From 7e8c764dc71dcd4b5a9c197d6c98ff90db00d1cd Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:03:54 +1000 Subject: [PATCH] Fix migrations not using tx (#5282) --- pkg/sqlite/migrations/32_postmigrate.go | 32 +++++++++++++------------ pkg/sqlite/migrations/32_premigrate.go | 4 ++-- pkg/sqlite/migrations/34_postmigrate.go | 4 ++-- pkg/sqlite/migrations/42_postmigrate.go | 28 +++++++++++----------- pkg/sqlite/migrations/45_postmigrate.go | 12 +++++----- pkg/sqlite/migrations/49_postmigrate.go | 4 ++-- pkg/sqlite/migrations/52_postmigrate.go | 6 ++--- pkg/sqlite/migrations/55_postmigrate.go | 4 ++-- pkg/sqlite/migrations/64_postmigrate.go | 4 ++-- 9 files changed, 50 insertions(+), 48 deletions(-) diff --git a/pkg/sqlite/migrations/32_postmigrate.go b/pkg/sqlite/migrations/32_postmigrate.go index 4dbd65df8..6a4cf3d00 100644 --- a/pkg/sqlite/migrations/32_postmigrate.go +++ b/pkg/sqlite/migrations/32_postmigrate.go @@ -74,7 +74,7 @@ func (m *schema32Migrator) migrateFolders(ctx context.Context) error { query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -94,12 +94,12 @@ func (m *schema32Migrator) migrateFolders(ctx context.Context) error { count++ parent := filepath.Dir(p) - parentID, zipFileID, err := m.createFolderHierarchy(parent) + parentID, zipFileID, err := m.createFolderHierarchy(tx, parent) if err != nil { return err } - _, err = m.db.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id) + _, err = tx.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id) if err != nil { return err } @@ -153,7 +153,7 @@ func (m *schema32Migrator) migrateFiles(ctx context.Context) error { query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit) if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -178,12 +178,12 @@ func (m *schema32Migrator) migrateFiles(ctx context.Context) error { parent := filepath.Dir(p) basename := filepath.Base(p) if parent != "." { - parentID, zipFileID, err := m.createFolderHierarchy(parent) + parentID, zipFileID, err := m.createFolderHierarchy(tx, parent) if err != nil { return err } - _, err = m.db.Exec("UPDATE `files` SET `parent_folder_id` = ?, `zip_file_id` = ?, `basename` = ? WHERE `id` = ?", parentID, zipFileID, basename, id) + _, err = tx.Exec("UPDATE `files` SET `parent_folder_id` = ?, `zip_file_id` = ?, `basename` = ? WHERE `id` = ?", parentID, zipFileID, basename, id) if err != nil { return fmt.Errorf("migrating file %s: %w", p, err) } @@ -245,16 +245,18 @@ func (m *schema32Migrator) deletePlaceholderFolder(ctx context.Context) error { return fmt.Errorf("not deleting placeholder folder because it has %d folders", result.Count) } - _, err := m.db.Exec("DELETE FROM `folders` WHERE `id` = 1") - return err + return m.withTxn(ctx, func(tx *sqlx.Tx) error { + _, err := tx.Exec("DELETE FROM `folders` WHERE `id` = 1") + return err + }) } -func (m *schema32Migrator) createFolderHierarchy(p string) (*int, sql.NullInt64, error) { +func (m *schema32Migrator) createFolderHierarchy(tx *sqlx.Tx, p string) (*int, sql.NullInt64, error) { parent := filepath.Dir(p) if parent == p { // get or create this folder - return m.getOrCreateFolder(p, nil, sql.NullInt64{}) + return m.getOrCreateFolder(tx, p, nil, sql.NullInt64{}) } var ( @@ -269,23 +271,23 @@ func (m *schema32Migrator) createFolderHierarchy(p string) (*int, sql.NullInt64, parentID = &foundEntry.id zipFileID = foundEntry.zipID } else { - parentID, zipFileID, err = m.createFolderHierarchy(parent) + parentID, zipFileID, err = m.createFolderHierarchy(tx, parent) if err != nil { return nil, sql.NullInt64{}, err } } - return m.getOrCreateFolder(p, parentID, zipFileID) + return m.getOrCreateFolder(tx, p, parentID, zipFileID) } -func (m *schema32Migrator) getOrCreateFolder(path string, parentID *int, zipFileID sql.NullInt64) (*int, sql.NullInt64, error) { +func (m *schema32Migrator) getOrCreateFolder(tx *sqlx.Tx, path string, parentID *int, zipFileID sql.NullInt64) (*int, sql.NullInt64, error) { foundEntry, ok := m.folderCache[path] if ok { return &foundEntry.id, foundEntry.zipID, nil } const query = "SELECT `id`, `zip_file_id` FROM `folders` WHERE `path` = ?" - rows, err := m.db.Query(query, path) + rows, err := tx.Query(query, path) if err != nil { return nil, sql.NullInt64{}, err } @@ -314,7 +316,7 @@ func (m *schema32Migrator) getOrCreateFolder(path string, parentID *int, zipFile } now := time.Now() - result, err := m.db.Exec(insertSQL, path, parentFolderID, zipFileID, time.Time{}, now, now) + result, err := tx.Exec(insertSQL, path, parentFolderID, zipFileID, time.Time{}, now, now) if err != nil { return nil, sql.NullInt64{}, fmt.Errorf("creating folder %s: %w", path, err) } diff --git a/pkg/sqlite/migrations/32_premigrate.go b/pkg/sqlite/migrations/32_premigrate.go index 12906f7d5..caba639bc 100644 --- a/pkg/sqlite/migrations/32_premigrate.go +++ b/pkg/sqlite/migrations/32_premigrate.go @@ -65,7 +65,7 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error { query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -100,7 +100,7 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error { logger.Infof("Correcting %q gallery to be zip-based.", p) - _, err = m.db.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id) + _, err = tx.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id) if err != nil { return err } diff --git a/pkg/sqlite/migrations/34_postmigrate.go b/pkg/sqlite/migrations/34_postmigrate.go index e167c9a97..769655cb6 100644 --- a/pkg/sqlite/migrations/34_postmigrate.go +++ b/pkg/sqlite/migrations/34_postmigrate.go @@ -88,7 +88,7 @@ func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, col query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -126,7 +126,7 @@ func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, col updateSQL := fmt.Sprintf("UPDATE `%s` SET %s WHERE `id` = ?", table, updateList) - _, err = m.db.Exec(updateSQL, args...) + _, err = tx.Exec(updateSQL, args...) if err != nil { return err } diff --git a/pkg/sqlite/migrations/42_postmigrate.go b/pkg/sqlite/migrations/42_postmigrate.go index afb0db9e7..42180ff0f 100644 --- a/pkg/sqlite/migrations/42_postmigrate.go +++ b/pkg/sqlite/migrations/42_postmigrate.go @@ -71,7 +71,7 @@ func (m *schema42Migrator) migrate(ctx context.Context) error { query += fmt.Sprintf(" ORDER BY `performer_id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -92,7 +92,7 @@ func (m *schema42Migrator) migrate(ctx context.Context) error { gotSome = true count++ - if err := m.migratePerformerAliases(id, aliases); err != nil { + if err := m.migratePerformerAliases(tx, id, aliases); err != nil { return err } } @@ -114,7 +114,7 @@ func (m *schema42Migrator) migrate(ctx context.Context) error { return nil } -func (m *schema42Migrator) migratePerformerAliases(id int, aliases string) error { +func (m *schema42Migrator) migratePerformerAliases(tx *sqlx.Tx, id int, aliases string) error { // split aliases by , or / aliasList := strings.FieldsFunc(aliases, func(r rune) bool { return strings.ContainsRune(",/", r) @@ -126,7 +126,7 @@ func (m *schema42Migrator) migratePerformerAliases(id int, aliases string) error } // delete the existing row - if _, err := m.db.Exec("DELETE FROM `performer_aliases` WHERE `performer_id` = ?", id); err != nil { + if _, err := tx.Exec("DELETE FROM `performer_aliases` WHERE `performer_id` = ?", id); err != nil { return err } @@ -140,7 +140,7 @@ func (m *schema42Migrator) migratePerformerAliases(id int, aliases string) error // insert aliases into table for _, alias := range aliasList { - _, err := m.db.Exec("INSERT INTO `performer_aliases` (`performer_id`, `alias`) VALUES (?, ?)", id, alias) + _, err := tx.Exec("INSERT INTO `performer_aliases` (`performer_id`, `alias`) VALUES (?, ?)", id, alias) if err != nil { return err } @@ -173,7 +173,7 @@ SELECT id, name FROM performers WHERE performers.name like '% (%)'` query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -194,7 +194,7 @@ SELECT id, name FROM performers WHERE performers.name like '% (%)'` lastID = id count++ - if err := m.massagePerformerName(id, name); err != nil { + if err := m.massagePerformerName(tx, id, name); err != nil { return err } } @@ -220,7 +220,7 @@ SELECT id, name FROM performers WHERE performers.name like '% (%)'` // the format "name (disambiguation)". var performerDisRE = regexp.MustCompile(`^((?:[^(\s]+\s)+)\(([^)]+)\)$`) -func (m *schema42Migrator) massagePerformerName(performerID int, name string) error { +func (m *schema42Migrator) massagePerformerName(tx *sqlx.Tx, performerID int, name string) error { r := performerDisRE.FindStringSubmatch(name) if len(r) != 3 { @@ -235,7 +235,7 @@ func (m *schema42Migrator) massagePerformerName(performerID int, name string) er logger.Infof("Separating %q into %q and disambiguation %q", name, newName, newDis) - _, err := m.db.Exec("UPDATE performers SET name = ?, disambiguation = ? WHERE id = ?", newName, newDis, performerID) + _, err := tx.Exec("UPDATE performers SET name = ?, disambiguation = ? WHERE id = ?", newName, newDis, performerID) if err != nil { return err } @@ -266,7 +266,7 @@ SELECT id, name FROM performers WHERE performers.disambiguation IS NULL AND EXIS query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -286,7 +286,7 @@ SELECT id, name FROM performers WHERE performers.disambiguation IS NULL AND EXIS gotSome = true count++ - if err := m.migrateDuplicatePerformer(id, name); err != nil { + if err := m.migrateDuplicatePerformer(tx, id, name); err != nil { return err } } @@ -308,13 +308,13 @@ SELECT id, name FROM performers WHERE performers.disambiguation IS NULL AND EXIS return nil } -func (m *schema42Migrator) migrateDuplicatePerformer(performerID int, name string) error { +func (m *schema42Migrator) migrateDuplicatePerformer(tx *sqlx.Tx, performerID int, name string) error { // get the highest value of disambiguation for this performer name query := ` SELECT disambiguation FROM performers WHERE name = ? ORDER BY disambiguation DESC LIMIT 1` var disambiguation sql.NullString - if err := m.db.Get(&disambiguation, query, name); err != nil { + if err := tx.Get(&disambiguation, query, name); err != nil { return err } @@ -333,7 +333,7 @@ SELECT disambiguation FROM performers WHERE name = ? ORDER BY disambiguation DES logger.Infof("Adding disambiguation '%d' for performer %q", newDisambiguation, name) - _, err := m.db.Exec("UPDATE performers SET disambiguation = ? WHERE id = ?", strconv.Itoa(newDisambiguation), performerID) + _, err := tx.Exec("UPDATE performers SET disambiguation = ? WHERE id = ?", strconv.Itoa(newDisambiguation), performerID) if err != nil { return err } diff --git a/pkg/sqlite/migrations/45_postmigrate.go b/pkg/sqlite/migrations/45_postmigrate.go index 3a2ee6702..9e2bb1f8f 100644 --- a/pkg/sqlite/migrations/45_postmigrate.go +++ b/pkg/sqlite/migrations/45_postmigrate.go @@ -161,7 +161,7 @@ func (m *schema45Migrator) migrateImagesTable(ctx context.Context, options migra query += fmt.Sprintf(" LIMIT %d", limit) - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -191,7 +191,7 @@ func (m *schema45Migrator) migrateImagesTable(ctx context.Context, options migra image := result[i+1].(*[]byte) if len(*image) > 0 { - if err := m.insertImage(*image, id, options.destTable, col.destCol); err != nil { + if err := m.insertImage(tx, *image, id, options.destTable, col.destCol); err != nil { return err } } @@ -202,7 +202,7 @@ func (m *schema45Migrator) migrateImagesTable(ctx context.Context, options migra "joinTable": options.joinTable, "joinIDCol": options.joinIDCol, }) - if _, err := m.db.Exec(deleteSQL, id); err != nil { + if _, err := tx.Exec(deleteSQL, id); err != nil { return err } } @@ -224,11 +224,11 @@ func (m *schema45Migrator) migrateImagesTable(ctx context.Context, options migra return nil } -func (m *schema45Migrator) insertImage(data []byte, id int, destTable string, destCol string) error { +func (m *schema45Migrator) insertImage(tx *sqlx.Tx, data []byte, id int, destTable string, destCol string) error { // calculate checksum and insert into blobs table checksum := md5.FromBytes(data) - if _, err := m.db.Exec("INSERT INTO `blobs` (`checksum`, `blob`) VALUES (?, ?) ON CONFLICT DO NOTHING", checksum, data); err != nil { + if _, err := tx.Exec("INSERT INTO `blobs` (`checksum`, `blob`) VALUES (?, ?) ON CONFLICT DO NOTHING", checksum, data); err != nil { return err } @@ -237,7 +237,7 @@ func (m *schema45Migrator) insertImage(data []byte, id int, destTable string, de "destTable": destTable, "destCol": destCol, }) - if _, err := m.db.Exec(updateSQL, checksum, id); err != nil { + if _, err := tx.Exec(updateSQL, checksum, id); err != nil { return err } diff --git a/pkg/sqlite/migrations/49_postmigrate.go b/pkg/sqlite/migrations/49_postmigrate.go index 67e128f2c..8ba900d16 100644 --- a/pkg/sqlite/migrations/49_postmigrate.go +++ b/pkg/sqlite/migrations/49_postmigrate.go @@ -112,7 +112,7 @@ type schema49Migrator struct { func (m *schema49Migrator) migrateSavedFilters(ctx context.Context) error { if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { - rows, err := m.db.Query("SELECT id, mode, find_filter FROM saved_filters ORDER BY id") + rows, err := tx.Query("SELECT id, mode, find_filter FROM saved_filters ORDER BY id") if err != nil { return err } @@ -147,7 +147,7 @@ func (m *schema49Migrator) migrateSavedFilters(ctx context.Context) error { return fmt.Errorf("failed to get display options for saved filter %s : %w", findFilter, err) } - _, err = m.db.Exec("UPDATE saved_filters SET find_filter = ?, object_filter = ?, ui_options = ? WHERE id = ?", newFindFilter, objectFilter, uiOptions, id) + _, err = tx.Exec("UPDATE saved_filters SET find_filter = ?, object_filter = ?, ui_options = ? WHERE id = ?", newFindFilter, objectFilter, uiOptions, id) if err != nil { return fmt.Errorf("failed to update saved filter %d: %w", id, err) } diff --git a/pkg/sqlite/migrations/52_postmigrate.go b/pkg/sqlite/migrations/52_postmigrate.go index f173d8087..8235b4cf0 100644 --- a/pkg/sqlite/migrations/52_postmigrate.go +++ b/pkg/sqlite/migrations/52_postmigrate.go @@ -34,7 +34,7 @@ func (m *schema52Migrator) migrate(ctx context.Context) error { query := "SELECT `folders`.`id`, `folders`.`path`, `parent_folder`.`path` FROM `folders` " + "INNER JOIN `folders` AS `parent_folder` ON `parent_folder`.`id` = `folders`.`parent_folder_id`" - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -64,7 +64,7 @@ func (m *schema52Migrator) migrate(ctx context.Context) error { // ensure the correct path is unique var v int - isEmptyErr := m.db.Get(&v, "SELECT 1 FROM folders WHERE path = ?", correctPath) + isEmptyErr := tx.Get(&v, "SELECT 1 FROM folders WHERE path = ?", correctPath) if isEmptyErr != nil && !errors.Is(isEmptyErr, sql.ErrNoRows) { return fmt.Errorf("error checking if correct path %s is unique: %w", correctPath, isEmptyErr) } @@ -75,7 +75,7 @@ func (m *schema52Migrator) migrate(ctx context.Context) error { continue } - if _, err := m.db.Exec("UPDATE folders SET path = ? WHERE id = ?", correctPath, id); err != nil { + if _, err := tx.Exec("UPDATE folders SET path = ? WHERE id = ?", correctPath, id); err != nil { return fmt.Errorf("error updating folder path %s to %s: %w", folderPath, correctPath, err) } } diff --git a/pkg/sqlite/migrations/55_postmigrate.go b/pkg/sqlite/migrations/55_postmigrate.go index 1a3a5c566..5db79a531 100644 --- a/pkg/sqlite/migrations/55_postmigrate.go +++ b/pkg/sqlite/migrations/55_postmigrate.go @@ -31,7 +31,7 @@ func (m *schema55Migrator) migrate(ctx context.Context) error { if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { query := "SELECT DISTINCT `scene_id`, `view_date` FROM `scenes_view_dates`" - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -53,7 +53,7 @@ func (m *schema55Migrator) migrate(ctx context.Context) error { } // convert the timestamp to the correct format - if _, err := m.db.Exec("UPDATE scenes_view_dates SET view_date = ? WHERE view_date = ?", utcTimestamp, viewDate.Timestamp); err != nil { + if _, err := tx.Exec("UPDATE scenes_view_dates SET view_date = ? WHERE view_date = ?", utcTimestamp, viewDate.Timestamp); err != nil { return fmt.Errorf("error correcting view date %s to %s: %w", viewDate.Timestamp, viewDate, err) } } diff --git a/pkg/sqlite/migrations/64_postmigrate.go b/pkg/sqlite/migrations/64_postmigrate.go index 5b0f31a25..023f7db1d 100644 --- a/pkg/sqlite/migrations/64_postmigrate.go +++ b/pkg/sqlite/migrations/64_postmigrate.go @@ -35,7 +35,7 @@ func (m *schema64Migrator) migrate(ctx context.Context) error { if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { query := "SELECT DISTINCT `scene_id`, `view_date` FROM `scenes_view_dates`" - rows, err := m.db.Query(query) + rows, err := tx.Query(query) if err != nil { return err } @@ -64,7 +64,7 @@ func (m *schema64Migrator) migrate(ctx context.Context) error { // convert the timestamp to the correct format logger.Debugf("correcting view date %q to UTC date %q for scene %d", viewDate.Timestamp, viewDate.Timestamp.UTC(), id) - r, err := m.db.Exec("UPDATE scenes_view_dates SET view_date = ? WHERE scene_id = ? AND (view_date = ? OR view_date = ?)", utcTimestamp, id, viewDate.Timestamp, viewDate) + r, err := tx.Exec("UPDATE scenes_view_dates SET view_date = ? WHERE scene_id = ? AND (view_date = ? OR view_date = ?)", utcTimestamp, id, viewDate.Timestamp, viewDate) if err != nil { return fmt.Errorf("error correcting view date %s to %s: %w", viewDate.Timestamp, viewDate, err) }