diff --git a/pkg/sqlite/migrations/32_postmigrate.go b/pkg/sqlite/migrations/32_postmigrate.go index 779d8f3b9..ed80c9765 100644 --- a/pkg/sqlite/migrations/32_postmigrate.go +++ b/pkg/sqlite/migrations/32_postmigrate.go @@ -54,39 +54,71 @@ type schema32Migrator struct { func (m *schema32Migrator) migrateFolders(ctx context.Context) error { logger.Infof("Migrating folders") - const query = "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`" + const ( + limit = 1000 + logEvery = 10000 + ) - rows, err := m.db.Query(query) - if err != nil { - return err - } - defer rows.Close() + lastID := 0 + count := 0 - for rows.Next() { - var id int - var p string + for { + gotSome := false - err := rows.Scan(&id, &p) - if err != nil { + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`" + + if lastID != 0 { + query += fmt.Sprintf("AND `folders`.`id` > %d ", lastID) + } + + query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit) + + rows, err := m.db.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var id int + var p string + + err := rows.Scan(&id, &p) + if err != nil { + return err + } + + lastID = id + gotSome = true + count++ + + parent := filepath.Dir(p) + parentID, zipFileID, err := m.createFolderHierarchy(parent) + if err != nil { + return err + } + + _, err = m.db.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id) + if err != nil { + return err + } + } + + return rows.Err() + }); err != nil { return err } - parent := filepath.Dir(p) - parentID, zipFileID, err := m.createFolderHierarchy(parent) - if err != nil { - return err + if !gotSome { + break } - _, err = m.db.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id) - if err != nil { - return err + if count%logEvery == 0 { + logger.Infof("Migrated %d folders", count) } } - if err := rows.Err(); err != nil { - return err - } - return nil } diff --git a/pkg/sqlite/migrations/32_premigrate.go b/pkg/sqlite/migrations/32_premigrate.go index e594c1f43..12906f7d5 100644 --- a/pkg/sqlite/migrations/32_premigrate.go +++ b/pkg/sqlite/migrations/32_premigrate.go @@ -2,6 +2,7 @@ package migrations import ( "context" + "fmt" "os" "github.com/jmoiron/sqlx" @@ -30,6 +31,11 @@ type schema32PreMigrator struct { } func (m *schema32PreMigrator) migrate(ctx context.Context) error { + const ( + limit = 1000 + logEvery = 10000 + ) + // query for galleries with zip = 0 and path not null result := struct { Count int `db:"count"` @@ -45,48 +51,73 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error { logger.Infof("Checking %d galleries for incorrect zip value...", result.Count) - if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { - const query = "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL ORDER BY `id`" - rows, err := m.db.Query(query) - if err != nil { + lastID := 0 + count := 0 + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL " + if lastID != 0 { + query += fmt.Sprintf("AND `id` > %d ", lastID) + } + + query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit) + + rows, err := m.db.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var id int + var p string + + err := rows.Scan(&id, &p) + if err != nil { + return err + } + + gotSome = true + lastID = id + count++ + + // if path does not exist, make no changes + // if it does exist and is a folder, then we ignore it + // otherwise set zip to 1 + info, err := os.Stat(p) + if err != nil { + logger.Warnf("unable to verify if %q is a folder due to error %v. Assuming folder-based.", p, err) + continue + } + + if info.IsDir() { + // ignore it + continue + } + + logger.Infof("Correcting %q gallery to be zip-based.", p) + + _, err = m.db.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id) + if err != nil { + return err + } + } + + return rows.Err() + }); err != nil { return err } - defer rows.Close() - for rows.Next() { - var id int - var p string - - err := rows.Scan(&id, &p) - if err != nil { - return err - } - - // if path does not exist, assume that it is a file and not a folder - // if it does exist and is a folder, then we ignore it - // otherwise set zip to 1 - info, err := os.Stat(p) - if err != nil { - logger.Warnf("unable to verify if %q is a folder due to error %v. Not migrating.", p, err) - continue - } - - if info.IsDir() { - // ignore it - continue - } - - logger.Infof("Correcting %q gallery to be zip-based.", p) - - _, err = m.db.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id) - if err != nil { - return err - } + if !gotSome { + break } - return rows.Err() - }); err != nil { - return err + if count%logEvery == 0 { + logger.Infof("Checked %d galleries", count) + } } return nil diff --git a/ui/v2.5/src/docs/en/MigrationNotes/32.md b/ui/v2.5/src/docs/en/MigrationNotes/32.md index f45c8f9b2..c19401aea 100644 --- a/ui/v2.5/src/docs/en/MigrationNotes/32.md +++ b/ui/v2.5/src/docs/en/MigrationNotes/32.md @@ -1,3 +1,5 @@ +**For best results, ensure that zip-based gallery paths are correct by performing a scan and clean of your library using v0.16.1 prior to running this migration.** + This migration significantly changes the way that stash stores information about your files. This migration is not reversible. After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored.