diff --git a/internal/manager/json_utils.go b/internal/manager/json_utils.go index f1ce60404..483794624 100644 --- a/internal/manager/json_utils.go +++ b/internal/manager/json_utils.go @@ -42,3 +42,7 @@ func (jp *jsonUtils) saveGallery(fn string, gallery *jsonschema.Gallery) error { func (jp *jsonUtils) saveFile(fn string, file jsonschema.DirEntry) error { return jsonschema.SaveFileFile(filepath.Join(jp.json.Files, fn), file) } + +func (jp *jsonUtils) saveSavedFilter(fn string, savedFilter *jsonschema.SavedFilter) error { + return jsonschema.SaveSavedFilterFile(filepath.Join(jp.json.SavedFilters, fn), savedFilter) +} diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index 19abba215..fe35c150d 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -23,6 +23,7 @@ import ( "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/performer" + "github.com/stashapp/stash/pkg/savedfilter" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/sliceutil" "github.com/stashapp/stash/pkg/sliceutil/stringslice" @@ -176,6 +177,7 @@ func (t *ExportTask) Start(ctx context.Context, wg *sync.WaitGroup) { t.ExportPerformers(ctx, workerCount) t.ExportStudios(ctx, workerCount) t.ExportTags(ctx, workerCount) + t.ExportSavedFilters(ctx, workerCount) return nil }) @@ -1186,3 +1188,62 @@ func (t *ExportTask) exportGroup(ctx context.Context, wg *sync.WaitGroup, jobCha } } } + +func (t *ExportTask) ExportSavedFilters(ctx context.Context, workers int) { + // don't export saved filters unless we're doing a full export + if !t.full { + return + } + + var wg sync.WaitGroup + + reader := t.repository.SavedFilter + var filters []*models.SavedFilter + var err error + filters, err = reader.All(ctx) + + if err != nil { + logger.Errorf("[saved filters] failed to fetch saved filters: %v", err) + } + + logger.Info("[saved filters] exporting") + startTime := time.Now() + + jobCh := make(chan *models.SavedFilter, workers*2) // make a buffered channel to feed workers + + for w := 0; w < workers; w++ { // create export Saved Filter workers + wg.Add(1) + go t.exportSavedFilter(ctx, &wg, jobCh) + } + + for i, savedFilter := range filters { + index := i + 1 + logger.Progressf("[saved filters] %d of %d", index, len(filters)) + + jobCh <- savedFilter // feed workers + } + + close(jobCh) + wg.Wait() + + logger.Infof("[saved filters] export complete in %s. %d workers used.", time.Since(startTime), workers) +} + +func (t *ExportTask) exportSavedFilter(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.SavedFilter) { + defer wg.Done() + + for thisFilter := range jobChan { + newJSON, err := savedfilter.ToJSON(ctx, thisFilter) + + if err != nil { + logger.Errorf("[saved filter] <%s> error getting saved filter JSON: %v", thisFilter.Name, err) + continue + } + + fn := newJSON.Filename() + + if err := t.json.saveSavedFilter(fn, newJSON); err != nil { + logger.Errorf("[saved filter] <%s> failed to save json: %v", fn, err) + } + } +} diff --git a/internal/manager/task_import.go b/internal/manager/task_import.go index 87185c661..8863a768d 100644 --- a/internal/manager/task_import.go +++ b/internal/manager/task_import.go @@ -20,6 +20,7 @@ import ( "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/performer" + "github.com/stashapp/stash/pkg/savedfilter" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/tag" @@ -124,6 +125,7 @@ func (t *ImportTask) Start(ctx context.Context) { } } + t.ImportSavedFilters(ctx) t.ImportTags(ctx) t.ImportPerformers(ctx) t.ImportStudios(ctx) @@ -779,3 +781,53 @@ func (t *ImportTask) ImportImages(ctx context.Context) { logger.Info("[images] import complete") } + +func (t *ImportTask) ImportSavedFilters(ctx context.Context) { + logger.Info("[saved filters] importing") + + path := t.json.json.SavedFilters + files, err := os.ReadDir(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + logger.Errorf("[saved filters] failed to read saved filters directory: %v", err) + } + + return + } + + r := t.repository + + for i, fi := range files { + index := i + 1 + savedFilterJSON, err := jsonschema.LoadSavedFilterFile(filepath.Join(path, fi.Name())) + if err != nil { + logger.Errorf("[saved filters] failed to read json: %v", err) + continue + } + + logger.Progressf("[saved filters] %d of %d", index, len(files)) + + if err := r.WithTxn(ctx, func(ctx context.Context) error { + return t.importSavedFilter(ctx, savedFilterJSON) + }); err != nil { + logger.Errorf("[saved filters] <%s> failed to import: %v", fi.Name(), err) + continue + } + } + + logger.Info("[saved filters] import complete") +} + +func (t *ImportTask) importSavedFilter(ctx context.Context, savedFilterJSON *jsonschema.SavedFilter) error { + importer := &savedfilter.Importer{ + ReaderWriter: t.repository.SavedFilter, + Input: *savedFilterJSON, + MissingRefBehaviour: t.MissingRefBehaviour, + } + + if err := performImport(ctx, importer, t.DuplicateBehaviour); err != nil { + return err + } + + return nil +} diff --git a/pkg/models/jsonschema/load.go b/pkg/models/jsonschema/load.go new file mode 100644 index 000000000..af0a79a6f --- /dev/null +++ b/pkg/models/jsonschema/load.go @@ -0,0 +1,31 @@ +package jsonschema + +import ( + "fmt" + "os" + + jsoniter "github.com/json-iterator/go" +) + +func loadFile[T any](filePath string) (*T, error) { + var ret T + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + var json = jsoniter.ConfigCompatibleWithStandardLibrary + jsonParser := json.NewDecoder(file) + err = jsonParser.Decode(&ret) + if err != nil { + return nil, err + } + return &ret, nil +} + +func saveFile[T any](filePath string, obj *T) error { + if obj == nil { + return fmt.Errorf("object must not be nil") + } + return marshalToFile(filePath, obj) +} diff --git a/pkg/models/jsonschema/saved_filter.go b/pkg/models/jsonschema/saved_filter.go new file mode 100644 index 000000000..ca828f3c2 --- /dev/null +++ b/pkg/models/jsonschema/saved_filter.go @@ -0,0 +1,27 @@ +package jsonschema + +import ( + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/models" +) + +type SavedFilter struct { + Mode models.FilterMode `db:"mode" json:"mode"` + Name string `db:"name" json:"name"` + FindFilter *models.FindFilterType `json:"find_filter"` + ObjectFilter map[string]interface{} `json:"object_filter"` + UIOptions map[string]interface{} `json:"ui_options"` +} + +func (s SavedFilter) Filename() string { + ret := fsutil.SanitiseBasename(s.Name + "_" + s.Mode.String()) + return ret + ".json" +} + +func LoadSavedFilterFile(filePath string) (*SavedFilter, error) { + return loadFile[SavedFilter](filePath) +} + +func SaveSavedFilterFile(filePath string, image *SavedFilter) error { + return saveFile[SavedFilter](filePath, image) +} diff --git a/pkg/models/paths/paths_json.go b/pkg/models/paths/paths_json.go index e6e302238..b2795409f 100644 --- a/pkg/models/paths/paths_json.go +++ b/pkg/models/paths/paths_json.go @@ -12,14 +12,15 @@ type JSONPaths struct { ScrapedFile string - Performers string - Scenes string - Images string - Galleries string - Studios string - Tags string - Groups string - Files string + Performers string + Scenes string + Images string + Galleries string + Studios string + Tags string + Groups string + Files string + SavedFilters string } func newJSONPaths(baseDir string) *JSONPaths { @@ -34,6 +35,7 @@ func newJSONPaths(baseDir string) *JSONPaths { jp.Groups = filepath.Join(baseDir, "movies") jp.Tags = filepath.Join(baseDir, "tags") jp.Files = filepath.Join(baseDir, "files") + jp.SavedFilters = filepath.Join(baseDir, "saved_filters") return &jp } @@ -52,6 +54,7 @@ func EmptyJSONDirs(baseDir string) { _ = fsutil.EmptyDir(jsonPaths.Groups) _ = fsutil.EmptyDir(jsonPaths.Tags) _ = fsutil.EmptyDir(jsonPaths.Files) + _ = fsutil.EmptyDir(jsonPaths.SavedFilters) } func EnsureJSONDirs(baseDir string) { @@ -83,4 +86,7 @@ func EnsureJSONDirs(baseDir string) { if err := fsutil.EnsureDir(jsonPaths.Files); err != nil { logger.Warnf("couldn't create directories for Files: %v", err) } + if err := fsutil.EnsureDir(jsonPaths.SavedFilters); err != nil { + logger.Warnf("couldn't create directories for Saved Filters: %v", err) + } } diff --git a/pkg/savedfilter/export.go b/pkg/savedfilter/export.go new file mode 100644 index 000000000..a52f618c9 --- /dev/null +++ b/pkg/savedfilter/export.go @@ -0,0 +1,19 @@ +package savedfilter + +import ( + "context" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" +) + +// ToJSON converts a SavedFilter object into its JSON equivalent. +func ToJSON(ctx context.Context, filter *models.SavedFilter) (*jsonschema.SavedFilter, error) { + return &jsonschema.SavedFilter{ + Name: filter.Name, + Mode: filter.Mode, + FindFilter: filter.FindFilter, + ObjectFilter: filter.ObjectFilter, + UIOptions: filter.UIOptions, + }, nil +} diff --git a/pkg/savedfilter/export_test.go b/pkg/savedfilter/export_test.go new file mode 100644 index 000000000..c05ec5535 --- /dev/null +++ b/pkg/savedfilter/export_test.go @@ -0,0 +1,91 @@ +package savedfilter + +import ( + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stretchr/testify/assert" + + "testing" +) + +const ( + savedFilterID = 1 + noImageID = 2 + errImageID = 3 + errAliasID = 4 + withParentsID = 5 + errParentsID = 6 +) + +const ( + filterName = "testFilter" + mode = models.FilterModeGalleries +) + +var ( + findFilter = models.FindFilterType{} + objectFilter = make(map[string]interface{}) + uiOptions = make(map[string]interface{}) +) + +func createSavedFilter(id int) models.SavedFilter { + return models.SavedFilter{ + ID: id, + Name: filterName, + Mode: mode, + FindFilter: &findFilter, + ObjectFilter: objectFilter, + UIOptions: uiOptions, + } +} + +func createJSONSavedFilter() *jsonschema.SavedFilter { + return &jsonschema.SavedFilter{ + Name: filterName, + Mode: mode, + FindFilter: &findFilter, + ObjectFilter: objectFilter, + UIOptions: uiOptions, + } +} + +type testScenario struct { + savedFilter models.SavedFilter + expected *jsonschema.SavedFilter + err bool +} + +var scenarios []testScenario + +func initTestTable() { + scenarios = []testScenario{ + { + createSavedFilter(savedFilterID), + createJSONSavedFilter(), + false, + }, + } +} + +func TestToJSON(t *testing.T) { + initTestTable() + + db := mocks.NewDatabase() + + for i, s := range scenarios { + savedFilter := s.savedFilter + json, err := ToJSON(testCtx, &savedFilter) + + switch { + case !s.err && err != nil: + t.Errorf("[%d] unexpected error: %s", i, err.Error()) + case s.err && err == nil: + t.Errorf("[%d] expected error not returned", i) + default: + assert.Equal(t, s.expected, json, "[%d]", i) + } + } + + db.AssertExpectations(t) +} diff --git a/pkg/savedfilter/import.go b/pkg/savedfilter/import.go new file mode 100644 index 000000000..451987e4c --- /dev/null +++ b/pkg/savedfilter/import.go @@ -0,0 +1,60 @@ +package savedfilter + +import ( + "context" + "fmt" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" +) + +type ImporterReaderWriter interface { + models.SavedFilterWriter +} + +type Importer struct { + ReaderWriter ImporterReaderWriter + Input jsonschema.SavedFilter + MissingRefBehaviour models.ImportMissingRefEnum + + savedFilter models.SavedFilter +} + +func (i *Importer) PreImport(ctx context.Context) error { + i.savedFilter = models.SavedFilter{ + Name: i.Input.Name, + Mode: i.Input.Mode, + FindFilter: i.Input.FindFilter, + ObjectFilter: i.Input.ObjectFilter, + UIOptions: i.Input.UIOptions, + } + + return nil +} + +func (i *Importer) PostImport(ctx context.Context, id int) error { + return nil +} + +func (i *Importer) Name() string { + return i.Input.Name +} + +func (i *Importer) FindExistingID(ctx context.Context) (*int, error) { + // for now, assume this is only imported in full, so we don't support updating existing filters + return nil, nil +} + +func (i *Importer) Create(ctx context.Context) (*int, error) { + err := i.ReaderWriter.Create(ctx, &i.savedFilter) + if err != nil { + return nil, fmt.Errorf("error creating saved filter: %v", err) + } + + id := i.savedFilter.ID + return &id, nil +} + +func (i *Importer) Update(ctx context.Context, id int) error { + return fmt.Errorf("updating existing saved filters is not supported") +} diff --git a/pkg/savedfilter/import_test.go b/pkg/savedfilter/import_test.go new file mode 100644 index 000000000..f0d975ee1 --- /dev/null +++ b/pkg/savedfilter/import_test.go @@ -0,0 +1,124 @@ +package savedfilter + +import ( + "context" + "errors" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +const ( + savedFilterNameErr = "savedFilterNameErr" + existingSavedFilterName = "existingSavedFilterName" + + existingFilterID = 100 +) + +var testCtx = context.Background() + +func TestImporterName(t *testing.T) { + i := Importer{ + Input: jsonschema.SavedFilter{ + Name: filterName, + }, + } + + assert.Equal(t, filterName, i.Name()) +} + +func TestImporterPreImport(t *testing.T) { + i := Importer{ + Input: jsonschema.SavedFilter{ + Name: filterName, + }, + } + + err := i.PreImport(testCtx) + assert.Nil(t, err) +} + +func TestImporterPostImport(t *testing.T) { + db := mocks.NewDatabase() + + i := Importer{ + ReaderWriter: db.SavedFilter, + Input: jsonschema.SavedFilter{}, + } + + err := i.PostImport(testCtx, savedFilterID) + assert.Nil(t, err) + + db.AssertExpectations(t) +} + +func TestImporterFindExistingID(t *testing.T) { + db := mocks.NewDatabase() + + i := Importer{ + ReaderWriter: db.SavedFilter, + Input: jsonschema.SavedFilter{ + Name: filterName, + }, + } + + id, err := i.FindExistingID(testCtx) + assert.Nil(t, id) + assert.Nil(t, err) +} + +func TestCreate(t *testing.T) { + db := mocks.NewDatabase() + + savedFilter := models.SavedFilter{ + Name: filterName, + } + + savedFilterErr := models.SavedFilter{ + Name: savedFilterNameErr, + } + + i := Importer{ + ReaderWriter: db.SavedFilter, + savedFilter: savedFilter, + } + + errCreate := errors.New("Create error") + db.SavedFilter.On("Create", testCtx, &savedFilter).Run(func(args mock.Arguments) { + t := args.Get(1).(*models.SavedFilter) + t.ID = savedFilterID + }).Return(nil).Once() + db.SavedFilter.On("Create", testCtx, &savedFilterErr).Return(errCreate).Once() + + id, err := i.Create(testCtx) + assert.Equal(t, savedFilterID, *id) + assert.Nil(t, err) + + i.savedFilter = savedFilterErr + id, err = i.Create(testCtx) + assert.Nil(t, id) + assert.NotNil(t, err) + + db.AssertExpectations(t) +} + +func TestUpdate(t *testing.T) { + db := mocks.NewDatabase() + + savedFilterErr := models.SavedFilter{ + Name: savedFilterNameErr, + } + + i := Importer{ + ReaderWriter: db.SavedFilter, + savedFilter: savedFilterErr, + } + + // Update is not currently supported + err := i.Update(testCtx, existingFilterID) + assert.NotNil(t, err) +} diff --git a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx index f445b4332..e093dc60a 100644 --- a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx @@ -520,7 +520,7 @@ export const DataManagementTasks: React.FC = ({ type="submit" onClick={() => onExport()} > - … +