diff --git a/pkg/models/find_filter.go b/pkg/models/find_filter.go index f684ca065..9934a9ea9 100644 --- a/pkg/models/find_filter.go +++ b/pkg/models/find_filter.go @@ -96,15 +96,15 @@ func (ff FindFilterType) GetPage() int { func (ff FindFilterType) GetPageSize() int { const defaultPerPage = 25 const minPerPage = 0 - const maxPerPage = 1000 if ff.PerPage == nil { return defaultPerPage } - if *ff.PerPage > maxPerPage { - return maxPerPage - } else if *ff.PerPage < minPerPage { + // removed the maxPerPage check. We already all -1 to indicate all results + // so there is no conceivable reason we should limit the page size + + if *ff.PerPage < minPerPage { // negative page sizes should return all results // this is a sanity check in case GetPageSize is // called with a negative page size. diff --git a/pkg/sqlite/batch.go b/pkg/sqlite/batch.go new file mode 100644 index 000000000..71ad5d354 --- /dev/null +++ b/pkg/sqlite/batch.go @@ -0,0 +1,20 @@ +package sqlite + +const defaultBatchSize = 1000 + +// batchExec executes the provided function in batches of the provided size. +func batchExec(ids []int, batchSize int, fn func(batch []int) error) error { + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + + batch := ids[i:end] + if err := fn(batch); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 622bfd8e8..b5e3a7a09 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -363,17 +363,23 @@ func (qb *GalleryStore) Find(ctx context.Context, id int) (*models.Gallery, erro } func (qb *GalleryStore) FindMany(ctx context.Context, ids []int) ([]*models.Gallery, error) { - q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(ids)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - galleries := make([]*models.Gallery, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - galleries[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + galleries[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range galleries { diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 1dbae4853..d5bb4e852 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -260,17 +260,23 @@ func (qb *ImageStore) Find(ctx context.Context, id int) (*models.Image, error) { } func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) { - q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(ids)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - images := make([]*models.Image, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - images[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + images[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range images { diff --git a/pkg/sqlite/movies.go b/pkg/sqlite/movies.go index e4d74cf9b..5d9d5cb36 100644 --- a/pkg/sqlite/movies.go +++ b/pkg/sqlite/movies.go @@ -70,17 +70,23 @@ func (qb *movieQueryBuilder) Find(ctx context.Context, id int) (*models.Movie, e func (qb *movieQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Movie, error) { tableMgr := movieTableMgr - q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - ret := make([]*models.Movie, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - ret[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + ret[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range ret { diff --git a/pkg/sqlite/performer.go b/pkg/sqlite/performer.go index b506a1309..024679007 100644 --- a/pkg/sqlite/performer.go +++ b/pkg/sqlite/performer.go @@ -299,17 +299,23 @@ func (qb *PerformerStore) Find(ctx context.Context, id int) (*models.Performer, func (qb *PerformerStore) FindMany(ctx context.Context, ids []int) ([]*models.Performer, error) { tableMgr := performerTableMgr - q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - ret := make([]*models.Performer, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - ret[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + ret[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range ret { diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index e7657677d..58c652a2c 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -364,18 +364,24 @@ func (qb *SceneStore) Find(ctx context.Context, id int) (*models.Scene, error) { } func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) { - table := qb.table() - q := qb.selectDataset().Prepared(true).Where(table.Col(idColumn).In(ids)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - scenes := make([]*models.Scene, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - scenes[i] = s + table := qb.table() + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := qb.selectDataset().Prepared(true).Where(table.Col(idColumn).In(batch)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + scenes[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range scenes { diff --git a/pkg/sqlite/studio.go b/pkg/sqlite/studio.go index 5a3013c39..f3dd963a8 100644 --- a/pkg/sqlite/studio.go +++ b/pkg/sqlite/studio.go @@ -80,17 +80,23 @@ func (qb *studioQueryBuilder) Find(ctx context.Context, id int) (*models.Studio, func (qb *studioQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Studio, error) { tableMgr := studioTableMgr - q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - ret := make([]*models.Studio, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - ret[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + ret[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range ret { diff --git a/pkg/sqlite/tag.go b/pkg/sqlite/tag.go index 49f4c9bbd..aa6232cb2 100644 --- a/pkg/sqlite/tag.go +++ b/pkg/sqlite/tag.go @@ -98,17 +98,23 @@ func (qb *tagQueryBuilder) Find(ctx context.Context, id int) (*models.Tag, error func (qb *tagQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Tag, error) { tableMgr := tagTableMgr - q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...)) - unsorted, err := qb.getMany(ctx, q) - if err != nil { - return nil, err - } - ret := make([]*models.Tag, len(ids)) - for _, s := range unsorted { - i := intslice.IntIndex(ids, s.ID) - ret[i] = s + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { + q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + for _, s := range unsorted { + i := intslice.IntIndex(ids, s.ID) + ret[i] = s + } + + return nil + }); err != nil { + return nil, err } for i := range ret { diff --git a/ui/v2.5/src/components/List/ListFilter.tsx b/ui/v2.5/src/components/List/ListFilter.tsx index f0e660328..830947b87 100644 --- a/ui/v2.5/src/components/List/ListFilter.tsx +++ b/ui/v2.5/src/components/List/ListFilter.tsx @@ -40,7 +40,6 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { useDebounce } from "src/hooks/debounce"; -const maxPageSize = 1000; interface IListFilterProps { onFilterUpdate: (newFilter: ListFilterModel) => void; filter: ListFilterModel; @@ -134,11 +133,6 @@ export const ListFilter: React.FC = ({ return; } - // don't allow page sizes over 1000 - if (pp > maxPageSize) { - pp = maxPageSize; - } - const newFilter = cloneDeep(filter); newFilter.itemsPerPage = pp; newFilter.currentPage = 1; @@ -377,7 +371,6 @@ export const ListFilter: React.FC = ({ ) => {