stash/pkg/sqlite/blob.go

package sqlite

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"io/fs"

	"github.com/doug-martin/goqu/v9"
	"github.com/doug-martin/goqu/v9/exp"
	"github.com/jmoiron/sqlx"
	"github.com/mattn/go-sqlite3"
	"github.com/stashapp/stash/pkg/file"
	"github.com/stashapp/stash/pkg/hash/md5"
	"github.com/stashapp/stash/pkg/logger"
	"github.com/stashapp/stash/pkg/sqlite/blob"
	"github.com/stashapp/stash/pkg/utils"
	"gopkg.in/guregu/null.v4"
)

const (
	blobTable          = "blobs"
	blobChecksumColumn = "checksum"
)

type BlobStoreOptions struct {
	// UseFilesystem should be true if blob data should be stored in the filesystem
	UseFilesystem bool
	// UseDatabase should be true if blob data should be stored in the database
	UseDatabase bool
	// Path is the filesystem path to use for storing blobs
	Path string
	// SupplementaryPaths are alternative filesystem paths that will be used to find blobs
	// No changes will be made to these filesystems
	SupplementaryPaths []string
}

type BlobStore struct {
	repository

	tableMgr *table

	fsStore *blob.FilesystemStore
	// supplementary stores
	otherStores []blob.FilesystemReader
	options     BlobStoreOptions
}

func NewBlobStore(options BlobStoreOptions) *BlobStore {
	fs := &file.OsFS{}

	ret := &BlobStore{
		repository: repository{
			tableName: blobTable,
			idColumn:  blobChecksumColumn,
		},

		tableMgr: blobTableMgr,

		fsStore: blob.NewFilesystemStore(options.Path, fs),
		options: options,
	}

	for _, otherPath := range options.SupplementaryPaths {
		ret.otherStores = append(ret.otherStores, *blob.NewReadonlyFilesystemStore(otherPath, fs))
	}

	return ret
}

type blobRow struct {
	Checksum string `db:"checksum"`
	Blob     []byte `db:"blob"`
}

func (qb *BlobStore) table() exp.IdentifierExpression {
	return qb.tableMgr.table
}

func (qb *BlobStore) Count(ctx context.Context) (int, error) {
	table := qb.table()
	q := dialect.From(table).Select(goqu.COUNT(table.Col(blobChecksumColumn)))

	var ret int
	if err := querySimple(ctx, q, &ret); err != nil {
		return 0, err
	}

	return ret, nil
}

// Write stores the data and its checksum in enabled stores.
// Always writes at least the checksum to the database.
func (qb *BlobStore) Write(ctx context.Context, data []byte) (string, error) {
	if !qb.options.UseDatabase && !qb.options.UseFilesystem {
		panic("no blob store configured")
	}

	if len(data) == 0 {
		return "", fmt.Errorf("cannot write empty data")
	}

	checksum := md5.FromBytes(data)

	// only write blob to the database if UseDatabase is true
	// always at least write the checksum
	var storedData []byte
	if qb.options.UseDatabase {
		storedData = data
	}

	if err := qb.write(ctx, checksum, storedData); err != nil {
		return "", fmt.Errorf("writing to database: %w", err)
	}

	if qb.options.UseFilesystem {
		if err := qb.fsStore.Write(ctx, checksum, data); err != nil {
			return "", fmt.Errorf("writing to filesystem: %w", err)
		}
	}

	return checksum, nil
}

func (qb *BlobStore) write(ctx context.Context, checksum string, data []byte) error {
	table := qb.table()
	q := dialect.Insert(table).Prepared(true).Rows(blobRow{
		Checksum: checksum,
		Blob:     data,
	}).OnConflict(goqu.DoNothing())

	_, err := exec(ctx, q)
	if err != nil {
		return fmt.Errorf("inserting into %s: %w", table, err)
	}

	return nil
}

func (qb *BlobStore) update(ctx context.Context, checksum string, data []byte) error {
	table := qb.table()
	q := dialect.Update(table).Prepared(true).Set(goqu.Record{
		"blob": data,
	}).Where(goqu.C(blobChecksumColumn).Eq(checksum))

	_, err := exec(ctx, q)
	if err != nil {
		return fmt.Errorf("updating %s: %w", table, err)
	}

	return nil
}

type ChecksumNotFoundError struct {
	Checksum string
}

func (e *ChecksumNotFoundError) Error() string {
	return fmt.Sprintf("checksum %s does not exist", e.Checksum)
}

type ChecksumBlobNotExistError struct {
	Checksum string
}

func (e *ChecksumBlobNotExistError) Error() string {
	return fmt.Sprintf("blob for checksum %s does not exist", e.Checksum)
}

func (qb *BlobStore) readSQL(ctx context.Context, querySQL string, args ...interface{}) ([]byte, string, error) {
	if !qb.options.UseDatabase && !qb.options.UseFilesystem {
		panic("no blob store configured")
	}

	// always try to get from the database first, even if set to use filesystem
	var row blobRow
	found := false
	const single = true
	if err := qb.queryFunc(ctx, querySQL, args, single, func(r *sqlx.Rows) error {
		found = true
		if err := r.StructScan(&row); err != nil {
			return err
		}

		return nil
	}); err != nil {
		return nil, "", fmt.Errorf("reading from database: %w", err)
	}

	if !found {
		// not found in the database - does not exist
		return nil, "", nil
	}

	checksum := row.Checksum

	if row.Blob != nil {
		return row.Blob, checksum, nil
	}

	// don't use the filesystem if not configured to do so
	if qb.options.UseFilesystem {
		ret, err := qb.readFromFilesystem(ctx, checksum)
		if err != nil {
			return nil, checksum, err
		}

		return ret, checksum, nil
	}

	return nil, checksum, &ChecksumBlobNotExistError{
		Checksum: checksum,
	}
}

func (qb *BlobStore) readFromFilesystem(ctx context.Context, checksum string) ([]byte, error) {
	// try to read from primary store first, then supplementaries
	fsStores := append([]blob.FilesystemReader{qb.fsStore.FilesystemReader}, qb.otherStores...)

	for _, fsStore := range fsStores {
		ret, err := fsStore.Read(ctx, checksum)
		if err == nil {
			return ret, nil
		}

		if !errors.Is(err, fs.ErrNotExist) {
			return nil, fmt.Errorf("reading from filesystem: %w", err)
		}
	}

	// blob not found - should not happen
	return nil, &ChecksumBlobNotExistError{
		Checksum: checksum,
	}
}

func (qb *BlobStore) EntryExists(ctx context.Context, checksum string) (bool, error) {
	q := dialect.From(qb.table()).Select(goqu.COUNT("*")).Where(qb.tableMgr.byID(checksum))

	var found int
	if err := querySimple(ctx, q, &found); err != nil {
		return false, fmt.Errorf("querying %s: %w", qb.table(), err)
	}

	return found != 0, nil
}

// Read reads the data from the database or filesystem, depending on which is enabled.
func (qb *BlobStore) Read(ctx context.Context, checksum string) ([]byte, error) {
	if !qb.options.UseDatabase && !qb.options.UseFilesystem {
		panic("no blob store configured")
	}

	// always try to get from the database first, even if set to use filesystem
	ret, err := qb.readFromDatabase(ctx, checksum)
	if err != nil {
		if !errors.Is(err, sql.ErrNoRows) {
			return nil, fmt.Errorf("reading from database: %w", err)
		}

		// not found in the database - does not exist
		return nil, &ChecksumNotFoundError{
			Checksum: checksum,
		}
	}

	if ret != nil {
		return ret, nil
	}

	// don't use the filesystem if not configured to do so
	if qb.options.UseFilesystem {
		return qb.readFromFilesystem(ctx, checksum)
	}

	// blob not found - should not happen
	return nil, &ChecksumBlobNotExistError{
		Checksum: checksum,
	}
}

func (qb *BlobStore) readFromDatabase(ctx context.Context, checksum string) ([]byte, error) {
	q := dialect.From(qb.table()).Select(qb.table().All()).Where(qb.tableMgr.byID(checksum))

	var row blobRow
	const single = true
	if err := queryFunc(ctx, q, single, func(r *sqlx.Rows) error {
		if err := r.StructScan(&row); err != nil {
			return err
		}

		return nil
	}); err != nil {
		return nil, fmt.Errorf("querying %s: %w", qb.table(), err)
	}

	return row.Blob, nil
}

// Delete marks a checksum as no longer in use by a single reference.
// If no references remain, the blob is deleted from the database and filesystem.
func (qb *BlobStore) Delete(ctx context.Context, checksum string) error {
	// try to delete the blob from the database
	if err := qb.delete(ctx, checksum); err != nil {
		if qb.isConstraintError(err) {
			// blob is still referenced - do not delete
			logger.Debugf("Blob %s is still referenced - not deleting", checksum)
			return nil
		}

		// unexpected error
		return fmt.Errorf("deleting from database: %w", err)
	}

	// blob was deleted from the database - delete from filesystem if enabled
	if qb.options.UseFilesystem {
		logger.Debugf("Deleting blob %s from filesystem", checksum)
		if err := qb.fsStore.Delete(ctx, checksum); err != nil {
			return fmt.Errorf("deleting from filesystem: %w", err)
		}
	}

	return nil
}

func (qb *BlobStore) isConstraintError(err error) bool {
	var sqliteError sqlite3.Error
	if errors.As(err, &sqliteError) {
		return sqliteError.Code == sqlite3.ErrConstraint
	}
	return false
}

func (qb *BlobStore) delete(ctx context.Context, checksum string) error {
	table := qb.table()

	q := dialect.Delete(table).Where(goqu.C(blobChecksumColumn).Eq(checksum))

	_, err := exec(ctx, q)
	if err != nil {
		return fmt.Errorf("deleting from %s: %w", table, err)
	}

	return nil
}

type blobJoinQueryBuilder struct {
	repository repository
	blobStore  *BlobStore

	joinTable string
}

func (qb *blobJoinQueryBuilder) GetImage(ctx context.Context, id int, blobCol string) ([]byte, error) {
	sqlQuery := utils.StrFormat(`
SELECT blobs.checksum, blobs.blob FROM {joinTable} INNER JOIN blobs ON {joinTable}.{joinCol} = blobs.checksum
WHERE {joinTable}.id = ?
`, utils.StrFormatMap{
		"joinTable": qb.joinTable,
		"joinCol":   blobCol,
	})

	ret, _, err := qb.blobStore.readSQL(ctx, sqlQuery, id)
	return ret, err
}

func (qb *blobJoinQueryBuilder) UpdateImage(ctx context.Context, id int, blobCol string, image []byte) error {
	if len(image) == 0 {
		return qb.DestroyImage(ctx, id, blobCol)
	}

	oldChecksum, err := qb.getChecksum(ctx, id, blobCol)
	if err != nil {
		return err
	}

	checksum, err := qb.blobStore.Write(ctx, image)
	if err != nil {
		return err
	}

	sqlQuery := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", qb.joinTable, blobCol)
	if _, err := dbWrapper.Exec(ctx, sqlQuery, checksum, id); err != nil {
		return err
	}

	// #3595 - delete the old blob if the checksum is different
	if oldChecksum != nil && *oldChecksum != checksum {
		if err := qb.blobStore.Delete(ctx, *oldChecksum); err != nil {
			return err
		}
	}

	return nil
}

func (qb *blobJoinQueryBuilder) getChecksum(ctx context.Context, id int, blobCol string) (*string, error) {
	sqlQuery := utils.StrFormat(`
SELECT {joinTable}.{joinCol} FROM {joinTable} WHERE {joinTable}.id = ?
`, utils.StrFormatMap{
		"joinTable": qb.joinTable,
		"joinCol":   blobCol,
	})

	var checksum null.String
	err := qb.repository.querySimple(ctx, sqlQuery, []interface{}{id}, &checksum)
	if err != nil {
		return nil, err
	}

	if !checksum.Valid {
		return nil, nil
	}

	return &checksum.String, nil
}

func (qb *blobJoinQueryBuilder) DestroyImage(ctx context.Context, id int, blobCol string) error {
	checksum, err := qb.getChecksum(ctx, id, blobCol)
	if err != nil {
		return err
	}

	if checksum == nil {
		// no image to delete
		return nil
	}

	updateQuery := fmt.Sprintf("UPDATE %s SET %s = NULL WHERE id = ?", qb.joinTable, blobCol)
	if _, err = dbWrapper.Exec(ctx, updateQuery, id); err != nil {
		return err
	}

	return qb.blobStore.Delete(ctx, *checksum)
}

func (qb *blobJoinQueryBuilder) HasImage(ctx context.Context, id int, blobCol string) (bool, error) {
	stmt := utils.StrFormat("SELECT COUNT(*) as count FROM (SELECT {joinCol} FROM {joinTable} WHERE id = ? AND {joinCol} IS NOT NULL LIMIT 1)", utils.StrFormatMap{
		"joinTable": qb.joinTable,
		"joinCol":   blobCol,
	})

	c, err := qb.repository.runCountQuery(ctx, stmt, []interface{}{id})
	if err != nil {
		return false, err
	}

	return c == 1, nil
}