From 987fa8078606ffde9874145a781c2d0bed079cd9 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Wed, 22 Nov 2023 10:01:11 +1100 Subject: [PATCH] Scraper and plugin manager (#4242) * Add package manager * Add SettingModal validate * Reverse modal button order * Add plugin package management * Refactor ClearableInput --- .gitignore | 8 - graphql/documents/data/config.graphql | 11 + graphql/documents/data/package.graphql | 8 + graphql/documents/mutations/plugins.graphql | 12 + graphql/documents/mutations/scrapers.graphql | 12 + graphql/documents/queries/plugins.graphql | 24 + .../queries/scrapers/scrapers.graphql | 24 + graphql/schema/schema.graphql | 29 + graphql/schema/types/config.graphql | 10 + graphql/schema/types/package.graphql | 36 + internal/api/resolver_mutation_configure.go | 18 + internal/api/resolver_mutation_package.go | 82 ++ internal/api/resolver_query_configuration.go | 2 + internal/api/resolver_query_package.go | 194 ++++ internal/manager/config/config.go | 59 + internal/manager/manager.go | 36 + internal/manager/task/packages.go | 134 +++ pkg/models/package.go | 12 + pkg/pkg/cache.go | 82 ++ pkg/pkg/manager.go | 268 +++++ pkg/pkg/pkg.go | 198 ++++ pkg/pkg/repository.go | 22 + pkg/pkg/repository_http.go | 205 ++++ pkg/pkg/repository_http_test.go | 55 + pkg/pkg/store.go | 158 +++ ui/v2.5/package.json | 2 +- .../List/Filters/SelectableFilter.tsx | 2 + ui/v2.5/src/components/Settings/Inputs.tsx | 3 + .../Settings/PluginPackageManager.tsx | 227 ++++ .../Settings/ScraperPackageManager.tsx | 221 ++++ .../Settings/SettingsPluginsPanel.tsx | 7 + .../Settings/SettingsScrapingPanel.tsx | 7 + ui/v2.5/src/components/Shared/Alert.tsx | 33 + .../src/components/Shared/ClearableInput.tsx | 12 +- ui/v2.5/src/components/Shared/Modal.tsx | 38 +- .../Shared/PackageManager/PackageManager.tsx | 1002 +++++++++++++++++ .../Shared/PackageManager/styles.scss | 100 ++ ui/v2.5/src/components/Shared/styles.scss | 18 +- ui/v2.5/src/core/StashService.ts | 37 + ui/v2.5/src/index.scss | 1 + ui/v2.5/src/locales/en-GB.json | 35 + ui/v2.5/src/utils/job.ts | 75 ++ 42 files changed, 3484 insertions(+), 35 deletions(-) create mode 100644 graphql/documents/data/package.graphql create mode 100644 graphql/schema/types/package.graphql create mode 100644 internal/api/resolver_mutation_package.go create mode 100644 internal/api/resolver_query_package.go create mode 100644 internal/manager/task/packages.go create mode 100644 pkg/models/package.go create mode 100644 pkg/pkg/cache.go create mode 100644 pkg/pkg/manager.go create mode 100644 pkg/pkg/pkg.go create mode 100644 pkg/pkg/repository.go create mode 100644 pkg/pkg/repository_http.go create mode 100644 pkg/pkg/repository_http_test.go create mode 100644 pkg/pkg/store.go create mode 100644 ui/v2.5/src/components/Settings/PluginPackageManager.tsx create mode 100644 ui/v2.5/src/components/Settings/ScraperPackageManager.tsx create mode 100644 ui/v2.5/src/components/Shared/Alert.tsx create mode 100644 ui/v2.5/src/components/Shared/PackageManager/PackageManager.tsx create mode 100644 ui/v2.5/src/components/Shared/PackageManager/styles.scss create mode 100644 ui/v2.5/src/utils/job.ts diff --git a/.gitignore b/.gitignore index 99d8dffed..533f297c1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,11 +21,6 @@ vendor # GraphQL generated output internal/api/generated_*.go -#### -# Jetbrains -#### - - #### # Visual Studio #### @@ -52,9 +47,6 @@ internal/api/generated_*.go .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml -# Goland Junk -pkg/pkg - #### # Random #### diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index cfec9336d..310200557 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -53,6 +53,17 @@ fragment ConfigGeneralData on ConfigGeneralResult { liveTranscodeInputArgs liveTranscodeOutputArgs drawFunscriptHeatmapRange + + scraperPackageSources { + name + url + local_path + } + pluginPackageSources { + name + url + local_path + } } fragment ConfigInterfaceData on ConfigInterfaceResult { diff --git a/graphql/documents/data/package.graphql b/graphql/documents/data/package.graphql new file mode 100644 index 000000000..23171e511 --- /dev/null +++ b/graphql/documents/data/package.graphql @@ -0,0 +1,8 @@ +fragment PackageData on Package { + package_id + name + version + date + metadata + sourceURL +} diff --git a/graphql/documents/mutations/plugins.graphql b/graphql/documents/mutations/plugins.graphql index e7785a6ed..e39611625 100644 --- a/graphql/documents/mutations/plugins.graphql +++ b/graphql/documents/mutations/plugins.graphql @@ -17,3 +17,15 @@ mutation ConfigurePlugin($plugin_id: ID!, $input: Map!) { mutation SetPluginsEnabled($enabledMap: BoolMap!) { setPluginsEnabled(enabledMap: $enabledMap) } + +mutation InstallPluginPackages($packages: [PackageSpecInput!]!) { + installPackages(type: Plugin, packages: $packages) +} + +mutation UpdatePluginPackages($packages: [PackageSpecInput!]!) { + updatePackages(type: Plugin, packages: $packages) +} + +mutation UninstallPluginPackages($packages: [PackageSpecInput!]!) { + uninstallPackages(type: Plugin, packages: $packages) +} diff --git a/graphql/documents/mutations/scrapers.graphql b/graphql/documents/mutations/scrapers.graphql index 3f186faec..30b1fb101 100644 --- a/graphql/documents/mutations/scrapers.graphql +++ b/graphql/documents/mutations/scrapers.graphql @@ -1,3 +1,15 @@ mutation ReloadScrapers { reloadScrapers } + +mutation InstallScraperPackages($packages: [PackageSpecInput!]!) { + installPackages(type: Scraper, packages: $packages) +} + +mutation UpdateScraperPackages($packages: [PackageSpecInput!]!) { + updatePackages(type: Scraper, packages: $packages) +} + +mutation UninstallScraperPackages($packages: [PackageSpecInput!]!) { + uninstallPackages(type: Scraper, packages: $packages) +} diff --git a/graphql/documents/queries/plugins.graphql b/graphql/documents/queries/plugins.graphql index 4c8bd0095..1f8506c44 100644 --- a/graphql/documents/queries/plugins.graphql +++ b/graphql/documents/queries/plugins.graphql @@ -43,3 +43,27 @@ query PluginTasks { } } } + +query InstalledPluginPackages { + installedPackages(type: Plugin) { + ...PackageData + } +} + +query InstalledPluginPackagesStatus { + installedPackages(type: Plugin) { + ...PackageData + upgrade { + ...PackageData + } + } +} + +query AvailablePluginPackages($source: String!) { + availablePackages(source: $source, type: Plugin) { + ...PackageData + requires { + package_id + } + } +} diff --git a/graphql/documents/queries/scrapers/scrapers.graphql b/graphql/documents/queries/scrapers/scrapers.graphql index b696de852..3a855c81f 100644 --- a/graphql/documents/queries/scrapers/scrapers.graphql +++ b/graphql/documents/queries/scrapers/scrapers.graphql @@ -119,3 +119,27 @@ query ScrapeMovieURL($url: String!) { ...ScrapedMovieData } } + +query InstalledScraperPackages { + installedPackages(type: Scraper) { + ...PackageData + } +} + +query InstalledScraperPackagesStatus { + installedPackages(type: Scraper) { + ...PackageData + upgrade { + ...PackageData + } + } +} + +query AvailableScraperPackages($source: String!) { + availablePackages(source: $source, type: Scraper) { + ...PackageData + requires { + package_id + } + } +} diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index ccd7e6a62..beef9e5d7 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -168,6 +168,12 @@ type Query { "List available plugin operations" pluginTasks: [PluginTask!] + # Packages + "List installed packages" + installedPackages(type: PackageType!): [Package!]! + "List available packages" + availablePackages(type: PackageType!, source: String!): [Package!]! + # Config "Returns the current, complete configuration" configuration: ConfigResult! @@ -381,6 +387,29 @@ type Mutation { ): ID! reloadPlugins: Boolean! + """ + Installs the given packages. + If a package is already installed, it will be updated if needed.. + If an error occurs when installing a package, the job will continue to install the remaining packages. + Returns the job ID + """ + installPackages(type: PackageType!, packages: [PackageSpecInput!]!): ID! + """ + Updates the given packages. + If a package is not installed, it will not be installed. + If a package does not need to be updated, it will not be updated. + If no packages are provided, all packages of the given type will be updated. + If an error occurs when updating a package, the job will continue to update the remaining packages. + Returns the job ID. + """ + updatePackages(type: PackageType!, packages: [PackageSpecInput!]): ID! + """ + Uninstalls the given packages. + If an error occurs when uninstalling a package, the job will continue to uninstall the remaining packages. + Returns the job ID + """ + uninstallPackages(type: PackageType!, packages: [PackageSpecInput!]!): ID! + stopJob(job_id: ID!): Boolean! stopAllJobs: Boolean! diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 90b2fde4c..7cbbe7755 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -167,6 +167,11 @@ input ConfigGeneralInput { stashBoxes: [StashBoxInput!] "Python path - resolved using path if unset" pythonPath: String + + "Source of scraper packages" + scraperPackageSources: [PackageSourceInput!] + "Source of plugin packages" + pluginPackageSources: [PackageSourceInput!] } type ConfigGeneralResult { @@ -280,6 +285,11 @@ type ConfigGeneralResult { stashBoxes: [StashBox!]! "Python path - resolved using path if unset" pythonPath: String! + + "Source of scraper packages" + scraperPackageSources: [PackageSource!]! + "Source of plugin packages" + pluginPackageSources: [PackageSource!]! } input ConfigDisableDropdownCreateInput { diff --git a/graphql/schema/types/package.graphql b/graphql/schema/types/package.graphql new file mode 100644 index 000000000..798d09547 --- /dev/null +++ b/graphql/schema/types/package.graphql @@ -0,0 +1,36 @@ +enum PackageType { + Scraper + Plugin +} + +type Package { + package_id: String! + name: String! + version: String + date: Timestamp + requires: [Package!]! + + sourceURL: String! + + "The available upgraded version of this package" + upgrade: Package + + metadata: Map! +} + +input PackageSpecInput { + id: String! + sourceURL: String! +} + +type PackageSource { + name: String + url: String! + local_path: String +} + +input PackageSourceInput { + name: String + url: String! + local_path: String +} diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index 2ba94ba16..6ae835837 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -347,6 +347,18 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen c.Set(config.DrawFunscriptHeatmapRange, input.DrawFunscriptHeatmapRange) } + refreshScraperSource := false + if input.ScraperPackageSources != nil { + c.Set(config.ScraperPackageSources, input.ScraperPackageSources) + refreshScraperSource = true + } + + refreshPluginSource := false + if input.PluginPackageSources != nil { + c.Set(config.PluginPackageSources, input.PluginPackageSources) + refreshPluginSource = true + } + if err := c.Write(); err != nil { return makeConfigGeneralResult(), err } @@ -361,6 +373,12 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen if refreshBlobStorage { manager.GetInstance().SetBlobStoreOptions() } + if refreshScraperSource { + manager.GetInstance().RefreshScraperSourceManager() + } + if refreshPluginSource { + manager.GetInstance().RefreshPluginSourceManager() + } return makeConfigGeneralResult(), nil } diff --git a/internal/api/resolver_mutation_package.go b/internal/api/resolver_mutation_package.go new file mode 100644 index 000000000..477e74c32 --- /dev/null +++ b/internal/api/resolver_mutation_package.go @@ -0,0 +1,82 @@ +package api + +import ( + "context" + "strconv" + + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/task" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" +) + +func refreshPackageType(typeArg PackageType) { + mgr := manager.GetInstance() + + if typeArg == PackageTypePlugin { + if err := mgr.PluginCache.LoadPlugins(); err != nil { + logger.Errorf("Error reading plugin configs: %v", err) + } + } else if typeArg == PackageTypeScraper { + if err := mgr.ScraperCache.ReloadScrapers(); err != nil { + logger.Errorf("Error reading scraper configs: %v", err) + } + } +} + +func (r *mutationResolver) InstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) { + pm, err := getPackageManager(typeArg) + if err != nil { + return "", err + } + + mgr := manager.GetInstance() + t := &task.InstallPackagesJob{ + PackagesJob: task.PackagesJob{ + PackageManager: pm, + OnComplete: func() { refreshPackageType(typeArg) }, + }, + Packages: packages, + } + jobID := mgr.JobManager.Add(ctx, "Installing packages...", t) + + return strconv.Itoa(jobID), nil +} + +func (r *mutationResolver) UpdatePackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) { + pm, err := getPackageManager(typeArg) + if err != nil { + return "", err + } + + mgr := manager.GetInstance() + t := &task.UpdatePackagesJob{ + PackagesJob: task.PackagesJob{ + PackageManager: pm, + OnComplete: func() { refreshPackageType(typeArg) }, + }, + Packages: packages, + } + jobID := mgr.JobManager.Add(ctx, "Updating packages...", t) + + return strconv.Itoa(jobID), nil +} + +func (r *mutationResolver) UninstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) { + pm, err := getPackageManager(typeArg) + if err != nil { + return "", err + } + + mgr := manager.GetInstance() + t := &task.UninstallPackagesJob{ + PackagesJob: task.PackagesJob{ + PackageManager: pm, + OnComplete: func() { refreshPackageType(typeArg) }, + }, + Packages: packages, + } + jobID := mgr.JobManager.Add(ctx, "Updating packages...", t) + + return strconv.Itoa(jobID), nil +} diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index ec3e33c49..c32172a15 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -127,6 +127,8 @@ func makeConfigGeneralResult() *ConfigGeneralResult { LiveTranscodeInputArgs: config.GetLiveTranscodeInputArgs(), LiveTranscodeOutputArgs: config.GetLiveTranscodeOutputArgs(), DrawFunscriptHeatmapRange: config.GetDrawFunscriptHeatmapRange(), + ScraperPackageSources: config.GetScraperPackageSources(), + PluginPackageSources: config.GetPluginPackageSources(), } } diff --git a/internal/api/resolver_query_package.go b/internal/api/resolver_query_package.go new file mode 100644 index 000000000..0ba3d9e9c --- /dev/null +++ b/internal/api/resolver_query_package.go @@ -0,0 +1,194 @@ +package api + +import ( + "context" + "errors" + "sort" + "strings" + + "github.com/99designs/gqlgen/graphql" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/pkg" + "github.com/stashapp/stash/pkg/sliceutil" +) + +var ErrInvalidPackageType = errors.New("invalid package type") + +func getPackageManager(typeArg PackageType) (*pkg.Manager, error) { + var pm *pkg.Manager + switch typeArg { + case PackageTypeScraper: + pm = manager.GetInstance().ScraperPackageManager + case PackageTypePlugin: + pm = manager.GetInstance().PluginPackageManager + default: + return nil, ErrInvalidPackageType + } + + return pm, nil +} + +func manifestToPackage(p pkg.Manifest) *Package { + ret := &Package{ + PackageID: p.ID, + Name: p.Name, + SourceURL: p.RepositoryURL, + } + + if len(p.Version) > 0 { + ret.Version = &p.Version + } + if !p.Date.IsZero() { + ret.Date = &p.Date.Time + } + + ret.Metadata = p.Metadata + if ret.Metadata == nil { + ret.Metadata = make(map[string]interface{}) + } + + return ret +} + +func remotePackageToPackage(p pkg.RemotePackage, index pkg.RemotePackageIndex) *Package { + ret := &Package{ + PackageID: p.ID, + Name: p.Name, + } + + if len(p.Version) > 0 { + ret.Version = &p.Version + } + if !p.Date.IsZero() { + ret.Date = &p.Date.Time + } + + ret.Metadata = p.Metadata + if ret.Metadata == nil { + ret.Metadata = make(map[string]interface{}) + } + + ret.SourceURL = p.Repository.Path() + + for _, r := range p.Requires { + // required packages must come from the same source + spec := models.PackageSpecInput{ + ID: r, + SourceURL: p.Repository.Path(), + } + + req, found := index[spec] + if !found { + // shouldn't happen, but we'll ignore it + continue + } + + ret.Requires = append(ret.Requires, remotePackageToPackage(req, index)) + } + + return ret +} + +func sortedPackageSpecKeys[V any](m map[models.PackageSpecInput]V) []models.PackageSpecInput { + // sort keys + var keys []models.PackageSpecInput + for k := range m { + keys = append(keys, k) + } + + sort.Slice(keys, func(i, j int) bool { + if strings.EqualFold(keys[i].ID, keys[j].ID) { + return keys[i].ID < keys[j].ID + } + + return strings.ToLower(keys[i].ID) < strings.ToLower(keys[j].ID) + }) + + return keys +} + +func (r *queryResolver) getInstalledPackagesWithUpgrades(ctx context.Context, pm *pkg.Manager) ([]*Package, error) { + // get all installed packages + installed, err := pm.ListInstalled(ctx) + if err != nil { + return nil, err + } + + // get remotes for all installed packages + allRemoteList, err := pm.ListInstalledRemotes(ctx, installed) + if err != nil { + return nil, err + } + + packageStatusIndex := pkg.MakePackageStatusIndex(installed, allRemoteList) + + ret := make([]*Package, len(packageStatusIndex)) + i := 0 + + for _, k := range sortedPackageSpecKeys(packageStatusIndex) { + v := packageStatusIndex[k] + p := manifestToPackage(*v.Local) + if v.Upgradable() { + pp := remotePackageToPackage(*v.Remote, allRemoteList) + p.Upgrade = pp + } + ret[i] = p + i++ + } + + return ret, nil +} + +func (r *queryResolver) InstalledPackages(ctx context.Context, typeArg PackageType) ([]*Package, error) { + pm, err := getPackageManager(typeArg) + if err != nil { + return nil, err + } + + installed, err := pm.ListInstalled(ctx) + if err != nil { + return nil, err + } + + var ret []*Package + + if sliceutil.Contains(graphql.CollectAllFields(ctx), "upgrade") { + ret, err = r.getInstalledPackagesWithUpgrades(ctx, pm) + if err != nil { + return nil, err + } + } else { + ret = make([]*Package, len(installed)) + i := 0 + for _, k := range sortedPackageSpecKeys(installed) { + ret[i] = manifestToPackage(installed[k]) + i++ + } + } + + return ret, nil +} + +func (r *queryResolver) AvailablePackages(ctx context.Context, typeArg PackageType, source string) ([]*Package, error) { + pm, err := getPackageManager(typeArg) + if err != nil { + return nil, err + } + + available, err := pm.ListRemote(ctx, source) + if err != nil { + return nil, err + } + + ret := make([]*Package, len(available)) + i := 0 + for _, k := range sortedPackageSpecKeys(available) { + p := available[k] + ret[i] = remotePackageToPackage(p, available) + + i++ + } + + return ret, nil +} diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index f856d842c..850548efc 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -21,6 +21,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/paths" + "github.com/stashapp/stash/pkg/sliceutil" "github.com/stashapp/stash/pkg/utils" ) @@ -137,6 +138,9 @@ const ( PluginsSettingPrefix = PluginsSetting + "." DisabledPlugins = "plugins.disabled" + PluginPackageSources = "plugins.package_sources" + ScraperPackageSources = "scrapers.package_sources" + // i18n Language = "language" @@ -1520,6 +1524,61 @@ func (i *Instance) ActivatePublicAccessTripwire(requestIP string) error { return i.Write() } +func (i *Instance) getPackageSources(key string) []*models.PackageSource { + var sources []*models.PackageSource + if err := i.unmarshalKey(key, &sources); err != nil { + logger.Warnf("error in unmarshalkey: %v", err) + } + + return sources +} + +func (i *Instance) GetPluginPackageSources() []*models.PackageSource { + return i.getPackageSources(PluginPackageSources) +} + +func (i *Instance) GetScraperPackageSources() []*models.PackageSource { + return i.getPackageSources(ScraperPackageSources) +} + +type packagePathGetter struct { + getterFn func() []*models.PackageSource +} + +func (g packagePathGetter) GetAllSourcePaths() []string { + p := g.getterFn() + var ret []string + for _, v := range p { + ret = sliceutil.AppendUnique(ret, v.LocalPath) + } + + return ret +} + +func (g packagePathGetter) GetSourcePath(srcURL string) string { + p := g.getterFn() + + for _, v := range p { + if v.URL == srcURL { + return v.LocalPath + } + } + + return "" +} + +func (i *Instance) GetPluginPackagePathGetter() packagePathGetter { + return packagePathGetter{ + getterFn: i.GetPluginPackageSources, + } +} + +func (i *Instance) GetScraperPackagePathGetter() packagePathGetter { + return packagePathGetter{ + getterFn: i.GetScraperPackageSources, + } +} + func (i *Instance) Validate() error { i.RLock() defer i.RUnlock() diff --git a/internal/manager/manager.go b/internal/manager/manager.go index c7486ff6c..b19936e1d 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net/http" "os" "path/filepath" "runtime" @@ -29,6 +30,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/paths" + "github.com/stashapp/stash/pkg/pkg" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scraper" @@ -130,6 +132,9 @@ type Manager struct { PluginCache *plugin.Cache ScraperCache *scraper.Cache + PluginPackageManager *pkg.Manager + ScraperPackageManager *pkg.Manager + DownloadStore *DownloadStore DLNAService *dlna.Service @@ -229,6 +234,9 @@ func initialize() error { dlnaRepository := dlna.NewRepository(repo) instance.DLNAService = dlna.NewService(dlnaRepository, cfg, &sceneServer) + instance.RefreshPluginSourceManager() + instance.RefreshScraperSourceManager() + if !cfg.IsNewSystem() { logger.Infof("using config file: %s", cfg.GetConfigFile()) @@ -280,6 +288,26 @@ func initialize() error { return nil } +func initialisePackageManager(localPath string, srcPathGetter pkg.SourcePathGetter, cachePathGetter pkg.CachePathGetter) *pkg.Manager { + const timeout = 10 * time.Second + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + Timeout: timeout, + } + + return &pkg.Manager{ + Local: &pkg.Store{ + BaseDir: localPath, + ManifestFile: pkg.ManifestFile, + }, + PackagePathGetter: srcPathGetter, + Client: httpClient, + CachePathGetter: cachePathGetter, + } +} + func videoFileFilter(ctx context.Context, f models.File) bool { return useAsVideo(f.Base().Path) } @@ -566,6 +594,14 @@ func (s *Manager) RefreshStreamManager() { s.StreamManager = ffmpeg.NewStreamManager(cacheDir, s.FFMPEG, s.FFProbe, s.Config, s.ReadLockManager) } +func (s *Manager) RefreshScraperSourceManager() { + s.ScraperPackageManager = initialisePackageManager(s.Config.GetScrapersPath(), s.Config.GetScraperPackagePathGetter(), s.Config) +} + +func (s *Manager) RefreshPluginSourceManager() { + s.PluginPackageManager = initialisePackageManager(s.Config.GetPluginsPath(), s.Config.GetPluginPackagePathGetter(), s.Config) +} + func setSetupDefaults(input *SetupInput) { if input.ConfigLocation == "" { input.ConfigLocation = filepath.Join(fsutil.GetHomeDirectory(), ".stash", "config.yml") diff --git a/internal/manager/task/packages.go b/internal/manager/task/packages.go new file mode 100644 index 000000000..af970362f --- /dev/null +++ b/internal/manager/task/packages.go @@ -0,0 +1,134 @@ +package task + +import ( + "context" + "fmt" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/pkg" +) + +type PackagesJob struct { + PackageManager *pkg.Manager + OnComplete func() +} + +func (j *PackagesJob) installPackage(ctx context.Context, p models.PackageSpecInput, progress *job.Progress) error { + defer progress.Increment() + + if err := j.PackageManager.Install(ctx, p); err != nil { + return fmt.Errorf("installing package: %w", err) + } + + return nil +} + +type InstallPackagesJob struct { + PackagesJob + Packages []*models.PackageSpecInput +} + +func (j *InstallPackagesJob) Execute(ctx context.Context, progress *job.Progress) { + progress.SetTotal(len(j.Packages)) + + for _, p := range j.Packages { + if job.IsCancelled(ctx) { + logger.Info("Cancelled installing packages") + return + } + + logger.Infof("Installing package %s", p.ID) + taskDesc := fmt.Sprintf("Installing %s", p.ID) + progress.ExecuteTask(taskDesc, func() { + if err := j.installPackage(ctx, *p, progress); err != nil { + logger.Errorf("Error installing package %s from %s: %v", p.ID, p.SourceURL, err) + } + }) + } + + if j.OnComplete != nil { + j.OnComplete() + } + + logger.Infof("Finished installing packages") +} + +type UpdatePackagesJob struct { + PackagesJob + Packages []*models.PackageSpecInput +} + +func (j *UpdatePackagesJob) Execute(ctx context.Context, progress *job.Progress) { + // if no packages are specified, update all + if len(j.Packages) == 0 { + installed, err := j.PackageManager.InstalledStatus(ctx) + if err != nil { + logger.Errorf("Error getting installed packages: %v", err) + return + } + + for _, p := range installed { + if p.Upgradable() { + j.Packages = append(j.Packages, &models.PackageSpecInput{ + ID: p.Local.ID, + SourceURL: p.Remote.Repository.Path(), + }) + } + } + } + + progress.SetTotal(len(j.Packages)) + + for _, p := range j.Packages { + if job.IsCancelled(ctx) { + logger.Info("Cancelled updating packages") + return + } + + logger.Infof("Updating package %s", p.ID) + taskDesc := fmt.Sprintf("Updating %s", p.ID) + progress.ExecuteTask(taskDesc, func() { + if err := j.installPackage(ctx, *p, progress); err != nil { + logger.Errorf("Error updating package %s from %s: %v", p.ID, p.SourceURL, err) + } + }) + } + + if j.OnComplete != nil { + j.OnComplete() + } + + logger.Infof("Finished updating packages") +} + +type UninstallPackagesJob struct { + PackagesJob + Packages []*models.PackageSpecInput +} + +func (j *UninstallPackagesJob) Execute(ctx context.Context, progress *job.Progress) { + progress.SetTotal(len(j.Packages)) + + for _, p := range j.Packages { + if job.IsCancelled(ctx) { + logger.Info("Cancelled installing packages") + return + } + + logger.Infof("Uninstalling package %s", p.ID) + taskDesc := fmt.Sprintf("Uninstalling %s", p.ID) + progress.ExecuteTask(taskDesc, func() { + if err := j.PackageManager.Uninstall(ctx, *p); err != nil { + logger.Errorf("Error uninstalling package %s: %v", p.ID, err) + } + }) + } + + if j.OnComplete != nil { + j.OnComplete() + } + + logger.Infof("Finished uninstalling packages") +} diff --git a/pkg/models/package.go b/pkg/models/package.go new file mode 100644 index 000000000..9fb2285b6 --- /dev/null +++ b/pkg/models/package.go @@ -0,0 +1,12 @@ +package models + +type PackageSpecInput struct { + ID string `json:"id"` + SourceURL string `json:"sourceURL"` +} + +type PackageSource struct { + Name *string `json:"name"` + LocalPath string `json:"localPath"` + URL string `json:"url"` +} diff --git a/pkg/pkg/cache.go b/pkg/pkg/cache.go new file mode 100644 index 000000000..f9a0e68f9 --- /dev/null +++ b/pkg/pkg/cache.go @@ -0,0 +1,82 @@ +package pkg + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/stashapp/stash/pkg/hash/md5" + "github.com/stashapp/stash/pkg/logger" +) + +const cacheSubDir = "package_lists" + +type repositoryCache struct { + cachePath string +} + +func (c *repositoryCache) path(url string) string { + // convert the url to md5 + hash := md5.FromString(url) + + return filepath.Join(c.cachePath, cacheSubDir, hash) +} + +func (c *repositoryCache) lastModified(url string) *time.Time { + if c == nil { + return nil + } + + path := c.path(url) + s, err := os.Stat(path) + if err != nil { + // ignore + logger.Debugf("error getting cached file %s: %v", path, err) + return nil + } + + ret := s.ModTime() + return &ret +} + +func (c *repositoryCache) getPackageList(url string) (io.ReadCloser, error) { + path := c.path(url) + ret, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to get file %q: %w", path, err) + } + + return ret, nil +} + +func (c *repositoryCache) cacheFile(url string, data io.ReadCloser) (io.ReadCloser, error) { + if c == nil { + return data, nil + } + + path := c.path(url) + + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + // ignore, just return the original file + logger.Debugf("error creating cache path %s: %v", filepath.Dir(path), err) + return data, nil + } + + f, err := os.Create(path) + if err != nil { + // ignore, just return the original file + logger.Debugf("error creating cached file %s: %v", path, err) + return data, nil + } + + defer data.Close() + if _, err := io.Copy(f, data); err != nil { + _ = f.Close() + return nil, fmt.Errorf("writing to cache file %s - %w", path, err) + } + + _ = f.Close() + return c.getPackageList(url) +} diff --git a/pkg/pkg/manager.go b/pkg/pkg/manager.go new file mode 100644 index 000000000..ba7b46532 --- /dev/null +++ b/pkg/pkg/manager.go @@ -0,0 +1,268 @@ +package pkg + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + + "github.com/stashapp/stash/pkg/models" +) + +type CachePathGetter interface { + GetCachePath() string +} + +// SourcePathGetter gets the source path for a given package URL. +type SourcePathGetter interface { + // GetAllSourcePaths gets all source paths. + GetAllSourcePaths() []string + + // GetSourcePath gets the source path for the given package URL. + GetSourcePath(srcURL string) string +} + +// Manager manages the installation of paks. +type Manager struct { + Local *Store + PackagePathGetter SourcePathGetter + CachePathGetter CachePathGetter + + Client *http.Client +} + +func (m *Manager) remoteFromURL(path string) (*httpRepository, error) { + u, err := url.Parse(path) + if err != nil { + return nil, fmt.Errorf("parsing path: %w", err) + } + + cachePath := m.CachePathGetter.GetCachePath() + var cache *repositoryCache + if cachePath != "" { + cache = &repositoryCache{cachePath: cachePath} + } + + return newHttpRepository(*u, m.Client, cache), nil +} + +func (m *Manager) ListInstalled(ctx context.Context) (LocalPackageIndex, error) { + paths := m.PackagePathGetter.GetAllSourcePaths() + + var installedList []Manifest + + for _, p := range paths { + store := m.Local.sub(p) + + srcList, err := store.List(ctx) + if err != nil { + return nil, fmt.Errorf("listing local packages: %w", err) + } + + installedList = append(installedList, srcList...) + } + + return localPackageIndexFromList(installedList), nil +} + +func (m *Manager) ListRemote(ctx context.Context, remoteURL string) (RemotePackageIndex, error) { + r, err := m.remoteFromURL(remoteURL) + if err != nil { + return nil, fmt.Errorf("creating remote repository: %w", err) + } + + list, err := r.List(ctx) + if err != nil { + return nil, fmt.Errorf("listing remote packages: %w", err) + } + + // add link to RemotePackage + for i := range list { + list[i].Repository = r + } + + ret := remotePackageIndexFromList(list) + + return ret, nil +} + +func (m *Manager) ListInstalledRemotes(ctx context.Context, installed LocalPackageIndex) (RemotePackageIndex, error) { + // get remotes for all installed packages + allRemoteList := make(RemotePackageIndex) + + remoteURLs := installed.remoteURLs() + for _, remoteURL := range remoteURLs { + remoteList, err := m.ListRemote(ctx, remoteURL) + if err != nil { + return nil, err + } + + allRemoteList.merge(remoteList) + } + + return allRemoteList, nil +} + +func (m *Manager) InstalledStatus(ctx context.Context) (PackageStatusIndex, error) { + // get all installed packages + installed, err := m.ListInstalled(ctx) + if err != nil { + return nil, err + } + + // get remotes for all installed packages + allRemoteList, err := m.ListInstalledRemotes(ctx, installed) + if err != nil { + return nil, err + } + + ret := MakePackageStatusIndex(installed, allRemoteList) + + return ret, nil +} + +func (m *Manager) packageByID(ctx context.Context, spec models.PackageSpecInput) (*RemotePackage, error) { + l, err := m.ListRemote(ctx, spec.SourceURL) + if err != nil { + return nil, err + } + + pkg, found := l[spec] + if !found { + return nil, nil + } + + return &pkg, nil +} + +func (m *Manager) getStore(remoteURL string) *Store { + srcPath := m.PackagePathGetter.GetSourcePath(remoteURL) + store := m.Local.sub(srcPath) + + return store +} + +func (m *Manager) Install(ctx context.Context, spec models.PackageSpecInput) error { + remote, err := m.remoteFromURL(spec.SourceURL) + if err != nil { + return fmt.Errorf("creating remote repository: %w", err) + } + + pkg, err := m.packageByID(ctx, spec) + if err != nil { + return fmt.Errorf("getting remote package: %w", err) + } + + fromRemote, err := remote.GetPackageZip(ctx, *pkg) + if err != nil { + return fmt.Errorf("getting remote package: %w", err) + } + + defer fromRemote.Close() + + d, err := io.ReadAll(fromRemote) + if err != nil { + return fmt.Errorf("reading package data: %w", err) + } + + sha := fmt.Sprintf("%x", sha256.Sum256(d)) + if sha != pkg.Sha256 { + return fmt.Errorf("package data (%s) does not match expected SHA256 (%s)", sha, pkg.Sha256) + } + + zr, err := zip.NewReader(bytes.NewReader(d), int64(len(d))) + if err != nil { + return fmt.Errorf("reading zip data: %w", err) + } + + store := m.getStore(spec.SourceURL) + + // uninstall existing package if present + if _, err := store.getManifest(ctx, pkg.ID); err == nil { + if err := m.deletePackageFiles(ctx, store, pkg.ID); err != nil { + return fmt.Errorf("uninstalling existing package: %w", err) + } + } + + if err := m.installPackage(*pkg, store, zr); err != nil { + return fmt.Errorf("installing package: %w", err) + } + + return nil +} + +func (m *Manager) installPackage(pkg RemotePackage, store *Store, zr *zip.Reader) error { + manifest := Manifest{ + ID: pkg.ID, + Name: pkg.Name, + Metadata: pkg.Metadata, + PackageVersion: pkg.PackageVersion, + RepositoryURL: pkg.Repository.Path(), + } + + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + + i, err := f.Open() + if err != nil { + return err + } + + fn := filepath.Clean(f.Name) + if err := store.writeFile(pkg.ID, fn, f.Mode(), i); err != nil { + i.Close() + return fmt.Errorf("writing file %q: %w", fn, err) + } + + i.Close() + manifest.Files = append(manifest.Files, fn) + } + + if err := store.writeManifest(pkg.ID, manifest); err != nil { + return fmt.Errorf("writing manifest: %w", err) + } + + return nil +} + +// Uninstall uninstalls the given package. +func (m *Manager) Uninstall(ctx context.Context, spec models.PackageSpecInput) error { + store := m.getStore(spec.SourceURL) + + if err := m.deletePackageFiles(ctx, store, spec.ID); err != nil { + return fmt.Errorf("deleting local package: %w", err) + } + + // also delete the directory + // ignore errors + _ = store.deletePackageDir(spec.ID) + + return nil +} + +func (m *Manager) deletePackageFiles(ctx context.Context, store *Store, id string) error { + manifest, err := store.getManifest(ctx, id) + if err != nil { + return fmt.Errorf("getting manifest: %w", err) + } + + for _, f := range manifest.Files { + if err := store.deleteFile(id, f); err != nil { + // ignore + continue + } + } + + if err := store.deleteManifest(id); err != nil { + return fmt.Errorf("deleting manifest: %w", err) + } + + return nil +} diff --git a/pkg/pkg/pkg.go b/pkg/pkg/pkg.go new file mode 100644 index 000000000..5bd27b404 --- /dev/null +++ b/pkg/pkg/pkg.go @@ -0,0 +1,198 @@ +package pkg + +import ( + "fmt" + "time" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil" +) + +const timeFormat = "2006-01-02 15:04:05 -0700" + +type Time struct { + time.Time +} + +func (t *Time) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + parsed, err := time.Parse(timeFormat, s) + if err != nil { + return err + } + t.Time = parsed + return nil +} + +func (t Time) MarshalYAML() (interface{}, error) { + return t.Format(timeFormat), nil +} + +type PackageMetadata map[string]interface{} + +type PackageVersion struct { + Version string `yaml:"version"` + Date Time `yaml:"date"` +} + +func (v PackageVersion) Upgradable(o PackageVersion) bool { + return o.Date.After(v.Date.Time) +} + +func (v PackageVersion) String() string { + ret := v.Version + if !v.Date.IsZero() { + date := v.Date.Format("2006-01-02") + if ret != "" { + ret += fmt.Sprintf(" (%s)", date) + } else { + ret = date + } + } + + return ret +} + +type PackageLocation struct { + // Path is the path to the package zip file. + // This may be relative or absolute. + Path string `yaml:"path"` + Sha256 string `yaml:"sha256"` +} + +type RemotePackage struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Repository remoteRepository `yaml:"-"` + Requires []string `yaml:"requires"` + Metadata PackageMetadata `yaml:"metadata"` + PackageVersion `yaml:",inline"` + PackageLocation `yaml:",inline"` +} + +func (p RemotePackage) PackageSpecInput() models.PackageSpecInput { + return models.PackageSpecInput{ + ID: p.ID, + SourceURL: p.Repository.Path(), + } +} + +type Manifest struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Metadata PackageMetadata `yaml:"metadata"` + PackageVersion `yaml:",inline"` + Requires []string `yaml:"requires"` + + RepositoryURL string `yaml:"source_repository"` + Files []string `yaml:"files"` +} + +func (m Manifest) PackageSpecInput() models.PackageSpecInput { + return models.PackageSpecInput{ + ID: m.ID, + SourceURL: m.RepositoryURL, + } +} + +// RemotePackageIndex is a map of package name to RemotePackage +type RemotePackageIndex map[models.PackageSpecInput]RemotePackage + +func (i RemotePackageIndex) merge(o RemotePackageIndex) { + for id, pkg := range o { + if existing, found := i[id]; found { + if existing.Date.After(pkg.Date.Time) { + continue + } + } + + i[id] = pkg + } +} + +func remotePackageIndexFromList(packages []RemotePackage) RemotePackageIndex { + index := make(RemotePackageIndex) + for _, pkg := range packages { + specInput := pkg.PackageSpecInput() + + // if package already exists in map, choose the newest + if existing, found := index[specInput]; found { + if existing.Date.After(pkg.Date.Time) { + continue + } + } + + index[specInput] = pkg + } + return index +} + +// LocalPackageIndex is a map of package name to RemotePackage +type LocalPackageIndex map[models.PackageSpecInput]Manifest + +func (i LocalPackageIndex) remoteURLs() []string { + var ret []string + + for _, pkg := range i { + ret = sliceutil.AppendUnique(ret, pkg.RepositoryURL) + } + + return ret +} + +func localPackageIndexFromList(packages []Manifest) LocalPackageIndex { + index := make(LocalPackageIndex) + for _, pkg := range packages { + index[pkg.PackageSpecInput()] = pkg + } + return index +} + +type PackageStatus struct { + Local *Manifest + Remote *RemotePackage +} + +func (s PackageStatus) Upgradable() bool { + if s.Local == nil || s.Remote == nil { + return false + } + + return s.Local.Upgradable(s.Remote.PackageVersion) +} + +type PackageStatusIndex map[models.PackageSpecInput]PackageStatus + +func MakePackageStatusIndex(installed LocalPackageIndex, remote RemotePackageIndex) PackageStatusIndex { + i := make(PackageStatusIndex) + + for spec, pkg := range installed { + pkgCopy := pkg + s := PackageStatus{ + Local: &pkgCopy, + } + + if remotePkg, found := remote[spec]; found { + s.Remote = &remotePkg + } + + i[spec] = s + } + + return i +} + +func (i PackageStatusIndex) Upgradable() []PackageStatus { + var ret []PackageStatus + + for _, s := range i { + if s.Upgradable() { + ret = append(ret, s) + } + } + + return ret +} diff --git a/pkg/pkg/repository.go b/pkg/pkg/repository.go new file mode 100644 index 000000000..d43115725 --- /dev/null +++ b/pkg/pkg/repository.go @@ -0,0 +1,22 @@ +package pkg + +import ( + "context" + "io" +) + +// remoteRepository is a repository that can be used to get paks from. +type remoteRepository interface { + RemotePackageLister + RemotePackageGetter + Path() string +} + +type RemotePackageLister interface { + // List returns all specs in the repository. + List(ctx context.Context) ([]RemotePackage, error) +} + +type RemotePackageGetter interface { + GetPackageZip(ctx context.Context, pkg RemotePackage) (io.ReadCloser, error) +} diff --git a/pkg/pkg/repository_http.go b/pkg/pkg/repository_http.go new file mode 100644 index 000000000..925d6ac5a --- /dev/null +++ b/pkg/pkg/repository_http.go @@ -0,0 +1,205 @@ +// Package http provides a repository implementation for HTTP. +package pkg + +import ( + "context" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path" + "time" + + "github.com/stashapp/stash/pkg/logger" + "gopkg.in/yaml.v2" +) + +// DefaultCacheTTL is the default time to live for the index cache. +const DefaultCacheTTL = 5 * time.Minute + +// httpRepository is a HTTP based repository. +// It is configured with a package list URL. Packages are located from the Path field of the package. +// +// The index is cached for the duration of CacheTTL. The first request after the cache expires will cause the index to be reloaded. +type httpRepository struct { + packageListURL url.URL + client *http.Client + + cache *repositoryCache +} + +// newHttpRepository creates a new Repository. If client is nil then http.DefaultClient is used. +func newHttpRepository(packageListURL url.URL, client *http.Client, cache *repositoryCache) *httpRepository { + if client == nil { + client = http.DefaultClient + } + return &httpRepository{ + packageListURL: packageListURL, + client: client, + cache: cache, + } +} + +func (r *httpRepository) Path() string { + return r.packageListURL.String() +} + +func (r *httpRepository) List(ctx context.Context) ([]RemotePackage, error) { + u := r.packageListURL + + // the package list URL may be file://, in which case we need to use the local file system + var ( + f io.ReadCloser + err error + ) + if u.Scheme == "file" { + f, err = r.getLocalFile(ctx, u.Path) + } else { + f, err = r.getFileCached(ctx, u) + } + + if err != nil { + return nil, fmt.Errorf("failed to get package list: %w", err) + } + + defer f.Close() + + data, err := io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("failed to read package list: %w", err) + } + + var index []RemotePackage + if err := yaml.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("reading package list: %w", err) + } + + return index, nil +} + +func isURL(s string) bool { + u, err := url.Parse(s) + return err == nil && u.Scheme != "" && (u.Scheme == "file" || u.Host != "") +} + +func (r *httpRepository) resolvePath(p string) url.URL { + // if the path can be resolved to a URL, then use that + if isURL(p) { + // isURL ensures URL is valid + u, _ := url.Parse(p) + return *u + } + + // otherwise, determine if the path is relative or absolute + // if it's relative, then join it with the package list URL + u := r.packageListURL + + if path.IsAbs(p) { + u.Path = p + } else { + u.Path = path.Join(path.Dir(u.Path), p) + } + + return u +} + +func (r *httpRepository) GetPackageZip(ctx context.Context, pkg RemotePackage) (io.ReadCloser, error) { + p := pkg.Path + + u := r.resolvePath(p) + + var ( + f io.ReadCloser + err error + ) + + // the package list URL may be file://, in which case we need to use the local file system + // the package zip path may be a URL. A remotely hosted list may _not_ use local files. + if u.Scheme == "file" { + if r.packageListURL.Scheme != "file" { + return nil, fmt.Errorf("%s is invalid for a remotely hosted package list", u.String()) + } + + f, err = r.getLocalFile(ctx, u.Path) + } else { + f, err = r.getFile(ctx, u) + } + + if err != nil { + return nil, fmt.Errorf("failed to get package file: %w", err) + } + + return f, nil +} + +// getFileCached tries to get the list from the local cache. +// If it is not found or is stale, then it gets it normally. +func (r *httpRepository) getFileCached(ctx context.Context, u url.URL) (io.ReadCloser, error) { + // check if the file is in the cache first + localModTime := r.cache.lastModified(u.String()) + + if localModTime != nil { + // get the update time of the file + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) + if err != nil { + // shouldn't happen + return nil, err + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get remote file: %w", err) + } + + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("failed to get remote file: %s", resp.Status) + } + + lastModified := resp.Header.Get("Last-Modified") + if lastModified != "" { + remoteModTime, _ := time.Parse(http.TimeFormat, lastModified) + + if !remoteModTime.After(*localModTime) { + logger.Debugf("cached version of %s is equal or newer than remote", u.String()) + return r.cache.getPackageList(u.String()) + } + } + + logger.Debugf("cached version of %s is older than remote", u.String()) + } + + return r.getFile(ctx, u) +} + +func (r *httpRepository) getFile(ctx context.Context, u url.URL) (io.ReadCloser, error) { + logger.Debugf("fetching %s", u.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + // shouldn't happen + return nil, err + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get remote file: %w", err) + } + + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("failed to get remote file: %s", resp.Status) + } + + return r.cache.cacheFile(u.String(), resp.Body) +} + +func (r *httpRepository) getLocalFile(ctx context.Context, path string) (fs.File, error) { + ret, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to get file %q: %w", path, err) + } + + return ret, nil +} + +var _ = remoteRepository(&httpRepository{}) diff --git a/pkg/pkg/repository_http_test.go b/pkg/pkg/repository_http_test.go new file mode 100644 index 000000000..e93ba8b8f --- /dev/null +++ b/pkg/pkg/repository_http_test.go @@ -0,0 +1,55 @@ +// Package http provides a repository implementation for HTTP. +package pkg + +import ( + "net/url" + "reflect" + "testing" +) + +func TestHttpRepository_resolvePath(t *testing.T) { + mustParse := func(s string) url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return *u + } + + tests := []struct { + name string + packageListURL url.URL + p string + want url.URL + }{ + { + name: "relative", + packageListURL: mustParse("https://example.com/foo/packages.yaml"), + p: "bar", + want: mustParse("https://example.com/foo/bar"), + }, + { + name: "absolute", + packageListURL: mustParse("https://example.com/foo/packages.yaml"), + p: "/bar", + want: mustParse("https://example.com/bar"), + }, + { + name: "different server", + packageListURL: mustParse("https://example.com/foo/packages.yaml"), + p: "http://example.org/bar", + want: mustParse("http://example.org/bar"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &httpRepository{ + packageListURL: tt.packageListURL, + } + got := r.resolvePath(tt.p) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("HttpRepository.resolvePath() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/pkg/store.go b/pkg/pkg/store.go new file mode 100644 index 000000000..4292edc1c --- /dev/null +++ b/pkg/pkg/store.go @@ -0,0 +1,158 @@ +package pkg + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +// ManifestFile is the default filename for the package manifest. +const ManifestFile = "manifest" + +// Store is a folder-based local repository. +// Packages are installed in their own directory under BaseDir. +// The package details are stored in a file named based on PackageFile. +type Store struct { + BaseDir string + // ManifestFile is the filename of the package file. + ManifestFile string +} + +// sub returns a new Store with the given path appended to the BaseDir. +func (r *Store) sub(path string) *Store { + if path == "" || path == "." { + return r + } + + return &Store{ + BaseDir: filepath.Join(r.BaseDir, path), + ManifestFile: r.ManifestFile, + } +} + +func (r *Store) List(ctx context.Context) ([]Manifest, error) { + e, err := os.ReadDir(r.BaseDir) + // ignore if directory cannot be read + if err != nil { + return nil, nil + } + + var ret []Manifest + + for _, ee := range e { + if !ee.IsDir() { + // ignore non-directories + continue + } + + pkg, err := r.getManifest(ctx, ee.Name()) + if err != nil { + // ignore if manifest does not exist + if errors.Is(err, os.ErrNotExist) { + continue + } + + return nil, err + } + + ret = append(ret, *pkg) + } + + return ret, nil +} + +func (r *Store) packageDir(id string) string { + return filepath.Join(r.BaseDir, id) +} + +func (r *Store) manifestPath(id string) string { + return filepath.Join(r.packageDir(id), r.ManifestFile) +} + +func (r *Store) getManifest(ctx context.Context, packageID string) (*Manifest, error) { + pfp := r.manifestPath(packageID) + + data, err := os.ReadFile(pfp) + if err != nil { + return nil, fmt.Errorf("reading manifest file %q: %w", pfp, err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("reading manifest file %q: %w", pfp, err) + } + + return &manifest, nil +} + +func (r *Store) ensurePackageExists(packageID string) error { + // ensure the manifest file exists + if _, err := os.Stat(r.manifestPath(packageID)); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("package %q does not exist", packageID) + } + } + + return nil +} + +func (r *Store) writeFile(packageID string, name string, mode fs.FileMode, i io.Reader) error { + fn := filepath.Join(r.packageDir(packageID), name) + + if err := os.MkdirAll(filepath.Dir(fn), os.ModePerm); err != nil { + return fmt.Errorf("creating directory %v: %w", fn, err) + } + + o, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + + defer o.Close() + + if _, err := io.Copy(o, i); err != nil { + return err + } + + return nil +} + +func (r *Store) writeManifest(packageID string, m Manifest) error { + pfp := r.manifestPath(packageID) + data, err := yaml.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling manifest: %w", err) + } + + if err := os.WriteFile(pfp, data, os.ModePerm); err != nil { + return fmt.Errorf("writing manifest file %q: %w", pfp, err) + } + + return nil +} + +func (r *Store) deleteFile(packageID string, name string) error { + // ensure the package exists + if err := r.ensurePackageExists(packageID); err != nil { + return err + } + + pkgDir := r.packageDir(packageID) + fp := filepath.Join(pkgDir, name) + + return os.Remove(fp) +} + +func (r *Store) deleteManifest(packageID string) error { + return r.deleteFile(packageID, r.ManifestFile) +} + +func (r *Store) deletePackageDir(packageID string) error { + return os.Remove(r.packageDir(packageID)) +} diff --git a/ui/v2.5/package.json b/ui/v2.5/package.json index 7542413c5..090541a25 100644 --- a/ui/v2.5/package.json +++ b/ui/v2.5/package.json @@ -8,7 +8,7 @@ "build": "vite build", "build-ci": "yarn run validate && yarn run build", "validate": "yarn run lint && yarn run check && yarn run format-check", - "lint": "yarn run lint:css && yarn run lint:js", + "lint": "yarn run lint:js && yarn run lint:css", "lint:css": "stylelint --cache \"src/**/*.scss\"", "lint:js": "eslint --cache src/", "check": "tsc --noEmit", diff --git a/ui/v2.5/src/components/List/Filters/SelectableFilter.tsx b/ui/v2.5/src/components/List/Filters/SelectableFilter.tsx index 3ce31bec7..42b37aae3 100644 --- a/ui/v2.5/src/components/List/Filters/SelectableFilter.tsx +++ b/ui/v2.5/src/components/List/Filters/SelectableFilter.tsx @@ -102,6 +102,7 @@ const SelectableFilter: React.FC = ({ onSelect, onUnselect, }) => { + const intl = useIntl(); const objects = useMemo(() => { return queryResults.filter( (p) => @@ -124,6 +125,7 @@ const SelectableFilter: React.FC = ({ focus={inputFocus} value={query} setValue={(v) => onQueryChange(v)} + placeholder={`${intl.formatMessage({ id: "actions.search" })}…`} />