diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 878f53298..329e9fb3b 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -64,6 +64,12 @@ fragment ConfigDLNAData on ConfigDLNAResult { interfaces } +fragment ConfigScrapingData on ConfigScrapingResult { + scraperUserAgent + scraperCertCheck + scraperCDPPath +} + fragment ConfigData on ConfigResult { general { ...ConfigGeneralData @@ -74,4 +80,7 @@ fragment ConfigData on ConfigResult { dlna { ...ConfigDLNAData } + scraping { + ...ConfigScrapingData + } } diff --git a/graphql/documents/mutations/config.graphql b/graphql/documents/mutations/config.graphql index 5d3e0e734..149d9bf28 100644 --- a/graphql/documents/mutations/config.graphql +++ b/graphql/documents/mutations/config.graphql @@ -24,6 +24,12 @@ mutation ConfigureDLNA($input: ConfigDLNAInput!) { } } +mutation ConfigureScraping($input: ConfigScrapingInput!) { + configureScraping(input: $input) { + ...ConfigScrapingData + } +} + mutation GenerateAPIKey($input: GenerateAPIKeyInput!) { generateAPIKey(input: $input) } diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 5ac192854..c0ba269ef 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -212,6 +212,7 @@ type Mutation { configureGeneral(input: ConfigGeneralInput!): ConfigGeneralResult! configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult! configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult! + configureScraping(input: ConfigScrapingInput!): ConfigScrapingResult! """Generate and set (or clear) API key""" generateAPIKey(input: GenerateAPIKeyInput!): String! diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index d9c3ae52c..d4251b82c 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -90,11 +90,11 @@ input ConfigGeneralInput { """Array of file regexp to exclude from Image Scans""" imageExcludes: [String!] """Scraper user agent string""" - scraperUserAgent: String + scraperUserAgent: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead") """Scraper CDP path. Path to chrome executable or remote address""" - scraperCDPPath: String + scraperCDPPath: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead") """Whether the scraper should check for invalid certificates""" - scraperCertCheck: Boolean! + scraperCertCheck: Boolean @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead") """Stash-box instances used for tagging""" stashBoxes: [StashBoxInput!]! } @@ -163,11 +163,11 @@ type ConfigGeneralResult { """Array of file regexp to exclude from Image Scans""" imageExcludes: [String!]! """Scraper user agent string""" - scraperUserAgent: String + scraperUserAgent: String @deprecated(reason: "use ConfigResult.scraping instead") """Scraper CDP path. Path to chrome executable or remote address""" - scraperCDPPath: String + scraperCDPPath: String @deprecated(reason: "use ConfigResult.scraping instead") """Whether the scraper should check for invalid certificates""" - scraperCertCheck: Boolean! + scraperCertCheck: Boolean! @deprecated(reason: "use ConfigResult.scraping instead") """Stash-box instances used for tagging""" stashBoxes: [StashBox!]! } @@ -244,11 +244,30 @@ type ConfigDLNAResult { interfaces: [String!]! } +input ConfigScrapingInput { + """Scraper user agent string""" + scraperUserAgent: String + """Scraper CDP path. Path to chrome executable or remote address""" + scraperCDPPath: String + """Whether the scraper should check for invalid certificates""" + scraperCertCheck: Boolean! +} + +type ConfigScrapingResult { + """Scraper user agent string""" + scraperUserAgent: String + """Scraper CDP path. Path to chrome executable or remote address""" + scraperCDPPath: String + """Whether the scraper should check for invalid certificates""" + scraperCertCheck: Boolean! +} + """All configuration settings""" type ConfigResult { general: ConfigGeneralResult! interface: ConfigInterfaceResult! dlna: ConfigDLNAResult! + scraping: ConfigScrapingResult! } """Directory structure of a path""" diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index 7154ff219..6c10647ca 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -178,7 +178,9 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co refreshScraperCache = true } - c.Set(config.ScraperCertCheck, input.ScraperCertCheck) + if input.ScraperCertCheck != nil { + c.Set(config.ScraperCertCheck, input.ScraperCertCheck) + } if input.StashBoxes != nil { if err := c.ValidateStashBoxes(input.StashBoxes); err != nil { @@ -291,6 +293,31 @@ func (r *mutationResolver) ConfigureDlna(ctx context.Context, input models.Confi return makeConfigDLNAResult(), nil } +func (r *mutationResolver) ConfigureScraping(ctx context.Context, input models.ConfigScrapingInput) (*models.ConfigScrapingResult, error) { + c := config.GetInstance() + + refreshScraperCache := false + if input.ScraperUserAgent != nil { + c.Set(config.ScraperUserAgent, input.ScraperUserAgent) + refreshScraperCache = true + } + + if input.ScraperCDPPath != nil { + c.Set(config.ScraperCDPPath, input.ScraperCDPPath) + refreshScraperCache = true + } + + c.Set(config.ScraperCertCheck, input.ScraperCertCheck) + if refreshScraperCache { + manager.GetInstance().RefreshScraperCache() + } + if err := c.Write(); err != nil { + return makeConfigScrapingResult(), err + } + + return makeConfigScrapingResult(), nil +} + func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) { c := config.GetInstance() diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index a3e226073..cf284142a 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -39,6 +39,7 @@ func makeConfigResult() *models.ConfigResult { General: makeConfigGeneralResult(), Interface: makeConfigInterfaceResult(), Dlna: makeConfigDLNAResult(), + Scraping: makeConfigScrapingResult(), } } @@ -132,3 +133,16 @@ func makeConfigDLNAResult() *models.ConfigDLNAResult { Interfaces: config.GetDLNAInterfaces(), } } + +func makeConfigScrapingResult() *models.ConfigScrapingResult { + config := config.GetInstance() + + scraperUserAgent := config.GetScraperUserAgent() + scraperCDPPath := config.GetScraperCDPPath() + + return &models.ConfigScrapingResult{ + ScraperUserAgent: &scraperUserAgent, + ScraperCertCheck: config.GetScraperCertCheck(), + ScraperCDPPath: &scraperCDPPath, + } +} diff --git a/ui/v2.5/src/App.tsx b/ui/v2.5/src/App.tsx index fa506466b..c1560d1f5 100755 --- a/ui/v2.5/src/App.tsx +++ b/ui/v2.5/src/App.tsx @@ -1,7 +1,7 @@ import React, { useEffect } from "react"; import { Route, Switch, useRouteMatch } from "react-router-dom"; import { IntlProvider } from "react-intl"; -import { merge } from "lodash"; +import { mergeWith } from "lodash"; import { ToastProvider } from "src/hooks/Toast"; import LightboxProvider from "src/hooks/Lightbox/context"; import { library } from "@fortawesome/fontawesome-svg-core"; @@ -59,11 +59,16 @@ export const App: React.FC = () => { const messageLanguage = languageMessageString(language); // use en-GB as default messages if any messages aren't found in the chosen language - const mergedMessages = merge( + const mergedMessages = mergeWith( // eslint-disable-next-line @typescript-eslint/no-explicit-any (locales as any)[defaultMessageLanguage], // eslint-disable-next-line @typescript-eslint/no-explicit-any - (locales as any)[messageLanguage] + (locales as any)[messageLanguage], + (objVal, srcVal) => { + if (srcVal === "") { + return objVal; + } + } ); const messages = flattenMessages(mergedMessages); diff --git a/ui/v2.5/src/components/Changelog/versions/v090.md b/ui/v2.5/src/components/Changelog/versions/v090.md index d656a7ce9..1c5d14414 100644 --- a/ui/v2.5/src/components/Changelog/versions/v090.md +++ b/ui/v2.5/src/components/Changelog/versions/v090.md @@ -3,6 +3,7 @@ * Added not equals/greater than/less than modifiers for resolution criteria. ([#1568](https://github.com/stashapp/stash/pull/1568)) ### 🎨 Improvements +* Moved scraping settings into the Scraping settings page. ([#1548](https://github.com/stashapp/stash/pull/1548)) * Show current scene details in tagger view. ([#1605](https://github.com/stashapp/stash/pull/1605)) * Removed stripes and added background colour to default performer images (old images can be downloaded from the PR link). ([#1609](https://github.com/stashapp/stash/pull/1609)) * Added pt-BR language option. ([#1587](https://github.com/stashapp/stash/pull/1587)) diff --git a/ui/v2.5/src/components/Settings/Settings.tsx b/ui/v2.5/src/components/Settings/Settings.tsx index 027cad854..49c8f6c38 100644 --- a/ui/v2.5/src/components/Settings/Settings.tsx +++ b/ui/v2.5/src/components/Settings/Settings.tsx @@ -9,7 +9,7 @@ import { SettingsInterfacePanel } from "./SettingsInterfacePanel/SettingsInterfa import { SettingsLogsPanel } from "./SettingsLogsPanel"; import { SettingsTasksPanel } from "./SettingsTasksPanel/SettingsTasksPanel"; import { SettingsPluginsPanel } from "./SettingsPluginsPanel"; -import { SettingsScrapersPanel } from "./SettingsScrapersPanel"; +import { SettingsScrapingPanel } from "./SettingsScrapingPanel"; import { SettingsToolsPanel } from "./SettingsToolsPanel"; import { SettingsDLNAPanel } from "./SettingsDLNAPanel"; @@ -54,8 +54,8 @@ export const Settings: React.FC = () => { - - + + @@ -93,8 +93,8 @@ export const Settings: React.FC = () => { - - + + diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index 4f942632d..c20383c05 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -126,13 +126,6 @@ export const SettingsConfigurationPanel: React.FC = () => { const [excludes, setExcludes] = useState([]); const [imageExcludes, setImageExcludes] = useState([]); - const [scraperUserAgent, setScraperUserAgent] = useState( - undefined - ); - const [scraperCDPPath, setScraperCDPPath] = useState( - undefined - ); - const [scraperCertCheck, setScraperCertCheck] = useState(true); const [stashBoxes, setStashBoxes] = useState([]); const { data, error, loading } = useConfiguration(); @@ -173,9 +166,6 @@ export const SettingsConfigurationPanel: React.FC = () => { galleryExtensions: commaDelimitedToList(galleryExtensions), excludes, imageExcludes, - scraperUserAgent, - scraperCDPPath, - scraperCertCheck, stashBoxes: stashBoxes.map( (b) => ({ @@ -223,9 +213,6 @@ export const SettingsConfigurationPanel: React.FC = () => { ); setExcludes(conf.general.excludes); setImageExcludes(conf.general.imageExcludes); - setScraperUserAgent(conf.general.scraperUserAgent ?? undefined); - setScraperCDPPath(conf.general.scraperCDPPath ?? undefined); - setScraperCertCheck(conf.general.scraperCertCheck); setStashBoxes( conf.general.stashBoxes.map((box, i) => ({ name: box?.name ?? undefined, @@ -830,59 +817,6 @@ export const SettingsConfigurationPanel: React.FC = () => { - -

{intl.formatMessage({ id: "config.general.scraping" })}

- -
- {intl.formatMessage({ id: "config.general.scraper_user_agent" })} -
- ) => - setScraperUserAgent(e.currentTarget.value) - } - /> - - {intl.formatMessage({ - id: "config.general.scraper_user_agent_desc", - })} - - - - -
- {intl.formatMessage({ id: "config.general.chrome_cdp_path" })} -
- ) => - setScraperCDPPath(e.currentTarget.value) - } - /> - - {intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })} - - - - - setScraperCertCheck(!scraperCertCheck)} - /> - - {intl.formatMessage({ - id: "config.general.check_for_insecure_certificates_desc", - })} - - - -
diff --git a/ui/v2.5/src/components/Settings/SettingsScrapersPanel.tsx b/ui/v2.5/src/components/Settings/SettingsScrapingPanel.tsx similarity index 63% rename from ui/v2.5/src/components/Settings/SettingsScrapersPanel.tsx rename to ui/v2.5/src/components/Settings/SettingsScrapingPanel.tsx index 86003cee3..51e81cd7c 100644 --- a/ui/v2.5/src/components/Settings/SettingsScrapersPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsScrapingPanel.tsx @@ -1,16 +1,18 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { Button } from "react-bootstrap"; +import { Button, Form } from "react-bootstrap"; import { mutateReloadScrapers, useListMovieScrapers, useListPerformerScrapers, useListSceneScrapers, useListGalleryScrapers, + useConfiguration, + useConfigureScraping, } from "src/core/StashService"; import { useToast } from "src/hooks"; import { TextUtils } from "src/utils"; -import { Icon, LoadingIndicator } from "src/components/Shared"; +import { CollapseButton, Icon, LoadingIndicator } from "src/components/Shared"; import { ScrapeType } from "src/core/generated-graphql"; interface IURLList { @@ -67,7 +69,7 @@ const URLList: React.FC = ({ urls }) => { return
    {getListItems()}
; }; -export const SettingsScrapersPanel: React.FC = () => { +export const SettingsScrapingPanel: React.FC = () => { const Toast = useToast(); const intl = useIntl(); const { @@ -87,17 +89,62 @@ export const SettingsScrapersPanel: React.FC = () => { loading: loadingMovies, } = useListMovieScrapers(); + const [scraperUserAgent, setScraperUserAgent] = useState( + undefined + ); + const [scraperCDPPath, setScraperCDPPath] = useState( + undefined + ); + const [scraperCertCheck, setScraperCertCheck] = useState(true); + + const { data, error } = useConfiguration(); + + const [updateScrapingConfig] = useConfigureScraping({ + scraperUserAgent, + scraperCDPPath, + scraperCertCheck, + }); + + useEffect(() => { + if (!data?.configuration || error) return; + + const conf = data.configuration; + if (conf.scraping) { + setScraperUserAgent(conf.scraping.scraperUserAgent ?? undefined); + setScraperCDPPath(conf.scraping.scraperCDPPath ?? undefined); + setScraperCertCheck(conf.scraping.scraperCertCheck); + } + }, [data, error]); + async function onReloadScrapers() { await mutateReloadScrapers().catch((e) => Toast.error(e)); } + async function onSave() { + try { + await updateScrapingConfig(); + Toast.success({ + content: intl.formatMessage( + { id: "toast.updated_entity" }, + { + entity: intl + .formatMessage({ id: "configuration" }) + .toLocaleLowerCase(), + } + ), + }); + } catch (e) { + Toast.error(e); + } + } + function renderPerformerScrapeTypes(types: ScrapeType[]) { const typeStrings = types .filter((t) => t !== ScrapeType.Fragment) .map((t) => { switch (t) { case ScrapeType.Name: - return intl.formatMessage({ id: "config.scrapers.search_by_name" }); + return intl.formatMessage({ id: "config.scraping.search_by_name" }); default: return t; } @@ -117,7 +164,7 @@ export const SettingsScrapersPanel: React.FC = () => { switch (t) { case ScrapeType.Fragment: return intl.formatMessage( - { id: "config.scrapers.entity_metadata" }, + { id: "config.scraping.entity_metadata" }, { entityType: intl.formatMessage({ id: "scene" }) } ); default: @@ -139,7 +186,7 @@ export const SettingsScrapersPanel: React.FC = () => { switch (t) { case ScrapeType.Fragment: return intl.formatMessage( - { id: "config.scrapers.entity_metadata" }, + { id: "config.scraping.entity_metadata" }, { entityType: intl.formatMessage({ id: "gallery" }) } ); default: @@ -161,7 +208,7 @@ export const SettingsScrapersPanel: React.FC = () => { switch (t) { case ScrapeType.Fragment: return intl.formatMessage( - { id: "config.scrapers.entity_metadata" }, + { id: "config.scraping.entity_metadata" }, { entityType: intl.formatMessage({ id: "movie" }) } ); default: @@ -195,7 +242,7 @@ export const SettingsScrapersPanel: React.FC = () => { return renderTable( intl.formatMessage( - { id: "config.scrapers.entity_scrapers" }, + { id: "config.scraping.entity_scrapers" }, { entityType: intl.formatMessage({ id: "scene" }) } ), elements @@ -217,7 +264,7 @@ export const SettingsScrapersPanel: React.FC = () => { return renderTable( intl.formatMessage( - { id: "config.scrapers.entity_scrapers" }, + { id: "config.scraping.entity_scrapers" }, { entityType: intl.formatMessage({ id: "gallery" }) } ), elements @@ -241,7 +288,7 @@ export const SettingsScrapersPanel: React.FC = () => { return renderTable( intl.formatMessage( - { id: "config.scrapers.entity_scrapers" }, + { id: "config.scraping.entity_scrapers" }, { entityType: intl.formatMessage({ id: "performer" }) } ), elements @@ -261,7 +308,7 @@ export const SettingsScrapersPanel: React.FC = () => { return renderTable( intl.formatMessage( - { id: "config.scrapers.entity_scrapers" }, + { id: "config.scraping.entity_scrapers" }, { entityType: intl.formatMessage({ id: "movie" }) } ), elements @@ -271,25 +318,24 @@ export const SettingsScrapersPanel: React.FC = () => { function renderTable(title: string, elements: JSX.Element[]) { if (elements.length > 0) { return ( -
-
{title}
+ {elements}
{intl.formatMessage({ id: "name" })} {intl.formatMessage({ - id: "config.scrapers.supported_types", + id: "config.scraping.supported_types", })} - {intl.formatMessage({ id: "config.scrapers.supported_urls" })} + {intl.formatMessage({ id: "config.scraping.supported_urls" })}
-
+ ); } } @@ -299,7 +345,63 @@ export const SettingsScrapersPanel: React.FC = () => { return ( <> -

{intl.formatMessage({ id: "config.categories.scrapers" })}

+ +

{intl.formatMessage({ id: "config.general.scraping" })}

+ +
+ {intl.formatMessage({ id: "config.general.scraper_user_agent" })} +
+ ) => + setScraperUserAgent(e.currentTarget.value) + } + /> + + {intl.formatMessage({ + id: "config.general.scraper_user_agent_desc", + })} + + + + +
+ {intl.formatMessage({ id: "config.general.chrome_cdp_path" })} +
+ ) => + setScraperCDPPath(e.currentTarget.value) + } + /> + + {intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })} + + + + + setScraperCertCheck(!scraperCertCheck)} + /> + + {intl.formatMessage({ + id: "config.general.check_for_insecure_certificates_desc", + })} + + + + +
+ +

{intl.formatMessage({ id: "config.scraping.scrapers" })}

+
+ +
+ + ); }; diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index b0adfdbc8..44fc90b10 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -761,6 +761,13 @@ export const useRemoveTempDLNAIP = () => GQL.useRemoveTempDlnaipMutation(); export const useLoggingSubscribe = () => GQL.useLoggingSubscribeSubscription(); +export const useConfigureScraping = (input: GQL.ConfigScrapingInput) => + GQL.useConfigureScrapingMutation({ + variables: { input }, + refetchQueries: getQueryNames([GQL.ConfigurationDocument]), + update: deleteCache([GQL.ConfigurationDocument]), + }); + export const querySystemStatus = () => client.query({ query: GQL.SystemStatusDocument, diff --git a/ui/v2.5/src/locales/de-DE.json b/ui/v2.5/src/locales/de-DE.json index e60a157c2..e22e410cf 100644 --- a/ui/v2.5/src/locales/de-DE.json +++ b/ui/v2.5/src/locales/de-DE.json @@ -151,7 +151,7 @@ "interface": "Oberfläche", "logs": "Protokoll", "plugins": "Plugins", - "scrapers": "Scraper", + "scraping": "Scraping", "tasks": "Aufgaben", "tools": "Werkzeuge" }, @@ -240,9 +240,10 @@ "hooks": "Hooks", "triggers_on": "Auslösen bei" }, - "scrapers": { + "scraping": { "entity_metadata": "{entityType} Metadaten", "entity_scrapers": "{entityType} Scraper", + "scrapers": "Scraper", "search_by_name": "Suche nach Name", "supported_types": "Unterstützte Typen", "supported_urls": "URLs" diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 5ea435dbc..b29a24db7 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -151,7 +151,7 @@ "interface": "Interface", "logs": "Logs", "plugins": "Plugins", - "scrapers": "Scrapers", + "scraping": "Scraping", "tasks": "Tasks", "tools": "Tools" }, @@ -240,9 +240,10 @@ "hooks": "Hooks", "triggers_on": "Triggers on" }, - "scrapers": { + "scraping": { "entity_metadata": "{entityType} Metadata", "entity_scrapers": "{entityType} scrapers", + "scrapers": "Scrapers", "search_by_name": "Search by name", "supported_types": "Supported types", "supported_urls": "URLs" diff --git a/ui/v2.5/src/locales/pt-BR.json b/ui/v2.5/src/locales/pt-BR.json index d7d4e620d..c56139dcd 100644 --- a/ui/v2.5/src/locales/pt-BR.json +++ b/ui/v2.5/src/locales/pt-BR.json @@ -151,7 +151,7 @@ "interface": "Interface", "logs": "Logs", "plugins": "Plugins", - "scrapers": "Scrapers", + "scraping": "Scraping", "tasks": "Tarefas", "tools": "Ferramentas" }, @@ -240,9 +240,10 @@ "hooks": "Hooks", "triggers_on": "Triggers on" }, - "scrapers": { + "scraping": { "entity_metadata": "{entityType} metadados", "entity_scrapers": "{entityType} scrapers", + "scrapers": "Scrapers", "search_by_name": "Buscar por nome", "supported_types": "Tipos suportados", "supported_urls": "URLs" diff --git a/ui/v2.5/src/locales/zh-TW.json b/ui/v2.5/src/locales/zh-TW.json index ebf1beeb3..83afac442 100644 --- a/ui/v2.5/src/locales/zh-TW.json +++ b/ui/v2.5/src/locales/zh-TW.json @@ -144,7 +144,7 @@ "interface": "介面", "logs": "日誌", "plugins": "插件", - "scrapers": "爬蟲", + "scraping": "爬蟲設定", "tasks": "排程", "tools": "工具" }, @@ -229,9 +229,10 @@ "logs": { "log_level": "日誌級別" }, - "scrapers": { + "scraping": { "entity_metadata": "{entityType}資訊", "entity_scrapers": "{entityType}爬蟲", + "scrapers": "爬蟲", "search_by_name": "透過名稱搜尋", "supported_types": "支援類型", "supported_urls": "支援網址"