Restructure scraping settings (#1548)

* Change scrapers overview into collapsible per type
* Move scraping configuration to (renamed) scrapers tab

Rename the Scrapers tab to Scraping and move the scraping configuration
to this tab.
This commit is contained in:
gitgiggety 2021-08-04 04:32:58 +02:00 committed by GitHub
parent 518be8ee70
commit ac41416cd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 242 additions and 107 deletions

View File

@ -64,6 +64,12 @@ fragment ConfigDLNAData on ConfigDLNAResult {
interfaces
}
fragment ConfigScrapingData on ConfigScrapingResult {
scraperUserAgent
scraperCertCheck
scraperCDPPath
}
fragment ConfigData on ConfigResult {
general {
...ConfigGeneralData
@ -74,4 +80,7 @@ fragment ConfigData on ConfigResult {
dlna {
...ConfigDLNAData
}
scraping {
...ConfigScrapingData
}
}

View File

@ -24,6 +24,12 @@ mutation ConfigureDLNA($input: ConfigDLNAInput!) {
}
}
mutation ConfigureScraping($input: ConfigScrapingInput!) {
configureScraping(input: $input) {
...ConfigScrapingData
}
}
mutation GenerateAPIKey($input: GenerateAPIKeyInput!) {
generateAPIKey(input: $input)
}

View File

@ -212,6 +212,7 @@ type Mutation {
configureGeneral(input: ConfigGeneralInput!): ConfigGeneralResult!
configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult!
configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult!
configureScraping(input: ConfigScrapingInput!): ConfigScrapingResult!
"""Generate and set (or clear) API key"""
generateAPIKey(input: GenerateAPIKeyInput!): String!

View File

@ -90,11 +90,11 @@ input ConfigGeneralInput {
"""Array of file regexp to exclude from Image Scans"""
imageExcludes: [String!]
"""Scraper user agent string"""
scraperUserAgent: String
scraperUserAgent: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
"""Scraper CDP path. Path to chrome executable or remote address"""
scraperCDPPath: String
scraperCDPPath: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
"""Whether the scraper should check for invalid certificates"""
scraperCertCheck: Boolean!
scraperCertCheck: Boolean @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
"""Stash-box instances used for tagging"""
stashBoxes: [StashBoxInput!]!
}
@ -163,11 +163,11 @@ type ConfigGeneralResult {
"""Array of file regexp to exclude from Image Scans"""
imageExcludes: [String!]!
"""Scraper user agent string"""
scraperUserAgent: String
scraperUserAgent: String @deprecated(reason: "use ConfigResult.scraping instead")
"""Scraper CDP path. Path to chrome executable or remote address"""
scraperCDPPath: String
scraperCDPPath: String @deprecated(reason: "use ConfigResult.scraping instead")
"""Whether the scraper should check for invalid certificates"""
scraperCertCheck: Boolean!
scraperCertCheck: Boolean! @deprecated(reason: "use ConfigResult.scraping instead")
"""Stash-box instances used for tagging"""
stashBoxes: [StashBox!]!
}
@ -244,11 +244,30 @@ type ConfigDLNAResult {
interfaces: [String!]!
}
input ConfigScrapingInput {
"""Scraper user agent string"""
scraperUserAgent: String
"""Scraper CDP path. Path to chrome executable or remote address"""
scraperCDPPath: String
"""Whether the scraper should check for invalid certificates"""
scraperCertCheck: Boolean!
}
type ConfigScrapingResult {
"""Scraper user agent string"""
scraperUserAgent: String
"""Scraper CDP path. Path to chrome executable or remote address"""
scraperCDPPath: String
"""Whether the scraper should check for invalid certificates"""
scraperCertCheck: Boolean!
}
"""All configuration settings"""
type ConfigResult {
general: ConfigGeneralResult!
interface: ConfigInterfaceResult!
dlna: ConfigDLNAResult!
scraping: ConfigScrapingResult!
}
"""Directory structure of a path"""

View File

@ -178,7 +178,9 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
refreshScraperCache = true
}
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
if input.ScraperCertCheck != nil {
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
}
if input.StashBoxes != nil {
if err := c.ValidateStashBoxes(input.StashBoxes); err != nil {
@ -291,6 +293,31 @@ func (r *mutationResolver) ConfigureDlna(ctx context.Context, input models.Confi
return makeConfigDLNAResult(), nil
}
func (r *mutationResolver) ConfigureScraping(ctx context.Context, input models.ConfigScrapingInput) (*models.ConfigScrapingResult, error) {
c := config.GetInstance()
refreshScraperCache := false
if input.ScraperUserAgent != nil {
c.Set(config.ScraperUserAgent, input.ScraperUserAgent)
refreshScraperCache = true
}
if input.ScraperCDPPath != nil {
c.Set(config.ScraperCDPPath, input.ScraperCDPPath)
refreshScraperCache = true
}
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
if refreshScraperCache {
manager.GetInstance().RefreshScraperCache()
}
if err := c.Write(); err != nil {
return makeConfigScrapingResult(), err
}
return makeConfigScrapingResult(), nil
}
func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) {
c := config.GetInstance()

View File

@ -39,6 +39,7 @@ func makeConfigResult() *models.ConfigResult {
General: makeConfigGeneralResult(),
Interface: makeConfigInterfaceResult(),
Dlna: makeConfigDLNAResult(),
Scraping: makeConfigScrapingResult(),
}
}
@ -132,3 +133,16 @@ func makeConfigDLNAResult() *models.ConfigDLNAResult {
Interfaces: config.GetDLNAInterfaces(),
}
}
func makeConfigScrapingResult() *models.ConfigScrapingResult {
config := config.GetInstance()
scraperUserAgent := config.GetScraperUserAgent()
scraperCDPPath := config.GetScraperCDPPath()
return &models.ConfigScrapingResult{
ScraperUserAgent: &scraperUserAgent,
ScraperCertCheck: config.GetScraperCertCheck(),
ScraperCDPPath: &scraperCDPPath,
}
}

View File

@ -1,7 +1,7 @@
import React, { useEffect } from "react";
import { Route, Switch, useRouteMatch } from "react-router-dom";
import { IntlProvider } from "react-intl";
import { merge } from "lodash";
import { mergeWith } from "lodash";
import { ToastProvider } from "src/hooks/Toast";
import LightboxProvider from "src/hooks/Lightbox/context";
import { library } from "@fortawesome/fontawesome-svg-core";
@ -59,11 +59,16 @@ export const App: React.FC = () => {
const messageLanguage = languageMessageString(language);
// use en-GB as default messages if any messages aren't found in the chosen language
const mergedMessages = merge(
const mergedMessages = mergeWith(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(locales as any)[defaultMessageLanguage],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(locales as any)[messageLanguage]
(locales as any)[messageLanguage],
(objVal, srcVal) => {
if (srcVal === "") {
return objVal;
}
}
);
const messages = flattenMessages(mergedMessages);

View File

@ -3,6 +3,7 @@
* Added not equals/greater than/less than modifiers for resolution criteria. ([#1568](https://github.com/stashapp/stash/pull/1568))
### 🎨 Improvements
* Moved scraping settings into the Scraping settings page. ([#1548](https://github.com/stashapp/stash/pull/1548))
* Show current scene details in tagger view. ([#1605](https://github.com/stashapp/stash/pull/1605))
* Removed stripes and added background colour to default performer images (old images can be downloaded from the PR link). ([#1609](https://github.com/stashapp/stash/pull/1609))
* Added pt-BR language option. ([#1587](https://github.com/stashapp/stash/pull/1587))

View File

@ -9,7 +9,7 @@ import { SettingsInterfacePanel } from "./SettingsInterfacePanel/SettingsInterfa
import { SettingsLogsPanel } from "./SettingsLogsPanel";
import { SettingsTasksPanel } from "./SettingsTasksPanel/SettingsTasksPanel";
import { SettingsPluginsPanel } from "./SettingsPluginsPanel";
import { SettingsScrapersPanel } from "./SettingsScrapersPanel";
import { SettingsScrapingPanel } from "./SettingsScrapingPanel";
import { SettingsToolsPanel } from "./SettingsToolsPanel";
import { SettingsDLNAPanel } from "./SettingsDLNAPanel";
@ -54,8 +54,8 @@ export const Settings: React.FC = () => {
</Nav.Link>
</Nav.Item>
<Nav.Item>
<Nav.Link eventKey="scrapers">
<FormattedMessage id="config.categories.scrapers" />
<Nav.Link eventKey="scraping">
<FormattedMessage id="config.categories.scraping" />
</Nav.Link>
</Nav.Item>
<Nav.Item>
@ -93,8 +93,8 @@ export const Settings: React.FC = () => {
<Tab.Pane eventKey="tools" unmountOnExit>
<SettingsToolsPanel />
</Tab.Pane>
<Tab.Pane eventKey="scrapers" unmountOnExit>
<SettingsScrapersPanel />
<Tab.Pane eventKey="scraping" unmountOnExit>
<SettingsScrapingPanel />
</Tab.Pane>
<Tab.Pane eventKey="plugins" unmountOnExit>
<SettingsPluginsPanel />

View File

@ -126,13 +126,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
const [excludes, setExcludes] = useState<string[]>([]);
const [imageExcludes, setImageExcludes] = useState<string[]>([]);
const [scraperUserAgent, setScraperUserAgent] = useState<string | undefined>(
undefined
);
const [scraperCDPPath, setScraperCDPPath] = useState<string | undefined>(
undefined
);
const [scraperCertCheck, setScraperCertCheck] = useState<boolean>(true);
const [stashBoxes, setStashBoxes] = useState<IStashBoxInstance[]>([]);
const { data, error, loading } = useConfiguration();
@ -173,9 +166,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
galleryExtensions: commaDelimitedToList(galleryExtensions),
excludes,
imageExcludes,
scraperUserAgent,
scraperCDPPath,
scraperCertCheck,
stashBoxes: stashBoxes.map(
(b) =>
({
@ -223,9 +213,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
);
setExcludes(conf.general.excludes);
setImageExcludes(conf.general.imageExcludes);
setScraperUserAgent(conf.general.scraperUserAgent ?? undefined);
setScraperCDPPath(conf.general.scraperCDPPath ?? undefined);
setScraperCertCheck(conf.general.scraperCertCheck);
setStashBoxes(
conf.general.stashBoxes.map((box, i) => ({
name: box?.name ?? undefined,
@ -830,59 +817,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
</Form.Group>
</Form.Group>
<Form.Group>
<h4>{intl.formatMessage({ id: "config.general.scraping" })}</h4>
<Form.Group id="scraperUserAgent">
<h6>
{intl.formatMessage({ id: "config.general.scraper_user_agent" })}
</h6>
<Form.Control
className="col col-sm-6 text-input"
defaultValue={scraperUserAgent}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setScraperUserAgent(e.currentTarget.value)
}
/>
<Form.Text className="text-muted">
{intl.formatMessage({
id: "config.general.scraper_user_agent_desc",
})}
</Form.Text>
</Form.Group>
<Form.Group id="scraperCDPPath">
<h6>
{intl.formatMessage({ id: "config.general.chrome_cdp_path" })}
</h6>
<Form.Control
className="col col-sm-6 text-input"
defaultValue={scraperCDPPath}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setScraperCDPPath(e.currentTarget.value)
}
/>
<Form.Text className="text-muted">
{intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })}
</Form.Text>
</Form.Group>
<Form.Group>
<Form.Check
id="scaper-cert-check"
checked={scraperCertCheck}
label={intl.formatMessage({
id: "config.general.check_for_insecure_certificates",
})}
onChange={() => setScraperCertCheck(!scraperCertCheck)}
/>
<Form.Text className="text-muted">
{intl.formatMessage({
id: "config.general.check_for_insecure_certificates_desc",
})}
</Form.Text>
</Form.Group>
</Form.Group>
<hr />
<Form.Group id="stashbox">

View File

@ -1,16 +1,18 @@
import React, { useState } from "react";
import React, { useEffect, useState } from "react";
import { FormattedMessage, useIntl } from "react-intl";
import { Button } from "react-bootstrap";
import { Button, Form } from "react-bootstrap";
import {
mutateReloadScrapers,
useListMovieScrapers,
useListPerformerScrapers,
useListSceneScrapers,
useListGalleryScrapers,
useConfiguration,
useConfigureScraping,
} from "src/core/StashService";
import { useToast } from "src/hooks";
import { TextUtils } from "src/utils";
import { Icon, LoadingIndicator } from "src/components/Shared";
import { CollapseButton, Icon, LoadingIndicator } from "src/components/Shared";
import { ScrapeType } from "src/core/generated-graphql";
interface IURLList {
@ -67,7 +69,7 @@ const URLList: React.FC<IURLList> = ({ urls }) => {
return <ul>{getListItems()}</ul>;
};
export const SettingsScrapersPanel: React.FC = () => {
export const SettingsScrapingPanel: React.FC = () => {
const Toast = useToast();
const intl = useIntl();
const {
@ -87,17 +89,62 @@ export const SettingsScrapersPanel: React.FC = () => {
loading: loadingMovies,
} = useListMovieScrapers();
const [scraperUserAgent, setScraperUserAgent] = useState<string | undefined>(
undefined
);
const [scraperCDPPath, setScraperCDPPath] = useState<string | undefined>(
undefined
);
const [scraperCertCheck, setScraperCertCheck] = useState<boolean>(true);
const { data, error } = useConfiguration();
const [updateScrapingConfig] = useConfigureScraping({
scraperUserAgent,
scraperCDPPath,
scraperCertCheck,
});
useEffect(() => {
if (!data?.configuration || error) return;
const conf = data.configuration;
if (conf.scraping) {
setScraperUserAgent(conf.scraping.scraperUserAgent ?? undefined);
setScraperCDPPath(conf.scraping.scraperCDPPath ?? undefined);
setScraperCertCheck(conf.scraping.scraperCertCheck);
}
}, [data, error]);
async function onReloadScrapers() {
await mutateReloadScrapers().catch((e) => Toast.error(e));
}
async function onSave() {
try {
await updateScrapingConfig();
Toast.success({
content: intl.formatMessage(
{ id: "toast.updated_entity" },
{
entity: intl
.formatMessage({ id: "configuration" })
.toLocaleLowerCase(),
}
),
});
} catch (e) {
Toast.error(e);
}
}
function renderPerformerScrapeTypes(types: ScrapeType[]) {
const typeStrings = types
.filter((t) => t !== ScrapeType.Fragment)
.map((t) => {
switch (t) {
case ScrapeType.Name:
return intl.formatMessage({ id: "config.scrapers.search_by_name" });
return intl.formatMessage({ id: "config.scraping.search_by_name" });
default:
return t;
}
@ -117,7 +164,7 @@ export const SettingsScrapersPanel: React.FC = () => {
switch (t) {
case ScrapeType.Fragment:
return intl.formatMessage(
{ id: "config.scrapers.entity_metadata" },
{ id: "config.scraping.entity_metadata" },
{ entityType: intl.formatMessage({ id: "scene" }) }
);
default:
@ -139,7 +186,7 @@ export const SettingsScrapersPanel: React.FC = () => {
switch (t) {
case ScrapeType.Fragment:
return intl.formatMessage(
{ id: "config.scrapers.entity_metadata" },
{ id: "config.scraping.entity_metadata" },
{ entityType: intl.formatMessage({ id: "gallery" }) }
);
default:
@ -161,7 +208,7 @@ export const SettingsScrapersPanel: React.FC = () => {
switch (t) {
case ScrapeType.Fragment:
return intl.formatMessage(
{ id: "config.scrapers.entity_metadata" },
{ id: "config.scraping.entity_metadata" },
{ entityType: intl.formatMessage({ id: "movie" }) }
);
default:
@ -195,7 +242,7 @@ export const SettingsScrapersPanel: React.FC = () => {
return renderTable(
intl.formatMessage(
{ id: "config.scrapers.entity_scrapers" },
{ id: "config.scraping.entity_scrapers" },
{ entityType: intl.formatMessage({ id: "scene" }) }
),
elements
@ -217,7 +264,7 @@ export const SettingsScrapersPanel: React.FC = () => {
return renderTable(
intl.formatMessage(
{ id: "config.scrapers.entity_scrapers" },
{ id: "config.scraping.entity_scrapers" },
{ entityType: intl.formatMessage({ id: "gallery" }) }
),
elements
@ -241,7 +288,7 @@ export const SettingsScrapersPanel: React.FC = () => {
return renderTable(
intl.formatMessage(
{ id: "config.scrapers.entity_scrapers" },
{ id: "config.scraping.entity_scrapers" },
{ entityType: intl.formatMessage({ id: "performer" }) }
),
elements
@ -261,7 +308,7 @@ export const SettingsScrapersPanel: React.FC = () => {
return renderTable(
intl.formatMessage(
{ id: "config.scrapers.entity_scrapers" },
{ id: "config.scraping.entity_scrapers" },
{ entityType: intl.formatMessage({ id: "movie" }) }
),
elements
@ -271,25 +318,24 @@ export const SettingsScrapersPanel: React.FC = () => {
function renderTable(title: string, elements: JSX.Element[]) {
if (elements.length > 0) {
return (
<div className="mb-2">
<h5>{title}</h5>
<CollapseButton text={title}>
<table className="scraper-table">
<thead>
<tr>
<th>{intl.formatMessage({ id: "name" })}</th>
<th>
{intl.formatMessage({
id: "config.scrapers.supported_types",
id: "config.scraping.supported_types",
})}
</th>
<th>
{intl.formatMessage({ id: "config.scrapers.supported_urls" })}
{intl.formatMessage({ id: "config.scraping.supported_urls" })}
</th>
</tr>
</thead>
<tbody>{elements}</tbody>
</table>
</div>
</CollapseButton>
);
}
}
@ -299,7 +345,63 @@ export const SettingsScrapersPanel: React.FC = () => {
return (
<>
<h4>{intl.formatMessage({ id: "config.categories.scrapers" })}</h4>
<Form.Group>
<h4>{intl.formatMessage({ id: "config.general.scraping" })}</h4>
<Form.Group id="scraperUserAgent">
<h6>
{intl.formatMessage({ id: "config.general.scraper_user_agent" })}
</h6>
<Form.Control
className="col col-sm-6 text-input"
defaultValue={scraperUserAgent}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setScraperUserAgent(e.currentTarget.value)
}
/>
<Form.Text className="text-muted">
{intl.formatMessage({
id: "config.general.scraper_user_agent_desc",
})}
</Form.Text>
</Form.Group>
<Form.Group id="scraperCDPPath">
<h6>
{intl.formatMessage({ id: "config.general.chrome_cdp_path" })}
</h6>
<Form.Control
className="col col-sm-6 text-input"
defaultValue={scraperCDPPath}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setScraperCDPPath(e.currentTarget.value)
}
/>
<Form.Text className="text-muted">
{intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })}
</Form.Text>
</Form.Group>
<Form.Group>
<Form.Check
id="scaper-cert-check"
checked={scraperCertCheck}
label={intl.formatMessage({
id: "config.general.check_for_insecure_certificates",
})}
onChange={() => setScraperCertCheck(!scraperCertCheck)}
/>
<Form.Text className="text-muted">
{intl.formatMessage({
id: "config.general.check_for_insecure_certificates_desc",
})}
</Form.Text>
</Form.Group>
</Form.Group>
<hr />
<h4>{intl.formatMessage({ id: "config.scraping.scrapers" })}</h4>
<div className="mb-3">
<Button onClick={() => onReloadScrapers()}>
<span className="fa-icon">
@ -317,6 +419,12 @@ export const SettingsScrapersPanel: React.FC = () => {
{renderPerformerScrapers()}
{renderMovieScrapers()}
</div>
<hr />
<Button variant="primary" onClick={() => onSave()}>
<FormattedMessage id="actions.save" />
</Button>
</>
);
};

View File

@ -761,6 +761,13 @@ export const useRemoveTempDLNAIP = () => GQL.useRemoveTempDlnaipMutation();
export const useLoggingSubscribe = () => GQL.useLoggingSubscribeSubscription();
export const useConfigureScraping = (input: GQL.ConfigScrapingInput) =>
GQL.useConfigureScrapingMutation({
variables: { input },
refetchQueries: getQueryNames([GQL.ConfigurationDocument]),
update: deleteCache([GQL.ConfigurationDocument]),
});
export const querySystemStatus = () =>
client.query<GQL.SystemStatusQuery>({
query: GQL.SystemStatusDocument,

View File

@ -151,7 +151,7 @@
"interface": "Oberfläche",
"logs": "Protokoll",
"plugins": "Plugins",
"scrapers": "Scraper",
"scraping": "Scraping",
"tasks": "Aufgaben",
"tools": "Werkzeuge"
},
@ -240,9 +240,10 @@
"hooks": "Hooks",
"triggers_on": "Auslösen bei"
},
"scrapers": {
"scraping": {
"entity_metadata": "{entityType} Metadaten",
"entity_scrapers": "{entityType} Scraper",
"scrapers": "Scraper",
"search_by_name": "Suche nach Name",
"supported_types": "Unterstützte Typen",
"supported_urls": "URLs"

View File

@ -151,7 +151,7 @@
"interface": "Interface",
"logs": "Logs",
"plugins": "Plugins",
"scrapers": "Scrapers",
"scraping": "Scraping",
"tasks": "Tasks",
"tools": "Tools"
},
@ -240,9 +240,10 @@
"hooks": "Hooks",
"triggers_on": "Triggers on"
},
"scrapers": {
"scraping": {
"entity_metadata": "{entityType} Metadata",
"entity_scrapers": "{entityType} scrapers",
"scrapers": "Scrapers",
"search_by_name": "Search by name",
"supported_types": "Supported types",
"supported_urls": "URLs"

View File

@ -151,7 +151,7 @@
"interface": "Interface",
"logs": "Logs",
"plugins": "Plugins",
"scrapers": "Scrapers",
"scraping": "Scraping",
"tasks": "Tarefas",
"tools": "Ferramentas"
},
@ -240,9 +240,10 @@
"hooks": "Hooks",
"triggers_on": "Triggers on"
},
"scrapers": {
"scraping": {
"entity_metadata": "{entityType} metadados",
"entity_scrapers": "{entityType} scrapers",
"scrapers": "Scrapers",
"search_by_name": "Buscar por nome",
"supported_types": "Tipos suportados",
"supported_urls": "URLs"

View File

@ -144,7 +144,7 @@
"interface": "介面",
"logs": "日誌",
"plugins": "插件",
"scrapers": "爬蟲",
"scraping": "爬蟲設定",
"tasks": "排程",
"tools": "工具"
},
@ -229,9 +229,10 @@
"logs": {
"log_level": "日誌級別"
},
"scrapers": {
"scraping": {
"entity_metadata": "{entityType}資訊",
"entity_scrapers": "{entityType}爬蟲",
"scrapers": "爬蟲",
"search_by_name": "透過名稱搜尋",
"supported_types": "支援類型",
"supported_urls": "支援網址"