From 84b6aa3cb24ac8d63793b2aef8b2ae8bf53c803f Mon Sep 17 00:00:00 2001 From: Kylart Date: Tue, 13 Oct 2020 21:08:21 +0200 Subject: [PATCH] Fixed GraphQL errors that occur with Anilist and implemented retry mechanics for Watch List information --- package-lock.json | 101 +++++++++++++----- package.json | 2 + src/main/externals/anilist/search.js | 6 +- src/main/externals/anilist/utils.js | 13 +++ src/main/services/localLists/info/index.js | 67 ++++++++++-- .../services/localLists/info/makeQuery.js | 12 ++- src/main/utils/graphql.js | 17 ++- 7 files changed, 169 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index 639f0de..b7d3f1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2198,6 +2198,17 @@ "unique-filename": "^1.1.1" } }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "optional": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, "cliui": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", @@ -2364,6 +2375,21 @@ "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", "dev": true }, + "vue-loader-v16": { + "version": "npm:vue-loader@16.0.0-beta.5", + "resolved": "https://registry.npmjs.org/vue-loader/-/vue-loader-16.0.0-beta.5.tgz", + "integrity": "sha512-ciWfzNefqWlmzKznCWY9hl+fPP4KlQ0A9MtHbJ/8DpyY+dAM8gDrjufIdxwTgC4szE4EZC3A6ip/BbrqM84GqA==", + "dev": true, + "optional": true, + "requires": { + "@types/mini-css-extract-plugin": "^0.9.1", + "chalk": "^3.0.0", + "hash-sum": "^2.0.0", + "loader-utils": "^1.2.3", + "merge-source-map": "^1.1.0", + "source-map": "^0.6.1" + } + }, "wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", @@ -6256,6 +6282,21 @@ } } }, + "cross-fetch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.6.tgz", + "integrity": "sha512-KBPUbqgFjzWlVcURG+Svp9TlhA5uliYtiNx/0r8nv0pdypeQCRJ9IaSIc3q/x3q8t3F75cHuwxVql1HFGHCNJQ==", + "requires": { + "node-fetch": "2.6.1" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + } + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -8800,6 +8841,11 @@ } } }, + "extract-files": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/extract-files/-/extract-files-9.0.0.tgz", + "integrity": "sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==" + }, "extract-zip": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz", @@ -9737,6 +9783,33 @@ "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", "dev": true }, + "graphql": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-15.3.0.tgz", + "integrity": "sha512-GTCJtzJmkFLWRfFJuoo9RWWa/FfamUHgiFosxi/X1Ani4AVWbeyBenZTNX6dM+7WSbbFfTo/25eh0LLkwHMw2w==" + }, + "graphql-request": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/graphql-request/-/graphql-request-3.1.0.tgz", + "integrity": "sha512-Flg2Bd4Ek9BDJ5qacZC/iYuiS3LroHxQTmlUnfqjo/6jKwowY25FVtoLTnssMCBrYspRYEYEIfF1GN8J3/o5JQ==", + "requires": { + "cross-fetch": "^3.0.5", + "extract-files": "^9.0.0", + "form-data": "^3.0.0" + }, + "dependencies": { + "form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "growl": { "version": "1.10.5", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", @@ -20024,34 +20097,6 @@ } } }, - "vue-loader-v16": { - "version": "npm:vue-loader@16.0.0-beta.5", - "resolved": "https://registry.npmjs.org/vue-loader/-/vue-loader-16.0.0-beta.5.tgz", - "integrity": "sha512-ciWfzNefqWlmzKznCWY9hl+fPP4KlQ0A9MtHbJ/8DpyY+dAM8gDrjufIdxwTgC4szE4EZC3A6ip/BbrqM84GqA==", - "dev": true, - "optional": true, - "requires": { - "@types/mini-css-extract-plugin": "^0.9.1", - "chalk": "^3.0.0", - "hash-sum": "^2.0.0", - "loader-utils": "^1.2.3", - "merge-source-map": "^1.1.0", - "source-map": "^0.6.1" - }, - "dependencies": { - "chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "dev": true, - "optional": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - } - } - }, "vue-router": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-3.4.3.tgz", diff --git a/package.json b/package.json index 93c7511..6be4546 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "buttercup": "^2.16.1", "chalk": "^4.1.0", "electron-updater": "^4.3.4", + "graphql": "^15.3.0", + "graphql-request": "^3.1.0", "lodash": "^4.17.20", "mal-scraper": "^2.7.2", "mime": "^2.4.6", diff --git a/src/main/externals/anilist/search.js b/src/main/externals/anilist/search.js index a550ec1..038a9a1 100644 --- a/src/main/externals/anilist/search.js +++ b/src/main/externals/anilist/search.js @@ -1,17 +1,17 @@ import { graphql } from '../../utils' -import { GRAPHQL_ENDPOINT } from './utils' +import { GRAPHQL_ENDPOINT, removeParenthesis } from './utils' import * as queries from './queries' import { formatSearch, formatInfo } from './helpers' async function searchTerm (term) { - const { data } = await graphql(GRAPHQL_ENDPOINT, queries.search, { term }) + const { data } = await graphql(GRAPHQL_ENDPOINT, queries.search, { term: removeParenthesis(term) }) return formatSearch(data) } async function fromName ({ name }) { - const { data } = await graphql(GRAPHQL_ENDPOINT, queries.info, { name }) + const { data } = await graphql(GRAPHQL_ENDPOINT, queries.info, { name: removeParenthesis(name) }) return formatInfo(data) } diff --git a/src/main/externals/anilist/utils.js b/src/main/externals/anilist/utils.js index 68e61ad..a540c52 100644 --- a/src/main/externals/anilist/utils.js +++ b/src/main/externals/anilist/utils.js @@ -5,3 +5,16 @@ export const CODE_URL = 'https://anilist.co/api/v2/oauth/authorize' export const TOKEN_URL = 'https://anilist.co/api/v2/oauth/token' export const REDIRECT_URI = 'kawanime-app://services?service=anilist' export const CLIENT_ID = config.anilist.clientId + +/** + * Remove parenthesis groups from a string. + * Because Anilist cannot handle them... + * + * @param {String} term + */ +export function removeParenthesis (term) { + return term + // Removes parenthesis groups + .replace(/\s*\([^)]*\)\s*/g, '') + .trim() +} diff --git a/src/main/services/localLists/info/index.js b/src/main/services/localLists/info/index.js index c2a845d..7eecc5c 100644 --- a/src/main/services/localLists/info/index.js +++ b/src/main/services/localLists/info/index.js @@ -12,27 +12,80 @@ const events = eventsList.localLists.info const keyPrefix = 'a' const NB_MAX_QUERIES = 20 +/** + * This method modifies its given arguments. + * Handles Not found errors in long GraphQL queries. It will feed the `failures` + * argument so that it's possible to retry failed queries without the Not Found + * names. + * + * @param {Error} error + * @param {Array} entries CurrentEntries of the query. Will be modified. + * @param {Array} failures Accumulator that will receive the error. Will be modified. + * + * @returns {undefined} + */ +function handleFailures (error, entries, failures) { + // Trying to find which query made the error + const { response: { errors }, query } = error + + errors.forEach(({ locations, status }) => { + // Only Not Found errors + if (status === 404) { + locations.forEach(({ line }) => { + const entryName = query.split('\n')[line - 1].match(/"([^)]+)"/)[0].slice(1, -1) + const entryIndex = entries.findIndex( + ({ name }) => { + return entryName === name + // treatment applied to headers + .replace(/"/g, '\\"') + .replace(/\s*\([^)]*\)\s*/g, '') + .trim() + } + ) + + if (entryIndex >= 0) { + entries.splice(entryIndex, 1) + } + }) + } + }) + + failures.push( + graphql(GRAPHQL_ENDPOINT, makeQuery(entries)) + .catch((error) => logger.error('Persistent Error for GraphQL request', error)) + ) +} + async function getInfo (entries) { // Let's make a maximum of 20 entries per query const queries = [] + const failures = [] const nbQueries = Math.floor(entries.length / NB_MAX_QUERIES) + 1 for (let i = 0; i < nbQueries; ++i) { const start = i * NB_MAX_QUERIES const end = start + NB_MAX_QUERIES - const query = makeQuery(entries.slice(start, end)) + const currentEntries = entries.slice(start, end) + const query = makeQuery(currentEntries) - queries.push(graphql(GRAPHQL_ENDPOINT, query).catch((err) => { - console.log(`Query #${i} failed`, err) - throw err - })) + queries.push( + graphql(GRAPHQL_ENDPOINT, query) + .catch((err) => handleFailures(err, currentEntries, failures)) + ) } - const results = await Promise.all(queries) + let results = await Promise.all(queries) + results = [ + ...results, + ...(await Promise.all(failures) + .catch((err) => logger.error('Could not save failures...', err)) + ) + ] return format( results + .filter(Boolean) .reduce((acc, { data }) => { Object.keys(data).forEach((key) => { acc[key] = data[key] @@ -74,7 +127,7 @@ async function handler (event, entries) { event.sender.send(events.success, storage) } catch (e) { - logger.error('An error occurred.', e.stack) + logger.error('Could not update watch list info.', e.message) event.sender.send(events.error, e.message) } } diff --git a/src/main/services/localLists/info/makeQuery.js b/src/main/services/localLists/info/makeQuery.js index 198cfea..f6efdd6 100644 --- a/src/main/services/localLists/info/makeQuery.js +++ b/src/main/services/localLists/info/makeQuery.js @@ -30,7 +30,15 @@ const CORE_QUERY = ` } ` -const getHeader = ({ key, name }) => `${keyPrefix}${key}: Media(search: "${encodeURIComponent(name).replace(/%20/g, ' ')}") {` +const getHeader = ({ key, name }) => { + const term = name + .replace(/"/g, '\\"') + // Removes parenthesis groups + .replace(/\s*\([^)]*\)\s*/g, '') + .trim() + + return `${keyPrefix}${key}: Media(search: "${term}") {` +} export default function (entries) { const queries = entries.reduce((acc, entry) => { @@ -44,5 +52,5 @@ export default function (entries) { const mainQuery = queries.join('\n') - return ['query {', mainQuery, '}'].join('\n') + return ['query KawAnime {', mainQuery, '}'].join('\n') } diff --git a/src/main/utils/graphql.js b/src/main/utils/graphql.js index 727c521..2165082 100644 --- a/src/main/utils/graphql.js +++ b/src/main/utils/graphql.js @@ -1,17 +1,16 @@ -import https from './https' +import { request, gql } from 'graphql-request' export default async function (url, query, variables, headers = {}, useCache = false) { try { - const response = await https.post(url, { - query, - variables - }, [], headers, useCache) + const data = await request(url, gql`${query}`, variables) - if (response.errors) throw new Error(response.errors[0].message) - - return response + return { data } } catch (e) { - console.log('FAILED QUERY', query, variables, headers) + e.query = query + e.variables = variables + e.headers = headers + e.url = url + throw e } }