Rooba
/
Diskernet
mirror of https://github.com/crisdosyago/Diskernet.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

"Adding search"

This commit is contained in:
Cris Stringfellow 2021-12-10 17:16:13 +08:00
parent e939adceb0
commit 9478a93729
3 changed files with 20 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
archivist.js
View File

@ -2,6 +2,7 @@ import hasha from 'hasha';
import {URL} from 'url'; import {URL} from 'url';
import path from 'path'; import path from 'path';
import fs from 'fs'; import fs from 'fs';
import FlexSearch from 'flexsearch';
import args from './args.js'; import args from './args.js';
import {APP_ROOT, context, sleep, DEBUG} from './common.js'; import {APP_ROOT, context, sleep, DEBUG} from './common.js';
import {connect} from './protocol.js'; import {connect} from './protocol.js';
@ -14,6 +15,11 @@ import {BLOCKED_BODY, BLOCKED_CODE, BLOCKED_HEADERS} from './blockedResponse.js'
// that holds the serialized requests // that holds the serialized requests
// that are saved on disk // that are saved on disk
let Fs, Mode, Close; let Fs, Mode, Close;
const {Index, registerCharset, registerLanguage} = FlexSearch;
const FLEX_OPTS = {
context: true,
};
const Flex = new Index(FLEX_OPTS);
const Cache = new Map(); const Cache = new Map();
const State = { const State = {
Cache, Cache,
@ -234,6 +240,7 @@ async function collect({chrome_port:port, mode} = {}) {
(Text, {nodeValue}) => Text + nodeValue + ' ', (Text, {nodeValue}) => Text + nodeValue + ' ',
'' ''
); );
if ( false ) { if ( false ) {
console.log({ console.log({
page : { page : {
@ -243,6 +250,7 @@ async function collect({chrome_port:port, mode} = {}) {
} }
}); });
} }
Flex.updateAsync(info.url, pageText);
} }
console.log(`Indexed ${info.url} to ${info.title}`); console.log(`Indexed ${info.url} to ${info.title}`);

11
package-lock.json generated
View File

@ -13,6 +13,7 @@
"chrome-launcher": "latest", "chrome-launcher": "latest",
"esm": "^3.2.25", "esm": "^3.2.25",
"express": "latest", "express": "latest",
"flexsearch": "^0.7.21",
"hasha": "latest", "hasha": "latest",
"node-fetch": "latest", "node-fetch": "latest",
"ws": "latest" "ws": "latest"
@ -1899,6 +1900,11 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/flexsearch": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.21.tgz",
"integrity": "sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg=="
},
"node_modules/forwarded": { "node_modules/forwarded": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -10891,6 +10897,11 @@
"path-exists": "^4.0.0" "path-exists": "^4.0.0"
} }
}, },
"flexsearch": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.21.tgz",
"integrity": "sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg=="
},
"forwarded": { "forwarded": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",

1
package.json
View File

@ -35,6 +35,7 @@
"chrome-launcher": "latest", "chrome-launcher": "latest",
"esm": "^3.2.25", "esm": "^3.2.25",
"express": "latest", "express": "latest",
"flexsearch": "^0.7.21",
"hasha": "latest", "hasha": "latest",
"node-fetch": "latest", "node-fetch": "latest",
"ws": "latest" "ws": "latest"