Good
This commit is contained in:
parent
e8ebec37cb
commit
df87afbed9
25
src/app.js
25
src/app.js
|
@ -42,13 +42,16 @@ start();
|
|||
async function start() {
|
||||
console.log(`Running in node...`);
|
||||
|
||||
process.on('unhandledrejection', cleanup);
|
||||
process.on('error', cleanup);
|
||||
process.on('beforeExit', cleanup);
|
||||
process.on('SIGBREAK', cleanup);
|
||||
process.on('unhandledRejection', cleanup);
|
||||
process.on('uncaughtException', cleanup);
|
||||
process.on('SIGHUP', cleanup);
|
||||
process.on('SIGINT', cleanup);
|
||||
process.on('SIGTERM', cleanup);
|
||||
process.on('beforeExit', cleanup);
|
||||
process.on('SIGINT', code => cleanup(code, 'signal', {exit:true}));
|
||||
process.on('SIGTERM', code => cleanup(code, 'signal', {exit:true}));
|
||||
process.on('SIGQUIT', code => cleanup(code, 'signal', {exit:true}));
|
||||
process.on('SIGBREAK', code => cleanup(code, 'signal', {exit:true}));
|
||||
process.on('SIGABRT', code => cleanup(code, 'signal', {exit:true}));
|
||||
|
||||
console.log(`Importing dependencies...`);
|
||||
const {launch:ChromeLaunch} = ChromeLauncher;
|
||||
|
@ -110,8 +113,8 @@ async function killChrome(wait = true) {
|
|||
}
|
||||
}
|
||||
|
||||
async function cleanup(reason) {
|
||||
console.log(`Cleanup called on reason: ${reason}`);
|
||||
async function cleanup(reason, err, {exit = false} = {}) {
|
||||
console.log(`Cleanup called on reason: ${reason}`, err);
|
||||
|
||||
if ( quitting ) {
|
||||
console.log(`Cleanup already called so not running again.`);
|
||||
|
@ -125,9 +128,11 @@ async function cleanup(reason) {
|
|||
|
||||
killChrome(false);
|
||||
|
||||
console.log(`Take a breath. Everything's done. 22120 is exiting in 3 seconds...`);
|
||||
if ( exit ) {
|
||||
console.log(`Take a breath. Everything's done. DiskerNet is exiting in 3 seconds...`);
|
||||
|
||||
await sleep(3000);
|
||||
await sleep(3000);
|
||||
|
||||
process.exit(0);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -211,9 +211,9 @@
|
|||
async function collect({chrome_port:port, mode} = {}) {
|
||||
const {library_path} = args;
|
||||
const exitHandlers = [];
|
||||
process.on('exit', runHandlers);
|
||||
process.on('beforeExit', runHandlers);
|
||||
process.on('SIGUSR2', runHandlers);
|
||||
process.on('beforeExit', runHandlers);
|
||||
process.on('exit', code => runHandlers(code, 'exit', {exit: true}));
|
||||
State.connection = State.connection || await connect({port});
|
||||
State.onExit = {
|
||||
addHandler(h) {
|
||||
|
@ -281,8 +281,8 @@
|
|||
|
||||
return Status.loaded;
|
||||
|
||||
function runHandlers(reason) {
|
||||
console.log('before exit running', exitHandlers, reason);
|
||||
async function runHandlers(reason, err, {exit = false} = {}) {
|
||||
console.log('before exit running', exitHandlers, {reason, err});
|
||||
while(exitHandlers.length) {
|
||||
const h = exitHandlers.shift();
|
||||
try {
|
||||
|
@ -291,7 +291,11 @@
|
|||
console.warn(`Error in exit handler`, h, e);
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
if ( exit ) {
|
||||
console.log(`Exiting in 3 seconds...`);
|
||||
await sleep(3000);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
function handleMessage(args) {
|
||||
|
@ -541,8 +545,8 @@
|
|||
if ( ! State.titles ) {
|
||||
State.titles = new Map();
|
||||
State.onExit.addHandler(() => {
|
||||
fs.writeFileSync(
|
||||
path.resolve('.', `titles-${(new Date).toISOString()}.txt`),
|
||||
Fs.writeFileSync(
|
||||
Path.resolve(args.CONFIG_DIR, `titles-${(new Date).toISOString()}.txt`),
|
||||
JSON.stringify([...State.titles.entries()], null, 2) + '\n'
|
||||
);
|
||||
});
|
||||
|
@ -1835,7 +1839,7 @@
|
|||
}
|
||||
if ( saveToFile ) {
|
||||
logName = `crawl-${(new Date).toISOString()}.urls.txt`;
|
||||
logStream = Fs.createWriteStream(logName, {flags:'as+'});
|
||||
logStream = Fs.createWriteStream(Path.resolve(args.CONFIG_DIR, logName), {flags:'as+'});
|
||||
}
|
||||
console.log('StartCrawl', {urls, timeout, depth, batchSize, saveToFile, minPageCrawlTime, maxPageCrawlTime, program});
|
||||
State.crawling = true;
|
||||
|
|
|
@ -7,7 +7,7 @@ const mode = process.argv[3] || 'save';
|
|||
const chrome_port = process.argv[4] || 9222;
|
||||
|
||||
const Pref = {};
|
||||
const CONFIG_DIR = path.resolve(os.homedir(), '.config', 'dosyago', 'DiskerNet');
|
||||
export const CONFIG_DIR = path.resolve(os.homedir(), '.config', 'dosyago', 'DiskerNet');
|
||||
fs.mkdirSync(CONFIG_DIR, {recursive:true});
|
||||
const pref_file = path.resolve(CONFIG_DIR, 'config.json');
|
||||
const cacheId = Math.random();
|
||||
|
@ -52,7 +52,8 @@ const args = {
|
|||
ndx_fts_index_dir,
|
||||
fuzzy_fts_index_dir,
|
||||
|
||||
results_per_page
|
||||
results_per_page,
|
||||
CONFIG_DIR
|
||||
};
|
||||
|
||||
export default args;
|
||||
|
|
Loading…
Reference in New Issue