This commit is contained in:
Cris Stringfellow 2023-01-14 16:19:39 +08:00
parent e8ebec37cb
commit df87afbed9
No known key found for this signature in database
3 changed files with 30 additions and 20 deletions

View File

@ -42,13 +42,16 @@ start();
async function start() {
console.log(`Running in node...`);
process.on('unhandledrejection', cleanup);
process.on('error', cleanup);
process.on('beforeExit', cleanup);
process.on('SIGBREAK', cleanup);
process.on('unhandledRejection', cleanup);
process.on('uncaughtException', cleanup);
process.on('SIGHUP', cleanup);
process.on('SIGINT', cleanup);
process.on('SIGTERM', cleanup);
process.on('beforeExit', cleanup);
process.on('SIGINT', code => cleanup(code, 'signal', {exit:true}));
process.on('SIGTERM', code => cleanup(code, 'signal', {exit:true}));
process.on('SIGQUIT', code => cleanup(code, 'signal', {exit:true}));
process.on('SIGBREAK', code => cleanup(code, 'signal', {exit:true}));
process.on('SIGABRT', code => cleanup(code, 'signal', {exit:true}));
console.log(`Importing dependencies...`);
const {launch:ChromeLaunch} = ChromeLauncher;
@ -110,8 +113,8 @@ async function killChrome(wait = true) {
}
}
async function cleanup(reason) {
console.log(`Cleanup called on reason: ${reason}`);
async function cleanup(reason, err, {exit = false} = {}) {
console.log(`Cleanup called on reason: ${reason}`, err);
if ( quitting ) {
console.log(`Cleanup already called so not running again.`);
@ -125,9 +128,11 @@ async function cleanup(reason) {
killChrome(false);
console.log(`Take a breath. Everything's done. 22120 is exiting in 3 seconds...`);
if ( exit ) {
console.log(`Take a breath. Everything's done. DiskerNet is exiting in 3 seconds...`);
await sleep(3000);
await sleep(3000);
process.exit(0);
process.exit(0);
}
}

View File

@ -211,9 +211,9 @@
async function collect({chrome_port:port, mode} = {}) {
const {library_path} = args;
const exitHandlers = [];
process.on('exit', runHandlers);
process.on('beforeExit', runHandlers);
process.on('SIGUSR2', runHandlers);
process.on('beforeExit', runHandlers);
process.on('exit', code => runHandlers(code, 'exit', {exit: true}));
State.connection = State.connection || await connect({port});
State.onExit = {
addHandler(h) {
@ -281,8 +281,8 @@
return Status.loaded;
function runHandlers(reason) {
console.log('before exit running', exitHandlers, reason);
async function runHandlers(reason, err, {exit = false} = {}) {
console.log('before exit running', exitHandlers, {reason, err});
while(exitHandlers.length) {
const h = exitHandlers.shift();
try {
@ -291,7 +291,11 @@
console.warn(`Error in exit handler`, h, e);
}
}
process.exit(0);
if ( exit ) {
console.log(`Exiting in 3 seconds...`);
await sleep(3000);
process.exit(0);
}
}
function handleMessage(args) {
@ -541,8 +545,8 @@
if ( ! State.titles ) {
State.titles = new Map();
State.onExit.addHandler(() => {
fs.writeFileSync(
path.resolve('.', `titles-${(new Date).toISOString()}.txt`),
Fs.writeFileSync(
Path.resolve(args.CONFIG_DIR, `titles-${(new Date).toISOString()}.txt`),
JSON.stringify([...State.titles.entries()], null, 2) + '\n'
);
});
@ -1835,7 +1839,7 @@
}
if ( saveToFile ) {
logName = `crawl-${(new Date).toISOString()}.urls.txt`;
logStream = Fs.createWriteStream(logName, {flags:'as+'});
logStream = Fs.createWriteStream(Path.resolve(args.CONFIG_DIR, logName), {flags:'as+'});
}
console.log('StartCrawl', {urls, timeout, depth, batchSize, saveToFile, minPageCrawlTime, maxPageCrawlTime, program});
State.crawling = true;

View File

@ -7,7 +7,7 @@ const mode = process.argv[3] || 'save';
const chrome_port = process.argv[4] || 9222;
const Pref = {};
const CONFIG_DIR = path.resolve(os.homedir(), '.config', 'dosyago', 'DiskerNet');
export const CONFIG_DIR = path.resolve(os.homedir(), '.config', 'dosyago', 'DiskerNet');
fs.mkdirSync(CONFIG_DIR, {recursive:true});
const pref_file = path.resolve(CONFIG_DIR, 'config.json');
const cacheId = Math.random();
@ -52,7 +52,8 @@ const args = {
ndx_fts_index_dir,
fuzzy_fts_index_dir,
results_per_page
results_per_page,
CONFIG_DIR
};
export default args;