Rooba
/
Diskernet
mirror of https://github.com/crisdosyago/Diskernet.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Build works

This commit is contained in:
Cris Stringfellow 2023-01-12 17:37:59 +08:00
parent 9c80cb8dc2
commit 79663006d9
No known key found for this signature in database
12 changed files with 641 additions and 9168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
.package.build.json
View File

@ -1,77 +0,0 @@
{
"name": "diskernet",
"version": "2.6.0",
"description": "Library server and an archivist browser controller.",
"main": "build/22120.js",
"module": "src/app.js",
"bin": {
"diskernet": "build/22120.js"
},
"scripts": {
"bundle": "npx rollup --config",
"start": "node src/app.js",
"setup": "bash ./scripts/build_setup.sh",
"build": "echo 'please run ./scripts/build_setup.sh first (one time only) and then ./scripts/compile.sh'",
"clean": "rm -rf build/* bin/*",
"test": "nodemon src/app.js",
"save": "nodemon src/app.js 22120 save",
"serve": "nodemon src/app.js 22120 serve",
"lint": "watch -n 5 npx eslint .",
"test-hl": "node src/highlighter.js",
"postpublish": "cp package.json .package.build.json",
"prepublishOnly": "npm run bundle && npx webpack ."
},
"repository": {
"type": "git",
"url": "git+https://github.com/dosyago/22120.git"
},
"pkg": {
"patches": {
"./node_modules/fetch-blob/streams.cjs": [
"Object.assign(globalThis, require('node:stream/web'))",
"Object.assign(globalThis, require('stream').web)"
]
},
"scripts": "build/*.js",
"assets": "public/**/*",
"outputPath": "bin"
},
"keywords": [
"web-archive",
"search-engine",
"self-hosted",
"offline",
"archivist",
"library"
],
"author": "@dosy",
"license": "AGPL-3.0",
"bugs": {
"url": "https://github.com/dosyago/22120/issues"
},
"homepage": "https://github.com/dosyago/22120#readme",
"dependencies": {
"chrome-launcher": "latest",
"express": "latest",
"flexsearch": "^0.7.21",
"fz-search": "^1.0.0",
"hasha": "latest",
"natural": "^5.1.11",
"ndx": "^1.0.2",
"ndx-query": "^1.0.1",
"ndx-serializable": "^1.0.0",
"node-fetch": "latest",
"ukkonen": "^1.4.0",
"ws": "latest"
},
"devDependencies": {
"eslint": "^8.4.1",
"esm": "^3.2.25",
"nexe": "^1.1.6",
"nodemon": "latest",
"npx": "^3.0.0",
"webpack": "latest",
"webpack-cli": "latest",
"rollup-plugin-terser": "^7.0.2"
}
}

67
.package.dev.json
View File

@ -1,67 +0,0 @@
{
"name": "diskernet",
"version": "2.4.11",
"type": "module",
"description": "Library server and an archivist browser controller.",
"main": "src/app.js",
"module": "dist/22120-module.js",
"bin": {
"diskernet": "build/22120.js"
},
"scripts": {
"bundle": "npx rollup --config",
"start": "node src/app.js",
"setup": "bash ./scripts/build_setup.sh",
"build": "echo 'please run ./scripts/build_setup.sh first (one time only) and then ./scripts/compile.sh'",
"clean": "rm -rf build/* bin/*",
"test": "nodemon src/app.js",
"save": "nodemon src/app.js 22120 save",
"serve": "nodemon src/app.js 22120 serve",
"lint": "watch -n 5 npx eslint .",
"test-hl": "node src/highlighter.js",
"postpublish": "cp package.json .package.build.json",
"prepublishOnly": "npm run bundle && npx webpack ."
},
"repository": {
"type": "git",
"url": "git+https://github.com/dosyago/22120.git"
},
"pkg": {
"scripts": "build/*.js",
"assets": "public/**/*",
"outputPath": "bin"
},
"keywords": [
"archivist",
"library"
],
"author": "@dosy",
"license": "AGPL-3.0",
"bugs": {
"url": "https://github.com/dosyago/22120/issues"
},
"homepage": "https://github.com/dosyago/22120#readme",
"dependencies": {
"chrome-launcher": "latest",
"express": "latest",
"flexsearch": "^0.7.21",
"fz-search": "^1.0.0",
"hasha": "latest",
"natural": "^5.1.11",
"ndx": "^1.0.2",
"ndx-query": "^1.0.1",
"ndx-serializable": "^1.0.0",
"node-fetch": "latest",
"ukkonen": "^1.4.0",
"ws": "latest"
},
"devDependencies": {
"eslint": "^8.4.1",
"esm": "^3.2.25",
"nexe": "^1.1.6",
"nodemon": "latest",
"npx": "^3.0.0",
"webpack": "latest",
"webpack-cli": "latest"
}
}

9386
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

20
package.json
View File

@ -6,12 +6,12 @@
"main": "src/app.js",
"module": "module/22120-module.js",
"bin": {
"diskernet": "dist/22120.cjs"
"diskernet": "build/out.cjs"
},
"scripts": {
"start": "node src/app.js",
"setup": "bash ./scripts/build_setup.sh",
"build": "echo 'please run ./scripts/build_setup.sh first (one time only) and then ./scripts/compile.sh'",
"build": "echo Ensure you 'npm run setup' first && bash ./scripts/compile.sh",
"clean": "rm -rf build/* bin/*",
"super-clean": "npm run clean || : && rm -rf node_modules || : && rm package-lock.json",
"test": "nodemon src/app.js",
@ -20,15 +20,17 @@
"lint": "watch -n 5 npx eslint .",
"test-hl": "node src/highlighter.js",
"postpublish": "cp package.json .package.build.json",
"prepublishOnly": "npm run bundle && npx webpack ."
"prepublishOnly": "./node_modules/.bin/esbuild src/app.js --bundle --outfile=build/out.cjs --platform=node"
},
"repository": {
"type": "git",
"url": "git+https://github.com/dosyago/22120.git"
},
"pkg": {
"scripts": "dist/**/*.js",
"assets": ["dist/**/*.js", "public/**/*"],
"scripts": "build/**/*.js",
"assets": [
"public/**/*"
],
"outputPath": "bin"
},
"keywords": [
@ -56,12 +58,8 @@
"ws": "latest"
},
"devDependencies": {
"esbuild": "0.16.17",
"eslint": "^8.4.1",
"esm": "^3.2.25",
"nexe": "^1.1.6",
"nodemon": "latest",
"npx": "^3.0.0",
"webpack": "latest",
"webpack-cli": "latest"
"nodemon": "latest"
}
}

48
scripts/compile.sh
View File

@ -1,48 +1,4 @@
#!/usr/bin/env bash
unset npm_config_prefix
source $HOME/.nvm/nvm.sh
. $HOME/.profile
nvm use --lts
echo "Cleaning old build and dist files..."
rm -rf build/* dist/*
echo "Setting build (CJS) mode..."
./scripts/go_build.sh
patch_required=$(grep -ER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*)
files=$(grep -rlER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*)
if [[ ! -z "$patch_required" ]]; then
while IFS= read -r file; do
#echo '--->' $file
#grep -q $file package.json
#if [ $? == 1 ]; then
echo '--->' $file "UNPATCHED!"
echo "Found an error!"
echo "Found something you need to patch before building"
echo "See: https://github.com/vercel/pkg/issues/1451"
echo
echo "$patch_required"
echo
echo "You need to add all these to pkg.patches to replace with require('stream').web"
./scripts/go_dev.sh
exit 1
#fi
#echo "OK"
done <<< $files
fi
echo "Bundling javascript..."
export NODE_ENV='production'
npx webpack
chmod +x ./build/22120.js
echo "Building for windows nix and macos..."
npx pkg --compress Gzip .
echo "Restoring dev (ES module) mode..."
./scripts/go_dev.sh
echo "Rebundling an es module for npm es module import..."
npm run bundle
./node_modules/.bin/esbuild src/app.js --bundle --outfile=build/out.cjs --platform=node
pkg --compress GZip .

48
scripts/old_compile.sh Executable file
View File

@ -0,0 +1,48 @@
#!/usr/bin/env bash
unset npm_config_prefix
source $HOME/.nvm/nvm.sh
. $HOME/.profile
nvm use --lts
echo "Cleaning old build and dist files..."
rm -rf build/* dist/*
echo "Setting build (CJS) mode..."
./scripts/go_build.sh
patch_required=$(grep -ER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*)
files=$(grep -rlER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*)
if [[ ! -z "$patch_required" ]]; then
while IFS= read -r file; do
#echo '--->' $file
#grep -q $file package.json
#if [ $? == 1 ]; then
echo '--->' $file "UNPATCHED!"
echo "Found an error!"
echo "Found something you need to patch before building"
echo "See: https://github.com/vercel/pkg/issues/1451"
echo
echo "$patch_required"
echo
echo "You need to add all these to pkg.patches to replace with require('stream').web"
./scripts/go_dev.sh
exit 1
#fi
#echo "OK"
done <<< $files
fi
echo "Bundling javascript..."
export NODE_ENV='production'
npx webpack
chmod +x ./build/22120.js
echo "Building for windows nix and macos..."
npx pkg --compress Gzip .
echo "Restoring dev (ES module) mode..."
./scripts/go_dev.sh
echo "Rebundling an es module for npm es module import..."
npm run bundle

54
src/.common.build.js
View File

@ -1,54 +0,0 @@
import path from 'path';
//import {fileURLToPath} from 'url';
export const DEBUG = process.env.DEBUG_22120 || false;
export const SHOW_FETCH = false;
export const CHECK_INTERVAL = 400;
export const TEXT_NODE = 3;
export const MAX_HIGHLIGHTABLE_LENGTH = 0; /* 0 is no max length for highlight */
export const MAX_TITLE_LENGTH = 140;
export const MAX_URL_LENGTH = 140;
export const MAX_HEAD = 140;
/* text nodes inside these elements that are ignored */
export const FORBIDDEN_TEXT_PARENT = new Set([
'STYLE',
'SCRIPT',
'NOSCRIPT',
/* we could remove these last two so as to index them as well */
'DATALIST',
'OPTION'
]);
export const ERROR_CODE_SAFE_TO_IGNORE = new Set([
-32000, /* message:
Can only get response body on requests captured after headers received.
* ignore because:
seems to only happen when new navigation aborts all
pending requests of the unloading page
*/
-32602, /* message:
Invalid InterceptionId.
* ignore because:
seems to only happen when new navigation aborts all
pending requests of the unloading page
*/
]);
export const SNIP_CONTEXT = 31;
export const NO_SANDBOX = process.env.DEBUG_22120 || false;
//export const APP_ROOT = '.';
export const APP_ROOT = __dirname;
//export const APP_ROOT = path.dirname(fileURLToPath(import.meta.url));
export const sleep = ms => new Promise(res => setTimeout(res, ms));
export function say(o) {
console.log(JSON.stringify(o));
}
export function clone(o) {
return JSON.parse(JSON.stringify(o));
}

54
src/.common.dev.js
View File

@ -1,54 +0,0 @@
import path from 'path';
import {fileURLToPath} from 'url';
export const DEBUG = process.env.DEBUG_22120 || false;
export const SHOW_FETCH = false;
export const CHECK_INTERVAL = 400;
export const TEXT_NODE = 3;
export const MAX_HIGHLIGHTABLE_LENGTH = 0; /* 0 is no max length for highlight */
export const MAX_TITLE_LENGTH = 140;
export const MAX_URL_LENGTH = 140;
export const MAX_HEAD = 140;
/* text nodes inside these elements that are ignored */
export const FORBIDDEN_TEXT_PARENT = new Set([
'STYLE',
'SCRIPT',
'NOSCRIPT',
/* we could remove these last two so as to index them as well */
'DATALIST',
'OPTION'
]);
export const ERROR_CODE_SAFE_TO_IGNORE = new Set([
-32000, /* message:
Can only get response body on requests captured after headers received.
* ignore because:
seems to only happen when new navigation aborts all
pending requests of the unloading page
*/
-32602, /* message:
Invalid InterceptionId.
* ignore because:
seems to only happen when new navigation aborts all
pending requests of the unloading page
*/
]);
export const SNIP_CONTEXT = 31;
export const NO_SANDBOX = process.env.DEBUG_22120 || false;
//export const APP_ROOT = '.';
//export const APP_ROOT = __dirname;
export const APP_ROOT = path.dirname(fileURLToPath(import.meta.url));
export const sleep = ms => new Promise(res => setTimeout(res, ms));
export function say(o) {
console.log(JSON.stringify(o));
}
export function clone(o) {
return JSON.parse(JSON.stringify(o));
}

14
src/app.js
View File

@ -1,3 +1,8 @@
import fs from 'fs';
import ChildProcess from 'child_process';
import ChromeLauncher from 'chrome-launcher';
import {DEBUG, sleep, NO_SANDBOX} from './common.js';
import Archivist from './archivist.js';
@ -30,7 +35,7 @@ const KILL_ON = {
linux: 'pkill -15 chrome',
};
let quitting, ChildProcess;
let quitting;
start();
@ -44,8 +49,7 @@ async function start() {
process.on('SIGTERM', cleanup);
console.log(`Importing dependencies...`);
const fs = await import('fs');
const {launch:ChromeLaunch} = await import('chrome-launcher');
const {launch:ChromeLaunch} = ChromeLauncher;
await killChrome();
@ -83,10 +87,6 @@ async function killChrome(wait = true) {
try {
if ( process.platform in KILL_ON ) {
console.log(`Attempting to shut running chrome...`);
if ( ! ChildProcess ) {
const {default:child_process} = await import('child_process');
ChildProcess = child_process;
}
const [err] = (await new Promise(
res => ChildProcess.exec(KILL_ON[process.platform], (...a) => res(a))
));

6
src/archivist.js
View File

@ -15,9 +15,9 @@
import readline from 'readline';
// search related
//import FlexSearch from 'flexsearch';
//const {Index: FTSIndex} = FlexSearch;
const {Index: FTSIndex} = require('flexsearch');
import FlexSearch from 'flexsearch';
const {Index: FTSIndex} = FlexSearch;
//const {Index: FTSIndex} = require('flexsearch');
import {
createIndex as NDX,
addDocumentToIndex as ndx,

2
src/common.js
View File

@ -41,7 +41,7 @@ export const NO_SANDBOX = process.env.DEBUG_22120 || false;
//export const APP_ROOT = '.';
//export const APP_ROOT = __dirname;
export const APP_ROOT = path.dirname(fileURLToPath(import.meta.url));
export const APP_ROOT = path.dirname(process.argv[0]);
export const sleep = ms => new Promise(res => setTimeout(res, ms));

33
webpack.config.cjs
View File

@ -1,33 +0,0 @@
const path = require('path');
const webpack = require('webpack');
module.exports = {
entry: "./src/app.js",
output: {
path: path.resolve(__dirname, 'dist'),
filename: "22120.cjs"
},
optimization: {
minimize: false
},
target: "node",
node: {
__dirname: false
},
externalsPresets: {
node: true
},
externals: [
],
plugins: [
new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }),
],
module: {
rules: [
{
test: /\.node$/,
loader: 'node-loader',
},
],
},
};