Merge pull request #817 from fosscord/dev/new-scripts

Add utility scripts

6 files changed, 228 insertions, 31 deletions

diff --git a/.gitignore b/.gitignore
index 607b4f5a..0f1d9332 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,6 @@ api/assets/plugins/*.js
 .idea/
 *.code-workspace
 
+
+*.log
+*.log.ansi
\ No newline at end of file
diff --git a/bundle/package.json b/bundle/package.json
index 311fab18..49aadeb8 100644
--- a/bundle/package.json
+++ b/bundle/package.json
@@ -4,7 +4,9 @@
 	"description": "",
 	"main": "src/start.js",
 	"scripts": {
-		"setup": "node scripts/install.js && npm install --no-optional && npx ts-patch install -s && npx patch-package --patch-dir ../api/patches/ && npm run build",
+		"setup": "node scripts/install.js && npm install --omit optional && ts-patch install -s && patch-package --patch-dir ../api/patches/ && npm run build",
+		"depclean": "node scripts/depclean.js",
+		"depcheck": "node scripts/depcheck.js",
 		"build": "node scripts/build.js",
 		"start": "node scripts/build.js && node dist/bundle/src/start.js",
 		"start:bundle": "node dist/bundle/src/start.js",
diff --git a/bundle/scripts/build.js b/bundle/scripts/build.js
index f73fb11a..f53a2223 100644
--- a/bundle/scripts/build.js
+++ b/bundle/scripts/build.js
@@ -2,39 +2,36 @@ const { execSync } = require("child_process");
 const path = require("path");
 const fs = require("fs");
 const { getSystemErrorMap } = require("util");
-const { argv } = require("process");
+const { argv, stdout, exit } = require("process");
+const { copyRecursiveSync, execIn, parts } = require('./utils');
 
-var steps = 2, i = 0;
+if(argv.includes("help")) {
+	console.log(`Fosscord build script help:
+Arguments:
+  clean			Cleans up previous builds
+  copyonly		Only copy source files, don't build (useful for updating assets)
+  verbose		Enable verbose logging
+  logerrors		Log build errors to console
+  pretty-errors		Pretty-print build errors
+  silent		No output to console or files.`);
+	exit(0);
+}
+
+let steps = 3, i = 0;
 if (argv.includes("clean")) steps++;
 if (argv.includes("copyonly")) steps--;
-const dirs = ["api", "util", "cdn", "gateway", "bundle"];
 
 const verbose = argv.includes("verbose") || argv.includes("v");
+const logerr = argv.includes("logerrors");
+const pretty = argv.includes("pretty-errors");
+const silent = argv.includes("silent");
 
-var copyRecursiveSync = function(src, dest) {
-	if(verbose) console.log(`cpsync: ${src} -> ${dest}`);
-	var exists = fs.existsSync(src);
-	if(!exists){
-		console.log(src + " doesn't exist, not copying!");
-		return;
-	}
-	var stats = exists && fs.statSync(src);
-	var isDirectory = exists && stats.isDirectory();
-	if (isDirectory) {
-	  fs.mkdirSync(dest, {recursive: true});
-	  fs.readdirSync(src).forEach(function(childItemName) {
-		copyRecursiveSync(path.join(src, childItemName),
-						  path.join(dest, childItemName));
-	  });
-	} else {
-	  fs.copyFileSync(src, dest);
-	}
-  };
+if(silent) console.error = console.log = function(){}
 
 if (argv.includes("clean")) {
 	console.log(`[${++i}/${steps}] Cleaning...`);
-	dirs.forEach((a) => {
-		var d = "../" + a + "/dist";
+	parts.forEach((a) => {
+		let d = "../" + a + "/dist";
 		if (fs.existsSync(d)) {
 			fs.rmSync(d, { recursive: true });
 			if (verbose) console.log(`Deleted ${d}!`);
@@ -42,11 +39,16 @@ if (argv.includes("clean")) {
 	});
 }
 
+console.log(`[${++i}/${steps}] Checking if dependencies were installed correctly...`);
+//exif-be-gone v1.3.0 doesnt build js, known bug
+if(!fs.existsSync(path.join(__dirname, "..", "node_modules", "exif-be-gone", "index.js")))
+	execIn("npm run build", path.join(__dirname, "..", "node_modules", "exif-be-gone"));
+
 console.log(`[${++i}/${steps}] Copying src files...`);
 copyRecursiveSync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
 copyRecursiveSync(path.join(__dirname, "..", "..", "api", "client_test"), path.join(__dirname, "..", "dist", "api", "client_test"));
 copyRecursiveSync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
-dirs.forEach((a) => {
+parts.forEach((a) => {
 	copyRecursiveSync("../" + a + "/src", "dist/" + a + "/src");
 	if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`);
 });
@@ -54,13 +56,16 @@ dirs.forEach((a) => {
 if (!argv.includes("copyonly")) {
 	console.log(`[${++i}/${steps}] Compiling src files ...`);
 
-	console.log(
+	let buildFlags = ''
+	if(pretty) buildFlags += '--pretty '
+
+	try {
 		execSync(
 			'node "' +
 				path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") +
 				'" -p "' +
 				path.join(__dirname, "..") +
-				'"',
+				'" ' + buildFlags,
 			{
 				cwd: path.join(__dirname, ".."),
 				shell: true,
@@ -68,6 +73,21 @@ if (!argv.includes("copyonly")) {
 				encoding: "utf8"
 			}
 		)
-	);
-}
-
+	} catch (error) {
+		if(verbose || logerr) {
+			error.stdout.split(/\r?\n/).forEach((line) => {
+				let _line = line.replace('dist/','',1);
+				if(!pretty && _line.includes('.ts(')) {
+					//reformat file path for easy jumping
+					_line = _line.replace('(',':',1).replace(',',':',1).replace(')','',1)
+				}
+				console.error(_line);
+			})
+		}
+		console.error(`Build failed! Please check build.log for info!`);
+		if(!silent){
+			if(pretty) fs.writeFileSync("build.log.ansi",  error.stdout);
+			fs.writeFileSync("build.log",  error.stdout.replaceAll(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, ''));
+		}
+	}
+}
\ No newline at end of file
diff --git a/bundle/scripts/depcheck.js b/bundle/scripts/depcheck.js
new file mode 100644
index 00000000..1957f794
--- /dev/null
+++ b/bundle/scripts/depcheck.js
@@ -0,0 +1,56 @@
+const path = require("path");
+const fs = require("fs");
+const { env } = require("process");
+const { execSync } = require("child_process");
+const { argv, stdout, exit } = require("process");
+
+const { execIn, getLines, parts } = require("./utils");
+
+let npmi_extra_flags = "";
+
+const resolveminor = argv.includes("resolveminor");
+if(argv.includes("nobuild")) npmi_extra_flags += "--ignore-scripts ";
+
+parts.forEach((part) => {
+	let partDir = path.join(__dirname, "..", "..", part);
+	let distDir = path.join(partDir, "dist");
+	console.log(`Checking updates for ${part} (${partDir})`);
+	if(part == "bundle") {
+		execIn(`npm run syncdeps`, partDir)
+	}
+    if(resolveminor) {
+        fs.rmSync(path.join(partDir, "node_modules"), {
+			recursive: true,
+			force: true,
+		});
+        execIn(`npm i --save --no-fund --no-audit --no-package-lock ${npmi_extra_flags}`, partDir)
+    }
+	let x = [
+		[
+			"pkg",
+			{
+				current: "1.0",
+				wanted: "2.0",
+				latest: "2.0",
+				dependent: "cdn",
+				location: "/usr/src/fosscord/bundle/node_packages/pkg",
+			},
+		],
+	];
+	x = Object.entries(
+		JSON.parse(execIn("npm outdated --json", partDir))
+	);
+	x.forEach((a) => {
+        let pkgname = a[0];
+        let pkginfo = a[1];
+        if(!pkginfo.current)
+            console.log(`MISSING ${pkgname}: ${pkginfo.current} -> ${pkginfo.wanted} (latest: ${pkginfo.latest})`);
+        else if(pkginfo.latest != pkginfo.wanted){
+            if(pkginfo.current != pkginfo.wanted) 
+                console.log(`MINOR   ${pkgname}: ${pkginfo.current} -> ${pkginfo.wanted}`);
+		    console.log(`MAJOR   ${pkgname}: ${pkginfo.current} -> ${pkginfo.latest}`);
+        }
+        else
+            console.log(`MINOR   ${pkgname}: ${pkginfo.current} -> ${pkginfo.wanted}`);
+	});
+});
diff --git a/bundle/scripts/depclean.js b/bundle/scripts/depclean.js
new file mode 100644
index 00000000..6d9a2260
--- /dev/null
+++ b/bundle/scripts/depclean.js
@@ -0,0 +1,63 @@
+const path = require("path");
+const fs = require("fs");
+const { env } = require("process");
+const { execSync } = require("child_process");
+const { argv, stdout, exit } = require("process");
+
+const { execIn, getLines, parts } = require('./utils');
+
+const bundleRequired = ["@ovos-media/ts-transform-paths"];
+const removeModules = argv.includes("cleanup");
+
+parts.forEach((part) => {
+	console.log(`Installing all packages for ${part}...`);
+	execIn("npm i", path.join(__dirname, "..", "..", part));
+});
+
+parts.forEach((part) => {
+	let partDir = path.join(__dirname, "..", "..", part);
+	let distDir = path.join(partDir, "dist");
+	let start = 0;
+	start = getLines(
+		execIn("npm ls --parseable --package-lock-only -a", partDir)
+	);
+	if (fs.existsSync(distDir))
+		fs.rmSync(distDir, {
+			recursive: true,
+			force: true,
+		});
+	let x = {
+		dependencies: [],
+		devDependencies: [],
+		invalidDirs: [],
+		invalidFiles: [],
+		missing: [],
+		using: [],
+	};
+	x = JSON.parse(execIn("npx depcheck --json", partDir).stdout);
+
+	fs.writeFileSync(
+		path.join(__dirname, "..", `depclean.out.${part}.json`),
+		JSON.stringify(x, null, "\t"),
+		{ encoding: "utf8" }
+	);
+
+	let depsToRemove = x.dependencies.join(" ");
+	if (depsToRemove) execIn(`npm r --save ${depsToRemove}`, partDir);
+
+	depsToRemove = x.devDependencies.join(" ");
+	if (depsToRemove) execIn(`npm r --save --dev ${depsToRemove}`, partDir);
+
+	if (removeModules && fs.existsSync(path.join(partDir, "node_modules")))
+		fs.rmSync(path.join(partDir, "node_modules"), {
+			recursive: true,
+			force: true,
+		});
+	let end = getLines(
+		execIn("npm ls --parseable --package-lock-only -a", partDir)
+	);
+	console.log(`${part}: ${start} -> ${end} (diff: ${start - end})`);
+});
+console.log("Installing required packages for bundle...");
+
+execIn(`npm i --save ${bundleRequired.join(" ")}`, path.join(__dirname, ".."));
diff --git a/bundle/scripts/utils.js b/bundle/scripts/utils.js
new file mode 100644
index 00000000..bf960532
--- /dev/null
+++ b/bundle/scripts/utils.js
@@ -0,0 +1,53 @@
+const path = require("path");
+const fs = require("fs");
+const { env } = require("process");
+const { execSync } = require("child_process");
+const { argv, stdout, exit } = require("process");
+
+const parts = ["api", "util", "cdn", "gateway", "bundle"];
+
+function copyRecursiveSync(src, dest) {
+	//if (verbose) console.log(`cpsync: ${src} -> ${dest}`);
+	let exists = fs.existsSync(src);
+	if (!exists) {
+		console.log(src + " doesn't exist, not copying!");
+		return;
+	}
+	let stats = exists && fs.statSync(src);
+	let isDirectory = exists && stats.isDirectory();
+	if (isDirectory) {
+		fs.mkdirSync(dest, { recursive: true });
+		fs.readdirSync(src).forEach(function (childItemName) {
+			copyRecursiveSync(
+				path.join(src, childItemName),
+				path.join(dest, childItemName)
+			);
+		});
+	} else {
+		fs.copyFileSync(src, dest);
+	}
+}
+
+function execIn(cmd, workdir) {
+	try {
+		return execSync(cmd, {
+			cwd: workdir,
+			shell: true,
+			env: process.env,
+			encoding: "utf-8",
+		});
+	} catch (error) {
+		return error.stdout;
+	}
+}
+
+function getLines(output) {
+	return output.split("\n").length;
+}
+
+module.exports = { 
+	//consts
+	parts,
+	//functions
+	copyRecursiveSync, execIn, getLines
+};