testing stuff dev/rory/paths_test

11 files changed, 136 insertions, 64 deletions

diff --git a/.gitignore b/.gitignore
index 8631a69d..fcdeeb3f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,5 @@ dbconf.json
 migrations.db
 
 assets/cache_src/
+
+package-lock.json
diff --git a/src/api/Server.ts b/src/api/Server.ts
index 560014d0..f4addb5a 100644
--- a/src/api/Server.ts
+++ b/src/api/Server.ts
@@ -1,4 +1,4 @@
-import { Config, getOrInitialiseDatabase, initEvent, registerRoutes } from "@fosscord/util";
+import { Config, getOrInitialiseDatabase, initEvent, Paths, registerRoutes, TestClientPaths } from "@fosscord/util";
 import { NextFunction, Request, Response, Router } from "express";
 import { Server, ServerOptions } from "lambert-server";
 import morgan from "morgan";
@@ -44,7 +44,7 @@ export class FosscordServer extends Server {
 				morgan("combined", {
 					skip: (req, res) => {
 						if(req.path.endsWith(".map")) return true;
-						if(req.path.includes("/assets/") && !fs.existsSync(path.join(__dirname, "..", "..", "..", "assets", req.path.split("/")[0].split('?')[0]))) return true;
+						if(req.path.includes("/assets/") && !fs.existsSync(path.join(Paths.AssetsPath, req.path.split("/")[0].split('?')[0]))) return true;
 						let skip = !(process.env["LOG_REQUESTS"]?.includes(res.statusCode.toString()) ?? false);
 						if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") skip = !skip;
 						return skip;
diff --git a/src/api/middlewares/TestClient.ts b/src/api/middlewares/TestClient.ts
index 0254a155..4b3dc5a0 100644
--- a/src/api/middlewares/TestClient.ts
+++ b/src/api/middlewares/TestClient.ts
@@ -1,4 +1,4 @@
-import { Config } from "@fosscord/util";
+import { Config, Paths, TestClientPaths } from "@fosscord/util";
 import express, { Application, Request, Response } from "express";
 import fs from "fs";
 import fetch, { Headers, Response as FetchResponse } from "node-fetch";
@@ -7,15 +7,15 @@ import { green } from "picocolors";
 import ProxyAgent from "proxy-agent";
 import { AssetCacheItem } from "../util/entities/AssetCacheItem";
 import { patchFile } from "..";
+import { createHash } from "crypto";
 
 const prettier = require("prettier");
-const AssetsPath = path.join(__dirname, "..", "..", "..", "assets");
 
 export default function TestClient(app: Application) {
 	const agent = new ProxyAgent();
 
 	//build client page
-	let html = fs.readFileSync(path.join(AssetsPath, "index.html"), { encoding: "utf8" });
+	let html = fs.readFileSync(TestClientPaths.Index, { encoding: "utf8" });
 	html = applyEnv(html);
 	html = applyInlinePlugins(html);
 	html = applyPlugins(html);
@@ -23,19 +23,15 @@ export default function TestClient(app: Application) {
 
 	//load asset cache
 	let newAssetCache: Map<string, AssetCacheItem> = new Map<string, AssetCacheItem>();
-	let assetCacheDir = path.join(AssetsPath, "cache");
-	if (process.env.ASSET_CACHE_DIR) assetCacheDir = process.env.ASSET_CACHE_DIR;
 
-	console.log(`[TestClient] ${green(`Using asset cache path: ${assetCacheDir}`)}`);
-	if (!fs.existsSync(assetCacheDir)) {
-		fs.mkdirSync(assetCacheDir);
-	}
-	if (fs.existsSync(path.join(assetCacheDir, "index.json"))) {
-		let rawdata = fs.readFileSync(path.join(assetCacheDir, "index.json"));
-		newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(rawdata.toString())));
+	console.log(`[TestClient] ${green(`Using asset cache path: ${TestClientPaths.CacheDir}`)}`);
+	if (!fs.existsSync(TestClientPaths.CacheDir)) {
+		fs.mkdirSync(TestClientPaths.CacheDir);
 	}
+	if (fs.existsSync(TestClientPaths.CacheIndex))
+		newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(fs.readFileSync(TestClientPaths.CacheIndex).toString())));
 
-	app.use("/assets", express.static(path.join(AssetsPath)));
+	app.use("/assets", express.static(path.join(Paths.AssetsPath)));
 	app.get("/assets/:file", async (req: Request, res: Response) => {
 		delete req.headers.host;
 		let response: FetchResponse;
@@ -58,6 +54,9 @@ export default function TestClient(app: Application) {
 					...req.headers
 				}
 			});
+			buffer = await response.buffer();
+			let hash = createHash("md5").update(buffer).digest('hex');
+			
 			//set cache info
 			assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers));
 			assetCacheItem.Key = req.params.file;
@@ -67,12 +66,12 @@ export default function TestClient(app: Application) {
 			if(response.status != 200) {
 				return res.status(404).send("Not found");
 			}
-			assetCacheItem.FilePath = path.join(assetCacheDir, req.params.file);
-			if(!fs.existsSync(assetCacheDir))
-				fs.mkdirSync(assetCacheDir);
-			fs.writeFileSync(path.join(assetCacheDir, "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
+			assetCacheItem.FilePath = path.join(TestClientPaths.CacheDir, req.params.file);
+			if(!fs.existsSync(TestClientPaths.CacheDir))
+				fs.mkdirSync(TestClientPaths.CacheDir);
+			fs.writeFileSync(TestClientPaths.CacheIndex, JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
 			//download file
-			fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, (await response.buffer()).toString()) : await response.buffer());
+			fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, buffer.toString()) : buffer);
 		}
 
 		assetCacheItem.Headers.forEach((value: string, name: string) => {
@@ -87,7 +86,7 @@ export default function TestClient(app: Application) {
 
 		if (!useTestClient) return res.send("Test client is disabled on this instance. Use a stand-alone client to connect this instance.");
 
-		res.send(fs.readFileSync(path.join(__dirname, "..", "..", "..", "assets", "developers.html"), { encoding: "utf8" }));
+		res.send(fs.readFileSync(TestClientPaths.Developers, { encoding: "utf8" }));
 	});
 	app.get("*", (req: Request, res: Response) => {
 		const { useTestClient } = Config.get().client;
@@ -118,7 +117,7 @@ function applyEnv(html: string): string {
 
 function applyPlugins(html: string): string {
 	// plugins
-	let files = fs.readdirSync(path.join(AssetsPath, "plugins"));
+	let files = fs.readdirSync(TestClientPaths.PluginsDir);
 	let plugins = "";
 	files.forEach((x) => {
 		if (x.endsWith(".js")) plugins += `<script src='/assets/plugins/${x}'></script>\n`;
@@ -128,7 +127,7 @@ function applyPlugins(html: string): string {
 
 function applyInlinePlugins(html: string): string {
 	// inline plugins
-	let files = fs.readdirSync(path.join(AssetsPath, "inline-plugins"));
+	let files = fs.readdirSync(TestClientPaths.InlinePluginsDir);
 	let plugins = "";
 	files.forEach((x) => {
 		if (x.endsWith(".js")) plugins += `<script src='/assets/inline-plugins/${x}'></script>\n\n`;
@@ -138,10 +137,10 @@ function applyInlinePlugins(html: string): string {
 
 function applyPreloadPlugins(html: string): string {
 	//preload plugins
-	let files = fs.readdirSync(path.join(AssetsPath, "preload-plugins"));
+	let files = fs.readdirSync(TestClientPaths.PreloadPluginsDir);
 	let plugins = "";
 	files.forEach((x) => {
-		if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(AssetsPath, "preload-plugins", x))}</script>\n`;
+		if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(TestClientPaths.PreloadPluginsDir, x))}</script>\n`;
 	});
 	return html.replaceAll("<!-- preload plugin marker -->", plugins);
 }
@@ -155,7 +154,13 @@ function stripHeaders(headers: Headers): Headers {
 		"transfer-encoding",
 		"expect-ct",
 		"access-control-allow-origin",
-		"content-encoding"
+		"content-encoding",
+		"cf-cache-status",
+		"cf-ray",
+		"server",
+		"etag",
+		"nel",
+		"report-to"
 	].forEach((headerName) => {
 		headers.delete(headerName);
 	});
diff --git a/src/api/middlewares/Translation.ts b/src/api/middlewares/Translation.ts
index 8e5e67e6..85b4ac06 100644
--- a/src/api/middlewares/Translation.ts
+++ b/src/api/middlewares/Translation.ts
@@ -1,3 +1,4 @@
+import { Paths } from "@fosscord/util";
 import { Router } from "express";
 import fs from "fs";
 import i18next from "i18next";
@@ -6,8 +7,8 @@ import i18nextBackend from "i18next-node-fs-backend";
 import path from "path";
 
 export async function initTranslation(router: Router) {
-	const languages = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales"));
-	const namespaces = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales", "en"));
+	const languages = fs.readdirSync(path.join(Paths.AssetsPath, "locales"));
+	const namespaces = fs.readdirSync(path.join(Paths.AssetsPath, "locales", "en"));
 	const ns = namespaces.filter((x) => x.endsWith(".json")).map((x) => x.slice(0, x.length - 5));
 
 	await i18next
diff --git a/src/api/routes/guilds/#guild_id/widget.png.ts b/src/api/routes/guilds/#guild_id/widget.png.ts
index 1c4ef29b..2b647276 100644
--- a/src/api/routes/guilds/#guild_id/widget.png.ts
+++ b/src/api/routes/guilds/#guild_id/widget.png.ts
@@ -1,5 +1,5 @@
 import { route } from "@fosscord/api";
-import { Guild, HTTPError } from "@fosscord/util";
+import { Guild, HTTPError, Paths } from "@fosscord/util";
 import { Request, Response, Router } from "express";
 import fs from "fs";
 import path from "path";
@@ -39,7 +39,7 @@ router.get("/", route({}), async (req: Request, res: Response) => {
 	}
 
 	// TODO: Widget style templates need Fosscord branding
-	const source = path.join(__dirname, "..", "..", "..", "..", "..", "assets", "widget", `${style}.png`);
+	const source = path.join(Paths.AssetsPath, "widget", `${style}.png`);
 	if (!fs.existsSync(source)) {
 		throw new HTTPError("Widget template does not exist.", 400);
 	}
diff --git a/src/api/util/TestClientPatcher.ts b/src/api/util/TestClientPatcher.ts
index 2e9bfafe..1c5f245d 100644
--- a/src/api/util/TestClientPatcher.ts
+++ b/src/api/util/TestClientPatcher.ts
@@ -1,30 +1,25 @@
 import path from "path";
 import fs from "fs";
+import { Paths, TestClientPaths } from "@fosscord/util";
 
-console.log('[TestClient] Loading private assets...');
 
-const privateAssetsRoot = path.join(__dirname, "..", "..", "..", "assets", "private");
-const iconsRoot = path.join(privateAssetsRoot, "icons");
-const icons = new Map<string, Buffer>();
-
-fs.readdirSync(iconsRoot).forEach(file => {
-    const fileName = path.basename(file);
-    //check if dir
-    if(fs.lstatSync(path.join(iconsRoot, file)).isDirectory()){
-        return;
-    }
-    icons.set(fileName,fs.readFileSync(path.join(iconsRoot,file)) as Buffer);
-});
-
-fs.readdirSync(path.join(iconsRoot, "custom")).forEach(file => {
-    const fileName = path.basename(file);
-    if(fs.lstatSync(path.join(iconsRoot,"custom", file)).isDirectory()){
-        return;
-    }
-    icons.set(fileName,fs.readFileSync(path.join(iconsRoot,"custom",file)) as Buffer);
-});
+function readAssets(): Map<string, Buffer> {
+    const icons = new Map<string, Buffer>();
+    
+    fs.readdirSync(Paths.IconPath).forEach(file => {
+        const fileName = path.basename(file);
+        //check if dir
+        if(fs.lstatSync(path.join(Paths.IconPath, file)).isDirectory()){
+            return;
+        }
+        if(fs.existsSync(path.join(Paths.CustomIconPath, fileName)))
+            icons.set(fileName,fs.readFileSync(path.join(Paths.CustomIconPath, fileName)) as Buffer);
+        else
+            icons.set(fileName,fs.readFileSync(path.join(Paths.IconPath, fileName)) as Buffer);
+    });
 
-console.log('[TestClient] Patcher ready!');
+    return icons;
+}
 
 export function patchFile(filePath: string, content: string): string {
     console.log(`[TestClient] Patching ${filePath}`);
@@ -32,10 +27,12 @@ export function patchFile(filePath: string, content: string): string {
     
     content = prettier(filePath, content);
     content = autoPatch(filePath, content);
+    content = applyPatches(filePath, content);
 
     console.log(`[TestClient] Patched ${filePath} in ${Date.now() - startTime}ms`);
     return content;
 }
+
 function prettier(filePath: string, content: string): string{
     let prettier = require("prettier");
     let parser;
@@ -91,7 +88,7 @@ function autoPatch(filePath: string, content: string): string{
     content = content.replaceAll(/--brand-experiment-(\d{1,4}): hsl\(235/g, '--brand-experiment-\$1: hsl(var(--brand-hue)')
 
     //logos
-    content = content.replace(/d: "M23\.0212.*/, `d: "${icons.get("homeIcon.path")!.toString()}"`);
+    content = content.replace(/d: "M23\.0212.*/, `d: "${readAssets().get("homeIcon.path")!.toString()}"`);
     content = content.replace('width: n, height: o, viewBox: "0 0 28 20"', 'width: 48, height: 48, viewBox: "0 0 48 48"');
 
     //undo webpacking
@@ -100,8 +97,18 @@ function autoPatch(filePath: string, content: string): string{
     content = content.replace(/!1/g, "false");
     // - real esmodule defs
     content = content.replace(/Object.defineProperty\((.), "__esModule", { value: (.*) }\);/g, '\$1.__esModule = \$2;');
-    
 
     console.log(`[TestClient] Autopatched ${path.basename(filePath)}!`);
     return content;
+}
+
+function applyPatches(filePath: string, content: string): string{
+    //get files in testclient_patches
+    const patches = fs.readdirSync(TestClientPaths.PatchDir);
+    for(let patch of patches){
+        //apply patch with git patch
+        const patchPath = path.join(TestClientPaths.PatchDir, patch);
+
+    }
+    return content;
 }
\ No newline at end of file
diff --git a/src/api/util/handlers/route.ts b/src/api/util/handlers/route.ts
index d43ae103..f8a57dcc 100644
--- a/src/api/util/handlers/route.ts
+++ b/src/api/util/handlers/route.ts
@@ -5,6 +5,7 @@ import {
 	FosscordApiErrors,
 	getPermission,
 	getRights,
+	Paths,
 	PermissionResolvable,
 	Permissions,
 	RightResolvable,
@@ -17,8 +18,7 @@ import { NextFunction, Request, Response } from "express";
 import fs from "fs";
 import path from "path";
 
-const SchemaPath = path.join(__dirname, "..", "..", "..", "..", "assets", "schemas.json");
-const schemas = JSON.parse(fs.readFileSync(SchemaPath, { encoding: "utf8" }));
+const schemas = JSON.parse(fs.readFileSync(Paths.SchemaPath, { encoding: "utf8" }));
 
 export const ajv = new Ajv({
 	allErrors: true,
diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts
index 1ab6a1d9..d9009aa9 100644
--- a/src/cdn/util/Storage.ts
+++ b/src/cdn/util/Storage.ts
@@ -5,6 +5,7 @@ import { S3 } from "@aws-sdk/client-s3";
 import fs from "fs";
 import { bgCyan, black } from "picocolors";
 import { S3Storage } from "./S3Storage";
+import { Paths } from "@fosscord/util";
 process.cwd();
 
 export interface Storage {
@@ -16,12 +17,7 @@ export interface Storage {
 let storage: Storage;
 
 if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
-	let location = process.env.STORAGE_LOCATION;
-	if (location) {
-		location = path.resolve(location);
-	} else {
-		location = path.join(process.cwd(), "files");
-	}
+	let location = Paths.CDNFilePath;
 	console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`);
 	//fse.ensureDirSync(location);
 	fs.mkdirSync(location, { recursive: true });
diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts
index b9f8365e..c67ab568 100644
--- a/src/util/util/Database.ts
+++ b/src/util/util/Database.ts
@@ -5,6 +5,7 @@ import { green, red, yellow } from "picocolors";
 import { exit } from "process";
 import "reflect-metadata";
 import { DataSource, DataSourceOptions, PrimaryColumn, PrimaryGeneratedColumn } from "typeorm";
+import { Paths } from ".";
 import * as Models from "../entities";
 import { BaseClass, BaseClassWithoutId } from "../entities";
 
@@ -41,7 +42,7 @@ function getDataSourceOptions(): DataSourceOptions {
 	const dbConnectionString = process.env.DATABASE || path.join(process.cwd(), "database.db");
 	const type = dbConnectionString.includes("://") ? dbConnectionString.split(":")[0]?.replace("+srv", "") : ("sqlite" as any);
 	const isSqlite = type.includes("sqlite");
-	const migrationsExist = fs.existsSync(path.join(__dirname, "..", "migrations", type));
+	const migrationsExist = fs.existsSync(path.join(Paths.MigrationsRoot, type));
 	//read env vars
 	const synchronizeInsteadOfMigrations = "DB_UNSAFE" in process.env;
 	const verboseDb = "DB_VERBOSE" in process.env;
@@ -94,7 +95,7 @@ function getDataSourceOptions(): DataSourceOptions {
 		bigNumberStrings: false,
 		supportBigNumbers: true,
 		name: "default",
-		migrations: synchronizeInsteadOfMigrations ? [] : [path.join(__dirname, "..", "migrations", type, "*.js")],
+		migrations: synchronizeInsteadOfMigrations ? [] : [path.join(Paths.MigrationsRoot, type, "*.js")],
 		migrationsRun: !synchronizeInsteadOfMigrations,
 		applicationName: `Fosscord Server`,
 	} as DataSourceOptions;
diff --git a/src/util/util/Paths.ts b/src/util/util/Paths.ts
new file mode 100644
index 00000000..20b16738
--- /dev/null
+++ b/src/util/util/Paths.ts
@@ -0,0 +1,59 @@
+import path from "path";
+import fs from "fs";
+
+
+
+
+
+export class ProjectPaths {
+    public static RuntimePath = path.join(__dirname, "..", "..");
+    public static ProjectRoot = path.join(ProjectPaths.RuntimePath, "..");
+    public static DataDir = path.join(ProjectPaths.ProjectRoot, "data");
+    public static ApiPath = path.join(ProjectPaths.RuntimePath, "api");
+    public static CdnPath = path.join(ProjectPaths.RuntimePath, "cdn");
+    public static GatewayPath = path.join(ProjectPaths.RuntimePath, "gateway");
+    public static UtilPath = path.join(ProjectPaths.RuntimePath, "util");
+}
+
+export class Paths {
+    public static AssetsPath = path.join(ProjectPaths.ProjectRoot, "assets");
+    public static PrivateAssetsPath = path.join(ProjectPaths.DataDir, "assets");
+    public static MigrationsRoot = path.join(ProjectPaths.UtilPath, "migrations");
+    public static CDNFilePath = path.resolve(process.env.STORAGE_LOCATION || path.join(ProjectPaths.ProjectRoot, "files"));
+    public static SchemaPath = path.join(ProjectPaths.DataDir, "schemas.json");
+    public static IconPath = path.join(Paths.AssetsPath, "icons");
+    public static CustomIconPath = path.join(Paths.AssetsPath, "icons", "custom");
+}
+
+export class TestClientPaths {
+    public static TestClientRoot = path.join(ProjectPaths.DataDir, "test-client");
+    public static TestClientCacheDir = process.env.TEST_CLIENT_CACHE_DIR || process.env.ASSET_CACHE_DIR || path.join(TestClientPaths.TestClientRoot, "cache");
+    public static Index = path.join(TestClientPaths.TestClientRoot, "index.html");
+    public static Developers = path.join(TestClientPaths.TestClientRoot, "developers.html");
+    public static PatchDir = path.join(TestClientPaths.TestClientRoot, "patches");
+    public static CacheDir = TestClientPaths.TestClientCacheDir;
+    public static CacheIndex = path.join(TestClientPaths.TestClientCacheDir, "index.json");
+    public static PluginsDir = path.join(TestClientPaths.TestClientRoot, "plugins");
+    public static PreloadPluginsDir = path.join(TestClientPaths.TestClientRoot, "preload-plugins");
+    public static InlinePluginsDir = path.join(TestClientPaths.TestClientRoot, "inline-plugins");
+}
+
+
+//warnings
+if(process.env.ASSET_CACHE_DIR) console.log(`[ENV/WARN] ASSET_CACHE_DIR is deprecated, please use TEST_CLIENT_CACHE_DIR instead!`);
+
+for(let key in ProjectPaths) {
+    if(!fs.existsSync((ProjectPaths as any)[key])) {
+        console.error(`[ERROR] ${(ProjectPaths as any)[key]} does not exist!`);
+    }
+}
+for(let key in Paths) {
+    if(!fs.existsSync((Paths as any)[key])) {
+        console.error(`[ERROR] ${(Paths as any)[key]} does not exist!`);
+    }
+}
+for(let key in TestClientPaths) {
+    if(!fs.existsSync((TestClientPaths as any)[key])) {
+        console.error(`[ERROR] ${(TestClientPaths as any)[key]} does not exist!`);
+    }
+}
diff --git a/src/util/util/index.ts b/src/util/util/index.ts
index 11f0b72a..af29787b 100644
--- a/src/util/util/index.ts
+++ b/src/util/util/index.ts
@@ -23,3 +23,4 @@ export * from "./Snowflake";
 export * from "./String";
 export * from "./Token";
 export * from "./TraverseDirectory";
+export * from "./Paths";
\ No newline at end of file