diff --git a/.gitignore b/.gitignore
index 8631a69d..fcdeeb3f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,5 @@ dbconf.json
migrations.db
assets/cache_src/
+
+package-lock.json
diff --git a/src/api/Server.ts b/src/api/Server.ts
index 560014d0..f4addb5a 100644
--- a/src/api/Server.ts
+++ b/src/api/Server.ts
@@ -1,4 +1,4 @@
-import { Config, getOrInitialiseDatabase, initEvent, registerRoutes } from "@fosscord/util";
+import { Config, getOrInitialiseDatabase, initEvent, Paths, registerRoutes, TestClientPaths } from "@fosscord/util";
import { NextFunction, Request, Response, Router } from "express";
import { Server, ServerOptions } from "lambert-server";
import morgan from "morgan";
@@ -44,7 +44,7 @@ export class FosscordServer extends Server {
morgan("combined", {
skip: (req, res) => {
if(req.path.endsWith(".map")) return true;
- if(req.path.includes("/assets/") && !fs.existsSync(path.join(__dirname, "..", "..", "..", "assets", req.path.split("/")[0].split('?')[0]))) return true;
+ if(req.path.includes("/assets/") && !fs.existsSync(path.join(Paths.AssetsPath, req.path.split("/")[0].split('?')[0]))) return true;
let skip = !(process.env["LOG_REQUESTS"]?.includes(res.statusCode.toString()) ?? false);
if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") skip = !skip;
return skip;
diff --git a/src/api/middlewares/TestClient.ts b/src/api/middlewares/TestClient.ts
index 0254a155..4b3dc5a0 100644
--- a/src/api/middlewares/TestClient.ts
+++ b/src/api/middlewares/TestClient.ts
@@ -1,4 +1,4 @@
-import { Config } from "@fosscord/util";
+import { Config, Paths, TestClientPaths } from "@fosscord/util";
import express, { Application, Request, Response } from "express";
import fs from "fs";
import fetch, { Headers, Response as FetchResponse } from "node-fetch";
@@ -7,15 +7,15 @@ import { green } from "picocolors";
import ProxyAgent from "proxy-agent";
import { AssetCacheItem } from "../util/entities/AssetCacheItem";
import { patchFile } from "..";
+import { createHash } from "crypto";
const prettier = require("prettier");
-const AssetsPath = path.join(__dirname, "..", "..", "..", "assets");
export default function TestClient(app: Application) {
const agent = new ProxyAgent();
//build client page
- let html = fs.readFileSync(path.join(AssetsPath, "index.html"), { encoding: "utf8" });
+ let html = fs.readFileSync(TestClientPaths.Index, { encoding: "utf8" });
html = applyEnv(html);
html = applyInlinePlugins(html);
html = applyPlugins(html);
@@ -23,19 +23,15 @@ export default function TestClient(app: Application) {
//load asset cache
let newAssetCache: Map<string, AssetCacheItem> = new Map<string, AssetCacheItem>();
- let assetCacheDir = path.join(AssetsPath, "cache");
- if (process.env.ASSET_CACHE_DIR) assetCacheDir = process.env.ASSET_CACHE_DIR;
- console.log(`[TestClient] ${green(`Using asset cache path: ${assetCacheDir}`)}`);
- if (!fs.existsSync(assetCacheDir)) {
- fs.mkdirSync(assetCacheDir);
- }
- if (fs.existsSync(path.join(assetCacheDir, "index.json"))) {
- let rawdata = fs.readFileSync(path.join(assetCacheDir, "index.json"));
- newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(rawdata.toString())));
+ console.log(`[TestClient] ${green(`Using asset cache path: ${TestClientPaths.CacheDir}`)}`);
+ if (!fs.existsSync(TestClientPaths.CacheDir)) {
+ fs.mkdirSync(TestClientPaths.CacheDir);
}
+ if (fs.existsSync(TestClientPaths.CacheIndex))
+ newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(fs.readFileSync(TestClientPaths.CacheIndex).toString())));
- app.use("/assets", express.static(path.join(AssetsPath)));
+ app.use("/assets", express.static(path.join(Paths.AssetsPath)));
app.get("/assets/:file", async (req: Request, res: Response) => {
delete req.headers.host;
let response: FetchResponse;
@@ -58,6 +54,9 @@ export default function TestClient(app: Application) {
...req.headers
}
});
+ buffer = await response.buffer();
+ let hash = createHash("md5").update(buffer).digest('hex');
+
//set cache info
assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers));
assetCacheItem.Key = req.params.file;
@@ -67,12 +66,12 @@ export default function TestClient(app: Application) {
if(response.status != 200) {
return res.status(404).send("Not found");
}
- assetCacheItem.FilePath = path.join(assetCacheDir, req.params.file);
- if(!fs.existsSync(assetCacheDir))
- fs.mkdirSync(assetCacheDir);
- fs.writeFileSync(path.join(assetCacheDir, "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
+ assetCacheItem.FilePath = path.join(TestClientPaths.CacheDir, req.params.file);
+ if(!fs.existsSync(TestClientPaths.CacheDir))
+ fs.mkdirSync(TestClientPaths.CacheDir);
+ fs.writeFileSync(TestClientPaths.CacheIndex, JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
//download file
- fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, (await response.buffer()).toString()) : await response.buffer());
+ fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, buffer.toString()) : buffer);
}
assetCacheItem.Headers.forEach((value: string, name: string) => {
@@ -87,7 +86,7 @@ export default function TestClient(app: Application) {
if (!useTestClient) return res.send("Test client is disabled on this instance. Use a stand-alone client to connect this instance.");
- res.send(fs.readFileSync(path.join(__dirname, "..", "..", "..", "assets", "developers.html"), { encoding: "utf8" }));
+ res.send(fs.readFileSync(TestClientPaths.Developers, { encoding: "utf8" }));
});
app.get("*", (req: Request, res: Response) => {
const { useTestClient } = Config.get().client;
@@ -118,7 +117,7 @@ function applyEnv(html: string): string {
function applyPlugins(html: string): string {
// plugins
- let files = fs.readdirSync(path.join(AssetsPath, "plugins"));
+ let files = fs.readdirSync(TestClientPaths.PluginsDir);
let plugins = "";
files.forEach((x) => {
if (x.endsWith(".js")) plugins += `<script src='/assets/plugins/${x}'></script>\n`;
@@ -128,7 +127,7 @@ function applyPlugins(html: string): string {
function applyInlinePlugins(html: string): string {
// inline plugins
- let files = fs.readdirSync(path.join(AssetsPath, "inline-plugins"));
+ let files = fs.readdirSync(TestClientPaths.InlinePluginsDir);
let plugins = "";
files.forEach((x) => {
if (x.endsWith(".js")) plugins += `<script src='/assets/inline-plugins/${x}'></script>\n\n`;
@@ -138,10 +137,10 @@ function applyInlinePlugins(html: string): string {
function applyPreloadPlugins(html: string): string {
//preload plugins
- let files = fs.readdirSync(path.join(AssetsPath, "preload-plugins"));
+ let files = fs.readdirSync(TestClientPaths.PreloadPluginsDir);
let plugins = "";
files.forEach((x) => {
- if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(AssetsPath, "preload-plugins", x))}</script>\n`;
+ if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(TestClientPaths.PreloadPluginsDir, x))}</script>\n`;
});
return html.replaceAll("<!-- preload plugin marker -->", plugins);
}
@@ -155,7 +154,13 @@ function stripHeaders(headers: Headers): Headers {
"transfer-encoding",
"expect-ct",
"access-control-allow-origin",
- "content-encoding"
+ "content-encoding",
+ "cf-cache-status",
+ "cf-ray",
+ "server",
+ "etag",
+ "nel",
+ "report-to"
].forEach((headerName) => {
headers.delete(headerName);
});
diff --git a/src/api/middlewares/Translation.ts b/src/api/middlewares/Translation.ts
index 8e5e67e6..85b4ac06 100644
--- a/src/api/middlewares/Translation.ts
+++ b/src/api/middlewares/Translation.ts
@@ -1,3 +1,4 @@
+import { Paths } from "@fosscord/util";
import { Router } from "express";
import fs from "fs";
import i18next from "i18next";
@@ -6,8 +7,8 @@ import i18nextBackend from "i18next-node-fs-backend";
import path from "path";
export async function initTranslation(router: Router) {
- const languages = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales"));
- const namespaces = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales", "en"));
+ const languages = fs.readdirSync(path.join(Paths.AssetsPath, "locales"));
+ const namespaces = fs.readdirSync(path.join(Paths.AssetsPath, "locales", "en"));
const ns = namespaces.filter((x) => x.endsWith(".json")).map((x) => x.slice(0, x.length - 5));
await i18next
diff --git a/src/api/routes/guilds/#guild_id/widget.png.ts b/src/api/routes/guilds/#guild_id/widget.png.ts
index 1c4ef29b..2b647276 100644
--- a/src/api/routes/guilds/#guild_id/widget.png.ts
+++ b/src/api/routes/guilds/#guild_id/widget.png.ts
@@ -1,5 +1,5 @@
import { route } from "@fosscord/api";
-import { Guild, HTTPError } from "@fosscord/util";
+import { Guild, HTTPError, Paths } from "@fosscord/util";
import { Request, Response, Router } from "express";
import fs from "fs";
import path from "path";
@@ -39,7 +39,7 @@ router.get("/", route({}), async (req: Request, res: Response) => {
}
// TODO: Widget style templates need Fosscord branding
- const source = path.join(__dirname, "..", "..", "..", "..", "..", "assets", "widget", `${style}.png`);
+ const source = path.join(Paths.AssetsPath, "widget", `${style}.png`);
if (!fs.existsSync(source)) {
throw new HTTPError("Widget template does not exist.", 400);
}
diff --git a/src/api/util/TestClientPatcher.ts b/src/api/util/TestClientPatcher.ts
index 2e9bfafe..1c5f245d 100644
--- a/src/api/util/TestClientPatcher.ts
+++ b/src/api/util/TestClientPatcher.ts
@@ -1,30 +1,25 @@
import path from "path";
import fs from "fs";
+import { Paths, TestClientPaths } from "@fosscord/util";
-console.log('[TestClient] Loading private assets...');
-const privateAssetsRoot = path.join(__dirname, "..", "..", "..", "assets", "private");
-const iconsRoot = path.join(privateAssetsRoot, "icons");
-const icons = new Map<string, Buffer>();
-
-fs.readdirSync(iconsRoot).forEach(file => {
- const fileName = path.basename(file);
- //check if dir
- if(fs.lstatSync(path.join(iconsRoot, file)).isDirectory()){
- return;
- }
- icons.set(fileName,fs.readFileSync(path.join(iconsRoot,file)) as Buffer);
-});
-
-fs.readdirSync(path.join(iconsRoot, "custom")).forEach(file => {
- const fileName = path.basename(file);
- if(fs.lstatSync(path.join(iconsRoot,"custom", file)).isDirectory()){
- return;
- }
- icons.set(fileName,fs.readFileSync(path.join(iconsRoot,"custom",file)) as Buffer);
-});
+function readAssets(): Map<string, Buffer> {
+ const icons = new Map<string, Buffer>();
+
+ fs.readdirSync(Paths.IconPath).forEach(file => {
+ const fileName = path.basename(file);
+ //check if dir
+ if(fs.lstatSync(path.join(Paths.IconPath, file)).isDirectory()){
+ return;
+ }
+ if(fs.existsSync(path.join(Paths.CustomIconPath, fileName)))
+ icons.set(fileName,fs.readFileSync(path.join(Paths.CustomIconPath, fileName)) as Buffer);
+ else
+ icons.set(fileName,fs.readFileSync(path.join(Paths.IconPath, fileName)) as Buffer);
+ });
-console.log('[TestClient] Patcher ready!');
+ return icons;
+}
export function patchFile(filePath: string, content: string): string {
console.log(`[TestClient] Patching ${filePath}`);
@@ -32,10 +27,12 @@ export function patchFile(filePath: string, content: string): string {
content = prettier(filePath, content);
content = autoPatch(filePath, content);
+ content = applyPatches(filePath, content);
console.log(`[TestClient] Patched ${filePath} in ${Date.now() - startTime}ms`);
return content;
}
+
function prettier(filePath: string, content: string): string{
let prettier = require("prettier");
let parser;
@@ -91,7 +88,7 @@ function autoPatch(filePath: string, content: string): string{
content = content.replaceAll(/--brand-experiment-(\d{1,4}): hsl\(235/g, '--brand-experiment-\$1: hsl(var(--brand-hue)')
//logos
- content = content.replace(/d: "M23\.0212.*/, `d: "${icons.get("homeIcon.path")!.toString()}"`);
+ content = content.replace(/d: "M23\.0212.*/, `d: "${readAssets().get("homeIcon.path")!.toString()}"`);
content = content.replace('width: n, height: o, viewBox: "0 0 28 20"', 'width: 48, height: 48, viewBox: "0 0 48 48"');
//undo webpacking
@@ -100,8 +97,18 @@ function autoPatch(filePath: string, content: string): string{
content = content.replace(/!1/g, "false");
// - real esmodule defs
content = content.replace(/Object.defineProperty\((.), "__esModule", { value: (.*) }\);/g, '\$1.__esModule = \$2;');
-
console.log(`[TestClient] Autopatched ${path.basename(filePath)}!`);
return content;
+}
+
+function applyPatches(filePath: string, content: string): string{
+ //get files in testclient_patches
+ const patches = fs.readdirSync(TestClientPaths.PatchDir);
+ for(let patch of patches){
+ //apply patch with git patch
+ const patchPath = path.join(TestClientPaths.PatchDir, patch);
+
+ }
+ return content;
}
\ No newline at end of file
diff --git a/src/api/util/handlers/route.ts b/src/api/util/handlers/route.ts
index d43ae103..f8a57dcc 100644
--- a/src/api/util/handlers/route.ts
+++ b/src/api/util/handlers/route.ts
@@ -5,6 +5,7 @@ import {
FosscordApiErrors,
getPermission,
getRights,
+ Paths,
PermissionResolvable,
Permissions,
RightResolvable,
@@ -17,8 +18,7 @@ import { NextFunction, Request, Response } from "express";
import fs from "fs";
import path from "path";
-const SchemaPath = path.join(__dirname, "..", "..", "..", "..", "assets", "schemas.json");
-const schemas = JSON.parse(fs.readFileSync(SchemaPath, { encoding: "utf8" }));
+const schemas = JSON.parse(fs.readFileSync(Paths.SchemaPath, { encoding: "utf8" }));
export const ajv = new Ajv({
allErrors: true,
diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts
index 1ab6a1d9..d9009aa9 100644
--- a/src/cdn/util/Storage.ts
+++ b/src/cdn/util/Storage.ts
@@ -5,6 +5,7 @@ import { S3 } from "@aws-sdk/client-s3";
import fs from "fs";
import { bgCyan, black } from "picocolors";
import { S3Storage } from "./S3Storage";
+import { Paths } from "@fosscord/util";
process.cwd();
export interface Storage {
@@ -16,12 +17,7 @@ export interface Storage {
let storage: Storage;
if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
- let location = process.env.STORAGE_LOCATION;
- if (location) {
- location = path.resolve(location);
- } else {
- location = path.join(process.cwd(), "files");
- }
+ let location = Paths.CDNFilePath;
console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`);
//fse.ensureDirSync(location);
fs.mkdirSync(location, { recursive: true });
diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts
index b9f8365e..c67ab568 100644
--- a/src/util/util/Database.ts
+++ b/src/util/util/Database.ts
@@ -5,6 +5,7 @@ import { green, red, yellow } from "picocolors";
import { exit } from "process";
import "reflect-metadata";
import { DataSource, DataSourceOptions, PrimaryColumn, PrimaryGeneratedColumn } from "typeorm";
+import { Paths } from ".";
import * as Models from "../entities";
import { BaseClass, BaseClassWithoutId } from "../entities";
@@ -41,7 +42,7 @@ function getDataSourceOptions(): DataSourceOptions {
const dbConnectionString = process.env.DATABASE || path.join(process.cwd(), "database.db");
const type = dbConnectionString.includes("://") ? dbConnectionString.split(":")[0]?.replace("+srv", "") : ("sqlite" as any);
const isSqlite = type.includes("sqlite");
- const migrationsExist = fs.existsSync(path.join(__dirname, "..", "migrations", type));
+ const migrationsExist = fs.existsSync(path.join(Paths.MigrationsRoot, type));
//read env vars
const synchronizeInsteadOfMigrations = "DB_UNSAFE" in process.env;
const verboseDb = "DB_VERBOSE" in process.env;
@@ -94,7 +95,7 @@ function getDataSourceOptions(): DataSourceOptions {
bigNumberStrings: false,
supportBigNumbers: true,
name: "default",
- migrations: synchronizeInsteadOfMigrations ? [] : [path.join(__dirname, "..", "migrations", type, "*.js")],
+ migrations: synchronizeInsteadOfMigrations ? [] : [path.join(Paths.MigrationsRoot, type, "*.js")],
migrationsRun: !synchronizeInsteadOfMigrations,
applicationName: `Fosscord Server`,
} as DataSourceOptions;
diff --git a/src/util/util/Paths.ts b/src/util/util/Paths.ts
new file mode 100644
index 00000000..20b16738
--- /dev/null
+++ b/src/util/util/Paths.ts
@@ -0,0 +1,59 @@
+import path from "path";
+import fs from "fs";
+
+
+
+
+
+export class ProjectPaths {
+ public static RuntimePath = path.join(__dirname, "..", "..");
+ public static ProjectRoot = path.join(ProjectPaths.RuntimePath, "..");
+ public static DataDir = path.join(ProjectPaths.ProjectRoot, "data");
+ public static ApiPath = path.join(ProjectPaths.RuntimePath, "api");
+ public static CdnPath = path.join(ProjectPaths.RuntimePath, "cdn");
+ public static GatewayPath = path.join(ProjectPaths.RuntimePath, "gateway");
+ public static UtilPath = path.join(ProjectPaths.RuntimePath, "util");
+}
+
+export class Paths {
+ public static AssetsPath = path.join(ProjectPaths.ProjectRoot, "assets");
+ public static PrivateAssetsPath = path.join(ProjectPaths.DataDir, "assets");
+ public static MigrationsRoot = path.join(ProjectPaths.UtilPath, "migrations");
+ public static CDNFilePath = path.resolve(process.env.STORAGE_LOCATION || path.join(ProjectPaths.ProjectRoot, "files"));
+ public static SchemaPath = path.join(ProjectPaths.DataDir, "schemas.json");
+ public static IconPath = path.join(Paths.AssetsPath, "icons");
+ public static CustomIconPath = path.join(Paths.AssetsPath, "icons", "custom");
+}
+
+export class TestClientPaths {
+ public static TestClientRoot = path.join(ProjectPaths.DataDir, "test-client");
+ public static TestClientCacheDir = process.env.TEST_CLIENT_CACHE_DIR || process.env.ASSET_CACHE_DIR || path.join(TestClientPaths.TestClientRoot, "cache");
+ public static Index = path.join(TestClientPaths.TestClientRoot, "index.html");
+ public static Developers = path.join(TestClientPaths.TestClientRoot, "developers.html");
+ public static PatchDir = path.join(TestClientPaths.TestClientRoot, "patches");
+ public static CacheDir = TestClientPaths.TestClientCacheDir;
+ public static CacheIndex = path.join(TestClientPaths.TestClientCacheDir, "index.json");
+ public static PluginsDir = path.join(TestClientPaths.TestClientRoot, "plugins");
+ public static PreloadPluginsDir = path.join(TestClientPaths.TestClientRoot, "preload-plugins");
+ public static InlinePluginsDir = path.join(TestClientPaths.TestClientRoot, "inline-plugins");
+}
+
+
+//warnings
+if(process.env.ASSET_CACHE_DIR) console.log(`[ENV/WARN] ASSET_CACHE_DIR is deprecated, please use TEST_CLIENT_CACHE_DIR instead!`);
+
+for(let key in ProjectPaths) {
+ if(!fs.existsSync((ProjectPaths as any)[key])) {
+ console.error(`[ERROR] ${(ProjectPaths as any)[key]} does not exist!`);
+ }
+}
+for(let key in Paths) {
+ if(!fs.existsSync((Paths as any)[key])) {
+ console.error(`[ERROR] ${(Paths as any)[key]} does not exist!`);
+ }
+}
+for(let key in TestClientPaths) {
+ if(!fs.existsSync((TestClientPaths as any)[key])) {
+ console.error(`[ERROR] ${(TestClientPaths as any)[key]} does not exist!`);
+ }
+}
diff --git a/src/util/util/index.ts b/src/util/util/index.ts
index 11f0b72a..af29787b 100644
--- a/src/util/util/index.ts
+++ b/src/util/util/index.ts
@@ -23,3 +23,4 @@ export * from "./Snowflake";
export * from "./String";
export * from "./Token";
export * from "./TraverseDirectory";
+export * from "./Paths";
\ No newline at end of file
|
