From cf3873d62ff30da691dae7f7424c6365b5a72edc Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 17 Jul 2022 18:52:41 +0200 Subject: Add local disk caching for fetched assets --- api/assets/inline-plugins/autoRegister.js | 62 ++++++++++ api/assets/inline-plugins/fosscord-login.js | 12 ++ api/assets/preload-plugins/autoRegister.js | 62 ---------- api/assets/preload-plugins/fosscord-login.js | 12 -- api/client_test/index.html | 1 + api/package.json | 2 +- api/src/middlewares/TestClient.ts | 165 +++++++++++++++++---------- api/src/util/entities/AssetCacheItem.ts | 16 +++ api/src/util/index.ts | 1 + 9 files changed, 196 insertions(+), 137 deletions(-) create mode 100644 api/assets/inline-plugins/autoRegister.js create mode 100644 api/assets/inline-plugins/fosscord-login.js delete mode 100644 api/assets/preload-plugins/autoRegister.js delete mode 100644 api/assets/preload-plugins/fosscord-login.js create mode 100644 api/src/util/entities/AssetCacheItem.ts (limited to 'api') diff --git a/api/assets/inline-plugins/autoRegister.js b/api/assets/inline-plugins/autoRegister.js new file mode 100644 index 00000000..bb0b903d --- /dev/null +++ b/api/assets/inline-plugins/autoRegister.js @@ -0,0 +1,62 @@ +// Auto register guest account: +const prefix = [ + "mysterious", + "adventurous", + "courageous", + "precious", + "cynical", + "flamer ", + "despicable", + "suspicious", + "gorgeous", + "impeccable", + "lovely", + "stunning", + "keyed", + "phoned", + "glorious", + "amazing", + "strange", + "arcane" +]; +const suffix = [ + "Anonymous", + "Boy", + "Lurker", + "Keyhitter", + "User", + "Enjoyer", + "Hunk", + "Coolstar", + "Wrestling", + "TylerTheCreator", + "Ad", + "Gamer", + "Games", + "Programmer" +]; + +Array.prototype.random = function () { + return this[Math.floor(Math.random() * this.length)]; +}; + +function _generateName() { + return `${prefix.random()}${suffix.random()}`; +} + +var token = JSON.parse(localStorage.getItem("token")); +if (!token && location.pathname !== "/login" && location.pathname !== "/register") { + fetch(`${window.GLOBAL_ENV.API_ENDPOINT}/auth/register`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: `${_generateName()}`, consent: true }) //${Date.now().toString().slice(-4)} + }) + .then((x) => x.json()) + .then((x) => { + localStorage.setItem("token", `"${x.token}"`); + if (!window.localStorage) { + // client already loaded -> need to reload to apply the newly registered user token + location.reload(); + } + }); +} diff --git a/api/assets/inline-plugins/fosscord-login.js b/api/assets/inline-plugins/fosscord-login.js new file mode 100644 index 00000000..38f82200 --- /dev/null +++ b/api/assets/inline-plugins/fosscord-login.js @@ -0,0 +1,12 @@ +// Remove `` from header when we're not accessing `/login` or `/register` +// fosscord-login.css replaces discord's TOS tooltip with something more fitting for fosscord, which when included in the main app, causes other tooltips +// to be affected, which is potentially unwanted. +// +// This script removes fosscord-login.css when a user reloads the page. From testing, it appears fosscord already properly removes +// fosscord-login.css after login is successful, but not if you reload the page after logging in. This script is to remove fosscord-login.css in +// that specific case. + +var token = JSON.parse(localStorage.getItem("token")); +if (!token && location.pathname !== "/login" && location.pathname !== "/register") { + document.getElementById("logincss").remove(); +} diff --git a/api/assets/preload-plugins/autoRegister.js b/api/assets/preload-plugins/autoRegister.js deleted file mode 100644 index bb0b903d..00000000 --- a/api/assets/preload-plugins/autoRegister.js +++ /dev/null @@ -1,62 +0,0 @@ -// Auto register guest account: -const prefix = [ - "mysterious", - "adventurous", - "courageous", - "precious", - "cynical", - "flamer ", - "despicable", - "suspicious", - "gorgeous", - "impeccable", - "lovely", - "stunning", - "keyed", - "phoned", - "glorious", - "amazing", - "strange", - "arcane" -]; -const suffix = [ - "Anonymous", - "Boy", - "Lurker", - "Keyhitter", - "User", - "Enjoyer", - "Hunk", - "Coolstar", - "Wrestling", - "TylerTheCreator", - "Ad", - "Gamer", - "Games", - "Programmer" -]; - -Array.prototype.random = function () { - return this[Math.floor(Math.random() * this.length)]; -}; - -function _generateName() { - return `${prefix.random()}${suffix.random()}`; -} - -var token = JSON.parse(localStorage.getItem("token")); -if (!token && location.pathname !== "/login" && location.pathname !== "/register") { - fetch(`${window.GLOBAL_ENV.API_ENDPOINT}/auth/register`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ username: `${_generateName()}`, consent: true }) //${Date.now().toString().slice(-4)} - }) - .then((x) => x.json()) - .then((x) => { - localStorage.setItem("token", `"${x.token}"`); - if (!window.localStorage) { - // client already loaded -> need to reload to apply the newly registered user token - location.reload(); - } - }); -} diff --git a/api/assets/preload-plugins/fosscord-login.js b/api/assets/preload-plugins/fosscord-login.js deleted file mode 100644 index 38f82200..00000000 --- a/api/assets/preload-plugins/fosscord-login.js +++ /dev/null @@ -1,12 +0,0 @@ -// Remove `` from header when we're not accessing `/login` or `/register` -// fosscord-login.css replaces discord's TOS tooltip with something more fitting for fosscord, which when included in the main app, causes other tooltips -// to be affected, which is potentially unwanted. -// -// This script removes fosscord-login.css when a user reloads the page. From testing, it appears fosscord already properly removes -// fosscord-login.css after login is successful, but not if you reload the page after logging in. This script is to remove fosscord-login.css in -// that specific case. - -var token = JSON.parse(localStorage.getItem("token")); -if (!token && location.pathname !== "/login" && location.pathname !== "/register") { - document.getElementById("logincss").remove(); -} diff --git a/api/client_test/index.html b/api/client_test/index.html index b438b492..7a4fe627 100644 --- a/api/client_test/index.html +++ b/api/client_test/index.html @@ -7,6 +7,7 @@ + diff --git a/api/package.json b/api/package.json index 29fa82a1..051666e9 100644 --- a/api/package.json +++ b/api/package.json @@ -48,7 +48,7 @@ "@types/jsonwebtoken": "^8.5.0", "@types/morgan": "^1.9.3", "@types/multer": "^1.4.5", - "@types/node": "^14.17.9", + "@types/node": "^14.18.22", "@types/node-fetch": "^2.5.5", "@types/supertest": "^2.0.11", "@zerollup/ts-transform-paths": "^1.7.18", diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index ecf87681..e52a5e59 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -1,54 +1,47 @@ import express, { Request, Response, Application } from "express"; -import fs from "fs"; +import fs, { writeFile } from "fs"; import path from "path"; -import fetch, { Response as FetchResponse } from "node-fetch"; +import fetch, { Response as FetchResponse, Headers } from "node-fetch"; import ProxyAgent from 'proxy-agent'; import { Config } from "@fosscord/util"; +import { AssetCacheItem } from "../util/entities/AssetCacheItem" +import { FileLogger } from "typeorm"; export default function TestClient(app: Application) { const agent = new ProxyAgent(); - const assetCache = new Map(); - const indexHTML = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); - - var html = indexHTML; - const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( - /(https?)?(:\/\/?)/g, - "" - ); - const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + //build client page + let html = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); + html = applyEnv(html); + html = applyInlinePlugins(html); + html = applyPlugins(html); + html = applyPreloadPlugins(html); - if (CDN_ENDPOINT) { - html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + //load asset cache + let newAssetCache: Map = new Map(); + if(!fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache"))) { + fs.mkdirSync(path.join(__dirname, "..", "..", "assets", "cache")); } - if (GATEWAY_ENDPOINT) { - html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + if(fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"))) { + let rawdata = fs.readFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json")); + newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - // inline plugins - var files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - var plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - // plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - //preload plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - - - app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); - + //define routes + app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; - var response: FetchResponse; - var buffer: Buffer; - const cache = assetCache.get(req.params.file); - if (!cache) { + let response: FetchResponse; + let buffer: Buffer; + let assetCacheItem: AssetCacheItem = new AssetCacheItem(req.params.file); + if(newAssetCache.has(req.params.file)){ + assetCacheItem = newAssetCache.get(req.params.file)!; + assetCacheItem.Headers.forEach((value: any, name: any) => { + res.set(name, value); + }); + } + else { + console.log(`CACHE MISS! Asset file: ${req.params.file}`); response = await fetch(`https://discord.com/assets/${req.params.file}`, { agent, // @ts-ignore @@ -56,34 +49,24 @@ export default function TestClient(app: Application) { ...req.headers } }); - buffer = await response.buffer(); - } else { - response = cache.response; - buffer = cache.buffer; + + //set cache info + assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers)); + assetCacheItem.FilePath = path.join(__dirname, "..", "..", "assets", "cache", req.params.file); + assetCacheItem.Key = req.params.file; + //add to cache and save + newAssetCache.set(req.params.file, assetCacheItem); + fs.writeFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4)); + //download file + fs.writeFileSync(assetCacheItem.FilePath, await response.buffer()); } - - response.headers.forEach((value, name) => { - if ( - [ - "content-length", - "content-security-policy", - "strict-transport-security", - "set-cookie", - "transfer-encoding", - "expect-ct", - "access-control-allow-origin", - "content-encoding" - ].includes(name.toLowerCase()) - ) { - return; - } + + assetCacheItem.Headers.forEach((value: string, name: string) => { res.set(name, value); }); - assetCache.set(req.params.file, { buffer, response }); - - return res.send(buffer); + return res.send(fs.readFileSync(assetCacheItem.FilePath)); }); - app.get("/developers*", (req: Request, res: Response) => { + app.get("/developers*", (_req: Request, res: Response) => { const { useTestClient } = Config.get().client; res.set("Cache-Control", "public, max-age=" + 60 * 60 * 24); res.set("content-type", "text/html"); @@ -104,4 +87,62 @@ export default function TestClient(app: Application) { res.send(html); }); + + +} + +function applyEnv(html: string): string { + const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( + /(https?)?(:\/\/?)/g, + "" + ); + const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + if (CDN_ENDPOINT) { + html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + } + if (GATEWAY_ENDPOINT) { + html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + } + return html; +} + +function applyPlugins(html: string): string { + // plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function applyInlinePlugins(html: string): string{ + // inline plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "inline-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n\n`; }); + return html.replaceAll("", plugins); +} + +function applyPreloadPlugins(html: string): string{ + //preload plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function stripHeaders(headers: Headers): Headers { + [ + "content-length", + "content-security-policy", + "strict-transport-security", + "set-cookie", + "transfer-encoding", + "expect-ct", + "access-control-allow-origin", + "content-encoding" + ].forEach(headerName => { + headers.delete(headerName); + }); + return headers; } \ No newline at end of file diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts new file mode 100644 index 00000000..7fb4e1e6 --- /dev/null +++ b/api/src/util/entities/AssetCacheItem.ts @@ -0,0 +1,16 @@ +import { Headers } from "node-fetch"; + +export class AssetCacheItem { + public Key: string; + public FilePath: string; + public Headers: any; + + constructor(key: string){ + this.Key = key; + } + /*constructor(key: string, filePath: string, headers: Headers) { + this.Key = key; + this.FilePath = filePath; + this.Headers = headers; + }*/ +} \ No newline at end of file diff --git a/api/src/util/index.ts b/api/src/util/index.ts index ffbcf24e..ac439371 100644 --- a/api/src/util/index.ts +++ b/api/src/util/index.ts @@ -6,3 +6,4 @@ export * from "./utility/RandomInviteID"; export * from "./handlers/route"; export * from "./utility/String"; export * from "./handlers/Voice"; +export * from "./entities/AssetCacheItem"; \ No newline at end of file -- cgit 1.5.1 From 6e9bd98711f488dd06d2ce93027f8d9ea4cd3ad5 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 31 Jul 2022 13:27:15 +0200 Subject: Make requested changes (pr792) --- .gitignore | 2 ++ api/src/middlewares/TestClient.ts | 1 - api/src/util/entities/AssetCacheItem.ts | 15 +-------------- fosscord-server.code-workspace | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) (limited to 'api') diff --git a/.gitignore b/.gitignore index cc60b305..607b4f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ config.json .vscode/settings.json api/assets/plugins/*.js .idea/ +*.code-workspace + diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index e52a5e59..7292868c 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -27,7 +27,6 @@ export default function TestClient(app: Application) { newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - //define routes app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts index 7fb4e1e6..160dece6 100644 --- a/api/src/util/entities/AssetCacheItem.ts +++ b/api/src/util/entities/AssetCacheItem.ts @@ -1,16 +1,3 @@ -import { Headers } from "node-fetch"; - export class AssetCacheItem { - public Key: string; - public FilePath: string; - public Headers: any; - - constructor(key: string){ - this.Key = key; - } - /*constructor(key: string, filePath: string, headers: Headers) { - this.Key = key; - this.FilePath = filePath; - this.Headers = headers; - }*/ + constructor(public Key: string, public FilePath: string = "", public Headers: any = null as any) {} } \ No newline at end of file diff --git a/fosscord-server.code-workspace b/fosscord-server.code-workspace index fff238f1..7491cb35 100644 --- a/fosscord-server.code-workspace +++ b/fosscord-server.code-workspace @@ -24,9 +24,5 @@ { "path": "webrtc" } - ], - "settings": { - "awooga.originalColorCustomizations": {}, - "workbench.colorCustomizations": {} - } + ] } -- cgit 1.5.1