From cf3873d62ff30da691dae7f7424c6365b5a72edc Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 17 Jul 2022 18:52:41 +0200 Subject: Add local disk caching for fetched assets --- api/src/middlewares/TestClient.ts | 165 ++++++++++++++++++++------------ api/src/util/entities/AssetCacheItem.ts | 16 ++++ api/src/util/index.ts | 1 + 3 files changed, 120 insertions(+), 62 deletions(-) create mode 100644 api/src/util/entities/AssetCacheItem.ts (limited to 'api/src') diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index ecf87681..e52a5e59 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -1,54 +1,47 @@ import express, { Request, Response, Application } from "express"; -import fs from "fs"; +import fs, { writeFile } from "fs"; import path from "path"; -import fetch, { Response as FetchResponse } from "node-fetch"; +import fetch, { Response as FetchResponse, Headers } from "node-fetch"; import ProxyAgent from 'proxy-agent'; import { Config } from "@fosscord/util"; +import { AssetCacheItem } from "../util/entities/AssetCacheItem" +import { FileLogger } from "typeorm"; export default function TestClient(app: Application) { const agent = new ProxyAgent(); - const assetCache = new Map(); - const indexHTML = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); - - var html = indexHTML; - const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( - /(https?)?(:\/\/?)/g, - "" - ); - const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + //build client page + let html = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); + html = applyEnv(html); + html = applyInlinePlugins(html); + html = applyPlugins(html); + html = applyPreloadPlugins(html); - if (CDN_ENDPOINT) { - html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + //load asset cache + let newAssetCache: Map = new Map(); + if(!fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache"))) { + fs.mkdirSync(path.join(__dirname, "..", "..", "assets", "cache")); } - if (GATEWAY_ENDPOINT) { - html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + if(fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"))) { + let rawdata = fs.readFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json")); + newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - // inline plugins - var files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - var plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - // plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - //preload plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - - - app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); - + //define routes + app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; - var response: FetchResponse; - var buffer: Buffer; - const cache = assetCache.get(req.params.file); - if (!cache) { + let response: FetchResponse; + let buffer: Buffer; + let assetCacheItem: AssetCacheItem = new AssetCacheItem(req.params.file); + if(newAssetCache.has(req.params.file)){ + assetCacheItem = newAssetCache.get(req.params.file)!; + assetCacheItem.Headers.forEach((value: any, name: any) => { + res.set(name, value); + }); + } + else { + console.log(`CACHE MISS! Asset file: ${req.params.file}`); response = await fetch(`https://discord.com/assets/${req.params.file}`, { agent, // @ts-ignore @@ -56,34 +49,24 @@ export default function TestClient(app: Application) { ...req.headers } }); - buffer = await response.buffer(); - } else { - response = cache.response; - buffer = cache.buffer; + + //set cache info + assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers)); + assetCacheItem.FilePath = path.join(__dirname, "..", "..", "assets", "cache", req.params.file); + assetCacheItem.Key = req.params.file; + //add to cache and save + newAssetCache.set(req.params.file, assetCacheItem); + fs.writeFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4)); + //download file + fs.writeFileSync(assetCacheItem.FilePath, await response.buffer()); } - - response.headers.forEach((value, name) => { - if ( - [ - "content-length", - "content-security-policy", - "strict-transport-security", - "set-cookie", - "transfer-encoding", - "expect-ct", - "access-control-allow-origin", - "content-encoding" - ].includes(name.toLowerCase()) - ) { - return; - } + + assetCacheItem.Headers.forEach((value: string, name: string) => { res.set(name, value); }); - assetCache.set(req.params.file, { buffer, response }); - - return res.send(buffer); + return res.send(fs.readFileSync(assetCacheItem.FilePath)); }); - app.get("/developers*", (req: Request, res: Response) => { + app.get("/developers*", (_req: Request, res: Response) => { const { useTestClient } = Config.get().client; res.set("Cache-Control", "public, max-age=" + 60 * 60 * 24); res.set("content-type", "text/html"); @@ -104,4 +87,62 @@ export default function TestClient(app: Application) { res.send(html); }); + + +} + +function applyEnv(html: string): string { + const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( + /(https?)?(:\/\/?)/g, + "" + ); + const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + if (CDN_ENDPOINT) { + html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + } + if (GATEWAY_ENDPOINT) { + html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + } + return html; +} + +function applyPlugins(html: string): string { + // plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function applyInlinePlugins(html: string): string{ + // inline plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "inline-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n\n`; }); + return html.replaceAll("", plugins); +} + +function applyPreloadPlugins(html: string): string{ + //preload plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function stripHeaders(headers: Headers): Headers { + [ + "content-length", + "content-security-policy", + "strict-transport-security", + "set-cookie", + "transfer-encoding", + "expect-ct", + "access-control-allow-origin", + "content-encoding" + ].forEach(headerName => { + headers.delete(headerName); + }); + return headers; } \ No newline at end of file diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts new file mode 100644 index 00000000..7fb4e1e6 --- /dev/null +++ b/api/src/util/entities/AssetCacheItem.ts @@ -0,0 +1,16 @@ +import { Headers } from "node-fetch"; + +export class AssetCacheItem { + public Key: string; + public FilePath: string; + public Headers: any; + + constructor(key: string){ + this.Key = key; + } + /*constructor(key: string, filePath: string, headers: Headers) { + this.Key = key; + this.FilePath = filePath; + this.Headers = headers; + }*/ +} \ No newline at end of file diff --git a/api/src/util/index.ts b/api/src/util/index.ts index ffbcf24e..ac439371 100644 --- a/api/src/util/index.ts +++ b/api/src/util/index.ts @@ -6,3 +6,4 @@ export * from "./utility/RandomInviteID"; export * from "./handlers/route"; export * from "./utility/String"; export * from "./handlers/Voice"; +export * from "./entities/AssetCacheItem"; \ No newline at end of file -- cgit 1.4.1 From 6e9bd98711f488dd06d2ce93027f8d9ea4cd3ad5 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 31 Jul 2022 13:27:15 +0200 Subject: Make requested changes (pr792) --- .gitignore | 2 ++ api/src/middlewares/TestClient.ts | 1 - api/src/util/entities/AssetCacheItem.ts | 15 +-------------- fosscord-server.code-workspace | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) (limited to 'api/src') diff --git a/.gitignore b/.gitignore index cc60b305..607b4f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ config.json .vscode/settings.json api/assets/plugins/*.js .idea/ +*.code-workspace + diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index e52a5e59..7292868c 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -27,7 +27,6 @@ export default function TestClient(app: Application) { newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - //define routes app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts index 7fb4e1e6..160dece6 100644 --- a/api/src/util/entities/AssetCacheItem.ts +++ b/api/src/util/entities/AssetCacheItem.ts @@ -1,16 +1,3 @@ -import { Headers } from "node-fetch"; - export class AssetCacheItem { - public Key: string; - public FilePath: string; - public Headers: any; - - constructor(key: string){ - this.Key = key; - } - /*constructor(key: string, filePath: string, headers: Headers) { - this.Key = key; - this.FilePath = filePath; - this.Headers = headers; - }*/ + constructor(public Key: string, public FilePath: string = "", public Headers: any = null as any) {} } \ No newline at end of file diff --git a/fosscord-server.code-workspace b/fosscord-server.code-workspace index fff238f1..7491cb35 100644 --- a/fosscord-server.code-workspace +++ b/fosscord-server.code-workspace @@ -24,9 +24,5 @@ { "path": "webrtc" } - ], - "settings": { - "awooga.originalColorCustomizations": {}, - "workbench.colorCustomizations": {} - } + ] } -- cgit 1.4.1