From cf3873d62ff30da691dae7f7424c6365b5a72edc Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 17 Jul 2022 18:52:41 +0200 Subject: Add local disk caching for fetched assets --- api/src/middlewares/TestClient.ts | 165 ++++++++++++++++++++------------ api/src/util/entities/AssetCacheItem.ts | 16 ++++ api/src/util/index.ts | 1 + 3 files changed, 120 insertions(+), 62 deletions(-) create mode 100644 api/src/util/entities/AssetCacheItem.ts (limited to 'api/src') diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index ecf87681..e52a5e59 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -1,54 +1,47 @@ import express, { Request, Response, Application } from "express"; -import fs from "fs"; +import fs, { writeFile } from "fs"; import path from "path"; -import fetch, { Response as FetchResponse } from "node-fetch"; +import fetch, { Response as FetchResponse, Headers } from "node-fetch"; import ProxyAgent from 'proxy-agent'; import { Config } from "@fosscord/util"; +import { AssetCacheItem } from "../util/entities/AssetCacheItem" +import { FileLogger } from "typeorm"; export default function TestClient(app: Application) { const agent = new ProxyAgent(); - const assetCache = new Map(); - const indexHTML = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); - - var html = indexHTML; - const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( - /(https?)?(:\/\/?)/g, - "" - ); - const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + //build client page + let html = fs.readFileSync(path.join(__dirname, "..", "..", "client_test", "index.html"), { encoding: "utf8" }); + html = applyEnv(html); + html = applyInlinePlugins(html); + html = applyPlugins(html); + html = applyPreloadPlugins(html); - if (CDN_ENDPOINT) { - html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + //load asset cache + let newAssetCache: Map = new Map(); + if(!fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache"))) { + fs.mkdirSync(path.join(__dirname, "..", "..", "assets", "cache")); } - if (GATEWAY_ENDPOINT) { - html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + if(fs.existsSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"))) { + let rawdata = fs.readFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json")); + newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - // inline plugins - var files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - var plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - // plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - //preload plugins - files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); - plugins = ""; - files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); - html = html.replaceAll("", plugins); - - - app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); - + //define routes + app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; - var response: FetchResponse; - var buffer: Buffer; - const cache = assetCache.get(req.params.file); - if (!cache) { + let response: FetchResponse; + let buffer: Buffer; + let assetCacheItem: AssetCacheItem = new AssetCacheItem(req.params.file); + if(newAssetCache.has(req.params.file)){ + assetCacheItem = newAssetCache.get(req.params.file)!; + assetCacheItem.Headers.forEach((value: any, name: any) => { + res.set(name, value); + }); + } + else { + console.log(`CACHE MISS! Asset file: ${req.params.file}`); response = await fetch(`https://discord.com/assets/${req.params.file}`, { agent, // @ts-ignore @@ -56,34 +49,24 @@ export default function TestClient(app: Application) { ...req.headers } }); - buffer = await response.buffer(); - } else { - response = cache.response; - buffer = cache.buffer; + + //set cache info + assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers)); + assetCacheItem.FilePath = path.join(__dirname, "..", "..", "assets", "cache", req.params.file); + assetCacheItem.Key = req.params.file; + //add to cache and save + newAssetCache.set(req.params.file, assetCacheItem); + fs.writeFileSync(path.join(__dirname, "..", "..", "assets", "cache", "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4)); + //download file + fs.writeFileSync(assetCacheItem.FilePath, await response.buffer()); } - - response.headers.forEach((value, name) => { - if ( - [ - "content-length", - "content-security-policy", - "strict-transport-security", - "set-cookie", - "transfer-encoding", - "expect-ct", - "access-control-allow-origin", - "content-encoding" - ].includes(name.toLowerCase()) - ) { - return; - } + + assetCacheItem.Headers.forEach((value: string, name: string) => { res.set(name, value); }); - assetCache.set(req.params.file, { buffer, response }); - - return res.send(buffer); + return res.send(fs.readFileSync(assetCacheItem.FilePath)); }); - app.get("/developers*", (req: Request, res: Response) => { + app.get("/developers*", (_req: Request, res: Response) => { const { useTestClient } = Config.get().client; res.set("Cache-Control", "public, max-age=" + 60 * 60 * 24); res.set("content-type", "text/html"); @@ -104,4 +87,62 @@ export default function TestClient(app: Application) { res.send(html); }); + + +} + +function applyEnv(html: string): string { + const CDN_ENDPOINT = (Config.get().cdn.endpointClient || Config.get()?.cdn.endpointPublic || process.env.CDN || "").replace( + /(https?)?(:\/\/?)/g, + "" + ); + const GATEWAY_ENDPOINT = Config.get().gateway.endpointClient || Config.get()?.gateway.endpointPublic || process.env.GATEWAY || ""; + + if (CDN_ENDPOINT) { + html = html.replace(/CDN_HOST: .+/, `CDN_HOST: \`${CDN_ENDPOINT}\`,`); + } + if (GATEWAY_ENDPOINT) { + html = html.replace(/GATEWAY_ENDPOINT: .+/, `GATEWAY_ENDPOINT: \`${GATEWAY_ENDPOINT}\`,`); + } + return html; +} + +function applyPlugins(html: string): string { + // plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function applyInlinePlugins(html: string): string{ + // inline plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "inline-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n\n`; }); + return html.replaceAll("", plugins); +} + +function applyPreloadPlugins(html: string): string{ + //preload plugins + let files = fs.readdirSync(path.join(__dirname, "..", "..", "assets", "preload-plugins")); + let plugins = ""; + files.forEach(x =>{if(x.endsWith(".js")) plugins += `\n`; }); + return html.replaceAll("", plugins); +} + +function stripHeaders(headers: Headers): Headers { + [ + "content-length", + "content-security-policy", + "strict-transport-security", + "set-cookie", + "transfer-encoding", + "expect-ct", + "access-control-allow-origin", + "content-encoding" + ].forEach(headerName => { + headers.delete(headerName); + }); + return headers; } \ No newline at end of file diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts new file mode 100644 index 00000000..7fb4e1e6 --- /dev/null +++ b/api/src/util/entities/AssetCacheItem.ts @@ -0,0 +1,16 @@ +import { Headers } from "node-fetch"; + +export class AssetCacheItem { + public Key: string; + public FilePath: string; + public Headers: any; + + constructor(key: string){ + this.Key = key; + } + /*constructor(key: string, filePath: string, headers: Headers) { + this.Key = key; + this.FilePath = filePath; + this.Headers = headers; + }*/ +} \ No newline at end of file diff --git a/api/src/util/index.ts b/api/src/util/index.ts index ffbcf24e..ac439371 100644 --- a/api/src/util/index.ts +++ b/api/src/util/index.ts @@ -6,3 +6,4 @@ export * from "./utility/RandomInviteID"; export * from "./handlers/route"; export * from "./utility/String"; export * from "./handlers/Voice"; +export * from "./entities/AssetCacheItem"; \ No newline at end of file -- cgit 1.5.1 From c73dbe9c5bffea1960cd1a113e62e6c69def386c Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sat, 23 Jul 2022 20:52:48 +1000 Subject: Moved user notes into separate table --- api/src/routes/users/@me/notes.ts | 39 ++++++++++++++++++++++++++++----------- util/src/entities/Note.ts | 18 ++++++++++++++++++ util/src/entities/User.ts | 4 +--- util/src/entities/index.ts | 3 ++- 4 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 util/src/entities/Note.ts (limited to 'api/src') diff --git a/api/src/routes/users/@me/notes.ts b/api/src/routes/users/@me/notes.ts index 4887b191..e3406d18 100644 --- a/api/src/routes/users/@me/notes.ts +++ b/api/src/routes/users/@me/notes.ts @@ -1,37 +1,54 @@ import { Request, Response, Router } from "express"; import { route } from "@fosscord/api"; -import { User, emitEvent } from "@fosscord/util"; +import { User, Note, emitEvent, Snowflake } from "@fosscord/util"; const router: Router = Router(); router.get("/:id", route({}), async (req: Request, res: Response) => { const { id } = req.params; - const user = await User.findOneOrFail({ where: { id: req.user_id }, select: ["notes"] }); - const note = user.notes[id]; + const note = await Note.findOneOrFail({ + where: { + owner: { id: req.user_id }, + target: { id: id }, + } + }); + return res.json({ - note: note, + note: note?.content, note_user_id: id, - user_id: user.id, + user_id: req.user_id, }); }); router.put("/:id", route({}), async (req: Request, res: Response) => { const { id } = req.params; - const user = await User.findOneOrFail({ where: { id: req.user_id } }); - const noteUser = await User.findOneOrFail({ where: { id: id }}); //if noted user does not exist throw + const owner = await User.findOneOrFail({ where: { id: req.user_id } }); + const target = await User.findOneOrFail({ where: { id: id } }); //if noted user does not exist throw const { note } = req.body; - await User.update({ id: req.user_id }, { notes: { ...user.notes, [noteUser.id]: note } }); + // await User.update({ id: req.user_id }, { notes: { ...user.notes, [noteUser.id]: note } }); + + if (await Note.findOne({ owner: { id: owner.id }, target: { id: target.id } })) { + Note.update( + { owner: { id: owner.id }, target: { id: target.id } }, + { owner, target, content: note } + ); + } + else { + Note.insert( + { id: Snowflake.generate(), owner, target, content: note } + ); + } await emitEvent({ event: "USER_NOTE_UPDATE", data: { note: note, - id: noteUser.id + id: target.id }, - user_id: user.id, - }) + user_id: owner.id, + }); return res.status(204); }); diff --git a/util/src/entities/Note.ts b/util/src/entities/Note.ts new file mode 100644 index 00000000..36017c5e --- /dev/null +++ b/util/src/entities/Note.ts @@ -0,0 +1,18 @@ +import { Column, Entity, JoinColumn, ManyToOne, Unique } from "typeorm"; +import { BaseClass } from "./BaseClass"; +import { User } from "./User"; + +@Entity("notes") +@Unique(["owner", "target"]) +export class Note extends BaseClass { + @JoinColumn({ name: "owner_id" }) + @ManyToOne(() => User, { onDelete: "CASCADE" }) + owner: User; + + @JoinColumn({ name: "target_id" }) + @ManyToOne(() => User, { onDelete: "CASCADE" }) + target: User; + + @Column() + content: string; +} \ No newline at end of file diff --git a/util/src/entities/User.ts b/util/src/entities/User.ts index 9b1c494e..8deb7ed5 100644 --- a/util/src/entities/User.ts +++ b/util/src/entities/User.ts @@ -5,6 +5,7 @@ import { Relationship } from "./Relationship"; import { ConnectedAccount } from "./ConnectedAccount"; import { Config, FieldErrors, Snowflake, trimSpecial } from ".."; import { Member, Session } from "."; +import { Note } from "./Note"; export enum PublicUserEnum { username, @@ -168,9 +169,6 @@ export class User extends BaseClass { @Column({ type: "simple-json", select: false }) extended_settings: string; - @Column({ type: "simple-json" }) - notes: { [key: string]: string }; //key is ID of user - toPublicUser() { const user: any = {}; PublicUserProjection.forEach((x) => { diff --git a/util/src/entities/index.ts b/util/src/entities/index.ts index f023d5a6..cb087136 100644 --- a/util/src/entities/index.ts +++ b/util/src/entities/index.ts @@ -27,4 +27,5 @@ export * from "./Template"; export * from "./User"; export * from "./VoiceState"; export * from "./Webhook"; -export * from "./ClientRelease"; \ No newline at end of file +export * from "./ClientRelease"; +export * from "./Note"; \ No newline at end of file -- cgit 1.5.1 From 78acb2a013d109e6a5179f8a9fe28f23819c5f3d Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sat, 23 Jul 2022 21:06:31 +1000 Subject: Delete Note if no content --- api/src/routes/users/@me/notes.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'api/src') diff --git a/api/src/routes/users/@me/notes.ts b/api/src/routes/users/@me/notes.ts index e3406d18..3c503942 100644 --- a/api/src/routes/users/@me/notes.ts +++ b/api/src/routes/users/@me/notes.ts @@ -27,18 +27,22 @@ router.put("/:id", route({}), async (req: Request, res: Response) => { const target = await User.findOneOrFail({ where: { id: id } }); //if noted user does not exist throw const { note } = req.body; - // await User.update({ id: req.user_id }, { notes: { ...user.notes, [noteUser.id]: note } }); - - if (await Note.findOne({ owner: { id: owner.id }, target: { id: target.id } })) { - Note.update( - { owner: { id: owner.id }, target: { id: target.id } }, - { owner, target, content: note } - ); + if (note && note.length) { + // upsert a note + if (await Note.findOne({ owner: { id: owner.id }, target: { id: target.id } })) { + Note.update( + { owner: { id: owner.id }, target: { id: target.id } }, + { owner, target, content: note } + ); + } + else { + Note.insert( + { id: Snowflake.generate(), owner, target, content: note } + ); + } } else { - Note.insert( - { id: Snowflake.generate(), owner, target, content: note } - ); + await Note.delete({ owner: { id: owner.id }, target: { id: target.id } }); } await emitEvent({ -- cgit 1.5.1 From f3b1e9b02641376177c4c358074e01f201ba0ff8 Mon Sep 17 00:00:00 2001 From: Puyodead1 Date: Sat, 30 Jul 2022 14:33:01 -0400 Subject: add discriminator to user modify schema --- api/assets/schemas.json | 5 +++++ api/src/routes/users/@me/index.ts | 1 + 2 files changed, 6 insertions(+) (limited to 'api/src') diff --git a/api/assets/schemas.json b/api/assets/schemas.json index ca42e676..f7efb888 100644 --- a/api/assets/schemas.json +++ b/api/assets/schemas.json @@ -12499,6 +12499,11 @@ "maxLength": 100, "type": "string" }, + "discriminator": { + "minLength": 4, + "maxLength": 4, + "type": "string" + }, "avatar": { "type": [ "null", diff --git a/api/src/routes/users/@me/index.ts b/api/src/routes/users/@me/index.ts index 1af413c4..7fc20457 100644 --- a/api/src/routes/users/@me/index.ts +++ b/api/src/routes/users/@me/index.ts @@ -11,6 +11,7 @@ export interface UserModifySchema { * @maxLength 100 */ username?: string; + discriminator?: string; avatar?: string | null; /** * @maxLength 1024 -- cgit 1.5.1 From 6e9bd98711f488dd06d2ce93027f8d9ea4cd3ad5 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Sun, 31 Jul 2022 13:27:15 +0200 Subject: Make requested changes (pr792) --- .gitignore | 2 ++ api/src/middlewares/TestClient.ts | 1 - api/src/util/entities/AssetCacheItem.ts | 15 +-------------- fosscord-server.code-workspace | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) (limited to 'api/src') diff --git a/.gitignore b/.gitignore index cc60b305..607b4f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ config.json .vscode/settings.json api/assets/plugins/*.js .idea/ +*.code-workspace + diff --git a/api/src/middlewares/TestClient.ts b/api/src/middlewares/TestClient.ts index e52a5e59..7292868c 100644 --- a/api/src/middlewares/TestClient.ts +++ b/api/src/middlewares/TestClient.ts @@ -27,7 +27,6 @@ export default function TestClient(app: Application) { newAssetCache = new Map(Object.entries(JSON.parse(rawdata.toString()))); } - //define routes app.use("/assets", express.static(path.join(__dirname, "..", "..", "assets"))); app.get("/assets/:file", async (req: Request, res: Response) => { delete req.headers.host; diff --git a/api/src/util/entities/AssetCacheItem.ts b/api/src/util/entities/AssetCacheItem.ts index 7fb4e1e6..160dece6 100644 --- a/api/src/util/entities/AssetCacheItem.ts +++ b/api/src/util/entities/AssetCacheItem.ts @@ -1,16 +1,3 @@ -import { Headers } from "node-fetch"; - export class AssetCacheItem { - public Key: string; - public FilePath: string; - public Headers: any; - - constructor(key: string){ - this.Key = key; - } - /*constructor(key: string, filePath: string, headers: Headers) { - this.Key = key; - this.FilePath = filePath; - this.Headers = headers; - }*/ + constructor(public Key: string, public FilePath: string = "", public Headers: any = null as any) {} } \ No newline at end of file diff --git a/fosscord-server.code-workspace b/fosscord-server.code-workspace index fff238f1..7491cb35 100644 --- a/fosscord-server.code-workspace +++ b/fosscord-server.code-workspace @@ -24,9 +24,5 @@ { "path": "webrtc" } - ], - "settings": { - "awooga.originalColorCustomizations": {}, - "workbench.colorCustomizations": {} - } + ] } -- cgit 1.5.1