diff options
author | Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> | 2022-08-13 02:00:50 +0200 |
---|---|---|
committer | TheArcaneBrony <myrainbowdash949@gmail.com> | 2022-08-13 22:00:55 +0200 |
commit | 5e86d7ab9c5200d794c3adb2b422d20a2aefd2ce (patch) | |
tree | 0a4b23ee96862077b21dea20cf71205709e15f7c /src/cdn | |
parent | try to update build script (diff) | |
download | server-5e86d7ab9c5200d794c3adb2b422d20a2aefd2ce.tar.xz |
restructure to single project
Diffstat (limited to 'src/cdn')
-rw-r--r-- | src/cdn/Server.ts | 80 | ||||
-rw-r--r-- | src/cdn/index.ts | 4 | ||||
-rw-r--r-- | src/cdn/routes/attachments.ts | 100 | ||||
-rw-r--r-- | src/cdn/routes/avatars.ts | 102 | ||||
-rw-r--r-- | src/cdn/routes/external.ts | 58 | ||||
-rw-r--r-- | src/cdn/routes/ping.ts | 9 | ||||
-rw-r--r-- | src/cdn/routes/role-icons.ts | 101 | ||||
-rw-r--r-- | src/cdn/start.ts | 13 | ||||
-rw-r--r-- | src/cdn/util/FileStorage.ts | 51 | ||||
-rw-r--r-- | src/cdn/util/S3Storage.ts | 60 | ||||
-rw-r--r-- | src/cdn/util/Storage.ts | 64 | ||||
-rw-r--r-- | src/cdn/util/index.ts | 3 | ||||
-rw-r--r-- | src/cdn/util/multer.ts | 10 |
13 files changed, 655 insertions, 0 deletions
diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts new file mode 100644 index 00000000..b27d3321 --- /dev/null +++ b/src/cdn/Server.ts @@ -0,0 +1,80 @@ +import { Server, ServerOptions } from "lambert-server"; +import { Config, getOrInitialiseDatabase, registerRoutes } from "@fosscord/util"; +import path from "path"; +import avatarsRoute from "./routes/avatars"; +import iconsRoute from "./routes/role-icons"; +import bodyParser from "body-parser"; + +export interface CDNServerOptions extends ServerOptions {} + +export class CDNServer extends Server { + public declare options: CDNServerOptions; + + constructor(options?: Partial<CDNServerOptions>) { + super(options); + } + + async start() { + await getOrInitialiseDatabase(); + await Config.init(); + this.app.use((req, res, next) => { + res.set("Access-Control-Allow-Origin", "*"); + // TODO: use better CSP policy + res.set( + "Content-security-policy", + "default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';" + ); + res.set( + "Access-Control-Allow-Headers", + req.header("Access-Control-Request-Headers") || "*" + ); + res.set( + "Access-Control-Allow-Methods", + req.header("Access-Control-Request-Methods") || "*" + ); + next(); + }); + this.app.use(bodyParser.json({ inflate: true, limit: "10mb" })); + + await registerRoutes(this, path.join(__dirname, "routes/")); + + this.app.use("/icons/", avatarsRoute); + this.log("verbose", "[Server] Route /icons registered"); + + this.app.use("/role-icons/", iconsRoute); + this.log("verbose", "[Server] Route /role-icons registered"); + + this.app.use("/emojis/", avatarsRoute); + this.log("verbose", "[Server] Route /emojis registered"); + + this.app.use("/stickers/", avatarsRoute); + this.log("verbose", "[Server] Route /stickers registered"); + + this.app.use("/banners/", avatarsRoute); + this.log("verbose", "[Server] Route /banners registered"); + + this.app.use("/splashes/", avatarsRoute); + this.log("verbose", "[Server] Route /splashes registered"); + + this.app.use("/app-icons/", avatarsRoute); + this.log("verbose", "[Server] Route /app-icons registered"); + + this.app.use("/app-assets/", avatarsRoute); + this.log("verbose", "[Server] Route /app-assets registered"); + + this.app.use("/discover-splashes/", avatarsRoute); + this.log("verbose", "[Server] Route /discover-splashes registered"); + + this.app.use("/team-icons/", avatarsRoute); + this.log("verbose", "[Server] Route /team-icons registered"); + + this.app.use("/channel-icons/", avatarsRoute); + this.log("verbose", "[Server] Route /channel-icons registered"); + + return super.start(); + } + + async stop() { + return super.stop(); + } +} diff --git a/src/cdn/index.ts b/src/cdn/index.ts new file mode 100644 index 00000000..a24300d6 --- /dev/null +++ b/src/cdn/index.ts @@ -0,0 +1,4 @@ +export * from "./Server"; +export * from "./util/FileStorage"; +export * from "./util/Storage"; +export * from "./util/multer"; diff --git a/src/cdn/routes/attachments.ts b/src/cdn/routes/attachments.ts new file mode 100644 index 00000000..723a6c03 --- /dev/null +++ b/src/cdn/routes/attachments.ts @@ -0,0 +1,100 @@ +import { Router, Response, Request } from "express"; +import { Config, Snowflake } from "@fosscord/util"; +import { storage } from "../util/Storage"; +import FileType from "file-type"; +import { HTTPError } from "@fosscord/util"; +import { multer } from "../util/multer"; +import imageSize from "image-size"; + +const router = Router(); + +const SANITIZED_CONTENT_TYPE = [ + "text/html", + "text/mhtml", + "multipart/related", + "application/xhtml+xml", +]; + +router.post( + "/:channel_id", + multer.single("file"), + async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("file missing"); + + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { channel_id } = req.params; + const filename = originalname + .replaceAll(" ", "_") + .replace(/[^a-zA-Z0-9._]+/g, ""); + const id = Snowflake.generate(); + const path = `attachments/${channel_id}/${id}/${filename}`; + + const endpoint = + Config.get()?.cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + let width; + let height; + if (mimetype.includes("image")) { + const dimensions = imageSize(buffer); + if (dimensions) { + width = dimensions.width; + height = dimensions.height; + } + } + + const file = { + id, + content_type: mimetype, + filename: filename, + size, + url: `${endpoint}/${path}`, + width, + height, + }; + + return res.json(file); + } +); + +router.get( + "/:channel_id/:id/:filename", + async (req: Request, res: Response) => { + const { channel_id, id, filename } = req.params; + + const file = await storage.get( + `attachments/${channel_id}/${id}/${filename}` + ); + if (!file) throw new HTTPError("File not found"); + const type = await FileType.fromBuffer(file); + let content_type = type?.mime || "application/octet-stream"; + + if (SANITIZED_CONTENT_TYPE.includes(content_type)) { + content_type = "application/octet-stream"; + } + + res.set("Content-Type", content_type); + res.set("Cache-Control", "public, max-age=31536000"); + + return res.send(file); + } +); + +router.delete( + "/:channel_id/:id/:filename", + async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + + const { channel_id, id, filename } = req.params; + const path = `attachments/${channel_id}/${id}/${filename}`; + + await storage.delete(path); + + return res.send({ success: true }); + } +); + +export default router; diff --git a/src/cdn/routes/avatars.ts b/src/cdn/routes/avatars.ts new file mode 100644 index 00000000..40705b2e --- /dev/null +++ b/src/cdn/routes/avatars.ts @@ -0,0 +1,102 @@ +import { Router, Response, Request } from "express"; +import { Config, Snowflake } from "@fosscord/util"; +import { storage } from "../util/Storage"; +import FileType from "file-type"; +import { HTTPError } from "@fosscord/util"; +import crypto from "crypto"; +import { multer } from "../util/multer"; + +// TODO: check premium and animated pfp are allowed in the config +// TODO: generate different sizes of icon +// TODO: generate different image types of icon +// TODO: delete old icons + +const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; +const STATIC_MIME_TYPES = [ + "image/png", + "image/jpeg", + "image/webp", + "image/svg+xml", + "image/svg", +]; +const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; + +const router = Router(); + +router.post( + "/:user_id", + multer.single("file"), + async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("Missing file"); + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { user_id } = req.params; + + let hash = crypto + .createHash("md5") + .update(Snowflake.generate()) + .digest("hex"); + + const type = await FileType.fromBuffer(buffer); + if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) + throw new HTTPError("Invalid file type"); + if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash + + const path = `avatars/${user_id}/${hash}`; + const endpoint = + Config.get().cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + + return res.json({ + id: hash, + content_type: type.mime, + size, + url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`, + }); + } +); + +router.get("/:user_id", async (req: Request, res: Response) => { + let { user_id } = req.params; + user_id = user_id.split(".")[0]; // remove .file extension + const path = `avatars/${user_id}`; + + const file = await storage.get(path); + if (!file) throw new HTTPError("not found", 404); + const type = await FileType.fromBuffer(file); + + res.set("Content-Type", type?.mime); + res.set("Cache-Control", "public, max-age=31536000"); + + return res.send(file); +}); + +router.get("/:user_id/:hash", async (req: Request, res: Response) => { + let { user_id, hash } = req.params; + hash = hash.split(".")[0]; // remove .file extension + const path = `avatars/${user_id}/${hash}`; + + const file = await storage.get(path); + if (!file) throw new HTTPError("not found", 404); + const type = await FileType.fromBuffer(file); + + res.set("Content-Type", type?.mime); + res.set("Cache-Control", "public, max-age=31536000"); + + return res.send(file); +}); + +router.delete("/:user_id/:id", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + const { user_id, id } = req.params; + const path = `avatars/${user_id}/${id}`; + + await storage.delete(path); + + return res.send({ success: true }); +}); + +export default router; diff --git a/src/cdn/routes/external.ts b/src/cdn/routes/external.ts new file mode 100644 index 00000000..c9441fc2 --- /dev/null +++ b/src/cdn/routes/external.ts @@ -0,0 +1,58 @@ +import { Router, Response, Request } from "express"; +import fetch from "node-fetch"; +import { HTTPError } from "@fosscord/util"; +import { Snowflake, Config } from "@fosscord/util"; +import { storage } from "../util/Storage"; +import FileType from "file-type"; + +// TODO: somehow handle the deletion of images posted to the /external route + +const router = Router(); +const DEFAULT_FETCH_OPTIONS: any = { + redirect: "follow", + follow: 1, + headers: { + "user-agent": + "Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)", + }, + size: 1024 * 1024 * 8, + compress: true, + method: "GET", +}; + +router.post("/", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + + if (!req.body) throw new HTTPError("Invalid Body"); + + const { url } = req.body; + if (!url || typeof url !== "string") throw new HTTPError("Invalid url"); + + const id = Snowflake.generate(); + + try { + const response = await fetch(url, DEFAULT_FETCH_OPTIONS); + const buffer = await response.buffer(); + + await storage.set(`/external/${id}`, buffer); + + res.send({ id }); + } catch (error) { + throw new HTTPError("Couldn't fetch website"); + } +}); + +router.get("/:id", async (req: Request, res: Response) => { + const { id } = req.params; + + const file = await storage.get(`/external/${id}`); + if (!file) throw new HTTPError("File not found"); + const result = await FileType.fromBuffer(file); + + res.set("Content-Type", result?.mime); + + return res.send(file); +}); + +export default router; diff --git a/src/cdn/routes/ping.ts b/src/cdn/routes/ping.ts new file mode 100644 index 00000000..38daf81e --- /dev/null +++ b/src/cdn/routes/ping.ts @@ -0,0 +1,9 @@ +import { Router, Response, Request } from "express"; + +const router = Router(); + +router.get("/", (req: Request, res: Response) => { + res.send("pong"); +}); + +export default router; diff --git a/src/cdn/routes/role-icons.ts b/src/cdn/routes/role-icons.ts new file mode 100644 index 00000000..2e5c42dd --- /dev/null +++ b/src/cdn/routes/role-icons.ts @@ -0,0 +1,101 @@ +import { Router, Response, Request } from "express"; +import { Config, Snowflake } from "@fosscord/util"; +import { storage } from "../util/Storage"; +import FileType from "file-type"; +import { HTTPError } from "@fosscord/util"; +import crypto from "crypto"; +import { multer } from "../util/multer"; + +//Role icons ---> avatars.ts modified + +// TODO: check user rights and perks and animated pfp are allowed in the policies +// TODO: generate different sizes of icon +// TODO: generate different image types of icon + +const STATIC_MIME_TYPES = [ + "image/png", + "image/jpeg", + "image/webp", + "image/svg+xml", + "image/svg", +]; +const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES]; + +const router = Router(); + +router.post( + "/:role_id", + multer.single("file"), + async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("Missing file"); + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { role_id } = req.params; + + let hash = crypto + .createHash("md5") + .update(Snowflake.generate()) + .digest("hex"); + + const type = await FileType.fromBuffer(buffer); + if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) + throw new HTTPError("Invalid file type"); + + const path = `role-icons/${role_id}/${hash}.png`; + const endpoint = + Config.get().cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + + return res.json({ + id: hash, + content_type: type.mime, + size, + url: `${endpoint}${req.baseUrl}/${role_id}/${hash}`, + }); + } +); + +router.get("/:role_id", async (req: Request, res: Response) => { + let { role_id } = req.params; + //role_id = role_id.split(".")[0]; // remove .file extension + const path = `role-icons/${role_id}`; + + const file = await storage.get(path); + if (!file) throw new HTTPError("not found", 404); + const type = await FileType.fromBuffer(file); + + res.set("Content-Type", type?.mime); + res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); + + return res.send(file); +}); + +router.get("/:role_id/:hash", async (req: Request, res: Response) => { + let { role_id, hash } = req.params; + //hash = hash.split(".")[0]; // remove .file extension + const path = `role-icons/${role_id}/${hash}`; + + const file = await storage.get(path); + if (!file) throw new HTTPError("not found", 404); + const type = await FileType.fromBuffer(file); + + res.set("Content-Type", type?.mime); + res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); + + return res.send(file); +}); + +router.delete("/:role_id/:id", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) + throw new HTTPError("Invalid request signature"); + const { role_id, id } = req.params; + const path = `role-icons/${role_id}/${id}`; + + await storage.delete(path); + + return res.send({ success: true }); +}); + +export default router; diff --git a/src/cdn/start.ts b/src/cdn/start.ts new file mode 100644 index 00000000..71681b40 --- /dev/null +++ b/src/cdn/start.ts @@ -0,0 +1,13 @@ +import dotenv from "dotenv"; +dotenv.config(); + +import { CDNServer } from "./Server"; +const server = new CDNServer({ port: Number(process.env.PORT) || 3003 }); +server + .start() + .then(() => { + console.log("[Server] started on :" + server.options.port); + }) + .catch((e) => console.error("[Server] Error starting: ", e)); + +module.exports = server; diff --git a/src/cdn/util/FileStorage.ts b/src/cdn/util/FileStorage.ts new file mode 100644 index 00000000..aee9d345 --- /dev/null +++ b/src/cdn/util/FileStorage.ts @@ -0,0 +1,51 @@ +import { Storage } from "./Storage"; +import fs from "fs"; +import { join, relative, dirname } from "path"; +import { Readable } from "stream"; +//import ExifTransformer = require("exif-be-gone"); +import ExifTransformer from "exif-be-gone"; + +// TODO: split stored files into separate folders named after cloned route + +function getPath(path: string) { + // STORAGE_LOCATION has a default value in start.ts + const root = process.env.STORAGE_LOCATION || "../"; + let filename = join(root, path); + + if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) + throw new Error("invalid path"); + return filename; +} + +export class FileStorage implements Storage { + async get(path: string): Promise<Buffer | null> { + path = getPath(path); + try { + return fs.readFileSync(path); + } catch (error) { + try { + const files = fs.readdirSync(path); + if (!files.length) return null; + return fs.readFileSync(join(path, files[0])); + } catch (error) { + return null; + } + } + } + + async set(path: string, value: any) { + path = getPath(path); + //fse.ensureDirSync(dirname(path)); + fs.mkdirSync(dirname(path), {recursive: true}); + + value = Readable.from(value); + const cleaned_file = fs.createWriteStream(path); + + return value.pipe(new ExifTransformer()).pipe(cleaned_file); + } + + async delete(path: string) { + //TODO we should delete the parent directory if empty + fs.unlinkSync(getPath(path)); + } +} diff --git a/src/cdn/util/S3Storage.ts b/src/cdn/util/S3Storage.ts new file mode 100644 index 00000000..c4066817 --- /dev/null +++ b/src/cdn/util/S3Storage.ts @@ -0,0 +1,60 @@ +import { S3 } from "@aws-sdk/client-s3"; +import { Readable } from "stream"; +import { Storage } from "./Storage"; + +const readableToBuffer = (readable: Readable): Promise<Buffer> => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + readable.on("data", (chunk) => chunks.push(chunk)); + readable.on("error", reject); + readable.on("end", () => resolve(Buffer.concat(chunks))); + }); + +export class S3Storage implements Storage { + public constructor( + private client: S3, + private bucket: string, + private basePath?: string + ) {} + + /** + * Always return a string, to ensure consistency. + */ + get bucketBasePath() { + return this.basePath ?? ""; + } + + async set(path: string, data: Buffer): Promise<void> { + await this.client.putObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}`, + Body: data, + }); + } + + async get(path: string): Promise<Buffer | null> { + try { + const s3Object = await this.client.getObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath ?? ""}${path}`, + }); + + if (!s3Object.Body) return null; + + const body = s3Object.Body; + + return await readableToBuffer(<Readable>body); + } catch (err) { + console.error(`[CDN] Unable to get S3 object at path ${path}.`); + console.error(err); + return null; + } + } + + async delete(path: string): Promise<void> { + await this.client.deleteObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}`, + }); + } +} diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts new file mode 100644 index 00000000..728804a0 --- /dev/null +++ b/src/cdn/util/Storage.ts @@ -0,0 +1,64 @@ +import { FileStorage } from "./FileStorage"; +import path from "path"; +//import fse from "fs-extra"; +import fs from "fs"; +import { bgCyan, black } from "picocolors"; +import { S3 } from "@aws-sdk/client-s3"; +import { S3Storage } from "./S3Storage"; +process.cwd(); + +export interface Storage { + set(path: string, data: Buffer): Promise<void>; + get(path: string): Promise<Buffer | null>; + delete(path: string): Promise<void>; +} + +let storage: Storage; + +if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { + let location = process.env.STORAGE_LOCATION; + if (location) { + location = path.resolve(location); + } else { + location = path.join(process.cwd(), "files"); + } + console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`); + //fse.ensureDirSync(location); + fs.mkdirSync(location, {recursive: true}); + process.env.STORAGE_LOCATION = location; + + storage = new FileStorage(); +} else if (process.env.STORAGE_PROVIDER === "s3") { + const region = process.env.STORAGE_REGION, + bucket = process.env.STORAGE_BUCKET; + + if (!region) { + console.error( + `[CDN] You must provide a region when using the S3 storage provider.` + ); + process.exit(1); + } + + if (!bucket) { + console.error( + `[CDN] You must provide a bucket when using the S3 storage provider.` + ); + process.exit(1); + } + + // in the S3 provider, this should be the root path in the bucket + let location = process.env.STORAGE_LOCATION; + + if (!location) { + console.warn( + `[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...` + ); + location = undefined; + } + + const client = new S3({ region }); + + storage = new S3Storage(client, bucket, location); +} + +export { storage }; diff --git a/src/cdn/util/index.ts b/src/cdn/util/index.ts new file mode 100644 index 00000000..07a5c31a --- /dev/null +++ b/src/cdn/util/index.ts @@ -0,0 +1,3 @@ +export * from "./FileStorage"; +export * from "./multer"; +export * from "./Storage"; diff --git a/src/cdn/util/multer.ts b/src/cdn/util/multer.ts new file mode 100644 index 00000000..bfdf6aff --- /dev/null +++ b/src/cdn/util/multer.ts @@ -0,0 +1,10 @@ +import multerConfig from "multer"; + +export const multer = multerConfig({ + storage: multerConfig.memoryStorage(), + limits: { + fields: 10, + files: 10, + fileSize: 1024 * 1024 * 100, // 100 mb + }, +}); |