diff options
Diffstat (limited to 'src/cdn')
-rw-r--r-- | src/cdn/Server.ts | 14 | ||||
-rw-r--r-- | src/cdn/index.ts | 2 | ||||
-rw-r--r-- | src/cdn/routes/attachments.ts | 145 | ||||
-rw-r--r-- | src/cdn/routes/avatars.ts | 78 | ||||
-rw-r--r-- | src/cdn/routes/external.ts | 15 | ||||
-rw-r--r-- | src/cdn/routes/ping.ts | 2 | ||||
-rw-r--r-- | src/cdn/routes/role-icons.ts | 76 | ||||
-rw-r--r-- | src/cdn/util/FileStorage.ts | 9 | ||||
-rw-r--r-- | src/cdn/util/S3Storage.ts | 12 | ||||
-rw-r--r-- | src/cdn/util/Storage.ts | 18 | ||||
-rw-r--r-- | src/cdn/util/multer.ts | 4 |
11 files changed, 148 insertions, 227 deletions
diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index b27d3321..ec5edc68 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -1,9 +1,9 @@ -import { Server, ServerOptions } from "lambert-server"; import { Config, getOrInitialiseDatabase, registerRoutes } from "@fosscord/util"; +import bodyParser from "body-parser"; +import { Server, ServerOptions } from "lambert-server"; import path from "path"; import avatarsRoute from "./routes/avatars"; import iconsRoute from "./routes/role-icons"; -import bodyParser from "body-parser"; export interface CDNServerOptions extends ServerOptions {} @@ -24,14 +24,8 @@ export class CDNServer extends Server { "Content-security-policy", "default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';" ); - res.set( - "Access-Control-Allow-Headers", - req.header("Access-Control-Request-Headers") || "*" - ); - res.set( - "Access-Control-Allow-Methods", - req.header("Access-Control-Request-Methods") || "*" - ); + res.set("Access-Control-Allow-Headers", req.header("Access-Control-Request-Headers") || "*"); + res.set("Access-Control-Allow-Methods", req.header("Access-Control-Request-Methods") || "*"); next(); }); this.app.use(bodyParser.json({ inflate: true, limit: "10mb" })); diff --git a/src/cdn/index.ts b/src/cdn/index.ts index a24300d6..e32fd606 100644 --- a/src/cdn/index.ts +++ b/src/cdn/index.ts @@ -1,4 +1,4 @@ export * from "./Server"; export * from "./util/FileStorage"; -export * from "./util/Storage"; export * from "./util/multer"; +export * from "./util/Storage"; diff --git a/src/cdn/routes/attachments.ts b/src/cdn/routes/attachments.ts index 723a6c03..013f03d8 100644 --- a/src/cdn/routes/attachments.ts +++ b/src/cdn/routes/attachments.ts @@ -1,100 +1,77 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; +import { Config, HTTPError, Snowflake } from "@fosscord/util"; +import { Request, Response, Router } from "express"; import FileType from "file-type"; -import { HTTPError } from "@fosscord/util"; -import { multer } from "../util/multer"; import imageSize from "image-size"; +import { multer } from "../util/multer"; +import { storage } from "../util/Storage"; const router = Router(); -const SANITIZED_CONTENT_TYPE = [ - "text/html", - "text/mhtml", - "multipart/related", - "application/xhtml+xml", -]; - -router.post( - "/:channel_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("file missing"); - - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { channel_id } = req.params; - const filename = originalname - .replaceAll(" ", "_") - .replace(/[^a-zA-Z0-9._]+/g, ""); - const id = Snowflake.generate(); - const path = `attachments/${channel_id}/${id}/${filename}`; - - const endpoint = - Config.get()?.cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - let width; - let height; - if (mimetype.includes("image")) { - const dimensions = imageSize(buffer); - if (dimensions) { - width = dimensions.width; - height = dimensions.height; - } +const SANITIZED_CONTENT_TYPE = ["text/html", "text/mhtml", "multipart/related", "application/xhtml+xml"]; + +router.post("/:channel_id", multer.single("file"), async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("file missing"); + + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { channel_id } = req.params; + const filename = originalname.replaceAll(" ", "_").replace(/[^a-zA-Z0-9._]+/g, ""); + const id = Snowflake.generate(); + const path = `attachments/${channel_id}/${id}/${filename}`; + + const endpoint = Config.get()?.cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + let width; + let height; + if (mimetype.includes("image")) { + const dimensions = imageSize(buffer); + if (dimensions) { + width = dimensions.width; + height = dimensions.height; } + } - const file = { - id, - content_type: mimetype, - filename: filename, - size, - url: `${endpoint}/${path}`, - width, - height, - }; - - return res.json(file); + const file = { + id, + content_type: mimetype, + filename: filename, + size, + url: `${endpoint}/${path}`, + width, + height + }; + + return res.json(file); +}); + +router.get("/:channel_id/:id/:filename", async (req: Request, res: Response) => { + const { channel_id, id, filename } = req.params; + + const file = await storage.get(`attachments/${channel_id}/${id}/${filename}`); + if (!file) throw new HTTPError("File not found"); + const type = await FileType.fromBuffer(file); + let content_type = type?.mime || "application/octet-stream"; + + if (SANITIZED_CONTENT_TYPE.includes(content_type)) { + content_type = "application/octet-stream"; } -); - -router.get( - "/:channel_id/:id/:filename", - async (req: Request, res: Response) => { - const { channel_id, id, filename } = req.params; - - const file = await storage.get( - `attachments/${channel_id}/${id}/${filename}` - ); - if (!file) throw new HTTPError("File not found"); - const type = await FileType.fromBuffer(file); - let content_type = type?.mime || "application/octet-stream"; - - if (SANITIZED_CONTENT_TYPE.includes(content_type)) { - content_type = "application/octet-stream"; - } - res.set("Content-Type", content_type); - res.set("Cache-Control", "public, max-age=31536000"); + res.set("Content-Type", content_type); + res.set("Cache-Control", "public, max-age=31536000"); - return res.send(file); - } -); + return res.send(file); +}); -router.delete( - "/:channel_id/:id/:filename", - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); +router.delete("/:channel_id/:id/:filename", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); - const { channel_id, id, filename } = req.params; - const path = `attachments/${channel_id}/${id}/${filename}`; + const { channel_id, id, filename } = req.params; + const path = `attachments/${channel_id}/${id}/${filename}`; - await storage.delete(path); + await storage.delete(path); - return res.send({ success: true }); - } -); + return res.send({ success: true }); +}); export default router; diff --git a/src/cdn/routes/avatars.ts b/src/cdn/routes/avatars.ts index 40705b2e..fa26938f 100644 --- a/src/cdn/routes/avatars.ts +++ b/src/cdn/routes/avatars.ts @@ -1,10 +1,9 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType from "file-type"; -import { HTTPError } from "@fosscord/util"; +import { Config, HTTPError, Snowflake } from "@fosscord/util"; import crypto from "crypto"; +import { Request, Response, Router } from "express"; +import FileType from "file-type"; import { multer } from "../util/multer"; +import { storage } from "../util/Storage"; // TODO: check premium and animated pfp are allowed in the config // TODO: generate different sizes of icon @@ -12,51 +11,35 @@ import { multer } from "../util/multer"; // TODO: delete old icons const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; -const STATIC_MIME_TYPES = [ - "image/png", - "image/jpeg", - "image/webp", - "image/svg+xml", - "image/svg", -]; +const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; const router = Router(); -router.post( - "/:user_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("Missing file"); - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { user_id } = req.params; - - let hash = crypto - .createHash("md5") - .update(Snowflake.generate()) - .digest("hex"); - - const type = await FileType.fromBuffer(buffer); - if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) - throw new HTTPError("Invalid file type"); - if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash - - const path = `avatars/${user_id}/${hash}`; - const endpoint = - Config.get().cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - - return res.json({ - id: hash, - content_type: type.mime, - size, - url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`, - }); - } -); +router.post("/:user_id", multer.single("file"), async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("Missing file"); + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { user_id } = req.params; + + let hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex"); + + const type = await FileType.fromBuffer(buffer); + if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type"); + if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash + + const path = `avatars/${user_id}/${hash}`; + const endpoint = Config.get().cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + + return res.json({ + id: hash, + content_type: type.mime, + size, + url: `${endpoint}${req.baseUrl}/${user_id}/${hash}` + }); +}); router.get("/:user_id", async (req: Request, res: Response) => { let { user_id } = req.params; @@ -89,8 +72,7 @@ router.get("/:user_id/:hash", async (req: Request, res: Response) => { }); router.delete("/:user_id/:id", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); const { user_id, id } = req.params; const path = `avatars/${user_id}/${id}`; diff --git a/src/cdn/routes/external.ts b/src/cdn/routes/external.ts index c9441fc2..7ccf9b8a 100644 --- a/src/cdn/routes/external.ts +++ b/src/cdn/routes/external.ts @@ -1,9 +1,8 @@ -import { Router, Response, Request } from "express"; +import { Config, HTTPError, Snowflake } from "@fosscord/util"; +import { Request, Response, Router } from "express"; +import FileType from "file-type"; import fetch from "node-fetch"; -import { HTTPError } from "@fosscord/util"; -import { Snowflake, Config } from "@fosscord/util"; import { storage } from "../util/Storage"; -import FileType from "file-type"; // TODO: somehow handle the deletion of images posted to the /external route @@ -12,17 +11,15 @@ const DEFAULT_FETCH_OPTIONS: any = { redirect: "follow", follow: 1, headers: { - "user-agent": - "Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)", + "user-agent": "Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)" }, size: 1024 * 1024 * 8, compress: true, - method: "GET", + method: "GET" }; router.post("/", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); if (!req.body) throw new HTTPError("Invalid Body"); diff --git a/src/cdn/routes/ping.ts b/src/cdn/routes/ping.ts index 38daf81e..420cf567 100644 --- a/src/cdn/routes/ping.ts +++ b/src/cdn/routes/ping.ts @@ -1,4 +1,4 @@ -import { Router, Response, Request } from "express"; +import { Request, Response, Router } from "express"; const router = Router(); diff --git a/src/cdn/routes/role-icons.ts b/src/cdn/routes/role-icons.ts index 2e5c42dd..768e194f 100644 --- a/src/cdn/routes/role-icons.ts +++ b/src/cdn/routes/role-icons.ts @@ -1,10 +1,9 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType from "file-type"; -import { HTTPError } from "@fosscord/util"; +import { Config, HTTPError, Snowflake } from "@fosscord/util"; import crypto from "crypto"; +import { Request, Response, Router } from "express"; +import FileType from "file-type"; import { multer } from "../util/multer"; +import { storage } from "../util/Storage"; //Role icons ---> avatars.ts modified @@ -12,50 +11,34 @@ import { multer } from "../util/multer"; // TODO: generate different sizes of icon // TODO: generate different image types of icon -const STATIC_MIME_TYPES = [ - "image/png", - "image/jpeg", - "image/webp", - "image/svg+xml", - "image/svg", -]; +const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES]; const router = Router(); -router.post( - "/:role_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("Missing file"); - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { role_id } = req.params; - - let hash = crypto - .createHash("md5") - .update(Snowflake.generate()) - .digest("hex"); - - const type = await FileType.fromBuffer(buffer); - if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) - throw new HTTPError("Invalid file type"); - - const path = `role-icons/${role_id}/${hash}.png`; - const endpoint = - Config.get().cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - - return res.json({ - id: hash, - content_type: type.mime, - size, - url: `${endpoint}${req.baseUrl}/${role_id}/${hash}`, - }); - } -); +router.post("/:role_id", multer.single("file"), async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + if (!req.file) throw new HTTPError("Missing file"); + const { buffer, mimetype, size, originalname, fieldname } = req.file; + const { role_id } = req.params; + + let hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex"); + + const type = await FileType.fromBuffer(buffer); + if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type"); + + const path = `role-icons/${role_id}/${hash}.png`; + const endpoint = Config.get().cdn.endpointPublic || "http://localhost:3003"; + + await storage.set(path, buffer); + + return res.json({ + id: hash, + content_type: type.mime, + size, + url: `${endpoint}${req.baseUrl}/${role_id}/${hash}` + }); +}); router.get("/:role_id", async (req: Request, res: Response) => { let { role_id } = req.params; @@ -88,8 +71,7 @@ router.get("/:role_id/:hash", async (req: Request, res: Response) => { }); router.delete("/:role_id/:id", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); const { role_id, id } = req.params; const path = `role-icons/${role_id}/${id}`; diff --git a/src/cdn/util/FileStorage.ts b/src/cdn/util/FileStorage.ts index aee9d345..fea013a6 100644 --- a/src/cdn/util/FileStorage.ts +++ b/src/cdn/util/FileStorage.ts @@ -1,7 +1,7 @@ -import { Storage } from "./Storage"; import fs from "fs"; -import { join, relative, dirname } from "path"; +import { dirname, join } from "path"; import { Readable } from "stream"; +import { Storage } from "./Storage"; //import ExifTransformer = require("exif-be-gone"); import ExifTransformer from "exif-be-gone"; @@ -12,8 +12,7 @@ function getPath(path: string) { const root = process.env.STORAGE_LOCATION || "../"; let filename = join(root, path); - if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) - throw new Error("invalid path"); + if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path"); return filename; } @@ -36,7 +35,7 @@ export class FileStorage implements Storage { async set(path: string, value: any) { path = getPath(path); //fse.ensureDirSync(dirname(path)); - fs.mkdirSync(dirname(path), {recursive: true}); + fs.mkdirSync(dirname(path), { recursive: true }); value = Readable.from(value); const cleaned_file = fs.createWriteStream(path); diff --git a/src/cdn/util/S3Storage.ts b/src/cdn/util/S3Storage.ts index c4066817..a7892e5e 100644 --- a/src/cdn/util/S3Storage.ts +++ b/src/cdn/util/S3Storage.ts @@ -11,11 +11,7 @@ const readableToBuffer = (readable: Readable): Promise<Buffer> => }); export class S3Storage implements Storage { - public constructor( - private client: S3, - private bucket: string, - private basePath?: string - ) {} + public constructor(private client: S3, private bucket: string, private basePath?: string) {} /** * Always return a string, to ensure consistency. @@ -28,7 +24,7 @@ export class S3Storage implements Storage { await this.client.putObject({ Bucket: this.bucket, Key: `${this.bucketBasePath}${path}`, - Body: data, + Body: data }); } @@ -36,7 +32,7 @@ export class S3Storage implements Storage { try { const s3Object = await this.client.getObject({ Bucket: this.bucket, - Key: `${this.bucketBasePath ?? ""}${path}`, + Key: `${this.bucketBasePath ?? ""}${path}` }); if (!s3Object.Body) return null; @@ -54,7 +50,7 @@ export class S3Storage implements Storage { async delete(path: string): Promise<void> { await this.client.deleteObject({ Bucket: this.bucket, - Key: `${this.bucketBasePath}${path}`, + Key: `${this.bucketBasePath}${path}` }); } } diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts index 728804a0..1ab6a1d9 100644 --- a/src/cdn/util/Storage.ts +++ b/src/cdn/util/Storage.ts @@ -1,9 +1,9 @@ -import { FileStorage } from "./FileStorage"; import path from "path"; +import { FileStorage } from "./FileStorage"; //import fse from "fs-extra"; +import { S3 } from "@aws-sdk/client-s3"; import fs from "fs"; import { bgCyan, black } from "picocolors"; -import { S3 } from "@aws-sdk/client-s3"; import { S3Storage } from "./S3Storage"; process.cwd(); @@ -24,7 +24,7 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { } console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`); //fse.ensureDirSync(location); - fs.mkdirSync(location, {recursive: true}); + fs.mkdirSync(location, { recursive: true }); process.env.STORAGE_LOCATION = location; storage = new FileStorage(); @@ -33,16 +33,12 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { bucket = process.env.STORAGE_BUCKET; if (!region) { - console.error( - `[CDN] You must provide a region when using the S3 storage provider.` - ); + console.error(`[CDN] You must provide a region when using the S3 storage provider.`); process.exit(1); } if (!bucket) { - console.error( - `[CDN] You must provide a bucket when using the S3 storage provider.` - ); + console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`); process.exit(1); } @@ -50,9 +46,7 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { let location = process.env.STORAGE_LOCATION; if (!location) { - console.warn( - `[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...` - ); + console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`); location = undefined; } diff --git a/src/cdn/util/multer.ts b/src/cdn/util/multer.ts index bfdf6aff..f56b0fb5 100644 --- a/src/cdn/util/multer.ts +++ b/src/cdn/util/multer.ts @@ -5,6 +5,6 @@ export const multer = multerConfig({ limits: { fields: 10, files: 10, - fileSize: 1024 * 1024 * 100, // 100 mb - }, + fileSize: 1024 * 1024 * 100 // 100 mb + } }); |