diff options
Diffstat (limited to 'cdn/src')
-rw-r--r-- | cdn/src/Server.ts | 80 | ||||
-rw-r--r-- | cdn/src/index.ts | 4 | ||||
-rw-r--r-- | cdn/src/routes/attachments.ts | 100 | ||||
-rw-r--r-- | cdn/src/routes/avatars.ts | 104 | ||||
-rw-r--r-- | cdn/src/routes/external.ts | 91 | ||||
-rw-r--r-- | cdn/src/routes/guilds.ts | 10 | ||||
-rw-r--r-- | cdn/src/routes/ping.ts | 9 | ||||
-rw-r--r-- | cdn/src/routes/role-icons.ts | 101 | ||||
-rw-r--r-- | cdn/src/start.ts | 13 | ||||
-rw-r--r-- | cdn/src/util/FileStorage.ts | 51 | ||||
-rw-r--r-- | cdn/src/util/S3Storage.ts | 60 | ||||
-rw-r--r-- | cdn/src/util/Storage.ts | 62 | ||||
-rw-r--r-- | cdn/src/util/index.ts | 3 | ||||
-rw-r--r-- | cdn/src/util/multer.ts | 10 |
14 files changed, 0 insertions, 698 deletions
diff --git a/cdn/src/Server.ts b/cdn/src/Server.ts deleted file mode 100644 index 5b395589..00000000 --- a/cdn/src/Server.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { Server, ServerOptions } from "lambert-server"; -import { Config, initDatabase, registerRoutes } from "@fosscord/util"; -import path from "path"; -import avatarsRoute from "./routes/avatars"; -import iconsRoute from "./routes/role-icons"; -import bodyParser from "body-parser"; - -export interface CDNServerOptions extends ServerOptions { } - -export class CDNServer extends Server { - public declare options: CDNServerOptions; - - constructor(options?: Partial<CDNServerOptions>) { - super(options); - } - - async start() { - await initDatabase(); - await Config.init(); - this.app.use((req, res, next) => { - res.set("Access-Control-Allow-Origin", "*"); - // TODO: use better CSP policy - res.set( - "Content-security-policy", - "default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';" - ); - res.set( - "Access-Control-Allow-Headers", - req.header("Access-Control-Request-Headers") || "*" - ); - res.set( - "Access-Control-Allow-Methods", - req.header("Access-Control-Request-Methods") || "*" - ); - next(); - }); - this.app.use(bodyParser.json({ inflate: true, limit: "10mb" })); - - await registerRoutes(this, path.join(__dirname, "routes/")); - - this.app.use("/icons/", avatarsRoute); - this.log("verbose", "[Server] Route /icons registered"); - - this.app.use("/role-icons/", iconsRoute); - this.log("verbose", "[Server] Route /role-icons registered"); - - this.app.use("/emojis/", avatarsRoute); - this.log("verbose", "[Server] Route /emojis registered"); - - this.app.use("/stickers/", avatarsRoute); - this.log("verbose", "[Server] Route /stickers registered"); - - this.app.use("/banners/", avatarsRoute); - this.log("verbose", "[Server] Route /banners registered"); - - this.app.use("/splashes/", avatarsRoute); - this.log("verbose", "[Server] Route /splashes registered"); - - this.app.use("/app-icons/", avatarsRoute); - this.log("verbose", "[Server] Route /app-icons registered"); - - this.app.use("/app-assets/", avatarsRoute); - this.log("verbose", "[Server] Route /app-assets registered"); - - this.app.use("/discover-splashes/", avatarsRoute); - this.log("verbose", "[Server] Route /discover-splashes registered"); - - this.app.use("/team-icons/", avatarsRoute); - this.log("verbose", "[Server] Route /team-icons registered"); - - this.app.use("/channel-icons/", avatarsRoute); - this.log("verbose", "[Server] Route /channel-icons registered"); - - return super.start(); - } - - async stop() { - return super.stop(); - } -} diff --git a/cdn/src/index.ts b/cdn/src/index.ts deleted file mode 100644 index a24300d6..00000000 --- a/cdn/src/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./Server"; -export * from "./util/FileStorage"; -export * from "./util/Storage"; -export * from "./util/multer"; diff --git a/cdn/src/routes/attachments.ts b/cdn/src/routes/attachments.ts deleted file mode 100644 index ae50bc48..00000000 --- a/cdn/src/routes/attachments.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType from "file-type"; -import { HTTPError } from "lambert-server"; -import { multer } from "../util/multer"; -import imageSize from "image-size"; - -const router = Router(); - -const SANITIZED_CONTENT_TYPE = [ - "text/html", - "text/mhtml", - "multipart/related", - "application/xhtml+xml", -]; - -router.post( - "/:channel_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("file missing"); - - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { channel_id } = req.params; - const filename = originalname - .replaceAll(" ", "_") - .replace(/[^a-zA-Z0-9._]+/g, ""); - const id = Snowflake.generate(); - const path = `attachments/${channel_id}/${id}/${filename}`; - - const endpoint = - Config.get()?.cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - var width; - var height; - if (mimetype.includes("image")) { - const dimensions = imageSize(buffer); - if (dimensions) { - width = dimensions.width; - height = dimensions.height; - } - } - - const file = { - id, - content_type: mimetype, - filename: filename, - size, - url: `${endpoint}/${path}`, - width, - height, - }; - - return res.json(file); - } -); - -router.get( - "/:channel_id/:id/:filename", - async (req: Request, res: Response) => { - const { channel_id, id, filename } = req.params; - - const file = await storage.get( - `attachments/${channel_id}/${id}/${filename}` - ); - if (!file) throw new HTTPError("File not found"); - const type = await FileType.fromBuffer(file); - let content_type = type?.mime || "application/octet-stream"; - - if (SANITIZED_CONTENT_TYPE.includes(content_type)) { - content_type = "application/octet-stream"; - } - - res.set("Content-Type", content_type); - res.set("Cache-Control", "public, max-age=31536000"); - - return res.send(file); - } -); - -router.delete( - "/:channel_id/:id/:filename", - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - - const { channel_id, id, filename } = req.params; - const path = `attachments/${channel_id}/${id}/${filename}`; - - await storage.delete(path); - - return res.send({ success: true }); - } -); - -export default router; diff --git a/cdn/src/routes/avatars.ts b/cdn/src/routes/avatars.ts deleted file mode 100644 index e5e25a4c..00000000 --- a/cdn/src/routes/avatars.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType from "file-type"; -import { HTTPError } from "lambert-server"; -import crypto from "crypto"; -import { multer } from "../util/multer"; - -// TODO: check premium and animated pfp are allowed in the config -// TODO: generate different sizes of icon -// TODO: generate different image types of icon -// TODO: delete old icons - -const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; -const STATIC_MIME_TYPES = [ - "image/png", - "image/jpeg", - "image/webp", - "image/svg+xml", - "image/svg", -]; -const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; - -const router = Router(); - -router.post( - "/:user_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("Missing file"); - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { user_id } = req.params; - - var hash = crypto - .createHash("md5") - .update(Snowflake.generate()) - .digest("hex"); - - const type = await FileType.fromBuffer(buffer); - if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) - throw new HTTPError("Invalid file type"); - if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash - - const path = `avatars/${user_id}/${hash}`; - const endpoint = - Config.get().cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - - return res.json({ - id: hash, - content_type: type.mime, - size, - url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`, - }); - } -); - -router.get("/:user_id", async (req: Request, res: Response) => { - var { user_id } = req.params; - user_id = user_id.split(".")[0]; // remove .file extension - const path = `avatars/${user_id}`; - - const file = await storage.get(path); - if (!file) throw new HTTPError("not found", 404); - const type = await FileType.fromBuffer(file); - - res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); - - return res.send(file); -}); - -export const getAvatar = async (req: Request, res: Response) => { - var { user_id, hash } = req.params; - hash = hash.split(".")[0]; // remove .file extension - const path = `avatars/${user_id}/${hash}`; - - const file = await storage.get(path); - if (!file) throw new HTTPError("not found", 404); - const type = await FileType.fromBuffer(file); - - res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); - - return res.send(file); -} - -router.get("/:user_id/:hash", getAvatar); - -router.delete("/:user_id/:id", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - const { user_id, id } = req.params; - const path = `avatars/${user_id}/${id}`; - - await storage.delete(path); - - return res.send({ success: true }); -}); - -export default router; diff --git a/cdn/src/routes/external.ts b/cdn/src/routes/external.ts deleted file mode 100644 index fc12c017..00000000 --- a/cdn/src/routes/external.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Router, Response, Request } from "express"; -import fetch from "node-fetch"; -import { HTTPError } from "lambert-server"; -import { Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType, { stream } from "file-type"; -import { Config } from "@fosscord/util"; -import sharp from "sharp"; - -// TODO: somehow handle the deletion of images posted to the /external route - -const router = Router(); -const DEFAULT_FETCH_OPTIONS: any = { - redirect: "follow", - follow: 1, - headers: { - "user-agent": - "Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)", - }, - size: 1024 * 1024 * 8, - compress: true, - method: "GET", -}; - -router.post("/", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - - if (!req.body) throw new HTTPError("Invalid Body"); - - const { url } = req.body; - if (!url || typeof url !== "string") throw new HTTPError("Invalid url"); - - const id = Snowflake.generate(); - - try { - const response = await fetch(url, DEFAULT_FETCH_OPTIONS); - const buffer = await response.buffer(); - - await storage.set(`/external/${id}`, buffer); - - res.send({ id }); - } catch (error) { - throw new HTTPError("Couldn't fetch website"); - } -}); - -router.get("/:id", async (req: Request, res: Response) => { - const { id } = req.params; - - const file = await storage.get(`/external/${id}`); - if (!file) throw new HTTPError("File not found"); - const result = await FileType.fromBuffer(file); - - res.set("Content-Type", result?.mime); - - return res.send(file); -}); - -// this method is gross lol don't care -router.get("/resize/:url", async (req: Request, res: Response) => { - const url = decodeURIComponent(req.params.url); - const { width, height } = req.query; - if (!width || !height) throw new HTTPError("Must provide width and height"); - - const { resizeHeightMax, resizeWidthMax } = Config.get().cdn; - const w = Math.min(parseInt(width as string), resizeWidthMax ?? 100); - const h = Math.min(parseInt(height as string), resizeHeightMax ?? 100); - if (w < 1 || h < 1) throw new HTTPError("Width and height must be greater than 0"); - - let buffer, response; - try { - response = await fetch(url, DEFAULT_FETCH_OPTIONS); - buffer = await response.buffer(); - } - catch (e) { - throw new HTTPError("Couldn't fetch website"); - } - - const resizedBuffer = await sharp(buffer) - .resize(parseInt(width as string), parseInt(height as string), { - fit: "inside", - }) - .toBuffer(); - - res.setHeader("Content-Disposition", "attachment"); - res.setHeader("Content-Type", response.headers.get("content-type") ?? "image/png"); - return res.end(resizedBuffer); -}); - -export default router; diff --git a/cdn/src/routes/guilds.ts b/cdn/src/routes/guilds.ts deleted file mode 100644 index 3c4b646c..00000000 --- a/cdn/src/routes/guilds.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Router } from "express"; -import { getAvatar } from "./avatars"; - -const router = Router(); - -// TODO: handle guild profiles -router.get("/:guild_id/users/:user_id/avatars/:hash", getAvatar); -router.get("/:guild_id/users/:user_id/banners/:hash", getAvatar); - -export default router; \ No newline at end of file diff --git a/cdn/src/routes/ping.ts b/cdn/src/routes/ping.ts deleted file mode 100644 index 38daf81e..00000000 --- a/cdn/src/routes/ping.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Router, Response, Request } from "express"; - -const router = Router(); - -router.get("/", (req: Request, res: Response) => { - res.send("pong"); -}); - -export default router; diff --git a/cdn/src/routes/role-icons.ts b/cdn/src/routes/role-icons.ts deleted file mode 100644 index 12aae8a4..00000000 --- a/cdn/src/routes/role-icons.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Router, Response, Request } from "express"; -import { Config, Snowflake } from "@fosscord/util"; -import { storage } from "../util/Storage"; -import FileType from "file-type"; -import { HTTPError } from "lambert-server"; -import crypto from "crypto"; -import { multer } from "../util/multer"; - -//Role icons ---> avatars.ts modified - -// TODO: check user rights and perks and animated pfp are allowed in the policies -// TODO: generate different sizes of icon -// TODO: generate different image types of icon - -const STATIC_MIME_TYPES = [ - "image/png", - "image/jpeg", - "image/webp", - "image/svg+xml", - "image/svg", -]; -const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES]; - -const router = Router(); - -router.post( - "/:role_id", - multer.single("file"), - async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - if (!req.file) throw new HTTPError("Missing file"); - const { buffer, mimetype, size, originalname, fieldname } = req.file; - const { role_id } = req.params; - - var hash = crypto - .createHash("md5") - .update(Snowflake.generate()) - .digest("hex"); - - const type = await FileType.fromBuffer(buffer); - if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) - throw new HTTPError("Invalid file type"); - - const path = `role-icons/${role_id}/${hash}.png`; - const endpoint = - Config.get().cdn.endpointPublic || "http://localhost:3003"; - - await storage.set(path, buffer); - - return res.json({ - id: hash, - content_type: type.mime, - size, - url: `${endpoint}${req.baseUrl}/${role_id}/${hash}`, - }); - } -); - -router.get("/:role_id", async (req: Request, res: Response) => { - var { role_id } = req.params; - //role_id = role_id.split(".")[0]; // remove .file extension - const path = `role-icons/${role_id}`; - - const file = await storage.get(path); - if (!file) throw new HTTPError("not found", 404); - const type = await FileType.fromBuffer(file); - - res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); - - return res.send(file); -}); - -router.get("/:role_id/:hash", async (req: Request, res: Response) => { - var { role_id, hash } = req.params; - //hash = hash.split(".")[0]; // remove .file extension - const path = `role-icons/${role_id}/${hash}`; - - const file = await storage.get(path); - if (!file) throw new HTTPError("not found", 404); - const type = await FileType.fromBuffer(file); - - res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); - - return res.send(file); -}); - -router.delete("/:role_id/:id", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) - throw new HTTPError("Invalid request signature"); - const { role_id, id } = req.params; - const path = `role-icons/${role_id}/${id}`; - - await storage.delete(path); - - return res.send({ success: true }); -}); - -export default router; diff --git a/cdn/src/start.ts b/cdn/src/start.ts deleted file mode 100644 index 71681b40..00000000 --- a/cdn/src/start.ts +++ /dev/null @@ -1,13 +0,0 @@ -import dotenv from "dotenv"; -dotenv.config(); - -import { CDNServer } from "./Server"; -const server = new CDNServer({ port: Number(process.env.PORT) || 3003 }); -server - .start() - .then(() => { - console.log("[Server] started on :" + server.options.port); - }) - .catch((e) => console.error("[Server] Error starting: ", e)); - -module.exports = server; diff --git a/cdn/src/util/FileStorage.ts b/cdn/src/util/FileStorage.ts deleted file mode 100644 index 84ecf556..00000000 --- a/cdn/src/util/FileStorage.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Storage } from "./Storage"; -import fs from "fs"; -import fse from "fs-extra"; -import { join, relative, dirname } from "path"; -import "missing-native-js-functions"; -import { Readable } from "stream"; -import ExifTransformer = require("exif-be-gone"); - -// TODO: split stored files into separate folders named after cloned route - -function getPath(path: string) { - // STORAGE_LOCATION has a default value in start.ts - const root = process.env.STORAGE_LOCATION || "../"; - var filename = join(root, path); - - if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) - throw new Error("invalid path"); - return filename; -} - -export class FileStorage implements Storage { - async get(path: string): Promise<Buffer | null> { - path = getPath(path); - try { - return fs.readFileSync(path); - } catch (error) { - try { - const files = fs.readdirSync(path); - if (!files.length) return null; - return fs.readFileSync(join(path, files[0])); - } catch (error) { - return null; - } - } - } - - async set(path: string, value: any) { - path = getPath(path); - fse.ensureDirSync(dirname(path)); - - value = Readable.from(value); - const cleaned_file = fs.createWriteStream(path); - - return value.pipe(new ExifTransformer()).pipe(cleaned_file); - } - - async delete(path: string) { - //TODO we should delete the parent directory if empty - fs.unlinkSync(getPath(path)); - } -} diff --git a/cdn/src/util/S3Storage.ts b/cdn/src/util/S3Storage.ts deleted file mode 100644 index c4066817..00000000 --- a/cdn/src/util/S3Storage.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { S3 } from "@aws-sdk/client-s3"; -import { Readable } from "stream"; -import { Storage } from "./Storage"; - -const readableToBuffer = (readable: Readable): Promise<Buffer> => - new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - readable.on("data", (chunk) => chunks.push(chunk)); - readable.on("error", reject); - readable.on("end", () => resolve(Buffer.concat(chunks))); - }); - -export class S3Storage implements Storage { - public constructor( - private client: S3, - private bucket: string, - private basePath?: string - ) {} - - /** - * Always return a string, to ensure consistency. - */ - get bucketBasePath() { - return this.basePath ?? ""; - } - - async set(path: string, data: Buffer): Promise<void> { - await this.client.putObject({ - Bucket: this.bucket, - Key: `${this.bucketBasePath}${path}`, - Body: data, - }); - } - - async get(path: string): Promise<Buffer | null> { - try { - const s3Object = await this.client.getObject({ - Bucket: this.bucket, - Key: `${this.bucketBasePath ?? ""}${path}`, - }); - - if (!s3Object.Body) return null; - - const body = s3Object.Body; - - return await readableToBuffer(<Readable>body); - } catch (err) { - console.error(`[CDN] Unable to get S3 object at path ${path}.`); - console.error(err); - return null; - } - } - - async delete(path: string): Promise<void> { - await this.client.deleteObject({ - Bucket: this.bucket, - Key: `${this.bucketBasePath}${path}`, - }); - } -} diff --git a/cdn/src/util/Storage.ts b/cdn/src/util/Storage.ts deleted file mode 100644 index 89dd5634..00000000 --- a/cdn/src/util/Storage.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { FileStorage } from "./FileStorage"; -import path from "path"; -import fse from "fs-extra"; -import { bgCyan, black } from "picocolors"; -import { S3 } from "@aws-sdk/client-s3"; -import { S3Storage } from "./S3Storage"; -process.cwd(); - -export interface Storage { - set(path: string, data: Buffer): Promise<void>; - get(path: string): Promise<Buffer | null>; - delete(path: string): Promise<void>; -} - -let storage: Storage; - -if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { - let location = process.env.STORAGE_LOCATION; - if (location) { - location = path.resolve(location); - } else { - location = path.join(process.cwd(), "files"); - } - console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`); - fse.ensureDirSync(location); - process.env.STORAGE_LOCATION = location; - - storage = new FileStorage(); -} else if (process.env.STORAGE_PROVIDER === "s3") { - const region = process.env.STORAGE_REGION, - bucket = process.env.STORAGE_BUCKET; - - if (!region) { - console.error( - `[CDN] You must provide a region when using the S3 storage provider.` - ); - process.exit(1); - } - - if (!bucket) { - console.error( - `[CDN] You must provide a bucket when using the S3 storage provider.` - ); - process.exit(1); - } - - // in the S3 provider, this should be the root path in the bucket - let location = process.env.STORAGE_LOCATION; - - if (!location) { - console.warn( - `[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...` - ); - location = undefined; - } - - const client = new S3({ region }); - - storage = new S3Storage(client, bucket, location); -} - -export { storage }; diff --git a/cdn/src/util/index.ts b/cdn/src/util/index.ts deleted file mode 100644 index 07a5c31a..00000000 --- a/cdn/src/util/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./FileStorage"; -export * from "./multer"; -export * from "./Storage"; diff --git a/cdn/src/util/multer.ts b/cdn/src/util/multer.ts deleted file mode 100644 index bfdf6aff..00000000 --- a/cdn/src/util/multer.ts +++ /dev/null @@ -1,10 +0,0 @@ -import multerConfig from "multer"; - -export const multer = multerConfig({ - storage: multerConfig.memoryStorage(), - limits: { - fields: 10, - files: 10, - fileSize: 1024 * 1024 * 100, // 100 mb - }, -}); |