13 files changed, 655 insertions, 0 deletions
diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts
new file mode 100644
index 00000000..b27d3321
--- /dev/null
+++ b/src/cdn/Server.ts
@@ -0,0 +1,80 @@
+import { Server, ServerOptions } from "lambert-server";
+import { Config, getOrInitialiseDatabase, registerRoutes } from "@fosscord/util";
+import path from "path";
+import avatarsRoute from "./routes/avatars";
+import iconsRoute from "./routes/role-icons";
+import bodyParser from "body-parser";
+
+export interface CDNServerOptions extends ServerOptions {}
+
+export class CDNServer extends Server {
+	public declare options: CDNServerOptions;
+
+	constructor(options?: Partial<CDNServerOptions>) {
+		super(options);
+	}
+
+	async start() {
+		await getOrInitialiseDatabase();
+		await Config.init();
+		this.app.use((req, res, next) => {
+			res.set("Access-Control-Allow-Origin", "*");
+			// TODO: use better CSP policy
+			res.set(
+				"Content-security-policy",
+				"default-src *  data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';"
+			);
+			res.set(
+				"Access-Control-Allow-Headers",
+				req.header("Access-Control-Request-Headers") || "*"
+			);
+			res.set(
+				"Access-Control-Allow-Methods",
+				req.header("Access-Control-Request-Methods") || "*"
+			);
+			next();
+		});
+		this.app.use(bodyParser.json({ inflate: true, limit: "10mb" }));
+
+		await registerRoutes(this, path.join(__dirname, "routes/"));
+
+		this.app.use("/icons/", avatarsRoute);
+		this.log("verbose", "[Server] Route /icons registered");
+
+		this.app.use("/role-icons/", iconsRoute);
+		this.log("verbose", "[Server] Route /role-icons registered");
+
+		this.app.use("/emojis/", avatarsRoute);
+		this.log("verbose", "[Server] Route /emojis registered");
+
+		this.app.use("/stickers/", avatarsRoute);
+		this.log("verbose", "[Server] Route /stickers registered");
+
+		this.app.use("/banners/", avatarsRoute);
+		this.log("verbose", "[Server] Route /banners registered");
+
+		this.app.use("/splashes/", avatarsRoute);
+		this.log("verbose", "[Server] Route /splashes registered");
+
+		this.app.use("/app-icons/", avatarsRoute);
+		this.log("verbose", "[Server] Route /app-icons registered");
+
+		this.app.use("/app-assets/", avatarsRoute);
+		this.log("verbose", "[Server] Route /app-assets registered");
+
+		this.app.use("/discover-splashes/", avatarsRoute);
+		this.log("verbose", "[Server] Route /discover-splashes registered");
+
+		this.app.use("/team-icons/", avatarsRoute);
+		this.log("verbose", "[Server] Route /team-icons registered");
+
+		this.app.use("/channel-icons/", avatarsRoute);
+		this.log("verbose", "[Server] Route /channel-icons registered");
+
+		return super.start();
+	}
+
+	async stop() {
+		return super.stop();
+	}
+}
diff --git a/src/cdn/index.ts b/src/cdn/index.ts
new file mode 100644
index 00000000..a24300d6
--- /dev/null
+++ b/src/cdn/index.ts
@@ -0,0 +1,4 @@
+export * from "./Server";
+export * from "./util/FileStorage";
+export * from "./util/Storage";
+export * from "./util/multer";
diff --git a/src/cdn/routes/attachments.ts b/src/cdn/routes/attachments.ts
new file mode 100644
index 00000000..723a6c03
--- /dev/null
+++ b/src/cdn/routes/attachments.ts
@@ -0,0 +1,100 @@
+import { Router, Response, Request } from "express";
+import { Config, Snowflake } from "@fosscord/util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+import { HTTPError } from "@fosscord/util";
+import { multer } from "../util/multer";
+import imageSize from "image-size";
+
+const router = Router();
+
+const SANITIZED_CONTENT_TYPE = [
+	"text/html",
+	"text/mhtml",
+	"multipart/related",
+	"application/xhtml+xml",
+];
+
+router.post(
+	"/:channel_id",
+	multer.single("file"),
+	async (req: Request, res: Response) => {
+		if (req.headers.signature !== Config.get().security.requestSignature)
+			throw new HTTPError("Invalid request signature");
+		if (!req.file) throw new HTTPError("file missing");
+
+		const { buffer, mimetype, size, originalname, fieldname } = req.file;
+		const { channel_id } = req.params;
+		const filename = originalname
+			.replaceAll(" ", "_")
+			.replace(/[^a-zA-Z0-9._]+/g, "");
+		const id = Snowflake.generate();
+		const path = `attachments/${channel_id}/${id}/${filename}`;
+
+		const endpoint =
+			Config.get()?.cdn.endpointPublic || "http://localhost:3003";
+
+		await storage.set(path, buffer);
+		let width;
+		let height;
+		if (mimetype.includes("image")) {
+			const dimensions = imageSize(buffer);
+			if (dimensions) {
+				width = dimensions.width;
+				height = dimensions.height;
+			}
+		}
+
+		const file = {
+			id,
+			content_type: mimetype,
+			filename: filename,
+			size,
+			url: `${endpoint}/${path}`,
+			width,
+			height,
+		};
+
+		return res.json(file);
+	}
+);
+
+router.get(
+	"/:channel_id/:id/:filename",
+	async (req: Request, res: Response) => {
+		const { channel_id, id, filename } = req.params;
+
+		const file = await storage.get(
+			`attachments/${channel_id}/${id}/${filename}`
+		);
+		if (!file) throw new HTTPError("File not found");
+		const type = await FileType.fromBuffer(file);
+		let content_type = type?.mime || "application/octet-stream";
+
+		if (SANITIZED_CONTENT_TYPE.includes(content_type)) {
+			content_type = "application/octet-stream";
+		}
+
+		res.set("Content-Type", content_type);
+		res.set("Cache-Control", "public, max-age=31536000");
+
+		return res.send(file);
+	}
+);
+
+router.delete(
+	"/:channel_id/:id/:filename",
+	async (req: Request, res: Response) => {
+		if (req.headers.signature !== Config.get().security.requestSignature)
+			throw new HTTPError("Invalid request signature");
+
+		const { channel_id, id, filename } = req.params;
+		const path = `attachments/${channel_id}/${id}/${filename}`;
+
+		await storage.delete(path);
+
+		return res.send({ success: true });
+	}
+);
+
+export default router;
diff --git a/src/cdn/routes/avatars.ts b/src/cdn/routes/avatars.ts
new file mode 100644
index 00000000..40705b2e
--- /dev/null
+++ b/src/cdn/routes/avatars.ts
@@ -0,0 +1,102 @@
+import { Router, Response, Request } from "express";
+import { Config, Snowflake } from "@fosscord/util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+import { HTTPError } from "@fosscord/util";
+import crypto from "crypto";
+import { multer } from "../util/multer";
+
+// TODO: check premium and animated pfp are allowed in the config
+// TODO: generate different sizes of icon
+// TODO: generate different image types of icon
+// TODO: delete old icons
+
+const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"];
+const STATIC_MIME_TYPES = [
+	"image/png",
+	"image/jpeg",
+	"image/webp",
+	"image/svg+xml",
+	"image/svg",
+];
+const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES];
+
+const router = Router();
+
+router.post(
+	"/:user_id",
+	multer.single("file"),
+	async (req: Request, res: Response) => {
+		if (req.headers.signature !== Config.get().security.requestSignature)
+			throw new HTTPError("Invalid request signature");
+		if (!req.file) throw new HTTPError("Missing file");
+		const { buffer, mimetype, size, originalname, fieldname } = req.file;
+		const { user_id } = req.params;
+
+		let hash = crypto
+			.createHash("md5")
+			.update(Snowflake.generate())
+			.digest("hex");
+
+		const type = await FileType.fromBuffer(buffer);
+		if (!type || !ALLOWED_MIME_TYPES.includes(type.mime))
+			throw new HTTPError("Invalid file type");
+		if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash
+
+		const path = `avatars/${user_id}/${hash}`;
+		const endpoint =
+			Config.get().cdn.endpointPublic || "http://localhost:3003";
+
+		await storage.set(path, buffer);
+
+		return res.json({
+			id: hash,
+			content_type: type.mime,
+			size,
+			url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`,
+		});
+	}
+);
+
+router.get("/:user_id", async (req: Request, res: Response) => {
+	let { user_id } = req.params;
+	user_id = user_id.split(".")[0]; // remove .file extension
+	const path = `avatars/${user_id}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000");
+
+	return res.send(file);
+});
+
+router.get("/:user_id/:hash", async (req: Request, res: Response) => {
+	let { user_id, hash } = req.params;
+	hash = hash.split(".")[0]; // remove .file extension
+	const path = `avatars/${user_id}/${hash}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000");
+
+	return res.send(file);
+});
+
+router.delete("/:user_id/:id", async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	const { user_id, id } = req.params;
+	const path = `avatars/${user_id}/${id}`;
+
+	await storage.delete(path);
+
+	return res.send({ success: true });
+});
+
+export default router;
diff --git a/src/cdn/routes/external.ts b/src/cdn/routes/external.ts
new file mode 100644
index 00000000..c9441fc2
--- /dev/null
+++ b/src/cdn/routes/external.ts
@@ -0,0 +1,58 @@
+import { Router, Response, Request } from "express";
+import fetch from "node-fetch";
+import { HTTPError } from "@fosscord/util";
+import { Snowflake, Config } from "@fosscord/util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+
+// TODO: somehow handle the deletion of images posted to the /external route
+
+const router = Router();
+const DEFAULT_FETCH_OPTIONS: any = {
+	redirect: "follow",
+	follow: 1,
+	headers: {
+		"user-agent":
+			"Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)",
+	},
+	size: 1024 * 1024 * 8,
+	compress: true,
+	method: "GET",
+};
+
+router.post("/", async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+
+	if (!req.body) throw new HTTPError("Invalid Body");
+
+	const { url } = req.body;
+	if (!url || typeof url !== "string") throw new HTTPError("Invalid url");
+
+	const id = Snowflake.generate();
+
+	try {
+		const response = await fetch(url, DEFAULT_FETCH_OPTIONS);
+		const buffer = await response.buffer();
+
+		await storage.set(`/external/${id}`, buffer);
+
+		res.send({ id });
+	} catch (error) {
+		throw new HTTPError("Couldn't fetch website");
+	}
+});
+
+router.get("/:id", async (req: Request, res: Response) => {
+	const { id } = req.params;
+
+	const file = await storage.get(`/external/${id}`);
+	if (!file) throw new HTTPError("File not found");
+	const result = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", result?.mime);
+
+	return res.send(file);
+});
+
+export default router;
diff --git a/src/cdn/routes/ping.ts b/src/cdn/routes/ping.ts
new file mode 100644
index 00000000..38daf81e
--- /dev/null
+++ b/src/cdn/routes/ping.ts
@@ -0,0 +1,9 @@
+import { Router, Response, Request } from "express";
+
+const router = Router();
+
+router.get("/", (req: Request, res: Response) => {
+	res.send("pong");
+});
+
+export default router;
diff --git a/src/cdn/routes/role-icons.ts b/src/cdn/routes/role-icons.ts
new file mode 100644
index 00000000..2e5c42dd
--- /dev/null
+++ b/src/cdn/routes/role-icons.ts
@@ -0,0 +1,101 @@
+import { Router, Response, Request } from "express";
+import { Config, Snowflake } from "@fosscord/util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+import { HTTPError } from "@fosscord/util";
+import crypto from "crypto";
+import { multer } from "../util/multer";
+
+//Role icons ---> avatars.ts modified
+
+// TODO: check user rights and perks and animated pfp are allowed in the policies
+// TODO: generate different sizes of icon
+// TODO: generate different image types of icon
+
+const STATIC_MIME_TYPES = [
+	"image/png",
+	"image/jpeg",
+	"image/webp",
+	"image/svg+xml",
+	"image/svg",
+];
+const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES];
+
+const router = Router();
+
+router.post(
+	"/:role_id",
+	multer.single("file"),
+	async (req: Request, res: Response) => {
+		if (req.headers.signature !== Config.get().security.requestSignature)
+			throw new HTTPError("Invalid request signature");
+		if (!req.file) throw new HTTPError("Missing file");
+		const { buffer, mimetype, size, originalname, fieldname } = req.file;
+		const { role_id } = req.params;
+
+		let hash = crypto
+			.createHash("md5")
+			.update(Snowflake.generate())
+			.digest("hex");
+
+		const type = await FileType.fromBuffer(buffer);
+		if (!type || !ALLOWED_MIME_TYPES.includes(type.mime))
+			throw new HTTPError("Invalid file type");
+
+		const path = `role-icons/${role_id}/${hash}.png`;
+		const endpoint =
+			Config.get().cdn.endpointPublic || "http://localhost:3003";
+
+		await storage.set(path, buffer);
+
+		return res.json({
+			id: hash,
+			content_type: type.mime,
+			size,
+			url: `${endpoint}${req.baseUrl}/${role_id}/${hash}`,
+		});
+	}
+);
+
+router.get("/:role_id", async (req: Request, res: Response) => {
+	let { role_id } = req.params;
+	//role_id = role_id.split(".")[0]; // remove .file extension
+	const path = `role-icons/${role_id}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000, must-revalidate");
+
+	return res.send(file);
+});
+
+router.get("/:role_id/:hash", async (req: Request, res: Response) => {
+	let { role_id, hash } = req.params;
+	//hash = hash.split(".")[0]; // remove .file extension
+	const path = `role-icons/${role_id}/${hash}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000, must-revalidate");
+
+	return res.send(file);
+});
+
+router.delete("/:role_id/:id", async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	const { role_id, id } = req.params;
+	const path = `role-icons/${role_id}/${id}`;
+
+	await storage.delete(path);
+
+	return res.send({ success: true });
+});
+
+export default router;
diff --git a/src/cdn/start.ts b/src/cdn/start.ts
new file mode 100644
index 00000000..71681b40
--- /dev/null
+++ b/src/cdn/start.ts
@@ -0,0 +1,13 @@
+import dotenv from "dotenv";
+dotenv.config();
+
+import { CDNServer } from "./Server";
+const server = new CDNServer({ port: Number(process.env.PORT) || 3003 });
+server
+	.start()
+	.then(() => {
+		console.log("[Server] started on :" + server.options.port);
+	})
+	.catch((e) => console.error("[Server] Error starting: ", e));
+
+module.exports = server;
diff --git a/src/cdn/util/FileStorage.ts b/src/cdn/util/FileStorage.ts
new file mode 100644
index 00000000..aee9d345
--- /dev/null
+++ b/src/cdn/util/FileStorage.ts
@@ -0,0 +1,51 @@
+import { Storage } from "./Storage";
+import fs from "fs";
+import { join, relative, dirname } from "path";
+import { Readable } from "stream";
+//import ExifTransformer = require("exif-be-gone");
+import ExifTransformer from "exif-be-gone";
+
+// TODO: split stored files into separate folders named after cloned route
+
+function getPath(path: string) {
+	// STORAGE_LOCATION has a default value in start.ts
+	const root = process.env.STORAGE_LOCATION || "../";
+	let filename = join(root, path);
+
+	if (path.indexOf("\0") !== -1 || !filename.startsWith(root))
+		throw new Error("invalid path");
+	return filename;
+}
+
+export class FileStorage implements Storage {
+	async get(path: string): Promise<Buffer | null> {
+		path = getPath(path);
+		try {
+			return fs.readFileSync(path);
+		} catch (error) {
+			try {
+				const files = fs.readdirSync(path);
+				if (!files.length) return null;
+				return fs.readFileSync(join(path, files[0]));
+			} catch (error) {
+				return null;
+			}
+		}
+	}
+
+	async set(path: string, value: any) {
+		path = getPath(path);
+		//fse.ensureDirSync(dirname(path));
+		fs.mkdirSync(dirname(path), {recursive: true});
+
+		value = Readable.from(value);
+		const cleaned_file = fs.createWriteStream(path);
+
+		return value.pipe(new ExifTransformer()).pipe(cleaned_file);
+	}
+
+	async delete(path: string) {
+		//TODO we should delete the parent directory if empty
+		fs.unlinkSync(getPath(path));
+	}
+}
diff --git a/src/cdn/util/S3Storage.ts b/src/cdn/util/S3Storage.ts
new file mode 100644
index 00000000..c4066817
--- /dev/null
+++ b/src/cdn/util/S3Storage.ts
@@ -0,0 +1,60 @@
+import { S3 } from "@aws-sdk/client-s3";
+import { Readable } from "stream";
+import { Storage } from "./Storage";
+
+const readableToBuffer = (readable: Readable): Promise<Buffer> =>
+	new Promise((resolve, reject) => {
+		const chunks: Buffer[] = [];
+		readable.on("data", (chunk) => chunks.push(chunk));
+		readable.on("error", reject);
+		readable.on("end", () => resolve(Buffer.concat(chunks)));
+	});
+
+export class S3Storage implements Storage {
+	public constructor(
+		private client: S3,
+		private bucket: string,
+		private basePath?: string
+	) {}
+
+	/**
+	 * Always return a string, to ensure consistency.
+	 */
+	get bucketBasePath() {
+		return this.basePath ?? "";
+	}
+
+	async set(path: string, data: Buffer): Promise<void> {
+		await this.client.putObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`,
+			Body: data,
+		});
+	}
+
+	async get(path: string): Promise<Buffer | null> {
+		try {
+			const s3Object = await this.client.getObject({
+				Bucket: this.bucket,
+				Key: `${this.bucketBasePath ?? ""}${path}`,
+			});
+
+			if (!s3Object.Body) return null;
+
+			const body = s3Object.Body;
+
+			return await readableToBuffer(<Readable>body);
+		} catch (err) {
+			console.error(`[CDN] Unable to get S3 object at path ${path}.`);
+			console.error(err);
+			return null;
+		}
+	}
+
+	async delete(path: string): Promise<void> {
+		await this.client.deleteObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`,
+		});
+	}
+}
diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts
new file mode 100644
index 00000000..728804a0
--- /dev/null
+++ b/src/cdn/util/Storage.ts
@@ -0,0 +1,64 @@
+import { FileStorage } from "./FileStorage";
+import path from "path";
+//import fse from "fs-extra";
+import fs from "fs";
+import { bgCyan, black } from "picocolors";
+import { S3 } from "@aws-sdk/client-s3";
+import { S3Storage } from "./S3Storage";
+process.cwd();
+
+export interface Storage {
+	set(path: string, data: Buffer): Promise<void>;
+	get(path: string): Promise<Buffer | null>;
+	delete(path: string): Promise<void>;
+}
+
+let storage: Storage;
+
+if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
+	let location = process.env.STORAGE_LOCATION;
+	if (location) {
+		location = path.resolve(location);
+	} else {
+		location = path.join(process.cwd(), "files");
+	}
+	console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`);
+	//fse.ensureDirSync(location);
+	fs.mkdirSync(location, {recursive: true});
+	process.env.STORAGE_LOCATION = location;
+
+	storage = new FileStorage();
+} else if (process.env.STORAGE_PROVIDER === "s3") {
+	const region = process.env.STORAGE_REGION,
+		bucket = process.env.STORAGE_BUCKET;
+
+	if (!region) {
+		console.error(
+			`[CDN] You must provide a region when using the S3 storage provider.`
+		);
+		process.exit(1);
+	}
+
+	if (!bucket) {
+		console.error(
+			`[CDN] You must provide a bucket when using the S3 storage provider.`
+		);
+		process.exit(1);
+	}
+
+	// in the S3 provider, this should be the root path in the bucket
+	let location = process.env.STORAGE_LOCATION;
+
+	if (!location) {
+		console.warn(
+			`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`
+		);
+		location = undefined;
+	}
+
+	const client = new S3({ region });
+
+	storage = new S3Storage(client, bucket, location);
+}
+
+export { storage };
diff --git a/src/cdn/util/index.ts b/src/cdn/util/index.ts
new file mode 100644
index 00000000..07a5c31a
--- /dev/null
+++ b/src/cdn/util/index.ts
@@ -0,0 +1,3 @@
+export * from "./FileStorage";
+export * from "./multer";
+export * from "./Storage";
diff --git a/src/cdn/util/multer.ts b/src/cdn/util/multer.ts
new file mode 100644
index 00000000..bfdf6aff
--- /dev/null
+++ b/src/cdn/util/multer.ts
@@ -0,0 +1,10 @@
+import multerConfig from "multer";
+
+export const multer = multerConfig({
+	storage: multerConfig.memoryStorage(),
+	limits: {
+		fields: 10,
+		files: 10,
+		fileSize: 1024 * 1024 * 100, // 100 mb
+	},
+});