Merge remote-tracking branch 'upstream/staging' into fix/categoryNames

4 files changed, 414 insertions, 0 deletions

diff --git a/src/api/util/handlers/Instance.ts b/src/api/util/handlers/Instance.ts
new file mode 100644
index 00000000..7c337270
--- /dev/null
+++ b/src/api/util/handlers/Instance.ts
@@ -0,0 +1,22 @@
+import { Config, Guild, Session } from "@fosscord/util";
+import { createQueryBuilder } from "typeorm";
+
+export async function initInstance() {
+	// TODO: clean up database and delete tombstone data
+	// TODO: set first user as instance administrator/or generate one if none exists and output it in the terminal
+
+	// create default guild and add it to auto join
+	// TODO: check if any current user is not part of autoJoinGuilds
+	const { autoJoin } = Config.get().guild;
+
+	if (autoJoin.enabled && !autoJoin.guilds?.length) {
+		let guild = await Guild.findOne({where: {}, order: {id: "ASC"}});
+		if (guild) {
+			// @ts-ignore
+			await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } });
+		}
+	}
+
+	// TODO: do no clear sessions for instance cluster
+	await Session.delete({});
+}
diff --git a/src/api/util/handlers/Message.ts b/src/api/util/handlers/Message.ts
new file mode 100644
index 00000000..ff5ece75
--- /dev/null
+++ b/src/api/util/handlers/Message.ts
@@ -0,0 +1,229 @@
+import {
+	Channel,
+	Embed,
+	emitEvent,
+	Guild,
+	Message,
+	MessageCreateEvent,
+	MessageUpdateEvent,
+	getPermission,
+	getRights,
+	CHANNEL_MENTION,
+	Snowflake,
+	USER_MENTION,
+	ROLE_MENTION,
+	Role,
+	EVERYONE_MENTION,
+	HERE_MENTION,
+	MessageType,
+	User,
+	Application,
+	Webhook,
+	Attachment,
+	Config,
+	MessageCreateSchema,
+} from "@fosscord/util";
+import { HTTPError } from "@fosscord/util";
+import fetch from "node-fetch";
+import cheerio from "cheerio";
+import { OrmUtils } from "@fosscord/util";
+
+const allow_empty = false;
+// TODO: check webhook, application, system author, stickers
+// TODO: embed gifs/videos/images
+
+const LINK_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/g;
+
+const DEFAULT_FETCH_OPTIONS: any = {
+	redirect: "follow",
+	follow: 1,
+	headers: {
+		"user-agent": "Mozilla/5.0 (compatible; Fosscord/1.0; +https://github.com/fosscord/fosscord)"
+	},
+	// size: 1024 * 1024 * 5, 	// grabbed from config later
+	compress: true,
+	method: "GET"
+};
+
+export async function handleMessage(opts: MessageOptions): Promise<Message> {
+	const channel = await Channel.findOneOrFail({ where: { id: opts.channel_id }, relations: ["recipients"] });
+	if (!channel || !opts.channel_id) throw new HTTPError("Channel not found", 404);
+
+	const message = OrmUtils.mergeDeep(new Message(), {
+		...opts,
+		sticker_items: opts.sticker_ids?.map((x) => ({ id: x })),
+		guild_id: channel.guild_id,
+		channel_id: opts.channel_id,
+		attachments: opts.attachments || [],
+		embeds: opts.embeds || [],
+		reactions: /*opts.reactions ||*/ [],
+		type: opts.type ?? 0
+	});
+
+	if (message.content && message.content.length > Config.get().limits.message.maxCharacters) {
+		throw new HTTPError("Content length over max character limit")
+	}
+
+	if (opts.author_id) {
+		message.author = await User.getPublicUser(opts.author_id);
+		const rights = await getRights(opts.author_id);
+		rights.hasThrow("SEND_MESSAGES");
+	}	
+	if (opts.application_id) {
+		message.application = await Application.findOneOrFail({ where: { id: opts.application_id } });
+	}
+	if (opts.webhook_id) {
+		message.webhook = await Webhook.findOneOrFail({ where: { id: opts.webhook_id } });
+	}
+	
+	const permission = await getPermission(opts.author_id, channel.guild_id, opts.channel_id);
+	permission.hasThrow("SEND_MESSAGES");
+	if (permission.cache.member) {
+		message.member = permission.cache.member;
+	}
+
+	if (opts.tts) permission.hasThrow("SEND_TTS_MESSAGES");
+	if (opts.message_reference) {
+		permission.hasThrow("READ_MESSAGE_HISTORY");
+		// code below has to be redone when we add custom message routing
+		if (message.guild_id !== null) {
+			const guild = await Guild.findOneOrFail({ where: { id: channel.guild_id } });
+			if (!guild.features.includes("CROSS_CHANNEL_REPLIES")) {
+				if (opts.message_reference.guild_id !== channel.guild_id) throw new HTTPError("You can only reference messages from this guild");
+				if (opts.message_reference.channel_id !== opts.channel_id) throw new HTTPError("You can only reference messages from this channel");
+			}
+		}
+		/** Q: should be checked if the referenced message exists? ANSWER: NO
+		 otherwise backfilling won't work **/
+		// @ts-ignore
+		message.type = MessageType.REPLY;
+	}
+
+	// TODO: stickers/activity
+	if (!allow_empty && (!opts.content && !opts.embeds?.length && !opts.attachments?.length && !opts.sticker_ids?.length)) {
+		throw new HTTPError("Empty messages are not allowed", 50006);
+	}
+
+	let content = opts.content;
+	let mention_channel_ids = [] as string[];
+	let mention_role_ids = [] as string[];
+	let mention_user_ids = [] as string[];
+	let mention_everyone = false;
+
+	if (content) { // TODO: explicit-only mentions
+		message.content = content.trim();
+		for (const [_, mention] of content.matchAll(CHANNEL_MENTION)) {
+			if (!mention_channel_ids.includes(mention)) mention_channel_ids.push(mention);
+		}
+
+		for (const [_, mention] of content.matchAll(USER_MENTION)) {
+			if (!mention_user_ids.includes(mention)) mention_user_ids.push(mention);
+		}
+
+		await Promise.all(
+			Array.from(content.matchAll(ROLE_MENTION)).map(async ([_, mention]) => {
+				const role = await Role.findOneOrFail({ where: { id: mention, guild_id: channel.guild_id } });
+				if (role.mentionable || permission.has("MANAGE_ROLES")) {
+					mention_role_ids.push(mention);
+				}
+			})
+		);
+
+		if (permission.has("MENTION_EVERYONE")) {
+			mention_everyone = !!content.match(EVERYONE_MENTION) || !!content.match(HERE_MENTION);
+		}
+	}
+
+	message.mention_channels = mention_channel_ids.map((x) => OrmUtils.mergeDeep(new Channel(), { id: x }));
+	message.mention_roles = mention_role_ids.map((x) => OrmUtils.mergeDeep(new Role(), { id: x }));
+	message.mentions = mention_user_ids.map((x) => OrmUtils.mergeDeep(new User(), { id: x }));
+	message.mention_everyone = mention_everyone;
+
+	// TODO: check and put it all in the body
+
+	return message;
+}
+
+// TODO: cache link result in db
+export async function postHandleMessage(message: Message) {
+	let links = message.content?.match(LINK_REGEX);
+	if (!links) return;
+
+	const data = { ...message };
+	data.embeds = data.embeds.filter((x) => x.type !== "link");
+
+	links = links.slice(0, 20); // embed max 20 links — TODO: make this configurable with instance policies
+
+	for (const link of links) {
+		try {
+			const request = await fetch(link, {
+				...DEFAULT_FETCH_OPTIONS,
+				size: Config.get().limits.message.maxEmbedDownloadSize,
+			});
+
+			const text = await request.text();
+			const $ = cheerio.load(text);
+
+			const title = $('meta[property="og:title"]').attr("content");
+			const provider_name = $('meta[property="og:site_name"]').text();
+			const author_name = $('meta[property="article:author"]').attr("content");
+			const description = $('meta[property="og:description"]').attr("content") || $('meta[property="description"]').attr("content");
+			const image = $('meta[property="og:image"]').attr("content");
+			const url = $('meta[property="og:url"]').attr("content");
+			// TODO: color
+			const embed: Embed = {
+				provider: {
+					url: link,
+					name: provider_name
+				}
+			};
+
+			if (author_name) embed.author = { name: author_name };
+			if (image) embed.thumbnail = { proxy_url: image, url: image };
+			if (title) embed.title = title;
+			if (url) embed.url = url;
+			if (description) embed.description = description;
+
+			if (title || description) {
+				data.embeds.push(embed);
+			}
+		} catch (error) {}
+	}
+
+	await Promise.all([
+		emitEvent({
+			event: "MESSAGE_UPDATE",
+			channel_id: message.channel_id,
+			data
+		} as MessageUpdateEvent),
+		Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds })
+	]);
+}
+
+export async function sendMessage(opts: MessageOptions) {
+	const message = await handleMessage({ ...opts, timestamp: new Date() });
+
+	//TODO: check this, removed toJSON call
+	await Promise.all([
+		Message.insert(message),
+		emitEvent({ event: "MESSAGE_CREATE", channel_id: opts.channel_id, data: message } as MessageCreateEvent)
+	]);
+
+	postHandleMessage(message).catch((e) => {}); // no await as it should catch error non-blockingly
+
+	return message;
+}
+
+interface MessageOptions extends MessageCreateSchema {
+	id?: string;
+	type?: MessageType;
+	pinned?: boolean;
+	author_id?: string;
+	webhook_id?: string;
+	application_id?: string;
+	embeds?: Embed[];
+	channel_id?: string;
+	attachments?: Attachment[];
+	edited_timestamp?: Date;
+	timestamp?: Date;
+}
diff --git a/src/api/util/handlers/Voice.ts b/src/api/util/handlers/Voice.ts
new file mode 100644
index 00000000..4d60eb91
--- /dev/null
+++ b/src/api/util/handlers/Voice.ts
@@ -0,0 +1,32 @@
+import { Config } from "@fosscord/util";
+import { distanceBetweenLocations, IPAnalysis } from "../utility/ipAddress";
+
+export async function getVoiceRegions(ipAddress: string, vip: boolean) {
+	const regions = Config.get().regions;
+	const availableRegions = regions.available.filter((ar) => (vip ? true : !ar.vip));
+	let optimalId = regions.default;
+
+	if (!regions.useDefaultAsOptimal) {
+		const clientIpAnalysis = await IPAnalysis(ipAddress);
+
+		let min = Number.POSITIVE_INFINITY;
+
+		for (let ar of availableRegions) {
+			//TODO the endpoint location should be saved in the database if not already present to prevent IPAnalysis call
+			const dist = distanceBetweenLocations(clientIpAnalysis, ar.location || (await IPAnalysis(ar.endpoint)));
+
+			if (dist < min) {
+				min = dist;
+				optimalId = ar.id;
+			}
+		}
+	}
+
+	return availableRegions.map((ar) => ({
+		id: ar.id,
+		name: ar.name,
+		custom: ar.custom,
+		deprecated: ar.deprecated,
+		optimal: ar.id === optimalId
+	}));
+}
diff --git a/src/api/util/handlers/route.ts b/src/api/util/handlers/route.ts
new file mode 100644
index 00000000..71e14955
--- /dev/null
+++ b/src/api/util/handlers/route.ts
@@ -0,0 +1,131 @@
+import {
+	DiscordApiErrors,
+	EVENT,
+	Event,
+	EventData,
+	FieldErrors,
+	FosscordApiErrors,
+	getPermission,
+	getRights,
+	PermissionResolvable,
+	Permissions,
+	RightResolvable,
+	Rights
+} from "@fosscord/util";
+import { NextFunction, Request, Response } from "express";
+import fs from "fs";
+import path from "path";
+import Ajv from "ajv";
+import { AnyValidateFunction } from "ajv/dist/core";
+import addFormats from "ajv-formats";
+
+const SchemaPath = path.join(__dirname, "..", "..", "..","..", "assets", "schemas.json");
+const schemas = JSON.parse(fs.readFileSync(SchemaPath, { encoding: "utf8" }));
+
+export const ajv = new Ajv({
+	allErrors: true,
+	parseDate: true,
+	allowDate: true,
+	schemas,
+	coerceTypes: true,
+	messages: true,
+	strict: true,
+	strictRequired: true
+});
+
+addFormats(ajv);
+
+declare global {
+	namespace Express {
+		interface Request {
+			permission?: Permissions;
+		}
+	}
+}
+
+export type RouteResponse = { status?: number; body?: `${string}Response`; headers?: Record<string, string> };
+
+export interface RouteOptions {
+	permission?: PermissionResolvable;
+	right?: RightResolvable;
+	body?: `${string}Schema`; // typescript interface name
+	test?: {
+		response?: RouteResponse;
+		body?: any;
+		path?: string;
+		event?: EVENT | EVENT[];
+		headers?: Record<string, string>;
+	};
+}
+
+// Normalizer is introduced to workaround https://github.com/ajv-validator/ajv/issues/1287
+// this removes null values as ajv doesn't treat them as undefined
+// normalizeBody allows to handle circular structures without issues
+// taken from https://github.com/serverless/serverless/blob/master/lib/classes/ConfigSchemaHandler/index.js#L30 (MIT license)
+const normalizeBody = (body: any = {}) => {
+	const normalizedObjectsSet = new WeakSet();
+	const normalizeObject = (object: any) => {
+		if (normalizedObjectsSet.has(object)) return;
+		normalizedObjectsSet.add(object);
+		if (Array.isArray(object)) {
+			for (const [index, value] of object.entries()) {
+				if (typeof value === "object") normalizeObject(value);
+			}
+		} else {
+			for (const [key, value] of Object.entries(object)) {
+				if (value == null) {
+					if (key === "icon" || key === "avatar" || key === "banner" || key === "splash" || key === "discovery_splash") continue;
+					delete object[key];
+				} else if (typeof value === "object") {
+					normalizeObject(value);
+				}
+			}
+		}
+	};
+	normalizeObject(body);
+	return body;
+};
+
+export function route(opts: RouteOptions) {
+	let validate: AnyValidateFunction<any> | undefined;
+	if (opts.body) {
+		validate = ajv.getSchema(opts.body);
+		if (!validate) throw new Error(`Body schema ${opts.body} not found`);
+	}
+
+	return async (req: Request, res: Response, next: NextFunction) => {
+		if (opts.permission) {
+			const required = new Permissions(opts.permission);
+			req.permission = await getPermission(req.user_id, req.params.guild_id, req.params.channel_id);
+
+			// bitfield comparison: check if user lacks certain permission
+			if (!req.permission.has(required)) {
+				throw DiscordApiErrors.MISSING_PERMISSIONS.withParams(opts.permission as string);
+			}
+		}
+
+		if (opts.right) {
+			const required = new Rights(opts.right);
+			req.rights = await getRights(req.user_id);
+
+			if (!req.rights || !req.rights.has(required)) {
+				throw FosscordApiErrors.MISSING_RIGHTS.withParams(opts.right as string);
+			}
+		}
+
+		if (validate) {
+			const valid = validate(normalizeBody(req.body));
+			if (!valid) {
+				const fields: Record<string, { code?: string; message: string }> = {};
+				if(process.env.LOG_INVALID_BODY) {
+					console.log(`Got invalid request: ${req.method} ${req.originalUrl}`)
+					console.log(req.body)
+					validate.errors?.forEach(x => console.log(x.params))
+				}
+				validate.errors?.forEach((x) => (fields[x.instancePath.slice(1)] = { code: x.keyword, message: x.message || "" }));
+				throw FieldErrors(fields);
+			}
+		}
+		next();
+	};
+}