From efc522a0c6a08e60de5efce0e076d61445fd9627 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:33:54 +0200 Subject: :construction: rate limit --- src/Server.ts | 8 +++- src/middlewares/Authentication.ts | 10 ++++- src/middlewares/RateLimit.ts | 61 ++++++++++++++++++++++++++++++- src/routes/guilds/#guild_id/widget.png.ts | 12 +++--- src/util/ipAddress.ts | 15 ++------ 5 files changed, 84 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/Server.ts b/src/Server.ts index c1e66bfd..54c8db0d 100644 --- a/src/Server.ts +++ b/src/Server.ts @@ -13,6 +13,7 @@ import express, { Router, Request, Response } from "express"; import fetch, { Response as FetchResponse } from "node-fetch"; import mongoose from "mongoose"; import path from "path"; +import RateLimit from "./middlewares/RateLimit"; // this will return the new updated document for findOneAndUpdate mongoose.set("returnOriginal", false); // https://mongoosejs.com/docs/api/model.html#model_Model.findOneAndUpdate @@ -54,7 +55,8 @@ export class FosscordServer extends Server { db.collection("roles").createIndex({ id: 1 }, { unique: true }), db.collection("emojis").createIndex({ id: 1 }, { unique: true }), db.collection("invites").createIndex({ code: 1 }, { unique: true }), - db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }) // after 0 seconds of expires_at the invite will get delete + db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }), // after 0 seconds of expires_at the invite will get delete + db.collection("ratelimits").createIndex({ created_at: 1 }, { expireAfterSeconds: 1000 }) ]); } @@ -67,6 +69,7 @@ export class FosscordServer extends Server { this.app.use(CORS); this.app.use(Authentication); + this.app.use(RateLimit({ count: 10, error: 10, window: 5 })); this.app.use(BodyParser({ inflate: true, limit: 1024 * 1024 * 2 })); const languages = await fs.readdir(path.join(__dirname, "..", "locales")); const namespaces = await fs.readdir(path.join(__dirname, "..", "locales", "en")); @@ -91,6 +94,9 @@ export class FosscordServer extends Server { const prefix = Router(); // @ts-ignore this.app = prefix; + prefix.use("/guilds/:id", RateLimit({ count: 10, window: 5 })); + prefix.use("/webhooks/:id", RateLimit({ count: 10, window: 5 })); + prefix.use("/channels/:id", RateLimit({ count: 10, window: 5 })); this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/")); app.use("/api", prefix); // allow unversioned requests diff --git a/src/middlewares/Authentication.ts b/src/middlewares/Authentication.ts index 4b38f1d4..76b335ad 100644 --- a/src/middlewares/Authentication.ts +++ b/src/middlewares/Authentication.ts @@ -11,10 +11,14 @@ export const NO_AUTHORIZATION_ROUTES = [ /^\/api(\/v\d+)?\/guilds\/\d+\/widget\.(json|png)/ ]; +export const API_PREFIX = /^\/api(\/v\d+)?/; +export const API_PREFIX_TRAILING_SLASH = /^\/api(\/v\d+)?\//; + declare global { namespace Express { interface Request { user_id: any; + user_bot: boolean; token: any; } } @@ -23,17 +27,19 @@ declare global { export async function Authentication(req: Request, res: Response, next: NextFunction) { if (req.method === "OPTIONS") return res.sendStatus(204); if (!req.url.startsWith("/api")) return next(); - if (req.url.startsWith("/api/v8/invites") && req.method === "GET") return next(); + const apiPath = req.url.replace(API_PREFIX, ""); + if (apiPath.startsWith("/invites") && req.method === "GET") return next(); if (NO_AUTHORIZATION_ROUTES.some((x) => x.test(req.url))) return next(); if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401)); try { const { jwtSecret } = Config.get().security; - const decoded: any = await checkToken(req.headers.authorization, jwtSecret); + const { decoded, user }: any = await checkToken(req.headers.authorization, jwtSecret); req.token = decoded; req.user_id = decoded.id; + req.user_bot = user.bot; return next(); } catch (error) { return next(new HTTPError(error.toString(), 400)); diff --git a/src/middlewares/RateLimit.ts b/src/middlewares/RateLimit.ts index e610d55b..ab69113e 100644 --- a/src/middlewares/RateLimit.ts +++ b/src/middlewares/RateLimit.ts @@ -1,5 +1,6 @@ -import { db, MongooseCache } from "@fosscord/server-util"; +import { db, MongooseCache, Bucket } from "@fosscord/server-util"; import { NextFunction, Request, Response } from "express"; +import { API_PREFIX, API_PREFIX_TRAILING_SLASH } from "./Authentication"; const Cache = new MongooseCache(db.collection("ratelimits"), [{ $match: { blocked: true } }], { onlyEvents: false, array: true }); @@ -22,10 +23,66 @@ TODO: use config values */ -export default function RateLimit(opts: { bucket?: string; window: number; count: number }) { +export default function RateLimit(opts: { + bucket?: string; + window: number; + count: number; + bot?: number; + error?: number; + webhook?: number; + oauth?: number; + GET?: number; + MODIFY?: number; +}) { Cache.init(); // will only initalize it once return async (req: Request, res: Response, next: NextFunction) => { + const bucket_id = req.path.replace(API_PREFIX_TRAILING_SLASH, ""); + const user_id = req.user_id; + const max_hits = req.user_bot ? opts.bot : opts.count; + const offender = Cache.data.find((x: Bucket) => x.user && x.id === bucket_id) as Bucket | null; + + if (offender && offender.blocked) { + const reset = offender.created_at.getTime() + opts.window; + const resetAfterMs = reset - Date.now(); + const resetAfterSec = resetAfterMs / 1000; + const global = bucket_id === "global"; + + return ( + res + .status(429) + .set("X-RateLimit-Limit", `${max_hits}`) + .set("X-RateLimit-Remaining", "0") + .set("X-RateLimit-Reset", `${reset}`) + .set("X-RateLimit-Reset-After", `${resetAfterSec}`) + .set("X-RateLimit-Global", `${global}`) + .set("Retry-After", `${Math.ceil(resetAfterSec)}`) + .set("X-RateLimit-Bucket", `${bucket_id}`) + // TODO: error rate limit message translation + .send({ message: "You are being rate limited.", retry_after: resetAfterSec, global }) + ); + } next(); + console.log(req.route); + + if (opts.error) { + res.once("finish", () => { + // check if error and increment error rate limit + }); + } + + db.collection("ratelimits").updateOne( + { bucket: bucket_id }, + { + $set: { + id: bucket_id, + user_id, + created_at: new Date(), + $cond: { if: { $gt: ["$hits", max_hits] }, then: true, else: false } + }, + $inc: { hits: 1 } + }, + { upsert: true } + ); }; } diff --git a/src/routes/guilds/#guild_id/widget.png.ts b/src/routes/guilds/#guild_id/widget.png.ts index ea947c5d..839a8129 100644 --- a/src/routes/guilds/#guild_id/widget.png.ts +++ b/src/routes/guilds/#guild_id/widget.png.ts @@ -1,9 +1,8 @@ import { Request, Response, Router } from "express"; import { GuildModel } from "@fosscord/server-util"; import { HTTPError } from "lambert-server"; -import { Image } from "canvas"; import fs from "fs"; -import path from "path" +import path from "path"; const router: Router = Router(); @@ -35,7 +34,7 @@ router.get("/", async (req: Request, res: Response) => { const sizeOf = require("image-size"); // TODO: Widget style templates need Fosscord branding - const source = path.join(__dirname, "..", "..", "..", "..", "assets","widget", `${style}.png`) + const source = path.join(__dirname, "..", "..", "..", "..", "assets", "widget", `${style}.png`); if (!fs.existsSync(source)) { throw new HTTPError("Widget template does not exist.", 400); } @@ -85,16 +84,17 @@ router.get("/", async (req: Request, res: Response) => { }); async function drawIcon(canvas: any, x: number, y: number, scale: number, icon: string) { - const img = new Image(); + // @ts-ignore + const img = new require("canvas").Image(); img.src = icon; - + // Do some canvas clipping magic! canvas.save(); canvas.beginPath(); const r = scale / 2; // use scale to determine radius canvas.arc(x + r, y + r, r, 0, 2 * Math.PI, false); // start circle at x, and y coords + radius to find center - + canvas.clip(); canvas.drawImage(img, x, y, scale, scale); diff --git a/src/util/ipAddress.ts b/src/util/ipAddress.ts index 74090d61..f2c8fd4d 100644 --- a/src/util/ipAddress.ts +++ b/src/util/ipAddress.ts @@ -1,13 +1,7 @@ import { Config } from "@fosscord/server-util"; import { Request } from "express"; // use ipdata package instead of simple fetch because of integrated caching -import IPData, { LookupResponse } from "ipdata"; - -var ipdata: IPData; -const cacheConfig = { - max: 1000, // max size - maxAge: 1000 * 60 * 60 * 24 // max age in ms (i.e. one day) -}; +import fetch from "node-fetch"; const exampleData = { ip: "", @@ -66,15 +60,14 @@ const exampleData = { status: 200 }; -export async function IPAnalysis(ip: string): Promise { +export async function IPAnalysis(ip: string): Promise { const { ipdataApiKey } = Config.get().security; if (!ipdataApiKey) return { ...exampleData, ip }; - if (!ipdata) ipdata = new IPData(ipdataApiKey, cacheConfig); - return await ipdata.lookup(ip); + return (await fetch(`https://api.ipdata.co/${ip}?api-key=${ipdataApiKey}`)).json(); } -export function isProxy(data: LookupResponse) { +export function isProxy(data: typeof exampleData) { if (!data || !data.asn || !data.threat) return false; if (data.asn.type !== "isp") return true; if (Object.values(data.threat).some((x) => x)) return true; -- cgit 1.5.1 From 79e1adc5ce5414eaed8b755603b2317fde22680e Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 1 Jul 2021 11:16:13 +0200 Subject: :sparkles: finished Rate Limit --- package-lock.json | 50 ++++++++++++++++---- package.json | 5 +- src/Server.ts | 4 +- src/middlewares/RateLimit.ts | 108 ++++++++++++++++++++++++++++++------------- 4 files changed, 122 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/package-lock.json b/package-lock.json index 0580a5ea..728f252d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@fosscord/server-util": "^1.3.21", + "@fosscord/server-util": "^1.3.23", "@types/jest": "^26.0.22", "@types/json-schema": "^7.0.7", "ajv": "^8.4.0", @@ -36,7 +36,8 @@ "mongoose-autopopulate": "^0.12.3", "mongoose-long": "^0.3.2", "multer": "^1.4.2", - "node-fetch": "^2.6.1" + "node-fetch": "^2.6.1", + "require_optional": "^1.0.1" }, "devDependencies": { "@types/bcrypt": "^3.0.0", @@ -549,9 +550,9 @@ } }, "node_modules/@fosscord/server-util": { - "version": "1.3.21", - "resolved": "https://registry.npmjs.org/@fosscord/server-util/-/server-util-1.3.21.tgz", - "integrity": "sha512-Mb4FqgkMeI2V+et5mpQkJqeijWNTjv79LZ6rRvgFdz+DwOw75S3A2vLMRqPX9lUB6uT9ZOMd5StQjR6YZNH9Sw==", + "version": "1.3.23", + "resolved": "https://registry.npmjs.org/@fosscord/server-util/-/server-util-1.3.23.tgz", + "integrity": "sha512-YxkuMwsJmMpCN4zGCq0LHvUuV9zlR8yTriquPqWfp5Sbj1DdFz7Qqo6wz6cRYb3WRIINouHhV60cbljmUqLIJQ==", "dependencies": { "@types/jsonwebtoken": "^8.5.0", "@types/mongoose-autopopulate": "^0.10.1", @@ -7846,6 +7847,23 @@ "node": ">=0.10.0" } }, + "node_modules/require_optional": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", + "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", + "dependencies": { + "resolve-from": "^2.0.0", + "semver": "^5.1.0" + } + }, + "node_modules/require_optional/node_modules/resolve-from": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", + "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -10635,9 +10653,9 @@ } }, "@fosscord/server-util": { - "version": "1.3.21", - "resolved": "https://registry.npmjs.org/@fosscord/server-util/-/server-util-1.3.21.tgz", - "integrity": "sha512-Mb4FqgkMeI2V+et5mpQkJqeijWNTjv79LZ6rRvgFdz+DwOw75S3A2vLMRqPX9lUB6uT9ZOMd5StQjR6YZNH9Sw==", + "version": "1.3.23", + "resolved": "https://registry.npmjs.org/@fosscord/server-util/-/server-util-1.3.23.tgz", + "integrity": "sha512-YxkuMwsJmMpCN4zGCq0LHvUuV9zlR8yTriquPqWfp5Sbj1DdFz7Qqo6wz6cRYb3WRIINouHhV60cbljmUqLIJQ==", "requires": { "@types/jsonwebtoken": "^8.5.0", "@types/mongoose-autopopulate": "^0.10.1", @@ -16816,6 +16834,22 @@ "is-finite": "^1.0.0" } }, + "require_optional": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", + "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", + "requires": { + "resolve-from": "^2.0.0", + "semver": "^5.1.0" + }, + "dependencies": { + "resolve-from": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", + "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=" + } + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/package.json b/package.json index 2d1882b4..71cfee4f 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/fosscord/fosscord-api#readme", "dependencies": { - "@fosscord/server-util": "^1.3.21", + "@fosscord/server-util": "^1.3.23", "@types/jest": "^26.0.22", "@types/json-schema": "^7.0.7", "ajv": "^8.4.0", @@ -56,7 +56,8 @@ "mongoose-autopopulate": "^0.12.3", "mongoose-long": "^0.3.2", "multer": "^1.4.2", - "node-fetch": "^2.6.1" + "node-fetch": "^2.6.1", + "require_optional": "^1.0.1" }, "devDependencies": { "@types/bcrypt": "^3.0.0", diff --git a/src/Server.ts b/src/Server.ts index 54c8db0d..326fcc5c 100644 --- a/src/Server.ts +++ b/src/Server.ts @@ -56,7 +56,7 @@ export class FosscordServer extends Server { db.collection("emojis").createIndex({ id: 1 }, { unique: true }), db.collection("invites").createIndex({ code: 1 }, { unique: true }), db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }), // after 0 seconds of expires_at the invite will get delete - db.collection("ratelimits").createIndex({ created_at: 1 }, { expireAfterSeconds: 1000 }) + db.collection("ratelimits").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }) ]); } @@ -69,7 +69,6 @@ export class FosscordServer extends Server { this.app.use(CORS); this.app.use(Authentication); - this.app.use(RateLimit({ count: 10, error: 10, window: 5 })); this.app.use(BodyParser({ inflate: true, limit: 1024 * 1024 * 2 })); const languages = await fs.readdir(path.join(__dirname, "..", "locales")); const namespaces = await fs.readdir(path.join(__dirname, "..", "locales", "en")); @@ -94,6 +93,7 @@ export class FosscordServer extends Server { const prefix = Router(); // @ts-ignore this.app = prefix; + prefix.use(RateLimit({ bucket: "global", count: 10, error: 10, window: 5, bot: 250 })); prefix.use("/guilds/:id", RateLimit({ count: 10, window: 5 })); prefix.use("/webhooks/:id", RateLimit({ count: 10, window: 5 })); prefix.use("/channels/:id", RateLimit({ count: 10, window: 5 })); diff --git a/src/middlewares/RateLimit.ts b/src/middlewares/RateLimit.ts index ab69113e..93d69236 100644 --- a/src/middlewares/RateLimit.ts +++ b/src/middlewares/RateLimit.ts @@ -1,8 +1,16 @@ import { db, MongooseCache, Bucket } from "@fosscord/server-util"; import { NextFunction, Request, Response } from "express"; -import { API_PREFIX, API_PREFIX_TRAILING_SLASH } from "./Authentication"; +import { getIpAdress } from "../util/ipAddress"; +import { API_PREFIX_TRAILING_SLASH } from "./Authentication"; -const Cache = new MongooseCache(db.collection("ratelimits"), [{ $match: { blocked: true } }], { onlyEvents: false, array: true }); +const Cache = new MongooseCache( + db.collection("ratelimits"), + [ + // TODO: uncomment $match and fix error: not receiving change events + // { $match: { blocked: true } } + ], + { onlyEvents: false, array: true } +); // Docs: https://discord.com/developers/docs/topics/rate-limits @@ -37,52 +45,86 @@ export default function RateLimit(opts: { Cache.init(); // will only initalize it once return async (req: Request, res: Response, next: NextFunction) => { - const bucket_id = req.path.replace(API_PREFIX_TRAILING_SLASH, ""); - const user_id = req.user_id; - const max_hits = req.user_bot ? opts.bot : opts.count; - const offender = Cache.data.find((x: Bucket) => x.user && x.id === bucket_id) as Bucket | null; + const bucket_id = opts.bucket || req.path.replace(API_PREFIX_TRAILING_SLASH, ""); + const user_id = req.user_id || getIpAdress(req); + var max_hits = opts.count; + if (opts.bot && req.user_bot) max_hits = opts.bot; + if (opts.GET && ["GET", "OPTIONS", "HEAD"].includes(req.method)) max_hits = opts.GET; + else if (opts.MODIFY && ["POST", "DELETE", "PATCH", "PUT"].includes(req.method)) max_hits = opts.MODIFY; + + const offender = Cache.data?.find((x: Bucket) => x.user_id == user_id && x.id === bucket_id) as Bucket | null; if (offender && offender.blocked) { - const reset = offender.created_at.getTime() + opts.window; + const reset = offender.expires_at.getTime(); const resetAfterMs = reset - Date.now(); const resetAfterSec = resetAfterMs / 1000; const global = bucket_id === "global"; + console.log("blocked", { resetAfterMs }); - return ( - res - .status(429) - .set("X-RateLimit-Limit", `${max_hits}`) - .set("X-RateLimit-Remaining", "0") - .set("X-RateLimit-Reset", `${reset}`) - .set("X-RateLimit-Reset-After", `${resetAfterSec}`) - .set("X-RateLimit-Global", `${global}`) - .set("Retry-After", `${Math.ceil(resetAfterSec)}`) - .set("X-RateLimit-Bucket", `${bucket_id}`) - // TODO: error rate limit message translation - .send({ message: "You are being rate limited.", retry_after: resetAfterSec, global }) - ); + if (resetAfterMs > 0) { + return ( + res + .status(429) + .set("X-RateLimit-Limit", `${max_hits}`) + .set("X-RateLimit-Remaining", "0") + .set("X-RateLimit-Reset", `${reset}`) + .set("X-RateLimit-Reset-After", `${resetAfterSec}`) + .set("X-RateLimit-Global", `${global}`) + .set("Retry-After", `${Math.ceil(resetAfterSec)}`) + .set("X-RateLimit-Bucket", `${bucket_id}`) + // TODO: error rate limit message translation + .send({ message: "You are being rate limited.", retry_after: resetAfterSec, global }) + ); + } else { + // mongodb ttl didn't update yet -> manually update/delete + db.collection("ratelimits").updateOne( + { id: bucket_id, user_id }, + { $set: { hits: 0, expires_at: new Date(Date.now() + opts.window * 1000), blocked: false } } + ); + } } next(); - console.log(req.route); if (opts.error) { res.once("finish", () => { // check if error and increment error rate limit + if (res.statusCode >= 400) { + // TODO: use config rate limit values + return hitRoute({ bucket_id: "error", user_id, max_hits: opts.error as number, window: opts.window }); + } }); } - db.collection("ratelimits").updateOne( - { bucket: bucket_id }, + return hitRoute({ user_id, bucket_id, max_hits, window: opts.window }); + }; +} + +function hitRoute(opts: { user_id: string; bucket_id: string; max_hits: number; window: number }) { + return db.collection("ratelimits").updateOne( + { id: opts.bucket_id, user_id: opts.user_id }, + [ { - $set: { - id: bucket_id, - user_id, - created_at: new Date(), - $cond: { if: { $gt: ["$hits", max_hits] }, then: true, else: false } - }, - $inc: { hits: 1 } + $replaceRoot: { + newRoot: { + // similar to $setOnInsert + $mergeObjects: [ + { + id: opts.bucket_id, + user_id: opts.user_id, + expires_at: new Date(Date.now() + opts.window * 1000) + }, + "$$ROOT" + ] + } + } }, - { upsert: true } - ); - }; + { + $set: { + hits: { $sum: [{ $ifNull: ["$hits", 0] }, 1] }, + blocked: { $gt: ["$hits", opts.max_hits] } + } + } + ], + { upsert: true } + ); } -- cgit 1.5.1