From ef4d4a318176c3e572adc17427a8b8c728a618ab Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:35:04 +0200 Subject: :construction: api --- api/src/middlewares/RateLimit.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'api/src/middlewares/RateLimit.ts') diff --git a/api/src/middlewares/RateLimit.ts b/api/src/middlewares/RateLimit.ts index acf92606..9601dad3 100644 --- a/api/src/middlewares/RateLimit.ts +++ b/api/src/middlewares/RateLimit.ts @@ -72,7 +72,7 @@ export default function RateLimit(opts: { offender.expires_at = new Date(Date.now() + opts.window * 1000); offender.blocked = false; // mongodb ttl didn't update yet -> manually update/delete - db.collection("ratelimits").updateOne({ id: bucket_id, user_id }, { $set: offender }); + db.collection("ratelimits").update({ id: bucket_id, user_id }, { $set: offender }); Cache.delete(user_id + bucket_id); } } @@ -132,7 +132,7 @@ export async function initRateLimits(app: Router) { async function hitRoute(opts: { user_id: string; bucket_id: string; max_hits: number; window: number }) { const filter = { id: opts.bucket_id, user_id: opts.user_id }; - const { value } = await db.collection("ratelimits").findOneAndUpdate( + const { value } = await db.collection("ratelimits").findOneOrFailAndUpdate( filter, { $setOnInsert: { @@ -158,7 +158,7 @@ async function hitRoute(opts: { user_id: string; bucket_id: string; max_hits: nu event: EventRateLimit, data: value }); - await db.collection("ratelimits").updateOne(filter, { $set: { blocked: true } }); + await db.collection("ratelimits").update(filter, { $set: { blocked: true } }); } else { Cache.delete(opts.user_id); } -- cgit 1.5.1 From e0f2a5548ddb8db509898d4913b503c9fbfa2279 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sun, 29 Aug 2021 16:58:23 +0200 Subject: fix rate limit --- api/src/middlewares/ErrorHandler.ts | 5 +- api/src/middlewares/RateLimit.ts | 95 +++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 48 deletions(-) (limited to 'api/src/middlewares/RateLimit.ts') diff --git a/api/src/middlewares/ErrorHandler.ts b/api/src/middlewares/ErrorHandler.ts index 8e2cd923..0ed37bb4 100644 --- a/api/src/middlewares/ErrorHandler.ts +++ b/api/src/middlewares/ErrorHandler.ts @@ -4,7 +4,7 @@ import { FieldError } from "../util/instanceOf"; // TODO: update with new body/typorm validation export function ErrorHandler(error: Error, req: Request, res: Response, next: NextFunction) { - if (!error) next(); + if (!error) return next(); try { let code = 400; @@ -18,7 +18,6 @@ export function ErrorHandler(error: Error, req: Request, res: Response, next: Ne message = error.message; errors = error.errors; } else { - console.error(error); if (req.server?.options?.production) { message = "Internal Server Error"; } @@ -27,7 +26,7 @@ export function ErrorHandler(error: Error, req: Request, res: Response, next: Ne if (httpcode > 511) httpcode = 400; - console.error(`[Error] ${code} ${req.url} ${message}`, errors || error); + console.error(`[Error] ${code} ${req.url}`, errors || error, "body:", req.body); res.status(httpcode).json({ code: code, message, errors }); } catch (error) { diff --git a/api/src/middlewares/RateLimit.ts b/api/src/middlewares/RateLimit.ts index 9601dad3..e0cf103a 100644 --- a/api/src/middlewares/RateLimit.ts +++ b/api/src/middlewares/RateLimit.ts @@ -1,6 +1,6 @@ -// @ts-nocheck -import { db, Bucket, Config, listenEvent, emitEvent } from "@fosscord/util"; +import { Config, listenEvent, emitEvent, RateLimit } from "@fosscord/util"; import { NextFunction, Request, Response, Router } from "express"; +import { LessThan } from "typeorm"; import { getIpAdress } from "../util/ipAddress"; import { API_PREFIX_TRAILING_SLASH } from "./Authentication"; @@ -18,10 +18,10 @@ TODO: different for methods (GET/POST) */ -var Cache = new Map(); -const EventRateLimit = "ratelimit"; +var Cache = new Map(); +const EventRateLimit = "RATELIMIT"; -export default function RateLimit(opts: { +export default function rateLimit(opts: { bucket?: string; window: number; count: number; @@ -36,15 +36,15 @@ export default function RateLimit(opts: { }): any { return async (req: Request, res: Response, next: NextFunction): Promise => { const bucket_id = opts.bucket || req.originalUrl.replace(API_PREFIX_TRAILING_SLASH, ""); - var user_id = getIpAdress(req); - if (!opts.onlyIp && req.user_id) user_id = req.user_id; + var executor_id = getIpAdress(req); + if (!opts.onlyIp && req.user_id) executor_id = req.user_id; var max_hits = opts.count; if (opts.bot && req.user_bot) max_hits = opts.bot; if (opts.GET && ["GET", "OPTIONS", "HEAD"].includes(req.method)) max_hits = opts.GET; else if (opts.MODIFY && ["POST", "DELETE", "PATCH", "PUT"].includes(req.method)) max_hits = opts.MODIFY; - const offender = Cache.get(user_id + bucket_id) as Bucket | null; + const offender = Cache.get(executor_id + bucket_id); if (offender && offender.blocked) { const reset = offender.expires_at.getTime(); @@ -72,12 +72,12 @@ export default function RateLimit(opts: { offender.expires_at = new Date(Date.now() + opts.window * 1000); offender.blocked = false; // mongodb ttl didn't update yet -> manually update/delete - db.collection("ratelimits").update({ id: bucket_id, user_id }, { $set: offender }); - Cache.delete(user_id + bucket_id); + RateLimit.delete({ id: bucket_id, executor_id }); + Cache.delete(executor_id + bucket_id); } } next(); - const hitRouteOpts = { bucket_id, user_id, max_hits, window: opts.window }; + const hitRouteOpts = { bucket_id, executor_id, max_hits, window: opts.window }; if (opts.error || opts.success) { res.once("finish", () => { @@ -97,69 +97,74 @@ export default function RateLimit(opts: { export async function initRateLimits(app: Router) { const { routes, global, ip, error } = Config.get().limits.rate; await listenEvent(EventRateLimit, (event) => { - Cache.set(event.channel_id, event.data); + Cache.set(event.channel_id as string, event.data); event.acknowledge?.(); }); + await RateLimit.delete({ expires_at: LessThan(new Date()) }); // clean up if not already deleted + const limits = await RateLimit.find({ blocked: true }); + limits.forEach((limit) => { + Cache.set(limit.executor_id, limit); + }); setInterval(() => { Cache.forEach((x, key) => { - if (Date.now() > x.expires_at) Cache.delete(key); + if (new Date() > x.expires_at) { + Cache.delete(key); + RateLimit.delete({ executor_id: key }); + } }); }, 1000 * 60 * 10); app.use( - RateLimit({ + rateLimit({ bucket: "global", onlyIp: true, ...ip }) ); - app.use(RateLimit({ bucket: "global", ...global })); + app.use(rateLimit({ bucket: "global", ...global })); app.use( - RateLimit({ + rateLimit({ bucket: "error", error: true, onlyIp: true, ...error }) ); - app.use("/guilds/:id", RateLimit(routes.guild)); - app.use("/webhooks/:id", RateLimit(routes.webhook)); - app.use("/channels/:id", RateLimit(routes.channel)); - app.use("/auth/login", RateLimit(routes.auth.login)); - app.use("/auth/register", RateLimit({ onlyIp: true, success: true, ...routes.auth.register })); + app.use("/guilds/:id", rateLimit(routes.guild)); + app.use("/webhooks/:id", rateLimit(routes.webhook)); + app.use("/channels/:id", rateLimit(routes.channel)); + app.use("/auth/login", rateLimit(routes.auth.login)); + app.use("/auth/register", rateLimit({ onlyIp: true, success: true, ...routes.auth.register })); } -async function hitRoute(opts: { user_id: string; bucket_id: string; max_hits: number; window: number }) { - const filter = { id: opts.bucket_id, user_id: opts.user_id }; - const { value } = await db.collection("ratelimits").findOneOrFailAndUpdate( - filter, - { - $setOnInsert: { - id: opts.bucket_id, - user_id: opts.user_id, - expires_at: new Date(Date.now() + opts.window * 1000) - }, - $inc: { - hits: 1 - } - // Conditionally update blocked doesn't work - }, - { upsert: true, returnDocument: "before" } - ); - if (!value) return; - const updateBlock = !value.blocked && value.hits >= opts.max_hits; +async function hitRoute(opts: { executor_id: string; bucket_id: string; max_hits: number; window: number }) { + var ratelimit = await RateLimit.findOne({ id: opts.bucket_id, executor_id: opts.executor_id }); + if (!ratelimit) { + ratelimit = new RateLimit({ + id: opts.bucket_id, + executor_id: opts.executor_id, + expires_at: new Date(Date.now() + opts.window * 1000), + hits: 0, + blocked: false + }); + } + + ratelimit.hits++; + + const updateBlock = !ratelimit.blocked && ratelimit.hits >= opts.max_hits; if (updateBlock) { - value.blocked = true; - Cache.set(opts.user_id + opts.bucket_id, value); + ratelimit.blocked = true; + Cache.set(opts.executor_id + opts.bucket_id, ratelimit); await emitEvent({ channel_id: EventRateLimit, event: EventRateLimit, - data: value + data: ratelimit }); - await db.collection("ratelimits").update(filter, { $set: { blocked: true } }); } else { - Cache.delete(opts.user_id); + Cache.delete(opts.executor_id); } + + await ratelimit.save(); } -- cgit 1.5.1 From d0292ef96bedde2dd23962c15b2a0446b8492fb0 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sun, 29 Aug 2021 17:12:46 +0200 Subject: :bug: convert bigint -> string --- api/src/middlewares/RateLimit.ts | 4 ++-- .../routes/channels/#channel_id/messages/index.ts | 2 +- api/src/routes/guilds/index.ts | 2 +- api/src/schema/Guild.ts | 2 +- bundle/database.db | Bin 225280 -> 225280 bytes gateway/src/opcodes/Identify.ts | 2 +- gateway/src/schema/Activity.ts | 4 ++-- util/src/entities/Application.ts | 2 +- util/src/interfaces/Event.ts | 6 +++--- 9 files changed, 12 insertions(+), 12 deletions(-) (limited to 'api/src/middlewares/RateLimit.ts') diff --git a/api/src/middlewares/RateLimit.ts b/api/src/middlewares/RateLimit.ts index e0cf103a..ed6b951a 100644 --- a/api/src/middlewares/RateLimit.ts +++ b/api/src/middlewares/RateLimit.ts @@ -1,6 +1,6 @@ import { Config, listenEvent, emitEvent, RateLimit } from "@fosscord/util"; import { NextFunction, Request, Response, Router } from "express"; -import { LessThan } from "typeorm"; +import { LessThan, MoreThan } from "typeorm"; import { getIpAdress } from "../util/ipAddress"; import { API_PREFIX_TRAILING_SLASH } from "./Authentication"; @@ -100,7 +100,7 @@ export async function initRateLimits(app: Router) { Cache.set(event.channel_id as string, event.data); event.acknowledge?.(); }); - await RateLimit.delete({ expires_at: LessThan(new Date()) }); // clean up if not already deleted + await RateLimit.delete({ expires_at: MoreThan(new Date()) }); // cleans up if not already deleted, morethan -> older date const limits = await RateLimit.find({ blocked: true }); limits.forEach((limit) => { Cache.set(limit.executor_id, limit); diff --git a/api/src/routes/channels/#channel_id/messages/index.ts b/api/src/routes/channels/#channel_id/messages/index.ts index 6307c022..17944548 100644 --- a/api/src/routes/channels/#channel_id/messages/index.ts +++ b/api/src/routes/channels/#channel_id/messages/index.ts @@ -77,7 +77,7 @@ router.get("/", async (req: Request, res: Response) => { delete x.user_ids; }); // @ts-ignore - if (!x.author) x.author = { discriminator: "0000", username: "Deleted User", public_flags: 0n, avatar: null }; + if (!x.author) x.author = { discriminator: "0000", username: "Deleted User", public_flags: "0", avatar: null }; return x; }); diff --git a/api/src/routes/guilds/index.ts b/api/src/routes/guilds/index.ts index c158c7d4..1e83cf13 100644 --- a/api/src/routes/guilds/index.ts +++ b/api/src/routes/guilds/index.ts @@ -47,7 +47,7 @@ router.post("/", check(GuildCreateSchema), async (req: Request, res: Response) = premium_tier: 0, public_updates_channel_id: undefined, rules_channel_id: undefined, - system_channel_flags: 0, + system_channel_flags: "0", system_channel_id: undefined, unavailable: false, vanity_url_code: undefined, diff --git a/api/src/schema/Guild.ts b/api/src/schema/Guild.ts index 01690ae9..3e98fe76 100644 --- a/api/src/schema/Guild.ts +++ b/api/src/schema/Guild.ts @@ -33,7 +33,7 @@ export const GuildUpdateSchema = { $icon: String, $verification_level: Number, $default_message_notifications: Number, - $system_channel_flags: Number, + $system_channel_flags: String, $system_channel_id: String, $explicit_content_filter: Number, $public_updates_channel_id: String, diff --git a/bundle/database.db b/bundle/database.db index 9572c45e..2d4abd49 100644 Binary files a/bundle/database.db and b/bundle/database.db differ diff --git a/gateway/src/opcodes/Identify.ts b/gateway/src/opcodes/Identify.ts index 5be2acce..87008998 100644 --- a/gateway/src/opcodes/Identify.ts +++ b/gateway/src/opcodes/Identify.ts @@ -42,7 +42,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { } } - const members = await Member.find({ where: { id: this.user_id }, relations: ["guilds"] }); + const members = await Member.find({ where: { id: this.user_id }, relations: ["guild"] }); const merged_members = members.map((x: any) => { const y = { ...x, user_id: x.id }; delete y.settings; diff --git a/gateway/src/schema/Activity.ts b/gateway/src/schema/Activity.ts index 0fd0592f..f1665efd 100644 --- a/gateway/src/schema/Activity.ts +++ b/gateway/src/schema/Activity.ts @@ -39,7 +39,7 @@ export const ActivitySchema = { $match: String, }, $instance: Boolean, - $flags: BigInt, + $flags: String, }, ], $since: Number, // unix time (in milliseconds) of when the client went idle, or null if the client is not idle @@ -79,7 +79,7 @@ export interface ActivitySchema { match?: string; // the secret for a specific instanced match }; instance?: boolean; - flags: bigint; // activity flags OR d together, describes what the payload includes + flags: string; // activity flags OR d together, describes what the payload includes } ]; since?: number; // unix time (in milliseconds) of when the client went idle, or null if the client is not idle diff --git a/util/src/entities/Application.ts b/util/src/entities/Application.ts index a87b5cea..b179d171 100644 --- a/util/src/entities/Application.ts +++ b/util/src/entities/Application.ts @@ -62,7 +62,7 @@ export class Application extends BaseClass { cover_image?: string; // the application's default rich presence invite cover image hash @Column() - flags: number; // the application's public flags + flags: string; // the application's public flags } export interface ApplicationCommand { diff --git a/util/src/interfaces/Event.ts b/util/src/interfaces/Event.ts index 0de55f71..814a8beb 100644 --- a/util/src/interfaces/Event.ts +++ b/util/src/interfaces/Event.ts @@ -36,7 +36,7 @@ export interface ReadyEventData { mobile: boolean; desktop: boolean; email: string | undefined; - flags: bigint; + flags: string; mfa_enabled: boolean; nsfw_allowed: boolean; phone: string | undefined; @@ -85,7 +85,7 @@ export interface ReadyEventData { }; application?: { id: string; - flags: bigint; + flags: string; }; merged_members?: Omit[][]; // probably all users who the user is in contact with @@ -95,7 +95,7 @@ export interface ReadyEventData { id: string; username: string; bot: boolean; - public_flags: bigint; + public_flags: string; }[]; } -- cgit 1.5.1 From 954700b2d5c4090fdf9d7d25beef3d6529afa8d3 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Mon, 30 Aug 2021 12:14:32 +0200 Subject: :zap: only local rate limit to prevent to much pressure on the database --- api/src/middlewares/RateLimit.ts | 70 +++++++++++++++++++++++++++++----------- util/src/entities/RateLimit.ts | 3 +- 2 files changed, 52 insertions(+), 21 deletions(-) (limited to 'api/src/middlewares/RateLimit.ts') diff --git a/api/src/middlewares/RateLimit.ts b/api/src/middlewares/RateLimit.ts index ed6b951a..dffbc0d9 100644 --- a/api/src/middlewares/RateLimit.ts +++ b/api/src/middlewares/RateLimit.ts @@ -1,11 +1,12 @@ -import { Config, listenEvent, emitEvent, RateLimit } from "@fosscord/util"; +import { Config, listenEvent } from "@fosscord/util"; import { NextFunction, Request, Response, Router } from "express"; -import { LessThan, MoreThan } from "typeorm"; import { getIpAdress } from "../util/ipAddress"; import { API_PREFIX_TRAILING_SLASH } from "./Authentication"; // Docs: https://discord.com/developers/docs/topics/rate-limits +// TODO: use better caching (e.g. redis) as else it creates to much pressure on the database + /* ? bucket limit? Max actions/sec per bucket? @@ -18,6 +19,14 @@ TODO: different for methods (GET/POST) */ +type RateLimit = { + id: "global" | "error" | string; + executor_id: string; + hits: number; + blocked: boolean; + expires_at: Date; +}; + var Cache = new Map(); const EventRateLimit = "RATELIMIT"; @@ -46,13 +55,22 @@ export default function rateLimit(opts: { const offender = Cache.get(executor_id + bucket_id); - if (offender && offender.blocked) { + if (offender) { const reset = offender.expires_at.getTime(); const resetAfterMs = reset - Date.now(); const resetAfterSec = resetAfterMs / 1000; - const global = bucket_id === "global"; - if (resetAfterMs > 0) { + if (resetAfterMs <= 0) { + offender.hits = 0; + offender.expires_at = new Date(Date.now() + opts.window * 1000); + offender.blocked = false; + + Cache.delete(executor_id + bucket_id); + } + + if (offender.blocked) { + const global = bucket_id === "global"; + console.log("blocked bucket: " + bucket_id, { resetAfterMs }); return ( res @@ -67,15 +85,9 @@ export default function rateLimit(opts: { // TODO: error rate limit message translation .send({ message: "You are being rate limited.", retry_after: resetAfterSec, global }) ); - } else { - offender.hits = 0; - offender.expires_at = new Date(Date.now() + opts.window * 1000); - offender.blocked = false; - // mongodb ttl didn't update yet -> manually update/delete - RateLimit.delete({ id: bucket_id, executor_id }); - Cache.delete(executor_id + bucket_id); } } + next(); const hitRouteOpts = { bucket_id, executor_id, max_hits, window: opts.window }; @@ -100,20 +112,20 @@ export async function initRateLimits(app: Router) { Cache.set(event.channel_id as string, event.data); event.acknowledge?.(); }); - await RateLimit.delete({ expires_at: MoreThan(new Date()) }); // cleans up if not already deleted, morethan -> older date - const limits = await RateLimit.find({ blocked: true }); - limits.forEach((limit) => { - Cache.set(limit.executor_id, limit); - }); + // await RateLimit.delete({ expires_at: LessThan(new Date().toISOString()) }); // cleans up if not already deleted, morethan -> older date + // const limits = await RateLimit.find({ blocked: true }); + // limits.forEach((limit) => { + // Cache.set(limit.executor_id, limit); + // }); setInterval(() => { Cache.forEach((x, key) => { if (new Date() > x.expires_at) { Cache.delete(key); - RateLimit.delete({ executor_id: key }); + // RateLimit.delete({ executor_id: key }); } }); - }, 1000 * 60 * 10); + }, 1000 * 60); app.use( rateLimit({ @@ -139,6 +151,25 @@ export async function initRateLimits(app: Router) { } async function hitRoute(opts: { executor_id: string; bucket_id: string; max_hits: number; window: number }) { + const id = opts.executor_id + opts.bucket_id; + var limit = Cache.get(id); + if (!limit) { + limit = { + id: opts.bucket_id, + executor_id: opts.executor_id, + expires_at: new Date(Date.now() + opts.window * 1000), + hits: 0, + blocked: false + }; + Cache.set(id, limit); + } + + limit.hits++; + if (limit.hits >= opts.max_hits) { + limit.blocked = true; + } + + /* var ratelimit = await RateLimit.findOne({ id: opts.bucket_id, executor_id: opts.executor_id }); if (!ratelimit) { ratelimit = new RateLimit({ @@ -167,4 +198,5 @@ async function hitRoute(opts: { executor_id: string; bucket_id: string; max_hits } await ratelimit.save(); + */ } diff --git a/util/src/entities/RateLimit.ts b/util/src/entities/RateLimit.ts index 49af0416..fa9c32c1 100644 --- a/util/src/entities/RateLimit.ts +++ b/util/src/entities/RateLimit.ts @@ -1,6 +1,5 @@ -import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; +import { Column, Entity } from "typeorm"; import { BaseClass } from "./BaseClass"; -import { User } from "./User"; @Entity("rate_limits") export class RateLimit extends BaseClass { -- cgit 1.5.1