From e3dda743afea2e994c119f492e5e378daa66810b Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Wed, 23 Jun 2021 18:06:00 +0200 Subject: :bug: fix checkToken --- src/util/Database.ts | 1 - src/util/checkToken.ts | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/util/Database.ts b/src/util/Database.ts index e5323ed6..3a0f0157 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -2,7 +2,6 @@ import "./MongoBigInt"; import mongoose, { Collection, Connection, LeanDocument } from "mongoose"; import { ChangeStream, ChangeEvent, Long } from "mongodb"; import EventEmitter from "events"; -import { Document } from "mongoose"; const uri = process.env.MONGO_URL || "mongodb://localhost:27017/fosscord?readPreference=secondaryPreferred"; // TODO: auto throw error if findOne doesn't find anything diff --git a/src/util/checkToken.ts b/src/util/checkToken.ts index 73ffb670..4a60195b 100644 --- a/src/util/checkToken.ts +++ b/src/util/checkToken.ts @@ -9,7 +9,8 @@ export function checkToken(token: string, jwtSecret: string): Promise { const user = await UserModel.findOne({ id: decoded.id }, { "user_data.valid_tokens_since": true }).exec(); if (!user) return rej("Invalid Token"); - if (decoded.iat * 1000 < user.user_data.valid_tokens_since.getTime()) return rej("Invalid Token"); + // we need to round it to seconds as it saved as seconds in jwt iat and valid_tokens_since is stored in milliseconds + if (decoded.iat * 1000 < user.user_data.valid_tokens_since.setSeconds(0, 0)) return rej("Invalid Token"); if (user.disabled) return rej("User disabled"); if (user.deleted) return rej("User not found"); -- cgit 1.5.1 From ae1fc50c84bb8eec2e7f9fa80c6a5c056b20f5b7 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 24 Jun 2021 08:52:08 +0200 Subject: Message reply --- src/models/Message.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src') diff --git a/src/models/Message.ts b/src/models/Message.ts index d3651ee2..3a9a68f2 100644 --- a/src/models/Message.ts +++ b/src/models/Message.ts @@ -342,6 +342,15 @@ MessageSchema.virtual("mention_channels", { autopopulate: { select: { id: true, guild_id: true, type: true, name: true } }, }); + +MessageSchema.virtual("referenced_message", { + ref: "Message", + localField: "message_reference.message_id", + foreignField: "id", + justOne: true, + autopopulate: true, +}); + MessageSchema.virtual("created_at").get(function (this: MessageDocument) { return new Date(Snowflake.deconstruct(this.id).timestamp); }); @@ -358,3 +367,4 @@ MessageSchema.set("removeResponse", ["mention_channel_ids", "mention_role_ids", // @ts-ignore export const MessageModel = db.model("Message", MessageSchema, "messages"); + -- cgit 1.5.1 From fe2d62cd477301402c0f5d5405b160d1242f5380 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 24 Jun 2021 08:52:24 +0200 Subject: :sparkles: allow bot tokens --- src/util/checkToken.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/util/checkToken.ts b/src/util/checkToken.ts index 4a60195b..d890e0e1 100644 --- a/src/util/checkToken.ts +++ b/src/util/checkToken.ts @@ -4,6 +4,7 @@ import { UserModel } from "../models"; export function checkToken(token: string, jwtSecret: string): Promise { return new Promise((res, rej) => { + token = token.replace("Bot ", ""); // TODO: proper bot support jwt.verify(token, jwtSecret, JWTOptions, async (err, decoded: any) => { if (err || !decoded) return rej("Invalid Token"); -- cgit 1.5.1 From c0032bf413cede1107ceb3f316fd488b681c30ac Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sun, 27 Jun 2021 11:24:31 +0200 Subject: :sparkles: Config: ipdataApiKey + register.blockProxies --- src/util/Config.ts | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/util/Config.ts b/src/util/Config.ts index f125bd18..30d216a4 100644 --- a/src/util/Config.ts +++ b/src/util/Config.ts @@ -88,6 +88,7 @@ export interface DefaultOptions { sitekey: string | null; secret: string | null; }; + ipdataApiKey: string | null; }; login: { requireCaptcha: boolean; @@ -107,6 +108,7 @@ export interface DefaultOptions { requireInvite: boolean; allowNewRegistration: boolean; allowMultipleAccounts: boolean; + blockProxies: boolean; password: { minLength: number; minNumbers: number; @@ -176,6 +178,7 @@ export const DefaultOptions: DefaultOptions = { sitekey: null, secret: null, }, + ipdataApiKey: "eca677b284b3bac29eb72f5e496aa9047f26543605efe99ff2ce35c9", }, login: { requireCaptcha: false, @@ -196,6 +199,7 @@ export const DefaultOptions: DefaultOptions = { requireCaptcha: true, allowNewRegistration: true, allowMultipleAccounts: true, + blockProxies: true, password: { minLength: 8, minNumbers: 2, -- cgit 1.5.1 From 7efdbe4027e39c6cc2571b4eba8ba174dd177962 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sun, 27 Jun 2021 23:11:04 +0200 Subject: :bug: fix mongoose cache --- src/util/Config.ts | 2 +- src/util/Database.ts | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/util/Config.ts b/src/util/Config.ts index 30d216a4..bd8559a8 100644 --- a/src/util/Config.ts +++ b/src/util/Config.ts @@ -4,7 +4,7 @@ import db, { MongooseCache } from "./Database"; import { Snowflake } from "./Snowflake"; import crypto from "crypto"; -var Config = new MongooseCache(db.collection("config"), [], { onlyEvents: false }); +var Config = new MongooseCache(db.collection("config"), [], { onlyEvents: false, array: false }); export default { init: async function init(defaultOpts: any = DefaultOptions) { diff --git a/src/util/Database.ts b/src/util/Database.ts index 3a0f0157..1b77629f 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -52,9 +52,11 @@ export class MongooseCache extends EventEmitter { public pipeline: Array>, public opts: { onlyEvents: boolean; + array?: boolean; } ) { super(); + if (this.opts.array == null) this.opts.array = true; } init = async () => { @@ -67,7 +69,8 @@ export class MongooseCache extends EventEmitter { if (!this.opts.onlyEvents) { const arr = await this.collection.aggregate(this.pipeline).toArray(); - this.data = arr.length ? arr[0] : arr; + if (this.opts.array) this.data = arr || []; + else this.data = arr?.[0]; } }; @@ -90,23 +93,34 @@ export class MongooseCache extends EventEmitter { change = (doc: ChangeEvent) => { try { - // @ts-ignore - if (doc.fullDocument) { - // @ts-ignore - if (!this.opts.onlyEvents) this.data = doc.fullDocument; - } - switch (doc.operationType) { case "dropDatabase": return this.destroy(); case "drop": return this.destroy(); case "delete": + if (!this.opts.onlyEvents) { + if (this.opts.array) { + this.data = this.data.filter((x: any) => doc.documentKey?._id?.equals(x._id)); + } else this.data = null; + } return this.emit("delete", doc.documentKey._id.toHexString()); case "insert": + if (!this.opts.onlyEvents) { + if (this.opts.array) this.data.push(doc.fullDocument); + else this.data = doc.fullDocument; + } return this.emit("insert", doc.fullDocument); case "update": case "replace": + if (!this.opts.onlyEvents) { + if (this.opts.array) { + const i = this.data.findIndex((x: any) => doc.fullDocument?._id?.equals(x._id)); + if (i == -1) this.data.push(doc.fullDocument); + else this.data[i] = doc.fullDocument; + } else this.data = doc.fullDocument; + } + return this.emit("change", doc.fullDocument); case "invalidate": return this.destroy(); @@ -119,6 +133,7 @@ export class MongooseCache extends EventEmitter { }; destroy = () => { + this.data = null; this.stream?.off("change", this.change); this.emit("close"); -- cgit 1.5.1 From 229e741bfdd4a6d8b7b2617eec5e88c626d29e00 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Mon, 28 Jun 2021 18:43:22 +0200 Subject: :sparkles: Rate Limit model --- src/models/RateLimit.ts | 26 ++++++++++++++++++++++++++ src/util/Database.ts | 28 +++++++++++++++++----------- 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/models/RateLimit.ts (limited to 'src') diff --git a/src/models/RateLimit.ts b/src/models/RateLimit.ts new file mode 100644 index 00000000..7e4ca3cc --- /dev/null +++ b/src/models/RateLimit.ts @@ -0,0 +1,26 @@ +import { Schema, Document, Types } from "mongoose"; +import db from "../util/Database"; +import { ChannelModel } from "./Channel"; +import { UserModel } from "./User"; +import { GuildModel } from "./Guild"; + +export interface Bucket { + id: "global" | string; // channel_239842397 | guild_238927349823 | webhook_238923423498 + user: string; + hits: number; + blocked: boolean; +} + +export interface BucketDocument extends Bucket, Document { + id: string; +} + +export const BucketSchema = new Schema({ + id: { type: String, required: true }, + user_id: { type: String, required: true }, // bot, user, oauth_application, webhook + hits: { type: Number, required: true }, // Number of times the user hit this bucket + blocked: { type: Boolean, required: true }, +}); + +// @ts-ignore +export const BucketModel = db.model("Bucket", BucketSchema, "ratelimits"); diff --git a/src/util/Database.ts b/src/util/Database.ts index 1b77629f..0732cb4e 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -46,6 +46,7 @@ export interface MongooseCache { export class MongooseCache extends EventEmitter { public stream: ChangeStream; public data: any; + public initalizing?: Promise; constructor( public collection: Collection, @@ -59,19 +60,24 @@ export class MongooseCache extends EventEmitter { if (this.opts.array == null) this.opts.array = true; } - init = async () => { - // @ts-ignore - this.stream = this.collection.watch(this.pipeline, { fullDocument: "updateLookup" }); + init = () => { + if (this.initalizing) return this.initalizing; + this.initalizing = new Promise(async (resolve, reject) => { + // @ts-ignore + this.stream = this.collection.watch(this.pipeline, { fullDocument: "updateLookup" }); - this.stream.on("change", this.change); - this.stream.on("close", this.destroy); - this.stream.on("error", console.error); + this.stream.on("change", this.change); + this.stream.on("close", this.destroy); + this.stream.on("error", console.error); - if (!this.opts.onlyEvents) { - const arr = await this.collection.aggregate(this.pipeline).toArray(); - if (this.opts.array) this.data = arr || []; - else this.data = arr?.[0]; - } + if (!this.opts.onlyEvents) { + const arr = await this.collection.aggregate(this.pipeline).toArray(); + if (this.opts.array) this.data = arr || []; + else this.data = arr?.[0]; + } + resolve(); + }); + return this.initalizing; }; changeStream = (pipeline: any) => { -- cgit 1.5.1 From e13bdc31272f382773a65508ab7bb5626d1c535e Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Wed, 30 Jun 2021 17:49:19 +0200 Subject: :sparkles: RateLimit Bucket --- src/models/RateLimit.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/models/RateLimit.ts b/src/models/RateLimit.ts index 7e4ca3cc..6ccfafe0 100644 --- a/src/models/RateLimit.ts +++ b/src/models/RateLimit.ts @@ -9,6 +9,7 @@ export interface Bucket { user: string; hits: number; blocked: boolean; + created_at: Date; } export interface BucketDocument extends Bucket, Document { @@ -20,6 +21,7 @@ export const BucketSchema = new Schema({ user_id: { type: String, required: true }, // bot, user, oauth_application, webhook hits: { type: Number, required: true }, // Number of times the user hit this bucket blocked: { type: Boolean, required: true }, + created_at: { type: Date, required: true }, }); // @ts-ignore -- cgit 1.5.1 From c187c4b7ac919f0f79afe965bf776bf4ee385493 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Wed, 30 Jun 2021 21:56:25 +0200 Subject: :sparkles: checkToken return user data --- src/models/index.ts | 1 + src/util/checkToken.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/models/index.ts b/src/models/index.ts index 004095db..4cc6ec2b 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -35,3 +35,4 @@ export * from "./Status"; export * from "./Role"; export * from "./User"; export * from "./VoiceState"; +export * from "./RateLimit"; diff --git a/src/util/checkToken.ts b/src/util/checkToken.ts index d890e0e1..e021a406 100644 --- a/src/util/checkToken.ts +++ b/src/util/checkToken.ts @@ -8,14 +8,17 @@ export function checkToken(token: string, jwtSecret: string): Promise { jwt.verify(token, jwtSecret, JWTOptions, async (err, decoded: any) => { if (err || !decoded) return rej("Invalid Token"); - const user = await UserModel.findOne({ id: decoded.id }, { "user_data.valid_tokens_since": true }).exec(); + const user = await UserModel.findOne( + { id: decoded.id }, + { "user_data.valid_tokens_since": true, bot: true } + ).exec(); if (!user) return rej("Invalid Token"); // we need to round it to seconds as it saved as seconds in jwt iat and valid_tokens_since is stored in milliseconds if (decoded.iat * 1000 < user.user_data.valid_tokens_since.setSeconds(0, 0)) return rej("Invalid Token"); if (user.disabled) return rej("User disabled"); if (user.deleted) return rej("User not found"); - return res(decoded); + return res({ decoded, user }); }); }); } -- cgit 1.5.1 From 3a8c7ec1c338d52b50a63d4e783820d4d5c55917 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 1 Jul 2021 11:13:12 +0200 Subject: :sparkles: Rate Limit Bucket --- src/models/RateLimit.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/models/RateLimit.ts b/src/models/RateLimit.ts index 6ccfafe0..6a0e1ffd 100644 --- a/src/models/RateLimit.ts +++ b/src/models/RateLimit.ts @@ -1,15 +1,12 @@ import { Schema, Document, Types } from "mongoose"; import db from "../util/Database"; -import { ChannelModel } from "./Channel"; -import { UserModel } from "./User"; -import { GuildModel } from "./Guild"; export interface Bucket { - id: "global" | string; // channel_239842397 | guild_238927349823 | webhook_238923423498 - user: string; + id: "global" | "error" | string; // channel_239842397 | guild_238927349823 | webhook_238923423498 + user_id: string; hits: number; blocked: boolean; - created_at: Date; + expires_at: Date; } export interface BucketDocument extends Bucket, Document { @@ -21,7 +18,7 @@ export const BucketSchema = new Schema({ user_id: { type: String, required: true }, // bot, user, oauth_application, webhook hits: { type: Number, required: true }, // Number of times the user hit this bucket blocked: { type: Boolean, required: true }, - created_at: { type: Date, required: true }, + expires_at: { type: Date, required: true }, }); // @ts-ignore -- cgit 1.5.1 From 137d719ecfad4529169d96d5f45cac9f263edbe8 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Wed, 7 Jul 2021 15:21:30 +0200 Subject: :bug: fix Config --- src/util/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/util/Config.ts b/src/util/Config.ts index bd8559a8..dfa942e7 100644 --- a/src/util/Config.ts +++ b/src/util/Config.ts @@ -9,7 +9,7 @@ var Config = new MongooseCache(db.collection("config"), [], { onlyEvents: false, export default { init: async function init(defaultOpts: any = DefaultOptions) { await Config.init(); - return this.set(Config.data.merge(defaultOpts)); + return this.set((Config.data || {}).merge(defaultOpts)); }, get: function get() { return Config.data; -- cgit 1.5.1 From cae178eefdd024414a3ba90b43e9d95fbc85ef3c Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 8 Jul 2021 16:44:14 +0200 Subject: :see_no_evil: :lock: hide db password in log --- src/util/Database.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/util/Database.ts b/src/util/Database.ts index 0732cb4e..ed596907 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -3,9 +3,10 @@ import mongoose, { Collection, Connection, LeanDocument } from "mongoose"; import { ChangeStream, ChangeEvent, Long } from "mongodb"; import EventEmitter from "events"; const uri = process.env.MONGO_URL || "mongodb://localhost:27017/fosscord?readPreference=secondaryPreferred"; +import { URL } from "url"; // TODO: auto throw error if findOne doesn't find anything -console.log(`[DB] connect: ${uri}`); +const url = new URL(uri.replace("mongodb://", "http://")); const connection = mongoose.createConnection(uri, { autoIndex: true, @@ -13,6 +14,7 @@ const connection = mongoose.createConnection(uri, { useUnifiedTopology: true, useFindAndModify: false, }); +console.log(`[DB] connect: mongodb://${url.username}@${url.host}${url.pathname}${url.search}`); export default connection; -- cgit 1.5.1 From 4c75b5cc7db95ff162fdd15ac02a41c76020a891 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sat, 10 Jul 2021 17:51:19 +0200 Subject: auto throw error if findOne doesn't find any doc --- src/models/index.ts | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/util/Database.ts | 1 - 2 files changed, 42 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/models/index.ts b/src/models/index.ts index 4cc6ec2b..61c8d85a 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,7 +1,42 @@ -import mongoose from "mongoose"; -import { Schema } from "mongoose"; +import mongoose, { Schema, Document } from "mongoose"; import mongooseAutoPopulate from "mongoose-autopopulate"; +type UpdateWithAggregationPipeline = UpdateAggregationStage[]; +type UpdateAggregationStage = + | { $addFields: any } + | { $set: any } + | { $project: any } + | { $unset: any } + | { $replaceRoot: any } + | { $replaceWith: any }; +type EnforceDocument = T extends Document ? T : T & Document & TMethods; + +declare module "mongoose" { + interface Model { + // removed null -> always return document -> throw error if it doesn't exist + findOne( + filter?: FilterQuery, + projection?: any | null, + options?: QueryOptions | null, + callback?: (err: CallbackError, doc: EnforceDocument) => void + ): QueryWithHelpers, EnforceDocument, TQueryHelpers>; + findOneAndUpdate( + filter?: FilterQuery, + update?: UpdateQuery | UpdateWithAggregationPipeline, + options?: QueryOptions | null, + callback?: (err: any, doc: EnforceDocument | null, res: any) => void + ): QueryWithHelpers, EnforceDocument, TQueryHelpers>; + } +} + +var HTTPError: any; + +try { + HTTPError = require("lambert-server").HTTPError; +} catch (e) { + HTTPError = Error; +} + mongoose.plugin(mongooseAutoPopulate); mongoose.plugin((schema: Schema, opts: any) => { @@ -17,6 +52,11 @@ mongoose.plugin((schema: Schema, opts: any) => { }); }, }); + schema.post("findOne", (doc, next) => { + if (!doc) return next(new HTTPError("Not found", 404)); + // @ts-ignore + return next(); + }); }); export * from "./Activity"; diff --git a/src/util/Database.ts b/src/util/Database.ts index ed596907..16e07d3b 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -5,7 +5,6 @@ import EventEmitter from "events"; const uri = process.env.MONGO_URL || "mongodb://localhost:27017/fosscord?readPreference=secondaryPreferred"; import { URL } from "url"; -// TODO: auto throw error if findOne doesn't find anything const url = new URL(uri.replace("mongodb://", "http://")); const connection = mongoose.createConnection(uri, { -- cgit 1.5.1 From 89b5f6963a7d0ecc4341d7de916a02ff9fcd6936 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sun, 18 Jul 2021 19:49:03 +0200 Subject: :loud_sound: fix log --- src/util/Database.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/util/Database.ts b/src/util/Database.ts index 16e07d3b..8c6847a8 100644 --- a/src/util/Database.ts +++ b/src/util/Database.ts @@ -13,7 +13,7 @@ const connection = mongoose.createConnection(uri, { useUnifiedTopology: true, useFindAndModify: false, }); -console.log(`[DB] connect: mongodb://${url.username}@${url.host}${url.pathname}${url.search}`); +console.log(`[Database] connect: mongodb://${url.username}@${url.host}${url.pathname}${url.search}`); export default connection; -- cgit 1.5.1 From 11ace49d8c1e75a41200bb45c0bd50c712d2e5d2 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Mon, 19 Jul 2021 22:42:07 +0200 Subject: :bug: fix findOne exists query --- src/models/index.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/models/index.ts b/src/models/index.ts index 61c8d85a..11a6fe37 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -52,10 +52,17 @@ mongoose.plugin((schema: Schema, opts: any) => { }); }, }); - schema.post("findOne", (doc, next) => { - if (!doc) return next(new HTTPError("Not found", 404)); - // @ts-ignore - return next(); + schema.post("findOne", function (doc, next) { + try { + // @ts-ignore + const isExistsQuery = JSON.stringify(this._userProvidedFields) === JSON.stringify({ _id: 1 }); + if (!doc && !isExistsQuery) return next(new HTTPError("Not found", 404)); + // @ts-ignore + return next(); + } catch (error) { + // @ts-ignore + next(); + } }); }); -- cgit 1.5.1