From 78dd8d99aaa18ad364c8525469d57a435aec2042 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Tue, 14 Mar 2023 00:06:41 +0100 Subject: Implement base logic for index page Signed-off-by: TheArcaneBrony --- .idea/.gitignore | 5 + .idea/codeStyles/Project.xml | 6 ++ .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/copyright/Fosscord.xml | 8 ++ .idea/copyright/profiles_settings.xml | 10 ++ .idea/fosscord-server.iml | 12 +++ .idea/inspectionProfiles/Project_Default.xml | 6 ++ .idea/jsLinters/eslint.xml | 6 ++ .idea/modules.xml | 8 ++ .idea/prettier.xml | 7 ++ .idea/sonarlint/issuestore/index.pb | 0 .idea/sonarlint/securityhotspotstore/index.pb | 0 .idea/vcs.xml | 6 ++ src/index/Server.ts | 125 +++++++++++++++++++++++++ src/index/routes/channels/#channel_id/index.ts | 116 +++++++++++++++++++++++ src/index/start.ts | 57 +++++++++++ src/util/entities/Application.ts | 36 +++---- src/util/entities/Channel.ts | 43 +++++---- src/util/util/Database.ts | 36 +++---- 19 files changed, 436 insertions(+), 56 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/copyright/Fosscord.xml create mode 100644 .idea/copyright/profiles_settings.xml create mode 100644 .idea/fosscord-server.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/jsLinters/eslint.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/prettier.xml create mode 100644 .idea/sonarlint/issuestore/index.pb create mode 100644 .idea/sonarlint/securityhotspotstore/index.pb create mode 100644 .idea/vcs.xml create mode 100644 src/index/Server.ts create mode 100644 src/index/routes/channels/#channel_id/index.ts create mode 100644 src/index/start.ts diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..b58b603f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 00000000..e942d55d --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 00000000..79ee123c --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/copyright/Fosscord.xml b/.idea/copyright/Fosscord.xml new file mode 100644 index 00000000..9453d207 --- /dev/null +++ b/.idea/copyright/Fosscord.xml @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 00000000..47c54d2a --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/fosscord-server.iml b/.idea/fosscord-server.iml new file mode 100644 index 00000000..0c8867d7 --- /dev/null +++ b/.idea/fosscord-server.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..03d9549e --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/jsLinters/eslint.xml b/.idea/jsLinters/eslint.xml new file mode 100644 index 00000000..541945bb --- /dev/null +++ b/.idea/jsLinters/eslint.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..043b5e1a --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/prettier.xml b/.idea/prettier.xml new file mode 100644 index 00000000..727b8b53 --- /dev/null +++ b/.idea/prettier.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/sonarlint/issuestore/index.pb b/.idea/sonarlint/issuestore/index.pb new file mode 100644 index 00000000..e69de29b diff --git a/.idea/sonarlint/securityhotspotstore/index.pb b/.idea/sonarlint/securityhotspotstore/index.pb new file mode 100644 index 00000000..e69de29b diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/index/Server.ts b/src/index/Server.ts new file mode 100644 index 00000000..0be5f0f8 --- /dev/null +++ b/src/index/Server.ts @@ -0,0 +1,125 @@ +/* + * Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + * Copyright (C) 2023 Fosscord and Fosscord Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { + Config, + initDatabase, + JSONReplacer, + registerRoutes, + Sentry, +} from "@fosscord/util"; +import { Request, Response, Router } from "express"; +import { Server, ServerOptions } from "lambert-server"; +import "missing-native-js-functions"; +import morgan from "morgan"; +import path from "path"; +import { red } from "picocolors"; +import { CORS, ErrorHandler, initRateLimits } from "@fosscord/api"; +import { initTranslation } from "../api/middlewares/Translation"; + +export type FosscordServerOptions = ServerOptions; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + server: FosscordServer; + } + } +} + +export class FosscordServer extends Server { + public declare options: FosscordServerOptions; + + constructor(opts?: Partial) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + super({ ...opts, errorHandler: false, jsonBody: false }); + } + + async start() { + await initDatabase(); + await Config.init(); + await Sentry.init(this.app); + + const logRequests = process.env["LOG_REQUESTS"] != undefined; + if (logRequests) { + this.app.use( + morgan("combined", { + skip: (req, res) => { + let skip = !( + process.env["LOG_REQUESTS"]?.includes( + res.statusCode.toString(), + ) ?? false + ); + if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") + skip = !skip; + return skip; + }, + }), + ); + } + + this.app.set("json replacer", JSONReplacer); + + this.app.use(CORS); + + const app = this.app; + const api = Router(); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + this.app = api; + + await initRateLimits(api); + await initTranslation(api); + + this.routes = await registerRoutes( + this, + path.join(__dirname, "routes", "/"), + ); + + // 404 is not an error in express, so this should not be an error middleware + // this is a fine place to put the 404 handler because its after we register the routes + // and since its not an error middleware, our error handler below still works. + api.use("*", (req: Request, res: Response) => { + res.status(404).json({ + message: "404 endpoint not found", + code: 0, + }); + }); + + this.app = app; + + //app.use("/__development", ) + //app.use("/__internals", ) + app.use("/index", api); + + this.app.use(ErrorHandler); + + Sentry.errorHandler(this.app); + + if (logRequests) + console.log( + red( + `Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`, + ), + ); + + return super.start(); + } +} diff --git a/src/index/routes/channels/#channel_id/index.ts b/src/index/routes/channels/#channel_id/index.ts new file mode 100644 index 00000000..dec04006 --- /dev/null +++ b/src/index/routes/channels/#channel_id/index.ts @@ -0,0 +1,116 @@ +/* + * Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + * Copyright (C) 2023 Fosscord and Fosscord Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { Channel, Message, User } from "@fosscord/util"; +import { Request, Response, Router } from "express"; +import { route } from "@fosscord/api"; + +const router: Router = Router(); +// TODO: delete channel +// TODO: Get channel + +router.get("/", route({}), async (req: Request, res: Response) => { + const { channel_id } = req.params; + const chunk_size = Number(req.query.chunk_size) || 100; + if (chunk_size > 1000) + return res + .status(400) + .send({ message: "chunk_size must be <= 1000", code: 50035 }); + + // Alicia - get channel + const channel = await Channel.findOne({ + where: { id: channel_id }, + relations: ["guild"], + }); + // Alicia - if channel not found, or not indexable... + if (!channel || !channel.indexable) + return res.status(404).send({ + message: "Unknown Channel (Is it indexable?)", + code: 10003, + }); + + res.setHeader("Transfer-Encoding", "chunked"); + // Alicia - base body... + res.write("context and info here...\n"); + + // Alicia - get messages + console.time("get channel"); + console.log("Creating query builders..."); + const messageQueryBuilder = await Message.createQueryBuilder(); + const userQueryBuilder = await User.createQueryBuilder(); + console.log(`Querying messages for channel ${channel_id}...`); + const msg_author_ids = await messageQueryBuilder + .where("channel_id = :channel_id", { channel_id }) + .orderBy("id::int8", "DESC") + .select("Message.id, Message.author_id") + .getRawMany(); + + const message_ids = msg_author_ids.map((message) => message.id); + const author_ids = msg_author_ids.map((message) => message.author_id); + + console.log( + `Got ${message_ids.length} messages for channel ${channel_id}...`, + ); + // Alicia - get authors + const author_results = await userQueryBuilder + .where("id IN (:...ids)", { + ids: author_ids.unique(), + }) + .distinctOn(["id"]) + .select("User.id, User.username, User.discriminator, User.avatar") + .getRawMany(); + // Alicia - turn result into dictionary + const authors: any = {}; + for (const author of author_results) { + authors[author.User_id] = author; + } + + console.log( + `Got ${author_results.length} authors for channel ${channel_id}...`, + ); + + // Alicia - write messages + while (message_ids.length > 0) { + const current_message_ids = message_ids.splice(0, chunk_size); + res.write(`\n`); + console.log(`${message_ids.length} remain...`); + const messages = await messageQueryBuilder + .select("*") + .where("id IN (:...ids)", { + ids: current_message_ids, + }) + .getRawMany(); + for (const message of messages) { + res.write(JSON.stringify(message) + "\n"); + } + } + // message_ids.slice(); + // for (const message_id of message_ids) { + // const message = await Message.findOne({ + // where: { id: message_id + "" }, + // }); + // res.write(JSON.stringify(message) + "\n"); + // res.write("\n"); + // } + + res.write(JSON.stringify(channel)); + console.timeEnd("get channel"); + res.end(); +}); + +export default router; diff --git a/src/index/start.ts b/src/index/start.ts new file mode 100644 index 00000000..7975d085 --- /dev/null +++ b/src/index/start.ts @@ -0,0 +1,57 @@ +/* + Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2023 Fosscord and Fosscord Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +require("module-alias/register"); +process.on("uncaughtException", console.error); +process.on("unhandledRejection", console.error); + +import "missing-native-js-functions"; +import { config } from "dotenv"; +config(); +import { FosscordServer } from "./Server"; +import cluster from "cluster"; +import os from "os"; +let cores = 1; +try { + cores = Number(process.env.THREADS) || os.cpus().length; +} catch { + console.log("[API] Failed to get thread count! Using 1..."); +} + +if (cluster.isPrimary && process.env.NODE_ENV == "production") { + console.log(`Primary ${process.pid} is running`); + + // Fork workers. + for (let i = 0; i < cores; i++) { + cluster.fork(); + } + + cluster.on("exit", (worker) => { + console.log(`worker ${worker.process.pid} died, restart worker`); + cluster.fork(); + }); +} else { + const port = Number(process.env.PORT) || 3001; + + const server = new FosscordServer({ port }); + server.start().catch(console.error); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + global.server = server; +} diff --git a/src/util/entities/Application.ts b/src/util/entities/Application.ts index 94709320..599a5128 100644 --- a/src/util/entities/Application.ts +++ b/src/util/entities/Application.ts @@ -1,20 +1,20 @@ /* - Fosscord: A FOSS re-implementation and extension of the Discord.com backend. - Copyright (C) 2023 Fosscord and Fosscord Contributors - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ + * Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + * Copyright (C) 2023 Fosscord and Fosscord Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ import { Column, Entity, JoinColumn, ManyToOne, OneToOne } from "typeorm"; import { BaseClass } from "./BaseClass"; @@ -55,8 +55,8 @@ export class Application extends BaseClass { owner: User; // TODO: enum this? https://discord.com/developers/docs/resources/application#application-object-application-flags - @Column() - flags: number = 0; + @Column({ default: 0 }) + flags: number; @Column({ type: "simple-array", nullable: true }) redirect_uris: string[] = []; diff --git a/src/util/entities/Channel.ts b/src/util/entities/Channel.ts index 1f128713..f86cf890 100644 --- a/src/util/entities/Channel.ts +++ b/src/util/entities/Channel.ts @@ -1,20 +1,20 @@ /* - Fosscord: A FOSS re-implementation and extension of the Discord.com backend. - Copyright (C) 2023 Fosscord and Fosscord Contributors - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ + * Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + * Copyright (C) 2023 Fosscord and Fosscord Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ import { Column, @@ -32,9 +32,9 @@ import { containsAll, emitEvent, getPermission, + InvisibleCharacters, Snowflake, trimSpecial, - InvisibleCharacters, } from "../util"; import { ChannelCreateEvent, ChannelRecipientRemoveEvent } from "../interfaces"; import { Recipient } from "./Recipient"; @@ -193,6 +193,9 @@ export class Channel extends BaseClass { default_thread_rate_limit_per_user: number = 0; // TODO: DM channel + @Column({ default: false }) + indexable: boolean; + static async createChannel( channel: Partial, user_id: string = "0", @@ -325,11 +328,11 @@ export class Channel extends BaseClass { // TODO: check config for max number of recipients /** if you want to disallow note to self channels, uncomment the conditional below - const otherRecipientsUsers = await User.find({ where: recipients.map((x) => ({ id: x })) }); - if (otherRecipientsUsers.length !== recipients.length) { + const otherRecipientsUsers = await User.find({ where: recipients.map((x) => ({ id: x })) }); + if (otherRecipientsUsers.length !== recipients.length) { throw new HTTPError("Recipient/s not found"); } - **/ + **/ const type = recipients.length > 1 ? ChannelType.GROUP_DM : ChannelType.DM; diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts index 64d7ca14..8bfd5426 100644 --- a/src/util/util/Database.ts +++ b/src/util/util/Database.ts @@ -1,23 +1,23 @@ /* - Fosscord: A FOSS re-implementation and extension of the Discord.com backend. - Copyright (C) 2023 Fosscord and Fosscord Contributors - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ + * Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + * Copyright (C) 2023 Fosscord and Fosscord Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ import { DataSource } from "typeorm"; -import { yellow, green, red } from "picocolors"; +import { green, red, yellow } from "picocolors"; import { Migration } from "../entities/Migration"; import { ConfigEntity } from "../entities/Config"; import { config } from "dotenv"; @@ -50,7 +50,7 @@ const DataSourceOptions = new DataSource({ database: isSqlite ? dbConnectionString : undefined, entities: [path.join(__dirname, "..", "entities", "*.js")], synchronize: !!process.env.DB_SYNC, - logging: false, + logging: !!process.env.DB_LOG, bigNumberStrings: false, supportBigNumbers: true, name: "default", -- cgit 1.4.1