From b9e545a445dfef8cf42d837df8e653cba7c1a736 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Sat, 14 Aug 2021 23:15:19 +0200 Subject: :art: clean up server bundle --- bundle/package-lock.json | 115 +++++++++++++++++++++----------------------- bundle/package.json | 6 +-- bundle/src/BundledServer.ts | 41 ++++++++++++++++ bundle/src/Database.ts | 43 +++++++++++++++++ bundle/src/Server.ts | 22 +++------ bundle/src/root.ts | 33 ------------- bundle/src/start.ts | 57 +++------------------- bundle/src/stats.ts | 22 +++++++++ 8 files changed, 176 insertions(+), 163 deletions(-) create mode 100644 bundle/src/BundledServer.ts create mode 100644 bundle/src/Database.ts delete mode 100644 bundle/src/root.ts create mode 100644 bundle/src/stats.ts (limited to 'bundle') diff --git a/bundle/package-lock.json b/bundle/package-lock.json index 58547168..b9beea95 100644 --- a/bundle/package-lock.json +++ b/bundle/package-lock.json @@ -20,7 +20,7 @@ "@types/node-os-utils": "^1.2.0", "async-exit-hook": "^2.0.1", "express": "^4.17.1", - "link": "^0.1.5", + "mongodb-memory-server": "^7.3.6", "mongodb-memory-server-global-4.4": "^7.3.6", "node-os-utils": "^1.3.5", "typescript": "^4.3.5" @@ -32,8 +32,17 @@ "license": "ISC", "dependencies": { "@fosscord/util": "file:../util", + "@types/amqplib": "^0.8.1", + "@types/bcrypt": "^5.0.0", + "@types/express": "^4.17.9", + "@types/i18next-node-fs-backend": "^2.1.0", "@types/jest": "^26.0.22", "@types/json-schema": "^7.0.7", + "@types/jsonwebtoken": "^8.5.0", + "@types/multer": "^1.4.5", + "@types/node": "^14.17.9", + "@types/node-fetch": "^2.5.7", + "@zerollup/ts-transform-paths": "^1.7.18", "ajv": "^8.4.0", "ajv-formats": "^2.1.0", "amqplib": "^0.8.0", @@ -60,23 +69,7 @@ "mongoose-long": "^0.3.2", "multer": "^1.4.2", "node-fetch": "^2.6.1", - "require_optional": "^1.0.1" - }, - "devDependencies": { - "@types/amqplib": "^0.8.1", - "@types/bcrypt": "^5.0.0", - "@types/express": "^4.17.9", - "@types/i18next-node-fs-backend": "^2.1.0", - "@types/jsonwebtoken": "^8.5.0", - "@types/multer": "^1.4.5", - "@types/node": "^14.17.9", - "@types/node-fetch": "^2.5.7", - "@zerollup/ts-transform-paths": "^1.7.18", - "0x": "^4.10.2", - "caxa": "^2.1.0", - "jest": "^26.6.3", - "saslprep": "^1.0.3", - "ts-node": "^9.1.1", + "require_optional": "^1.0.1", "ts-node-dev": "^1.1.6", "typescript": "^4.1.2" } @@ -87,6 +80,15 @@ "license": "ISC", "dependencies": { "@fosscord/util": "file:../util", + "@types/body-parser": "^1.19.0", + "@types/btoa": "^1.2.3", + "@types/dotenv": "^8.2.0", + "@types/express": "^4.17.12", + "@types/fs-extra": "^9.0.12", + "@types/multer": "^1.4.7", + "@types/node": "^14.17.0", + "@types/node-fetch": "^2.5.7", + "@types/uuid": "^8.3.0", "body-parser": "^1.19.0", "btoa": "^1.2.1", "cheerio": "^1.0.0-rc.5", @@ -102,18 +104,8 @@ "missing-native-js-functions": "^1.0.8", "multer": "^1.4.2", "node-fetch": "^2.6.1", + "typescript": "^4.1.2", "uuid": "^8.3.2" - }, - "devDependencies": { - "@types/body-parser": "^1.19.0", - "@types/btoa": "^1.2.3", - "@types/dotenv": "^8.2.0", - "@types/express": "^4.17.12", - "@types/fs-extra": "^9.0.12", - "@types/multer": "^1.4.7", - "@types/node": "^14.17.0", - "@types/node-fetch": "^2.5.7", - "@types/uuid": "^8.3.0" } }, "../gateway": { @@ -122,6 +114,11 @@ "license": "ISC", "dependencies": { "@fosscord/util": "file:../util", + "@types/amqplib": "^0.8.1", + "@types/jsonwebtoken": "^8.5.0", + "@types/mongoose-autopopulate": "^0.10.1", + "@types/uuid": "^8.3.0", + "@types/ws": "^7.4.0", "ajv": "^8.5.0", "amqplib": "^0.8.0", "dotenv": "^8.2.0", @@ -130,17 +127,10 @@ "missing-native-js-functions": "^1.2.3", "mongoose-autopopulate": "^0.12.3", "node-fetch": "^2.6.1", + "ts-node-dev": "^1.1.6", "typescript": "^4.2.3", "uuid": "^8.3.2", "ws": "^7.4.2" - }, - "devDependencies": { - "@types/amqplib": "^0.8.1", - "@types/jsonwebtoken": "^8.5.0", - "@types/mongoose-autopopulate": "^0.10.1", - "@types/uuid": "^8.3.0", - "@types/ws": "^7.4.0", - "ts-node-dev": "^1.1.6" } }, "../util": { @@ -148,13 +138,14 @@ "version": "1.3.55", "hasInstallScript": true, "license": "GPLV3", - "devDependencies": { + "dependencies": { "@types/amqplib": "^0.8.1", "@types/jsonwebtoken": "^8.5.0", "@types/mongodb": "^3.6.9", "@types/mongoose-autopopulate": "^0.10.1", "@types/mongoose-lean-virtuals": "^0.5.1", "@types/node": "^14.17.9", + "@types/node-fetch": "^2.5.12", "ajv": "^8.5.0", "amqplib": "^0.8.0", "dot-prop": "^6.0.1", @@ -164,6 +155,7 @@ "mongodb": "^3.6.9", "mongoose": "^5.13.7", "mongoose-autopopulate": "^0.12.3", + "node-fetch": "^2.6.1", "typescript": "^4.1.3" } }, @@ -814,18 +806,6 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, - "node_modules/link": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/link/-/link-0.1.5.tgz", - "integrity": "sha1-f8eqW9P2rFoZxSVaI/QgfWhS6S4=", - "bin": { - "link.js": "Source/Node/link.js", - "linkjs": "Source/Node/link.js" - }, - "engines": { - "node": "> 0.4.x < 0.9.0" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -998,6 +978,19 @@ } } }, + "node_modules/mongodb-memory-server": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/mongodb-memory-server/-/mongodb-memory-server-7.3.6.tgz", + "integrity": "sha512-JfuY7uJD9TZxq6C4YVuCjiP2v0V/NYb19Wki6rFovdJvkgxGp5/KXKaazu47leECYAtJc2ajW1c009M3hRM+6A==", + "hasInstallScript": true, + "dependencies": { + "mongodb-memory-server-core": "7.3.6", + "tslib": "^2.3.0" + }, + "engines": { + "node": ">=12.22.0" + } + }, "node_modules/mongodb-memory-server-core": { "version": "7.3.6", "resolved": "https://registry.npmjs.org/mongodb-memory-server-core/-/mongodb-memory-server-core-7.3.6.tgz", @@ -1573,7 +1566,6 @@ "@types/node": "^14.17.9", "@types/node-fetch": "^2.5.7", "@zerollup/ts-transform-paths": "^1.7.18", - "0x": "^4.10.2", "ajv": "^8.4.0", "ajv-formats": "^2.1.0", "amqplib": "^0.8.0", @@ -1581,7 +1573,6 @@ "atomically": "^1.7.0", "bcrypt": "^5.0.1", "body-parser": "^1.19.0", - "caxa": "^2.1.0", "cheerio": "^1.0.0-rc.9", "dot-prop": "^6.0.1", "dotenv": "^8.2.0", @@ -1593,7 +1584,6 @@ "i18next-http-middleware": "^3.1.3", "i18next-node-fs-backend": "^2.1.3", "image-size": "^1.0.0", - "jest": "^26.6.3", "jsonwebtoken": "^8.5.1", "lambert-server": "^1.2.8", "missing-native-js-functions": "^1.2.6", @@ -1603,8 +1593,6 @@ "multer": "^1.4.2", "node-fetch": "^2.6.1", "require_optional": "^1.0.1", - "saslprep": "^1.0.3", - "ts-node": "^9.1.1", "ts-node-dev": "^1.1.6", "typescript": "^4.1.2" } @@ -1637,6 +1625,7 @@ "missing-native-js-functions": "^1.0.8", "multer": "^1.4.2", "node-fetch": "^2.6.1", + "typescript": "^4.1.2", "uuid": "^8.3.2" } }, @@ -1672,6 +1661,7 @@ "@types/mongoose-autopopulate": "^0.10.1", "@types/mongoose-lean-virtuals": "^0.5.1", "@types/node": "^14.17.9", + "@types/node-fetch": "^2.5.12", "ajv": "^8.5.0", "amqplib": "^0.8.0", "dot-prop": "^6.0.1", @@ -1681,6 +1671,7 @@ "mongodb": "^3.6.9", "mongoose": "^5.13.7", "mongoose-autopopulate": "^0.12.3", + "node-fetch": "^2.6.1", "typescript": "^4.1.3" } }, @@ -2168,11 +2159,6 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, - "link": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/link/-/link-0.1.5.tgz", - "integrity": "sha1-f8eqW9P2rFoZxSVaI/QgfWhS6S4=" - }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -2274,6 +2260,15 @@ "saslprep": "^1.0.0" } }, + "mongodb-memory-server": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/mongodb-memory-server/-/mongodb-memory-server-7.3.6.tgz", + "integrity": "sha512-JfuY7uJD9TZxq6C4YVuCjiP2v0V/NYb19Wki6rFovdJvkgxGp5/KXKaazu47leECYAtJc2ajW1c009M3hRM+6A==", + "requires": { + "mongodb-memory-server-core": "7.3.6", + "tslib": "^2.3.0" + } + }, "mongodb-memory-server-core": { "version": "7.3.6", "resolved": "https://registry.npmjs.org/mongodb-memory-server-core/-/mongodb-memory-server-core-7.3.6.tgz", diff --git a/bundle/package.json b/bundle/package.json index 210afe5f..928cd5fb 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -10,8 +10,7 @@ "build:api": "cd ../api/ && npm run build", "build:cdn": "cd ../cdn/ && npm run build", "build:gateway": "cd ../gateway/ && npm run build", - "bundle": "npm run build && node dist/start.js", - "start": "npm run build && node dist/root.js", + "start": "npm run build && node dist/start.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -36,8 +35,7 @@ "@types/node-os-utils": "^1.2.0", "async-exit-hook": "^2.0.1", "express": "^4.17.1", - "link": "^0.1.5", - "mongodb-memory-server-global-4.4": "^7.3.6", + "mongodb-memory-server": "^7.3.6", "node-os-utils": "^1.3.5", "typescript": "^4.3.5" } diff --git a/bundle/src/BundledServer.ts b/bundle/src/BundledServer.ts new file mode 100644 index 00000000..6951c1c4 --- /dev/null +++ b/bundle/src/BundledServer.ts @@ -0,0 +1,41 @@ +process.on("unhandledRejection", console.error); +process.on("uncaughtException", console.error); + +import http from "http"; +import { FosscordServer as APIServer } from "@fosscord/api"; +import { Server as GatewayServer } from "@fosscord/gateway"; +import { CDNServer } from "@fosscord/cdn/"; +import express from "express"; +import { Config } from "../../util/dist"; + +const app = express(); +const server = http.createServer(); +const port = Number(process.env.PORT) || 8080; +const production = true; +server.on("request", app); + +// @ts-ignore +const api = new APIServer({ server, port, production, app }); +// @ts-ignore +const cdn = new CDNServer({ server, port, production, app }); +// @ts-ignore +const gateway = new GatewayServer({ server, port, production }); + +async function main() { + await Config.set({ + cdn: { + endpointClient: "${location.host}", + endpoint: `http://localhost:${port}`, + }, + gateway: { + endpointClient: '${location.protocol === "https:" ? "wss://" : "ws://"}${location.host}', + endpoint: `ws://localhost:${port}`, + }, + }); + + await api.start(); + await cdn.start(); + await gateway.start(); +} + +main().catch(console.error); diff --git a/bundle/src/Database.ts b/bundle/src/Database.ts new file mode 100644 index 00000000..0efd2471 --- /dev/null +++ b/bundle/src/Database.ts @@ -0,0 +1,43 @@ +import fs from "fs"; +import { MongoMemoryServer } from "mongodb-memory-server"; +import path from "path"; +import exitHook from "async-exit-hook"; +console.log(process.arch, process.platform); +if (process.arch == "ia32") { + Object.defineProperty(process, "arch", { + value: "x64", + }); +} + +export async function setupDatabase() { + const dbPath = path.join(__dirname, "..", "..", "db"); + const dbName = "fosscord"; + const storageEngine = "wiredTiger"; + const port = 27020; + const ip = "127.0.0.1"; + var mongod: MongoMemoryServer; + fs.mkdirSync(dbPath, { recursive: true }); + + exitHook((callback: any) => { + (async () => { + console.log(`Stopping MongoDB ...`); + await mongod.stop(); + console.log(`Stopped MongoDB`); + callback(); + })(); + }); + + console.log(`[Database] starting ...`); + mongod = new MongoMemoryServer({ + instance: { + port, + ip, + dbName, + dbPath, + storageEngine, + auth: false, // by default `mongod` is started with '--noauth', start `mongod` with '--auth' + }, + }); + await mongod.start(); + process.env.MONGO_URL = mongod.getUri(dbName); +} diff --git a/bundle/src/Server.ts b/bundle/src/Server.ts index 6951c1c4..42a3f3c3 100644 --- a/bundle/src/Server.ts +++ b/bundle/src/Server.ts @@ -1,35 +1,27 @@ process.on("unhandledRejection", console.error); process.on("uncaughtException", console.error); -import http from "http"; import { FosscordServer as APIServer } from "@fosscord/api"; import { Server as GatewayServer } from "@fosscord/gateway"; import { CDNServer } from "@fosscord/cdn/"; -import express from "express"; import { Config } from "../../util/dist"; -const app = express(); -const server = http.createServer(); -const port = Number(process.env.PORT) || 8080; const production = true; -server.on("request", app); -// @ts-ignore -const api = new APIServer({ server, port, production, app }); -// @ts-ignore -const cdn = new CDNServer({ server, port, production, app }); -// @ts-ignore -const gateway = new GatewayServer({ server, port, production }); +const api = new APIServer({ production, port: Number(process.env.API_PORT) || 3001 }); +const gateway = new GatewayServer({ port: Number(process.env.GATEWAY_PORT) || 3002 }); +const cdn = new CDNServer({ production, port: Number(process.env.CDN_PORT) || 3003 }); async function main() { await Config.set({ cdn: { endpointClient: "${location.host}", - endpoint: `http://localhost:${port}`, + endpoint: `http://localhost:${cdn.options.port}`, }, gateway: { - endpointClient: '${location.protocol === "https:" ? "wss://" : "ws://"}${location.host}', - endpoint: `ws://localhost:${port}`, + endpointClient: + '${location.protocol === "https:" ? "wss://" : "ws://"}${location.hostname}:' + gateway.port, + endpoint: `ws://localhost:${gateway.port}`, }, }); diff --git a/bundle/src/root.ts b/bundle/src/root.ts deleted file mode 100644 index 42a3f3c3..00000000 --- a/bundle/src/root.ts +++ /dev/null @@ -1,33 +0,0 @@ -process.on("unhandledRejection", console.error); -process.on("uncaughtException", console.error); - -import { FosscordServer as APIServer } from "@fosscord/api"; -import { Server as GatewayServer } from "@fosscord/gateway"; -import { CDNServer } from "@fosscord/cdn/"; -import { Config } from "../../util/dist"; - -const production = true; - -const api = new APIServer({ production, port: Number(process.env.API_PORT) || 3001 }); -const gateway = new GatewayServer({ port: Number(process.env.GATEWAY_PORT) || 3002 }); -const cdn = new CDNServer({ production, port: Number(process.env.CDN_PORT) || 3003 }); - -async function main() { - await Config.set({ - cdn: { - endpointClient: "${location.host}", - endpoint: `http://localhost:${cdn.options.port}`, - }, - gateway: { - endpointClient: - '${location.protocol === "https:" ? "wss://" : "ws://"}${location.hostname}:' + gateway.port, - endpoint: `ws://localhost:${gateway.port}`, - }, - }); - - await api.start(); - await cdn.start(); - await gateway.start(); -} - -main().catch(console.error); diff --git a/bundle/src/start.ts b/bundle/src/start.ts index ee012c15..581c3d9a 100644 --- a/bundle/src/start.ts +++ b/bundle/src/start.ts @@ -1,64 +1,19 @@ -import fs from "fs"; -import { MongoMemoryServer } from "mongodb-memory-server-global-4.4"; -import path from "path"; +process.env.MONGOMS_DEBUG = "true"; + import cluster from "cluster"; import os from "os"; -import osu from "node-os-utils"; -import exitHook from "async-exit-hook"; +import { setupDatabase } from "./Database"; +import { initStats } from "./stats"; // TODO: add tcp socket event transmission const cores = 1 || Number(process.env.threads) || os.cpus().length; if (cluster.isMaster && !process.env.masterStarted) { - const dbPath = path.join(__dirname, "..", "..", "db"); - const dbName = "fosscord"; - const storageEngine = "wiredTiger"; - const port = 27020; - const ip = "127.0.0.1"; - var mongod: MongoMemoryServer; - fs.mkdirSync(dbPath, { recursive: true }); - - exitHook((callback: any) => { - (async () => { - console.log(`Stopping MongoDB ...`); - await mongod.stop(); - console.log(`Stopped MongoDB`); - callback(); - })(); - }); - process.env.masterStarted = "true"; - setInterval(async () => { - const [cpuUsed, memory, network] = await Promise.all([osu.cpu.usage(), osu.mem.info(), osu.netstat.inOut()]); - if (typeof network === "object") { - console.log(`Network: in ${network.total.inputMb}mb | out ${network.total.outputMb}mb`); - } - - console.log( - `[CPU] ${cpuUsed.toFixed(2)}% | [Memory] ${memory.usedMemMb.toFixed(0)}mb/${memory.totalMemMb.toFixed(0)}mb` - ); - }, 1000 * 60); - (async () => { - console.log(`[Database] starting ...`); - mongod = new MongoMemoryServer({ - instance: { - port, - ip, - dbName, - dbPath, - storageEngine, - auth: false, // by default `mongod` is started with '--noauth', start `mongod` with '--auth' - }, - }); - await mongod.start(); - process.env.MONGO_URL = mongod.getUri(dbName); - - console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`); - console.log(`[System] ${await osu.os.oos()} ${os.arch()}`); - console.log(`[Database] started`); - console.log(`[Process] running with pid: ${process.pid}`); + initStats(); + await setupDatabase(); if (cores === 1) { require("./Server.js"); diff --git a/bundle/src/stats.ts b/bundle/src/stats.ts new file mode 100644 index 00000000..c621ed75 --- /dev/null +++ b/bundle/src/stats.ts @@ -0,0 +1,22 @@ +import os from "os"; +import osu from "node-os-utils"; + +export function initStats() { + console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`); + console.log(`[System] ${os.platform()} ${os.arch()}`); + console.log(`[Database] started`); + console.log(`[Process] running with pid: ${process.pid}`); + + setInterval(async () => { + const [cpuUsed, memory, network] = await Promise.all([osu.cpu.usage(), osu.mem.info(), osu.netstat.inOut()]); + if (typeof network === "object") { + console.log(`[Network]: in ${network.total.inputMb}mb | out ${network.total.outputMb}mb`); + } + + console.log( + `[CPU] ${cpuUsed.toFixed(2)}% | [Memory] ${Math.round( + process.memoryUsage().rss / 1024 / 1024 + )}mb/${memory.totalMemMb.toFixed(0)}mb` + ); + }, 1000 * 60); +} -- cgit 1.4.1