From af73cb69938bc4c16454d74301a02e230c373e6f Mon Sep 17 00:00:00 2001 From: developomp Date: Wed, 20 Oct 2021 09:14:48 +0900 Subject: prettier formatted /cdn --- cdn/README.md | 27 ++++++++- cdn/package.json | 130 ++++++++++++++++++++-------------------- cdn/src/util/S3Storage.ts | 20 +++---- cdn/src/util/Storage.ts | 19 +++--- cdn/tests/cdn_endpoints.test.js | 63 +++++++++++++------ cdn/tsconfig.json | 9 ++- 6 files changed, 164 insertions(+), 104 deletions(-) (limited to 'cdn') diff --git a/cdn/README.md b/cdn/README.md index 6cd6186a..7d8e99b0 100644 --- a/cdn/README.md +++ b/cdn/README.md @@ -1,20 +1,26 @@ # Fosscord-CDN + CDN for Fosscord ## Run localy: + ``` npm i node dist/ ``` ## Endpoints: + ### POST `/attachments/` + ``` Content-Type: form-data attachment: File (binary-data) ``` + ##### Returns: + ``` { "success": boolean, // true @@ -23,20 +29,28 @@ attachment: File (binary-data) "filename": string // "lakdoiauej.png" } ``` + ### GET `/attachments//` + ``` requests image from database with given and ``` + ##### Returns: + ``` Content-Type: image/ Image ``` + ### DELETE `/attachments//` + ``` deletes database entry ``` + ##### Returns: + ``` Content-Type: application/json @@ -49,7 +63,8 @@ Content-Type: application/json
_(endpoints for crawler):_ -### POST `/external` + +### POST `/external` ``` requests crawling of `og:`metadata and the download of the `og:image` property @@ -59,7 +74,9 @@ Content-Type: application/json body: {"url": URL} // "https://discord.com" ``` + ##### Returns: + ``` Content-Type: application/json @@ -72,17 +89,23 @@ Content-Type: application/json "ogType": string // "website" } ``` + ### GET `/external//` -- requests cached crawled image + +- requests cached crawled image + ``` url-params: :id // aHR0cHM6Ly9kaXNjb3JkLmNvbQ== :filename // discord.png ``` + ``` /external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png ``` + ##### Returns: + ``` Content-Type: image/ Image diff --git a/cdn/package.json b/cdn/package.json index 318f1632..fec43785 100644 --- a/cdn/package.json +++ b/cdn/package.json @@ -1,67 +1,67 @@ { - "name": "@fosscord/cdn", - "version": "1.0.0", - "description": "cdn for fosscord", - "main": "dist/index.js", - "types": "src/index.ts", - "scripts": { - "test": "npm run build && jest --coverage ./tests", - "build": "npx tsc -p .", - "start": "node dist/start.js" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/fosscord/fosscord-server.git" - }, - "keywords": [], - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/fosscord/fosscord-server/issues" - }, - "homepage": "https://github.com/fosscord/fosscord-server#readme", - "devDependencies": { - "@types/amqplib": "^0.8.1", - "@types/body-parser": "^1.19.0", - "@types/btoa": "^1.2.3", - "@types/dotenv": "^8.2.0", - "@types/express": "^4.17.12", - "@types/fs-extra": "^9.0.12", - "@types/jsonwebtoken": "^8.5.0", - "@types/multer": "^1.4.7", - "@types/node": "^14.17.0", - "@types/node-fetch": "^2.5.7", - "@zerollup/ts-transform-paths": "^1.7.18", - "ts-patch": "^1.4.4" - }, - "dependencies": { - "@aws-sdk/client-s3": "^3.36.1", - "@aws-sdk/node-http-handler": "^3.36.0", - "@fosscord/util": "file:../util", - "body-parser": "^1.19.0", - "btoa": "^1.2.1", - "dotenv": "^10.0.0", - "exif-be-gone": "^1.2.0", - "express": "^4.17.1", - "express-async-errors": "^3.1.1", - "file-type": "^16.5.0", - "form-data": "^4.0.0", - "fs-extra": "^10.0.0", - "image-size": "^1.0.0", - "jest": "^27.0.6", - "lambert-db": "^1.2.3", - "lambert-server": "^1.2.12", - "missing-native-js-functions": "^1.2.17", - "multer": "^1.4.2", - "nanocolors": "^0.2.12", - "node-fetch": "^2.6.1", - "supertest": "^6.1.6", - "typescript": "^4.1.2" - }, - "jest": { - "setupFilesAfterEnv": [ - "/jest/setup.js" - ], - "verbose": true - } + "name": "@fosscord/cdn", + "version": "1.0.0", + "description": "cdn for fosscord", + "main": "dist/index.js", + "types": "src/index.ts", + "scripts": { + "test": "npm run build && jest --coverage ./tests", + "build": "npx tsc -p .", + "start": "node dist/start.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/fosscord/fosscord-server.git" + }, + "keywords": [], + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/fosscord/fosscord-server/issues" + }, + "homepage": "https://github.com/fosscord/fosscord-server#readme", + "devDependencies": { + "@types/amqplib": "^0.8.1", + "@types/body-parser": "^1.19.0", + "@types/btoa": "^1.2.3", + "@types/dotenv": "^8.2.0", + "@types/express": "^4.17.12", + "@types/fs-extra": "^9.0.12", + "@types/jsonwebtoken": "^8.5.0", + "@types/multer": "^1.4.7", + "@types/node": "^14.17.0", + "@types/node-fetch": "^2.5.7", + "@zerollup/ts-transform-paths": "^1.7.18", + "ts-patch": "^1.4.4" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.36.1", + "@aws-sdk/node-http-handler": "^3.36.0", + "@fosscord/util": "file:../util", + "body-parser": "^1.19.0", + "btoa": "^1.2.1", + "dotenv": "^10.0.0", + "exif-be-gone": "^1.2.0", + "express": "^4.17.1", + "express-async-errors": "^3.1.1", + "file-type": "^16.5.0", + "form-data": "^4.0.0", + "fs-extra": "^10.0.0", + "image-size": "^1.0.0", + "jest": "^27.0.6", + "lambert-db": "^1.2.3", + "lambert-server": "^1.2.12", + "missing-native-js-functions": "^1.2.17", + "multer": "^1.4.2", + "nanocolors": "^0.2.12", + "node-fetch": "^2.6.1", + "supertest": "^6.1.6", + "typescript": "^4.1.2" + }, + "jest": { + "setupFilesAfterEnv": [ + "/jest/setup.js" + ], + "verbose": true + } } diff --git a/cdn/src/util/S3Storage.ts b/cdn/src/util/S3Storage.ts index df5bc19c..c4066817 100644 --- a/cdn/src/util/S3Storage.ts +++ b/cdn/src/util/S3Storage.ts @@ -5,30 +5,30 @@ import { Storage } from "./Storage"; const readableToBuffer = (readable: Readable): Promise => new Promise((resolve, reject) => { const chunks: Buffer[] = []; - readable.on('data', chunk => chunks.push(chunk)); - readable.on('error', reject); - readable.on('end', () => resolve(Buffer.concat(chunks))); + readable.on("data", (chunk) => chunks.push(chunk)); + readable.on("error", reject); + readable.on("end", () => resolve(Buffer.concat(chunks))); }); export class S3Storage implements Storage { public constructor( private client: S3, private bucket: string, - private basePath?: string, + private basePath?: string ) {} /** * Always return a string, to ensure consistency. */ get bucketBasePath() { - return this.basePath ?? ''; + return this.basePath ?? ""; } async set(path: string, data: Buffer): Promise { await this.client.putObject({ Bucket: this.bucket, Key: `${this.bucketBasePath}${path}`, - Body: data + Body: data, }); } @@ -36,15 +36,15 @@ export class S3Storage implements Storage { try { const s3Object = await this.client.getObject({ Bucket: this.bucket, - Key: `${this.bucketBasePath ?? ''}${path}` + Key: `${this.bucketBasePath ?? ""}${path}`, }); if (!s3Object.Body) return null; const body = s3Object.Body; - return await readableToBuffer( body); - } catch(err) { + return await readableToBuffer(body); + } catch (err) { console.error(`[CDN] Unable to get S3 object at path ${path}.`); console.error(err); return null; @@ -54,7 +54,7 @@ export class S3Storage implements Storage { async delete(path: string): Promise { await this.client.deleteObject({ Bucket: this.bucket, - Key: `${this.bucketBasePath}${path}` + Key: `${this.bucketBasePath}${path}`, }); } } diff --git a/cdn/src/util/Storage.ts b/cdn/src/util/Storage.ts index 3332f21c..f45fe44a 100644 --- a/cdn/src/util/Storage.ts +++ b/cdn/src/util/Storage.ts @@ -2,7 +2,7 @@ import { FileStorage } from "./FileStorage"; import path from "path"; import fse from "fs-extra"; import { bgCyan, black } from "nanocolors"; -import { S3 } from '@aws-sdk/client-s3'; +import { S3 } from "@aws-sdk/client-s3"; import { S3Storage } from "./S3Storage"; process.cwd(); @@ -27,17 +27,20 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { storage = new FileStorage(); } else if (process.env.STORAGE_PROVIDER === "s3") { - const - region = process.env.STORAGE_REGION, + const region = process.env.STORAGE_REGION, bucket = process.env.STORAGE_BUCKET; if (!region) { - console.error(`[CDN] You must provide a region when using the S3 storage provider.`); + console.error( + `[CDN] You must provide a region when using the S3 storage provider.` + ); process.exit(1); } if (!bucket) { - console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`); + console.error( + `[CDN] You must provide a bucket when using the S3 storage provider.` + ); process.exit(1); } @@ -45,8 +48,10 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { let location = process.env.STORAGE_LOCATION; if (!location) { - console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`); - location = undefined; + console.warn( + `[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...` + ); + location = undefined; } const client = new S3({ region }); diff --git a/cdn/tests/cdn_endpoints.test.js b/cdn/tests/cdn_endpoints.test.js index a133d0dd..8cf16748 100644 --- a/cdn/tests/cdn_endpoints.test.js +++ b/cdn/tests/cdn_endpoints.test.js @@ -11,7 +11,12 @@ if (!process.env.STORAGE_PROVIDER) process.env.STORAGE_PROVIDER = "file"; if (process.env.STORAGE_PROVIDER === "file") { if (process.env.STORAGE_LOCATION) { if (!process.env.STORAGE_LOCATION.startsWith("/")) { - process.env.STORAGE_LOCATION = path.join(__dirname, "..", process.env.STORAGE_LOCATION, "/"); + process.env.STORAGE_LOCATION = path.join( + __dirname, + "..", + process.env.STORAGE_LOCATION, + "/" + ); } } else { process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/"); @@ -67,7 +72,9 @@ describe("/attachments", () => { .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); expect(response.statusCode).toBe(200); - expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); + expect(response.headers["content-type"]).toEqual( + expect.stringContaining("json") + ); expect(response.body.url).toBeDefined(); }); }); @@ -79,9 +86,11 @@ describe("/attachments", () => { .post("/attachments/123456789") .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); - request.get(response.body.url.replace("http://localhost:3003", "")).then((x) => { - expect(x.statusCode).toBe(200); - }); + request + .get(response.body.url.replace("http://localhost:3003", "")) + .then((x) => { + expect(x.statusCode).toBe(200); + }); }); }); }); @@ -92,9 +101,13 @@ describe("/attachments", () => { .post("/attachments/123456789") .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); - request.delete(response.body.url.replace("http://localhost:3003", "")).then((x) => { - expect(x.body.success).toBeDefined(); - }); + request + .delete( + response.body.url.replace("http://localhost:3003", "") + ) + .then((x) => { + expect(x.body.success).toBeDefined(); + }); }); }); }); @@ -123,7 +136,9 @@ describe("/avatars", () => { .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); expect(response.statusCode).toBe(200); - expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); + expect(response.headers["content-type"]).toEqual( + expect.stringContaining("json") + ); expect(response.body.url).toBeDefined(); }); }); @@ -135,9 +150,11 @@ describe("/avatars", () => { .post("/avatars/123456789") .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); - request.get(response.body.url.replace("http://localhost:3003", "")).then((x) => { - expect(x.statusCode).toBe(200); - }); + request + .get(response.body.url.replace("http://localhost:3003", "")) + .then((x) => { + expect(x.statusCode).toBe(200); + }); }); }); }); @@ -148,9 +165,13 @@ describe("/avatars", () => { .post("/avatars/123456789") .set({ signature: Config.get().security.requestSignature }) .attach("file", __dirname + "/antman.jpg"); - request.delete(response.body.url.replace("http://localhost:3003", "")).then((x) => { - expect(x.body.success).toBeDefined(); - }); + request + .delete( + response.body.url.replace("http://localhost:3003", "") + ) + .then((x) => { + expect(x.body.success).toBeDefined(); + }); }); }); }); @@ -177,9 +198,13 @@ describe("/external", () => { const response = await request .post("/external") .set({ signature: Config.get().security.requestSignature }) - .send({ url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp" }); + .send({ + url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp", + }); expect(response.statusCode).toBe(200); - expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); + expect(response.headers["content-type"]).toEqual( + expect.stringContaining("json") + ); expect(response.body.id).toBeDefined(); }); }); @@ -201,7 +226,9 @@ describe("/external", () => { let response = await request .post("/external") .set({ signature: Config.get().security.requestSignature }) - .send({ url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp" }); + .send({ + url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp", + }); request.get(`external/${response.body.id}`).then((x) => { expect(x.statusCode).toBe(200); }); diff --git a/cdn/tsconfig.json b/cdn/tsconfig.json index 249dffff..64ab18f4 100644 --- a/cdn/tsconfig.json +++ b/cdn/tsconfig.json @@ -7,7 +7,10 @@ "incremental": true /* Enable incremental compilation */, "target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, - "lib": ["ES2015", "dom"] /* Specify library files to be included in the compilation. */, + "lib": [ + "ES2015", + "dom" + ] /* Specify library files to be included in the compilation. */, "allowJs": true /* Allow javascript files to be compiled. */, "checkJs": true /* Report errors in .js files. */, // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ @@ -48,7 +51,9 @@ // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "typeRoots": [], /* List of folders to include type definitions from. */ - "types": ["node"] /* Type declaration files to be included in compilation. */, + "types": [ + "node" + ] /* Type declaration files to be included in compilation. */, // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ -- cgit 1.4.1