diff --git a/scripts/openapi.js b/scripts/openapi.js
index a1188c54..ab7de525 100644
--- a/scripts/openapi.js
+++ b/scripts/openapi.js
@@ -16,6 +16,9 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
+const { Stopwatch } = require("../dist/util/util/Stopwatch");
+const totalSw = Stopwatch.startNew();
+
require("module-alias/register");
const getRouteDescriptions = require("./util/getRouteDescriptions");
const path = require("path");
@@ -83,7 +86,7 @@ function combineSchemas(schemas) {
for (const key in definitions) {
if (!schemaRegEx.test(key)) {
- console.error(`${bgRedBright("ERROR")} Invalid schema name: ${key}, context:`, definitions[key]);
+ console.error(` \x1b[5m${bgRedBright("ERROR")}\x1b[25m Invalid schema name: ${key}, context:`, definitions[key]);
continue;
}
specification.components = specification.components || {};
@@ -240,7 +243,7 @@ async function main() {
fs.writeFileSync(openapiPath, JSON.stringify(specification, null, 4).replaceAll("#/definitions", "#/components/schemas").replaceAll("bigint", "number"));
console.log("Wrote OpenAPI specification to", openapiPath);
- console.log("Specification contains", Object.keys(specification.paths).length, "paths and", Object.keys(specification.components.schemas).length, "schemas.");
+ console.log("Specification contains", Object.keys(specification.paths).length, "paths and", Object.keys(specification.components.schemas).length, "schemas in", Number(totalSw.elapsed().totalMilliseconds + "." + totalSw.elapsed().microseconds), "ms.");
}
main();
diff --git a/scripts/schema.js b/scripts/schema.js
index 4d62f91f..d5cea3a4 100644
--- a/scripts/schema.js
+++ b/scripts/schema.js
@@ -19,8 +19,8 @@
/*
Regenerates the `spacebarchat/server/assets/schemas.json` file, used for API/Gateway input validation.
*/
-
-const scriptStartTime = new Date();
+const { Stopwatch } = require("../dist/util/util/Stopwatch");
+const totalSw = Stopwatch.startNew();
const conWarn = console.warn;
console.warn = (...args) => {
@@ -32,9 +32,10 @@ console.warn = (...args) => {
const path = require("path");
const fs = require("fs");
+const fsp = require("fs/promises");
const TJS = require("typescript-json-schema");
const walk = require("./util/walk");
-const { redBright, yellowBright, bgRedBright, yellow } = require("picocolors");
+const { redBright, yellowBright, bgRedBright, yellow, greenBright, green, cyanBright } = require("picocolors");
const schemaPath = path.join(__dirname, "..", "assets", "schemas.json");
const exclusionList = JSON.parse(fs.readFileSync(path.join(__dirname, "schemaExclusions.json"), { encoding: "utf8" }));
@@ -47,6 +48,20 @@ const settings = {
defaultProps: false,
};
+const baseClassProperties = [
+ // BaseClass methods
+ "toJSON",
+ "hasId",
+ "save",
+ "remove",
+ "softRemove",
+ "recover",
+ "reload",
+ "assign",
+ "_do_validate", // ?
+ "hasId", // ?
+];
+
const ExcludeAndWarn = [...exclusionList.manualWarn, ...exclusionList.manualWarnRe.map((r) => new RegExp(r))];
const Excluded = [...exclusionList.manual, ...exclusionList.manualRe.map((r) => new RegExp(r)), ...exclusionList.auto.map((r) => r.value)];
const Included = [...exclusionList.include, ...exclusionList.includeRe.map((r) => new RegExp(r))];
@@ -97,12 +112,12 @@ const excludedLambdas = [
}
},
(n, s) => {
- if (s.properties && Object.keys(s.properties).every(x=> x[0] === x[0].toUpperCase())) {
+ if (s.properties && Object.keys(s.properties).every((x) => x[0] === x[0].toUpperCase())) {
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as all its properties have uppercase characters.`);
exclusionList.auto.push({ value: n, reason: "Schema with only uppercase properties" });
return true;
}
- }
+ },
// (n, s) => {
// if (JSON.stringify(s).length <= 300) {
// console.log({n, s});
@@ -115,17 +130,27 @@ function includesMatch(haystack, needles, log = false) {
const match = needle instanceof RegExp ? needle.test(haystack) : haystack === needle;
if (match) {
if (log) console.warn(redBright("[WARN]:"), "Excluding schema", haystack, "due to match with", needle);
- return true;
+ return needle;
}
}
- return false;
+ return null;
}
-function main() {
+async function main() {
+ const stepSw = Stopwatch.startNew();
+
+ process.stdout.write("Loading program... ");
const program = TJS.programFromConfig(path.join(__dirname, "..", "tsconfig.json"), walk(path.join(__dirname, "..", "src", "schemas")));
const generator = TJS.buildGenerator(program, settings);
- if (!generator || !program) return;
+ if (!generator || !program) {
+ console.log(redBright("Failed to create schema generator."));
+ return;
+ }
+ const elapsedLoad = stepSw.getElapsedAndReset();
+ process.stdout.write("Done in " + yellowBright(elapsedLoad.totalMilliseconds + "." + elapsedLoad.microseconds) + " ms\n");
+
+ process.stdout.write("Generating schema list... ");
let schemas = generator.getUserSymbols().filter((x) => {
return (
(x.endsWith("Schema") || x.endsWith("Response") || x.startsWith("API")) &&
@@ -138,42 +163,34 @@ function main() {
(includesMatch(x, Included) || (!includesMatch(x, ExcludeAndWarn, true) && !includesMatch(x, Excluded)))
);
});
- //.sort((a,b) => a.localeCompare(b));
+ //.sort((a,b) => a.localeCompare(b));
+
+ const elapsedList = stepSw.getElapsedAndReset();
+ process.stdout.write("Done in " + yellowBright(elapsedList.totalMilliseconds + "." + elapsedList.microseconds) + " ms\n");
+ console.log("Found", yellowBright(schemas.length), "schemas to process.");
- var definitions = {};
+ let definitions = {};
+ let nestedDefinitions = {};
+ let writePromises = [];
if (process.env.WRITE_SCHEMA_DIR === "true") {
- fs.rmSync("schemas", { recursive: true, force: true });
- fs.mkdirSync("schemas");
+ fs.rmSync("schemas_orig", { recursive: true, force: true });
+ fs.mkdirSync("schemas_orig");
+
+ fs.rmSync("schemas_nested", { recursive: true, force: true });
+ fs.mkdirSync("schemas_nested");
+
+ fs.rmSync("schemas_final", { recursive: true, force: true });
+ fs.mkdirSync("schemas_final");
}
+ const schemaSw = Stopwatch.startNew();
for (const name of schemas) {
- const startTime = new Date();
process.stdout.write(`Processing schema ${name}... `);
const part = TJS.generateSchema(program, name, settings, [], generator);
if (!part) continue;
- // this is a hack. want some want to check if its a @column, instead
- if (part.properties) {
- for (let key in part.properties) {
- if (
- [
- // BaseClass methods
- "toJSON",
- "hasId",
- "save",
- "remove",
- "softRemove",
- "recover",
- "reload",
- "assign",
- ].includes(key)
- ) {
- delete part.properties[key];
- continue;
- }
- }
- }
+ filterSchema(part);
if (definitions[name]) {
process.stdout.write(yellow(` [ERROR] Duplicate schema name detected: ${name}. Overwriting previous schema.`));
@@ -183,20 +200,111 @@ function main() {
continue;
}
- if (process.env.WRITE_SCHEMA_DIR === "true") fs.writeFileSync(path.join("schemas", `${name}.json`), JSON.stringify(part, null, 4));
+ if (process.env.WRITE_SCHEMA_DIR === "true") writePromises.push(fsp.writeFile(path.join("schemas_orig", `${name}.json`), JSON.stringify(part, null, 4)));
+
+ // testing:
+ function mergeDefs(schemaName, schema) {
+ if (schema.definitions) {
+ // schema["x-sb-defs"] = Object.keys(schema.definitions);
+ process.stdout.write(cyanBright("Processing nested... "));
+ for (const defKey in schema.definitions) {
+ if (definitions[defKey] && deepEqual(definitions[defKey], schema.definitions[defKey])) {
+ // console.log("Definition", defKey, "from schema", schemaName, "is identical to existing definition, skipping.");
+ schema.definitions = Object.fromEntries(Object.entries(schema.definitions).filter(([k, v]) => k !== defKey));
+ process.stdout.write(greenBright("T"));
+ } else if (!nestedDefinitions[defKey]) {
+ nestedDefinitions[defKey] = schema.definitions[defKey];
+ schema.definitions = Object.fromEntries(Object.entries(schema.definitions).filter(([k, v]) => k !== defKey));
+ // console.log("Tracking sub-definition", defKey, "from schema", schemaName);
+ process.stdout.write(green("N"));
+ } else if (!deepEqual(nestedDefinitions[defKey], schema.definitions[defKey])) {
+ console.log(redBright("[ERROR]"), "Conflicting nested definition for", defKey, "found in schema", schemaName);
+ console.log(columnizedObjectDiff(nestedDefinitions[defKey], schema.definitions[defKey], true));
+ } else {
+ // console.log("Definition", defKey, "from schema", schemaName, "is identical to existing definition, skipping.");
+ schema.definitions = Object.fromEntries(Object.entries(schema.definitions).filter(([k, v]) => k !== defKey));
+ process.stdout.write(greenBright("M"));
+ }
+ }
+ if (Object.keys(schema.definitions).length === 0) {
+ process.stdout.write(greenBright("✓ "));
+ delete schema.definitions;
+ } else {
+ console.log("Remaining definitions in schema", schemaName, "after merge:", Object.keys(schema.definitions));
+ }
+ }
+ }
+ mergeDefs(name, part);
- process.stdout.write("Done in " + yellowBright(new Date() - startTime) + " ms, " + yellowBright(JSON.stringify(part).length) + " bytes (unformatted) ");
- if (new Date() - startTime >= 20) console.log(bgRedBright("[SLOW]"));
+ const elapsed = schemaSw.getElapsedAndReset();
+ process.stdout.write("Done in " + yellowBright(elapsed.totalMilliseconds + "." + elapsed.microseconds) + " ms, " + yellowBright(JSON.stringify(part).length) + " bytes (unformatted) ");
+ if (elapsed.totalMilliseconds >= 20) console.log(bgRedBright("\x1b[5m[SLOW]\x1b[25m"));
else console.log();
definitions = { ...definitions, [name]: { ...part } };
}
+ console.log("Processed", Object.keys(definitions).length, "schemas in", Number(stepSw.elapsed().totalMilliseconds + "." + stepSw.elapsed().microseconds), "ms.");
+
+ console.log("Merging nested definitions into main definitions...");
+ let isNewLine = true;
+ for (const defKey in nestedDefinitions) {
+ if (!includesMatch(defKey, Included, false)) {
+ const bannedMatch = includesMatch(defKey, ExcludeAndWarn, false) ?? includesMatch(defKey, Excluded, false);
+ if (bannedMatch !== null) {
+ // console.log(yellowBright("\n[WARN]"), "Skipping nested definition", defKey, "as it matched a banned format.");
+ console.log((isNewLine ? "" : "\n") + redBright("WARNING") + " Excluding schema " + yellowBright(defKey) + " due to match with " + redBright(bannedMatch));
+ isNewLine = true;
+ continue;
+ }
+ }
+
+ nestedDefinitions[defKey]["$schema"] = "http://json-schema.org/draft-07/schema#";
+ if (definitions[defKey]) {
+ if (!deepEqual(definitions[defKey], nestedDefinitions[defKey])) {
+ if (Object.keys(definitions[defKey]).every((k) => k === "$ref" || k === "$schema")) {
+ definitions[defKey] = nestedDefinitions[defKey];
+ console.log(yellowBright("\nWARNING"), "Overwriting definition for", defKey, "with nested definition (ref/schema only).");
+ isNewLine = true;
+ } else {
+ console.log(redBright("\nERROR"), "Conflicting definition for", defKey, "found in main definitions.");
+ console.log(columnizedObjectDiff(definitions[defKey], nestedDefinitions[defKey], true));
+ console.log("Keys:", Object.keys(definitions[defKey]), Object.keys(nestedDefinitions[defKey]));
+ isNewLine = true;
+ }
+ } else {
+ // console.log("Definition", defKey, "is identical to existing definition, skipping.");
+ }
+ } else {
+ definitions[defKey] = nestedDefinitions[defKey];
+ if (isNewLine) {
+ process.stdout.write("Adding nested definitions to main definitions: ");
+ isNewLine = false;
+ } else process.stdout.write("\x1b[4D, ");
+ process.stdout.write(yellowBright(defKey) + "... ");
+ }
+ }
deleteOneOfKindUndefinedRecursive(definitions, "$");
+ if (process.env.WRITE_SCHEMA_DIR === "true") {
+ await Promise.all(writePromises);
+ await Promise.all(
+ Object.keys(definitions).map(async (name) => {
+ await fsp.writeFile(path.join("schemas_final", `${name}.json`), JSON.stringify(definitions[name], null, 4));
+ // console.log("Wrote schema", name, "to schemas/");
+ }),
+ );
+ await Promise.all(
+ Object.keys(nestedDefinitions).map(async (name) => {
+ await fsp.writeFile(path.join("schemas_nested", `${name}.json`), JSON.stringify(nestedDefinitions[name], null, 4));
+ // console.log("Wrote schema", name, "to schemas_nested/");
+ }),
+ );
+ }
+
fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4));
fs.writeFileSync(__dirname + "/schemaExclusions.json", JSON.stringify(exclusionList, null, 4));
- console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath, "in", new Date() - scriptStartTime, "ms.");
+ console.log("\nSuccessfully wrote", Object.keys(definitions).length, "schemas to", schemaPath, "in", Number(totalSw.elapsed().totalMilliseconds + "." + totalSw.elapsed().microseconds), "ms,", fs.statSync(schemaPath).size, "bytes.");
}
function deleteOneOfKindUndefinedRecursive(obj, path) {
@@ -212,4 +320,56 @@ function deleteOneOfKindUndefinedRecursive(obj, path) {
return false;
}
+function filterSchema(schema) {
+ // this is a hack. we may want to check if its a @column instead
+ if (schema.properties) {
+ for (let key in schema.properties) {
+ if (baseClassProperties.includes(key)) {
+ delete schema.properties[key];
+ }
+ }
+ }
+
+ if (schema.required) schema.required = schema.required.filter((x) => !baseClassProperties.includes(x));
+
+ // recurse into own definitions
+ if (schema.definitions)
+ for (const defKey in schema.definitions) {
+ filterSchema(schema.definitions[defKey]);
+ }
+}
+
+function deepEqual(a, b) {
+ if (a === b) return true;
+
+ if (typeof a !== "object" || typeof b !== "object" || a == null || b == null) {
+ return false;
+ }
+
+ const keysA = Object.keys(a);
+ const keysB = Object.keys(b);
+
+ if (keysA.length !== keysB.length) return false;
+
+ for (const key of keysA) {
+ if (!keysB.includes(key) || !deepEqual(a[key], b[key])) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+function columnizedObjectDiff(a, b, trackEqual = false) {
+ const diffs = { left: {}, right: {}, ...(trackEqual ? { equal: {} } : {}) };
+ const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
+ for (const key of keys) {
+ if (!deepEqual(a[key], b[key])) {
+ diffs.left[key] = a[key];
+ diffs.right[key] = b[key];
+ } else if (trackEqual) diffs.equal[key] = a[key];
+ }
+ return diffs;
+}
+
main();
diff --git a/scripts/schemaExclusions.json b/scripts/schemaExclusions.json
index 8eb7955f..72a010f8 100644
--- a/scripts/schemaExclusions.json
+++ b/scripts/schemaExclusions.json
@@ -2,7 +2,9 @@
"include": [
"MessageInteractionSchema"
],
- "includeRe": [],
+ "includeRe": [
+ "^MessageComponentType\\..*"
+ ],
"manual": [
"DefaultSchema",
"Schema",
@@ -40,7 +42,9 @@
"ListTagsForResourceResponse",
"GetContactResponse",
"LibraryResponse",
- "LibraryLocalResponse"
+ "LibraryLocalResponse",
+ "SchemaTraits",
+ "TraitsSchema"
],
"manualRe": [
".*\\.Response$",
@@ -48,8 +52,6 @@
".*\\..*",
"^Axios",
"^Internal",
- "^Record<",
- "^Omit<",
"^ListContact(s|Lists)Response$",
"^APIKeyConfiguration\\..*",
"^AccountSetting\\..*",
@@ -82,8 +84,7 @@
],
"manualWarn": [],
"manualWarnRe": [
- "^Record",
- "^Partial"
+ ".*<.*>$"
],
"auto": [
{
diff --git a/scripts/util/getRouteDescriptions.js b/scripts/util/getRouteDescriptions.js
index 54c8fe19..9f329f32 100644
--- a/scripts/util/getRouteDescriptions.js
+++ b/scripts/util/getRouteDescriptions.js
@@ -2,7 +2,7 @@ const express = require("express");
const path = require("path");
const { traverseDirectory } = require("lambert-server");
const RouteUtility = require("../../dist/api/util/handlers/route.js");
-const { bgRedBright } = require("picocolors");
+const { bgRedBright, greenBright, yellowBright, blueBright, redBright, underline, bold } = require("picocolors");
const methods = ["get", "post", "put", "delete", "patch"];
const routes = new Map();
@@ -14,23 +14,47 @@ let currentPath = "";
If someone could fix that I'd really appreciate it, but for now just, don't do that :p
*/
+function colorizeMethod(method) {
+ switch (method.toLowerCase()) {
+ case "get":
+ return greenBright(method.toUpperCase());
+ case "post":
+ return yellowBright(method.toUpperCase());
+ case "put":
+ return blueBright(method.toUpperCase());
+ case "delete":
+ return redBright(method.toUpperCase());
+ case "patch":
+ return yellowBright(method.toUpperCase());
+ default:
+ return method.toUpperCase();
+ }
+}
+
+function formatPath(path) {
+ return path
+ .replace(/:(\w+)/g, underline(":$1"))
+ .replace(/#(\w+)/g, underline("#$1"))
+ ;
+}
+
/**
* @param {string} file
- * @param {string} method
- * @param {string} prefix
- * @param {string} path
+ * @param {string} apiMethod
+ * @param {string} apiPathPrefix
+ * @param {string} apiPath
* @param args
*/
-function proxy(file, method, prefix, path, ...args) {
+function proxy(file, apiMethod, apiPathPrefix, apiPath, ...args) {
const opts = args.find((x) => x?.prototype?.OPTS_MARKER == true);
if (!opts)
return console.error(
- `${bgRedBright("ERROR")} ${file} has route without route() description middleware`,
+ ` \x1b[5m${bgRedBright("ERROR")}\x1b[25m ${file.replace(path.resolve(__dirname, "..", "..", "dist"), "/src/")} has route without route() description middleware: ${colorizeMethod(apiMethod)} ${formatPath(apiPath)}`,
);
- console.log(`${method.toUpperCase().padStart("OPTIONS".length)} ${prefix + path}`);
+ console.log(`${colorizeMethod(apiMethod).padStart("DELETE".length + 10)} ${formatPath(apiPathPrefix + apiPath)}`);
opts.file = file.replace("/dist/", "/src/").replace(".js", ".ts");
- routes.set(prefix + path + "|" + method, opts());
+ routes.set(apiPathPrefix + apiPath + "|" + apiMethod, opts());
}
express.Router = () => {
@@ -56,7 +80,7 @@ module.exports = function getRouteDescriptions() {
currentFile = file;
currentPath = file.replace(root.slice(0, -1), "");
- currentPath = currentPath.split(".").slice(0, -1).join("."); // trancate .js/.ts file extension of path
+ currentPath = currentPath.split(".").slice(0, -1).join("."); // truncate .js/.ts file extension of path
currentPath = currentPath.replaceAll("#", ":").replaceAll("\\", "/"); // replace # with : for path parameters and windows paths with slashes
if (currentPath.endsWith("/index"))
currentPath = currentPath.slice(0, "/index".length * -1); // delete index from path
|
