diff --git a/scripts/configTemplates/bundle.json b/scripts/configTemplates/bundle.json
new file mode 100644
index 00000000..e975a1d9
--- /dev/null
+++ b/scripts/configTemplates/bundle.json
@@ -0,0 +1,15 @@
+{
+ "gateway": {
+ "endpointPublic": "ws://localhost:3001"
+ },
+ "cdn": {
+ "endpointPrivate": "http://localhost:3001",
+ "endpointPublic": "http://localhost:3001"
+ },
+ "api": {
+ "endpointPublic": "http://localhost:3001/api/v9/"
+ },
+ "rabbitmq": {
+ "host": null
+ }
+}
\ No newline at end of file
diff --git a/scripts/configTemplates/split.json b/scripts/configTemplates/split.json
new file mode 100644
index 00000000..f58adc8d
--- /dev/null
+++ b/scripts/configTemplates/split.json
@@ -0,0 +1,15 @@
+{
+ "api": {
+ "endpointPublic": "http://localhost:3001/api/v9/"
+ },
+ "cdn": {
+ "endpointPublic": "http://localhost:3003",
+ "endpointPrivate": "http://localhost:3003"
+ },
+ "gateway": {
+ "endpointPublic": "ws://localhost:3002"
+ },
+ "rabbitmq": {
+ "host": "amqp://guest:guest@localhost/"
+ }
+}
\ No newline at end of file
diff --git a/src/applyConfig.ts b/src/applyConfig.ts
new file mode 100755
index 00000000..e414b2a2
--- /dev/null
+++ b/src/applyConfig.ts
@@ -0,0 +1,19 @@
+import { bgRedBright } from "picocolors";
+
+require("dotenv").config({ quiet: true });
+import moduleAlias from "module-alias";
+moduleAlias();
+
+process.env.CONFIG_PATH = process.argv[2] || "";
+process.env.CONFIG_MODE = "overwrite";
+process.env.CONFIG_WRITEBACK = "false";
+
+import { initDatabase } from "@spacebar/util";
+import { Config } from "@spacebar/util";
+import { EnvConfig } from "@spacebar/util";
+
+(async () => {
+ console.log("Env config:", JSON.stringify(EnvConfig.get(), null, 2));
+ await initDatabase();
+ await Config.init();
+})();
diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts
index 1ca455ff..c0e1e09c 100644
--- a/src/util/util/Config.ts
+++ b/src/util/util/Config.ts
@@ -18,180 +18,245 @@
import { existsSync } from "fs";
import fs from "fs/promises";
-import { OrmUtils } from "..";
+import { EnvConfig, OrmUtils } from "..";
import { ConfigValue } from "../config";
import { ConfigEntity } from "../entities/Config";
import { JsonValue } from "@protobuf-ts/runtime";
+import { yellow, yellowBright } from "picocolors";
-// TODO: yaml instead of json
-const overridePath = process.env.CONFIG_PATH ?? "";
-
-let config: ConfigValue;
-let pairs: ConfigEntity[];
+let cachedConfig: ConfigValue;
+let cachedConfigWithOverrides: ConfigValue | undefined;
+let cachedPairs: ConfigEntity[];
// TODO: use events to inform about config updates
// Config keys are separated with _
export class Config {
- public static async init(force: boolean = false) {
- if (config && !force) return config;
- console.log("[Config] Loading configuration...");
- if (!process.env.CONFIG_PATH) {
- pairs = await validateConfig();
- config = pairsToConfig(pairs);
- } else {
- console.log(`[Config] Using CONFIG_PATH rather than database`);
- if (existsSync(process.env.CONFIG_PATH)) {
- const file = JSON.parse((await fs.readFile(process.env.CONFIG_PATH)).toString());
- config = file;
- } else config = new ConfigValue();
- pairs = generatePairs(config);
- }
+ public static async init(force: boolean = false) {
+ if ((cachedConfigWithOverrides || cachedConfig) && !force) return cachedConfigWithOverrides ?? cachedConfig;
+ console.log("[Config] Loading configuration...");
+
+ const { path: jsonPath, enabled: jsonEnabled, mode: jsonMode, writebackEnabled: jsonWritebackEnabled } = EnvConfig.get().configuration;
+ let jsonConfig: ConfigValue;
+
+ if (jsonEnabled) console.log(`[Config] Using JSON configuration file at ${jsonPath} in '${jsonMode}' mode.`);
+
+ if (jsonEnabled && jsonMode === "single") {
+ if (!existsSync(jsonPath)) throw new Error(`[Config] CONFIG_PATH does not exist, but CONFIG_MODE is set to 'single'. Please ensure the file at ${jsonPath} exists.`);
+ jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString());
+
+ // merge with defaults to allow partial configs
+ cachedConfig = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, jsonConfig);
+ return await saveConfig(cachedConfig); // handle writeback if enabled
+ }
+
+ if (jsonEnabled && existsSync(jsonPath) && jsonMode === "overwrite") {
+ console.log(`[Config] Loading configuration from JSON file at ${jsonPath}...`);
+ jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString());
+ console.log("[Config] Overwriting database configuration with JSON configuration...");
+ await saveConfigToDatabaseAtomic(jsonConfig);
+ }
+
+ console.log("[Config] Loading configuration from database...");
+ cachedPairs = await validateConfig();
+ cachedConfig = pairsToConfig(cachedPairs);
+
+ // If a config doesn't exist, create it.
+ if (Object.keys(cachedConfig).length == 0) cachedConfig = new ConfigValue();
+
+ cachedConfig = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, cachedConfig);
+
+ let ret = await this.set(cachedConfig);
+
+ if (jsonEnabled && existsSync(jsonPath) && jsonMode === "override") {
+ console.log(`[Config] Loading configuration from JSON file at ${jsonPath}...`);
+ jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString());
+ console.log("[Config] Overriding database configuration values with JSON configuration...");
+ ret = cachedConfigWithOverrides = OrmUtils.mergeDeep({}, cachedConfig, jsonConfig);
+ }
- // If a config doesn't exist, create it.
- if (Object.keys(config).length == 0) config = new ConfigValue();
+ const changes = getChanges(ret!);
+ for (const [key, change] of Object.entries(changes)) {
+ if (change.type === "changed") {
+ console.log(yellowBright(`[Config] Setting '${key}' has been changed from '${JSON.stringify(change.old)}' to '${JSON.stringify(change.new)}'`));
+ } else if (change.type === "unknown") {
+ console.log(yellow(`[Config] Unknown setting '${key}' with value '${JSON.stringify(change.new)}'`));
+ }
+ }
- config = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, config);
+ validateFinalConfig(ret!);
+ return ret;
+ }
- await this.set(config);
- validateFinalConfig(config);
- return config;
- }
- public static get() {
- if (!config) {
- // If we haven't initialised the config yet, return default config.
- // Typeorm instantiates each entity once when initialising database,
- // which means when we use config values as default values in entity classes,
- // the config isn't initialised yet and would throw an error about the config being undefined.
+ public static get() {
+ if (!cachedConfig) {
+ // If we haven't initialised the config yet, return default config.
+ // Typeorm instantiates each entity once when initialising database,
+ // which means when we use config values as default values in entity classes,
+ // the config isn't initialised yet and would throw an error about the config being undefined.
- return new ConfigValue();
- }
+ return new ConfigValue();
+ }
- return config;
- }
- public static set(val: Partial<ConfigValue>) {
- if (!config || !val) return;
- config = OrmUtils.mergeDeep(config);
+ return cachedConfigWithOverrides ?? cachedConfig;
+ }
+ public static set(val: Partial<ConfigValue>) {
+ if (!cachedConfig || !val) return;
+ cachedConfig = OrmUtils.mergeDeep(cachedConfig, val);
- return applyConfig(config);
- }
+ return saveConfig(val);
+ }
}
// TODO: better types
-const generatePairs = (obj: object | null, key = ""): ConfigEntity[] => {
- if (typeof obj == "object" && obj != null) {
- return Object.keys(obj)
- .map((k) =>
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- generatePairs((obj as any)[k], key ? `${key}_${k}` : k),
- )
- .flat();
- }
+function generatePairs(obj: object | null, key = ""): ConfigEntity[] {
+ if (typeof obj == "object" && obj != null) {
+ return Object.keys(obj)
+ .map((k) =>
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ generatePairs((obj as any)[k], key ? `${key}_${k}` : k),
+ )
+ .flat();
+ }
- const ret = new ConfigEntity();
- ret.key = key;
- ret.value = obj;
- return [ret];
-};
+ const ret = new ConfigEntity();
+ ret.key = key;
+ ret.value = obj;
+ return [ret];
+}
+
+async function saveConfig(val: Partial<ConfigValue>) {
+ const merged = OrmUtils.mergeDeep({}, cachedConfig, val) as ConfigValue;
+ if (EnvConfig.get().configuration.enabled) {
+ if (EnvConfig.get().configuration.writebackEnabled) {
+ await fs.writeFile(EnvConfig.get().configuration.path, JSON.stringify(merged, null, 4));
+ } else {
+ console.log("[Config/WARN] JSON config file in use, and writeback is disabled!");
+ console.log("[Config/WARN] Programmatic config changes will not be persisted, and your config will not get updated!");
+ console.log("[Config/WARN] Please check regularly to make adjustments as necessary!");
+ }
-async function applyConfig(val: ConfigValue) {
- if (process.env.CONFIG_PATH)
- if (!process.env.CONFIG_READONLY) await fs.writeFile(overridePath, JSON.stringify(val, null, 4));
- else console.log("[WARNING] JSON config file in use, and writing is disabled! Programmatic config changes will not be persisted, and your config will not get updated!");
- else {
- const pairs = generatePairs(val);
- // keys are sorted to try to influence database order...
- await Promise.all(pairs.sort((x, y) => (x.key > y.key ? 1 : -1)).map((pair) => pair.save()));
- }
- return val;
+ if (EnvConfig.get().configuration.mode == "overwrite") await saveConfigToDatabaseAtomic(val);
+ } else await saveConfigToDatabaseAtomic(val); // not using a JSON file
+
+ return merged;
+}
+
+// Atomically save changes to database
+async function saveConfigToDatabaseAtomic(val: Partial<ConfigValue>) {
+ const pairs = generatePairs(val);
+ const pairsToUpdate =
+ cachedPairs === undefined ? pairs : pairs.filter((p) => cachedPairs.some((cp) => cp.key === p.key && JSON.stringify(cp.value) !== JSON.stringify(p.value)));
+
+ if (pairsToUpdate.length > 0) console.log("[Config] Atomic update:", pairsToUpdate);
+
+ // keys are sorted to try to influence database order...
+ await Promise.all(pairsToUpdate.sort((x, y) => (x.key > y.key ? 1 : -1)).map((pair) => pair.save()));
}
function pairsToConfig(pairs: ConfigEntity[]) {
- // TODO: typings
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const value: any = {};
+ // TODO: typings
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const value: any = {};
- pairs.forEach((p) => {
- const keys = p.key.split("_");
- let obj = value;
- let prev = "";
- let prevObj = obj;
- let i = 0;
+ pairs.forEach((p) => {
+ const keys = p.key.split("_");
+ let obj = value;
+ let prev = "";
+ let prevObj = obj;
+ let i = 0;
- for (const key of keys) {
- if (!isNaN(Number(key)) && !prevObj[prev]?.length) prevObj[prev] = obj = [];
- if (i++ === keys.length - 1) obj[key] = p.value;
- else if (!obj[key]) obj[key] = {};
+ for (const key of keys) {
+ if (!isNaN(Number(key)) && !prevObj[prev]?.length) prevObj[prev] = obj = [];
+ if (i++ === keys.length - 1) obj[key] = p.value;
+ else if (!obj[key]) obj[key] = {};
- prev = key;
- prevObj = obj;
- obj = obj[key];
- }
- });
+ prev = key;
+ prevObj = obj;
+ obj = obj[key];
+ }
+ });
- return value as ConfigValue;
+ return value as ConfigValue;
}
const validateConfig = async () => {
- let hasErrored = false;
- const totalStartTime = new Date();
- const config = await ConfigEntity.find({ select: { key: true } });
+ let hasErrored = false;
+ const totalStartTime = new Date();
+ const config = await ConfigEntity.find({ select: { key: true }, order: { key: "ASC" } });
- for (const row in config) {
- // extension methods...
- if (typeof config[row] === "function") continue;
+ for (const row in config) {
+ // extension methods...
+ if (typeof config[row] === "function") continue;
- try {
- const found = await ConfigEntity.findOne({
- where: { key: config[row].key },
- });
- if (!found) continue;
- config[row] = found;
- } catch (e) {
- console.error(`Config key '${config[row].key}' has invalid JSON value : ${(e as Error)?.message}`);
- hasErrored = true;
- }
- }
+ try {
+ const found = await ConfigEntity.findOne({
+ where: { key: config[row].key },
+ });
+ if (!found) continue;
+ config[row] = found;
+ } catch (e) {
+ console.error(`Config key '${config[row].key}' has invalid JSON value: ${(e as Error)?.message}`);
+ hasErrored = true;
+ }
+ }
- console.log("[Config] Total config load time:", new Date().getTime() - totalStartTime.getTime(), "ms");
+ console.log("[Config] Total config load time:", new Date().getTime() - totalStartTime.getTime(), "ms");
- if (hasErrored) {
- console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration");
- process.exit(1);
- }
+ if (hasErrored) {
+ console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration");
+ process.exit(1);
+ }
- return config;
+ return config;
};
function validateFinalConfig(config: ConfigValue) {
- let hasErrors = false;
- function assertConfig(path: string, condition: (val: JsonValue) => boolean, recommendedValue: string) {
- // _ to separate keys
- const keys = path.split("_");
- let obj: never = config as never;
+ let hasErrors = false;
+ function assertConfig(path: string, condition: (val: JsonValue) => boolean, recommendedValue: string) {
+ // _ to separate keys
+ const keys = path.split("_");
+ let obj: never = config as never;
+
+ for (const key of keys) {
+ if (obj == null || !(key in obj)) {
+ console.warn(`[Config] Missing config value for '${path}'. Recommended value: ${recommendedValue}`);
+ return;
+ }
+ obj = obj[key];
+ }
- for (const key of keys) {
- if (obj == null || !(key in obj)) {
- console.warn(`[Config] Missing config value for '${path}'. Recommended value: ${recommendedValue}`);
- return;
- }
- obj = obj[key];
- }
+ if (!condition(obj)) {
+ console.warn(`[Config] Invalid config value for '${path}': ${obj}. Recommended value: ${recommendedValue}`);
+ hasErrors = true;
+ }
+ }
- if (!condition(obj)) {
- console.warn(`[Config] Invalid config value for '${path}': ${obj}. Recommended value: ${recommendedValue}`);
- hasErrors = true;
- }
- }
+ assertConfig("api_endpointPublic", (v) => v != null, 'A valid public API endpoint URL, ex. "http://localhost:3001/api/v9"');
+ assertConfig("cdn_endpointPublic", (v) => v != null, 'A valid public CDN endpoint URL, ex. "http://localhost:3003/"');
+ assertConfig("cdn_endpointPrivate", (v) => v != null, 'A valid private CDN endpoint URL, ex. "http://localhost:3003/" - must be routable from the API server!');
+ assertConfig("gateway_endpointPublic", (v) => v != null, 'A valid public gateway endpoint URL, ex. "ws://localhost:3002/"');
+
+ if (hasErrors) {
+ console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration");
+ console.error("[Config] Hint: if you're just testing with bundle (`npm run start`), you can set all endpoint URLs to [proto]://localhost:3001");
+ process.exit(1);
+ } else console.log("[Config] Configuration validated successfully.");
+}
- assertConfig("api_endpointPublic", (v) => v != null, 'A valid public API endpoint URL, ex. "http://localhost:3001/api/v9"');
- assertConfig("cdn_endpointPublic", (v) => v != null, 'A valid public CDN endpoint URL, ex. "http://localhost:3003/"');
- assertConfig("cdn_endpointPrivate", (v) => v != null, 'A valid private CDN endpoint URL, ex. "http://localhost:3003/" - must be routable from the API server!');
- assertConfig("gateway_endpointPublic", (v) => v != null, 'A valid public gateway endpoint URL, ex. "ws://localhost:3002/"');
+function getChanges(config: ConfigValue) {
+ const defaultPairs = generatePairs(new ConfigValue());
+ const newPairs = generatePairs(cachedConfig);
+ const ignoredKeys = ["general_instanceId", "security_requestSignature", "security_jwtSecret", "security_cdnSignatureKey"];
+ const changes: { [key: string]: { type: "changed" | "unknown"; old?: string | number | boolean | null | undefined; new: string | number | boolean | null | undefined } } = {};
- if (hasErrors) {
- console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration");
- console.error("[Config] Hint: if you're just testing with bundle (`npm run start`), you can set all endpoint URLs to [proto]://localhost:3001");
- process.exit(1);
- } else console.log("[Config] Configuration validated successfully.");
+ for (const newPair of newPairs) {
+ const defaultPair = defaultPairs.find((p) => p.key === newPair.key);
+ if (defaultPair && JSON.stringify(defaultPair.value) !== JSON.stringify(newPair.value) && !ignoredKeys.includes(newPair.key)) {
+ changes[newPair.key] = { type: "changed", old: defaultPair.value, new: newPair.value };
+ } else if (!defaultPair) {
+ changes[newPair.key] = { type: "unknown", new: newPair.value };
+ }
+ }
+ return changes;
}
|
