summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--scripts/benchmark/connections.js82
-rw-r--r--scripts/stress/identify.js53
-rw-r--r--scripts/stress/users.js (renamed from scripts/benchmark/users.js)0
-rw-r--r--src/gateway/opcodes/Heartbeat.ts2
-rw-r--r--src/gateway/opcodes/Identify.ts552
-rw-r--r--src/gateway/util/Capabilities.ts26
-rw-r--r--src/gateway/util/WebSocket.ts2
-rw-r--r--src/gateway/util/index.ts1
-rw-r--r--src/util/dtos/ReadyGuildDTO.ts70
-rw-r--r--src/util/entities/Channel.ts7
-rw-r--r--src/util/entities/Guild.ts1
-rw-r--r--src/util/entities/Role.ts3
-rw-r--r--src/util/entities/User.ts9
-rw-r--r--src/util/interfaces/Event.ts36
14 files changed, 514 insertions, 330 deletions
diff --git a/scripts/benchmark/connections.js b/scripts/benchmark/connections.js
deleted file mode 100644
index 4246c646..00000000
--- a/scripts/benchmark/connections.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
-	Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
-	Copyright (C) 2023 Spacebar and Spacebar Contributors
-	
-	This program is free software: you can redistribute it and/or modify
-	it under the terms of the GNU Affero General Public License as published
-	by the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-	
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU Affero General Public License for more details.
-	
-	You should have received a copy of the GNU Affero General Public License
-	along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
-require("dotenv").config();
-const cluster = require("cluster");
-const WebSocket = require("ws");
-const endpoint = process.env.GATEWAY || "ws://localhost:3001";
-const connections = Number(process.env.CONNECTIONS) || 50;
-const token = process.env.TOKEN;
-var cores = 1;
-try {
-	cores = Number(process.env.THREADS) || os.cpus().length;
-} catch {
-	console.log("[Bundle] Failed to get thread count! Using 1...");
-}
-
-if (!token) {
-	console.error("TOKEN env var missing");
-	process.exit();
-}
-
-if (cluster.isMaster) {
-	for (let i = 0; i < cores; i++) {
-		cluster.fork();
-	}
-
-	cluster.on("exit", (worker, code, signal) => {
-		console.log(`worker ${worker.process.pid} died`);
-	});
-} else {
-	for (let i = 0; i < connections; i++) {
-		connect();
-	}
-}
-
-function connect() {
-	const client = new WebSocket(endpoint);
-	client.on("message", (data) => {
-		data = JSON.parse(data);
-
-		switch (data.op) {
-			case 10:
-				client.interval = setInterval(() => {
-					client.send(JSON.stringify({ op: 1 }));
-				}, data.d.heartbeat_interval);
-
-				client.send(
-					JSON.stringify({
-						op: 2,
-						d: {
-							token,
-							properties: {},
-						},
-					}),
-				);
-
-				break;
-		}
-	});
-	client.once("close", (code, reason) => {
-		clearInterval(client.interval);
-		connect();
-	});
-	client.on("error", (err) => {
-		// console.log(err);
-	});
-}
diff --git a/scripts/stress/identify.js b/scripts/stress/identify.js
new file mode 100644
index 00000000..7dea5e4d
--- /dev/null
+++ b/scripts/stress/identify.js
@@ -0,0 +1,53 @@
+/* eslint-env node */
+
+require("dotenv").config();
+const { OPCODES } = require("../../dist/gateway/util/Constants.js");
+const WebSocket = require("ws");
+const ENDPOINT = `ws://localhost:3002?v=9&encoding=json`;
+const TOKEN =
+	"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjEwOTMxMTgwMjgzNjA1MzYxMDYiLCJpYXQiOjE2ODA2OTE5MDB9.9ByCqDvC4mIutW8nM7WhVCtGuKW08UimPnmBeNw-K0E";
+const TOTAL_ITERATIONS = 500;
+
+const doTimedIdentify = () =>
+	new Promise((resolve) => {
+		let start;
+		const ws = new WebSocket(ENDPOINT);
+		ws.on("message", (data) => {
+			const parsed = JSON.parse(data);
+
+			switch (parsed.op) {
+				case OPCODES.Hello:
+					// send identify
+					start = performance.now();
+					ws.send(
+						JSON.stringify({
+							op: OPCODES.Identify,
+							d: {
+								token: TOKEN,
+								properties: {},
+							},
+						}),
+					);
+					break;
+				case OPCODES.Dispatch:
+					if (parsed.t == "READY") {
+						ws.close();
+						return resolve(performance.now() - start);
+					}
+
+					break;
+			}
+		});
+	});
+
+(async () => {
+	const perfs = [];
+	while (perfs.length < TOTAL_ITERATIONS) {
+		const ret = await doTimedIdentify();
+		perfs.push(ret);
+		// console.log(`${perfs.length}/${TOTAL_ITERATIONS} - this: ${Math.floor(ret)}ms`)
+	}
+
+	const avg = perfs.reduce((prev, curr) => prev + curr) / (perfs.length - 1);
+	console.log(`Average identify time: ${Math.floor(avg * 100) / 100}ms`);
+})();
diff --git a/scripts/benchmark/users.js b/scripts/stress/users.js
index 20f9f7c3..20f9f7c3 100644
--- a/scripts/benchmark/users.js
+++ b/scripts/stress/users.js
diff --git a/src/gateway/opcodes/Heartbeat.ts b/src/gateway/opcodes/Heartbeat.ts
index 7866c3e9..b9b62be3 100644
--- a/src/gateway/opcodes/Heartbeat.ts
+++ b/src/gateway/opcodes/Heartbeat.ts
@@ -25,5 +25,5 @@ export async function onHeartbeat(this: WebSocket) {
 
 	setHeartbeat(this);
 
-	await Send(this, { op: 11 });
+	await Send(this, { op: 11, d: {} });
 }
diff --git a/src/gateway/opcodes/Identify.ts b/src/gateway/opcodes/Identify.ts
index 98fae3ed..5816c308 100644
--- a/src/gateway/opcodes/Identify.ts
+++ b/src/gateway/opcodes/Identify.ts
@@ -16,7 +16,15 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { WebSocket, Payload } from "@spacebar/gateway";
+import {
+	WebSocket,
+	Payload,
+	setupListener,
+	Capabilities,
+	CLOSECODES,
+	OPCODES,
+	Send,
+} from "@spacebar/gateway";
 import {
 	checkToken,
 	Intents,
@@ -26,7 +34,6 @@ import {
 	Session,
 	EVENTEnum,
 	Config,
-	PublicMember,
 	PublicUser,
 	PrivateUserProjection,
 	ReadState,
@@ -36,301 +43,390 @@ import {
 	PrivateSessionProjection,
 	MemberPrivateProjection,
 	PresenceUpdateEvent,
-	UserSettings,
 	IdentifySchema,
 	DefaultUserGuildSettings,
-	UserGuildSettings,
 	ReadyGuildDTO,
 	Guild,
-	UserTokenData,
-	ConnectedAccount,
+	PublicUserProjection,
+	ReadyUserGuildSettingsEntries,
+	UserSettings,
+	Permissions,
+	DMChannel,
+	GuildOrUnavailable,
+	Recipient,
 } from "@spacebar/util";
-import { Send } from "../util/Send";
-import { CLOSECODES, OPCODES } from "../util/Constants";
-import { setupListener } from "../listener/listener";
-// import experiments from "./experiments.json";
-const experiments: unknown[] = [];
 import { check } from "./instanceOf";
-import { Recipient } from "@spacebar/util";
 
 // TODO: user sharding
 // TODO: check privileged intents, if defined in the config
-// TODO: check if already identified
 
-// TODO: Refactor identify ( and lazyrequest, tbh )
+const getUserFromToken = async (token: string): Promise<string | null> => {
+	try {
+		const { jwtSecret } = Config.get().security;
+		const { decoded } = await checkToken(token, jwtSecret);
+		return decoded.id;
+	} catch (e) {
+		console.error(`[Gateway] Invalid token`, e);
+		return null;
+	}
+};
 
 export async function onIdentify(this: WebSocket, data: Payload) {
+	if (this.user_id) {
+		// we've already identified
+		return this.close(CLOSECODES.Already_authenticated);
+	}
+
 	clearTimeout(this.readyTimeout);
-	// TODO: is this needed now that we use `json-bigint`?
-	if (typeof data.d?.client_state?.highest_last_message_id === "number")
-		data.d.client_state.highest_last_message_id += "";
-	check.call(this, IdentifySchema, data.d);
 
+	// Check payload matches schema
+	check.call(this, IdentifySchema, data.d);
 	const identify: IdentifySchema = data.d;
 
-	let decoded: UserTokenData["decoded"];
-	try {
-		const { jwtSecret } = Config.get().security;
-		decoded = (await checkToken(identify.token, jwtSecret)).decoded; // will throw an error if invalid
-	} catch (error) {
-		console.error("invalid token", error);
-		return this.close(CLOSECODES.Authentication_failed);
-	}
-	this.user_id = decoded.id;
-	const session_id = this.session_id;
-
-	const [
-		user,
-		read_states,
-		members,
-		recipients,
-		session,
-		application,
-		connected_accounts,
-	] = await Promise.all([
-		User.findOneOrFail({
-			where: { id: this.user_id },
-			relations: ["relationships", "relationships.to", "settings"],
-			select: [...PrivateUserProjection, "relationships"],
-		}),
-		ReadState.find({ where: { user_id: this.user_id } }),
-		Member.find({
-			where: { id: this.user_id },
-			select: MemberPrivateProjection,
-			relations: [
-				"guild",
-				"guild.channels",
-				"guild.emojis",
-				"guild.roles",
-				"guild.stickers",
-				"user",
-				"roles",
-			],
-		}),
-		Recipient.find({
-			where: { user_id: this.user_id, closed: false },
-			relations: [
-				"channel",
-				"channel.recipients",
-				"channel.recipients.user",
-			],
-			// TODO: public user selection
-		}),
-		// save the session and delete it when the websocket is closed
-		Session.create({
-			user_id: this.user_id,
-			session_id: session_id,
-			// TODO: check if status is only one of: online, dnd, offline, idle
-			status: identify.presence?.status || "offline", //does the session always start as online?
-			client_info: {
-				//TODO read from identity
-				client: "desktop",
-				os: identify.properties?.os,
-				version: 0,
-			},
-			activities: [],
-		}).save(),
-		Application.findOne({ where: { id: this.user_id } }),
-		ConnectedAccount.find({ where: { user_id: this.user_id } }),
-	]);
+	this.capabilities = new Capabilities(identify.capabilities || 0);
 
-	if (!user) return this.close(CLOSECODES.Authentication_failed);
-	if (!user.settings) {
-		user.settings = new UserSettings();
-		await user.settings.save();
-	}
+	// Check auth
+	// TODO: the checkToken call will fetch user, and then we have to refetch with different select
+	// checkToken should be able to select what we want
+	const user_id = await getUserFromToken(identify.token);
+	if (!user_id) return this.close(CLOSECODES.Authentication_failed);
+	this.user_id = user_id;
 
-	if (!identify.intents) identify.intents = BigInt("0x6ffffffff");
+	// Check intents
+	if (!identify.intents) identify.intents = 30064771071n; // TODO: what is this number?
 	this.intents = new Intents(identify.intents);
+
+	// TODO: actually do intent things.
+
+	// Validate sharding
 	if (identify.shard) {
 		this.shard_id = identify.shard[0];
 		this.shard_count = identify.shard[1];
+
 		if (
 			this.shard_count == null ||
 			this.shard_id == null ||
-			this.shard_id >= this.shard_count ||
+			this.shard_id > this.shard_count ||
 			this.shard_id < 0 ||
 			this.shard_count <= 0
 		) {
-			console.log(identify.shard);
+			// TODO: why do we even care about this right now?
+			console.log(
+				`[Gateway] Invalid sharding from ${user_id}: ${identify.shard}`,
+			);
 			return this.close(CLOSECODES.Invalid_shard);
 		}
 	}
-	let users: PublicUser[] = [];
 
-	const merged_members = members.map((x: Member) => {
+	// Generate a new gateway session ( id is already made, just save it in db )
+	const session = Session.create({
+		user_id: this.user_id,
+		session_id: this.session_id,
+		status: identify.presence?.status || "online",
+		client_info: {
+			client: identify.properties?.$device,
+			os: identify.properties?.os,
+			version: 0,
+		},
+		activities: identify.presence?.activities, // TODO: validation
+	});
+
+	// Get from database:
+	// * the current user,
+	// * the users read states
+	// * guild members for this user
+	// * recipients ( dm channels )
+	// * the bot application, if it exists
+	const [, user, application, read_states, members, recipients] =
+		await Promise.all([
+			session.save(),
+
+			// TODO: Refactor checkToken to allow us to skip this additional query
+			User.findOneOrFail({
+				where: { id: this.user_id },
+				relations: ["relationships", "relationships.to", "settings"],
+				select: [...PrivateUserProjection, "relationships"],
+			}),
+
+			Application.findOne({
+				where: { id: this.user_id },
+				select: ["id", "flags"],
+			}),
+
+			ReadState.find({
+				where: { user_id: this.user_id },
+				select: [
+					"id",
+					"channel_id",
+					"last_message_id",
+					"last_pin_timestamp",
+					"mention_count",
+				],
+			}),
+
+			Member.find({
+				where: { id: this.user_id },
+				select: {
+					// We only want some member props
+					...Object.fromEntries(
+						MemberPrivateProjection.map((x) => [x, true]),
+					),
+					settings: true, // guild settings
+					roles: { id: true }, // the full role is fetched from the `guild` relation
+
+					// TODO: we don't really need every property of
+					// guild channels, emoji, roles, stickers
+					// but we do want almost everything from guild.
+					// How do you do that without just enumerating the guild props?
+					guild: true,
+				},
+				relations: [
+					"guild",
+					"guild.channels",
+					"guild.emojis",
+					"guild.roles",
+					"guild.stickers",
+					"roles",
+
+					// For these entities, `user` is always just the logged in user we fetched above
+					// "user",
+				],
+			}),
+
+			Recipient.find({
+				where: { user_id: this.user_id, closed: false },
+				relations: [
+					"channel",
+					"channel.recipients",
+					"channel.recipients.user",
+				],
+				select: {
+					channel: {
+						id: true,
+						flags: true,
+						// is_spam: true,	// TODO
+						last_message_id: true,
+						last_pin_timestamp: true,
+						type: true,
+						icon: true,
+						name: true,
+						owner_id: true,
+						recipients: {
+							// we don't actually need this ID or any other information about the recipient info,
+							// but typeorm does not select anything from the users relation of recipients unless we select
+							// at least one column.
+							id: true,
+							// We only want public user data for each dm channel
+							user: Object.fromEntries(
+								PublicUserProjection.map((x) => [x, true]),
+							),
+						},
+					},
+				},
+			}),
+		]);
+
+	// We forgot to migrate user settings from the JSON column of `users`
+	// to the `user_settings` table theyre in now,
+	// so for instances that migrated, users may not have a `user_settings` row.
+	if (!user.settings) {
+		user.settings = new UserSettings();
+		await user.settings.save();
+	}
+
+	// Generate merged_members
+	const merged_members = members.map((x) => {
 		return [
 			{
 				...x,
 				roles: x.roles.map((x) => x.id),
+
+				// add back user, which we don't fetch from db
+				// TODO: For guild profiles, this may need to be changed.
+				// TODO: The only field required in the user prop is `id`,
+				// but our types are annoying so I didn't bother.
+				user: user.toPublicUser(),
+
+				guild: {
+					id: x.guild.id,
+				},
 				settings: undefined,
-				guild: undefined,
 			},
 		];
-	}) as PublicMember[][];
-	// TODO: This type is bad.
-	let guilds: Partial<Guild>[] = members.map((x) => ({
-		...x.guild,
-		joined_at: x.joined_at,
-	}));
+	});
+
+	// Populated with guilds 'unavailable' currently
+	// Just for bots
+	const pending_guilds: Guild[] = [];
+
+	// Generate guilds list ( make them unavailable if user is bot )
+	const guilds: GuildOrUnavailable[] = members.map((member) => {
+		// filter guild channels we don't have permission to view
+		// TODO: check if this causes issues when the user is granted other roles?
+		member.guild.channels = member.guild.channels.filter((channel) => {
+			const perms = Permissions.finalPermission({
+				user: {
+					id: member.id,
+					roles: member.roles.map((x) => x.id),
+				},
+				guild: member.guild,
+				channel,
+			});
 
-	const pending_guilds: typeof guilds = [];
-	if (user.bot)
-		guilds = guilds.map((guild) => {
-			pending_guilds.push(guild);
-			return { id: guild.id, unavailable: true };
+			return perms.has("VIEW_CHANNEL");
 		});
 
-	// TODO: Rewrite this. Perhaps a DTO?
-	const user_guild_settings_entries = members.map((x) => ({
-		...DefaultUserGuildSettings,
-		...x.settings,
-		guild_id: x.guild.id,
-		channel_overrides: Object.entries(
-			x.settings.channel_overrides ?? {},
-		).map((y) => ({
-			...y[1],
-			channel_id: y[0],
-		})),
-	})) as unknown as UserGuildSettings[];
-
-	const channels = recipients.map((x) => {
-		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-		//@ts-ignore
-		x.channel.recipients = x.channel.recipients.map((x) =>
-			x.user.toPublicUser(),
-		);
-		//TODO is this needed? check if users in group dm that are not friends are sent in the READY event
-		users = users.concat(x.channel.recipients as unknown as User[]);
-		if (x.channel.isDm()) {
-			x.channel.recipients = x.channel.recipients?.filter(
-				(x) => x.id !== this.user_id,
-			);
+		if (user.bot) {
+			pending_guilds.push(member.guild);
+			return { id: member.guild.id, unavailable: true };
 		}
-		return x.channel;
-	});
 
-	for (const relation of user.relationships) {
-		const related_user = relation.to;
-		const public_related_user = {
-			username: related_user.username,
-			discriminator: related_user.discriminator,
-			id: related_user.id,
-			public_flags: related_user.public_flags,
-			avatar: related_user.avatar,
-			bot: related_user.bot,
-			bio: related_user.bio,
-			premium_since: user.premium_since,
-			premium_type: user.premium_type,
-			accent_color: related_user.accent_color,
+		return {
+			...member.guild.toJSON(),
+			joined_at: member.joined_at,
 		};
-		users.push(public_related_user);
-	}
+	});
+
+	// Generate user_guild_settings
+	const user_guild_settings_entries: ReadyUserGuildSettingsEntries[] =
+		members.map((x) => ({
+			...DefaultUserGuildSettings,
+			...x.settings,
+			guild_id: x.guild_id,
+			channel_overrides: Object.entries(
+				x.settings.channel_overrides ?? {},
+			).map((y) => ({
+				...y[1],
+				channel_id: y[0],
+			})),
+		}));
+
+	// Popultaed with users from private channels, relationships.
+	// Uses a set to dedupe for us.
+	const users: Set<PublicUser> = new Set();
+
+	// Generate dm channels from recipients list. Append recipients to `users` list
+	const channels = recipients
+		.filter(({ channel }) => channel.isDm())
+		.map((r) => {
+			// TODO: fix the types of Recipient
+			// Their channels are only ever private (I think) and thus are always DM channels
+			const channel = r.channel as DMChannel;
+
+			// Remove ourself from the list of other users in dm channel
+			channel.recipients = channel.recipients.filter(
+				(recipient) => recipient.user.id !== this.user_id,
+			);
+
+			const channelUsers = channel.recipients?.map((recipient) =>
+				recipient.user.toPublicUser(),
+			);
+
+			if (channelUsers && channelUsers.length > 0)
+				channelUsers.forEach((user) => users.add(user));
 
-	setImmediate(async () => {
-		// run in seperate "promise context" because ready payload is not dependent on those events
+			return {
+				id: channel.id,
+				flags: channel.flags,
+				last_message_id: channel.last_message_id,
+				type: channel.type,
+				recipients: channelUsers || [],
+				is_spam: false, // TODO
+			};
+		});
+
+	// From user relationships ( friends ), also append to `users` list
+	user.relationships.forEach((x) => users.add(x.to.toPublicUser()));
+
+	// Send SESSIONS_REPLACE and PRESENCE_UPDATE
+	const allSessions = (
+		await Session.find({
+			where: { user_id: this.user_id },
+			select: PrivateSessionProjection,
+		})
+	).map((x) => ({
+		// TODO how is active determined?
+		// in our lazy request impl, we just pick the 'most relevant' session
+		active: x.session_id == session.session_id,
+		activities: x.activities,
+		client_info: x.client_info,
+		// TODO: what does all mean?
+		session_id: x.session_id == session.session_id ? "all" : x.session_id,
+		status: x.status,
+	}));
+
+	Promise.all([
 		emitEvent({
 			event: "SESSIONS_REPLACE",
 			user_id: this.user_id,
-			data: await Session.find({
-				where: { user_id: this.user_id },
-				select: PrivateSessionProjection,
-			}),
-		} as SessionsReplace);
+			data: allSessions,
+		} as SessionsReplace),
 		emitEvent({
 			event: "PRESENCE_UPDATE",
 			user_id: this.user_id,
 			data: {
-				user: await User.getPublicUser(this.user_id),
+				user: user.toPublicUser(),
 				activities: session.activities,
-				client_status: session?.client_info,
+				client_status: session.client_info,
 				status: session.status,
 			},
-		} as PresenceUpdateEvent);
-	});
+		} as PresenceUpdateEvent),
+	]);
 
-	read_states.forEach((s: Partial<ReadState>) => {
-		s.id = s.channel_id;
-		delete s.user_id;
-		delete s.channel_id;
-	});
+	// Build READY
 
-	const privateUser = {
-		avatar: user.avatar,
-		mobile: user.mobile,
-		desktop: user.desktop,
-		discriminator: user.discriminator,
-		email: user.email,
-		flags: user.flags,
-		id: user.id,
-		mfa_enabled: user.mfa_enabled,
-		nsfw_allowed: user.nsfw_allowed,
-		phone: user.phone,
-		premium: user.premium,
-		premium_type: user.premium_type,
-		public_flags: user.public_flags,
-		premium_usage_flags: user.premium_usage_flags,
-		purchased_flags: user.purchased_flags,
-		username: user.username,
-		verified: user.verified,
-		bot: user.bot,
-		accent_color: user.accent_color,
-		banner: user.banner,
-		bio: user.bio,
-		premium_since: user.premium_since,
-	};
+	read_states.forEach((x) => {
+		x.id = x.channel_id;
+	});
 
 	const d: ReadyEventData = {
 		v: 9,
-		application: {
-			id: application?.id ?? "",
-			flags: application?.flags ?? 0,
-		}, //TODO: check this code!
-		user: privateUser,
+		application: application
+			? { id: application.id, flags: application.flags }
+			: undefined,
+		user: user.toPrivateUser(),
 		user_settings: user.settings,
-		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-		// @ts-ignore
-		guilds: guilds.map((x: Guild & { joined_at: Date }) => {
-			return {
-				...new ReadyGuildDTO(x).toJSON(),
-				guild_hashes: {},
-				joined_at: x.joined_at,
-				name: x.name,
-				icon: x.icon,
-			};
-		}),
-		guild_experiments: [], // TODO
-		geo_ordered_rtc_regions: [], // TODO
+		guilds: this.capabilities.has(Capabilities.FLAGS.CLIENT_STATE_V2)
+			? guilds.map((x) => new ReadyGuildDTO(x).toJSON())
+			: guilds,
 		relationships: user.relationships.map((x) => x.toPublicRelationship()),
 		read_state: {
 			entries: read_states,
 			partial: false,
-			version: 304128,
+			version: 0, // TODO
 		},
 		user_guild_settings: {
 			entries: user_guild_settings_entries,
-			partial: false, // TODO partial
-			version: 642,
+			partial: false,
+			version: 0, // TODO
 		},
 		private_channels: channels,
-		session_id: session_id,
-		analytics_token: "", // TODO
-		connected_accounts,
+		session_id: this.session_id,
+		country_code: user.settings.locale, // TODO: do ip analysis instead
+		users: Array.from(users),
+		merged_members: merged_members,
+		sessions: allSessions,
+
 		consents: {
 			personalization: {
 				consented: false, // TODO
 			},
 		},
-		country_code: user.settings.locale,
-		friend_suggestion_count: 0, // TODO
-		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-		// @ts-ignore
-		experiments: experiments, // TODO
-		guild_join_requests: [], // TODO what is this?
-		users: users.filter((x) => x).unique(),
-		merged_members: merged_members,
-		// shard // TODO: only for user sharding
-		sessions: [], // TODO:
+		experiments: [],
+		guild_join_requests: [],
+		connected_accounts: [],
+		guild_experiments: [],
+		geo_ordered_rtc_regions: [],
+		api_code_version: 1,
+		friend_suggestion_count: 0,
+		analytics_token: "",
+		tutorial: null,
+		resume_gateway_url:
+			Config.get().gateway.endpointClient ||
+			Config.get().gateway.endpointPublic ||
+			"ws://127.0.0.1:3001",
+		session_type: "normal", // TODO
+		auth_session_id_hash: "", // TODO
 
 		// lol hack whatever
 		required_action:
@@ -339,7 +435,7 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 				: undefined,
 	};
 
-	// TODO: send real proper data structure
+	// Send READY
 	await Send(this, {
 		op: OPCODES.Dispatch,
 		t: EVENTEnum.Ready,
@@ -347,23 +443,23 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		d,
 	});
 
+	// If we're a bot user, send GUILD_CREATE for each unavailable guild
 	await Promise.all(
-		pending_guilds.map((guild) =>
+		pending_guilds.map((x) =>
 			Send(this, {
 				op: OPCODES.Dispatch,
 				t: EVENTEnum.GuildCreate,
 				s: this.sequence++,
-				d: guild,
-			})?.catch(console.error),
+				d: x,
+			})?.catch((e) =>
+				console.error(`[Gateway] error when sending bot guilds`, e),
+			),
 		),
 	);
 
 	//TODO send READY_SUPPLEMENTAL
 	//TODO send GUILD_MEMBER_LIST_UPDATE
-	//TODO send SESSIONS_REPLACE
 	//TODO send VOICE_STATE_UPDATE to let the client know if another device is already connected to a voice channel
 
 	await setupListener.call(this);
-
-	// console.log(`${this.ipAddress} identified as ${d.user.id}`);
 }
diff --git a/src/gateway/util/Capabilities.ts b/src/gateway/util/Capabilities.ts
new file mode 100644
index 00000000..6c94bb45
--- /dev/null
+++ b/src/gateway/util/Capabilities.ts
@@ -0,0 +1,26 @@
+import { BitField, BitFieldResolvable, BitFlag } from "@spacebar/util";
+
+export type CapabilityResolvable = BitFieldResolvable | CapabilityString;
+type CapabilityString = keyof typeof Capabilities.FLAGS;
+
+export class Capabilities extends BitField {
+	static FLAGS = {
+		// Thanks, Opencord!
+		// https://github.com/MateriiApps/OpenCord/blob/master/app/src/main/java/com/xinto/opencord/gateway/io/Capabilities.kt
+		LAZY_USER_NOTES: BitFlag(0),
+		NO_AFFINE_USER_IDS: BitFlag(1),
+		VERSIONED_READ_STATES: BitFlag(2),
+		VERSIONED_USER_GUILD_SETTINGS: BitFlag(3),
+		DEDUPLICATE_USER_OBJECTS: BitFlag(4),
+		PRIORITIZED_READY_PAYLOAD: BitFlag(5),
+		MULTIPLE_GUILD_EXPERIMENT_POPULATIONS: BitFlag(6),
+		NON_CHANNEL_READ_STATES: BitFlag(7),
+		AUTH_TOKEN_REFRESH: BitFlag(8),
+		USER_SETTINGS_PROTO: BitFlag(9),
+		CLIENT_STATE_V2: BitFlag(10),
+		PASSIVE_GUILD_UPDATE: BitFlag(11),
+	};
+
+	any = (capability: CapabilityResolvable) => super.any(capability);
+	has = (capability: CapabilityResolvable) => super.has(capability);
+}
diff --git a/src/gateway/util/WebSocket.ts b/src/gateway/util/WebSocket.ts
index 972129c7..833756ff 100644
--- a/src/gateway/util/WebSocket.ts
+++ b/src/gateway/util/WebSocket.ts
@@ -19,6 +19,7 @@
 import { Intents, ListenEventOpts, Permissions } from "@spacebar/util";
 import WS from "ws";
 import { Deflate, Inflate } from "fast-zlib";
+import { Capabilities } from "./Capabilities";
 // import { Client } from "@spacebar/webrtc";
 
 export interface WebSocket extends WS {
@@ -40,5 +41,6 @@ export interface WebSocket extends WS {
 	events: Record<string, undefined | (() => unknown)>;
 	member_events: Record<string, () => unknown>;
 	listen_options: ListenEventOpts;
+	capabilities?: Capabilities;
 	// client?: Client;
 }
diff --git a/src/gateway/util/index.ts b/src/gateway/util/index.ts
index 627f12b2..6ef694d9 100644
--- a/src/gateway/util/index.ts
+++ b/src/gateway/util/index.ts
@@ -21,3 +21,4 @@ export * from "./Send";
 export * from "./SessionUtils";
 export * from "./Heartbeat";
 export * from "./WebSocket";
+export * from "./Capabilities";
diff --git a/src/util/dtos/ReadyGuildDTO.ts b/src/util/dtos/ReadyGuildDTO.ts
index b21afe74..7ca268a0 100644
--- a/src/util/dtos/ReadyGuildDTO.ts
+++ b/src/util/dtos/ReadyGuildDTO.ts
@@ -18,13 +18,45 @@
 
 import {
 	Channel,
+	ChannelOverride,
+	ChannelType,
 	Emoji,
 	Guild,
-	PublicMember,
+	PublicUser,
 	Role,
 	Sticker,
+	UserGuildSettings,
+	PublicMember,
 } from "../entities";
 
+// TODO: this is not the best place for this type
+export type ReadyUserGuildSettingsEntries = Omit<
+	UserGuildSettings,
+	"channel_overrides"
+> & {
+	channel_overrides: (ChannelOverride & { channel_id: string })[];
+};
+
+// TODO: probably should move somewhere else
+export interface ReadyPrivateChannel {
+	id: string;
+	flags: number;
+	is_spam: boolean;
+	last_message_id?: string;
+	recipients: PublicUser[];
+	type: ChannelType.DM | ChannelType.GROUP_DM;
+}
+
+export type GuildOrUnavailable =
+	| { id: string; unavailable: boolean }
+	| (Guild & { joined_at?: Date; unavailable: boolean });
+
+const guildIsAvailable = (
+	guild: GuildOrUnavailable,
+): guild is Guild & { joined_at: Date; unavailable: false } => {
+	return guild.unavailable == false;
+};
+
 export interface IReadyGuildDTO {
 	application_command_counts?: { 1: number; 2: number; 3: number }; // ????????????
 	channels: Channel[];
@@ -65,12 +97,21 @@ export interface IReadyGuildDTO {
 		max_members: number | undefined;
 		nsfw_level: number | undefined;
 		hub_type?: unknown | null; // ????
+
+		home_header: null; // TODO
+		latest_onboarding_question_id: null; // TODO
+		safety_alerts_channel_id: null; // TODO
+		max_stage_video_channel_users: 50; // TODO
+		nsfw: boolean;
+		id: string;
 	};
 	roles: Role[];
 	stage_instances: unknown[];
 	stickers: Sticker[];
 	threads: unknown[];
 	version: string;
+	guild_hashes: unknown;
+	unavailable: boolean;
 }
 
 export class ReadyGuildDTO implements IReadyGuildDTO {
@@ -113,14 +154,30 @@ export class ReadyGuildDTO implements IReadyGuildDTO {
 		max_members: number | undefined;
 		nsfw_level: number | undefined;
 		hub_type?: unknown | null; // ????
+
+		home_header: null; // TODO
+		latest_onboarding_question_id: null; // TODO
+		safety_alerts_channel_id: null; // TODO
+		max_stage_video_channel_users: 50; // TODO
+		nsfw: boolean;
+		id: string;
 	};
 	roles: Role[];
 	stage_instances: unknown[];
 	stickers: Sticker[];
 	threads: unknown[];
 	version: string;
+	guild_hashes: unknown;
+	unavailable: boolean;
+	joined_at: Date;
+
+	constructor(guild: GuildOrUnavailable) {
+		if (!guildIsAvailable(guild)) {
+			this.id = guild.id;
+			this.unavailable = true;
+			return;
+		}
 
-	constructor(guild: Guild) {
 		this.application_command_counts = {
 			1: 5,
 			2: 2,
@@ -164,12 +221,21 @@ export class ReadyGuildDTO implements IReadyGuildDTO {
 			max_members: guild.max_members,
 			nsfw_level: guild.nsfw_level,
 			hub_type: null,
+
+			home_header: null,
+			id: guild.id,
+			latest_onboarding_question_id: null,
+			max_stage_video_channel_users: 50, // TODO
+			nsfw: guild.nsfw,
+			safety_alerts_channel_id: null,
 		};
 		this.roles = guild.roles;
 		this.stage_instances = [];
 		this.stickers = guild.stickers;
 		this.threads = [];
 		this.version = "1"; // ??????
+		this.guild_hashes = {};
+		this.joined_at = guild.joined_at;
 	}
 
 	toJSON() {
diff --git a/src/util/entities/Channel.ts b/src/util/entities/Channel.ts
index 9ce04848..72490028 100644
--- a/src/util/entities/Channel.ts
+++ b/src/util/entities/Channel.ts
@@ -482,3 +482,10 @@ export enum ChannelPermissionOverwriteType {
 	member = 1,
 	group = 2,
 }
+
+export interface DMChannel extends Omit<Channel, "type" | "recipients"> {
+	type: ChannelType.DM | ChannelType.GROUP_DM;
+	recipients: Recipient[];
+
+	// TODO: probably more props
+}
diff --git a/src/util/entities/Guild.ts b/src/util/entities/Guild.ts
index e8454986..64dc2420 100644
--- a/src/util/entities/Guild.ts
+++ b/src/util/entities/Guild.ts
@@ -353,6 +353,7 @@ export class Guild extends BaseClass {
 			position: 0,
 			icon: undefined,
 			unicode_emoji: undefined,
+			flags: 0, // TODO?
 		}).save();
 
 		if (!body.channels || !body.channels.length)
diff --git a/src/util/entities/Role.ts b/src/util/entities/Role.ts
index 85877c12..3ae5efc1 100644
--- a/src/util/entities/Role.ts
+++ b/src/util/entities/Role.ts
@@ -66,4 +66,7 @@ export class Role extends BaseClass {
 		integration_id?: string;
 		premium_subscriber?: boolean;
 	};
+
+	@Column()
+	flags: number;
 }
diff --git a/src/util/entities/User.ts b/src/util/entities/User.ts
index df9af328..b30a7c58 100644
--- a/src/util/entities/User.ts
+++ b/src/util/entities/User.ts
@@ -280,6 +280,15 @@ export class User extends BaseClass {
 		return user as PublicUser;
 	}
 
+	toPrivateUser() {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const user: any = {};
+		PrivateUserProjection.forEach((x) => {
+			user[x] = this[x];
+		});
+		return user as UserPrivate;
+	}
+
 	static async getPublicUser(user_id: string, opts?: FindOneOptions<User>) {
 		return await User.findOneOrFail({
 			where: { id: user_id },
diff --git a/src/util/interfaces/Event.ts b/src/util/interfaces/Event.ts
index 76a5f8d0..7ddb763d 100644
--- a/src/util/interfaces/Event.ts
+++ b/src/util/interfaces/Event.ts
@@ -28,7 +28,6 @@ import {
 	Role,
 	Emoji,
 	PublicMember,
-	UserGuildSettings,
 	Guild,
 	Channel,
 	PublicUser,
@@ -40,6 +39,10 @@ import {
 	UserSettings,
 	IReadyGuildDTO,
 	ReadState,
+	UserPrivate,
+	ReadyUserGuildSettingsEntries,
+	ReadyPrivateChannel,
+	GuildOrUnavailable,
 } from "@spacebar/util";
 
 export interface Event {
@@ -68,22 +71,10 @@ export interface PublicRelationship {
 
 export interface ReadyEventData {
 	v: number;
-	user: PublicUser & {
-		mobile: boolean;
-		desktop: boolean;
-		email: string | undefined;
-		flags: string;
-		mfa_enabled: boolean;
-		nsfw_allowed: boolean;
-		phone: string | undefined;
-		premium: boolean;
-		premium_type: number;
-		verified: boolean;
-		bot: boolean;
-	};
-	private_channels: Channel[]; // this will be empty for bots
+	user: UserPrivate;
+	private_channels: ReadyPrivateChannel[]; // this will be empty for bots
 	session_id: string; // resuming
-	guilds: IReadyGuildDTO[];
+	guilds: IReadyGuildDTO[] | GuildOrUnavailable[]; // depends on capability
 	analytics_token?: string;
 	connected_accounts?: ConnectedAccount[];
 	consents?: {
@@ -115,7 +106,7 @@ export interface ReadyEventData {
 		version: number;
 	};
 	user_guild_settings?: {
-		entries: UserGuildSettings[];
+		entries: ReadyUserGuildSettingsEntries[];
 		version: number;
 		partial: boolean;
 	};
@@ -127,6 +118,17 @@ export interface ReadyEventData {
 	// probably all users who the user is in contact with
 	users?: PublicUser[];
 	sessions: unknown[];
+	api_code_version: number;
+	tutorial: number | null;
+	resume_gateway_url: string;
+	session_type: string;
+	auth_session_id_hash: string;
+	required_action?:
+		| "REQUIRE_VERIFIED_EMAIL"
+		| "REQUIRE_VERIFIED_PHONE"
+		| "REQUIRE_CAPTCHA" // TODO: allow these to be triggered
+		| "TOS_UPDATE_ACKNOWLEDGMENT"
+		| "AGREEMENTS";
 }
 
 export interface ReadyEvent extends Event {
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-11-09 23:28:22 +0000