Refactor to mono-repo + upgrade packages

8 files changed, 700 insertions, 0 deletions

diff --git a/src/webrtc/opcodes/BackendVersion.ts b/src/webrtc/opcodes/BackendVersion.ts
new file mode 100644
index 00000000..b4b61c7d
--- /dev/null
+++ b/src/webrtc/opcodes/BackendVersion.ts
@@ -0,0 +1,6 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+
+export async function onBackendVersion(this: WebSocket, data: Payload) {
+	await Send(this, { op: VoiceOPCodes.VOICE_BACKEND_VERSION, d: { voice: "0.8.43", rtc_worker: "0.3.26" } });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Heartbeat.ts b/src/webrtc/opcodes/Heartbeat.ts
new file mode 100644
index 00000000..1b6c5bcd
--- /dev/null
+++ b/src/webrtc/opcodes/Heartbeat.ts
@@ -0,0 +1,9 @@
+import { CLOSECODES, Payload, Send, setHeartbeat, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+
+export async function onHeartbeat(this: WebSocket, data: Payload) {
+	setHeartbeat(this);
+	if (isNaN(data.d)) return this.close(CLOSECODES.Decode_error);
+
+	await Send(this, { op: VoiceOPCodes.HEARTBEAT_ACK, d: data.d });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Identify.ts b/src/webrtc/opcodes/Identify.ts
new file mode 100644
index 00000000..19a575ab
--- /dev/null
+++ b/src/webrtc/opcodes/Identify.ts
@@ -0,0 +1,60 @@
+import { CLOSECODES, Payload, Send, WebSocket } from "@fosscord/gateway";
+import { validateSchema, VoiceIdentifySchema, VoiceState } from "@fosscord/util";
+import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@fosscord/webrtc";
+import SemanticSDP from "semantic-sdp";
+const defaultSDP = require("./sdp.json");
+
+export async function onIdentify(this: WebSocket, data: Payload) {
+	clearTimeout(this.readyTimeout);
+	const { server_id, user_id, session_id, token, streams, video } = validateSchema("VoiceIdentifySchema", data.d) as VoiceIdentifySchema;
+
+	const voiceState = await VoiceState.findOne({ where: { guild_id: server_id, user_id, token, session_id } });
+	if (!voiceState) return this.close(CLOSECODES.Authentication_failed);
+
+	this.user_id = user_id;
+	this.session_id = session_id;
+	const sdp = SemanticSDP.SDPInfo.expand(defaultSDP);
+	sdp.setDTLS(SemanticSDP.DTLSInfo.expand({ setup: "actpass", hash: "sha-256", fingerprint: endpoint.getDTLSFingerprint() }));
+
+	this.client = {
+		websocket: this,
+		out: {
+			tracks: new Map()
+		},
+		in: {
+			audio_ssrc: 0,
+			video_ssrc: 0,
+			rtx_ssrc: 0
+		},
+		sdp,
+		channel_id: voiceState.channel_id
+	};
+
+	const clients = getClients(voiceState.channel_id)!;
+	clients.add(this.client);
+
+	this.on("close", () => {
+		clients.delete(this.client!);
+	});
+
+	await Send(this, {
+		op: VoiceOPCodes.READY,
+		d: {
+			streams: [
+				// { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false }
+			],
+			ssrc: -1,
+			port: endpoint.getLocalPort(),
+			modes: [
+				"aead_aes256_gcm_rtpsize",
+				"aead_aes256_gcm",
+				"xsalsa20_poly1305_lite_rtpsize",
+				"xsalsa20_poly1305_lite",
+				"xsalsa20_poly1305_suffix",
+				"xsalsa20_poly1305"
+			],
+			ip: PublicIP,
+			experiments: []
+		}
+	});
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/SelectProtocol.ts b/src/webrtc/opcodes/SelectProtocol.ts
new file mode 100644
index 00000000..a3579b34
--- /dev/null
+++ b/src/webrtc/opcodes/SelectProtocol.ts
@@ -0,0 +1,46 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { SelectProtocolSchema, validateSchema } from "@fosscord/util";
+import { endpoint, PublicIP, VoiceOPCodes } from "@fosscord/webrtc";
+import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp";
+
+export async function onSelectProtocol(this: WebSocket, payload: Payload) {
+	if (!this.client) return;
+
+	const data = validateSchema("SelectProtocolSchema", payload.d) as SelectProtocolSchema;
+
+	const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!);
+	this.client.sdp!.setICE(offer.getICE());
+	this.client.sdp!.setDTLS(offer.getDTLS());
+
+	const transport = endpoint.createTransport(this.client.sdp!);
+	this.client.transport = transport;
+	transport.setRemoteProperties(this.client.sdp!);
+	transport.setLocalProperties(this.client.sdp!);
+
+	const dtls = transport.getLocalDTLSInfo();
+	const ice = transport.getLocalICEInfo();
+	const port = endpoint.getLocalPort();
+	const fingerprint = dtls.getHash() + " " + dtls.getFingerprint();
+	const candidates = transport.getLocalCandidates();
+	const candidate = candidates[0];
+
+	const answer =
+		`m=audio ${port} ICE/SDP`
+		+ `a=fingerprint:${fingerprint}`
+		+ `c=IN IP4 ${PublicIP}`
+		+ `a=rtcp:${port}`
+		+ `a=ice-ufrag:${ice.getUfrag()}`
+		+ `a=ice-pwd:${ice.getPwd()}`
+		+ `a=fingerprint:${fingerprint}`
+		+ `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`;
+
+	await Send(this, {
+		op: VoiceOPCodes.SELECT_PROTOCOL_ACK,
+		d: {
+			video_codec: "H264",
+			sdp: answer,
+			media_session_id: this.session_id,
+			audio_codec: "opus"
+		}
+	});
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Speaking.ts b/src/webrtc/opcodes/Speaking.ts
new file mode 100644
index 00000000..e2227040
--- /dev/null
+++ b/src/webrtc/opcodes/Speaking.ts
@@ -0,0 +1,22 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { getClients, VoiceOPCodes } from "../util";
+
+// {"speaking":1,"delay":5,"ssrc":2805246727}
+
+export async function onSpeaking(this: WebSocket, data: Payload) {
+	if (!this.client) return;
+
+	getClients(this.client.channel_id).forEach((client) => {
+		if (client === this.client) return;
+		const ssrc = this.client!.out.tracks.get(client.websocket.user_id);
+
+		Send(client.websocket, {
+			op: VoiceOPCodes.SPEAKING,
+			d: {
+				user_id: client.websocket.user_id,
+				speaking: data.d.speaking,
+				ssrc: ssrc?.audio_ssrc || 0
+			}
+		});
+	});
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Video.ts b/src/webrtc/opcodes/Video.ts
new file mode 100644
index 00000000..ff20d5a9
--- /dev/null
+++ b/src/webrtc/opcodes/Video.ts
@@ -0,0 +1,118 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { validateSchema, VoiceVideoSchema } from "@fosscord/util";
+import { channels, getClients, VoiceOPCodes } from "@fosscord/webrtc";
+import { IncomingStreamTrack, SSRCs } from "medooze-media-server";
+import SemanticSDP from "semantic-sdp";
+
+export async function onVideo(this: WebSocket, payload: Payload) {
+	if (!this.client) return;
+	const { transport, channel_id } = this.client;
+	if (!transport) return;
+	const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema;
+
+	await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } });
+
+	const id = "stream" + this.user_id;
+
+	var stream = this.client.in.stream!;
+	if (!stream) {
+		stream = this.client.transport!.createIncomingStream(
+			// @ts-ignore
+			SemanticSDP.StreamInfo.expand({
+				id,
+				// @ts-ignore
+				tracks: []
+			})
+		);
+		this.client.in.stream = stream;
+
+		const interval = setInterval(() => {
+			for (const track of stream.getTracks()) {
+				for (const layer of Object.values(track.getStats())) {
+					console.log(track.getId(), layer.total);
+				}
+			}
+		}, 5000);
+
+		stream.on("stopped", () => {
+			console.log("stream stopped");
+			clearInterval(interval);
+		});
+		this.on("close", () => {
+			transport!.stop();
+		});
+		const out = transport.createOutgoingStream(
+			// @ts-ignore
+			SemanticSDP.StreamInfo.expand({
+				id: "out" + this.user_id,
+				// @ts-ignore
+				tracks: []
+			})
+		);
+		this.client.out.stream = out;
+
+		const clients = channels.get(channel_id)!;
+
+		clients.forEach((client) => {
+			if (client.websocket.user_id === this.user_id) return;
+			if (!client.in.stream) return;
+
+			client.in.stream?.getTracks().forEach((track) => {
+				attachTrack.call(this, track, client.websocket.user_id);
+			});
+		});
+	}
+
+	if (d.audio_ssrc) {
+		handleSSRC.call(this, "audio", { media: d.audio_ssrc, rtx: d.audio_ssrc + 1 });
+	}
+	if (d.video_ssrc && d.rtx_ssrc) {
+		handleSSRC.call(this, "video", { media: d.video_ssrc, rtx: d.rtx_ssrc });
+	}
+}
+
+function attachTrack(this: WebSocket, track: IncomingStreamTrack, user_id: string) {
+	if (!this.client) return;
+	const outTrack = this.client.transport!.createOutgoingStreamTrack(track.getMedia());
+	outTrack.attachTo(track);
+	this.client.out.stream!.addTrack(outTrack);
+	var ssrcs = this.client.out.tracks.get(user_id)!;
+	if (!ssrcs) ssrcs = this.client.out.tracks.set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 }).get(user_id)!;
+
+	if (track.getMedia() === "audio") {
+		ssrcs.audio_ssrc = outTrack.getSSRCs().media!;
+	} else if (track.getMedia() === "video") {
+		ssrcs.video_ssrc = outTrack.getSSRCs().media!;
+		ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!;
+	}
+
+	Send(this, {
+		op: VoiceOPCodes.VIDEO,
+		d: {
+			user_id: user_id,
+			...ssrcs
+		} as VoiceVideoSchema
+	});
+}
+
+function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) {
+	if (!this.client) return;
+	const stream = this.client.in.stream!;
+	const transport = this.client.transport!;
+
+	const id = type + ssrcs.media;
+	var track = stream.getTrack(id);
+	if (!track) {
+		console.log("createIncomingStreamTrack", id);
+		track = transport.createIncomingStreamTrack(type, { id, ssrcs });
+		stream.addTrack(track);
+
+		const clients = getClients(this.client.channel_id)!;
+		clients.forEach((client) => {
+			if (client.websocket.user_id === this.user_id) return;
+			if (!client.out.stream) return;
+
+			attachTrack.call(this, track, client.websocket.user_id);
+		});
+	}
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/index.ts b/src/webrtc/opcodes/index.ts
new file mode 100644
index 00000000..8c664cce
--- /dev/null
+++ b/src/webrtc/opcodes/index.ts
@@ -0,0 +1,19 @@
+import { Payload, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+import { onBackendVersion } from "./BackendVersion";
+import { onHeartbeat } from "./Heartbeat";
+import { onIdentify } from "./Identify";
+import { onSelectProtocol } from "./SelectProtocol";
+import { onSpeaking } from "./Speaking";
+import { onVideo } from "./Video";
+
+export type OPCodeHandler = (this: WebSocket, data: Payload) => any;
+
+export default {
+	[VoiceOPCodes.HEARTBEAT]: onHeartbeat,
+	[VoiceOPCodes.IDENTIFY]: onIdentify,
+	[VoiceOPCodes.VOICE_BACKEND_VERSION]: onBackendVersion,
+	[VoiceOPCodes.VIDEO]: onVideo,
+	[VoiceOPCodes.SPEAKING]: onSpeaking,
+	[VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol
+};
\ No newline at end of file
diff --git a/src/webrtc/opcodes/sdp.json b/src/webrtc/opcodes/sdp.json
new file mode 100644
index 00000000..4867b9c7
--- /dev/null
+++ b/src/webrtc/opcodes/sdp.json
@@ -0,0 +1,420 @@
+{
+	"version": 0,
+	"streams": [],
+	"medias": [
+		{
+			"id": "0",
+			"type": "audio",
+			"direction": "sendrecv",
+			"codecs": [
+				{
+					"codec": "opus",
+					"type": 111,
+					"channels": 2,
+					"params": {
+						"minptime": "10",
+						"useinbandfec": "1"
+					},
+					"rtcpfbs": [
+						{
+							"id": "transport-cc"
+						}
+					]
+				}
+			],
+			"extensions": {
+				"1": "urn:ietf:params:rtp-hdrext:ssrc-audio-level",
+				"2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
+				"3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
+				"4": "urn:ietf:params:rtp-hdrext:sdes:mid"
+			}
+		},
+		{
+			"id": "1",
+			"type": "video",
+			"direction": "sendrecv",
+			"codecs": [
+				{
+					"codec": "VP8",
+					"type": 96,
+					"rtx": 97,
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "VP9",
+					"type": 98,
+					"rtx": 99,
+					"params": {
+						"profile-id": "0"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "VP9",
+					"type": 100,
+					"rtx": 101,
+					"params": {
+						"profile-id": "2"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "VP9",
+					"type": 102,
+					"rtx": 122,
+					"params": {
+						"profile-id": "1"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 127,
+					"rtx": 121,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "1",
+						"profile-level-id": "42001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 125,
+					"rtx": 107,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "0",
+						"profile-level-id": "42001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 108,
+					"rtx": 109,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "1",
+						"profile-level-id": "42e01f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 124,
+					"rtx": 120,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "0",
+						"profile-level-id": "42e01f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 123,
+					"rtx": 119,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "1",
+						"profile-level-id": "4d001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 35,
+					"rtx": 36,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "0",
+						"profile-level-id": "4d001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 37,
+					"rtx": 38,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "1",
+						"profile-level-id": "f4001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 39,
+					"rtx": 40,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "0",
+						"profile-level-id": "f4001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				},
+				{
+					"codec": "H264",
+					"type": 114,
+					"rtx": 115,
+					"params": {
+						"level-asymmetry-allowed": "1",
+						"packetization-mode": "1",
+						"profile-level-id": "64001f"
+					},
+					"rtcpfbs": [
+						{
+							"id": "goog-remb"
+						},
+						{
+							"id": "transport-cc"
+						},
+						{
+							"id": "ccm",
+							"params": ["fir"]
+						},
+						{
+							"id": "nack"
+						},
+						{
+							"id": "nack",
+							"params": ["pli"]
+						}
+					]
+				}
+			],
+			"extensions": {
+				"2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
+				"3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
+				"4": "urn:ietf:params:rtp-hdrext:sdes:mid",
+				"5": "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay",
+				"6": "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type",
+				"7": "http://www.webrtc.org/experiments/rtp-hdrext/video-timing",
+				"8": "http://www.webrtc.org/experiments/rtp-hdrext/color-space",
+				"10": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id",
+				"11": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id",
+				"13": "urn:3gpp:video-orientation",
+				"14": "urn:ietf:params:rtp-hdrext:toffset"
+			}
+		}
+	],
+	"candidates": []
+}
\ No newline at end of file