diff --git a/src/webrtc/opcodes/BackendVersion.ts b/src/webrtc/opcodes/BackendVersion.ts
new file mode 100644
index 00000000..b4b61c7d
--- /dev/null
+++ b/src/webrtc/opcodes/BackendVersion.ts
@@ -0,0 +1,6 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+
+export async function onBackendVersion(this: WebSocket, data: Payload) {
+ await Send(this, { op: VoiceOPCodes.VOICE_BACKEND_VERSION, d: { voice: "0.8.43", rtc_worker: "0.3.26" } });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Heartbeat.ts b/src/webrtc/opcodes/Heartbeat.ts
new file mode 100644
index 00000000..1b6c5bcd
--- /dev/null
+++ b/src/webrtc/opcodes/Heartbeat.ts
@@ -0,0 +1,9 @@
+import { CLOSECODES, Payload, Send, setHeartbeat, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+
+export async function onHeartbeat(this: WebSocket, data: Payload) {
+ setHeartbeat(this);
+ if (isNaN(data.d)) return this.close(CLOSECODES.Decode_error);
+
+ await Send(this, { op: VoiceOPCodes.HEARTBEAT_ACK, d: data.d });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Identify.ts b/src/webrtc/opcodes/Identify.ts
new file mode 100644
index 00000000..19a575ab
--- /dev/null
+++ b/src/webrtc/opcodes/Identify.ts
@@ -0,0 +1,60 @@
+import { CLOSECODES, Payload, Send, WebSocket } from "@fosscord/gateway";
+import { validateSchema, VoiceIdentifySchema, VoiceState } from "@fosscord/util";
+import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@fosscord/webrtc";
+import SemanticSDP from "semantic-sdp";
+const defaultSDP = require("./sdp.json");
+
+export async function onIdentify(this: WebSocket, data: Payload) {
+ clearTimeout(this.readyTimeout);
+ const { server_id, user_id, session_id, token, streams, video } = validateSchema("VoiceIdentifySchema", data.d) as VoiceIdentifySchema;
+
+ const voiceState = await VoiceState.findOne({ where: { guild_id: server_id, user_id, token, session_id } });
+ if (!voiceState) return this.close(CLOSECODES.Authentication_failed);
+
+ this.user_id = user_id;
+ this.session_id = session_id;
+ const sdp = SemanticSDP.SDPInfo.expand(defaultSDP);
+ sdp.setDTLS(SemanticSDP.DTLSInfo.expand({ setup: "actpass", hash: "sha-256", fingerprint: endpoint.getDTLSFingerprint() }));
+
+ this.client = {
+ websocket: this,
+ out: {
+ tracks: new Map()
+ },
+ in: {
+ audio_ssrc: 0,
+ video_ssrc: 0,
+ rtx_ssrc: 0
+ },
+ sdp,
+ channel_id: voiceState.channel_id
+ };
+
+ const clients = getClients(voiceState.channel_id)!;
+ clients.add(this.client);
+
+ this.on("close", () => {
+ clients.delete(this.client!);
+ });
+
+ await Send(this, {
+ op: VoiceOPCodes.READY,
+ d: {
+ streams: [
+ // { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false }
+ ],
+ ssrc: -1,
+ port: endpoint.getLocalPort(),
+ modes: [
+ "aead_aes256_gcm_rtpsize",
+ "aead_aes256_gcm",
+ "xsalsa20_poly1305_lite_rtpsize",
+ "xsalsa20_poly1305_lite",
+ "xsalsa20_poly1305_suffix",
+ "xsalsa20_poly1305"
+ ],
+ ip: PublicIP,
+ experiments: []
+ }
+ });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/SelectProtocol.ts b/src/webrtc/opcodes/SelectProtocol.ts
new file mode 100644
index 00000000..a3579b34
--- /dev/null
+++ b/src/webrtc/opcodes/SelectProtocol.ts
@@ -0,0 +1,46 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { SelectProtocolSchema, validateSchema } from "@fosscord/util";
+import { endpoint, PublicIP, VoiceOPCodes } from "@fosscord/webrtc";
+import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp";
+
+export async function onSelectProtocol(this: WebSocket, payload: Payload) {
+ if (!this.client) return;
+
+ const data = validateSchema("SelectProtocolSchema", payload.d) as SelectProtocolSchema;
+
+ const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!);
+ this.client.sdp!.setICE(offer.getICE());
+ this.client.sdp!.setDTLS(offer.getDTLS());
+
+ const transport = endpoint.createTransport(this.client.sdp!);
+ this.client.transport = transport;
+ transport.setRemoteProperties(this.client.sdp!);
+ transport.setLocalProperties(this.client.sdp!);
+
+ const dtls = transport.getLocalDTLSInfo();
+ const ice = transport.getLocalICEInfo();
+ const port = endpoint.getLocalPort();
+ const fingerprint = dtls.getHash() + " " + dtls.getFingerprint();
+ const candidates = transport.getLocalCandidates();
+ const candidate = candidates[0];
+
+ const answer =
+ `m=audio ${port} ICE/SDP`
+ + `a=fingerprint:${fingerprint}`
+ + `c=IN IP4 ${PublicIP}`
+ + `a=rtcp:${port}`
+ + `a=ice-ufrag:${ice.getUfrag()}`
+ + `a=ice-pwd:${ice.getPwd()}`
+ + `a=fingerprint:${fingerprint}`
+ + `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`;
+
+ await Send(this, {
+ op: VoiceOPCodes.SELECT_PROTOCOL_ACK,
+ d: {
+ video_codec: "H264",
+ sdp: answer,
+ media_session_id: this.session_id,
+ audio_codec: "opus"
+ }
+ });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Speaking.ts b/src/webrtc/opcodes/Speaking.ts
new file mode 100644
index 00000000..e2227040
--- /dev/null
+++ b/src/webrtc/opcodes/Speaking.ts
@@ -0,0 +1,22 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { getClients, VoiceOPCodes } from "../util";
+
+// {"speaking":1,"delay":5,"ssrc":2805246727}
+
+export async function onSpeaking(this: WebSocket, data: Payload) {
+ if (!this.client) return;
+
+ getClients(this.client.channel_id).forEach((client) => {
+ if (client === this.client) return;
+ const ssrc = this.client!.out.tracks.get(client.websocket.user_id);
+
+ Send(client.websocket, {
+ op: VoiceOPCodes.SPEAKING,
+ d: {
+ user_id: client.websocket.user_id,
+ speaking: data.d.speaking,
+ ssrc: ssrc?.audio_ssrc || 0
+ }
+ });
+ });
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/Video.ts b/src/webrtc/opcodes/Video.ts
new file mode 100644
index 00000000..ff20d5a9
--- /dev/null
+++ b/src/webrtc/opcodes/Video.ts
@@ -0,0 +1,118 @@
+import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { validateSchema, VoiceVideoSchema } from "@fosscord/util";
+import { channels, getClients, VoiceOPCodes } from "@fosscord/webrtc";
+import { IncomingStreamTrack, SSRCs } from "medooze-media-server";
+import SemanticSDP from "semantic-sdp";
+
+export async function onVideo(this: WebSocket, payload: Payload) {
+ if (!this.client) return;
+ const { transport, channel_id } = this.client;
+ if (!transport) return;
+ const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema;
+
+ await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } });
+
+ const id = "stream" + this.user_id;
+
+ var stream = this.client.in.stream!;
+ if (!stream) {
+ stream = this.client.transport!.createIncomingStream(
+ // @ts-ignore
+ SemanticSDP.StreamInfo.expand({
+ id,
+ // @ts-ignore
+ tracks: []
+ })
+ );
+ this.client.in.stream = stream;
+
+ const interval = setInterval(() => {
+ for (const track of stream.getTracks()) {
+ for (const layer of Object.values(track.getStats())) {
+ console.log(track.getId(), layer.total);
+ }
+ }
+ }, 5000);
+
+ stream.on("stopped", () => {
+ console.log("stream stopped");
+ clearInterval(interval);
+ });
+ this.on("close", () => {
+ transport!.stop();
+ });
+ const out = transport.createOutgoingStream(
+ // @ts-ignore
+ SemanticSDP.StreamInfo.expand({
+ id: "out" + this.user_id,
+ // @ts-ignore
+ tracks: []
+ })
+ );
+ this.client.out.stream = out;
+
+ const clients = channels.get(channel_id)!;
+
+ clients.forEach((client) => {
+ if (client.websocket.user_id === this.user_id) return;
+ if (!client.in.stream) return;
+
+ client.in.stream?.getTracks().forEach((track) => {
+ attachTrack.call(this, track, client.websocket.user_id);
+ });
+ });
+ }
+
+ if (d.audio_ssrc) {
+ handleSSRC.call(this, "audio", { media: d.audio_ssrc, rtx: d.audio_ssrc + 1 });
+ }
+ if (d.video_ssrc && d.rtx_ssrc) {
+ handleSSRC.call(this, "video", { media: d.video_ssrc, rtx: d.rtx_ssrc });
+ }
+}
+
+function attachTrack(this: WebSocket, track: IncomingStreamTrack, user_id: string) {
+ if (!this.client) return;
+ const outTrack = this.client.transport!.createOutgoingStreamTrack(track.getMedia());
+ outTrack.attachTo(track);
+ this.client.out.stream!.addTrack(outTrack);
+ var ssrcs = this.client.out.tracks.get(user_id)!;
+ if (!ssrcs) ssrcs = this.client.out.tracks.set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 }).get(user_id)!;
+
+ if (track.getMedia() === "audio") {
+ ssrcs.audio_ssrc = outTrack.getSSRCs().media!;
+ } else if (track.getMedia() === "video") {
+ ssrcs.video_ssrc = outTrack.getSSRCs().media!;
+ ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!;
+ }
+
+ Send(this, {
+ op: VoiceOPCodes.VIDEO,
+ d: {
+ user_id: user_id,
+ ...ssrcs
+ } as VoiceVideoSchema
+ });
+}
+
+function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) {
+ if (!this.client) return;
+ const stream = this.client.in.stream!;
+ const transport = this.client.transport!;
+
+ const id = type + ssrcs.media;
+ var track = stream.getTrack(id);
+ if (!track) {
+ console.log("createIncomingStreamTrack", id);
+ track = transport.createIncomingStreamTrack(type, { id, ssrcs });
+ stream.addTrack(track);
+
+ const clients = getClients(this.client.channel_id)!;
+ clients.forEach((client) => {
+ if (client.websocket.user_id === this.user_id) return;
+ if (!client.out.stream) return;
+
+ attachTrack.call(this, track, client.websocket.user_id);
+ });
+ }
+}
\ No newline at end of file
diff --git a/src/webrtc/opcodes/index.ts b/src/webrtc/opcodes/index.ts
new file mode 100644
index 00000000..8c664cce
--- /dev/null
+++ b/src/webrtc/opcodes/index.ts
@@ -0,0 +1,19 @@
+import { Payload, WebSocket } from "@fosscord/gateway";
+import { VoiceOPCodes } from "../util";
+import { onBackendVersion } from "./BackendVersion";
+import { onHeartbeat } from "./Heartbeat";
+import { onIdentify } from "./Identify";
+import { onSelectProtocol } from "./SelectProtocol";
+import { onSpeaking } from "./Speaking";
+import { onVideo } from "./Video";
+
+export type OPCodeHandler = (this: WebSocket, data: Payload) => any;
+
+export default {
+ [VoiceOPCodes.HEARTBEAT]: onHeartbeat,
+ [VoiceOPCodes.IDENTIFY]: onIdentify,
+ [VoiceOPCodes.VOICE_BACKEND_VERSION]: onBackendVersion,
+ [VoiceOPCodes.VIDEO]: onVideo,
+ [VoiceOPCodes.SPEAKING]: onSpeaking,
+ [VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol
+};
\ No newline at end of file
diff --git a/src/webrtc/opcodes/sdp.json b/src/webrtc/opcodes/sdp.json
new file mode 100644
index 00000000..4867b9c7
--- /dev/null
+++ b/src/webrtc/opcodes/sdp.json
@@ -0,0 +1,420 @@
+{
+ "version": 0,
+ "streams": [],
+ "medias": [
+ {
+ "id": "0",
+ "type": "audio",
+ "direction": "sendrecv",
+ "codecs": [
+ {
+ "codec": "opus",
+ "type": 111,
+ "channels": 2,
+ "params": {
+ "minptime": "10",
+ "useinbandfec": "1"
+ },
+ "rtcpfbs": [
+ {
+ "id": "transport-cc"
+ }
+ ]
+ }
+ ],
+ "extensions": {
+ "1": "urn:ietf:params:rtp-hdrext:ssrc-audio-level",
+ "2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
+ "3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
+ "4": "urn:ietf:params:rtp-hdrext:sdes:mid"
+ }
+ },
+ {
+ "id": "1",
+ "type": "video",
+ "direction": "sendrecv",
+ "codecs": [
+ {
+ "codec": "VP8",
+ "type": 96,
+ "rtx": 97,
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "VP9",
+ "type": 98,
+ "rtx": 99,
+ "params": {
+ "profile-id": "0"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "VP9",
+ "type": 100,
+ "rtx": 101,
+ "params": {
+ "profile-id": "2"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "VP9",
+ "type": 102,
+ "rtx": 122,
+ "params": {
+ "profile-id": "1"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 127,
+ "rtx": 121,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "1",
+ "profile-level-id": "42001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 125,
+ "rtx": 107,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "0",
+ "profile-level-id": "42001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 108,
+ "rtx": 109,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "1",
+ "profile-level-id": "42e01f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 124,
+ "rtx": 120,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "0",
+ "profile-level-id": "42e01f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 123,
+ "rtx": 119,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "1",
+ "profile-level-id": "4d001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 35,
+ "rtx": 36,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "0",
+ "profile-level-id": "4d001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 37,
+ "rtx": 38,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "1",
+ "profile-level-id": "f4001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 39,
+ "rtx": 40,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "0",
+ "profile-level-id": "f4001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ },
+ {
+ "codec": "H264",
+ "type": 114,
+ "rtx": 115,
+ "params": {
+ "level-asymmetry-allowed": "1",
+ "packetization-mode": "1",
+ "profile-level-id": "64001f"
+ },
+ "rtcpfbs": [
+ {
+ "id": "goog-remb"
+ },
+ {
+ "id": "transport-cc"
+ },
+ {
+ "id": "ccm",
+ "params": ["fir"]
+ },
+ {
+ "id": "nack"
+ },
+ {
+ "id": "nack",
+ "params": ["pli"]
+ }
+ ]
+ }
+ ],
+ "extensions": {
+ "2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
+ "3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
+ "4": "urn:ietf:params:rtp-hdrext:sdes:mid",
+ "5": "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay",
+ "6": "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type",
+ "7": "http://www.webrtc.org/experiments/rtp-hdrext/video-timing",
+ "8": "http://www.webrtc.org/experiments/rtp-hdrext/color-space",
+ "10": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id",
+ "11": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id",
+ "13": "urn:3gpp:video-orientation",
+ "14": "urn:ietf:params:rtp-hdrext:toffset"
+ }
+ }
+ ],
+ "candidates": []
+}
\ No newline at end of file
|
