diff options
Diffstat (limited to 'src/webrtc/opcodes')
| -rw-r--r-- | src/webrtc/opcodes/BackendVersion.ts | 6 | ||||
| -rw-r--r-- | src/webrtc/opcodes/Heartbeat.ts | 9 | ||||
| -rw-r--r-- | src/webrtc/opcodes/Identify.ts | 60 | ||||
| -rw-r--r-- | src/webrtc/opcodes/SelectProtocol.ts | 46 | ||||
| -rw-r--r-- | src/webrtc/opcodes/Speaking.ts | 22 | ||||
| -rw-r--r-- | src/webrtc/opcodes/Video.ts | 118 | ||||
| -rw-r--r-- | src/webrtc/opcodes/index.ts | 19 | ||||
| -rw-r--r-- | src/webrtc/opcodes/sdp.json | 420 |
8 files changed, 700 insertions, 0 deletions
diff --git a/src/webrtc/opcodes/BackendVersion.ts b/src/webrtc/opcodes/BackendVersion.ts new file mode 100644 index 00000000..b4b61c7d --- /dev/null +++ b/src/webrtc/opcodes/BackendVersion.ts @@ -0,0 +1,6 @@ +import { Payload, Send, WebSocket } from "@fosscord/gateway"; +import { VoiceOPCodes } from "../util"; + +export async function onBackendVersion(this: WebSocket, data: Payload) { + await Send(this, { op: VoiceOPCodes.VOICE_BACKEND_VERSION, d: { voice: "0.8.43", rtc_worker: "0.3.26" } }); +} \ No newline at end of file diff --git a/src/webrtc/opcodes/Heartbeat.ts b/src/webrtc/opcodes/Heartbeat.ts new file mode 100644 index 00000000..1b6c5bcd --- /dev/null +++ b/src/webrtc/opcodes/Heartbeat.ts @@ -0,0 +1,9 @@ +import { CLOSECODES, Payload, Send, setHeartbeat, WebSocket } from "@fosscord/gateway"; +import { VoiceOPCodes } from "../util"; + +export async function onHeartbeat(this: WebSocket, data: Payload) { + setHeartbeat(this); + if (isNaN(data.d)) return this.close(CLOSECODES.Decode_error); + + await Send(this, { op: VoiceOPCodes.HEARTBEAT_ACK, d: data.d }); +} \ No newline at end of file diff --git a/src/webrtc/opcodes/Identify.ts b/src/webrtc/opcodes/Identify.ts new file mode 100644 index 00000000..19a575ab --- /dev/null +++ b/src/webrtc/opcodes/Identify.ts @@ -0,0 +1,60 @@ +import { CLOSECODES, Payload, Send, WebSocket } from "@fosscord/gateway"; +import { validateSchema, VoiceIdentifySchema, VoiceState } from "@fosscord/util"; +import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@fosscord/webrtc"; +import SemanticSDP from "semantic-sdp"; +const defaultSDP = require("./sdp.json"); + +export async function onIdentify(this: WebSocket, data: Payload) { + clearTimeout(this.readyTimeout); + const { server_id, user_id, session_id, token, streams, video } = validateSchema("VoiceIdentifySchema", data.d) as VoiceIdentifySchema; + + const voiceState = await VoiceState.findOne({ where: { guild_id: server_id, user_id, token, session_id } }); + if (!voiceState) return this.close(CLOSECODES.Authentication_failed); + + this.user_id = user_id; + this.session_id = session_id; + const sdp = SemanticSDP.SDPInfo.expand(defaultSDP); + sdp.setDTLS(SemanticSDP.DTLSInfo.expand({ setup: "actpass", hash: "sha-256", fingerprint: endpoint.getDTLSFingerprint() })); + + this.client = { + websocket: this, + out: { + tracks: new Map() + }, + in: { + audio_ssrc: 0, + video_ssrc: 0, + rtx_ssrc: 0 + }, + sdp, + channel_id: voiceState.channel_id + }; + + const clients = getClients(voiceState.channel_id)!; + clients.add(this.client); + + this.on("close", () => { + clients.delete(this.client!); + }); + + await Send(this, { + op: VoiceOPCodes.READY, + d: { + streams: [ + // { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false } + ], + ssrc: -1, + port: endpoint.getLocalPort(), + modes: [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + ip: PublicIP, + experiments: [] + } + }); +} \ No newline at end of file diff --git a/src/webrtc/opcodes/SelectProtocol.ts b/src/webrtc/opcodes/SelectProtocol.ts new file mode 100644 index 00000000..a3579b34 --- /dev/null +++ b/src/webrtc/opcodes/SelectProtocol.ts @@ -0,0 +1,46 @@ +import { Payload, Send, WebSocket } from "@fosscord/gateway"; +import { SelectProtocolSchema, validateSchema } from "@fosscord/util"; +import { endpoint, PublicIP, VoiceOPCodes } from "@fosscord/webrtc"; +import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp"; + +export async function onSelectProtocol(this: WebSocket, payload: Payload) { + if (!this.client) return; + + const data = validateSchema("SelectProtocolSchema", payload.d) as SelectProtocolSchema; + + const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!); + this.client.sdp!.setICE(offer.getICE()); + this.client.sdp!.setDTLS(offer.getDTLS()); + + const transport = endpoint.createTransport(this.client.sdp!); + this.client.transport = transport; + transport.setRemoteProperties(this.client.sdp!); + transport.setLocalProperties(this.client.sdp!); + + const dtls = transport.getLocalDTLSInfo(); + const ice = transport.getLocalICEInfo(); + const port = endpoint.getLocalPort(); + const fingerprint = dtls.getHash() + " " + dtls.getFingerprint(); + const candidates = transport.getLocalCandidates(); + const candidate = candidates[0]; + + const answer = + `m=audio ${port} ICE/SDP` + + `a=fingerprint:${fingerprint}` + + `c=IN IP4 ${PublicIP}` + + `a=rtcp:${port}` + + `a=ice-ufrag:${ice.getUfrag()}` + + `a=ice-pwd:${ice.getPwd()}` + + `a=fingerprint:${fingerprint}` + + `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`; + + await Send(this, { + op: VoiceOPCodes.SELECT_PROTOCOL_ACK, + d: { + video_codec: "H264", + sdp: answer, + media_session_id: this.session_id, + audio_codec: "opus" + } + }); +} \ No newline at end of file diff --git a/src/webrtc/opcodes/Speaking.ts b/src/webrtc/opcodes/Speaking.ts new file mode 100644 index 00000000..e2227040 --- /dev/null +++ b/src/webrtc/opcodes/Speaking.ts @@ -0,0 +1,22 @@ +import { Payload, Send, WebSocket } from "@fosscord/gateway"; +import { getClients, VoiceOPCodes } from "../util"; + +// {"speaking":1,"delay":5,"ssrc":2805246727} + +export async function onSpeaking(this: WebSocket, data: Payload) { + if (!this.client) return; + + getClients(this.client.channel_id).forEach((client) => { + if (client === this.client) return; + const ssrc = this.client!.out.tracks.get(client.websocket.user_id); + + Send(client.websocket, { + op: VoiceOPCodes.SPEAKING, + d: { + user_id: client.websocket.user_id, + speaking: data.d.speaking, + ssrc: ssrc?.audio_ssrc || 0 + } + }); + }); +} \ No newline at end of file diff --git a/src/webrtc/opcodes/Video.ts b/src/webrtc/opcodes/Video.ts new file mode 100644 index 00000000..ff20d5a9 --- /dev/null +++ b/src/webrtc/opcodes/Video.ts @@ -0,0 +1,118 @@ +import { Payload, Send, WebSocket } from "@fosscord/gateway"; +import { validateSchema, VoiceVideoSchema } from "@fosscord/util"; +import { channels, getClients, VoiceOPCodes } from "@fosscord/webrtc"; +import { IncomingStreamTrack, SSRCs } from "medooze-media-server"; +import SemanticSDP from "semantic-sdp"; + +export async function onVideo(this: WebSocket, payload: Payload) { + if (!this.client) return; + const { transport, channel_id } = this.client; + if (!transport) return; + const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema; + + await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } }); + + const id = "stream" + this.user_id; + + var stream = this.client.in.stream!; + if (!stream) { + stream = this.client.transport!.createIncomingStream( + // @ts-ignore + SemanticSDP.StreamInfo.expand({ + id, + // @ts-ignore + tracks: [] + }) + ); + this.client.in.stream = stream; + + const interval = setInterval(() => { + for (const track of stream.getTracks()) { + for (const layer of Object.values(track.getStats())) { + console.log(track.getId(), layer.total); + } + } + }, 5000); + + stream.on("stopped", () => { + console.log("stream stopped"); + clearInterval(interval); + }); + this.on("close", () => { + transport!.stop(); + }); + const out = transport.createOutgoingStream( + // @ts-ignore + SemanticSDP.StreamInfo.expand({ + id: "out" + this.user_id, + // @ts-ignore + tracks: [] + }) + ); + this.client.out.stream = out; + + const clients = channels.get(channel_id)!; + + clients.forEach((client) => { + if (client.websocket.user_id === this.user_id) return; + if (!client.in.stream) return; + + client.in.stream?.getTracks().forEach((track) => { + attachTrack.call(this, track, client.websocket.user_id); + }); + }); + } + + if (d.audio_ssrc) { + handleSSRC.call(this, "audio", { media: d.audio_ssrc, rtx: d.audio_ssrc + 1 }); + } + if (d.video_ssrc && d.rtx_ssrc) { + handleSSRC.call(this, "video", { media: d.video_ssrc, rtx: d.rtx_ssrc }); + } +} + +function attachTrack(this: WebSocket, track: IncomingStreamTrack, user_id: string) { + if (!this.client) return; + const outTrack = this.client.transport!.createOutgoingStreamTrack(track.getMedia()); + outTrack.attachTo(track); + this.client.out.stream!.addTrack(outTrack); + var ssrcs = this.client.out.tracks.get(user_id)!; + if (!ssrcs) ssrcs = this.client.out.tracks.set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 }).get(user_id)!; + + if (track.getMedia() === "audio") { + ssrcs.audio_ssrc = outTrack.getSSRCs().media!; + } else if (track.getMedia() === "video") { + ssrcs.video_ssrc = outTrack.getSSRCs().media!; + ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!; + } + + Send(this, { + op: VoiceOPCodes.VIDEO, + d: { + user_id: user_id, + ...ssrcs + } as VoiceVideoSchema + }); +} + +function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) { + if (!this.client) return; + const stream = this.client.in.stream!; + const transport = this.client.transport!; + + const id = type + ssrcs.media; + var track = stream.getTrack(id); + if (!track) { + console.log("createIncomingStreamTrack", id); + track = transport.createIncomingStreamTrack(type, { id, ssrcs }); + stream.addTrack(track); + + const clients = getClients(this.client.channel_id)!; + clients.forEach((client) => { + if (client.websocket.user_id === this.user_id) return; + if (!client.out.stream) return; + + attachTrack.call(this, track, client.websocket.user_id); + }); + } +} \ No newline at end of file diff --git a/src/webrtc/opcodes/index.ts b/src/webrtc/opcodes/index.ts new file mode 100644 index 00000000..8c664cce --- /dev/null +++ b/src/webrtc/opcodes/index.ts @@ -0,0 +1,19 @@ +import { Payload, WebSocket } from "@fosscord/gateway"; +import { VoiceOPCodes } from "../util"; +import { onBackendVersion } from "./BackendVersion"; +import { onHeartbeat } from "./Heartbeat"; +import { onIdentify } from "./Identify"; +import { onSelectProtocol } from "./SelectProtocol"; +import { onSpeaking } from "./Speaking"; +import { onVideo } from "./Video"; + +export type OPCodeHandler = (this: WebSocket, data: Payload) => any; + +export default { + [VoiceOPCodes.HEARTBEAT]: onHeartbeat, + [VoiceOPCodes.IDENTIFY]: onIdentify, + [VoiceOPCodes.VOICE_BACKEND_VERSION]: onBackendVersion, + [VoiceOPCodes.VIDEO]: onVideo, + [VoiceOPCodes.SPEAKING]: onSpeaking, + [VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol +}; \ No newline at end of file diff --git a/src/webrtc/opcodes/sdp.json b/src/webrtc/opcodes/sdp.json new file mode 100644 index 00000000..4867b9c7 --- /dev/null +++ b/src/webrtc/opcodes/sdp.json @@ -0,0 +1,420 @@ +{ + "version": 0, + "streams": [], + "medias": [ + { + "id": "0", + "type": "audio", + "direction": "sendrecv", + "codecs": [ + { + "codec": "opus", + "type": 111, + "channels": 2, + "params": { + "minptime": "10", + "useinbandfec": "1" + }, + "rtcpfbs": [ + { + "id": "transport-cc" + } + ] + } + ], + "extensions": { + "1": "urn:ietf:params:rtp-hdrext:ssrc-audio-level", + "2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", + "3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", + "4": "urn:ietf:params:rtp-hdrext:sdes:mid" + } + }, + { + "id": "1", + "type": "video", + "direction": "sendrecv", + "codecs": [ + { + "codec": "VP8", + "type": 96, + "rtx": 97, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "VP9", + "type": 98, + "rtx": 99, + "params": { + "profile-id": "0" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "VP9", + "type": 100, + "rtx": 101, + "params": { + "profile-id": "2" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "VP9", + "type": 102, + "rtx": 122, + "params": { + "profile-id": "1" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 127, + "rtx": 121, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "1", + "profile-level-id": "42001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 125, + "rtx": 107, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "0", + "profile-level-id": "42001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 108, + "rtx": 109, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "1", + "profile-level-id": "42e01f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 124, + "rtx": 120, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "0", + "profile-level-id": "42e01f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 123, + "rtx": 119, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "1", + "profile-level-id": "4d001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 35, + "rtx": 36, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "0", + "profile-level-id": "4d001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 37, + "rtx": 38, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "1", + "profile-level-id": "f4001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 39, + "rtx": 40, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "0", + "profile-level-id": "f4001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + }, + { + "codec": "H264", + "type": 114, + "rtx": 115, + "params": { + "level-asymmetry-allowed": "1", + "packetization-mode": "1", + "profile-level-id": "64001f" + }, + "rtcpfbs": [ + { + "id": "goog-remb" + }, + { + "id": "transport-cc" + }, + { + "id": "ccm", + "params": ["fir"] + }, + { + "id": "nack" + }, + { + "id": "nack", + "params": ["pli"] + } + ] + } + ], + "extensions": { + "2": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", + "3": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", + "4": "urn:ietf:params:rtp-hdrext:sdes:mid", + "5": "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay", + "6": "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type", + "7": "http://www.webrtc.org/experiments/rtp-hdrext/video-timing", + "8": "http://www.webrtc.org/experiments/rtp-hdrext/color-space", + "10": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id", + "11": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id", + "13": "urn:3gpp:video-orientation", + "14": "urn:ietf:params:rtp-hdrext:toffset" + } + } + ], + "candidates": [] +} \ No newline at end of file |