From 85bd49f5b87957644264a393515baa4152fd7561 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 02:37:38 +1100 Subject: boilerplate stuff --- webrtc/src/opcodes/SelectProtocol.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 webrtc/src/opcodes/SelectProtocol.ts (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts new file mode 100644 index 00000000..f1732dd9 --- /dev/null +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -0,0 +1,16 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onSelectProtocol(socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.SESSION_DESCRIPTION, + d: { + video_codec: "H264", + secret_key: new Array(32).fill(null).map(x => Math.random() * 256), + mode: "aead_aes256_gcm_rtpsize", + media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + audio_codec: "opus", + } + })); +} \ No newline at end of file -- cgit 1.5.1 From b1dc6b34ddb0c27b99f4203043d4f2ba0fcee375 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 03:35:30 +1100 Subject: messing around with things I don't understand --- webrtc/src/Server.ts | 70 +++++++++++++++++++++++++++++++++--- webrtc/src/opcodes/Connect.ts | 3 +- webrtc/src/opcodes/Heartbeat.ts | 3 +- webrtc/src/opcodes/Identify.ts | 7 ++-- webrtc/src/opcodes/Resume.ts | 3 +- webrtc/src/opcodes/SelectProtocol.ts | 5 +-- webrtc/src/opcodes/Speaking.ts | 3 +- 7 files changed, 81 insertions(+), 13 deletions(-) (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 06a36df9..cdda10ec 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,15 +1,19 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, CLOSECODES, Payload, OPCODES } from "@fosscord/gateway"; +import { WebSocket, Payload, } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; -import { setHeartbeat } from "./util" -import mediasoup from "mediasoup"; +import { setHeartbeat } from "./util"; +import * as mediasoup from "mediasoup"; +import { types as MediasoupTypes } from "mediasoup"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; export class Server { public ws: WebSocketServer; + public mediasoupWorkers: MediasoupTypes.Worker[] = []; + public mediasoupRouters: MediasoupTypes.Router[] = []; + public mediasoupTransports: MediasoupTypes.Transport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -23,9 +27,9 @@ export class Server { const payload: Payload = JSON.parse(message); if (OPCodeHandlers[payload.op]) - await OPCodeHandlers[payload.op](socket, payload); + await OPCodeHandlers[payload.op].call(this, socket, payload); else - console.error(`Unimplemented`, payload) + console.error(`Unimplemented`, payload); }); }); } @@ -34,7 +38,63 @@ export class Server { // @ts-ignore await initDatabase(); await Config.init(); + await this.createWorkers(); console.log("[DB] connected"); console.log(`[WebRTC] online on 0.0.0.0:${port}`); } + + async createWorkers(): Promise { + const numWorkers = 1; + for (let i = 0; i < numWorkers; i++) { + const worker = await mediasoup.createWorker(); + if (!worker) return; + + worker.on("died", () => { + console.error("mediasoup worker died"); + }); + + worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { + console.log("new router"); + + this.mediasoupRouters.push(router); + + router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { + console.log("new transport"); + + this.mediasoupTransports.push(transport); + }) + + await router.createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true + }); + }); + + await worker.createRouter({ + mediaCodecs: [ + { + kind: "audio", + mimeType: "audio/opus", + clockRate: 48000, + channels: 2 + }, + { + kind: "video", + mimeType: "video/H264", + clockRate: 90000, + parameters: + { + "packetization-mode": 1, + "profile-level-id": "42e01f", + "level-asymmetry-allowed": 1 + } + } + ] + }); + + this.mediasoupWorkers.push(worker); + } + } } diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5cc66506..5db11638 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onConnect(socket: WebSocket, data: Payload) { +export async function onConnect(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 04150e36..06d6bcb1 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,7 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { setHeartbeat } from "./../util"; +import { Server } from "../Server" -export async function onHeartbeat(socket: WebSocket, data: Payload) { +export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { await setHeartbeat(socket); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 2026d7c9..6043a460 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,14 +1,17 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onIdentify(socket: WebSocket, data: Payload) { +export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { ssrc: 1, ip: "127.0.0.1", - port: 3005, + + //@ts-ignore + port: this.mediasoupTransports[0].iceCandidates.port, modes: [ "xsalsa20_poly1305", "xsalsa20_poly1305_suffix", diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index de21eba6..dcd4f4cd 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onResume(socket: WebSocket, data: Payload) { +export async function onResume(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index f1732dd9..fcc45855 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,15 +1,16 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSelectProtocol(socket: WebSocket, data: Payload) { +export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { video_codec: "H264", secret_key: new Array(32).fill(null).map(x => Math.random() * 256), mode: "aead_aes256_gcm_rtpsize", - media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + media_session_id: this.mediasoupTransports[0].id, audio_codec: "opus", } })); diff --git a/webrtc/src/opcodes/Speaking.ts b/webrtc/src/opcodes/Speaking.ts index 14f86b3c..861a7c3d 100644 --- a/webrtc/src/opcodes/Speaking.ts +++ b/webrtc/src/opcodes/Speaking.ts @@ -1,6 +1,7 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSpeaking(socket: WebSocket, data: Payload) { +export async function onSpeaking(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file -- cgit 1.5.1 From 2e573cc3056da60a45556179687fb4c720af1227 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:59:26 +1100 Subject: more fuckery --- webrtc/package-lock.json | 14 ++++++++ webrtc/package.json | 1 + webrtc/src/Server.ts | 31 ++++++++++------ webrtc/src/opcodes/Connect.ts | 4 +++ webrtc/src/opcodes/Identify.ts | 68 ++++++++++++++++++++++++++++++++--- webrtc/src/opcodes/SelectProtocol.ts | 70 +++++++++++++++++++++++++++++++++--- webrtc/src/start.ts | 6 ++++ 7 files changed, 175 insertions(+), 19 deletions(-) (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 6c3726dc..43bb8cd8 100644 --- a/webrtc/package-lock.json +++ b/webrtc/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", "tsconfig-paths": "^3.12.0", @@ -210,6 +211,14 @@ "node": ">=0.3.1" } }, + "node_modules/dotenv": { + "version": "12.0.4", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-12.0.4.tgz", + "integrity": "sha512-oWdqbSywffzH1l4WXKPHWA0TWYpqp7IyLfqjipT4upoIFS0HPMqtNotykQpD4iIg0BqtNmdgPCh2WMvMt7yTiw==", + "engines": { + "node": ">=12" + } + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -672,6 +681,11 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "dotenv": { + "version": "12.0.4", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-12.0.4.tgz", + "integrity": "sha512-oWdqbSywffzH1l4WXKPHWA0TWYpqp7IyLfqjipT4upoIFS0HPMqtNotykQpD4iIg0BqtNmdgPCh2WMvMt7yTiw==" + }, "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", diff --git a/webrtc/package.json b/webrtc/package.json index 8c66245d..d5a994a1 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -18,6 +18,7 @@ "typescript": "^4.3.2" }, "dependencies": { + "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", "tsconfig-paths": "^3.12.0", diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index cdda10ec..1d2e73e7 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -54,21 +54,32 @@ export class Server { }); worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { - console.log("new router"); + console.log("new router created [id:%s]", router.id); this.mediasoupRouters.push(router); - router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { - console.log("new transport"); + router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + console.log("new transport created [id:%s]", transport.id); - this.mediasoupTransports.push(transport); - }) + await transport.enableTraceEvent(); + + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { + console.log("new producer created [id:%s]", producer.id); + }); + + transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { + console.log("new consumer created [id:%s]", consumer.id); + }); - await router.createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], - enableUdp: true, - enableTcp: true, - preferUdp: true + transport.observer.on("newdataproducer", (dataProducer) => { + console.log("new data producer created [id:%s]", dataProducer.id); + }); + + transport.on("trace", (trace) => { + console.log(trace); + }); + + this.mediasoupTransports.push(transport); }); }); diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5db11638..b312d6f2 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -3,4 +3,8 @@ import { Payload } from "./index"; import { Server } from "../Server" export async function onConnect(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 15, + d: { any: 100 } + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6043a460..6bbed04c 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,9 +1,67 @@ import { WebSocket } from "@fosscord/gateway"; -import { Payload } from "./index" +import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; + +const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { + var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true, + }); + + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + var producer = await transport.produce( + { + kind: "audio", + rtpParameters: + { + mid: "1", + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + id: i + 1, + uri: x, + })) + } + }); + + const consumer = await transport.consume( + { + producerId: producer.id, + rtpCapabilities: + { + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + kind: "audio", + preferredId: i + 1, + uri: x, + })) + } + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { @@ -11,11 +69,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) ip: "127.0.0.1", //@ts-ignore - port: this.mediasoupTransports[0].iceCandidates.port, + port: transport.iceCandidates[0].port, modes: [ "xsalsa20_poly1305", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305_lite", + // "xsalsa20_poly1305_suffix", + // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, }, diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index fcc45855..24e8ef5f 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,17 +1,79 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; + +/* + { + op: 1, + d: { + protocol: "webrtc", + data: " + a=extmap-allow-mixed + a=ice-ufrag:ilWh + a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-options:trickle + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid + a=rtpmap:111 opus/48000/2 + a=extmap:14 urn:ietf:params:rtp-hdrext:toffset + a=extmap:13 urn:3gpp:video-orientation + a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay + a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type + a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing + a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space + a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id + a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id + a=rtpmap:96 VP8/90000 + a=rtpmap:97 rtx/90000 + ", + sdp: "same data as in d.data? also not documented by discord", + codecs: [ + { + name: "opus", + type: "audio", + priority: 1000, + payload_type: 111, + rtx_payload_type: null, + }, + { + name: "H264", + type: "video", + priority: 1000, + payload_type: 102, + rtx_payload_type: 121, + }, + { + name: "VP8", + type: "video", + priority: 2000, + payload_type: 96, + rtx_payload_type: 97, + }, + { + name: "VP9", + type: "video", + priority: 3000, + payload_type: 98, + rtx_payload_type: 99, + }, + ], + rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", + }, + } +*/ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: "H264", + video_codec: data.d.codecs.find((x: any) => x.type === "video").name, secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "aead_aes256_gcm_rtpsize", + mode: "xsalsa20_poly1305", media_session_id: this.mediasoupTransports[0].id, - audio_codec: "opus", + audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, } })); } \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 5614982d..299bfce8 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,4 +1,10 @@ +import { config } from "dotenv"; +config(); + import { Server } from "./Server"; +//testing +process.env.DATABASE = "../bundle/database.db"; + const server = new Server(); server.listen(); \ No newline at end of file -- cgit 1.5.1 From 4847351daa226cbd71fc8676b6be7516c6e76253 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 21 Jan 2022 21:04:45 +1100 Subject: mmmm --- webrtc/package-lock.json | 27 ++++++++++++ webrtc/package.json | 2 + webrtc/src/opcodes/Identify.ts | 79 +++++++++++++----------------------- webrtc/src/opcodes/SelectProtocol.ts | 37 +++++++++++++++++ webrtc/src/opcodes/index.ts | 2 + 5 files changed, 96 insertions(+), 51 deletions(-) (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 43bb8cd8..afba7e76 100644 --- a/webrtc/package-lock.json +++ b/webrtc/package-lock.json @@ -12,11 +12,13 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" }, "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -78,6 +80,12 @@ "integrity": "sha512-7EIraBEyRHEe7CH+Fm1XvgqU6uwZN8Q7jppJGcqjROMT29qhAuuOxYB1uEY5UMYQKEmA5D+5tBnhdaPXSsLONA==", "dev": true }, + "node_modules/@types/sdp-transform": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@types/sdp-transform/-/sdp-transform-2.4.5.tgz", + "integrity": "sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==", + "dev": true + }, "node_modules/@types/ws": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.4.tgz", @@ -392,6 +400,14 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "node_modules/sdp-transform": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.1.tgz", + "integrity": "sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==", + "bin": { + "sdp-verify": "checker.js" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -599,6 +615,12 @@ "integrity": "sha512-7EIraBEyRHEe7CH+Fm1XvgqU6uwZN8Q7jppJGcqjROMT29qhAuuOxYB1uEY5UMYQKEmA5D+5tBnhdaPXSsLONA==", "dev": true }, + "@types/sdp-transform": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@types/sdp-transform/-/sdp-transform-2.4.5.tgz", + "integrity": "sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==", + "dev": true + }, "@types/ws": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.4.tgz", @@ -817,6 +839,11 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "sdp-transform": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.1.tgz", + "integrity": "sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/webrtc/package.json b/webrtc/package.json index d5a994a1..b9bac356 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -13,6 +13,7 @@ "license": "ISC", "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -21,6 +22,7 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" } diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6bbed04c..c31870c8 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -2,10 +2,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; -import * as mediasoup from "mediasoup"; -import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; - -const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ @@ -15,52 +11,31 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) preferUdp: true, }); - const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; - const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; - - var producer = await transport.produce( - { - kind: "audio", - rtpParameters: - { - mid: "1", - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - id: i + 1, - uri: x, - })) - } - }); - - const consumer = await transport.consume( - { - producerId: producer.id, - rtpCapabilities: - { - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - kind: "audio", - preferredId: i + 1, - uri: x, - })) - } - }); + /* + //discord proper sends: + { + "streams": [ + { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, + { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 1311884, + "port": 50008, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.214.158", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] + } + */ socket.send(JSON.stringify({ op: VoiceOPCodes.READY, @@ -71,11 +46,13 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) //@ts-ignore port: transport.iceCandidates[0].port, modes: [ - "xsalsa20_poly1305", + "aead_aes256_gcm_rtpsize", + // "xsalsa20_poly1305", // "xsalsa20_poly1305_suffix", // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, + experiments: [], }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 24e8ef5f..08335ade 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -2,6 +2,9 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; +import * as sdpTransform from 'sdp-transform'; /* { @@ -66,6 +69,40 @@ import { Server } from "../Server"; */ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + const transport = this.mediasoupTransports[0]; //whatever + + const res = sdpTransform.parse(data.d.sdp); + + /* + res.media.map(x => x.rtp).flat(1).map(x => ({ + codec: x.codec, + payloadType: x.payload, + clockRate: x.rate as number, + mimeType: `audio/${x.codec}`, + })), + */ + + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 2fe69c38..36d30e7d 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -32,4 +32,6 @@ export default { //op 11? [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 //op 13? + //op 15? + //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} }; \ No newline at end of file -- cgit 1.5.1 From 958d570574ecd5e3e510fa8203edf27b8760763c Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 4 Feb 2022 18:46:09 +1100 Subject: ;jondfgsk --- webrtc/src/Server.ts | 19 ++++----------- webrtc/src/opcodes/Identify.ts | 7 +++--- webrtc/src/opcodes/SelectProtocol.ts | 46 +++++++++++++++++++++++------------- 3 files changed, 37 insertions(+), 35 deletions(-) (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d2e73e7..dcbf216a 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,5 +1,5 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, Payload, } from "@fosscord/gateway"; +import { WebSocket, Payload, CLOSECODES } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; import { setHeartbeat } from "./util"; @@ -28,8 +28,10 @@ export class Server { if (OPCodeHandlers[payload.op]) await OPCodeHandlers[payload.op].call(this, socket, payload); - else + else { console.error(`Unimplemented`, payload); + socket.close(CLOSECODES.Unknown_opcode); + } }); }); } @@ -46,7 +48,7 @@ export class Server { async createWorkers(): Promise { const numWorkers = 1; for (let i = 0; i < numWorkers; i++) { - const worker = await mediasoup.createWorker(); + const worker = await mediasoup.createWorker({ logLevel: "debug" }); if (!worker) return; worker.on("died", () => { @@ -91,17 +93,6 @@ export class Server { clockRate: 48000, channels: 2 }, - { - kind: "video", - mimeType: "video/H264", - clockRate: 90000, - parameters: - { - "packetization-mode": 1, - "profile-level-id": "42e01f", - "level-asymmetry-allowed": 1 - } - } ] }); diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index c31870c8..82f327be 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -5,7 +5,7 @@ import { Server } from "../Server"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], + listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, preferUdp: true, @@ -40,10 +40,9 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { + streams: [], ssrc: 1, - ip: "127.0.0.1", - - //@ts-ignore + ip: transport.iceCandidates[0].ip, port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 08335ade..36527a8b 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -68,6 +68,8 @@ import * as sdpTransform from 'sdp-transform'; } */ +var test_hasMadeProducer = false; + export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; @@ -85,23 +87,33 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })) - }, - paused: false, - }); + if (!test_hasMadeProducer) { + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + + const consumer = await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }) + + test_hasMadeProducer = true; + } socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, -- cgit 1.5.1 From aa8a9eea6b5475d949ee9124a3f47d8166d019fc Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 19:15:33 +1100 Subject: augh --- bundle/package.json | 2 +- util/src/util/Constants.ts | 5 +-- webrtc/src/Server.ts | 12 +++++-- webrtc/src/opcodes/Heartbeat.ts | 4 +-- webrtc/src/opcodes/Identify.ts | 37 ++++++++++++++++---- webrtc/src/opcodes/SelectProtocol.ts | 68 ++++++++++++++++++++++++++++-------- webrtc/src/opcodes/Version.ts | 14 ++++++++ webrtc/src/opcodes/index.ts | 3 ++ webrtc/src/start.ts | 6 ++-- webrtc/src/util/Heartbeat.ts | 21 ++++++----- 10 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 webrtc/src/opcodes/Version.ts (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/bundle/package.json b/bundle/package.json index 0b00b325..aedd963b 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -112,4 +112,4 @@ "typescript-json-schema": "^0.50.1", "ws": "^7.4.2" } -} +} \ No newline at end of file diff --git a/util/src/util/Constants.ts b/util/src/util/Constants.ts index d5315767..42a2c274 100644 --- a/util/src/util/Constants.ts +++ b/util/src/util/Constants.ts @@ -77,8 +77,9 @@ export const VoiceOPCodes = { RESUME: 7, HELLO: 8, RESUMED: 9, - CLIENT_CONNECT: 12, - CLIENT_DISCONNECT: 13, + CLIENT_CONNECT: 12, // incorrect, op 12 is probably used for video + CLIENT_DISCONNECT: 13, // incorrect + VERSION: 16, //not documented }; export const Events = { diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 0145a221..1d18d6d1 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,6 +6,8 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; +import Net from "net"; + var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -13,7 +15,7 @@ export class Server { public ws: WebSocketServer; public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; - public mediasoupTransports: MediasoupTypes.Transport[] = []; + public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -26,7 +28,7 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - console.log(payload); + // console.log(payload); if (OPCodeHandlers[payload.op]) try { @@ -68,9 +70,13 @@ export class Server { this.mediasoupRouters.push(router); - router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); + transport.observer.on("sctpstatechange", (state) => { + console.log(state) + }); + await transport.enableTraceEvent(); transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 06d6bcb1..47f33f76 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,8 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; -import { setHeartbeat } from "./../util"; +import { setHeartbeat } from "../util"; import { Server } from "../Server" export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { - await setHeartbeat(socket); + await setHeartbeat(socket, data.d); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index e965e3de..d7da5c7c 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -28,12 +28,12 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } ); const user = session.user; - const guild = await Guild.findOneOrFail({ id: data.d.server_id }); + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); if (!guild.members.find(x => x.id === user.id)) return socket.close(CLOSECODES.Invalid_intent); - var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, @@ -66,13 +66,39 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } */ + + + /* + { + "streams": [ + { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 129860, + "port": 50003, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.213.251", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ]; + }; + */ + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { - streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))], - ssrc: 1, + streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], + ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: transport.iceCandidates[0].port, + port: "50001", modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -81,7 +107,6 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - heartbeat_interval: 1, experiments: [], }, })); diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 36527a8b..a957e14f 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -87,42 +87,82 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") + if (!test_hasMadeProducer) { const producer = await transport.produce({ kind: "audio", rtpParameters: { mid: "audio", codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, }], headerExtensions: res.ext?.map(x => ({ id: x.value, uri: x.uri, - })) + })), }, paused: false, }); - + const consumer = await transport.consume({ producerId: producer.id, - paused: false, + paused: true, rtpCapabilities, - }) - + }); + test_hasMadeProducer = true; } + /* server sends sdp: + + m=audio 50021 ICE/SDP //same port as sent in READY + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.213.132 //same IP as sent in READY + a=rtcp:50021 //same port? + a=ice-ufrag:rTmX + a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT + + */ + + + var test = { + "video_codec": "H264", + "sdp": ` + m=audio 50011 ICE/SDP\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + c=IN IP4 109.200.214.156\n + a=rtcp:50011\n + a=ice-ufrag:d0aZ\n + a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, + "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", + "audio_codec": "opus" + }; + + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: data.d.codecs.find((x: any) => x.type === "video").name, - secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "xsalsa20_poly1305", - media_session_id: this.mediasoupTransports[0].id, - audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, + video_codec: videoCodec?.substring(6) || undefined, + // mode: "xsalsa20_poly1305", + media_session_id: transport.id, + audio_codec: audioCodec?.mimeType.substring(6), + sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` + + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + + `a=ice-pwd:${transport.iceParameters.password}\n` + + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` + + `a=candidate:1 1 ${transport.iceCandidates[0].protocol} ${transport.iceCandidates[0].priority} ${transport.iceCandidates[0].ip} ${transport.iceCandidates[0].port} typ ${transport.iceCandidates[0].type}` } })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Version.ts b/webrtc/src/opcodes/Version.ts new file mode 100644 index 00000000..0ea6eb4d --- /dev/null +++ b/webrtc/src/opcodes/Version.ts @@ -0,0 +1,14 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { setHeartbeat } from "../util"; +import { Server } from "../Server" + +export async function onVersion(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 16, + d: { + voice: "0.8.31", //version numbers? + rtc_worker: "0.3.18", + } + })) +} \ No newline at end of file diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 9b1eb270..d0f40bc2 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -15,6 +15,8 @@ import { onSpeaking } from "./Speaking"; import { onResume } from "./Resume"; import { onConnect } from "./Connect"; +import { onVersion } from "./Version"; + export type OPCodeHandler = (this: WebSocket, data: Payload) => any; export default { @@ -34,4 +36,5 @@ export default { //op 13? //op 15? //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} + [VoiceOPCodes.VERSION]: onVersion, }; \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 299bfce8..98f06ad5 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,10 +1,10 @@ +//testing +process.env.DATABASE = "../bundle/database.db"; + import { config } from "dotenv"; config(); import { Server } from "./Server"; -//testing -process.env.DATABASE = "../bundle/database.db"; - const server = new Server(); server.listen(); \ No newline at end of file diff --git a/webrtc/src/util/Heartbeat.ts b/webrtc/src/util/Heartbeat.ts index 7b5ed9cd..8c5e3a7a 100644 --- a/webrtc/src/util/Heartbeat.ts +++ b/webrtc/src/util/Heartbeat.ts @@ -1,18 +1,23 @@ import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { VoiceOPCodes } from "@fosscord/util"; -export async function setHeartbeat(socket: WebSocket) { +export async function setHeartbeat(socket: WebSocket, nonce?: Number) { if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout); socket.heartbeatTimeout = setTimeout(() => { return socket.close(CLOSECODES.Session_timed_out); }, 1000 * 45); - socket.send(JSON.stringify({ - op: VoiceOPCodes.HEARTBEAT_ACK, - d: { - v: 6, - heartbeat_interval: 13750, - } - })); + if (!nonce) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.HELLO, + d: { + v: 5, + heartbeat_interval: 13750, + } + })); + } + else { + socket.send(JSON.stringify({ op: VoiceOPCodes.HEARTBEAT_ACK, d: nonce })); + } } \ No newline at end of file -- cgit 1.5.1 From 69bcbf0475c4c5a84477d3e4cb862894db3052da Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 22:57:37 +1100 Subject: VOICE CONNECTS!!! Dtls stuck on "connecting" state + currently no way to edit/inspect packets received or use own packet format in mediasoup ( fork? ) --- webrtc/src/Server.ts | 23 ++-- webrtc/src/opcodes/Connect.ts | 32 +++++- webrtc/src/opcodes/Identify.ts | 63 ++--------- webrtc/src/opcodes/Resume.ts | 20 +++- webrtc/src/opcodes/SelectProtocol.ts | 200 ++++++++++++++++------------------- 5 files changed, 165 insertions(+), 173 deletions(-) (limited to 'webrtc/src/opcodes/SelectProtocol.ts') diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d18d6d1..42b82c6a 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,7 +6,7 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; -import Net from "net"; +import udp from "dgram"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -16,6 +16,8 @@ export class Server { public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; + public mediasoupProducers: MediasoupTypes.Producer[] = []; + public mediasoupConsumers: MediasoupTypes.Consumer[] = []; constructor() { this.ws = new WebSocketServer({ @@ -28,8 +30,6 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - // console.log(payload); - if (OPCodeHandlers[payload.op]) try { await OPCodeHandlers[payload.op].call(this, socket, payload); @@ -44,6 +44,7 @@ export class Server { } }); }); + } async listen(): Promise { @@ -73,18 +74,26 @@ export class Server { router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); - transport.observer.on("sctpstatechange", (state) => { - console.log(state) - }); - await transport.enableTraceEvent(); + transport.on("connect", () => { + console.log("transport connect") + }) + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { console.log("new producer created [id:%s]", producer.id); + + this.mediasoupProducers.push(producer); }); transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { console.log("new consumer created [id:%s]", consumer.id); + + this.mediasoupConsumers.push(consumer); + + consumer.on("rtp", (rtpPacket) => { + console.log(rtpPacket); + }); }); transport.observer.on("newdataproducer", (dataProducer) => { diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index b312d6f2..1f874a44 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -2,8 +2,38 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +/* +Sent by client: + +{ + "op": 12, + "d": { + "audio_ssrc": 0, + "video_ssrc": 0, + "rtx_ssrc": 0, + "streams": [ + { + "type": "video", + "rid": "100", + "ssrc": 0, + "active": false, + "quality": 100, + "rtx_ssrc": 0, + "max_bitrate": 2500000, + "max_framerate": 20, + "max_resolution": { + "type": "fixed", + "width": 1280, + "height": 720 + } + } + ] + } +} +*/ + export async function onConnect(this: Server, socket: WebSocket, data: Payload) { - socket.send(JSON.stringify({ + socket.send(JSON.stringify({ //what is op 15? op: 15, d: { any: 100 } })) diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index d7da5c7c..9baa16e3 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -34,71 +34,20 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify return socket.close(CLOSECODES.Invalid_intent); var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], + listenIps: [{ ip: "10.22.64.69" }], enableUdp: true, enableTcp: true, preferUdp: true, + enableSctp: true, }); - /* - //discord proper sends: - { - "streams": [ - { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, - { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 1311884, - "port": 50008, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.214.158", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ] - } - */ - - - - /* - { - "streams": [ - { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 129860, - "port": 50003, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.213.251", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ]; - }; - */ - socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: "50001", + port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -107,7 +56,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - experiments: [], + experiments: [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index dcd4f4cd..856b550c 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,6 +1,24 @@ -import { WebSocket } from "@fosscord/gateway"; +import { CLOSECODES, WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +import { Guild, Session, VoiceOPCodes } from "@fosscord/util"; export async function onResume(this: Server, socket: WebSocket, data: Payload) { + const session = await Session.findOneOrFail( + { session_id: data.d.session_id, }, + { + where: { user_id: data.d.user_id }, + relations: ["user"] + } + ); + const user = session.user; + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); + + if (!guild.members.find(x => x.id === user.id)) + return socket.close(CLOSECODES.Invalid_intent); + + socket.send(JSON.stringify({ + op: VoiceOPCodes.RESUMED, + d: null, + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index a957e14f..dc9d2b88 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -6,15 +6,18 @@ import * as mediasoup from "mediasoup"; import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; import * as sdpTransform from 'sdp-transform'; + /* - { - op: 1, - d: { - protocol: "webrtc", - data: " + + Sent by client: +{ + "op": 1, + "d": { + "protocol": "webrtc", + "data": " a=extmap-allow-mixed - a=ice-ufrag:ilWh - a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-ufrag:vNxb + a=ice-pwd:tZvpbVPYEKcnW0gGRPq0OOnh a=ice-options:trickle a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time @@ -32,43 +35,63 @@ import * as sdpTransform from 'sdp-transform'; a=rtpmap:96 VP8/90000 a=rtpmap:97 rtx/90000 ", - sdp: "same data as in d.data? also not documented by discord", - codecs: [ - { - name: "opus", - type: "audio", - priority: 1000, - payload_type: 111, - rtx_payload_type: null, - }, - { - name: "H264", - type: "video", - priority: 1000, - payload_type: 102, - rtx_payload_type: 121, - }, - { - name: "VP8", - type: "video", - priority: 2000, - payload_type: 96, - rtx_payload_type: 97, - }, - { - name: "VP9", - type: "video", - priority: 3000, - payload_type: 98, - rtx_payload_type: 99, - }, + "codecs": [ + { + "name": "opus", + "type": "audio", + "priority": 1000, + "payload_type": 111, + "rtx_payload_type": null + }, + { + "name": "H264", + "type": "video", + "priority": 1000, + "payload_type": 102, + "rtx_payload_type": 121 + }, + { + "name": "VP8", + "type": "video", + "priority": 2000, + "payload_type": 96, + "rtx_payload_type": 97 + }, + { + "name": "VP9", + "type": "video", + "priority": 3000, + "payload_type": 98, + "rtx_payload_type": 99 + } ], - rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", - }, + "rtc_connection_id": "3faa0b80-b3e2-4bae-b291-273801fbb7ab" + } +} + +Sent by server: + +{ + "op": 4, + "d": { + "video_codec": "H264", + "sdp": " + m=audio 50001 ICE/SDP + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.214.158 + a=rtcp:50001 + a=ice-ufrag:CLzn + a=ice-pwd:qEmIcNwigd07mu46Ok0XCh + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.214.158 50001 typ host + ", + "media_session_id": "807955cb953e98c5b90704cf048e81ec", + "audio_codec": "opus" } +} + */ -var test_hasMadeProducer = false; export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; @@ -78,87 +101,46 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa const res = sdpTransform.parse(data.d.sdp); - /* - res.media.map(x => x.rtp).flat(1).map(x => ({ - codec: x.codec, - payloadType: x.payload, - clockRate: x.rate as number, - mimeType: `audio/${x.codec}`, + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video"); + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio"); + + const producer = this.mediasoupProducers[0] || await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, })), - */ - - const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType - const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") - - if (!test_hasMadeProducer) { - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: audioCodec!.clockRate, - payloadType: audioCodec!.preferredPayloadType as number, - mimeType: audioCodec!.mimeType, - channels: audioCodec?.channels, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })), - }, - paused: false, - }); - - const consumer = await transport.consume({ - producerId: producer.id, - paused: true, - rtpCapabilities, - }); - - test_hasMadeProducer = true; - } - - /* server sends sdp: - - m=audio 50021 ICE/SDP //same port as sent in READY - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - c=IN IP4 109.200.213.132 //same IP as sent in READY - a=rtcp:50021 //same port? - a=ice-ufrag:rTmX - a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT - - */ - + }, + paused: false, + }); - var test = { - "video_codec": "H264", - "sdp": ` - m=audio 50011 ICE/SDP\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - c=IN IP4 109.200.214.156\n - a=rtcp:50011\n - a=ice-ufrag:d0aZ\n - a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, - "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", - "audio_codec": "opus" - }; + console.log("can consume: " + this.mediasoupRouters[0].canConsume({ producerId: producer.id, rtpCapabilities: rtpCapabilities })); + const consumer = this.mediasoupConsumers[0] || await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }); socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: videoCodec?.substring(6) || undefined, - // mode: "xsalsa20_poly1305", + video_codec: videoCodec?.mimeType?.substring(6) || undefined, + mode: "xsalsa20_poly1305_lite", media_session_id: transport.id, audio_codec: audioCodec?.mimeType.substring(6), sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` - + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=rtcp: ${transport.iceCandidates[0].port}\n` + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + `a=ice-pwd:${transport.iceParameters.password}\n` + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` -- cgit 1.5.1