diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts
index 1d18d6d1..42b82c6a 100644
--- a/webrtc/src/Server.ts
+++ b/webrtc/src/Server.ts
@@ -6,7 +6,7 @@ import { setHeartbeat } from "./util";
import * as mediasoup from "mediasoup";
import { types as MediasoupTypes } from "mediasoup";
-import Net from "net";
+import udp from "dgram";
var port = Number(process.env.PORT);
if (isNaN(port)) port = 3004;
@@ -16,6 +16,8 @@ export class Server {
public mediasoupWorkers: MediasoupTypes.Worker[] = [];
public mediasoupRouters: MediasoupTypes.Router[] = [];
public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = [];
+ public mediasoupProducers: MediasoupTypes.Producer[] = [];
+ public mediasoupConsumers: MediasoupTypes.Consumer[] = [];
constructor() {
this.ws = new WebSocketServer({
@@ -28,8 +30,6 @@ export class Server {
socket.on("message", async (message: string) => {
const payload: Payload = JSON.parse(message);
- // console.log(payload);
-
if (OPCodeHandlers[payload.op])
try {
await OPCodeHandlers[payload.op].call(this, socket, payload);
@@ -44,6 +44,7 @@ export class Server {
}
});
});
+
}
async listen(): Promise<void> {
@@ -73,18 +74,26 @@ export class Server {
router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => {
console.log("new transport created [id:%s]", transport.id);
- transport.observer.on("sctpstatechange", (state) => {
- console.log(state)
- });
-
await transport.enableTraceEvent();
+ transport.on("connect", () => {
+ console.log("transport connect")
+ })
+
transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => {
console.log("new producer created [id:%s]", producer.id);
+
+ this.mediasoupProducers.push(producer);
});
transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => {
console.log("new consumer created [id:%s]", consumer.id);
+
+ this.mediasoupConsumers.push(consumer);
+
+ consumer.on("rtp", (rtpPacket) => {
+ console.log(rtpPacket);
+ });
});
transport.observer.on("newdataproducer", (dataProducer) => {
diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts
index b312d6f2..1f874a44 100644
--- a/webrtc/src/opcodes/Connect.ts
+++ b/webrtc/src/opcodes/Connect.ts
@@ -2,8 +2,38 @@ import { WebSocket } from "@fosscord/gateway";
import { Payload } from "./index";
import { Server } from "../Server"
+/*
+Sent by client:
+
+{
+ "op": 12,
+ "d": {
+ "audio_ssrc": 0,
+ "video_ssrc": 0,
+ "rtx_ssrc": 0,
+ "streams": [
+ {
+ "type": "video",
+ "rid": "100",
+ "ssrc": 0,
+ "active": false,
+ "quality": 100,
+ "rtx_ssrc": 0,
+ "max_bitrate": 2500000,
+ "max_framerate": 20,
+ "max_resolution": {
+ "type": "fixed",
+ "width": 1280,
+ "height": 720
+ }
+ }
+ ]
+ }
+}
+*/
+
export async function onConnect(this: Server, socket: WebSocket, data: Payload) {
- socket.send(JSON.stringify({
+ socket.send(JSON.stringify({ //what is op 15?
op: 15,
d: { any: 100 }
}))
diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts
index d7da5c7c..9baa16e3 100644
--- a/webrtc/src/opcodes/Identify.ts
+++ b/webrtc/src/opcodes/Identify.ts
@@ -34,71 +34,20 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
return socket.close(CLOSECODES.Invalid_intent);
var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({
- listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }],
+ listenIps: [{ ip: "10.22.64.69" }],
enableUdp: true,
enableTcp: true,
preferUdp: true,
+ enableSctp: true,
});
- /*
- //discord proper sends:
- {
- "streams": [
- { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false },
- { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false }
- ],
- "ssrc": 1311884,
- "port": 50008,
- "modes": [
- "aead_aes256_gcm_rtpsize",
- "aead_aes256_gcm",
- "xsalsa20_poly1305_lite_rtpsize",
- "xsalsa20_poly1305_lite",
- "xsalsa20_poly1305_suffix",
- "xsalsa20_poly1305"
- ],
- "ip": "109.200.214.158",
- "experiments": [
- "bwe_conservative_link_estimate",
- "bwe_remote_locus_client",
- "fixed_keyframe_interval"
- ]
- }
- */
-
-
-
- /*
- {
- "streams": [
- { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false }
- ],
- "ssrc": 129860,
- "port": 50003,
- "modes": [
- "aead_aes256_gcm_rtpsize",
- "aead_aes256_gcm",
- "xsalsa20_poly1305_lite_rtpsize",
- "xsalsa20_poly1305_lite",
- "xsalsa20_poly1305_suffix",
- "xsalsa20_poly1305"
- ],
- "ip": "109.200.213.251",
- "experiments": [
- "bwe_conservative_link_estimate",
- "bwe_remote_locus_client",
- "fixed_keyframe_interval"
- ];
- };
- */
-
socket.send(JSON.stringify({
op: VoiceOPCodes.READY,
d: {
streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))],
ssrc: Math.floor(Math.random() * 10000),
ip: transport.iceCandidates[0].ip,
- port: "50001",
+ port: transport.iceCandidates[0].port,
modes: [
"aead_aes256_gcm_rtpsize",
"aead_aes256_gcm",
@@ -107,7 +56,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
"xsalsa20_poly1305_suffix",
"xsalsa20_poly1305"
],
- experiments: [],
+ experiments: [
+ "bwe_conservative_link_estimate",
+ "bwe_remote_locus_client",
+ "fixed_keyframe_interval"
+ ]
},
}));
}
\ No newline at end of file
diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts
index dcd4f4cd..856b550c 100644
--- a/webrtc/src/opcodes/Resume.ts
+++ b/webrtc/src/opcodes/Resume.ts
@@ -1,6 +1,24 @@
-import { WebSocket } from "@fosscord/gateway";
+import { CLOSECODES, WebSocket } from "@fosscord/gateway";
import { Payload } from "./index";
import { Server } from "../Server"
+import { Guild, Session, VoiceOPCodes } from "@fosscord/util";
export async function onResume(this: Server, socket: WebSocket, data: Payload) {
+ const session = await Session.findOneOrFail(
+ { session_id: data.d.session_id, },
+ {
+ where: { user_id: data.d.user_id },
+ relations: ["user"]
+ }
+ );
+ const user = session.user;
+ const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] });
+
+ if (!guild.members.find(x => x.id === user.id))
+ return socket.close(CLOSECODES.Invalid_intent);
+
+ socket.send(JSON.stringify({
+ op: VoiceOPCodes.RESUMED,
+ d: null,
+ }))
}
\ No newline at end of file
diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts
index a957e14f..dc9d2b88 100644
--- a/webrtc/src/opcodes/SelectProtocol.ts
+++ b/webrtc/src/opcodes/SelectProtocol.ts
@@ -6,15 +6,18 @@ import * as mediasoup from "mediasoup";
import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters";
import * as sdpTransform from 'sdp-transform';
+
/*
- {
- op: 1,
- d: {
- protocol: "webrtc",
- data: "
+
+ Sent by client:
+{
+ "op": 1,
+ "d": {
+ "protocol": "webrtc",
+ "data": "
a=extmap-allow-mixed
- a=ice-ufrag:ilWh
- a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ
+ a=ice-ufrag:vNxb
+ a=ice-pwd:tZvpbVPYEKcnW0gGRPq0OOnh
a=ice-options:trickle
a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
@@ -32,43 +35,63 @@ import * as sdpTransform from 'sdp-transform';
a=rtpmap:96 VP8/90000
a=rtpmap:97 rtx/90000
",
- sdp: "same data as in d.data? also not documented by discord",
- codecs: [
- {
- name: "opus",
- type: "audio",
- priority: 1000,
- payload_type: 111,
- rtx_payload_type: null,
- },
- {
- name: "H264",
- type: "video",
- priority: 1000,
- payload_type: 102,
- rtx_payload_type: 121,
- },
- {
- name: "VP8",
- type: "video",
- priority: 2000,
- payload_type: 96,
- rtx_payload_type: 97,
- },
- {
- name: "VP9",
- type: "video",
- priority: 3000,
- payload_type: 98,
- rtx_payload_type: 99,
- },
+ "codecs": [
+ {
+ "name": "opus",
+ "type": "audio",
+ "priority": 1000,
+ "payload_type": 111,
+ "rtx_payload_type": null
+ },
+ {
+ "name": "H264",
+ "type": "video",
+ "priority": 1000,
+ "payload_type": 102,
+ "rtx_payload_type": 121
+ },
+ {
+ "name": "VP8",
+ "type": "video",
+ "priority": 2000,
+ "payload_type": 96,
+ "rtx_payload_type": 97
+ },
+ {
+ "name": "VP9",
+ "type": "video",
+ "priority": 3000,
+ "payload_type": 98,
+ "rtx_payload_type": 99
+ }
],
- rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104",
- },
+ "rtc_connection_id": "3faa0b80-b3e2-4bae-b291-273801fbb7ab"
+ }
+}
+
+Sent by server:
+
+{
+ "op": 4,
+ "d": {
+ "video_codec": "H264",
+ "sdp": "
+ m=audio 50001 ICE/SDP
+ a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+ c=IN IP4 109.200.214.158
+ a=rtcp:50001
+ a=ice-ufrag:CLzn
+ a=ice-pwd:qEmIcNwigd07mu46Ok0XCh
+ a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+ a=candidate:1 1 UDP 4261412862 109.200.214.158 50001 typ host
+ ",
+ "media_session_id": "807955cb953e98c5b90704cf048e81ec",
+ "audio_codec": "opus"
}
+}
+
*/
-var test_hasMadeProducer = false;
export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) {
const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities;
@@ -78,87 +101,46 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa
const res = sdpTransform.parse(data.d.sdp);
- /*
- res.media.map(x => x.rtp).flat(1).map(x => ({
- codec: x.codec,
- payloadType: x.payload,
- clockRate: x.rate as number,
- mimeType: `audio/${x.codec}`,
+ const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video");
+ const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio");
+
+ const producer = this.mediasoupProducers[0] || await transport.produce({
+ kind: "audio",
+ rtpParameters: {
+ mid: "audio",
+ codecs: [{
+ clockRate: audioCodec!.clockRate,
+ payloadType: audioCodec!.preferredPayloadType as number,
+ mimeType: audioCodec!.mimeType,
+ channels: audioCodec?.channels,
+ }],
+ headerExtensions: res.ext?.map(x => ({
+ id: x.value,
+ uri: x.uri,
})),
- */
-
- const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType
- const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio")
-
- if (!test_hasMadeProducer) {
- const producer = await transport.produce({
- kind: "audio",
- rtpParameters: {
- mid: "audio",
- codecs: [{
- clockRate: audioCodec!.clockRate,
- payloadType: audioCodec!.preferredPayloadType as number,
- mimeType: audioCodec!.mimeType,
- channels: audioCodec?.channels,
- }],
- headerExtensions: res.ext?.map(x => ({
- id: x.value,
- uri: x.uri,
- })),
- },
- paused: false,
- });
-
- const consumer = await transport.consume({
- producerId: producer.id,
- paused: true,
- rtpCapabilities,
- });
-
- test_hasMadeProducer = true;
- }
-
- /* server sends sdp:
-
- m=audio 50021 ICE/SDP //same port as sent in READY
- a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
- c=IN IP4 109.200.213.132 //same IP as sent in READY
- a=rtcp:50021 //same port?
- a=ice-ufrag:rTmX
- a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA
- a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
- a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT
-
- */
-
+ },
+ paused: false,
+ });
- var test = {
- "video_codec": "H264",
- "sdp": `
- m=audio 50011 ICE/SDP\n
- a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
- c=IN IP4 109.200.214.156\n
- a=rtcp:50011\n
- a=ice-ufrag:d0aZ\n
- a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n
- a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
- a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`,
- "media_session_id": "9e18c981687f2de5399edd5cb3f3babf",
- "audio_codec": "opus"
- };
+ console.log("can consume: " + this.mediasoupRouters[0].canConsume({ producerId: producer.id, rtpCapabilities: rtpCapabilities }));
+ const consumer = this.mediasoupConsumers[0] || await transport.consume({
+ producerId: producer.id,
+ paused: false,
+ rtpCapabilities,
+ });
socket.send(JSON.stringify({
op: VoiceOPCodes.SESSION_DESCRIPTION,
d: {
- video_codec: videoCodec?.substring(6) || undefined,
- // mode: "xsalsa20_poly1305",
+ video_codec: videoCodec?.mimeType?.substring(6) || undefined,
+ mode: "xsalsa20_poly1305_lite",
media_session_id: transport.id,
audio_codec: audioCodec?.mimeType.substring(6),
sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n`
+ `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n`
+ `c=IN IPV4 ${transport.iceCandidates[0].ip}\n`
- + `a=rtcp:${transport.iceCandidates[0].port}\n`
+ + `a=rtcp: ${transport.iceCandidates[0].port}\n`
+ `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n`
+ `a=ice-pwd:${transport.iceParameters.password}\n`
+ `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n`
|
