Windows screenshare/video call support, general call improvements (#1725)

* Initial support for d3d11 qml video item

* Windows screenshare support, D3D11

* misc fixes

* further window visibility checks

* preview updates

* fix qml preview

* fix compositor

* add libnice plugin dep

* re-run formatter

* final formatter fix [skip ci]

* fix tumbleweed build

---------

Co-authored-by: Joseph Donofry <rubberduckie3554@gmail.com>

4 files changed, 237 insertions, 64 deletions

diff --git a/src/voip/CallManager.cpp b/src/voip/CallManager.cpp
index ac3ec8ee..58aaa1a4 100644
--- a/src/voip/CallManager.cpp
+++ b/src/voip/CallManager.cpp
@@ -34,6 +34,10 @@
 #include <xcb/xcb_ewmh.h>
 #endif
 
+#ifdef Q_OS_WINDOWS
+#include <Windows.h>
+#endif
+
 #ifdef GSTREAMER_AVAILABLE
 extern "C"
 {
@@ -82,12 +86,12 @@ CallManager::CallManager(QObject *parent)
 {
 #ifdef GSTREAMER_AVAILABLE
     std::string errorMessage;
-    if (session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
-        screenShareTypes_.push_back(ScreenShareType::XDP);
-        screenShareType_ = ScreenShareType::XDP;
-    }
 
-    if (std::getenv("DISPLAY")) {
+    if (QGuiApplication::platformName() == QStringLiteral("windows") &&
+        session_.havePlugins(true, true, ScreenShareType::D3D11, &errorMessage)) {
+        screenShareType_ = ScreenShareType::D3D11;
+        screenShareTypes_.push_back(ScreenShareType::D3D11);
+    } else if (std::getenv("DISPLAY")) {
         screenShareTypes_.push_back(ScreenShareType::X11);
         if (QGuiApplication::platformName() != QStringLiteral("wayland")) {
             // Selected by default
@@ -96,6 +100,12 @@ CallManager::CallManager(QObject *parent)
                 std::swap(screenShareTypes_[0], screenShareTypes_[1]);
         }
     }
+
+    if (QGuiApplication::platformName() != QStringLiteral("windows") &&
+        session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
+        screenShareTypes_.push_back(ScreenShareType::XDP);
+        screenShareType_ = ScreenShareType::XDP;
+    }
 #endif
 
     connect(
@@ -254,7 +264,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
 
 #ifdef GSTREAMER_AVAILABLE
     if (callType == CallType::SCREEN) {
-        if (screenShareType_ == ScreenShareType::X11) {
+        if (screenShareType_ == ScreenShareType::X11 ||
+            screenShareType_ == ScreenShareType::D3D11) {
             if (windows_.empty() || windowIndex >= windows_.size()) {
                 nhlog::ui()->error("WebRTC: window index out of range");
                 return;
@@ -270,8 +281,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
 #endif
 
     if (haveCallInvite_) {
-        nhlog::ui()->debug(
-          "WebRTC: Discarding outbound call for inbound call. localUser is polite party");
+        nhlog::ui()->debug("WebRTC: Discarding outbound call for inbound call. "
+                           "localUser is polite party");
         if (callParty_ == callee->user_id) {
             if (callType == callType_)
                 acceptInvite();
@@ -305,7 +316,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
     playRingtone(QUrl(QStringLiteral("qrc:/media/media/ringback.ogg")), true);
 
     uint32_t shareWindowId =
-      callType == CallType::SCREEN && screenShareType_ == ScreenShareType::X11
+      callType == CallType::SCREEN &&
+          (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11)
         ? windows_[windowIndex].second
         : 0;
     if (!session_.createOffer(callType, screenShareType_, shareWindowId)) {
@@ -337,7 +349,7 @@ callHangUpReasonString(CallHangUp::Reason reason)
         return "User";
     }
 }
-}
+} // namespace
 
 void
 CallManager::hangUp(CallHangUp::Reason reason)
@@ -511,8 +523,8 @@ CallManager::handleEvent(const RoomEvent<CallInvite> &callInviteEvent)
 void
 CallManager::acceptInvite()
 {
-    // if call was accepted/rejected elsewhere and m.call.select_answer is received
-    // before acceptInvite
+    // if call was accepted/rejected elsewhere and m.call.select_answer is
+    // received before acceptInvite
     if (!haveCallInvite_)
         return;
 
@@ -699,7 +711,8 @@ CallManager::handleEvent(const RoomEvent<CallReject> &callRejectEvent)
 
     if (callRejectEvent.content.call_id == callid_) {
         if (session_.state() == webrtc::State::OFFERSENT) {
-            // only accept reject if webrtc is in OFFERSENT state, else call has been accepted
+            // only accept reject if webrtc is in OFFERSENT state, else call has been
+            // accepted
             emit newMessage(
               roomid_,
               CallSelectAnswer{
@@ -861,7 +874,7 @@ bool
 CallManager::screenShareReady() const
 {
 #ifdef GSTREAMER_AVAILABLE
-    if (screenShareType_ == ScreenShareType::X11) {
+    if (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11) {
         return true;
     } else {
         return ScreenCastPortal::instance().ready();
@@ -875,12 +888,15 @@ QStringList
 CallManager::screenShareTypeList()
 {
     QStringList ret;
-    ret.reserve(2);
+    ret.reserve(3);
     for (ScreenShareType type : screenShareTypes_) {
         switch (type) {
         case ScreenShareType::X11:
             ret.append(tr("X11"));
             break;
+        case ScreenShareType::D3D11:
+            ret.append("DirectX 11");
+            break;
         case ScreenShareType::XDP:
             ret.append(tr("PipeWire"));
             break;
@@ -893,8 +909,10 @@ CallManager::screenShareTypeList()
 QStringList
 CallManager::windowList()
 {
-    if (!(std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) !=
-          screenShareTypes_.end())) {
+    if (std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) ==
+          screenShareTypes_.end() &&
+        std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::D3D11) ==
+          screenShareTypes_.end()) {
         return {};
     }
 
@@ -950,6 +968,32 @@ CallManager::windowList()
         xcb_ewmh_get_windows_reply_wipe(&clients);
     }
 #endif
+#ifdef Q_OS_WINDOWS
+    for (HWND windowHandle = GetTopWindow(nullptr); windowHandle != nullptr;
+         windowHandle      = GetNextWindow(windowHandle, GW_HWNDNEXT)) {
+        if (!IsWindowVisible(windowHandle))
+            continue;
+
+        int titleLength = GetWindowTextLengthW(windowHandle);
+        if (titleLength == 0)
+            continue;
+
+        if (GetWindowLong(windowHandle, GWL_EXSTYLE) & WS_EX_TOOLWINDOW)
+            continue;
+
+        TITLEBARINFO titleInfo;
+        titleInfo.cbSize = sizeof(titleInfo);
+        GetTitleBarInfo(windowHandle, &titleInfo);
+        if (titleInfo.rgstate[0] & STATE_SYSTEM_INVISIBLE)
+            continue;
+
+        wchar_t *windowTitle = new wchar_t[titleLength + 1];
+        GetWindowTextW(windowHandle, windowTitle, titleLength + 1);
+
+        windows_.push_back(
+          {QString::fromWCharArray(windowTitle), reinterpret_cast<uint64_t>(windowHandle)});
+    }
+#endif
     QStringList ret;
     assert(windows_.size() < std::numeric_limits<int>::max());
     ret.reserve(static_cast<int>(windows_.size()));
@@ -1007,11 +1051,13 @@ make_preview_sink()
 {
     if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
         return gst_element_factory_make("waylandsink", nullptr);
+    } else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
+        return gst_element_factory_make("d3d11videosink", nullptr);
     } else {
         return gst_element_factory_make("ximagesink", nullptr);
     }
 }
-}
+} // namespace
 #endif
 
 void
@@ -1032,6 +1078,12 @@ CallManager::previewWindow(unsigned int index) const
         return;
     }
 
+    if (screenShareType_ == ScreenShareType::D3D11 &&
+        (windows_.empty() || index >= windows_.size())) {
+        nhlog::ui()->error("D3D11 screencast not available");
+        return;
+    }
+
     auto settings = ChatPage::instance()->userSettings();
 
     pipe_ = gst_pipeline_new(nullptr);
@@ -1066,6 +1118,19 @@ CallManager::previewWindow(unsigned int index) const
 
         gst_bin_add(GST_BIN(pipe_), ximagesrc);
         screencastsrc = ximagesrc;
+    } else if (screenShareType_ == ScreenShareType::D3D11) {
+        GstElement *d3d11screensrc = gst_element_factory_make("d3d11screencapturesrc", nullptr);
+        if (!d3d11screensrc) {
+            nhlog::ui()->error("Failed to create d3d11screencapturesrc");
+            gst_object_unref(pipe_);
+            pipe_ = nullptr;
+            return;
+        }
+        g_object_set(d3d11screensrc, "window-handle", windows_[index].second, nullptr);
+        g_object_set(d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
+
+        gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
+        screencastsrc = d3d11screensrc;
     } else {
         ScreenCastPortal &sc_portal            = ScreenCastPortal::instance();
         const ScreenCastPortal::Stream *stream = sc_portal.getStream();
@@ -1171,6 +1236,6 @@ getTurnURIs(const mtx::responses::TurnServer &turnServer)
     }
     return ret;
 }
-}
+} // namespace
 
 #include "moc_CallManager.cpp"
diff --git a/src/voip/CallManager.h b/src/voip/CallManager.h
index c0fd0831..2490fbf8 100644
--- a/src/voip/CallManager.h
+++ b/src/voip/CallManager.h
@@ -133,7 +133,11 @@ private:
     QTimer turnServerTimer_;
     QMediaPlayer player_;
     std::vector<webrtc::ScreenShareType> screenShareTypes_;
+#ifndef Q_OS_WINDOWS
     std::vector<std::pair<QString, uint32_t>> windows_;
+#else
+    std::vector<std::pair<QString, uint64_t>> windows_;
+#endif
     std::vector<std::string> rejectCallPartyIDs_;
 
     template<typename T>
diff --git a/src/voip/WebRTCSession.cpp b/src/voip/WebRTCSession.cpp
index a10e24ef..ba1d5424 100644
--- a/src/voip/WebRTCSession.cpp
+++ b/src/voip/WebRTCSession.cpp
@@ -23,6 +23,7 @@
 #include "voip/ScreenCastPortal.h"
 
 #ifdef GSTREAMER_AVAILABLE
+#include "MainWindow.h"
 extern "C"
 {
 #include "gst/gl/gstgldisplay.h"
@@ -330,33 +331,66 @@ GstElement *
 newVideoSinkChain(GstElement *pipe)
 {
     // use compositor for now; acceleration needs investigation
-    GstElement *queue      = gst_element_factory_make("queue", nullptr);
-    GstElement *compositor = gst_element_factory_make("compositor", "compositor");
-    GstElement *glupload   = gst_element_factory_make("glupload", nullptr);
-    GstElement *qmlglsink  = gst_element_factory_make("qml6glsink", nullptr);
-    GstElement *glsinkbin  = gst_element_factory_make("glsinkbin", nullptr);
+    GstElement *queue = gst_element_factory_make("queue", nullptr);
+
+    auto graphicsApi       = MainWindow::instance()->graphicsApi();
+    GstElement *compositor = gst_element_factory_make(
+      graphicsApi == QSGRendererInterface::OpenGL ? "compositor" : "d3d11compositor", "compositor");
     g_object_set(compositor, "background", 1, nullptr);
-    g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
-    g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
-    gst_bin_add_many(GST_BIN(pipe), queue, compositor, glupload, glsinkbin, nullptr);
-    gst_element_link_many(queue, compositor, glupload, glsinkbin, nullptr);
-    gst_element_sync_state_with_parent(queue);
-    gst_element_sync_state_with_parent(compositor);
-    gst_element_sync_state_with_parent(glupload);
-    gst_element_sync_state_with_parent(glsinkbin);
-
-    // to propagate context (hopefully)
-    gst_element_set_state(qmlglsink, GST_STATE_READY);
-
-    // Workaround: On wayland, when egl is used, gstreamer might terminate the display connection.
-    // Prevent that by "leaking" a reference to the display. See
-    // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743
-    if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
-        auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay");
-        if (context) {
-            GstGLDisplay *display;
-            gst_context_get_gl_display(context, &display);
+    switch (graphicsApi) {
+    case QSGRendererInterface::OpenGL: {
+        GstElement *glupload       = gst_element_factory_make("glupload", nullptr);
+        GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
+        GstElement *qmlglsink      = gst_element_factory_make("qml6glsink", nullptr);
+        GstElement *glsinkbin      = gst_element_factory_make("glsinkbin", nullptr);
+
+        g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
+        g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
+        gst_bin_add_many(
+          GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
+        gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
+
+        gst_element_sync_state_with_parent(queue);
+        gst_element_sync_state_with_parent(compositor);
+        gst_element_sync_state_with_parent(glupload);
+        gst_element_sync_state_with_parent(glcolorconvert);
+        gst_element_sync_state_with_parent(glsinkbin);
+
+        // to propagate context (hopefully)
+        gst_element_set_state(qmlglsink, GST_STATE_READY);
+
+        // Workaround: On wayland, when egl is used, gstreamer might terminate the display
+        // connection. Prevent that by "leaking" a reference to the display. See
+        // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743
+        if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
+            auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay");
+            if (context) {
+                GstGLDisplay *display;
+                gst_context_get_gl_display(context, &display);
+            }
         }
+    } break;
+    case QSGRendererInterface::Direct3D11: {
+        GstElement *d3d11upload       = gst_element_factory_make("d3d11upload", nullptr);
+        GstElement *d3d11colorconvert = gst_element_factory_make("d3d11colorconvert", nullptr);
+        GstElement *qmld3d11sink      = gst_element_factory_make("qml6d3d11sink", nullptr);
+
+        g_object_set(qmld3d11sink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
+        gst_bin_add_many(
+          GST_BIN(pipe), queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
+        gst_element_link_many(
+          queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
+
+        gst_element_sync_state_with_parent(queue);
+        gst_element_sync_state_with_parent(compositor);
+        gst_element_sync_state_with_parent(d3d11upload);
+        gst_element_sync_state_with_parent(d3d11colorconvert);
+
+        // to propagate context (hopefully)
+        gst_element_set_state(qmld3d11sink, GST_STATE_READY);
+    } break;
+    default:
+        break;
     }
 
     return queue;
@@ -606,20 +640,20 @@ WebRTCSession::havePlugins(bool isVideo,
     if (!initialised_ && !init(errorMessage))
         return false;
 
-    static constexpr std::initializer_list<const char *> audio_elements = {
-      "audioconvert",
-      "audioresample",
-      "autoaudiosink",
-      "capsfilter",
-      "decodebin",
-      "opusenc",
-      "queue",
-      "rtpopuspay",
-      "volume",
-      "webrtcbin",
-    };
-
-    static constexpr std::initializer_list<const char *> video_elements = {
+    static constexpr std::initializer_list<const char *> audio_elements = {"audioconvert",
+                                                                           "audioresample",
+                                                                           "autoaudiosink",
+                                                                           "capsfilter",
+                                                                           "decodebin",
+                                                                           "opusenc",
+                                                                           "queue",
+                                                                           "rtpopuspay",
+                                                                           "volume",
+                                                                           "webrtcbin",
+                                                                           "nicesrc",
+                                                                           "nicesink"};
+
+    static constexpr std::initializer_list<const char *> gl_video_elements = {
       "compositor",
       "glsinkbin",
       "glupload",
@@ -631,6 +665,19 @@ WebRTCSession::havePlugins(bool isVideo,
       "vp8enc",
     };
 
+    static constexpr std::initializer_list<const char *> d3d11_video_elements = {
+      "compositor",
+      "d3d11colorconvert",
+      "d3d11videosink",
+      "d3d11upload",
+      "qml6d3d11sink",
+      "rtpvp8pay",
+      "tee",
+      "videoconvert",
+      "videoscale",
+      "vp8enc",
+    };
+
     std::string strError("Missing GStreamer elements: ");
     GstRegistry *registry = gst_registry_get();
 
@@ -654,8 +701,19 @@ WebRTCSession::havePlugins(bool isVideo,
     haveVoicePlugins_ = check_plugins(audio_elements);
 
     // check both elements at once
-    if (isVideo)
-        haveVideoPlugins_ = check_plugins(video_elements);
+    if (isVideo) {
+        switch (MainWindow::instance()->graphicsApi()) {
+        case QSGRendererInterface::OpenGL:
+            haveVideoPlugins_ = check_plugins(gl_video_elements);
+            break;
+        case QSGRendererInterface::Direct3D11:
+            haveVideoPlugins_ = check_plugins(d3d11_video_elements);
+            break;
+        default:
+            haveVideoPlugins_ = false;
+            break;
+        }
+    }
 
     bool haveScreensharePlugins = false;
     if (isScreenshare) {
@@ -663,6 +721,8 @@ WebRTCSession::havePlugins(bool isVideo,
         if (haveScreensharePlugins) {
             if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
                 haveScreensharePlugins = check_plugins({"waylandsink"});
+            } else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
+                haveScreensharePlugins = check_plugins({"d3d11videosink"});
             } else {
                 haveScreensharePlugins = check_plugins({"ximagesink"});
             }
@@ -670,6 +730,9 @@ WebRTCSession::havePlugins(bool isVideo,
         if (haveScreensharePlugins) {
             if (screenShareType == ScreenShareType::X11) {
                 haveScreensharePlugins = check_plugins({"ximagesrc"});
+            } else if (screenShareType == ScreenShareType::D3D11) {
+                haveScreensharePlugins =
+                  check_plugins({"d3d11screencapturesrc", "d3d11download", "d3d11convert"});
             } else {
                 haveScreensharePlugins = check_plugins({"pipewiresrc"});
             }
@@ -685,9 +748,19 @@ WebRTCSession::havePlugins(bool isVideo,
     }
 
     if (isVideo || isScreenshare) {
-        // load qmlglsink to register GStreamer's GstGLVideoItem QML type
-        GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
-        gst_object_unref(qmlglsink);
+        switch (MainWindow::instance()->graphicsApi()) {
+        case QSGRendererInterface::OpenGL: {
+            // load qmlglsink to register GStreamer's GstGLVideoItem QML type
+            GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
+            gst_object_unref(qmlglsink);
+        } break;
+        case QSGRendererInterface::Direct3D11: {
+            GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr);
+            gst_object_unref(qmld3d11sink);
+        } break;
+        default:
+            break;
+        }
     }
     return true;
 }
@@ -1027,6 +1100,36 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
 
             gst_bin_add(GST_BIN(pipe_), ximagesrc);
             screencastsrc = ximagesrc;
+        } else if (screenShareType_ == ScreenShareType::D3D11) {
+            GstElement *d3d11screensrc =
+              gst_element_factory_make("d3d11screencapturesrc", "screenshare");
+            if (!d3d11screensrc) {
+                nhlog::ui()->error("WebRTC: failed to create d3d11screencapturesrc");
+                gst_object_unref(pipe_);
+                pipe_ = nullptr;
+                return false;
+            }
+            g_object_set(
+              d3d11screensrc, "window-handle", static_cast<guint64>(shareWindowId_), nullptr);
+            g_object_set(
+              d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
+            g_object_set(d3d11screensrc, "do-timestamp", (gboolean)1, nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
+
+            GstElement *d3d11convert = gst_element_factory_make("d3d11convert", nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11convert);
+            if (!gst_element_link(d3d11screensrc, d3d11convert)) {
+                nhlog::ui()->error("WebRTC: failed to link d3d11screencapturesrc -> d3d11convert");
+                return false;
+            }
+
+            GstElement *d3d11download = gst_element_factory_make("d3d11download", nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11download);
+            if (!gst_element_link(d3d11convert, d3d11download)) {
+                nhlog::ui()->error("WebRTC: failed to link d3d11convert -> d3d11download");
+                return false;
+            }
+            screencastsrc = d3d11download;
         } else {
             ScreenCastPortal &sc_portal = ScreenCastPortal::instance();
             GstElement *pipewiresrc     = gst_element_factory_make("pipewiresrc", "screenshare");
diff --git a/src/voip/WebRTCSession.h b/src/voip/WebRTCSession.h
index 3357bff7..20c32110 100644
--- a/src/voip/WebRTCSession.h
+++ b/src/voip/WebRTCSession.h
@@ -31,7 +31,8 @@ Q_ENUM_NS(CallType)
 enum class ScreenShareType
 {
     X11,
-    XDP
+    XDP,
+    D3D11
 };
 Q_ENUM_NS(ScreenShareType)