From 550c80525a1633edc983a7fe0d1dae11220cb35f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 14 Oct 2021 22:53:11 +0200 Subject: Move voip and encryption stuff into their own directories --- src/Cache.cpp | 2 +- src/CallDevices.cpp | 385 ------- src/CallDevices.h | 47 - src/CallManager.cpp | 680 ------------ src/CallManager.h | 117 --- src/ChatPage.cpp | 6 +- src/DeviceVerificationFlow.cpp | 849 --------------- src/DeviceVerificationFlow.h | 248 ----- src/MainWindow.cpp | 2 +- src/Olm.cpp | 1612 ----------------------------- src/Olm.h | 125 --- src/SelfVerificationStatus.cpp | 249 ----- src/SelfVerificationStatus.h | 43 - src/UserSettingsPage.cpp | 5 +- src/WebRTCSession.cpp | 1155 --------------------- src/WebRTCSession.h | 117 --- src/encryption/DeviceVerificationFlow.cpp | 849 +++++++++++++++ src/encryption/DeviceVerificationFlow.h | 248 +++++ src/encryption/Olm.cpp | 1612 +++++++++++++++++++++++++++++ src/encryption/Olm.h | 125 +++ src/encryption/SelfVerificationStatus.cpp | 249 +++++ src/encryption/SelfVerificationStatus.h | 43 + src/timeline/EventStore.cpp | 1 - src/timeline/EventStore.h | 2 +- src/timeline/InputBar.cpp | 1 - src/timeline/TimelineModel.cpp | 2 +- src/timeline/TimelineViewManager.cpp | 4 +- src/timeline/TimelineViewManager.h | 4 +- src/ui/NhekoGlobalObject.cpp | 2 +- src/ui/UserProfile.cpp | 2 +- src/voip/CallDevices.cpp | 385 +++++++ src/voip/CallDevices.h | 47 + src/voip/CallManager.cpp | 680 ++++++++++++ src/voip/CallManager.h | 117 +++ src/voip/WebRTCSession.cpp | 1155 +++++++++++++++++++++ src/voip/WebRTCSession.h | 117 +++ 36 files changed, 5642 insertions(+), 5645 deletions(-) delete mode 100644 src/CallDevices.cpp delete mode 100644 src/CallDevices.h delete mode 100644 src/CallManager.cpp delete mode 100644 src/CallManager.h delete mode 100644 src/DeviceVerificationFlow.cpp delete mode 100644 src/DeviceVerificationFlow.h delete mode 100644 src/Olm.cpp delete mode 100644 src/Olm.h delete mode 100644 src/SelfVerificationStatus.cpp delete mode 100644 src/SelfVerificationStatus.h delete mode 100644 src/WebRTCSession.cpp delete mode 100644 src/WebRTCSession.h create mode 100644 src/encryption/DeviceVerificationFlow.cpp create mode 100644 src/encryption/DeviceVerificationFlow.h create mode 100644 src/encryption/Olm.cpp create mode 100644 src/encryption/Olm.h create mode 100644 src/encryption/SelfVerificationStatus.cpp create mode 100644 src/encryption/SelfVerificationStatus.h create mode 100644 src/voip/CallDevices.cpp create mode 100644 src/voip/CallDevices.h create mode 100644 src/voip/CallManager.cpp create mode 100644 src/voip/CallManager.h create mode 100644 src/voip/WebRTCSession.cpp create mode 100644 src/voip/WebRTCSession.h (limited to 'src') diff --git a/src/Cache.cpp b/src/Cache.cpp index ea3dd525..ecfbe6c9 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -30,9 +30,9 @@ #include "EventAccessors.h" #include "Logging.h" #include "MatrixClient.h" -#include "Olm.h" #include "UserSettingsPage.h" #include "Utils.h" +#include "encryption/Olm.h" //! Should be changed when a breaking change occurs in the cache format. //! This will reset client's data. diff --git a/src/CallDevices.cpp b/src/CallDevices.cpp deleted file mode 100644 index be185470..00000000 --- a/src/CallDevices.cpp +++ /dev/null @@ -1,385 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include -#include -#include - -#include "CallDevices.h" -#include "ChatPage.h" -#include "Logging.h" -#include "UserSettingsPage.h" - -#ifdef GSTREAMER_AVAILABLE -extern "C" -{ -#include "gst/gst.h" -} -#endif - -CallDevices::CallDevices() - : QObject() -{} - -#ifdef GSTREAMER_AVAILABLE -namespace { - -struct AudioSource -{ - std::string name; - GstDevice *device; -}; - -struct VideoSource -{ - struct Caps - { - std::string resolution; - std::vector frameRates; - }; - std::string name; - GstDevice *device; - std::vector caps; -}; - -std::vector audioSources_; -std::vector videoSources_; - -using FrameRate = std::pair; -std::optional -getFrameRate(const GValue *value) -{ - if (GST_VALUE_HOLDS_FRACTION(value)) { - gint num = gst_value_get_fraction_numerator(value); - gint den = gst_value_get_fraction_denominator(value); - return FrameRate{num, den}; - } - return std::nullopt; -} - -void -addFrameRate(std::vector &rates, const FrameRate &rate) -{ - constexpr double minimumFrameRate = 15.0; - if (static_cast(rate.first) / rate.second >= minimumFrameRate) - rates.push_back(std::to_string(rate.first) + "/" + std::to_string(rate.second)); -} - -void -setDefaultDevice(bool isVideo) -{ - auto settings = ChatPage::instance()->userSettings(); - if (isVideo && settings->camera().isEmpty()) { - const VideoSource &camera = videoSources_.front(); - settings->setCamera(QString::fromStdString(camera.name)); - settings->setCameraResolution(QString::fromStdString(camera.caps.front().resolution)); - settings->setCameraFrameRate( - QString::fromStdString(camera.caps.front().frameRates.front())); - } else if (!isVideo && settings->microphone().isEmpty()) { - settings->setMicrophone(QString::fromStdString(audioSources_.front().name)); - } -} - -void -addDevice(GstDevice *device) -{ - if (!device) - return; - - gchar *name = gst_device_get_display_name(device); - gchar *type = gst_device_get_device_class(device); - bool isVideo = !std::strncmp(type, "Video", 5); - g_free(type); - nhlog::ui()->debug("WebRTC: {} device added: {}", isVideo ? "video" : "audio", name); - if (!isVideo) { - audioSources_.push_back({name, device}); - g_free(name); - setDefaultDevice(false); - return; - } - - GstCaps *gstcaps = gst_device_get_caps(device); - if (!gstcaps) { - nhlog::ui()->debug("WebRTC: unable to get caps for {}", name); - g_free(name); - return; - } - - VideoSource source{name, device, {}}; - g_free(name); - guint nCaps = gst_caps_get_size(gstcaps); - for (guint i = 0; i < nCaps; ++i) { - GstStructure *structure = gst_caps_get_structure(gstcaps, i); - const gchar *struct_name = gst_structure_get_name(structure); - if (!std::strcmp(struct_name, "video/x-raw")) { - gint widthpx, heightpx; - if (gst_structure_get(structure, - "width", - G_TYPE_INT, - &widthpx, - "height", - G_TYPE_INT, - &heightpx, - nullptr)) { - VideoSource::Caps caps; - caps.resolution = std::to_string(widthpx) + "x" + std::to_string(heightpx); - const GValue *value = gst_structure_get_value(structure, "framerate"); - if (auto fr = getFrameRate(value); fr) - addFrameRate(caps.frameRates, *fr); - else if (GST_VALUE_HOLDS_FRACTION_RANGE(value)) { - addFrameRate(caps.frameRates, - *getFrameRate(gst_value_get_fraction_range_min(value))); - addFrameRate(caps.frameRates, - *getFrameRate(gst_value_get_fraction_range_max(value))); - } else if (GST_VALUE_HOLDS_LIST(value)) { - guint nRates = gst_value_list_get_size(value); - for (guint j = 0; j < nRates; ++j) { - const GValue *rate = gst_value_list_get_value(value, j); - if (auto frate = getFrameRate(rate); frate) - addFrameRate(caps.frameRates, *frate); - } - } - if (!caps.frameRates.empty()) - source.caps.push_back(std::move(caps)); - } - } - } - gst_caps_unref(gstcaps); - videoSources_.push_back(std::move(source)); - setDefaultDevice(true); -} - -template -bool -removeDevice(T &sources, GstDevice *device, bool changed) -{ - if (auto it = std::find_if( - sources.begin(), sources.end(), [device](const auto &s) { return s.device == device; }); - it != sources.end()) { - nhlog::ui()->debug( - std::string("WebRTC: device ") + (changed ? "changed: " : "removed: ") + "{}", it->name); - gst_object_unref(device); - sources.erase(it); - return true; - } - return false; -} - -void -removeDevice(GstDevice *device, bool changed) -{ - if (device) { - if (removeDevice(audioSources_, device, changed) || - removeDevice(videoSources_, device, changed)) - return; - } -} - -gboolean -newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data G_GNUC_UNUSED) -{ - switch (GST_MESSAGE_TYPE(msg)) { - case GST_MESSAGE_DEVICE_ADDED: { - GstDevice *device; - gst_message_parse_device_added(msg, &device); - addDevice(device); - emit CallDevices::instance().devicesChanged(); - break; - } - case GST_MESSAGE_DEVICE_REMOVED: { - GstDevice *device; - gst_message_parse_device_removed(msg, &device); - removeDevice(device, false); - emit CallDevices::instance().devicesChanged(); - break; - } - case GST_MESSAGE_DEVICE_CHANGED: { - GstDevice *device; - GstDevice *oldDevice; - gst_message_parse_device_changed(msg, &device, &oldDevice); - removeDevice(oldDevice, true); - addDevice(device); - break; - } - default: - break; - } - return TRUE; -} - -template -std::vector -deviceNames(T &sources, const std::string &defaultDevice) -{ - std::vector ret; - ret.reserve(sources.size()); - for (const auto &s : sources) - ret.push_back(s.name); - - // move default device to top of the list - if (auto it = std::find(ret.begin(), ret.end(), defaultDevice); it != ret.end()) - std::swap(ret.front(), *it); - - return ret; -} - -std::optional -getVideoSource(const std::string &cameraName) -{ - if (auto it = std::find_if(videoSources_.cbegin(), - videoSources_.cend(), - [&cameraName](const auto &s) { return s.name == cameraName; }); - it != videoSources_.cend()) { - return *it; - } - return std::nullopt; -} - -std::pair -tokenise(std::string_view str, char delim) -{ - std::pair ret; - ret.first = std::atoi(str.data()); - auto pos = str.find_first_of(delim); - ret.second = std::atoi(str.data() + pos + 1); - return ret; -} -} - -void -CallDevices::init() -{ - static GstDeviceMonitor *monitor = nullptr; - if (!monitor) { - monitor = gst_device_monitor_new(); - GstCaps *caps = gst_caps_new_empty_simple("audio/x-raw"); - gst_device_monitor_add_filter(monitor, "Audio/Source", caps); - gst_device_monitor_add_filter(monitor, "Audio/Duplex", caps); - gst_caps_unref(caps); - caps = gst_caps_new_empty_simple("video/x-raw"); - gst_device_monitor_add_filter(monitor, "Video/Source", caps); - gst_device_monitor_add_filter(monitor, "Video/Duplex", caps); - gst_caps_unref(caps); - - GstBus *bus = gst_device_monitor_get_bus(monitor); - gst_bus_add_watch(bus, newBusMessage, nullptr); - gst_object_unref(bus); - if (!gst_device_monitor_start(monitor)) { - nhlog::ui()->error("WebRTC: failed to start device monitor"); - return; - } - } -} - -bool -CallDevices::haveMic() const -{ - return !audioSources_.empty(); -} - -bool -CallDevices::haveCamera() const -{ - return !videoSources_.empty(); -} - -std::vector -CallDevices::names(bool isVideo, const std::string &defaultDevice) const -{ - return isVideo ? deviceNames(videoSources_, defaultDevice) - : deviceNames(audioSources_, defaultDevice); -} - -std::vector -CallDevices::resolutions(const std::string &cameraName) const -{ - std::vector ret; - if (auto s = getVideoSource(cameraName); s) { - ret.reserve(s->caps.size()); - for (const auto &c : s->caps) - ret.push_back(c.resolution); - } - return ret; -} - -std::vector -CallDevices::frameRates(const std::string &cameraName, const std::string &resolution) const -{ - if (auto s = getVideoSource(cameraName); s) { - if (auto it = std::find_if(s->caps.cbegin(), - s->caps.cend(), - [&](const auto &c) { return c.resolution == resolution; }); - it != s->caps.cend()) - return it->frameRates; - } - return {}; -} - -GstDevice * -CallDevices::audioDevice() const -{ - std::string name = ChatPage::instance()->userSettings()->microphone().toStdString(); - if (auto it = std::find_if(audioSources_.cbegin(), - audioSources_.cend(), - [&name](const auto &s) { return s.name == name; }); - it != audioSources_.cend()) { - nhlog::ui()->debug("WebRTC: microphone: {}", name); - return it->device; - } else { - nhlog::ui()->error("WebRTC: unknown microphone: {}", name); - return nullptr; - } -} - -GstDevice * -CallDevices::videoDevice(std::pair &resolution, std::pair &frameRate) const -{ - auto settings = ChatPage::instance()->userSettings(); - std::string name = settings->camera().toStdString(); - if (auto s = getVideoSource(name); s) { - nhlog::ui()->debug("WebRTC: camera: {}", name); - resolution = tokenise(settings->cameraResolution().toStdString(), 'x'); - frameRate = tokenise(settings->cameraFrameRate().toStdString(), '/'); - nhlog::ui()->debug("WebRTC: camera resolution: {}x{}", resolution.first, resolution.second); - nhlog::ui()->debug("WebRTC: camera frame rate: {}/{}", frameRate.first, frameRate.second); - return s->device; - } else { - nhlog::ui()->error("WebRTC: unknown camera: {}", name); - return nullptr; - } -} - -#else - -bool -CallDevices::haveMic() const -{ - return false; -} - -bool -CallDevices::haveCamera() const -{ - return false; -} - -std::vector -CallDevices::names(bool, const std::string &) const -{ - return {}; -} - -std::vector -CallDevices::resolutions(const std::string &) const -{ - return {}; -} - -std::vector -CallDevices::frameRates(const std::string &, const std::string &) const -{ - return {}; -} - -#endif diff --git a/src/CallDevices.h b/src/CallDevices.h deleted file mode 100644 index d30ce644..00000000 --- a/src/CallDevices.h +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include -#include -#include - -#include - -typedef struct _GstDevice GstDevice; - -class CallDevices : public QObject -{ - Q_OBJECT - -public: - static CallDevices &instance() - { - static CallDevices instance; - return instance; - } - - bool haveMic() const; - bool haveCamera() const; - std::vector names(bool isVideo, const std::string &defaultDevice) const; - std::vector resolutions(const std::string &cameraName) const; - std::vector frameRates(const std::string &cameraName, - const std::string &resolution) const; - -signals: - void devicesChanged(); - -private: - CallDevices(); - - friend class WebRTCSession; - void init(); - GstDevice *audioDevice() const; - GstDevice *videoDevice(std::pair &resolution, std::pair &frameRate) const; - -public: - CallDevices(CallDevices const &) = delete; - void operator=(CallDevices const &) = delete; -}; diff --git a/src/CallManager.cpp b/src/CallManager.cpp deleted file mode 100644 index 0f701b0d..00000000 --- a/src/CallManager.cpp +++ /dev/null @@ -1,680 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "Cache.h" -#include "CallDevices.h" -#include "CallManager.h" -#include "ChatPage.h" -#include "Logging.h" -#include "MatrixClient.h" -#include "UserSettingsPage.h" -#include "Utils.h" - -#include "mtx/responses/turn_server.hpp" - -#ifdef XCB_AVAILABLE -#include -#include -#endif - -#ifdef GSTREAMER_AVAILABLE -extern "C" -{ -#include "gst/gst.h" -} -#endif - -Q_DECLARE_METATYPE(std::vector) -Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) -Q_DECLARE_METATYPE(mtx::responses::TurnServer) - -using namespace mtx::events; -using namespace mtx::events::msg; - -using webrtc::CallType; - -namespace { -std::vector -getTurnURIs(const mtx::responses::TurnServer &turnServer); -} - -CallManager::CallManager(QObject *parent) - : QObject(parent) - , session_(WebRTCSession::instance()) - , turnServerTimer_(this) -{ - qRegisterMetaType>(); - qRegisterMetaType(); - qRegisterMetaType(); - - connect( - &session_, - &WebRTCSession::offerCreated, - this, - [this](const std::string &sdp, const std::vector &candidates) { - nhlog::ui()->debug("WebRTC: call id: {} - sending offer", callid_); - emit newMessage(roomid_, CallInvite{callid_, sdp, "0", timeoutms_}); - emit newMessage(roomid_, CallCandidates{callid_, candidates, "0"}); - std::string callid(callid_); - QTimer::singleShot(timeoutms_, this, [this, callid]() { - if (session_.state() == webrtc::State::OFFERSENT && callid == callid_) { - hangUp(CallHangUp::Reason::InviteTimeOut); - emit ChatPage::instance()->showNotification("The remote side failed to pick up."); - } - }); - }); - - connect( - &session_, - &WebRTCSession::answerCreated, - this, - [this](const std::string &sdp, const std::vector &candidates) { - nhlog::ui()->debug("WebRTC: call id: {} - sending answer", callid_); - emit newMessage(roomid_, CallAnswer{callid_, sdp, "0"}); - emit newMessage(roomid_, CallCandidates{callid_, candidates, "0"}); - }); - - connect(&session_, - &WebRTCSession::newICECandidate, - this, - [this](const CallCandidates::Candidate &candidate) { - nhlog::ui()->debug("WebRTC: call id: {} - sending ice candidate", callid_); - emit newMessage(roomid_, CallCandidates{callid_, {candidate}, "0"}); - }); - - connect(&turnServerTimer_, &QTimer::timeout, this, &CallManager::retrieveTurnServer); - - connect( - this, &CallManager::turnServerRetrieved, this, [this](const mtx::responses::TurnServer &res) { - nhlog::net()->info("TURN server(s) retrieved from homeserver:"); - nhlog::net()->info("username: {}", res.username); - nhlog::net()->info("ttl: {} seconds", res.ttl); - for (const auto &u : res.uris) - nhlog::net()->info("uri: {}", u); - - // Request new credentials close to expiry - // See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00 - turnURIs_ = getTurnURIs(res); - uint32_t ttl = std::max(res.ttl, UINT32_C(3600)); - if (res.ttl < 3600) - nhlog::net()->warn("Setting ttl to 1 hour"); - turnServerTimer_.setInterval(ttl * 1000 * 0.9); - }); - - connect(&session_, &WebRTCSession::stateChanged, this, [this](webrtc::State state) { - switch (state) { - case webrtc::State::DISCONNECTED: - playRingtone(QUrl("qrc:/media/media/callend.ogg"), false); - clear(); - break; - case webrtc::State::ICEFAILED: { - QString error("Call connection failed."); - if (turnURIs_.empty()) - error += " Your homeserver has no configured TURN server."; - emit ChatPage::instance()->showNotification(error); - hangUp(CallHangUp::Reason::ICEFailed); - break; - } - default: - break; - } - emit newCallState(); - }); - - connect( - &CallDevices::instance(), &CallDevices::devicesChanged, this, &CallManager::devicesChanged); - - connect( - &player_, &QMediaPlayer::mediaStatusChanged, this, [this](QMediaPlayer::MediaStatus status) { - if (status == QMediaPlayer::LoadedMedia) - player_.play(); - }); - - connect(&player_, - QOverload::of(&QMediaPlayer::error), - [this](QMediaPlayer::Error error) { - stopRingtone(); - switch (error) { - case QMediaPlayer::FormatError: - case QMediaPlayer::ResourceError: - nhlog::ui()->error("WebRTC: valid ringtone file not found"); - break; - case QMediaPlayer::AccessDeniedError: - nhlog::ui()->error("WebRTC: access to ringtone file denied"); - break; - default: - nhlog::ui()->error("WebRTC: unable to play ringtone"); - break; - } - }); -} - -void -CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int windowIndex) -{ - if (isOnCall()) - return; - if (callType == CallType::SCREEN) { - if (!screenShareSupported()) - return; - if (windows_.empty() || windowIndex >= windows_.size()) { - nhlog::ui()->error("WebRTC: window index out of range"); - return; - } - } - - auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); - if (roomInfo.member_count != 2) { - emit ChatPage::instance()->showNotification("Calls are limited to 1:1 rooms."); - return; - } - - std::string errorMessage; - if (!session_.havePlugins(false, &errorMessage) || - ((callType == CallType::VIDEO || callType == CallType::SCREEN) && - !session_.havePlugins(true, &errorMessage))) { - emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); - return; - } - - callType_ = callType; - roomid_ = roomid; - session_.setTurnServers(turnURIs_); - generateCallID(); - std::string strCallType = - callType_ == CallType::VOICE ? "voice" : (callType_ == CallType::VIDEO ? "video" : "screen"); - nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType); - std::vector members(cache::getMembers(roomid.toStdString())); - const RoomMember &callee = - members.front().user_id == utils::localUser() ? members.back() : members.front(); - callParty_ = callee.user_id; - callPartyDisplayName_ = callee.display_name.isEmpty() ? callee.user_id : callee.display_name; - callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); - emit newInviteState(); - playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); - if (!session_.createOffer(callType, - callType == CallType::SCREEN ? windows_[windowIndex].second : 0)) { - emit ChatPage::instance()->showNotification("Problem setting up call."); - endCall(); - } -} - -namespace { -std::string -callHangUpReasonString(CallHangUp::Reason reason) -{ - switch (reason) { - case CallHangUp::Reason::ICEFailed: - return "ICE failed"; - case CallHangUp::Reason::InviteTimeOut: - return "Invite time out"; - default: - return "User"; - } -} -} - -void -CallManager::hangUp(CallHangUp::Reason reason) -{ - if (!callid_.empty()) { - nhlog::ui()->debug( - "WebRTC: call id: {} - hanging up ({})", callid_, callHangUpReasonString(reason)); - emit newMessage(roomid_, CallHangUp{callid_, "0", reason}); - endCall(); - } -} - -void -CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) -{ -#ifdef GSTREAMER_AVAILABLE - if (handleEvent(event) || handleEvent(event) || - handleEvent(event) || handleEvent(event)) - return; -#else - (void)event; -#endif -} - -template -bool -CallManager::handleEvent(const mtx::events::collections::TimelineEvents &event) -{ - if (std::holds_alternative>(event)) { - handleEvent(std::get>(event)); - return true; - } - return false; -} - -void -CallManager::handleEvent(const RoomEvent &callInviteEvent) -{ - const char video[] = "m=video"; - const std::string &sdp = callInviteEvent.content.sdp; - bool isVideo = std::search(sdp.cbegin(), - sdp.cend(), - std::cbegin(video), - std::cend(video) - 1, - [](unsigned char c1, unsigned char c2) { - return std::tolower(c1) == std::tolower(c2); - }) != sdp.cend(); - - nhlog::ui()->debug("WebRTC: call id: {} - incoming {} CallInvite from {}", - callInviteEvent.content.call_id, - (isVideo ? "video" : "voice"), - callInviteEvent.sender); - - if (callInviteEvent.content.call_id.empty()) - return; - - auto roomInfo = cache::singleRoomInfo(callInviteEvent.room_id); - if (isOnCall() || roomInfo.member_count != 2) { - emit newMessage( - QString::fromStdString(callInviteEvent.room_id), - CallHangUp{callInviteEvent.content.call_id, "0", CallHangUp::Reason::InviteTimeOut}); - return; - } - - const QString &ringtone = ChatPage::instance()->userSettings()->ringtone(); - if (ringtone != "Mute") - playRingtone(ringtone == "Default" ? QUrl("qrc:/media/media/ring.ogg") - : QUrl::fromLocalFile(ringtone), - true); - roomid_ = QString::fromStdString(callInviteEvent.room_id); - callid_ = callInviteEvent.content.call_id; - remoteICECandidates_.clear(); - - std::vector members(cache::getMembers(callInviteEvent.room_id)); - const RoomMember &caller = - members.front().user_id == utils::localUser() ? members.back() : members.front(); - callParty_ = caller.user_id; - callPartyDisplayName_ = caller.display_name.isEmpty() ? caller.user_id : caller.display_name; - callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); - - haveCallInvite_ = true; - callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; - inviteSDP_ = callInviteEvent.content.sdp; - emit newInviteState(); -} - -void -CallManager::acceptInvite() -{ - if (!haveCallInvite_) - return; - - stopRingtone(); - std::string errorMessage; - if (!session_.havePlugins(false, &errorMessage) || - (callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) { - emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); - hangUp(); - return; - } - - session_.setTurnServers(turnURIs_); - if (!session_.acceptOffer(inviteSDP_)) { - emit ChatPage::instance()->showNotification("Problem setting up call."); - hangUp(); - return; - } - session_.acceptICECandidates(remoteICECandidates_); - remoteICECandidates_.clear(); - haveCallInvite_ = false; - emit newInviteState(); -} - -void -CallManager::handleEvent(const RoomEvent &callCandidatesEvent) -{ - if (callCandidatesEvent.sender == utils::localUser().toStdString()) - return; - - nhlog::ui()->debug("WebRTC: call id: {} - incoming CallCandidates from {}", - callCandidatesEvent.content.call_id, - callCandidatesEvent.sender); - - if (callid_ == callCandidatesEvent.content.call_id) { - if (isOnCall()) - session_.acceptICECandidates(callCandidatesEvent.content.candidates); - else { - // CallInvite has been received and we're awaiting localUser to accept or - // reject the call - for (const auto &c : callCandidatesEvent.content.candidates) - remoteICECandidates_.push_back(c); - } - } -} - -void -CallManager::handleEvent(const RoomEvent &callAnswerEvent) -{ - nhlog::ui()->debug("WebRTC: call id: {} - incoming CallAnswer from {}", - callAnswerEvent.content.call_id, - callAnswerEvent.sender); - - if (callAnswerEvent.sender == utils::localUser().toStdString() && - callid_ == callAnswerEvent.content.call_id) { - if (!isOnCall()) { - emit ChatPage::instance()->showNotification("Call answered on another device."); - stopRingtone(); - haveCallInvite_ = false; - emit newInviteState(); - } - return; - } - - if (isOnCall() && callid_ == callAnswerEvent.content.call_id) { - stopRingtone(); - if (!session_.acceptAnswer(callAnswerEvent.content.sdp)) { - emit ChatPage::instance()->showNotification("Problem setting up call."); - hangUp(); - } - } -} - -void -CallManager::handleEvent(const RoomEvent &callHangUpEvent) -{ - nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}", - callHangUpEvent.content.call_id, - callHangUpReasonString(callHangUpEvent.content.reason), - callHangUpEvent.sender); - - if (callid_ == callHangUpEvent.content.call_id) - endCall(); -} - -void -CallManager::toggleMicMute() -{ - session_.toggleMicMute(); - emit micMuteChanged(); -} - -bool -CallManager::callsSupported() -{ -#ifdef GSTREAMER_AVAILABLE - return true; -#else - return false; -#endif -} - -bool -CallManager::screenShareSupported() -{ - return std::getenv("DISPLAY") && !std::getenv("WAYLAND_DISPLAY"); -} - -QStringList -CallManager::devices(bool isVideo) const -{ - QStringList ret; - const QString &defaultDevice = isVideo ? ChatPage::instance()->userSettings()->camera() - : ChatPage::instance()->userSettings()->microphone(); - std::vector devices = - CallDevices::instance().names(isVideo, defaultDevice.toStdString()); - ret.reserve(devices.size()); - std::transform(devices.cbegin(), devices.cend(), std::back_inserter(ret), [](const auto &d) { - return QString::fromStdString(d); - }); - - return ret; -} - -void -CallManager::generateCallID() -{ - using namespace std::chrono; - uint64_t ms = duration_cast(system_clock::now().time_since_epoch()).count(); - callid_ = "c" + std::to_string(ms); -} - -void -CallManager::clear() -{ - roomid_.clear(); - callParty_.clear(); - callPartyDisplayName_.clear(); - callPartyAvatarUrl_.clear(); - callid_.clear(); - callType_ = CallType::VOICE; - haveCallInvite_ = false; - emit newInviteState(); - inviteSDP_.clear(); - remoteICECandidates_.clear(); -} - -void -CallManager::endCall() -{ - stopRingtone(); - session_.end(); - clear(); -} - -void -CallManager::refreshTurnServer() -{ - turnURIs_.clear(); - turnServerTimer_.start(2000); -} - -void -CallManager::retrieveTurnServer() -{ - http::client()->get_turn_server( - [this](const mtx::responses::TurnServer &res, mtx::http::RequestErr err) { - if (err) { - turnServerTimer_.setInterval(5000); - return; - } - emit turnServerRetrieved(res); - }); -} - -void -CallManager::playRingtone(const QUrl &ringtone, bool repeat) -{ - static QMediaPlaylist playlist; - playlist.clear(); - playlist.setPlaybackMode(repeat ? QMediaPlaylist::CurrentItemInLoop - : QMediaPlaylist::CurrentItemOnce); - playlist.addMedia(ringtone); - player_.setVolume(100); - player_.setPlaylist(&playlist); -} - -void -CallManager::stopRingtone() -{ - player_.setPlaylist(nullptr); -} - -QStringList -CallManager::windowList() -{ - windows_.clear(); - windows_.push_back({tr("Entire screen"), 0}); - -#ifdef XCB_AVAILABLE - std::unique_ptr> connection( - xcb_connect(nullptr, nullptr), [](xcb_connection_t *c) { xcb_disconnect(c); }); - if (xcb_connection_has_error(connection.get())) { - nhlog::ui()->error("Failed to connect to X server"); - return {}; - } - - xcb_ewmh_connection_t ewmh; - if (!xcb_ewmh_init_atoms_replies( - &ewmh, xcb_ewmh_init_atoms(connection.get(), &ewmh), nullptr)) { - nhlog::ui()->error("Failed to connect to EWMH server"); - return {}; - } - std::unique_ptr> - ewmhconnection(&ewmh, [](xcb_ewmh_connection_t *c) { xcb_ewmh_connection_wipe(c); }); - - for (int i = 0; i < ewmh.nb_screens; i++) { - xcb_ewmh_get_windows_reply_t clients; - if (!xcb_ewmh_get_client_list_reply( - &ewmh, xcb_ewmh_get_client_list(&ewmh, i), &clients, nullptr)) { - nhlog::ui()->error("Failed to request window list"); - return {}; - } - - for (uint32_t w = 0; w < clients.windows_len; w++) { - xcb_window_t window = clients.windows[w]; - - std::string name; - xcb_ewmh_get_utf8_strings_reply_t data; - auto getName = [](xcb_ewmh_get_utf8_strings_reply_t *r) { - std::string name(r->strings, r->strings_len); - xcb_ewmh_get_utf8_strings_reply_wipe(r); - return name; - }; - - xcb_get_property_cookie_t cookie = xcb_ewmh_get_wm_name(&ewmh, window); - if (xcb_ewmh_get_wm_name_reply(&ewmh, cookie, &data, nullptr)) - name = getName(&data); - - cookie = xcb_ewmh_get_wm_visible_name(&ewmh, window); - if (xcb_ewmh_get_wm_visible_name_reply(&ewmh, cookie, &data, nullptr)) - name = getName(&data); - - windows_.push_back({QString::fromStdString(name), window}); - } - xcb_ewmh_get_windows_reply_wipe(&clients); - } -#endif - QStringList ret; - ret.reserve(windows_.size()); - for (const auto &w : windows_) - ret.append(w.first); - - return ret; -} - -#ifdef GSTREAMER_AVAILABLE -namespace { - -GstElement *pipe_ = nullptr; -unsigned int busWatchId_ = 0; - -gboolean -newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer G_GNUC_UNUSED) -{ - switch (GST_MESSAGE_TYPE(msg)) { - case GST_MESSAGE_EOS: - if (pipe_) { - gst_element_set_state(GST_ELEMENT(pipe_), GST_STATE_NULL); - gst_object_unref(pipe_); - pipe_ = nullptr; - } - if (busWatchId_) { - g_source_remove(busWatchId_); - busWatchId_ = 0; - } - break; - default: - break; - } - return TRUE; -} -} -#endif - -void -CallManager::previewWindow(unsigned int index) const -{ -#ifdef GSTREAMER_AVAILABLE - if (windows_.empty() || index >= windows_.size() || !gst_is_initialized()) - return; - - GstElement *ximagesrc = gst_element_factory_make("ximagesrc", nullptr); - if (!ximagesrc) { - nhlog::ui()->error("Failed to create ximagesrc"); - return; - } - GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); - GstElement *videoscale = gst_element_factory_make("videoscale", nullptr); - GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); - GstElement *ximagesink = gst_element_factory_make("ximagesink", nullptr); - - g_object_set(ximagesrc, "use-damage", FALSE, nullptr); - g_object_set(ximagesrc, "show-pointer", FALSE, nullptr); - g_object_set(ximagesrc, "xid", windows_[index].second, nullptr); - - GstCaps *caps = gst_caps_new_simple( - "video/x-raw", "width", G_TYPE_INT, 480, "height", G_TYPE_INT, 360, nullptr); - g_object_set(capsfilter, "caps", caps, nullptr); - gst_caps_unref(caps); - - pipe_ = gst_pipeline_new(nullptr); - gst_bin_add_many( - GST_BIN(pipe_), ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr); - if (!gst_element_link_many( - ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr)) { - nhlog::ui()->error("Failed to link preview window elements"); - gst_object_unref(pipe_); - pipe_ = nullptr; - return; - } - if (gst_element_set_state(pipe_, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { - nhlog::ui()->error("Unable to start preview pipeline"); - gst_object_unref(pipe_); - pipe_ = nullptr; - return; - } - - GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_)); - busWatchId_ = gst_bus_add_watch(bus, newBusMessage, nullptr); - gst_object_unref(bus); -#else - (void)index; -#endif -} - -namespace { -std::vector -getTurnURIs(const mtx::responses::TurnServer &turnServer) -{ - // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp) - // where username and password are percent-encoded - std::vector ret; - for (const auto &uri : turnServer.uris) { - if (auto c = uri.find(':'); c == std::string::npos) { - nhlog::ui()->error("Invalid TURN server uri: {}", uri); - continue; - } else { - std::string scheme = std::string(uri, 0, c); - if (scheme != "turn" && scheme != "turns") { - nhlog::ui()->error("Invalid TURN server uri: {}", uri); - continue; - } - - QString encodedUri = - QString::fromStdString(scheme) + "://" + - QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" + - QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" + - QString::fromStdString(std::string(uri, ++c)); - ret.push_back(encodedUri.toStdString()); - } - } - return ret; -} -} diff --git a/src/CallManager.h b/src/CallManager.h deleted file mode 100644 index 22f31814..00000000 --- a/src/CallManager.h +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include -#include - -#include -#include -#include -#include - -#include "CallDevices.h" -#include "WebRTCSession.h" -#include "mtx/events/collections.hpp" -#include "mtx/events/voip.hpp" - -namespace mtx::responses { -struct TurnServer; -} - -class QStringList; -class QUrl; - -class CallManager : public QObject -{ - Q_OBJECT - Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState) - Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState) - Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState) - Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState) - Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) - Q_PROPERTY(QString callPartyDisplayName READ callPartyDisplayName NOTIFY newInviteState) - Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) - Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) - Q_PROPERTY(bool haveLocalPiP READ haveLocalPiP NOTIFY newCallState) - Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) - Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) - Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) - Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT) - -public: - CallManager(QObject *); - - bool haveCallInvite() const { return haveCallInvite_; } - bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; } - webrtc::CallType callType() const { return callType_; } - webrtc::State callState() const { return session_.state(); } - QString callParty() const { return callParty_; } - QString callPartyDisplayName() const { return callPartyDisplayName_; } - QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } - bool isMicMuted() const { return session_.isMicMuted(); } - bool haveLocalPiP() const { return session_.haveLocalPiP(); } - QStringList mics() const { return devices(false); } - QStringList cameras() const { return devices(true); } - void refreshTurnServer(); - - static bool callsSupported(); - static bool screenShareSupported(); - -public slots: - void sendInvite(const QString &roomid, webrtc::CallType, unsigned int windowIndex = 0); - void syncEvent(const mtx::events::collections::TimelineEvents &event); - void toggleMicMute(); - void toggleLocalPiP() { session_.toggleLocalPiP(); } - void acceptInvite(); - void hangUp(mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); - QStringList windowList(); - void previewWindow(unsigned int windowIndex) const; - -signals: - void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &); - void newMessage(const QString &roomid, const mtx::events::msg::CallCandidates &); - void newMessage(const QString &roomid, const mtx::events::msg::CallAnswer &); - void newMessage(const QString &roomid, const mtx::events::msg::CallHangUp &); - void newInviteState(); - void newCallState(); - void micMuteChanged(); - void devicesChanged(); - void turnServerRetrieved(const mtx::responses::TurnServer &); - -private slots: - void retrieveTurnServer(); - -private: - WebRTCSession &session_; - QString roomid_; - QString callParty_; - QString callPartyDisplayName_; - QString callPartyAvatarUrl_; - std::string callid_; - const uint32_t timeoutms_ = 120000; - webrtc::CallType callType_ = webrtc::CallType::VOICE; - bool haveCallInvite_ = false; - std::string inviteSDP_; - std::vector remoteICECandidates_; - std::vector turnURIs_; - QTimer turnServerTimer_; - QMediaPlayer player_; - std::vector> windows_; - - template - bool handleEvent(const mtx::events::collections::TimelineEvents &event); - void handleEvent(const mtx::events::RoomEvent &); - void handleEvent(const mtx::events::RoomEvent &); - void handleEvent(const mtx::events::RoomEvent &); - void handleEvent(const mtx::events::RoomEvent &); - void answerInvite(const mtx::events::msg::CallInvite &, bool isVideo); - void generateCallID(); - QStringList devices(bool isVideo) const; - void clear(); - void endCall(); - void playRingtone(const QUrl &ringtone, bool repeat); - void stopRingtone(); -}; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 673f39ee..9239e342 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -12,19 +12,19 @@ #include "AvatarProvider.h" #include "Cache.h" #include "Cache_p.h" -#include "CallManager.h" #include "ChatPage.h" -#include "DeviceVerificationFlow.h" #include "EventAccessors.h" #include "Logging.h" #include "MainWindow.h" #include "MatrixClient.h" -#include "Olm.h" #include "UserSettingsPage.h" #include "Utils.h" +#include "encryption/DeviceVerificationFlow.h" +#include "encryption/Olm.h" #include "ui/OverlayModal.h" #include "ui/Theme.h" #include "ui/UserProfile.h" +#include "voip/CallManager.h" #include "notifications/Manager.h" diff --git a/src/DeviceVerificationFlow.cpp b/src/DeviceVerificationFlow.cpp deleted file mode 100644 index 2481d4f9..00000000 --- a/src/DeviceVerificationFlow.cpp +++ /dev/null @@ -1,849 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "DeviceVerificationFlow.h" - -#include "Cache.h" -#include "Cache_p.h" -#include "ChatPage.h" -#include "Logging.h" -#include "Utils.h" -#include "timeline/TimelineModel.h" - -#include -#include -#include - -static constexpr int TIMEOUT = 2 * 60 * 1000; // 2 minutes - -namespace msgs = mtx::events::msg; - -static mtx::events::msg::KeyVerificationMac -key_verification_mac(mtx::crypto::SAS *sas, - mtx::identifiers::User sender, - const std::string &senderDevice, - mtx::identifiers::User receiver, - const std::string &receiverDevice, - const std::string &transactionId, - std::map keys); - -DeviceVerificationFlow::DeviceVerificationFlow(QObject *, - DeviceVerificationFlow::Type flow_type, - TimelineModel *model, - QString userID, - QString deviceId_) - : sender(false) - , type(flow_type) - , deviceId(deviceId_) - , model_(model) -{ - timeout = new QTimer(this); - timeout->setSingleShot(true); - this->sas = olm::client()->sas_init(); - this->isMacVerified = false; - - auto user_id = userID.toStdString(); - this->toClient = mtx::identifiers::parse(user_id); - cache::client()->query_keys( - user_id, [user_id, this](const UserKeyCache &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to query device keys: {},{}", - mtx::errors::to_string(err->matrix_error.errcode), - static_cast(err->status_code)); - return; - } - - if (!this->deviceId.isEmpty() && - (res.device_keys.find(deviceId.toStdString()) == res.device_keys.end())) { - nhlog::net()->warn("no devices retrieved {}", user_id); - return; - } - - this->their_keys = res; - }); - - cache::client()->query_keys( - http::client()->user_id().to_string(), - [this](const UserKeyCache &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to query device keys: {},{}", - mtx::errors::to_string(err->matrix_error.errcode), - static_cast(err->status_code)); - return; - } - - if (res.master_keys.keys.empty()) - return; - - if (auto status = cache::verificationStatus(http::client()->user_id().to_string()); - status && status->user_verified == crypto::Trust::Verified) - this->our_trusted_master_key = res.master_keys.keys.begin()->second; - }); - - if (model) { - connect( - this->model_, &TimelineModel::updateFlowEventId, this, [this](std::string event_id_) { - this->relation.rel_type = mtx::common::RelationType::Reference; - this->relation.event_id = event_id_; - this->transaction_id = event_id_; - }); - } - - connect(timeout, &QTimer::timeout, this, [this]() { - nhlog::crypto()->info("verification: timeout"); - if (state_ != Success && state_ != Failed) - this->cancelVerification(DeviceVerificationFlow::Error::Timeout); - }); - - connect(ChatPage::instance(), - &ChatPage::receivedDeviceVerificationStart, - this, - &DeviceVerificationFlow::handleStartMessage); - connect(ChatPage::instance(), - &ChatPage::receivedDeviceVerificationAccept, - this, - [this](const mtx::events::msg::KeyVerificationAccept &msg) { - nhlog::crypto()->info("verification: received accept"); - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - if ((msg.key_agreement_protocol == "curve25519-hkdf-sha256") && - (msg.hash == "sha256") && - (msg.message_authentication_code == "hkdf-hmac-sha256")) { - this->commitment = msg.commitment; - if (std::find(msg.short_authentication_string.begin(), - msg.short_authentication_string.end(), - mtx::events::msg::SASMethods::Emoji) != - msg.short_authentication_string.end()) { - this->method = mtx::events::msg::SASMethods::Emoji; - } else { - this->method = mtx::events::msg::SASMethods::Decimal; - } - this->mac_method = msg.message_authentication_code; - this->sendVerificationKey(); - } else { - this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); - } - }); - - connect(ChatPage::instance(), - &ChatPage::receivedDeviceVerificationCancel, - this, - [this](const mtx::events::msg::KeyVerificationCancel &msg) { - nhlog::crypto()->info("verification: received cancel"); - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - error_ = User; - emit errorChanged(); - setState(Failed); - }); - - connect( - ChatPage::instance(), - &ChatPage::receivedDeviceVerificationKey, - this, - [this](const mtx::events::msg::KeyVerificationKey &msg) { - nhlog::crypto()->info("verification: received key"); - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - - if (sender) { - if (state_ != WaitingForOtherToAccept) { - this->cancelVerification(OutOfOrder); - return; - } - } else { - if (state_ != WaitingForKeys) { - this->cancelVerification(OutOfOrder); - return; - } - } - - this->sas->set_their_key(msg.key); - std::string info; - if (this->sender == true) { - info = "MATRIX_KEY_VERIFICATION_SAS|" + http::client()->user_id().to_string() + "|" + - http::client()->device_id() + "|" + this->sas->public_key() + "|" + - this->toClient.to_string() + "|" + this->deviceId.toStdString() + "|" + - msg.key + "|" + this->transaction_id; - } else { - info = "MATRIX_KEY_VERIFICATION_SAS|" + this->toClient.to_string() + "|" + - this->deviceId.toStdString() + "|" + msg.key + "|" + - http::client()->user_id().to_string() + "|" + http::client()->device_id() + - "|" + this->sas->public_key() + "|" + this->transaction_id; - } - - nhlog::ui()->info("Info is: '{}'", info); - - if (this->sender == false) { - this->sendVerificationKey(); - } else { - if (this->commitment != mtx::crypto::bin2base64_unpadded(mtx::crypto::sha256( - msg.key + this->canonical_json.dump()))) { - this->cancelVerification(DeviceVerificationFlow::Error::MismatchedCommitment); - return; - } - } - - if (this->method == mtx::events::msg::SASMethods::Emoji) { - this->sasList = this->sas->generate_bytes_emoji(info); - setState(CompareEmoji); - } else if (this->method == mtx::events::msg::SASMethods::Decimal) { - this->sasList = this->sas->generate_bytes_decimal(info); - setState(CompareNumber); - } - }); - - connect( - ChatPage::instance(), - &ChatPage::receivedDeviceVerificationMac, - this, - [this](const mtx::events::msg::KeyVerificationMac &msg) { - nhlog::crypto()->info("verification: received mac"); - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - - std::map key_list; - std::string key_string; - for (const auto &mac : msg.mac) { - for (const auto &[deviceid, key] : their_keys.device_keys) { - (void)deviceid; - if (key.keys.count(mac.first)) - key_list[mac.first] = key.keys.at(mac.first); - } - - if (their_keys.master_keys.keys.count(mac.first)) - key_list[mac.first] = their_keys.master_keys.keys[mac.first]; - if (their_keys.user_signing_keys.keys.count(mac.first)) - key_list[mac.first] = their_keys.user_signing_keys.keys[mac.first]; - if (their_keys.self_signing_keys.keys.count(mac.first)) - key_list[mac.first] = their_keys.self_signing_keys.keys[mac.first]; - } - auto macs = key_verification_mac(sas.get(), - toClient, - this->deviceId.toStdString(), - http::client()->user_id(), - http::client()->device_id(), - this->transaction_id, - key_list); - - for (const auto &[key, mac] : macs.mac) { - if (mac != msg.mac.at(key)) { - this->cancelVerification(DeviceVerificationFlow::Error::KeyMismatch); - return; - } - } - - if (msg.keys == macs.keys) { - mtx::requests::KeySignaturesUpload req; - if (utils::localUser().toStdString() == this->toClient.to_string()) { - // self verification, sign master key with device key, if we - // verified it - for (const auto &mac : msg.mac) { - if (their_keys.master_keys.keys.count(mac.first)) { - json j = their_keys.master_keys; - j.erase("signatures"); - j.erase("unsigned"); - mtx::crypto::CrossSigningKeys master_key = j; - master_key.signatures[utils::localUser().toStdString()] - ["ed25519:" + http::client()->device_id()] = - olm::client()->sign_message(j.dump()); - req.signatures[utils::localUser().toStdString()] - [master_key.keys.at(mac.first)] = master_key; - } else if (mac.first == "ed25519:" + this->deviceId.toStdString()) { - // Sign their device key with self signing key - - auto device_id = this->deviceId.toStdString(); - - if (their_keys.device_keys.count(device_id)) { - json j = their_keys.device_keys.at(device_id); - j.erase("signatures"); - j.erase("unsigned"); - - auto secret = cache::secret( - mtx::secret_storage::secrets::cross_signing_self_signing); - if (!secret) - continue; - auto ssk = mtx::crypto::PkSigning::from_seed(*secret); - - mtx::crypto::DeviceKeys dev = j; - dev.signatures[utils::localUser().toStdString()] - ["ed25519:" + ssk.public_key()] = ssk.sign(j.dump()); - - req.signatures[utils::localUser().toStdString()][device_id] = dev; - } - } - } - } else { - // Sign their master key with user signing key - for (const auto &mac : msg.mac) { - if (their_keys.master_keys.keys.count(mac.first)) { - json j = their_keys.master_keys; - j.erase("signatures"); - j.erase("unsigned"); - - auto secret = - cache::secret(mtx::secret_storage::secrets::cross_signing_user_signing); - if (!secret) - continue; - auto usk = mtx::crypto::PkSigning::from_seed(*secret); - - mtx::crypto::CrossSigningKeys master_key = j; - master_key.signatures[utils::localUser().toStdString()] - ["ed25519:" + usk.public_key()] = usk.sign(j.dump()); - - req.signatures[toClient.to_string()][master_key.keys.at(mac.first)] = - master_key; - } - } - } - - if (!req.signatures.empty()) { - http::client()->keys_signatures_upload( - req, - [](const mtx::responses::KeySignaturesUpload &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("failed to upload signatures: {},{}", - mtx::errors::to_string(err->matrix_error.errcode), - static_cast(err->status_code)); - } - - for (const auto &[user_id, tmp] : res.errors) - for (const auto &[key_id, e] : tmp) - nhlog::net()->error("signature error for user {} and key " - "id {}: {}, {}", - user_id, - key_id, - mtx::errors::to_string(e.errcode), - e.error); - }); - } - - this->isMacVerified = true; - this->acceptDevice(); - } else { - this->cancelVerification(DeviceVerificationFlow::Error::KeyMismatch); - } - }); - - connect(ChatPage::instance(), - &ChatPage::receivedDeviceVerificationReady, - this, - [this](const mtx::events::msg::KeyVerificationReady &msg) { - nhlog::crypto()->info("verification: received ready"); - if (!sender) { - if (msg.from_device != http::client()->device_id()) { - error_ = User; - emit errorChanged(); - setState(Failed); - } - - return; - } - - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - else { - this->deviceId = QString::fromStdString(msg.from_device); - } - } - this->startVerificationRequest(); - }); - - connect(ChatPage::instance(), - &ChatPage::receivedDeviceVerificationDone, - this, - [this](const mtx::events::msg::KeyVerificationDone &msg) { - nhlog::crypto()->info("verification: received done"); - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - nhlog::ui()->info("Flow done on other side"); - }); - - timeout->start(TIMEOUT); -} - -QString -DeviceVerificationFlow::state() -{ - switch (state_) { - case PromptStartVerification: - return "PromptStartVerification"; - case CompareEmoji: - return "CompareEmoji"; - case CompareNumber: - return "CompareNumber"; - case WaitingForKeys: - return "WaitingForKeys"; - case WaitingForOtherToAccept: - return "WaitingForOtherToAccept"; - case WaitingForMac: - return "WaitingForMac"; - case Success: - return "Success"; - case Failed: - return "Failed"; - default: - return ""; - } -} - -void -DeviceVerificationFlow::next() -{ - if (sender) { - switch (state_) { - case PromptStartVerification: - sendVerificationRequest(); - break; - case CompareEmoji: - case CompareNumber: - sendVerificationMac(); - break; - case WaitingForKeys: - case WaitingForOtherToAccept: - case WaitingForMac: - case Success: - case Failed: - nhlog::db()->error("verification: Invalid state transition!"); - break; - } - } else { - switch (state_) { - case PromptStartVerification: - if (canonical_json.is_null()) - sendVerificationReady(); - else // legacy path without request and ready - acceptVerificationRequest(); - break; - case CompareEmoji: - [[fallthrough]]; - case CompareNumber: - sendVerificationMac(); - break; - case WaitingForKeys: - case WaitingForOtherToAccept: - case WaitingForMac: - case Success: - case Failed: - nhlog::db()->error("verification: Invalid state transition!"); - break; - } - } -} - -QString -DeviceVerificationFlow::getUserId() -{ - return QString::fromStdString(this->toClient.to_string()); -} - -QString -DeviceVerificationFlow::getDeviceId() -{ - return this->deviceId; -} - -bool -DeviceVerificationFlow::getSender() -{ - return this->sender; -} - -std::vector -DeviceVerificationFlow::getSasList() -{ - return this->sasList; -} - -bool -DeviceVerificationFlow::isSelfVerification() const -{ - return this->toClient.to_string() == http::client()->user_id().to_string(); -} - -void -DeviceVerificationFlow::setEventId(std::string event_id_) -{ - this->relation.rel_type = mtx::common::RelationType::Reference; - this->relation.event_id = event_id_; - this->transaction_id = event_id_; -} - -void -DeviceVerificationFlow::handleStartMessage(const mtx::events::msg::KeyVerificationStart &msg, - std::string) -{ - if (msg.transaction_id.has_value()) { - if (msg.transaction_id.value() != this->transaction_id) - return; - } else if (msg.relations.references()) { - if (msg.relations.references() != this->relation.event_id) - return; - } - if ((std::find(msg.key_agreement_protocols.begin(), - msg.key_agreement_protocols.end(), - "curve25519-hkdf-sha256") != msg.key_agreement_protocols.end()) && - (std::find(msg.hashes.begin(), msg.hashes.end(), "sha256") != msg.hashes.end()) && - (std::find(msg.message_authentication_codes.begin(), - msg.message_authentication_codes.end(), - "hkdf-hmac-sha256") != msg.message_authentication_codes.end())) { - if (std::find(msg.short_authentication_string.begin(), - msg.short_authentication_string.end(), - mtx::events::msg::SASMethods::Emoji) != - msg.short_authentication_string.end()) { - this->method = mtx::events::msg::SASMethods::Emoji; - } else if (std::find(msg.short_authentication_string.begin(), - msg.short_authentication_string.end(), - mtx::events::msg::SASMethods::Decimal) != - msg.short_authentication_string.end()) { - this->method = mtx::events::msg::SASMethods::Decimal; - } else { - this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); - return; - } - if (!sender) - this->canonical_json = nlohmann::json(msg); - else { - if (utils::localUser().toStdString() < this->toClient.to_string()) { - this->canonical_json = nlohmann::json(msg); - } - } - - if (state_ != PromptStartVerification) - this->acceptVerificationRequest(); - } else { - this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); - } -} - -//! accepts a verification -void -DeviceVerificationFlow::acceptVerificationRequest() -{ - mtx::events::msg::KeyVerificationAccept req; - - req.method = mtx::events::msg::VerificationMethods::SASv1; - req.key_agreement_protocol = "curve25519-hkdf-sha256"; - req.hash = "sha256"; - req.message_authentication_code = "hkdf-hmac-sha256"; - if (this->method == mtx::events::msg::SASMethods::Emoji) - req.short_authentication_string = {mtx::events::msg::SASMethods::Emoji}; - else if (this->method == mtx::events::msg::SASMethods::Decimal) - req.short_authentication_string = {mtx::events::msg::SASMethods::Decimal}; - req.commitment = mtx::crypto::bin2base64_unpadded( - mtx::crypto::sha256(this->sas->public_key() + this->canonical_json.dump())); - - send(req); - setState(WaitingForKeys); -} -//! responds verification request -void -DeviceVerificationFlow::sendVerificationReady() -{ - mtx::events::msg::KeyVerificationReady req; - - req.from_device = http::client()->device_id(); - req.methods = {mtx::events::msg::VerificationMethods::SASv1}; - - send(req); - setState(WaitingForKeys); -} -//! accepts a verification -void -DeviceVerificationFlow::sendVerificationDone() -{ - mtx::events::msg::KeyVerificationDone req; - - send(req); -} -//! starts the verification flow -void -DeviceVerificationFlow::startVerificationRequest() -{ - mtx::events::msg::KeyVerificationStart req; - - req.from_device = http::client()->device_id(); - req.method = mtx::events::msg::VerificationMethods::SASv1; - req.key_agreement_protocols = {"curve25519-hkdf-sha256"}; - req.hashes = {"sha256"}; - req.message_authentication_codes = {"hkdf-hmac-sha256"}; - req.short_authentication_string = {mtx::events::msg::SASMethods::Decimal, - mtx::events::msg::SASMethods::Emoji}; - - if (this->type == DeviceVerificationFlow::Type::ToDevice) { - mtx::requests::ToDeviceMessages body; - req.transaction_id = this->transaction_id; - this->canonical_json = nlohmann::json(req); - } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { - req.relations.relations.push_back(this->relation); - // Set synthesized to surpress the nheko relation extensions - req.relations.synthesized = true; - this->canonical_json = nlohmann::json(req); - } - send(req); - setState(WaitingForOtherToAccept); -} -//! sends a verification request -void -DeviceVerificationFlow::sendVerificationRequest() -{ - mtx::events::msg::KeyVerificationRequest req; - - req.from_device = http::client()->device_id(); - req.methods = {mtx::events::msg::VerificationMethods::SASv1}; - - if (this->type == DeviceVerificationFlow::Type::ToDevice) { - QDateTime currentTime = QDateTime::currentDateTimeUtc(); - - req.timestamp = (uint64_t)currentTime.toMSecsSinceEpoch(); - - } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { - req.to = this->toClient.to_string(); - req.msgtype = "m.key.verification.request"; - req.body = "User is requesting to verify keys with you. However, your client does " - "not support this method, so you will need to use the legacy method of " - "key verification."; - } - - send(req); - setState(WaitingForOtherToAccept); -} -//! cancels a verification flow -void -DeviceVerificationFlow::cancelVerification(DeviceVerificationFlow::Error error_code) -{ - if (state_ == State::Success || state_ == State::Failed) - return; - - mtx::events::msg::KeyVerificationCancel req; - - if (error_code == DeviceVerificationFlow::Error::UnknownMethod) { - req.code = "m.unknown_method"; - req.reason = "unknown method received"; - } else if (error_code == DeviceVerificationFlow::Error::MismatchedCommitment) { - req.code = "m.mismatched_commitment"; - req.reason = "commitment didn't match"; - } else if (error_code == DeviceVerificationFlow::Error::MismatchedSAS) { - req.code = "m.mismatched_sas"; - req.reason = "sas didn't match"; - } else if (error_code == DeviceVerificationFlow::Error::KeyMismatch) { - req.code = "m.key_match"; - req.reason = "keys did not match"; - } else if (error_code == DeviceVerificationFlow::Error::Timeout) { - req.code = "m.timeout"; - req.reason = "timed out"; - } else if (error_code == DeviceVerificationFlow::Error::User) { - req.code = "m.user"; - req.reason = "user cancelled the verification"; - } else if (error_code == DeviceVerificationFlow::Error::OutOfOrder) { - req.code = "m.unexpected_message"; - req.reason = "received messages out of order"; - } - - this->error_ = error_code; - emit errorChanged(); - this->setState(Failed); - - send(req); -} -//! sends the verification key -void -DeviceVerificationFlow::sendVerificationKey() -{ - mtx::events::msg::KeyVerificationKey req; - - req.key = this->sas->public_key(); - - send(req); -} - -mtx::events::msg::KeyVerificationMac -key_verification_mac(mtx::crypto::SAS *sas, - mtx::identifiers::User sender, - const std::string &senderDevice, - mtx::identifiers::User receiver, - const std::string &receiverDevice, - const std::string &transactionId, - std::map keys) -{ - mtx::events::msg::KeyVerificationMac req; - - std::string info = "MATRIX_KEY_VERIFICATION_MAC" + sender.to_string() + senderDevice + - receiver.to_string() + receiverDevice + transactionId; - - std::string key_list; - bool first = true; - for (const auto &[key_id, key] : keys) { - req.mac[key_id] = sas->calculate_mac(key, info + key_id); - - if (!first) - key_list += ","; - key_list += key_id; - first = false; - } - - req.keys = sas->calculate_mac(key_list, info + "KEY_IDS"); - - return req; -} - -//! sends the mac of the keys -void -DeviceVerificationFlow::sendVerificationMac() -{ - std::map key_list; - key_list["ed25519:" + http::client()->device_id()] = olm::client()->identity_keys().ed25519; - - // send our master key, if we trust it - if (!this->our_trusted_master_key.empty()) - key_list["ed25519:" + our_trusted_master_key] = our_trusted_master_key; - - mtx::events::msg::KeyVerificationMac req = key_verification_mac(sas.get(), - http::client()->user_id(), - http::client()->device_id(), - this->toClient, - this->deviceId.toStdString(), - this->transaction_id, - key_list); - - send(req); - - setState(WaitingForMac); - acceptDevice(); -} -//! Completes the verification flow -void -DeviceVerificationFlow::acceptDevice() -{ - if (!isMacVerified) { - setState(WaitingForMac); - } else if (state_ == WaitingForMac) { - cache::markDeviceVerified(this->toClient.to_string(), this->deviceId.toStdString()); - this->sendVerificationDone(); - setState(Success); - - // Request secrets. We should probably check somehow, if a device knowns about the - // secrets. - if (utils::localUser().toStdString() == this->toClient.to_string() && - (!cache::secret(mtx::secret_storage::secrets::cross_signing_self_signing) || - !cache::secret(mtx::secret_storage::secrets::cross_signing_user_signing))) { - olm::request_cross_signing_keys(); - } - } -} - -void -DeviceVerificationFlow::unverify() -{ - cache::markDeviceUnverified(this->toClient.to_string(), this->deviceId.toStdString()); - - emit refreshProfile(); -} - -QSharedPointer -DeviceVerificationFlow::NewInRoomVerification(QObject *parent_, - TimelineModel *timelineModel_, - const mtx::events::msg::KeyVerificationRequest &msg, - QString other_user_, - QString event_id_) -{ - QSharedPointer flow( - new DeviceVerificationFlow(parent_, - Type::RoomMsg, - timelineModel_, - other_user_, - QString::fromStdString(msg.from_device))); - - flow->setEventId(event_id_.toStdString()); - - if (std::find(msg.methods.begin(), - msg.methods.end(), - mtx::events::msg::VerificationMethods::SASv1) == msg.methods.end()) { - flow->cancelVerification(UnknownMethod); - } - - return flow; -} -QSharedPointer -DeviceVerificationFlow::NewToDeviceVerification(QObject *parent_, - const mtx::events::msg::KeyVerificationRequest &msg, - QString other_user_, - QString txn_id_) -{ - QSharedPointer flow(new DeviceVerificationFlow( - parent_, Type::ToDevice, nullptr, other_user_, QString::fromStdString(msg.from_device))); - flow->transaction_id = txn_id_.toStdString(); - - if (std::find(msg.methods.begin(), - msg.methods.end(), - mtx::events::msg::VerificationMethods::SASv1) == msg.methods.end()) { - flow->cancelVerification(UnknownMethod); - } - - return flow; -} -QSharedPointer -DeviceVerificationFlow::NewToDeviceVerification(QObject *parent_, - const mtx::events::msg::KeyVerificationStart &msg, - QString other_user_, - QString txn_id_) -{ - QSharedPointer flow(new DeviceVerificationFlow( - parent_, Type::ToDevice, nullptr, other_user_, QString::fromStdString(msg.from_device))); - flow->transaction_id = txn_id_.toStdString(); - - flow->handleStartMessage(msg, ""); - - return flow; -} -QSharedPointer -DeviceVerificationFlow::InitiateUserVerification(QObject *parent_, - TimelineModel *timelineModel_, - QString userid) -{ - QSharedPointer flow( - new DeviceVerificationFlow(parent_, Type::RoomMsg, timelineModel_, userid, "")); - flow->sender = true; - return flow; -} -QSharedPointer -DeviceVerificationFlow::InitiateDeviceVerification(QObject *parent_, QString userid, QString device) -{ - QSharedPointer flow( - new DeviceVerificationFlow(parent_, Type::ToDevice, nullptr, userid, device)); - - flow->sender = true; - flow->transaction_id = http::client()->generate_txn_id(); - - return flow; -} diff --git a/src/DeviceVerificationFlow.h b/src/DeviceVerificationFlow.h deleted file mode 100644 index f71fa337..00000000 --- a/src/DeviceVerificationFlow.h +++ /dev/null @@ -1,248 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include - -#include -#include - -#include "CacheCryptoStructs.h" -#include "Logging.h" -#include "MatrixClient.h" -#include "Olm.h" -#include "timeline/TimelineModel.h" - -class QTimer; - -using sas_ptr = std::unique_ptr; - -// clang-format off -/* - * Stolen from fluffy chat :D - * - * State | +-------------+ +-----------+ | - * | | AliceDevice | | BobDevice | | - * | | (sender) | | | | - * | +-------------+ +-----------+ | - * promptStartVerify | | | | - * | o | (m.key.verification.request) | | - * | p |-------------------------------->| (ASK FOR VERIFICATION REQUEST) | - * waitForOtherAccept | t | | | promptStartVerify - * && | i | (m.key.verification.ready) | | - * no commitment | o |<--------------------------------| | - * && | n | | | - * no canonical_json | a | (m.key.verification.start) | | waitingForKeys - * | l |<--------------------------------| Not sending to prevent the glare resolve| && no commitment - * | | | | && no canonical_json - * | | m.key.verification.start | | - * waitForOtherAccept | |-------------------------------->| (IF NOT ALREADY ASKED, | - * && | | | ASK FOR VERIFICATION REQUEST) | promptStartVerify, if not accepted - * canonical_json | | m.key.verification.accept | | - * | |<--------------------------------| | - * waitForOtherAccept | | | | waitingForKeys - * && | | m.key.verification.key | | && canonical_json - * commitment | |-------------------------------->| | && commitment - * | | | | - * | | m.key.verification.key | | - * | |<--------------------------------| | - * compareEmoji/Number| | | | compareEmoji/Number - * | | COMPARE EMOJI / NUMBERS | | - * | | | | - * waitingForMac | | m.key.verification.mac | | waitingForMac - * | success |<------------------------------->| success | - * | | | | - * success/fail | | m.key.verification.done | | success/fail - * | |<------------------------------->| | - */ -// clang-format on -class DeviceVerificationFlow : public QObject -{ - Q_OBJECT - Q_PROPERTY(QString state READ state NOTIFY stateChanged) - Q_PROPERTY(Error error READ error NOTIFY errorChanged) - Q_PROPERTY(QString userId READ getUserId CONSTANT) - Q_PROPERTY(QString deviceId READ getDeviceId CONSTANT) - Q_PROPERTY(bool sender READ getSender CONSTANT) - Q_PROPERTY(std::vector sasList READ getSasList CONSTANT) - Q_PROPERTY(bool isDeviceVerification READ isDeviceVerification CONSTANT) - Q_PROPERTY(bool isSelfVerification READ isSelfVerification CONSTANT) - -public: - enum State - { - PromptStartVerification, - WaitingForOtherToAccept, - WaitingForKeys, - CompareEmoji, - CompareNumber, - WaitingForMac, - Success, - Failed, - }; - Q_ENUM(State) - - enum Type - { - ToDevice, - RoomMsg - }; - - enum Error - { - UnknownMethod, - MismatchedCommitment, - MismatchedSAS, - KeyMismatch, - Timeout, - User, - OutOfOrder, - }; - Q_ENUM(Error) - - static QSharedPointer NewInRoomVerification( - QObject *parent_, - TimelineModel *timelineModel_, - const mtx::events::msg::KeyVerificationRequest &msg, - QString other_user_, - QString event_id_); - static QSharedPointer NewToDeviceVerification( - QObject *parent_, - const mtx::events::msg::KeyVerificationRequest &msg, - QString other_user_, - QString txn_id_); - static QSharedPointer NewToDeviceVerification( - QObject *parent_, - const mtx::events::msg::KeyVerificationStart &msg, - QString other_user_, - QString txn_id_); - static QSharedPointer - InitiateUserVerification(QObject *parent_, TimelineModel *timelineModel_, QString userid); - static QSharedPointer InitiateDeviceVerification(QObject *parent, - QString userid, - QString device); - - // getters - QString state(); - Error error() { return error_; } - QString getUserId(); - QString getDeviceId(); - bool getSender(); - std::vector getSasList(); - QString transactionId() { return QString::fromStdString(this->transaction_id); } - // setters - void setDeviceId(QString deviceID); - void setEventId(std::string event_id); - bool isDeviceVerification() const - { - return this->type == DeviceVerificationFlow::Type::ToDevice; - } - bool isSelfVerification() const; - - void callback_fn(const UserKeyCache &res, mtx::http::RequestErr err, std::string user_id); - -public slots: - //! unverifies a device - void unverify(); - //! Continues the flow - void next(); - //! Cancel the flow - void cancel() { cancelVerification(User); } - -signals: - void refreshProfile(); - void stateChanged(); - void errorChanged(); - -private: - DeviceVerificationFlow(QObject *, - DeviceVerificationFlow::Type flow_type, - TimelineModel *model, - QString userID, - QString deviceId_); - void setState(State state) - { - if (state != state_) { - state_ = state; - emit stateChanged(); - } - } - - void handleStartMessage(const mtx::events::msg::KeyVerificationStart &msg, std::string); - //! sends a verification request - void sendVerificationRequest(); - //! accepts a verification request - void sendVerificationReady(); - //! completes the verification flow(); - void sendVerificationDone(); - //! accepts a verification - void acceptVerificationRequest(); - //! starts the verification flow - void startVerificationRequest(); - //! cancels a verification flow - void cancelVerification(DeviceVerificationFlow::Error error_code); - //! sends the verification key - void sendVerificationKey(); - //! sends the mac of the keys - void sendVerificationMac(); - //! Completes the verification flow - void acceptDevice(); - - std::string transaction_id; - - bool sender; - Type type; - mtx::identifiers::User toClient; - QString deviceId; - - // public part of our master key, when trusted or empty - std::string our_trusted_master_key; - - mtx::events::msg::SASMethods method = mtx::events::msg::SASMethods::Emoji; - QTimer *timeout = nullptr; - sas_ptr sas; - std::string mac_method; - std::string commitment; - nlohmann::json canonical_json; - - std::vector sasList; - UserKeyCache their_keys; - TimelineModel *model_; - mtx::common::Relation relation; - - State state_ = PromptStartVerification; - Error error_ = UnknownMethod; - - bool isMacVerified = false; - - template - void send(T msg) - { - if (this->type == DeviceVerificationFlow::Type::ToDevice) { - mtx::requests::ToDeviceMessages body; - msg.transaction_id = this->transaction_id; - body[this->toClient][deviceId.toStdString()] = msg; - - http::client()->send_to_device( - this->transaction_id, body, [](mtx::http::RequestErr err) { - if (err) - nhlog::net()->warn("failed to send verification to_device message: {} {}", - err->matrix_error.error, - static_cast(err->status_code)); - }); - } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { - if constexpr (!std::is_same_v) { - msg.relations.relations.push_back(this->relation); - // Set synthesized to surpress the nheko relation extensions - msg.relations.synthesized = true; - } - (model_)->sendMessageEvent(msg, mtx::events::to_device_content_to_type); - } - - nhlog::net()->debug("Sent verification step: {} in state: {}", - mtx::events::to_string(mtx::events::to_device_content_to_type), - state().toStdString()); - } -}; diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index c8eb2d24..34db0d1d 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -26,11 +26,11 @@ #include "TrayIcon.h" #include "UserSettingsPage.h" #include "Utils.h" -#include "WebRTCSession.h" #include "WelcomePage.h" #include "ui/LoadingIndicator.h" #include "ui/OverlayModal.h" #include "ui/SnackBar.h" +#include "voip/WebRTCSession.h" #include "dialogs/CreateRoom.h" diff --git a/src/Olm.cpp b/src/Olm.cpp deleted file mode 100644 index 14c97984..00000000 --- a/src/Olm.cpp +++ /dev/null @@ -1,1612 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "Olm.h" - -#include -#include - -#include -#include - -#include -#include - -#include "Cache.h" -#include "Cache_p.h" -#include "ChatPage.h" -#include "DeviceVerificationFlow.h" -#include "Logging.h" -#include "MatrixClient.h" -#include "UserSettingsPage.h" -#include "Utils.h" - -namespace { -auto client_ = std::make_unique(); - -std::map request_id_to_secret_name; - -constexpr auto MEGOLM_ALGO = "m.megolm.v1.aes-sha2"; -} - -namespace olm { -static void -backup_session_key(const MegolmSessionIndex &idx, - const GroupSessionData &data, - mtx::crypto::InboundGroupSessionPtr &session); - -void -from_json(const nlohmann::json &obj, OlmMessage &msg) -{ - if (obj.at("type") != "m.room.encrypted") - throw std::invalid_argument("invalid type for olm message"); - - if (obj.at("content").at("algorithm") != OLM_ALGO) - throw std::invalid_argument("invalid algorithm for olm message"); - - msg.sender = obj.at("sender"); - msg.sender_key = obj.at("content").at("sender_key"); - msg.ciphertext = obj.at("content") - .at("ciphertext") - .get>(); -} - -mtx::crypto::OlmClient * -client() -{ - return client_.get(); -} - -static void -handle_secret_request(const mtx::events::DeviceEvent *e, - const std::string &sender) -{ - using namespace mtx::events; - - if (e->content.action != mtx::events::msg::RequestAction::Request) - return; - - auto local_user = http::client()->user_id(); - - if (sender != local_user.to_string()) - return; - - auto verificationStatus = cache::verificationStatus(local_user.to_string()); - - if (!verificationStatus) - return; - - auto deviceKeys = cache::userKeys(local_user.to_string()); - if (!deviceKeys) - return; - - if (std::find(verificationStatus->verified_devices.begin(), - verificationStatus->verified_devices.end(), - e->content.requesting_device_id) == verificationStatus->verified_devices.end()) - return; - - // this is a verified device - mtx::events::DeviceEvent secretSend; - secretSend.type = EventType::SecretSend; - secretSend.content.request_id = e->content.request_id; - - auto secret = cache::client()->secret(e->content.name); - if (!secret) - return; - secretSend.content.secret = secret.value(); - - send_encrypted_to_device_messages( - {{local_user.to_string(), {{e->content.requesting_device_id}}}}, secretSend); - - nhlog::net()->info("Sent secret '{}' to ({},{})", - e->content.name, - local_user.to_string(), - e->content.requesting_device_id); -} - -void -handle_to_device_messages(const std::vector &msgs) -{ - if (msgs.empty()) - return; - nhlog::crypto()->info("received {} to_device messages", msgs.size()); - nlohmann::json j_msg; - - for (const auto &msg : msgs) { - j_msg = std::visit([](auto &e) { return json(e); }, std::move(msg)); - if (j_msg.count("type") == 0) { - nhlog::crypto()->warn("received message with no type field: {}", j_msg.dump(2)); - continue; - } - - std::string msg_type = j_msg.at("type"); - - if (msg_type == to_string(mtx::events::EventType::RoomEncrypted)) { - try { - olm::OlmMessage olm_msg = j_msg; - cache::client()->query_keys( - olm_msg.sender, [olm_msg](const UserKeyCache &userKeys, mtx::http::RequestErr e) { - if (e) { - nhlog::crypto()->error("Failed to query user keys, dropping olm " - "message"); - return; - } - handle_olm_message(std::move(olm_msg), userKeys); - }); - } catch (const nlohmann::json::exception &e) { - nhlog::crypto()->warn( - "parsing error for olm message: {} {}", e.what(), j_msg.dump(2)); - } catch (const std::invalid_argument &e) { - nhlog::crypto()->warn( - "validation error for olm message: {} {}", e.what(), j_msg.dump(2)); - } - - } else if (msg_type == to_string(mtx::events::EventType::RoomKeyRequest)) { - nhlog::crypto()->warn("handling key request event: {}", j_msg.dump(2)); - try { - mtx::events::DeviceEvent req = j_msg; - if (req.content.action == mtx::events::msg::RequestAction::Request) - handle_key_request_message(req); - else - nhlog::crypto()->warn("ignore key request (unhandled action): {}", - req.content.request_id); - } catch (const nlohmann::json::exception &e) { - nhlog::crypto()->warn( - "parsing error for key_request message: {} {}", e.what(), j_msg.dump(2)); - } - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationAccept)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationAccept(message.content); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationRequest)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationRequest(message.content, - message.sender); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationCancel)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationCancel(message.content); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationKey)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationKey(message.content); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationMac)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationMac(message.content); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationStart)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationStart(message.content, message.sender); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationReady)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationReady(message.content); - } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationDone)) { - auto message = - std::get>(msg); - ChatPage::instance()->receivedDeviceVerificationDone(message.content); - } else if (auto e = - std::get_if>(&msg)) { - handle_secret_request(e, e->sender); - } else { - nhlog::crypto()->warn("unhandled event: {}", j_msg.dump(2)); - } - } -} - -void -handle_olm_message(const OlmMessage &msg, const UserKeyCache &otherUserDeviceKeys) -{ - nhlog::crypto()->info("sender : {}", msg.sender); - nhlog::crypto()->info("sender_key: {}", msg.sender_key); - - if (msg.sender_key == olm::client()->identity_keys().ed25519) { - nhlog::crypto()->warn("Ignoring olm message from ourselves!"); - return; - } - - const auto my_key = olm::client()->identity_keys().curve25519; - - bool failed_decryption = false; - - for (const auto &cipher : msg.ciphertext) { - // We skip messages not meant for the current device. - if (cipher.first != my_key) { - nhlog::crypto()->debug( - "Skipping message for {} since we are {}.", cipher.first, my_key); - continue; - } - - const auto type = cipher.second.type; - nhlog::crypto()->info("type: {}", type == 0 ? "OLM_PRE_KEY" : "OLM_MESSAGE"); - - auto payload = try_olm_decryption(msg.sender_key, cipher.second); - - if (payload.is_null()) { - // Check for PRE_KEY message - if (cipher.second.type == 0) { - payload = handle_pre_key_olm_message(msg.sender, msg.sender_key, cipher.second); - } else { - nhlog::crypto()->error("Undecryptable olm message!"); - failed_decryption = true; - continue; - } - } - - if (!payload.is_null()) { - mtx::events::collections::DeviceEvents device_event; - - // Other properties are included in order to prevent an attacker from - // publishing someone else's curve25519 keys as their own and subsequently - // claiming to have sent messages which they didn't. sender must correspond - // to the user who sent the event, recipient to the local user, and - // recipient_keys to the local ed25519 key. - std::string receiver_ed25519 = payload["recipient_keys"]["ed25519"]; - if (receiver_ed25519.empty() || - receiver_ed25519 != olm::client()->identity_keys().ed25519) { - nhlog::crypto()->warn("Decrypted event doesn't include our ed25519: {}", - payload.dump()); - return; - } - std::string receiver = payload["recipient"]; - if (receiver.empty() || receiver != http::client()->user_id().to_string()) { - nhlog::crypto()->warn("Decrypted event doesn't include our user_id: {}", - payload.dump()); - return; - } - - // Clients must confirm that the sender_key and the ed25519 field value - // under the keys property match the keys returned by /keys/query for the - // given user, and must also verify the signature of the payload. Without - // this check, a client cannot be sure that the sender device owns the - // private part of the ed25519 key it claims to have in the Olm payload. - // This is crucial when the ed25519 key corresponds to a verified device. - std::string sender_ed25519 = payload["keys"]["ed25519"]; - if (sender_ed25519.empty()) { - nhlog::crypto()->warn("Decrypted event doesn't include sender ed25519: {}", - payload.dump()); - return; - } - - bool from_their_device = false; - for (auto [device_id, key] : otherUserDeviceKeys.device_keys) { - auto c_key = key.keys.find("curve25519:" + device_id); - auto e_key = key.keys.find("ed25519:" + device_id); - - if (c_key == key.keys.end() || e_key == key.keys.end()) { - nhlog::crypto()->warn("Skipping device {} as we have no keys for it.", - device_id); - } else if (c_key->second == msg.sender_key && e_key->second == sender_ed25519) { - from_their_device = true; - break; - } - } - if (!from_their_device) { - nhlog::crypto()->warn("Decrypted event isn't sent from a device " - "listed by that user! {}", - payload.dump()); - return; - } - - { - std::string msg_type = payload["type"]; - json event_array = json::array(); - event_array.push_back(payload); - - std::vector temp_events; - mtx::responses::utils::parse_device_events(event_array, temp_events); - if (temp_events.empty()) { - nhlog::crypto()->warn("Decrypted unknown event: {}", payload.dump()); - return; - } - device_event = temp_events.at(0); - } - - using namespace mtx::events; - if (auto e1 = std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationAccept(e1->content); - } else if (auto e2 = - std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationRequest(e2->content, e2->sender); - } else if (auto e3 = - std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationCancel(e3->content); - } else if (auto e4 = std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationKey(e4->content); - } else if (auto e5 = std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationMac(e5->content); - } else if (auto e6 = - std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationStart(e6->content, e6->sender); - } else if (auto e7 = - std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationReady(e7->content); - } else if (auto e8 = - std::get_if>(&device_event)) { - ChatPage::instance()->receivedDeviceVerificationDone(e8->content); - } else if (auto roomKey = std::get_if>(&device_event)) { - create_inbound_megolm_session(*roomKey, msg.sender_key, sender_ed25519); - } else if (auto forwardedRoomKey = - std::get_if>(&device_event)) { - forwardedRoomKey->content.forwarding_curve25519_key_chain.push_back(msg.sender_key); - import_inbound_megolm_session(*forwardedRoomKey); - } else if (auto e = std::get_if>(&device_event)) { - auto local_user = http::client()->user_id(); - - if (msg.sender != local_user.to_string()) - return; - - auto secret_name = request_id_to_secret_name.find(e->content.request_id); - - if (secret_name != request_id_to_secret_name.end()) { - nhlog::crypto()->info("Received secret: {}", secret_name->second); - - mtx::events::msg::SecretRequest secretRequest{}; - secretRequest.action = mtx::events::msg::RequestAction::Cancellation; - secretRequest.requesting_device_id = http::client()->device_id(); - secretRequest.request_id = e->content.request_id; - - auto verificationStatus = cache::verificationStatus(local_user.to_string()); - - if (!verificationStatus) - return; - - auto deviceKeys = cache::userKeys(local_user.to_string()); - std::string sender_device_id; - if (deviceKeys) { - for (auto &[dev, key] : deviceKeys->device_keys) { - if (key.keys["curve25519:" + dev] == msg.sender_key) { - sender_device_id = dev; - break; - } - } - } - - std::map> - body; - - for (const auto &dev : verificationStatus->verified_devices) { - if (dev != secretRequest.requesting_device_id && dev != sender_device_id) - body[local_user][dev] = secretRequest; - } - - http::client()->send_to_device( - http::client()->generate_txn_id(), - body, - [name = secret_name->second](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("Failed to send request cancellation " - "for secrect " - "'{}'", - name); - } - }); - - nhlog::crypto()->info("Storing secret {}", secret_name->second); - cache::client()->storeSecret(secret_name->second, e->content.secret); - - request_id_to_secret_name.erase(secret_name); - } - - } else if (auto sec_req = std::get_if>(&device_event)) { - handle_secret_request(sec_req, msg.sender); - } - - return; - } else { - failed_decryption = true; - } - } - - if (failed_decryption) { - try { - std::map> targets; - for (auto [device_id, key] : otherUserDeviceKeys.device_keys) { - if (key.keys.at("curve25519:" + device_id) == msg.sender_key) - targets[msg.sender].push_back(device_id); - } - - send_encrypted_to_device_messages( - targets, mtx::events::DeviceEvent{}, true); - nhlog::crypto()->info( - "Recovering from broken olm channel with {}:{}", msg.sender, msg.sender_key); - } catch (std::exception &e) { - nhlog::crypto()->error("Failed to recover from broken olm sessions: {}", e.what()); - } - } -} - -nlohmann::json -handle_pre_key_olm_message(const std::string &sender, - const std::string &sender_key, - const mtx::events::msg::OlmCipherContent &content) -{ - nhlog::crypto()->info("opening olm session with {}", sender); - - mtx::crypto::OlmSessionPtr inbound_session = nullptr; - try { - inbound_session = olm::client()->create_inbound_session_from(sender_key, content.body); - - // We also remove the one time key used to establish that - // session so we'll have to update our copy of the account object. - cache::saveOlmAccount(olm::client()->save(cache::client()->pickleSecret())); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to create inbound session with {}: {}", sender, e.what()); - return {}; - } - - if (!mtx::crypto::matches_inbound_session_from( - inbound_session.get(), sender_key, content.body)) { - nhlog::crypto()->warn("inbound olm session doesn't match sender's key ({})", sender); - return {}; - } - - mtx::crypto::BinaryBuf output; - try { - output = olm::client()->decrypt_message(inbound_session.get(), content.type, content.body); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to decrypt olm message {}: {}", content.body, e.what()); - return {}; - } - - auto plaintext = json::parse(std::string((char *)output.data(), output.size())); - nhlog::crypto()->debug("decrypted message: \n {}", plaintext.dump(2)); - - try { - nhlog::crypto()->debug("New olm session: {}", - mtx::crypto::session_id(inbound_session.get())); - cache::saveOlmSession( - sender_key, std::move(inbound_session), QDateTime::currentMSecsSinceEpoch()); - } catch (const lmdb::error &e) { - nhlog::db()->warn("failed to save inbound olm session from {}: {}", sender, e.what()); - } - - return plaintext; -} - -mtx::events::msg::Encrypted -encrypt_group_message(const std::string &room_id, const std::string &device_id, nlohmann::json body) -{ - using namespace mtx::events; - using namespace mtx::identifiers; - - auto own_user_id = http::client()->user_id().to_string(); - - auto members = cache::client()->getMembersWithKeys( - room_id, UserSettings::instance()->onlyShareKeysWithVerifiedUsers()); - - std::map> sendSessionTo; - mtx::crypto::OutboundGroupSessionPtr session = nullptr; - GroupSessionData group_session_data; - - if (cache::outboundMegolmSessionExists(room_id)) { - auto res = cache::getOutboundMegolmSession(room_id); - auto encryptionSettings = cache::client()->roomEncryptionSettings(room_id); - mtx::events::state::Encryption defaultSettings; - - // rotate if we crossed the limits for this key - if (res.data.message_index < - encryptionSettings.value_or(defaultSettings).rotation_period_msgs && - (QDateTime::currentMSecsSinceEpoch() - res.data.timestamp) < - encryptionSettings.value_or(defaultSettings).rotation_period_ms) { - auto member_it = members.begin(); - auto session_member_it = res.data.currently.keys.begin(); - auto session_member_it_end = res.data.currently.keys.end(); - - while (member_it != members.end() || session_member_it != session_member_it_end) { - if (member_it == members.end()) { - // a member left, purge session! - nhlog::crypto()->debug("Rotating megolm session because of left member"); - break; - } - - if (session_member_it == session_member_it_end) { - // share with all remaining members - while (member_it != members.end()) { - sendSessionTo[member_it->first] = {}; - - if (member_it->second) - for (const auto &dev : member_it->second->device_keys) - if (member_it->first != own_user_id || dev.first != device_id) - sendSessionTo[member_it->first].push_back(dev.first); - - ++member_it; - } - - session = std::move(res.session); - break; - } - - if (member_it->first > session_member_it->first) { - // a member left, purge session - nhlog::crypto()->debug("Rotating megolm session because of left member"); - break; - } else if (member_it->first < session_member_it->first) { - // new member, send them the session at this index - sendSessionTo[member_it->first] = {}; - - if (member_it->second) { - for (const auto &dev : member_it->second->device_keys) - if (member_it->first != own_user_id || dev.first != device_id) - sendSessionTo[member_it->first].push_back(dev.first); - } - - ++member_it; - } else { - // compare devices - bool device_removed = false; - for (const auto &dev : session_member_it->second.deviceids) { - if (!member_it->second || - !member_it->second->device_keys.count(dev.first)) { - device_removed = true; - break; - } - } - - if (device_removed) { - // device removed, rotate session! - nhlog::crypto()->debug("Rotating megolm session because of removed " - "device of {}", - member_it->first); - break; - } - - // check for new devices to share with - if (member_it->second) - for (const auto &dev : member_it->second->device_keys) - if (!session_member_it->second.deviceids.count(dev.first) && - (member_it->first != own_user_id || dev.first != device_id)) - sendSessionTo[member_it->first].push_back(dev.first); - - ++member_it; - ++session_member_it; - if (member_it == members.end() && session_member_it == session_member_it_end) { - // all devices match or are newly added - session = std::move(res.session); - } - } - } - } - - group_session_data = std::move(res.data); - } - - if (!session) { - nhlog::ui()->debug("creating new outbound megolm session"); - - // Create a new outbound megolm session. - session = olm::client()->init_outbound_group_session(); - const auto session_id = mtx::crypto::session_id(session.get()); - const auto session_key = mtx::crypto::session_key(session.get()); - - // Saving the new megolm session. - GroupSessionData session_data{}; - session_data.message_index = 0; - session_data.timestamp = QDateTime::currentMSecsSinceEpoch(); - session_data.sender_claimed_ed25519_key = olm::client()->identity_keys().ed25519; - - sendSessionTo.clear(); - - for (const auto &[user, devices] : members) { - sendSessionTo[user] = {}; - session_data.currently.keys[user] = {}; - if (devices) { - for (const auto &[device_id_, key] : devices->device_keys) { - (void)key; - if (device_id != device_id_ || user != own_user_id) { - sendSessionTo[user].push_back(device_id_); - session_data.currently.keys[user].deviceids[device_id_] = 0; - } - } - } - } - - { - MegolmSessionIndex index; - index.room_id = room_id; - index.session_id = session_id; - index.sender_key = olm::client()->identity_keys().curve25519; - auto megolm_session = olm::client()->init_inbound_group_session(session_key); - backup_session_key(index, session_data, megolm_session); - cache::saveInboundMegolmSession(index, std::move(megolm_session), session_data); - } - - cache::saveOutboundMegolmSession(room_id, session_data, session); - group_session_data = std::move(session_data); - } - - mtx::events::DeviceEvent megolm_payload{}; - megolm_payload.content.algorithm = MEGOLM_ALGO; - megolm_payload.content.room_id = room_id; - megolm_payload.content.session_id = mtx::crypto::session_id(session.get()); - megolm_payload.content.session_key = mtx::crypto::session_key(session.get()); - megolm_payload.type = mtx::events::EventType::RoomKey; - - if (!sendSessionTo.empty()) - olm::send_encrypted_to_device_messages(sendSessionTo, megolm_payload); - - // relations shouldn't be encrypted... - mtx::common::Relations relations = mtx::common::parse_relations(body["content"]); - - auto payload = olm::client()->encrypt_group_message(session.get(), body.dump()); - - // Prepare the m.room.encrypted event. - msg::Encrypted data; - data.ciphertext = std::string((char *)payload.data(), payload.size()); - data.sender_key = olm::client()->identity_keys().curve25519; - data.session_id = mtx::crypto::session_id(session.get()); - data.device_id = device_id; - data.algorithm = MEGOLM_ALGO; - data.relations = relations; - - group_session_data.message_index = olm_outbound_group_session_message_index(session.get()); - nhlog::crypto()->debug("next message_index {}", group_session_data.message_index); - - // update current set of members for the session with the new members and that message_index - for (const auto &[user, devices] : sendSessionTo) { - if (!group_session_data.currently.keys.count(user)) - group_session_data.currently.keys[user] = {}; - - for (const auto &device_id_ : devices) { - if (!group_session_data.currently.keys[user].deviceids.count(device_id_)) - group_session_data.currently.keys[user].deviceids[device_id_] = - group_session_data.message_index; - } - } - - // We need to re-pickle the session after we send a message to save the new message_index. - cache::updateOutboundMegolmSession(room_id, group_session_data, session); - - return data; -} - -nlohmann::json -try_olm_decryption(const std::string &sender_key, const mtx::events::msg::OlmCipherContent &msg) -{ - auto session_ids = cache::getOlmSessions(sender_key); - - nhlog::crypto()->info("attempt to decrypt message with {} known session_ids", - session_ids.size()); - - for (const auto &id : session_ids) { - auto session = cache::getOlmSession(sender_key, id); - - if (!session) { - nhlog::crypto()->warn("Unknown olm session: {}:{}", sender_key, id); - continue; - } - - mtx::crypto::BinaryBuf text; - - try { - text = olm::client()->decrypt_message(session->get(), msg.type, msg.body); - nhlog::crypto()->debug("Updated olm session: {}", - mtx::crypto::session_id(session->get())); - cache::saveOlmSession( - id, std::move(session.value()), QDateTime::currentMSecsSinceEpoch()); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->debug("failed to decrypt olm message ({}, {}) with {}: {}", - msg.type, - sender_key, - id, - e.what()); - continue; - } catch (const lmdb::error &e) { - nhlog::crypto()->critical("failed to save session: {}", e.what()); - return {}; - } - - try { - return json::parse(std::string_view((char *)text.data(), text.size())); - } catch (const json::exception &e) { - nhlog::crypto()->critical("failed to parse the decrypted session msg: {} {}", - e.what(), - std::string_view((char *)text.data(), text.size())); - } - } - - return {}; -} - -void -create_inbound_megolm_session(const mtx::events::DeviceEvent &roomKey, - const std::string &sender_key, - const std::string &sender_ed25519) -{ - MegolmSessionIndex index; - index.room_id = roomKey.content.room_id; - index.session_id = roomKey.content.session_id; - index.sender_key = sender_key; - - try { - GroupSessionData data{}; - data.forwarding_curve25519_key_chain = {sender_key}; - data.sender_claimed_ed25519_key = sender_ed25519; - - auto megolm_session = - olm::client()->init_inbound_group_session(roomKey.content.session_key); - backup_session_key(index, data, megolm_session); - cache::saveInboundMegolmSession(index, std::move(megolm_session), data); - } catch (const lmdb::error &e) { - nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); - return; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to create inbound megolm session: {}", e.what()); - return; - } - - nhlog::crypto()->info( - "established inbound megolm session ({}, {})", roomKey.content.room_id, roomKey.sender); - - ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); -} - -void -import_inbound_megolm_session( - const mtx::events::DeviceEvent &roomKey) -{ - MegolmSessionIndex index; - index.room_id = roomKey.content.room_id; - index.session_id = roomKey.content.session_id; - index.sender_key = roomKey.content.sender_key; - - try { - auto megolm_session = - olm::client()->import_inbound_group_session(roomKey.content.session_key); - - GroupSessionData data{}; - data.forwarding_curve25519_key_chain = roomKey.content.forwarding_curve25519_key_chain; - data.sender_claimed_ed25519_key = roomKey.content.sender_claimed_ed25519_key; - // may have come from online key backup, so we can't trust it... - data.trusted = false; - // if we got it forwarded from the sender, assume it is trusted. They may still have - // used key backup, but it is unlikely. - if (roomKey.content.forwarding_curve25519_key_chain.size() == 1 && - roomKey.content.forwarding_curve25519_key_chain.back() == roomKey.content.sender_key) { - data.trusted = true; - } - - backup_session_key(index, data, megolm_session); - cache::saveInboundMegolmSession(index, std::move(megolm_session), data); - } catch (const lmdb::error &e) { - nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); - return; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to import inbound megolm session: {}", e.what()); - return; - } - - nhlog::crypto()->info( - "established inbound megolm session ({}, {})", roomKey.content.room_id, roomKey.sender); - - ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); -} - -void -backup_session_key(const MegolmSessionIndex &idx, - const GroupSessionData &data, - mtx::crypto::InboundGroupSessionPtr &session) -{ - try { - if (!UserSettings::instance()->useOnlineKeyBackup()) { - // Online key backup disabled - return; - } - - auto backupVersion = cache::client()->backupVersion(); - if (!backupVersion) { - // no trusted OKB - return; - } - - using namespace mtx::crypto; - - auto decryptedSecret = cache::secret(mtx::secret_storage::secrets::megolm_backup_v1); - if (!decryptedSecret) { - // no backup key available - return; - } - auto sessionDecryptionKey = to_binary_buf(base642bin(*decryptedSecret)); - - auto public_key = mtx::crypto::CURVE25519_public_key_from_private(sessionDecryptionKey); - - mtx::responses::backup::SessionData sessionData; - sessionData.algorithm = mtx::crypto::MEGOLM_ALGO; - sessionData.forwarding_curve25519_key_chain = data.forwarding_curve25519_key_chain; - sessionData.sender_claimed_keys["ed25519"] = data.sender_claimed_ed25519_key; - sessionData.sender_key = idx.sender_key; - sessionData.session_key = mtx::crypto::export_session(session.get(), -1); - - auto encrypt_session = mtx::crypto::encrypt_session(sessionData, public_key); - - mtx::responses::backup::SessionBackup bk; - bk.first_message_index = olm_inbound_group_session_first_known_index(session.get()); - bk.forwarded_count = data.forwarding_curve25519_key_chain.size(); - bk.is_verified = false; - bk.session_data = std::move(encrypt_session); - - http::client()->put_room_keys( - backupVersion->version, - idx.room_id, - idx.session_id, - bk, - [idx](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to backup session key ({}:{}): {} ({})", - idx.room_id, - idx.session_id, - err->matrix_error.error, - static_cast(err->status_code)); - } else { - nhlog::crypto()->debug( - "backed up session key ({}:{})", idx.room_id, idx.session_id); - } - }); - } catch (std::exception &e) { - nhlog::net()->warn("failed to backup session key: {}", e.what()); - } -} - -void -mark_keys_as_published() -{ - olm::client()->mark_keys_as_published(); - cache::saveOlmAccount(olm::client()->save(cache::client()->pickleSecret())); -} - -void -lookup_keybackup(const std::string room, const std::string session_id) -{ - if (!UserSettings::instance()->useOnlineKeyBackup()) { - // Online key backup disabled - return; - } - - auto backupVersion = cache::client()->backupVersion(); - if (!backupVersion) { - // no trusted OKB - return; - } - - using namespace mtx::crypto; - - auto decryptedSecret = cache::secret(mtx::secret_storage::secrets::megolm_backup_v1); - if (!decryptedSecret) { - // no backup key available - return; - } - auto sessionDecryptionKey = to_binary_buf(base642bin(*decryptedSecret)); - - http::client()->room_keys( - backupVersion->version, - room, - session_id, - [room, session_id, sessionDecryptionKey](const mtx::responses::backup::SessionBackup &bk, - mtx::http::RequestErr err) { - if (err) { - if (err->status_code != 404) - nhlog::crypto()->error("Failed to dowload key {}:{}: {} - {}", - room, - session_id, - mtx::errors::to_string(err->matrix_error.errcode), - err->matrix_error.error); - return; - } - try { - auto session = decrypt_session(bk.session_data, sessionDecryptionKey); - - if (session.algorithm != mtx::crypto::MEGOLM_ALGO) - // don't know this algorithm - return; - - MegolmSessionIndex index; - index.room_id = room; - index.session_id = session_id; - index.sender_key = session.sender_key; - - GroupSessionData data{}; - data.forwarding_curve25519_key_chain = session.forwarding_curve25519_key_chain; - data.sender_claimed_ed25519_key = session.sender_claimed_keys["ed25519"]; - // online key backup can't be trusted, because anyone can upload to it. - data.trusted = false; - - auto megolm_session = - olm::client()->import_inbound_group_session(session.session_key); - - if (!cache::inboundMegolmSessionExists(index) || - olm_inbound_group_session_first_known_index(megolm_session.get()) < - olm_inbound_group_session_first_known_index( - cache::getInboundMegolmSession(index).get())) { - cache::saveInboundMegolmSession(index, std::move(megolm_session), data); - - nhlog::crypto()->info("imported inbound megolm session " - "from key backup ({}, {})", - room, - session_id); - - // call on UI thread - QTimer::singleShot(0, ChatPage::instance(), [index] { - ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); - }); - } - } catch (const lmdb::error &e) { - nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); - return; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to import inbound megolm session: {}", e.what()); - return; - } - }); -} - -void -send_key_request_for(mtx::events::EncryptedEvent e, - const std::string &request_id, - bool cancel) -{ - using namespace mtx::events; - - nhlog::crypto()->debug("sending key request: sender_key {}, session_id {}", - e.content.sender_key, - e.content.session_id); - - mtx::events::msg::KeyRequest request; - request.action = cancel ? mtx::events::msg::RequestAction::Cancellation - : mtx::events::msg::RequestAction::Request; - - request.algorithm = MEGOLM_ALGO; - request.room_id = e.room_id; - request.sender_key = e.content.sender_key; - request.session_id = e.content.session_id; - request.request_id = request_id; - request.requesting_device_id = http::client()->device_id(); - - nhlog::crypto()->debug("m.room_key_request: {}", json(request).dump(2)); - - std::map> body; - body[mtx::identifiers::parse(e.sender)][e.content.device_id] = request; - body[http::client()->user_id()]["*"] = request; - - http::client()->send_to_device( - http::client()->generate_txn_id(), body, [e](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to send " - "send_to_device " - "message: {}", - err->matrix_error.error); - } - - nhlog::net()->info( - "m.room_key_request sent to {}:{} and your own devices", e.sender, e.content.device_id); - }); - - // http::client()->room_keys -} - -void -handle_key_request_message(const mtx::events::DeviceEvent &req) -{ - if (req.content.algorithm != MEGOLM_ALGO) { - nhlog::crypto()->debug("ignoring key request {} with invalid algorithm: {}", - req.content.request_id, - req.content.algorithm); - return; - } - - // Check if we were the sender of the session being requested (unless it is actually us - // requesting the session). - if (req.sender != http::client()->user_id().to_string() && - req.content.sender_key != olm::client()->identity_keys().curve25519) { - nhlog::crypto()->debug( - "ignoring key request {} because we did not create the requested session: " - "\nrequested({}) ours({})", - req.content.request_id, - req.content.sender_key, - olm::client()->identity_keys().curve25519); - return; - } - - // Check that the requested session_id and the one we have saved match. - MegolmSessionIndex index{}; - index.room_id = req.content.room_id; - index.session_id = req.content.session_id; - index.sender_key = req.content.sender_key; - - // Check if we have the keys for the requested session. - auto sessionData = cache::getMegolmSessionData(index); - if (!sessionData) { - nhlog::crypto()->warn("requested session not found in room: {}", req.content.room_id); - return; - } - - const auto session = cache::getInboundMegolmSession(index); - if (!session) { - nhlog::crypto()->warn("No session with id {} in db", req.content.session_id); - return; - } - - if (!cache::isRoomMember(req.sender, req.content.room_id)) { - nhlog::crypto()->warn("user {} that requested the session key is not member of the room {}", - req.sender, - req.content.room_id); - return; - } - - // check if device is verified - auto verificationStatus = cache::verificationStatus(req.sender); - bool verifiedDevice = false; - if (verificationStatus && - // Share keys, if the option to share with trusted users is enabled or with yourself - (ChatPage::instance()->userSettings()->shareKeysWithTrustedUsers() || - req.sender == http::client()->user_id().to_string())) { - for (const auto &dev : verificationStatus->verified_devices) { - if (dev == req.content.requesting_device_id) { - verifiedDevice = true; - nhlog::crypto()->debug("Verified device: {}", dev); - break; - } - } - } - - bool shouldSeeKeys = false; - uint64_t minimumIndex = -1; - if (sessionData->currently.keys.count(req.sender)) { - if (sessionData->currently.keys.at(req.sender) - .deviceids.count(req.content.requesting_device_id)) { - shouldSeeKeys = true; - minimumIndex = sessionData->currently.keys.at(req.sender) - .deviceids.at(req.content.requesting_device_id); - } - } - - if (!verifiedDevice && !shouldSeeKeys) { - nhlog::crypto()->debug("ignoring key request for room {}", req.content.room_id); - return; - } - - if (verifiedDevice) { - // share the minimum index we have - minimumIndex = -1; - } - - try { - auto session_key = mtx::crypto::export_session(session.get(), minimumIndex); - - // - // Prepare the m.room_key event. - // - mtx::events::msg::ForwardedRoomKey forward_key{}; - forward_key.algorithm = MEGOLM_ALGO; - forward_key.room_id = index.room_id; - forward_key.session_id = index.session_id; - forward_key.session_key = session_key; - forward_key.sender_key = index.sender_key; - - // TODO(Nico): Figure out if this is correct - forward_key.sender_claimed_ed25519_key = sessionData->sender_claimed_ed25519_key; - forward_key.forwarding_curve25519_key_chain = sessionData->forwarding_curve25519_key_chain; - - send_megolm_key_to_device(req.sender, req.content.requesting_device_id, forward_key); - } catch (std::exception &e) { - nhlog::crypto()->error("Failed to forward session key: {}", e.what()); - } -} - -void -send_megolm_key_to_device(const std::string &user_id, - const std::string &device_id, - const mtx::events::msg::ForwardedRoomKey &payload) -{ - mtx::events::DeviceEvent room_key; - room_key.content = payload; - room_key.type = mtx::events::EventType::ForwardedRoomKey; - - std::map> targets; - targets[user_id] = {device_id}; - send_encrypted_to_device_messages(targets, room_key); - nhlog::crypto()->debug("Forwarded key to {}:{}", user_id, device_id); -} - -DecryptionResult -decryptEvent(const MegolmSessionIndex &index, - const mtx::events::EncryptedEvent &event, - bool dont_write_db) -{ - try { - if (!cache::client()->inboundMegolmSessionExists(index)) { - return {DecryptionErrorCode::MissingSession, std::nullopt, std::nullopt}; - } - } catch (const lmdb::error &e) { - return {DecryptionErrorCode::DbError, e.what(), std::nullopt}; - } - - // TODO: Lookup index,event_id,origin_server_ts tuple for replay attack errors - - std::string msg_str; - try { - auto session = cache::client()->getInboundMegolmSession(index); - if (!session) { - return {DecryptionErrorCode::MissingSession, std::nullopt, std::nullopt}; - } - - auto sessionData = - cache::client()->getMegolmSessionData(index).value_or(GroupSessionData{}); - - auto res = olm::client()->decrypt_group_message(session.get(), event.content.ciphertext); - msg_str = std::string((char *)res.data.data(), res.data.size()); - - if (!event.event_id.empty() && event.event_id[0] == '$') { - auto oldIdx = sessionData.indices.find(res.message_index); - if (oldIdx != sessionData.indices.end()) { - if (oldIdx->second != event.event_id) - return {DecryptionErrorCode::ReplayAttack, std::nullopt, std::nullopt}; - } else if (!dont_write_db) { - sessionData.indices[res.message_index] = event.event_id; - cache::client()->saveInboundMegolmSession(index, std::move(session), sessionData); - } - } - } catch (const lmdb::error &e) { - return {DecryptionErrorCode::DbError, e.what(), std::nullopt}; - } catch (const mtx::crypto::olm_exception &e) { - if (e.error_code() == mtx::crypto::OlmErrorCode::UNKNOWN_MESSAGE_INDEX) - return {DecryptionErrorCode::MissingSessionIndex, e.what(), std::nullopt}; - return {DecryptionErrorCode::DecryptionFailed, e.what(), std::nullopt}; - } - - try { - // Add missing fields for the event. - json body = json::parse(msg_str); - body["event_id"] = event.event_id; - body["sender"] = event.sender; - body["origin_server_ts"] = event.origin_server_ts; - body["unsigned"] = event.unsigned_data; - - // relations are unencrypted in content... - mtx::common::add_relations(body["content"], event.content.relations); - - mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json(body, te); - - return {DecryptionErrorCode::NoError, std::nullopt, std::move(te.data)}; - } catch (std::exception &e) { - return {DecryptionErrorCode::ParsingFailed, e.what(), std::nullopt}; - } -} - -crypto::Trust -calculate_trust(const std::string &user_id, const MegolmSessionIndex &index) -{ - auto status = cache::client()->verificationStatus(user_id); - auto megolmData = cache::client()->getMegolmSessionData(index); - crypto::Trust trustlevel = crypto::Trust::Unverified; - - if (megolmData && megolmData->trusted && status.verified_device_keys.count(index.sender_key)) - trustlevel = status.verified_device_keys.at(index.sender_key); - - return trustlevel; -} - -//! Send encrypted to device messages, targets is a map from userid to device ids or {} for all -//! devices -void -send_encrypted_to_device_messages(const std::map> targets, - const mtx::events::collections::DeviceEvents &event, - bool force_new_session) -{ - static QMap, qint64> rateLimit; - - nlohmann::json ev_json = std::visit([](const auto &e) { return json(e); }, event); - - std::map> keysToQuery; - mtx::requests::ClaimKeys claims; - std::map> - messages; - std::map> pks; - - auto our_curve = olm::client()->identity_keys().curve25519; - - for (const auto &[user, devices] : targets) { - auto deviceKeys = cache::client()->userKeys(user); - - // no keys for user, query them - if (!deviceKeys) { - keysToQuery[user] = devices; - continue; - } - - auto deviceTargets = devices; - if (devices.empty()) { - deviceTargets.clear(); - for (const auto &[device, keys] : deviceKeys->device_keys) { - (void)keys; - deviceTargets.push_back(device); - } - } - - for (const auto &device : deviceTargets) { - if (!deviceKeys->device_keys.count(device)) { - keysToQuery[user] = {}; - break; - } - - auto d = deviceKeys->device_keys.at(device); - - if (!d.keys.count("curve25519:" + device) || !d.keys.count("ed25519:" + device)) { - nhlog::crypto()->warn("Skipping device {} since it has no keys!", device); - continue; - } - - auto device_curve = d.keys.at("curve25519:" + device); - if (device_curve == our_curve) { - nhlog::crypto()->warn("Skipping our own device, since sending " - "ourselves olm messages makes no sense."); - continue; - } - - auto session = cache::getLatestOlmSession(device_curve); - if (!session || force_new_session) { - auto currentTime = QDateTime::currentSecsSinceEpoch(); - if (rateLimit.value(QPair(user, device)) + 60 * 60 * 10 < currentTime) { - claims.one_time_keys[user][device] = mtx::crypto::SIGNED_CURVE25519; - pks[user][device].ed25519 = d.keys.at("ed25519:" + device); - pks[user][device].curve25519 = d.keys.at("curve25519:" + device); - - rateLimit.insert(QPair(user, device), currentTime); - } else { - nhlog::crypto()->warn("Not creating new session with {}:{} " - "because of rate limit", - user, - device); - } - continue; - } - - messages[mtx::identifiers::parse(user)][device] = - olm::client() - ->create_olm_encrypted_content(session->get(), - ev_json, - UserId(user), - d.keys.at("ed25519:" + device), - device_curve) - .get(); - - try { - nhlog::crypto()->debug("Updated olm session: {}", - mtx::crypto::session_id(session->get())); - cache::saveOlmSession(d.keys.at("curve25519:" + device), - std::move(*session), - QDateTime::currentMSecsSinceEpoch()); - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to save outbound olm session: {}", e.what()); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to pickle outbound olm session: {}", e.what()); - } - } - } - - if (!messages.empty()) - http::client()->send_to_device( - http::client()->generate_txn_id(), messages, [](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to send " - "send_to_device " - "message: {}", - err->matrix_error.error); - } - }); - - auto BindPks = [ev_json](decltype(pks) pks_temp) { - return [pks = pks_temp, ev_json](const mtx::responses::ClaimKeys &res, - mtx::http::RequestErr) { - std::map> - messages; - for (const auto &[user_id, retrieved_devices] : res.one_time_keys) { - nhlog::net()->debug("claimed keys for {}", user_id); - if (retrieved_devices.size() == 0) { - nhlog::net()->debug("no one-time keys found for user_id: {}", user_id); - continue; - } - - for (const auto &rd : retrieved_devices) { - const auto device_id = rd.first; - - nhlog::net()->debug("{} : \n {}", device_id, rd.second.dump(2)); - - if (rd.second.empty() || !rd.second.begin()->contains("key")) { - nhlog::net()->warn("Skipping device {} as it has no key.", device_id); - continue; - } - - auto otk = rd.second.begin()->at("key"); - - auto sign_key = pks.at(user_id).at(device_id).ed25519; - auto id_key = pks.at(user_id).at(device_id).curve25519; - - // Verify signature - { - auto signedKey = *rd.second.begin(); - std::string signature = - signedKey["signatures"][user_id].value("ed25519:" + device_id, ""); - - if (signature.empty() || !mtx::crypto::ed25519_verify_signature( - sign_key, signedKey, signature)) { - nhlog::net()->warn("Skipping device {} as its one time key " - "has an invalid signature.", - device_id); - continue; - } - } - - auto session = olm::client()->create_outbound_session(id_key, otk); - - messages[mtx::identifiers::parse(user_id)][device_id] = - olm::client() - ->create_olm_encrypted_content( - session.get(), ev_json, UserId(user_id), sign_key, id_key) - .get(); - - try { - nhlog::crypto()->debug("Updated olm session: {}", - mtx::crypto::session_id(session.get())); - cache::saveOlmSession( - id_key, std::move(session), QDateTime::currentMSecsSinceEpoch()); - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to save outbound olm session: {}", e.what()); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to pickle outbound olm session: {}", - e.what()); - } - } - nhlog::net()->info("send_to_device: {}", user_id); - } - - if (!messages.empty()) - http::client()->send_to_device( - http::client()->generate_txn_id(), messages, [](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to send " - "send_to_device " - "message: {}", - err->matrix_error.error); - } - }); - }; - }; - - if (!claims.one_time_keys.empty()) - http::client()->claim_keys(claims, BindPks(pks)); - - if (!keysToQuery.empty()) { - mtx::requests::QueryKeys req; - req.device_keys = keysToQuery; - http::client()->query_keys( - req, - [ev_json, BindPks, our_curve](const mtx::responses::QueryKeys &res, - mtx::http::RequestErr err) { - if (err) { - nhlog::net()->warn("failed to query device keys: {} {}", - err->matrix_error.error, - static_cast(err->status_code)); - return; - } - - nhlog::net()->info("queried keys"); - - cache::client()->updateUserKeys(cache::nextBatchToken(), res); - - mtx::requests::ClaimKeys claim_keys; - - std::map> deviceKeys; - - for (const auto &user : res.device_keys) { - for (const auto &dev : user.second) { - const auto user_id = ::UserId(dev.second.user_id); - const auto device_id = DeviceId(dev.second.device_id); - - if (user_id.get() == http::client()->user_id().to_string() && - device_id.get() == http::client()->device_id()) - continue; - - const auto device_keys = dev.second.keys; - const auto curveKey = "curve25519:" + device_id.get(); - const auto edKey = "ed25519:" + device_id.get(); - - if ((device_keys.find(curveKey) == device_keys.end()) || - (device_keys.find(edKey) == device_keys.end())) { - nhlog::net()->debug("ignoring malformed keys for device {}", - device_id.get()); - continue; - } - - DevicePublicKeys pks; - pks.ed25519 = device_keys.at(edKey); - pks.curve25519 = device_keys.at(curveKey); - - if (pks.curve25519 == our_curve) { - nhlog::crypto()->warn("Skipping our own device, since sending " - "ourselves olm messages makes no sense."); - continue; - } - - try { - if (!mtx::crypto::verify_identity_signature( - dev.second, device_id, user_id)) { - nhlog::crypto()->warn("failed to verify identity keys: {}", - json(dev.second).dump(2)); - continue; - } - } catch (const json::exception &e) { - nhlog::crypto()->warn("failed to parse device key json: {}", e.what()); - continue; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->warn("failed to verify device key json: {}", e.what()); - continue; - } - - auto currentTime = QDateTime::currentSecsSinceEpoch(); - if (rateLimit.value(QPair(user.first, device_id.get())) + 60 * 60 * 10 < - currentTime) { - deviceKeys[user_id].emplace(device_id, pks); - claim_keys.one_time_keys[user.first][device_id] = - mtx::crypto::SIGNED_CURVE25519; - - rateLimit.insert(QPair(user.first, device_id.get()), currentTime); - } else { - nhlog::crypto()->warn("Not creating new session with {}:{} " - "because of rate limit", - user.first, - device_id.get()); - continue; - } - - nhlog::net()->info("{}", device_id.get()); - nhlog::net()->info(" curve25519 {}", pks.curve25519); - nhlog::net()->info(" ed25519 {}", pks.ed25519); - } - } - - if (!claim_keys.one_time_keys.empty()) - http::client()->claim_keys(claim_keys, BindPks(deviceKeys)); - }); - } -} - -void -request_cross_signing_keys() -{ - mtx::events::msg::SecretRequest secretRequest{}; - secretRequest.action = mtx::events::msg::RequestAction::Request; - secretRequest.requesting_device_id = http::client()->device_id(); - - auto local_user = http::client()->user_id(); - - auto verificationStatus = cache::verificationStatus(local_user.to_string()); - - if (!verificationStatus) - return; - - auto request = [&](std::string secretName) { - secretRequest.name = secretName; - secretRequest.request_id = "ss." + http::client()->generate_txn_id(); - - request_id_to_secret_name[secretRequest.request_id] = secretRequest.name; - - std::map> - body; - - for (const auto &dev : verificationStatus->verified_devices) { - if (dev != secretRequest.requesting_device_id) - body[local_user][dev] = secretRequest; - } - - http::client()->send_to_device( - http::client()->generate_txn_id(), - body, - [request_id = secretRequest.request_id, secretName](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("Failed to send request for secrect '{}'", secretName); - // Cancel request on UI thread - QTimer::singleShot(1, cache::client(), [request_id]() { - request_id_to_secret_name.erase(request_id); - }); - return; - } - }); - - for (const auto &dev : verificationStatus->verified_devices) { - if (dev != secretRequest.requesting_device_id) - body[local_user][dev].action = mtx::events::msg::RequestAction::Cancellation; - } - - // timeout after 15 min - QTimer::singleShot(15 * 60 * 1000, [secretRequest, body]() { - if (request_id_to_secret_name.count(secretRequest.request_id)) { - request_id_to_secret_name.erase(secretRequest.request_id); - http::client()->send_to_device( - http::client()->generate_txn_id(), - body, - [secretRequest](mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("Failed to cancel request for secrect '{}'", - secretRequest.name); - return; - } - }); - } - }); - }; - - request(mtx::secret_storage::secrets::cross_signing_self_signing); - request(mtx::secret_storage::secrets::cross_signing_user_signing); - request(mtx::secret_storage::secrets::megolm_backup_v1); -} - -namespace { -void -unlock_secrets(const std::string &key, - const std::map &secrets) -{ - http::client()->secret_storage_key( - key, - [secrets](mtx::secret_storage::AesHmacSha2KeyDescription keyDesc, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("Failed to download secret storage key"); - return; - } - - emit ChatPage::instance()->downloadedSecrets(keyDesc, secrets); - }); -} -} - -void -download_cross_signing_keys() -{ - using namespace mtx::secret_storage; - http::client()->secret_storage_secret( - secrets::megolm_backup_v1, [](Secret secret, mtx::http::RequestErr err) { - std::optional backup_key; - if (!err) - backup_key = secret; - - http::client()->secret_storage_secret( - secrets::cross_signing_self_signing, - [backup_key](Secret secret, mtx::http::RequestErr err) { - std::optional self_signing_key; - if (!err) - self_signing_key = secret; - - http::client()->secret_storage_secret( - secrets::cross_signing_user_signing, - [backup_key, self_signing_key](Secret secret, mtx::http::RequestErr err) { - std::optional user_signing_key; - if (!err) - user_signing_key = secret; - - std::map> - secrets; - - if (backup_key && !backup_key->encrypted.empty()) - secrets[backup_key->encrypted.begin()->first][secrets::megolm_backup_v1] = - backup_key->encrypted.begin()->second; - if (self_signing_key && !self_signing_key->encrypted.empty()) - secrets[self_signing_key->encrypted.begin()->first] - [secrets::cross_signing_self_signing] = - self_signing_key->encrypted.begin()->second; - if (user_signing_key && !user_signing_key->encrypted.empty()) - secrets[user_signing_key->encrypted.begin()->first] - [secrets::cross_signing_user_signing] = - user_signing_key->encrypted.begin()->second; - - for (const auto &[key, secrets] : secrets) - unlock_secrets(key, secrets); - }); - }); - }); -} - -} // namespace olm diff --git a/src/Olm.h b/src/Olm.h deleted file mode 100644 index 44e2b8ed..00000000 --- a/src/Olm.h +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include -#include -#include -#include - -#include - -constexpr auto OLM_ALGO = "m.olm.v1.curve25519-aes-sha2"; - -namespace olm { -Q_NAMESPACE - -enum DecryptionErrorCode -{ - NoError, - MissingSession, // Session was not found, retrieve from backup or request from other devices - // and try again - MissingSessionIndex, // Session was found, but it does not reach back enough to this index, - // retrieve from backup or request from other devices and try again - DbError, // DB read failed - DecryptionFailed, // libolm error - ParsingFailed, // Failed to parse the actual event - ReplayAttack, // Megolm index reused -}; -Q_ENUM_NS(DecryptionErrorCode) - -struct DecryptionResult -{ - DecryptionErrorCode error; - std::optional error_message; - std::optional event; -}; - -struct OlmMessage -{ - std::string sender_key; - std::string sender; - - using RecipientKey = std::string; - std::map ciphertext; -}; - -void -from_json(const nlohmann::json &obj, OlmMessage &msg); - -mtx::crypto::OlmClient * -client(); - -void -handle_to_device_messages(const std::vector &msgs); - -nlohmann::json -try_olm_decryption(const std::string &sender_key, - const mtx::events::msg::OlmCipherContent &content); - -void -handle_olm_message(const OlmMessage &msg, const UserKeyCache &otherUserDeviceKeys); - -//! Establish a new inbound megolm session with the decrypted payload from olm. -void -create_inbound_megolm_session(const mtx::events::DeviceEvent &roomKey, - const std::string &sender_key, - const std::string &sender_ed25519); -void -import_inbound_megolm_session( - const mtx::events::DeviceEvent &roomKey); -void -lookup_keybackup(const std::string room, const std::string session_id); - -nlohmann::json -handle_pre_key_olm_message(const std::string &sender, - const std::string &sender_key, - const mtx::events::msg::OlmCipherContent &content); - -mtx::events::msg::Encrypted -encrypt_group_message(const std::string &room_id, - const std::string &device_id, - nlohmann::json body); - -//! Decrypt an event. Use dont_write_db to prevent db writes when already in a write transaction. -DecryptionResult -decryptEvent(const MegolmSessionIndex &index, - const mtx::events::EncryptedEvent &event, - bool dont_write_db = false); -crypto::Trust -calculate_trust(const std::string &user_id, const MegolmSessionIndex &index); - -void -mark_keys_as_published(); - -//! Request the encryption keys from sender's device for the given event. -void -send_key_request_for(mtx::events::EncryptedEvent e, - const std::string &request_id, - bool cancel = false); - -void -handle_key_request_message(const mtx::events::DeviceEvent &); - -void -send_megolm_key_to_device(const std::string &user_id, - const std::string &device_id, - const mtx::events::msg::ForwardedRoomKey &payload); - -//! Send encrypted to device messages, targets is a map from userid to device ids or {} for all -//! devices -void -send_encrypted_to_device_messages(const std::map> targets, - const mtx::events::collections::DeviceEvents &event, - bool force_new_session = false); - -//! Request backup and signing keys and cache them locally -void -request_cross_signing_keys(); -//! Download backup and signing keys and cache them locally -void -download_cross_signing_keys(); - -} // namespace olm diff --git a/src/SelfVerificationStatus.cpp b/src/SelfVerificationStatus.cpp deleted file mode 100644 index d75a2109..00000000 --- a/src/SelfVerificationStatus.cpp +++ /dev/null @@ -1,249 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "SelfVerificationStatus.h" - -#include "Cache_p.h" -#include "Logging.h" -#include "MainWindow.h" -#include "MatrixClient.h" -#include "Olm.h" -#include "ui/UIA.h" - -#include - -SelfVerificationStatus::SelfVerificationStatus(QObject *o) - : QObject(o) -{ - connect(MainWindow::instance(), &MainWindow::reload, this, [this] { - connect(cache::client(), - &Cache::selfUnverified, - this, - &SelfVerificationStatus::invalidate, - Qt::UniqueConnection); - invalidate(); - }); -} - -void -SelfVerificationStatus::setupCrosssigning(bool useSSSS, QString password, bool useOnlineKeyBackup) -{ - nhlog::db()->info("Clicked setup crossigning"); - - auto xsign_keys = olm::client()->create_crosssigning_keys(); - - if (!xsign_keys) { - nhlog::crypto()->critical("Failed to setup cross-signing keys!"); - emit setupFailed(tr("Failed to create keys for cross-signing!")); - return; - } - - cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_master, - xsign_keys->private_master_key); - cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_self_signing, - xsign_keys->private_self_signing_key); - cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_user_signing, - xsign_keys->private_user_signing_key); - - std::optional okb; - if (useOnlineKeyBackup) { - okb = olm::client()->create_online_key_backup(xsign_keys->private_master_key); - if (!okb) { - nhlog::crypto()->critical("Failed to setup online key backup!"); - emit setupFailed(tr("Failed to create keys for online key backup!")); - return; - } - - cache::client()->storeSecret( - mtx::secret_storage::secrets::megolm_backup_v1, - mtx::crypto::bin2base64(mtx::crypto::to_string(okb->privateKey))); - - http::client()->post_backup_version( - okb->backupVersion.algorithm, - okb->backupVersion.auth_data, - [](const mtx::responses::Version &v, mtx::http::RequestErr e) { - if (e) { - nhlog::net()->error("error setting up online key backup: {} {} {} {}", - e->parse_error, - e->status_code, - e->error_code, - e->matrix_error.error); - } else { - nhlog::crypto()->info("Set up online key backup: '{}'", v.version); - } - }); - } - - std::optional ssss; - if (useSSSS) { - ssss = olm::client()->create_ssss_key(password.toStdString()); - if (!ssss) { - nhlog::crypto()->critical("Failed to setup secure server side secret storage!"); - emit setupFailed(tr("Failed to create keys secure server side secret storage!")); - return; - } - - auto master = mtx::crypto::PkSigning::from_seed(xsign_keys->private_master_key); - nlohmann::json j = ssss->keyDescription; - j.erase("signatures"); - ssss->keyDescription - .signatures[http::client()->user_id().to_string()]["ed25519:" + master.public_key()] = - master.sign(j.dump()); - - http::client()->upload_secret_storage_key( - ssss->keyDescription.name, ssss->keyDescription, [](mtx::http::RequestErr) {}); - http::client()->set_secret_storage_default_key(ssss->keyDescription.name, - [](mtx::http::RequestErr) {}); - - auto uploadSecret = [ssss](const std::string &key_name, const std::string &secret) { - mtx::secret_storage::Secret s; - s.encrypted[ssss->keyDescription.name] = - mtx::crypto::encrypt(secret, ssss->privateKey, key_name); - http::client()->upload_secret_storage_secret( - key_name, s, [key_name](mtx::http::RequestErr) { - nhlog::crypto()->info("Uploaded secret: {}", key_name); - }); - }; - - uploadSecret(mtx::secret_storage::secrets::cross_signing_master, - xsign_keys->private_master_key); - uploadSecret(mtx::secret_storage::secrets::cross_signing_self_signing, - xsign_keys->private_self_signing_key); - uploadSecret(mtx::secret_storage::secrets::cross_signing_user_signing, - xsign_keys->private_user_signing_key); - - if (okb) - uploadSecret(mtx::secret_storage::secrets::megolm_backup_v1, - mtx::crypto::bin2base64(mtx::crypto::to_string(okb->privateKey))); - } - - mtx::requests::DeviceSigningUpload device_sign{}; - device_sign.master_key = xsign_keys->master_key; - device_sign.self_signing_key = xsign_keys->self_signing_key; - device_sign.user_signing_key = xsign_keys->user_signing_key; - http::client()->device_signing_upload( - device_sign, - UIA::instance()->genericHandler(tr("Encryption Setup")), - [this, ssss, xsign_keys](mtx::http::RequestErr e) { - if (e) { - nhlog::crypto()->critical("Failed to upload cross signing keys: {}", - e->matrix_error.error); - - emit setupFailed(tr("Encryption setup failed: %1") - .arg(QString::fromStdString(e->matrix_error.error))); - return; - } - nhlog::crypto()->info("Crosssigning keys uploaded!"); - - auto deviceKeys = cache::client()->userKeys(http::client()->user_id().to_string()); - if (deviceKeys) { - auto myKey = deviceKeys->device_keys.at(http::client()->device_id()); - if (myKey.user_id == http::client()->user_id().to_string() && - myKey.device_id == http::client()->device_id() && - myKey.keys["ed25519:" + http::client()->device_id()] == - olm::client()->identity_keys().ed25519 && - myKey.keys["curve25519:" + http::client()->device_id()] == - olm::client()->identity_keys().curve25519) { - json j = myKey; - j.erase("signatures"); - j.erase("unsigned"); - - auto ssk = - mtx::crypto::PkSigning::from_seed(xsign_keys->private_self_signing_key); - myKey.signatures[http::client()->user_id().to_string()] - ["ed25519:" + ssk.public_key()] = ssk.sign(j.dump()); - mtx::requests::KeySignaturesUpload req; - req.signatures[http::client()->user_id().to_string()] - [http::client()->device_id()] = myKey; - - http::client()->keys_signatures_upload( - req, - [](const mtx::responses::KeySignaturesUpload &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("failed to upload signatures: {},{}", - mtx::errors::to_string(err->matrix_error.errcode), - static_cast(err->status_code)); - } - - for (const auto &[user_id, tmp] : res.errors) - for (const auto &[key_id, e] : tmp) - nhlog::net()->error("signature error for user {} and key " - "id {}: {}, {}", - user_id, - key_id, - mtx::errors::to_string(e.errcode), - e.error); - }); - } - } - - if (ssss) { - auto k = QString::fromStdString(mtx::crypto::key_to_recoverykey(ssss->privateKey)); - - QString r; - for (int i = 0; i < k.size(); i += 4) - r += k.mid(i, 4) + " "; - - emit showRecoveryKey(r.trimmed()); - } else { - emit setupCompleted(); - } - }); -} - -void -SelfVerificationStatus::verifyMasterKey() -{ - nhlog::db()->info("Clicked verify master key"); -} - -void -SelfVerificationStatus::verifyUnverifiedDevices() -{ - nhlog::db()->info("Clicked verify unverified devices"); -} - -void -SelfVerificationStatus::invalidate() -{ - nhlog::db()->info("Invalidating self verification status"); - auto keys = cache::client()->userKeys(http::client()->user_id().to_string()); - if (!keys) { - cache::client()->query_keys(http::client()->user_id().to_string(), - [](const UserKeyCache &, mtx::http::RequestErr) {}); - return; - } - - if (keys->master_keys.keys.empty()) { - if (status_ != SelfVerificationStatus::NoMasterKey) { - this->status_ = SelfVerificationStatus::NoMasterKey; - emit statusChanged(); - } - return; - } - - auto verifStatus = cache::client()->verificationStatus(http::client()->user_id().to_string()); - - if (!verifStatus.user_verified) { - if (status_ != SelfVerificationStatus::UnverifiedMasterKey) { - this->status_ = SelfVerificationStatus::UnverifiedMasterKey; - emit statusChanged(); - } - return; - } - - if (verifStatus.unverified_device_count > 0) { - if (status_ != SelfVerificationStatus::UnverifiedDevices) { - this->status_ = SelfVerificationStatus::UnverifiedDevices; - emit statusChanged(); - } - return; - } - - if (status_ != SelfVerificationStatus::AllVerified) { - this->status_ = SelfVerificationStatus::AllVerified; - emit statusChanged(); - return; - } -} diff --git a/src/SelfVerificationStatus.h b/src/SelfVerificationStatus.h deleted file mode 100644 index 8cb54df6..00000000 --- a/src/SelfVerificationStatus.h +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include - -class SelfVerificationStatus : public QObject -{ - Q_OBJECT - - Q_PROPERTY(Status status READ status NOTIFY statusChanged) - -public: - SelfVerificationStatus(QObject *o = nullptr); - enum Status - { - AllVerified, - NoMasterKey, - UnverifiedMasterKey, - UnverifiedDevices, - }; - Q_ENUM(Status) - - Q_INVOKABLE void setupCrosssigning(bool useSSSS, QString password, bool useOnlineKeyBackup); - Q_INVOKABLE void verifyMasterKey(); - Q_INVOKABLE void verifyUnverifiedDevices(); - - Status status() const { return status_; } - -signals: - void statusChanged(); - void setupCompleted(); - void showRecoveryKey(QString key); - void setupFailed(QString message); - -public slots: - void invalidate(); - -private: - Status status_ = AllVerified; -}; diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index cc1f8206..340709a6 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -26,14 +25,14 @@ #include #include "Cache.h" -#include "CallDevices.h" #include "Config.h" #include "MatrixClient.h" -#include "Olm.h" #include "UserSettingsPage.h" #include "Utils.h" +#include "encryption/Olm.h" #include "ui/FlatButton.h" #include "ui/ToggleButton.h" +#include "voip/CallDevices.h" #include "config/nheko.h" diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp deleted file mode 100644 index 801a365c..00000000 --- a/src/WebRTCSession.cpp +++ /dev/null @@ -1,1155 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "CallDevices.h" -#include "ChatPage.h" -#include "Logging.h" -#include "UserSettingsPage.h" -#include "WebRTCSession.h" - -#ifdef GSTREAMER_AVAILABLE -extern "C" -{ -#include "gst/gst.h" -#include "gst/sdp/sdp.h" - -#define GST_USE_UNSTABLE_API -#include "gst/webrtc/webrtc.h" -} -#endif - -// https://github.com/vector-im/riot-web/issues/10173 -#define STUN_SERVER "stun://turn.matrix.org:3478" - -Q_DECLARE_METATYPE(webrtc::CallType) -Q_DECLARE_METATYPE(webrtc::State) - -using webrtc::CallType; -using webrtc::State; - -WebRTCSession::WebRTCSession() - : devices_(CallDevices::instance()) -{ - qRegisterMetaType(); - qmlRegisterUncreatableMetaObject( - webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum"); - - qRegisterMetaType(); - qmlRegisterUncreatableMetaObject( - webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum"); - - connect(this, &WebRTCSession::stateChanged, this, &WebRTCSession::setState); - init(); -} - -bool -WebRTCSession::init(std::string *errorMessage) -{ -#ifdef GSTREAMER_AVAILABLE - if (initialised_) - return true; - - GError *error = nullptr; - if (!gst_init_check(nullptr, nullptr, &error)) { - std::string strError("WebRTC: failed to initialise GStreamer: "); - if (error) { - strError += error->message; - g_error_free(error); - } - nhlog::ui()->error(strError); - if (errorMessage) - *errorMessage = strError; - return false; - } - - initialised_ = true; - gchar *version = gst_version_string(); - nhlog::ui()->info("WebRTC: initialised {}", version); - g_free(version); - devices_.init(); - return true; -#else - (void)errorMessage; - return false; -#endif -} - -#ifdef GSTREAMER_AVAILABLE -namespace { - -std::string localsdp_; -std::vector localcandidates_; -bool haveAudioStream_ = false; -bool haveVideoStream_ = false; -GstPad *localPiPSinkPad_ = nullptr; -GstPad *remotePiPSinkPad_ = nullptr; - -gboolean -newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data) -{ - WebRTCSession *session = static_cast(user_data); - switch (GST_MESSAGE_TYPE(msg)) { - case GST_MESSAGE_EOS: - nhlog::ui()->error("WebRTC: end of stream"); - session->end(); - break; - case GST_MESSAGE_ERROR: - GError *error; - gchar *debug; - gst_message_parse_error(msg, &error, &debug); - nhlog::ui()->error( - "WebRTC: error from element {}: {}", GST_OBJECT_NAME(msg->src), error->message); - g_clear_error(&error); - g_free(debug); - session->end(); - break; - default: - break; - } - return TRUE; -} - -GstWebRTCSessionDescription * -parseSDP(const std::string &sdp, GstWebRTCSDPType type) -{ - GstSDPMessage *msg; - gst_sdp_message_new(&msg); - if (gst_sdp_message_parse_buffer((guint8 *)sdp.c_str(), sdp.size(), msg) == GST_SDP_OK) { - return gst_webrtc_session_description_new(type, msg); - } else { - nhlog::ui()->error("WebRTC: failed to parse remote session description"); - gst_sdp_message_free(msg); - return nullptr; - } -} - -void -setLocalDescription(GstPromise *promise, gpointer webrtc) -{ - const GstStructure *reply = gst_promise_get_reply(promise); - gboolean isAnswer = gst_structure_id_has_field(reply, g_quark_from_string("answer")); - GstWebRTCSessionDescription *gstsdp = nullptr; - gst_structure_get( - reply, isAnswer ? "answer" : "offer", GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &gstsdp, nullptr); - gst_promise_unref(promise); - g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr); - - gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp); - localsdp_ = std::string(sdp); - g_free(sdp); - gst_webrtc_session_description_free(gstsdp); - - nhlog::ui()->debug( - "WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_); -} - -void -createOffer(GstElement *webrtc) -{ - // create-offer first, then set-local-description - GstPromise *promise = gst_promise_new_with_change_func(setLocalDescription, webrtc, nullptr); - g_signal_emit_by_name(webrtc, "create-offer", nullptr, promise); -} - -void -createAnswer(GstPromise *promise, gpointer webrtc) -{ - // create-answer first, then set-local-description - gst_promise_unref(promise); - promise = gst_promise_new_with_change_func(setLocalDescription, webrtc, nullptr); - g_signal_emit_by_name(webrtc, "create-answer", nullptr, promise); -} - -void -iceGatheringStateChanged(GstElement *webrtc, - GParamSpec *pspec G_GNUC_UNUSED, - gpointer user_data G_GNUC_UNUSED) -{ - GstWebRTCICEGatheringState newState; - g_object_get(webrtc, "ice-gathering-state", &newState, nullptr); - if (newState == GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE) { - nhlog::ui()->debug("WebRTC: GstWebRTCICEGatheringState -> Complete"); - if (WebRTCSession::instance().isOffering()) { - emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_); - emit WebRTCSession::instance().stateChanged(State::OFFERSENT); - } else { - emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_); - emit WebRTCSession::instance().stateChanged(State::ANSWERSENT); - } - } -} - -void -addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, - guint mlineIndex, - gchar *candidate, - gpointer G_GNUC_UNUSED) -{ - nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate); - localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); -} - -void -iceConnectionStateChanged(GstElement *webrtc, - GParamSpec *pspec G_GNUC_UNUSED, - gpointer user_data G_GNUC_UNUSED) -{ - GstWebRTCICEConnectionState newState; - g_object_get(webrtc, "ice-connection-state", &newState, nullptr); - switch (newState) { - case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING: - nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking"); - emit WebRTCSession::instance().stateChanged(State::CONNECTING); - break; - case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED: - nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed"); - emit WebRTCSession::instance().stateChanged(State::ICEFAILED); - break; - default: - break; - } -} - -// https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1164 -struct KeyFrameRequestData -{ - GstElement *pipe = nullptr; - GstElement *decodebin = nullptr; - gint packetsLost = 0; - guint timerid = 0; - std::string statsField; -} keyFrameRequestData_; - -void -sendKeyFrameRequest() -{ - GstPad *sinkpad = gst_element_get_static_pad(keyFrameRequestData_.decodebin, "sink"); - if (!gst_pad_push_event(sinkpad, - gst_event_new_custom(GST_EVENT_CUSTOM_UPSTREAM, - gst_structure_new_empty("GstForceKeyUnit")))) - nhlog::ui()->error("WebRTC: key frame request failed"); - else - nhlog::ui()->debug("WebRTC: sent key frame request"); - - gst_object_unref(sinkpad); -} - -void -testPacketLoss_(GstPromise *promise, gpointer G_GNUC_UNUSED) -{ - const GstStructure *reply = gst_promise_get_reply(promise); - gint packetsLost = 0; - GstStructure *rtpStats; - if (!gst_structure_get( - reply, keyFrameRequestData_.statsField.c_str(), GST_TYPE_STRUCTURE, &rtpStats, nullptr)) { - nhlog::ui()->error("WebRTC: get-stats: no field: {}", keyFrameRequestData_.statsField); - gst_promise_unref(promise); - return; - } - gst_structure_get_int(rtpStats, "packets-lost", &packetsLost); - gst_structure_free(rtpStats); - gst_promise_unref(promise); - if (packetsLost > keyFrameRequestData_.packetsLost) { - nhlog::ui()->debug("WebRTC: inbound video lost packet count: {}", packetsLost); - keyFrameRequestData_.packetsLost = packetsLost; - sendKeyFrameRequest(); - } -} - -gboolean -testPacketLoss(gpointer G_GNUC_UNUSED) -{ - if (keyFrameRequestData_.pipe) { - GstElement *webrtc = gst_bin_get_by_name(GST_BIN(keyFrameRequestData_.pipe), "webrtcbin"); - GstPromise *promise = gst_promise_new_with_change_func(testPacketLoss_, nullptr, nullptr); - g_signal_emit_by_name(webrtc, "get-stats", nullptr, promise); - gst_object_unref(webrtc); - return TRUE; - } - return FALSE; -} - -void -setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointer G_GNUC_UNUSED) -{ - if (!std::strcmp( - gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(gst_element_get_factory(element))), - "rtpvp8depay")) - g_object_set(element, "wait-for-keyframe", TRUE, nullptr); -} - -GstElement * -newAudioSinkChain(GstElement *pipe) -{ - GstElement *queue = gst_element_factory_make("queue", nullptr); - GstElement *convert = gst_element_factory_make("audioconvert", nullptr); - GstElement *resample = gst_element_factory_make("audioresample", nullptr); - GstElement *sink = gst_element_factory_make("autoaudiosink", nullptr); - gst_bin_add_many(GST_BIN(pipe), queue, convert, resample, sink, nullptr); - gst_element_link_many(queue, convert, resample, sink, nullptr); - gst_element_sync_state_with_parent(queue); - gst_element_sync_state_with_parent(convert); - gst_element_sync_state_with_parent(resample); - gst_element_sync_state_with_parent(sink); - return queue; -} - -GstElement * -newVideoSinkChain(GstElement *pipe) -{ - // use compositor for now; acceleration needs investigation - GstElement *queue = gst_element_factory_make("queue", nullptr); - GstElement *compositor = gst_element_factory_make("compositor", "compositor"); - GstElement *glupload = gst_element_factory_make("glupload", nullptr); - GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr); - GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); - GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); - g_object_set(compositor, "background", 1, nullptr); - g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); - g_object_set(glsinkbin, "sink", qmlglsink, nullptr); - gst_bin_add_many( - GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); - gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); - gst_element_sync_state_with_parent(queue); - gst_element_sync_state_with_parent(compositor); - gst_element_sync_state_with_parent(glupload); - gst_element_sync_state_with_parent(glcolorconvert); - gst_element_sync_state_with_parent(glsinkbin); - return queue; -} - -std::pair -getResolution(GstPad *pad) -{ - std::pair ret; - GstCaps *caps = gst_pad_get_current_caps(pad); - const GstStructure *s = gst_caps_get_structure(caps, 0); - gst_structure_get_int(s, "width", &ret.first); - gst_structure_get_int(s, "height", &ret.second); - gst_caps_unref(caps); - return ret; -} - -std::pair -getResolution(GstElement *pipe, const gchar *elementName, const gchar *padName) -{ - GstElement *element = gst_bin_get_by_name(GST_BIN(pipe), elementName); - GstPad *pad = gst_element_get_static_pad(element, padName); - auto ret = getResolution(pad); - gst_object_unref(pad); - gst_object_unref(element); - return ret; -} - -std::pair -getPiPDimensions(const std::pair &resolution, int fullWidth, double scaleFactor) -{ - int pipWidth = fullWidth * scaleFactor; - int pipHeight = static_cast(resolution.second) / resolution.first * pipWidth; - return {pipWidth, pipHeight}; -} - -void -addLocalPiP(GstElement *pipe, const std::pair &videoCallSize) -{ - // embed localUser's camera into received video (CallType::VIDEO) - // OR embed screen share into received video (CallType::SCREEN) - GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); - if (!tee) - return; - - GstElement *queue = gst_element_factory_make("queue", nullptr); - gst_bin_add(GST_BIN(pipe), queue); - gst_element_link(tee, queue); - gst_element_sync_state_with_parent(queue); - gst_object_unref(tee); - - GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor"); - localPiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); - g_object_set(localPiPSinkPad_, "zorder", 2, nullptr); - - bool isVideo = WebRTCSession::instance().callType() == CallType::VIDEO; - const gchar *element = isVideo ? "camerafilter" : "screenshare"; - const gchar *pad = isVideo ? "sink" : "src"; - auto resolution = getResolution(pipe, element, pad); - auto pipSize = getPiPDimensions(resolution, videoCallSize.first, 0.25); - nhlog::ui()->debug("WebRTC: local picture-in-picture: {}x{}", pipSize.first, pipSize.second); - g_object_set(localPiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); - gint offset = videoCallSize.first / 80; - g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr); - - GstPad *srcpad = gst_element_get_static_pad(queue, "src"); - if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, localPiPSinkPad_))) - nhlog::ui()->error("WebRTC: failed to link local PiP elements"); - gst_object_unref(srcpad); - gst_object_unref(compositor); -} - -void -addRemotePiP(GstElement *pipe) -{ - // embed localUser's camera into screen image being shared - if (remotePiPSinkPad_) { - auto camRes = getResolution(pipe, "camerafilter", "sink"); - auto shareRes = getResolution(pipe, "screenshare", "src"); - auto pipSize = getPiPDimensions(camRes, shareRes.first, 0.2); - nhlog::ui()->debug( - "WebRTC: screen share picture-in-picture: {}x{}", pipSize.first, pipSize.second); - - gint offset = shareRes.first / 100; - g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr); - g_object_set(remotePiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); - g_object_set(remotePiPSinkPad_, - "xpos", - shareRes.first - pipSize.first - offset, - "ypos", - shareRes.second - pipSize.second - offset, - nullptr); - } -} - -void -addLocalVideo(GstElement *pipe) -{ - GstElement *queue = newVideoSinkChain(pipe); - GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); - GstPad *srcpad = gst_element_get_request_pad(tee, "src_%u"); - GstPad *sinkpad = gst_element_get_static_pad(queue, "sink"); - if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, sinkpad))) - nhlog::ui()->error("WebRTC: failed to link videosrctee -> video sink chain"); - gst_object_unref(srcpad); -} - -void -linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) -{ - GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); - GstCaps *sinkcaps = gst_pad_get_current_caps(sinkpad); - const GstStructure *structure = gst_caps_get_structure(sinkcaps, 0); - - gchar *mediaType = nullptr; - guint ssrc = 0; - gst_structure_get( - structure, "media", G_TYPE_STRING, &mediaType, "ssrc", G_TYPE_UINT, &ssrc, nullptr); - gst_caps_unref(sinkcaps); - gst_object_unref(sinkpad); - - WebRTCSession *session = &WebRTCSession::instance(); - GstElement *queue = nullptr; - if (!std::strcmp(mediaType, "audio")) { - nhlog::ui()->debug("WebRTC: received incoming audio stream"); - haveAudioStream_ = true; - queue = newAudioSinkChain(pipe); - } else if (!std::strcmp(mediaType, "video")) { - nhlog::ui()->debug("WebRTC: received incoming video stream"); - if (!session->getVideoItem()) { - g_free(mediaType); - nhlog::ui()->error("WebRTC: video call item not set"); - return; - } - haveVideoStream_ = true; - keyFrameRequestData_.statsField = - std::string("rtp-inbound-stream-stats_") + std::to_string(ssrc); - queue = newVideoSinkChain(pipe); - auto videoCallSize = getResolution(newpad); - nhlog::ui()->info( - "WebRTC: incoming video resolution: {}x{}", videoCallSize.first, videoCallSize.second); - addLocalPiP(pipe, videoCallSize); - } else { - g_free(mediaType); - nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); - return; - } - - GstPad *queuepad = gst_element_get_static_pad(queue, "sink"); - if (queuepad) { - if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad))) - nhlog::ui()->error("WebRTC: unable to link new pad"); - else { - if (session->callType() == CallType::VOICE || - (haveAudioStream_ && (haveVideoStream_ || session->isRemoteVideoRecvOnly()))) { - emit session->stateChanged(State::CONNECTED); - if (haveVideoStream_) { - keyFrameRequestData_.pipe = pipe; - keyFrameRequestData_.decodebin = decodebin; - keyFrameRequestData_.timerid = - g_timeout_add_seconds(3, testPacketLoss, nullptr); - } - addRemotePiP(pipe); - if (session->isRemoteVideoRecvOnly()) - addLocalVideo(pipe); - } - } - gst_object_unref(queuepad); - } - g_free(mediaType); -} - -void -addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe) -{ - if (GST_PAD_DIRECTION(newpad) != GST_PAD_SRC) - return; - - nhlog::ui()->debug("WebRTC: received incoming stream"); - GstElement *decodebin = gst_element_factory_make("decodebin", nullptr); - // hardware decoding needs investigation; eg rendering fails if vaapi plugin installed - g_object_set(decodebin, "force-sw-decoders", TRUE, nullptr); - g_signal_connect(decodebin, "pad-added", G_CALLBACK(linkNewPad), pipe); - g_signal_connect(decodebin, "element-added", G_CALLBACK(setWaitForKeyFrame), nullptr); - gst_bin_add(GST_BIN(pipe), decodebin); - gst_element_sync_state_with_parent(decodebin); - GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); - if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, sinkpad))) - nhlog::ui()->error("WebRTC: unable to link decodebin"); - gst_object_unref(sinkpad); -} - -bool -contains(std::string_view str1, std::string_view str2) -{ - return std::search(str1.cbegin(), - str1.cend(), - str2.cbegin(), - str2.cend(), - [](unsigned char c1, unsigned char c2) { - return std::tolower(c1) == std::tolower(c2); - }) != str1.cend(); -} - -bool -getMediaAttributes(const GstSDPMessage *sdp, - const char *mediaType, - const char *encoding, - int &payloadType, - bool &recvOnly, - bool &sendOnly) -{ - payloadType = -1; - recvOnly = false; - sendOnly = false; - for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) { - const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex); - if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) { - recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr; - sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr; - const gchar *rtpval = nullptr; - for (guint n = 0; n == 0 || rtpval; ++n) { - rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n); - if (rtpval && contains(rtpval, encoding)) { - payloadType = std::atoi(rtpval); - break; - } - } - return true; - } - } - return false; -} -} - -bool -WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) -{ - if (!initialised_ && !init(errorMessage)) - return false; - if (!isVideo && haveVoicePlugins_) - return true; - if (isVideo && haveVideoPlugins_) - return true; - - const gchar *voicePlugins[] = {"audioconvert", - "audioresample", - "autodetect", - "dtls", - "nice", - "opus", - "playback", - "rtpmanager", - "srtp", - "volume", - "webrtc", - nullptr}; - - const gchar *videoPlugins[] = { - "compositor", "opengl", "qmlgl", "rtp", "videoconvert", "vpx", nullptr}; - - std::string strError("Missing GStreamer plugins: "); - const gchar **needed = isVideo ? videoPlugins : voicePlugins; - bool &havePlugins = isVideo ? haveVideoPlugins_ : haveVoicePlugins_; - havePlugins = true; - GstRegistry *registry = gst_registry_get(); - for (guint i = 0; i < g_strv_length((gchar **)needed); i++) { - GstPlugin *plugin = gst_registry_find_plugin(registry, needed[i]); - if (!plugin) { - havePlugins = false; - strError += std::string(needed[i]) + " "; - continue; - } - gst_object_unref(plugin); - } - if (!havePlugins) { - nhlog::ui()->error(strError); - if (errorMessage) - *errorMessage = strError; - return false; - } - - if (isVideo) { - // load qmlglsink to register GStreamer's GstGLVideoItem QML type - GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); - gst_object_unref(qmlglsink); - } - return true; -} - -bool -WebRTCSession::createOffer(CallType callType, uint32_t shareWindowId) -{ - clear(); - isOffering_ = true; - callType_ = callType; - shareWindowId_ = shareWindowId; - - // opus and vp8 rtp payload types must be defined dynamically - // therefore from the range [96-127] - // see for example https://tools.ietf.org/html/rfc7587 - constexpr int opusPayloadType = 111; - constexpr int vp8PayloadType = 96; - return startPipeline(opusPayloadType, vp8PayloadType); -} - -bool -WebRTCSession::acceptOffer(const std::string &sdp) -{ - nhlog::ui()->debug("WebRTC: received offer:\n{}", sdp); - if (state_ != State::DISCONNECTED) - return false; - - clear(); - GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER); - if (!offer) - return false; - - int opusPayloadType; - bool recvOnly; - bool sendOnly; - if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) { - if (opusPayloadType == -1) { - nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding"); - gst_webrtc_session_description_free(offer); - return false; - } - } else { - nhlog::ui()->error("WebRTC: remote offer - no audio media"); - gst_webrtc_session_description_free(offer); - return false; - } - - int vp8PayloadType; - bool isVideo = getMediaAttributes( - offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_, isRemoteVideoSendOnly_); - if (isVideo && vp8PayloadType == -1) { - nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding"); - gst_webrtc_session_description_free(offer); - return false; - } - callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; - - if (!startPipeline(opusPayloadType, vp8PayloadType)) { - gst_webrtc_session_description_free(offer); - return false; - } - - // avoid a race that sometimes leaves the generated answer without media tracks (a=ssrc - // lines) - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - - // set-remote-description first, then create-answer - GstPromise *promise = gst_promise_new_with_change_func(createAnswer, webrtc_, nullptr); - g_signal_emit_by_name(webrtc_, "set-remote-description", offer, promise); - gst_webrtc_session_description_free(offer); - return true; -} - -bool -WebRTCSession::acceptAnswer(const std::string &sdp) -{ - nhlog::ui()->debug("WebRTC: received answer:\n{}", sdp); - if (state_ != State::OFFERSENT) - return false; - - GstWebRTCSessionDescription *answer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_ANSWER); - if (!answer) { - end(); - return false; - } - - if (callType_ != CallType::VOICE) { - int unused; - if (!getMediaAttributes( - answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_, isRemoteVideoSendOnly_)) - isRemoteVideoRecvOnly_ = true; - } - - g_signal_emit_by_name(webrtc_, "set-remote-description", answer, nullptr); - gst_webrtc_session_description_free(answer); - return true; -} - -void -WebRTCSession::acceptICECandidates( - const std::vector &candidates) -{ - if (state_ >= State::INITIATED) { - for (const auto &c : candidates) { - nhlog::ui()->debug( - "WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate); - if (!c.candidate.empty()) { - g_signal_emit_by_name( - webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str()); - } - } - } -} - -bool -WebRTCSession::startPipeline(int opusPayloadType, int vp8PayloadType) -{ - if (state_ != State::DISCONNECTED) - return false; - - emit stateChanged(State::INITIATING); - - if (!createPipeline(opusPayloadType, vp8PayloadType)) { - end(); - return false; - } - - webrtc_ = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); - - if (ChatPage::instance()->userSettings()->useStunServer()) { - nhlog::ui()->info("WebRTC: setting STUN server: {}", STUN_SERVER); - g_object_set(webrtc_, "stun-server", STUN_SERVER, nullptr); - } - - for (const auto &uri : turnServers_) { - nhlog::ui()->info("WebRTC: setting TURN server: {}", uri); - gboolean udata; - g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata)); - } - if (turnServers_.empty()) - nhlog::ui()->warn("WebRTC: no TURN server provided"); - - // generate the offer when the pipeline goes to PLAYING - if (isOffering_) - g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(::createOffer), nullptr); - - // on-ice-candidate is emitted when a local ICE candidate has been gathered - g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr); - - // capture ICE failure - g_signal_connect( - webrtc_, "notify::ice-connection-state", G_CALLBACK(iceConnectionStateChanged), nullptr); - - // incoming streams trigger pad-added - gst_element_set_state(pipe_, GST_STATE_READY); - g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_); - - // capture ICE gathering completion - g_signal_connect( - webrtc_, "notify::ice-gathering-state", G_CALLBACK(iceGatheringStateChanged), nullptr); - - // webrtcbin lifetime is the same as that of the pipeline - gst_object_unref(webrtc_); - - // start the pipeline - GstStateChangeReturn ret = gst_element_set_state(pipe_, GST_STATE_PLAYING); - if (ret == GST_STATE_CHANGE_FAILURE) { - nhlog::ui()->error("WebRTC: unable to start pipeline"); - end(); - return false; - } - - GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_)); - busWatchId_ = gst_bus_add_watch(bus, newBusMessage, this); - gst_object_unref(bus); - emit stateChanged(State::INITIATED); - return true; -} - -bool -WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType) -{ - GstDevice *device = devices_.audioDevice(); - if (!device) - return false; - - GstElement *source = gst_device_create_element(device, nullptr); - GstElement *volume = gst_element_factory_make("volume", "srclevel"); - GstElement *convert = gst_element_factory_make("audioconvert", nullptr); - GstElement *resample = gst_element_factory_make("audioresample", nullptr); - GstElement *queue1 = gst_element_factory_make("queue", nullptr); - GstElement *opusenc = gst_element_factory_make("opusenc", nullptr); - GstElement *rtp = gst_element_factory_make("rtpopuspay", nullptr); - GstElement *queue2 = gst_element_factory_make("queue", nullptr); - GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); - - GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp", - "media", - G_TYPE_STRING, - "audio", - "encoding-name", - G_TYPE_STRING, - "OPUS", - "payload", - G_TYPE_INT, - opusPayloadType, - nullptr); - g_object_set(capsfilter, "caps", rtpcaps, nullptr); - gst_caps_unref(rtpcaps); - - GstElement *webrtcbin = gst_element_factory_make("webrtcbin", "webrtcbin"); - g_object_set(webrtcbin, "bundle-policy", GST_WEBRTC_BUNDLE_POLICY_MAX_BUNDLE, nullptr); - - pipe_ = gst_pipeline_new(nullptr); - gst_bin_add_many(GST_BIN(pipe_), - source, - volume, - convert, - resample, - queue1, - opusenc, - rtp, - queue2, - capsfilter, - webrtcbin, - nullptr); - - if (!gst_element_link_many(source, - volume, - convert, - resample, - queue1, - opusenc, - rtp, - queue2, - capsfilter, - webrtcbin, - nullptr)) { - nhlog::ui()->error("WebRTC: failed to link audio pipeline elements"); - return false; - } - - return callType_ == CallType::VOICE || isRemoteVideoSendOnly_ - ? true - : addVideoPipeline(vp8PayloadType); -} - -bool -WebRTCSession::addVideoPipeline(int vp8PayloadType) -{ - // allow incoming video calls despite localUser having no webcam - if (callType_ == CallType::VIDEO && !devices_.haveCamera()) - return !isOffering_; - - auto settings = ChatPage::instance()->userSettings(); - GstElement *camerafilter = nullptr; - GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); - GstElement *tee = gst_element_factory_make("tee", "videosrctee"); - gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr); - if (callType_ == CallType::VIDEO || (settings->screenSharePiP() && devices_.haveCamera())) { - std::pair resolution; - std::pair frameRate; - GstDevice *device = devices_.videoDevice(resolution, frameRate); - if (!device) - return false; - - GstElement *camera = gst_device_create_element(device, nullptr); - GstCaps *caps = gst_caps_new_simple("video/x-raw", - "width", - G_TYPE_INT, - resolution.first, - "height", - G_TYPE_INT, - resolution.second, - "framerate", - GST_TYPE_FRACTION, - frameRate.first, - frameRate.second, - nullptr); - camerafilter = gst_element_factory_make("capsfilter", "camerafilter"); - g_object_set(camerafilter, "caps", caps, nullptr); - gst_caps_unref(caps); - - gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr); - if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) { - nhlog::ui()->error("WebRTC: failed to link camera elements"); - return false; - } - if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) { - nhlog::ui()->error("WebRTC: failed to link camerafilter -> tee"); - return false; - } - } - - if (callType_ == CallType::SCREEN) { - nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", - settings->screenShareFrameRate()); - nhlog::ui()->debug("WebRTC: screen share picture-in-picture: {}", - settings->screenSharePiP()); - nhlog::ui()->debug("WebRTC: screen share request remote camera: {}", - settings->screenShareRemoteVideo()); - nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}", - settings->screenShareHideCursor()); - - GstElement *ximagesrc = gst_element_factory_make("ximagesrc", "screenshare"); - if (!ximagesrc) { - nhlog::ui()->error("WebRTC: failed to create ximagesrc"); - return false; - } - g_object_set(ximagesrc, "use-damage", FALSE, nullptr); - g_object_set(ximagesrc, "xid", shareWindowId_, nullptr); - g_object_set(ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr); - - GstCaps *caps = gst_caps_new_simple("video/x-raw", - "framerate", - GST_TYPE_FRACTION, - settings->screenShareFrameRate(), - 1, - nullptr); - GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); - g_object_set(capsfilter, "caps", caps, nullptr); - gst_caps_unref(caps); - gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr); - - if (settings->screenSharePiP() && devices_.haveCamera()) { - GstElement *compositor = gst_element_factory_make("compositor", nullptr); - g_object_set(compositor, "background", 1, nullptr); - gst_bin_add(GST_BIN(pipe_), compositor); - if (!gst_element_link_many(ximagesrc, compositor, capsfilter, tee, nullptr)) { - nhlog::ui()->error("WebRTC: failed to link screen share elements"); - return false; - } - - GstPad *srcpad = gst_element_get_static_pad(camerafilter, "src"); - remotePiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); - if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, remotePiPSinkPad_))) { - nhlog::ui()->error("WebRTC: failed to link camerafilter -> compositor"); - gst_object_unref(srcpad); - return false; - } - gst_object_unref(srcpad); - } else if (!gst_element_link_many(ximagesrc, videoconvert, capsfilter, tee, nullptr)) { - nhlog::ui()->error("WebRTC: failed to link screen share elements"); - return false; - } - } - - GstElement *queue = gst_element_factory_make("queue", nullptr); - GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); - g_object_set(vp8enc, "deadline", 1, nullptr); - g_object_set(vp8enc, "error-resilient", 1, nullptr); - GstElement *rtpvp8pay = gst_element_factory_make("rtpvp8pay", nullptr); - GstElement *rtpqueue = gst_element_factory_make("queue", nullptr); - GstElement *rtpcapsfilter = gst_element_factory_make("capsfilter", nullptr); - GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp", - "media", - G_TYPE_STRING, - "video", - "encoding-name", - G_TYPE_STRING, - "VP8", - "payload", - G_TYPE_INT, - vp8PayloadType, - nullptr); - g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr); - gst_caps_unref(rtpcaps); - - gst_bin_add_many(GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr); - - GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); - if (!gst_element_link_many( - tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) { - nhlog::ui()->error("WebRTC: failed to link rtp video elements"); - gst_object_unref(webrtcbin); - return false; - } - - if (callType_ == CallType::SCREEN && - !ChatPage::instance()->userSettings()->screenShareRemoteVideo()) { - GArray *transceivers; - g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers); - GstWebRTCRTPTransceiver *transceiver = - g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1); - transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; - g_array_unref(transceivers); - } - - gst_object_unref(webrtcbin); - return true; -} - -bool -WebRTCSession::haveLocalPiP() const -{ - if (state_ >= State::INITIATED) { - if (callType_ == CallType::VOICE || isRemoteVideoRecvOnly_) - return false; - else if (callType_ == CallType::SCREEN) - return true; - else { - GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); - if (tee) { - gst_object_unref(tee); - return true; - } - } - } - return false; -} - -bool -WebRTCSession::isMicMuted() const -{ - if (state_ < State::INITIATED) - return false; - - GstElement *srclevel = gst_bin_get_by_name(GST_BIN(pipe_), "srclevel"); - gboolean muted; - g_object_get(srclevel, "mute", &muted, nullptr); - gst_object_unref(srclevel); - return muted; -} - -bool -WebRTCSession::toggleMicMute() -{ - if (state_ < State::INITIATED) - return false; - - GstElement *srclevel = gst_bin_get_by_name(GST_BIN(pipe_), "srclevel"); - gboolean muted; - g_object_get(srclevel, "mute", &muted, nullptr); - g_object_set(srclevel, "mute", !muted, nullptr); - gst_object_unref(srclevel); - return !muted; -} - -void -WebRTCSession::toggleLocalPiP() -{ - if (localPiPSinkPad_) { - guint zorder; - g_object_get(localPiPSinkPad_, "zorder", &zorder, nullptr); - g_object_set(localPiPSinkPad_, "zorder", zorder ? 0 : 2, nullptr); - } -} - -void -WebRTCSession::clear() -{ - callType_ = webrtc::CallType::VOICE; - isOffering_ = false; - isRemoteVideoRecvOnly_ = false; - isRemoteVideoSendOnly_ = false; - videoItem_ = nullptr; - pipe_ = nullptr; - webrtc_ = nullptr; - busWatchId_ = 0; - shareWindowId_ = 0; - haveAudioStream_ = false; - haveVideoStream_ = false; - localPiPSinkPad_ = nullptr; - remotePiPSinkPad_ = nullptr; - localsdp_.clear(); - localcandidates_.clear(); -} - -void -WebRTCSession::end() -{ - nhlog::ui()->debug("WebRTC: ending session"); - keyFrameRequestData_ = KeyFrameRequestData{}; - if (pipe_) { - gst_element_set_state(pipe_, GST_STATE_NULL); - gst_object_unref(pipe_); - pipe_ = nullptr; - if (busWatchId_) { - g_source_remove(busWatchId_); - busWatchId_ = 0; - } - } - - clear(); - if (state_ != State::DISCONNECTED) - emit stateChanged(State::DISCONNECTED); -} - -#else - -bool -WebRTCSession::havePlugins(bool, std::string *) -{ - return false; -} - -bool -WebRTCSession::haveLocalPiP() const -{ - return false; -} - -bool WebRTCSession::createOffer(webrtc::CallType, uint32_t) { return false; } - -bool -WebRTCSession::acceptOffer(const std::string &) -{ - return false; -} - -bool -WebRTCSession::acceptAnswer(const std::string &) -{ - return false; -} - -void -WebRTCSession::acceptICECandidates(const std::vector &) -{} - -bool -WebRTCSession::isMicMuted() const -{ - return false; -} - -bool -WebRTCSession::toggleMicMute() -{ - return false; -} - -void -WebRTCSession::toggleLocalPiP() -{} - -void -WebRTCSession::end() -{} - -#endif diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h deleted file mode 100644 index 56c0a295..00000000 --- a/src/WebRTCSession.h +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-FileCopyrightText: 2021 Nheko Contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include -#include - -#include - -#include "mtx/events/voip.hpp" - -typedef struct _GstElement GstElement; -class CallDevices; -class QQuickItem; - -namespace webrtc { -Q_NAMESPACE - -enum class CallType -{ - VOICE, - VIDEO, - SCREEN // localUser is sharing screen -}; -Q_ENUM_NS(CallType) - -enum class State -{ - DISCONNECTED, - ICEFAILED, - INITIATING, - INITIATED, - OFFERSENT, - ANSWERSENT, - CONNECTING, - CONNECTED - -}; -Q_ENUM_NS(State) -} - -class WebRTCSession : public QObject -{ - Q_OBJECT - -public: - static WebRTCSession &instance() - { - static WebRTCSession instance; - return instance; - } - - bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); - webrtc::CallType callType() const { return callType_; } - webrtc::State state() const { return state_; } - bool haveLocalPiP() const; - bool isOffering() const { return isOffering_; } - bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } - bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } - - bool createOffer(webrtc::CallType, uint32_t shareWindowId); - bool acceptOffer(const std::string &sdp); - bool acceptAnswer(const std::string &sdp); - void acceptICECandidates(const std::vector &); - - bool isMicMuted() const; - bool toggleMicMute(); - void toggleLocalPiP(); - void end(); - - void setTurnServers(const std::vector &uris) { turnServers_ = uris; } - - void setVideoItem(QQuickItem *item) { videoItem_ = item; } - QQuickItem *getVideoItem() const { return videoItem_; } - -signals: - void offerCreated(const std::string &sdp, - const std::vector &); - void answerCreated(const std::string &sdp, - const std::vector &); - void newICECandidate(const mtx::events::msg::CallCandidates::Candidate &); - void stateChanged(webrtc::State); - -private slots: - void setState(webrtc::State state) { state_ = state; } - -private: - WebRTCSession(); - - CallDevices &devices_; - bool initialised_ = false; - bool haveVoicePlugins_ = false; - bool haveVideoPlugins_ = false; - webrtc::CallType callType_ = webrtc::CallType::VOICE; - webrtc::State state_ = webrtc::State::DISCONNECTED; - bool isOffering_ = false; - bool isRemoteVideoRecvOnly_ = false; - bool isRemoteVideoSendOnly_ = false; - QQuickItem *videoItem_ = nullptr; - GstElement *pipe_ = nullptr; - GstElement *webrtc_ = nullptr; - unsigned int busWatchId_ = 0; - std::vector turnServers_; - uint32_t shareWindowId_ = 0; - - bool init(std::string *errorMessage = nullptr); - bool startPipeline(int opusPayloadType, int vp8PayloadType); - bool createPipeline(int opusPayloadType, int vp8PayloadType); - bool addVideoPipeline(int vp8PayloadType); - void clear(); - -public: - WebRTCSession(WebRTCSession const &) = delete; - void operator=(WebRTCSession const &) = delete; -}; diff --git a/src/encryption/DeviceVerificationFlow.cpp b/src/encryption/DeviceVerificationFlow.cpp new file mode 100644 index 00000000..2481d4f9 --- /dev/null +++ b/src/encryption/DeviceVerificationFlow.cpp @@ -0,0 +1,849 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "DeviceVerificationFlow.h" + +#include "Cache.h" +#include "Cache_p.h" +#include "ChatPage.h" +#include "Logging.h" +#include "Utils.h" +#include "timeline/TimelineModel.h" + +#include +#include +#include + +static constexpr int TIMEOUT = 2 * 60 * 1000; // 2 minutes + +namespace msgs = mtx::events::msg; + +static mtx::events::msg::KeyVerificationMac +key_verification_mac(mtx::crypto::SAS *sas, + mtx::identifiers::User sender, + const std::string &senderDevice, + mtx::identifiers::User receiver, + const std::string &receiverDevice, + const std::string &transactionId, + std::map keys); + +DeviceVerificationFlow::DeviceVerificationFlow(QObject *, + DeviceVerificationFlow::Type flow_type, + TimelineModel *model, + QString userID, + QString deviceId_) + : sender(false) + , type(flow_type) + , deviceId(deviceId_) + , model_(model) +{ + timeout = new QTimer(this); + timeout->setSingleShot(true); + this->sas = olm::client()->sas_init(); + this->isMacVerified = false; + + auto user_id = userID.toStdString(); + this->toClient = mtx::identifiers::parse(user_id); + cache::client()->query_keys( + user_id, [user_id, this](const UserKeyCache &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to query device keys: {},{}", + mtx::errors::to_string(err->matrix_error.errcode), + static_cast(err->status_code)); + return; + } + + if (!this->deviceId.isEmpty() && + (res.device_keys.find(deviceId.toStdString()) == res.device_keys.end())) { + nhlog::net()->warn("no devices retrieved {}", user_id); + return; + } + + this->their_keys = res; + }); + + cache::client()->query_keys( + http::client()->user_id().to_string(), + [this](const UserKeyCache &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to query device keys: {},{}", + mtx::errors::to_string(err->matrix_error.errcode), + static_cast(err->status_code)); + return; + } + + if (res.master_keys.keys.empty()) + return; + + if (auto status = cache::verificationStatus(http::client()->user_id().to_string()); + status && status->user_verified == crypto::Trust::Verified) + this->our_trusted_master_key = res.master_keys.keys.begin()->second; + }); + + if (model) { + connect( + this->model_, &TimelineModel::updateFlowEventId, this, [this](std::string event_id_) { + this->relation.rel_type = mtx::common::RelationType::Reference; + this->relation.event_id = event_id_; + this->transaction_id = event_id_; + }); + } + + connect(timeout, &QTimer::timeout, this, [this]() { + nhlog::crypto()->info("verification: timeout"); + if (state_ != Success && state_ != Failed) + this->cancelVerification(DeviceVerificationFlow::Error::Timeout); + }); + + connect(ChatPage::instance(), + &ChatPage::receivedDeviceVerificationStart, + this, + &DeviceVerificationFlow::handleStartMessage); + connect(ChatPage::instance(), + &ChatPage::receivedDeviceVerificationAccept, + this, + [this](const mtx::events::msg::KeyVerificationAccept &msg) { + nhlog::crypto()->info("verification: received accept"); + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + if ((msg.key_agreement_protocol == "curve25519-hkdf-sha256") && + (msg.hash == "sha256") && + (msg.message_authentication_code == "hkdf-hmac-sha256")) { + this->commitment = msg.commitment; + if (std::find(msg.short_authentication_string.begin(), + msg.short_authentication_string.end(), + mtx::events::msg::SASMethods::Emoji) != + msg.short_authentication_string.end()) { + this->method = mtx::events::msg::SASMethods::Emoji; + } else { + this->method = mtx::events::msg::SASMethods::Decimal; + } + this->mac_method = msg.message_authentication_code; + this->sendVerificationKey(); + } else { + this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); + } + }); + + connect(ChatPage::instance(), + &ChatPage::receivedDeviceVerificationCancel, + this, + [this](const mtx::events::msg::KeyVerificationCancel &msg) { + nhlog::crypto()->info("verification: received cancel"); + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + error_ = User; + emit errorChanged(); + setState(Failed); + }); + + connect( + ChatPage::instance(), + &ChatPage::receivedDeviceVerificationKey, + this, + [this](const mtx::events::msg::KeyVerificationKey &msg) { + nhlog::crypto()->info("verification: received key"); + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + + if (sender) { + if (state_ != WaitingForOtherToAccept) { + this->cancelVerification(OutOfOrder); + return; + } + } else { + if (state_ != WaitingForKeys) { + this->cancelVerification(OutOfOrder); + return; + } + } + + this->sas->set_their_key(msg.key); + std::string info; + if (this->sender == true) { + info = "MATRIX_KEY_VERIFICATION_SAS|" + http::client()->user_id().to_string() + "|" + + http::client()->device_id() + "|" + this->sas->public_key() + "|" + + this->toClient.to_string() + "|" + this->deviceId.toStdString() + "|" + + msg.key + "|" + this->transaction_id; + } else { + info = "MATRIX_KEY_VERIFICATION_SAS|" + this->toClient.to_string() + "|" + + this->deviceId.toStdString() + "|" + msg.key + "|" + + http::client()->user_id().to_string() + "|" + http::client()->device_id() + + "|" + this->sas->public_key() + "|" + this->transaction_id; + } + + nhlog::ui()->info("Info is: '{}'", info); + + if (this->sender == false) { + this->sendVerificationKey(); + } else { + if (this->commitment != mtx::crypto::bin2base64_unpadded(mtx::crypto::sha256( + msg.key + this->canonical_json.dump()))) { + this->cancelVerification(DeviceVerificationFlow::Error::MismatchedCommitment); + return; + } + } + + if (this->method == mtx::events::msg::SASMethods::Emoji) { + this->sasList = this->sas->generate_bytes_emoji(info); + setState(CompareEmoji); + } else if (this->method == mtx::events::msg::SASMethods::Decimal) { + this->sasList = this->sas->generate_bytes_decimal(info); + setState(CompareNumber); + } + }); + + connect( + ChatPage::instance(), + &ChatPage::receivedDeviceVerificationMac, + this, + [this](const mtx::events::msg::KeyVerificationMac &msg) { + nhlog::crypto()->info("verification: received mac"); + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + + std::map key_list; + std::string key_string; + for (const auto &mac : msg.mac) { + for (const auto &[deviceid, key] : their_keys.device_keys) { + (void)deviceid; + if (key.keys.count(mac.first)) + key_list[mac.first] = key.keys.at(mac.first); + } + + if (their_keys.master_keys.keys.count(mac.first)) + key_list[mac.first] = their_keys.master_keys.keys[mac.first]; + if (their_keys.user_signing_keys.keys.count(mac.first)) + key_list[mac.first] = their_keys.user_signing_keys.keys[mac.first]; + if (their_keys.self_signing_keys.keys.count(mac.first)) + key_list[mac.first] = their_keys.self_signing_keys.keys[mac.first]; + } + auto macs = key_verification_mac(sas.get(), + toClient, + this->deviceId.toStdString(), + http::client()->user_id(), + http::client()->device_id(), + this->transaction_id, + key_list); + + for (const auto &[key, mac] : macs.mac) { + if (mac != msg.mac.at(key)) { + this->cancelVerification(DeviceVerificationFlow::Error::KeyMismatch); + return; + } + } + + if (msg.keys == macs.keys) { + mtx::requests::KeySignaturesUpload req; + if (utils::localUser().toStdString() == this->toClient.to_string()) { + // self verification, sign master key with device key, if we + // verified it + for (const auto &mac : msg.mac) { + if (their_keys.master_keys.keys.count(mac.first)) { + json j = their_keys.master_keys; + j.erase("signatures"); + j.erase("unsigned"); + mtx::crypto::CrossSigningKeys master_key = j; + master_key.signatures[utils::localUser().toStdString()] + ["ed25519:" + http::client()->device_id()] = + olm::client()->sign_message(j.dump()); + req.signatures[utils::localUser().toStdString()] + [master_key.keys.at(mac.first)] = master_key; + } else if (mac.first == "ed25519:" + this->deviceId.toStdString()) { + // Sign their device key with self signing key + + auto device_id = this->deviceId.toStdString(); + + if (their_keys.device_keys.count(device_id)) { + json j = their_keys.device_keys.at(device_id); + j.erase("signatures"); + j.erase("unsigned"); + + auto secret = cache::secret( + mtx::secret_storage::secrets::cross_signing_self_signing); + if (!secret) + continue; + auto ssk = mtx::crypto::PkSigning::from_seed(*secret); + + mtx::crypto::DeviceKeys dev = j; + dev.signatures[utils::localUser().toStdString()] + ["ed25519:" + ssk.public_key()] = ssk.sign(j.dump()); + + req.signatures[utils::localUser().toStdString()][device_id] = dev; + } + } + } + } else { + // Sign their master key with user signing key + for (const auto &mac : msg.mac) { + if (their_keys.master_keys.keys.count(mac.first)) { + json j = their_keys.master_keys; + j.erase("signatures"); + j.erase("unsigned"); + + auto secret = + cache::secret(mtx::secret_storage::secrets::cross_signing_user_signing); + if (!secret) + continue; + auto usk = mtx::crypto::PkSigning::from_seed(*secret); + + mtx::crypto::CrossSigningKeys master_key = j; + master_key.signatures[utils::localUser().toStdString()] + ["ed25519:" + usk.public_key()] = usk.sign(j.dump()); + + req.signatures[toClient.to_string()][master_key.keys.at(mac.first)] = + master_key; + } + } + } + + if (!req.signatures.empty()) { + http::client()->keys_signatures_upload( + req, + [](const mtx::responses::KeySignaturesUpload &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("failed to upload signatures: {},{}", + mtx::errors::to_string(err->matrix_error.errcode), + static_cast(err->status_code)); + } + + for (const auto &[user_id, tmp] : res.errors) + for (const auto &[key_id, e] : tmp) + nhlog::net()->error("signature error for user {} and key " + "id {}: {}, {}", + user_id, + key_id, + mtx::errors::to_string(e.errcode), + e.error); + }); + } + + this->isMacVerified = true; + this->acceptDevice(); + } else { + this->cancelVerification(DeviceVerificationFlow::Error::KeyMismatch); + } + }); + + connect(ChatPage::instance(), + &ChatPage::receivedDeviceVerificationReady, + this, + [this](const mtx::events::msg::KeyVerificationReady &msg) { + nhlog::crypto()->info("verification: received ready"); + if (!sender) { + if (msg.from_device != http::client()->device_id()) { + error_ = User; + emit errorChanged(); + setState(Failed); + } + + return; + } + + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + else { + this->deviceId = QString::fromStdString(msg.from_device); + } + } + this->startVerificationRequest(); + }); + + connect(ChatPage::instance(), + &ChatPage::receivedDeviceVerificationDone, + this, + [this](const mtx::events::msg::KeyVerificationDone &msg) { + nhlog::crypto()->info("verification: received done"); + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + nhlog::ui()->info("Flow done on other side"); + }); + + timeout->start(TIMEOUT); +} + +QString +DeviceVerificationFlow::state() +{ + switch (state_) { + case PromptStartVerification: + return "PromptStartVerification"; + case CompareEmoji: + return "CompareEmoji"; + case CompareNumber: + return "CompareNumber"; + case WaitingForKeys: + return "WaitingForKeys"; + case WaitingForOtherToAccept: + return "WaitingForOtherToAccept"; + case WaitingForMac: + return "WaitingForMac"; + case Success: + return "Success"; + case Failed: + return "Failed"; + default: + return ""; + } +} + +void +DeviceVerificationFlow::next() +{ + if (sender) { + switch (state_) { + case PromptStartVerification: + sendVerificationRequest(); + break; + case CompareEmoji: + case CompareNumber: + sendVerificationMac(); + break; + case WaitingForKeys: + case WaitingForOtherToAccept: + case WaitingForMac: + case Success: + case Failed: + nhlog::db()->error("verification: Invalid state transition!"); + break; + } + } else { + switch (state_) { + case PromptStartVerification: + if (canonical_json.is_null()) + sendVerificationReady(); + else // legacy path without request and ready + acceptVerificationRequest(); + break; + case CompareEmoji: + [[fallthrough]]; + case CompareNumber: + sendVerificationMac(); + break; + case WaitingForKeys: + case WaitingForOtherToAccept: + case WaitingForMac: + case Success: + case Failed: + nhlog::db()->error("verification: Invalid state transition!"); + break; + } + } +} + +QString +DeviceVerificationFlow::getUserId() +{ + return QString::fromStdString(this->toClient.to_string()); +} + +QString +DeviceVerificationFlow::getDeviceId() +{ + return this->deviceId; +} + +bool +DeviceVerificationFlow::getSender() +{ + return this->sender; +} + +std::vector +DeviceVerificationFlow::getSasList() +{ + return this->sasList; +} + +bool +DeviceVerificationFlow::isSelfVerification() const +{ + return this->toClient.to_string() == http::client()->user_id().to_string(); +} + +void +DeviceVerificationFlow::setEventId(std::string event_id_) +{ + this->relation.rel_type = mtx::common::RelationType::Reference; + this->relation.event_id = event_id_; + this->transaction_id = event_id_; +} + +void +DeviceVerificationFlow::handleStartMessage(const mtx::events::msg::KeyVerificationStart &msg, + std::string) +{ + if (msg.transaction_id.has_value()) { + if (msg.transaction_id.value() != this->transaction_id) + return; + } else if (msg.relations.references()) { + if (msg.relations.references() != this->relation.event_id) + return; + } + if ((std::find(msg.key_agreement_protocols.begin(), + msg.key_agreement_protocols.end(), + "curve25519-hkdf-sha256") != msg.key_agreement_protocols.end()) && + (std::find(msg.hashes.begin(), msg.hashes.end(), "sha256") != msg.hashes.end()) && + (std::find(msg.message_authentication_codes.begin(), + msg.message_authentication_codes.end(), + "hkdf-hmac-sha256") != msg.message_authentication_codes.end())) { + if (std::find(msg.short_authentication_string.begin(), + msg.short_authentication_string.end(), + mtx::events::msg::SASMethods::Emoji) != + msg.short_authentication_string.end()) { + this->method = mtx::events::msg::SASMethods::Emoji; + } else if (std::find(msg.short_authentication_string.begin(), + msg.short_authentication_string.end(), + mtx::events::msg::SASMethods::Decimal) != + msg.short_authentication_string.end()) { + this->method = mtx::events::msg::SASMethods::Decimal; + } else { + this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); + return; + } + if (!sender) + this->canonical_json = nlohmann::json(msg); + else { + if (utils::localUser().toStdString() < this->toClient.to_string()) { + this->canonical_json = nlohmann::json(msg); + } + } + + if (state_ != PromptStartVerification) + this->acceptVerificationRequest(); + } else { + this->cancelVerification(DeviceVerificationFlow::Error::UnknownMethod); + } +} + +//! accepts a verification +void +DeviceVerificationFlow::acceptVerificationRequest() +{ + mtx::events::msg::KeyVerificationAccept req; + + req.method = mtx::events::msg::VerificationMethods::SASv1; + req.key_agreement_protocol = "curve25519-hkdf-sha256"; + req.hash = "sha256"; + req.message_authentication_code = "hkdf-hmac-sha256"; + if (this->method == mtx::events::msg::SASMethods::Emoji) + req.short_authentication_string = {mtx::events::msg::SASMethods::Emoji}; + else if (this->method == mtx::events::msg::SASMethods::Decimal) + req.short_authentication_string = {mtx::events::msg::SASMethods::Decimal}; + req.commitment = mtx::crypto::bin2base64_unpadded( + mtx::crypto::sha256(this->sas->public_key() + this->canonical_json.dump())); + + send(req); + setState(WaitingForKeys); +} +//! responds verification request +void +DeviceVerificationFlow::sendVerificationReady() +{ + mtx::events::msg::KeyVerificationReady req; + + req.from_device = http::client()->device_id(); + req.methods = {mtx::events::msg::VerificationMethods::SASv1}; + + send(req); + setState(WaitingForKeys); +} +//! accepts a verification +void +DeviceVerificationFlow::sendVerificationDone() +{ + mtx::events::msg::KeyVerificationDone req; + + send(req); +} +//! starts the verification flow +void +DeviceVerificationFlow::startVerificationRequest() +{ + mtx::events::msg::KeyVerificationStart req; + + req.from_device = http::client()->device_id(); + req.method = mtx::events::msg::VerificationMethods::SASv1; + req.key_agreement_protocols = {"curve25519-hkdf-sha256"}; + req.hashes = {"sha256"}; + req.message_authentication_codes = {"hkdf-hmac-sha256"}; + req.short_authentication_string = {mtx::events::msg::SASMethods::Decimal, + mtx::events::msg::SASMethods::Emoji}; + + if (this->type == DeviceVerificationFlow::Type::ToDevice) { + mtx::requests::ToDeviceMessages body; + req.transaction_id = this->transaction_id; + this->canonical_json = nlohmann::json(req); + } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { + req.relations.relations.push_back(this->relation); + // Set synthesized to surpress the nheko relation extensions + req.relations.synthesized = true; + this->canonical_json = nlohmann::json(req); + } + send(req); + setState(WaitingForOtherToAccept); +} +//! sends a verification request +void +DeviceVerificationFlow::sendVerificationRequest() +{ + mtx::events::msg::KeyVerificationRequest req; + + req.from_device = http::client()->device_id(); + req.methods = {mtx::events::msg::VerificationMethods::SASv1}; + + if (this->type == DeviceVerificationFlow::Type::ToDevice) { + QDateTime currentTime = QDateTime::currentDateTimeUtc(); + + req.timestamp = (uint64_t)currentTime.toMSecsSinceEpoch(); + + } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { + req.to = this->toClient.to_string(); + req.msgtype = "m.key.verification.request"; + req.body = "User is requesting to verify keys with you. However, your client does " + "not support this method, so you will need to use the legacy method of " + "key verification."; + } + + send(req); + setState(WaitingForOtherToAccept); +} +//! cancels a verification flow +void +DeviceVerificationFlow::cancelVerification(DeviceVerificationFlow::Error error_code) +{ + if (state_ == State::Success || state_ == State::Failed) + return; + + mtx::events::msg::KeyVerificationCancel req; + + if (error_code == DeviceVerificationFlow::Error::UnknownMethod) { + req.code = "m.unknown_method"; + req.reason = "unknown method received"; + } else if (error_code == DeviceVerificationFlow::Error::MismatchedCommitment) { + req.code = "m.mismatched_commitment"; + req.reason = "commitment didn't match"; + } else if (error_code == DeviceVerificationFlow::Error::MismatchedSAS) { + req.code = "m.mismatched_sas"; + req.reason = "sas didn't match"; + } else if (error_code == DeviceVerificationFlow::Error::KeyMismatch) { + req.code = "m.key_match"; + req.reason = "keys did not match"; + } else if (error_code == DeviceVerificationFlow::Error::Timeout) { + req.code = "m.timeout"; + req.reason = "timed out"; + } else if (error_code == DeviceVerificationFlow::Error::User) { + req.code = "m.user"; + req.reason = "user cancelled the verification"; + } else if (error_code == DeviceVerificationFlow::Error::OutOfOrder) { + req.code = "m.unexpected_message"; + req.reason = "received messages out of order"; + } + + this->error_ = error_code; + emit errorChanged(); + this->setState(Failed); + + send(req); +} +//! sends the verification key +void +DeviceVerificationFlow::sendVerificationKey() +{ + mtx::events::msg::KeyVerificationKey req; + + req.key = this->sas->public_key(); + + send(req); +} + +mtx::events::msg::KeyVerificationMac +key_verification_mac(mtx::crypto::SAS *sas, + mtx::identifiers::User sender, + const std::string &senderDevice, + mtx::identifiers::User receiver, + const std::string &receiverDevice, + const std::string &transactionId, + std::map keys) +{ + mtx::events::msg::KeyVerificationMac req; + + std::string info = "MATRIX_KEY_VERIFICATION_MAC" + sender.to_string() + senderDevice + + receiver.to_string() + receiverDevice + transactionId; + + std::string key_list; + bool first = true; + for (const auto &[key_id, key] : keys) { + req.mac[key_id] = sas->calculate_mac(key, info + key_id); + + if (!first) + key_list += ","; + key_list += key_id; + first = false; + } + + req.keys = sas->calculate_mac(key_list, info + "KEY_IDS"); + + return req; +} + +//! sends the mac of the keys +void +DeviceVerificationFlow::sendVerificationMac() +{ + std::map key_list; + key_list["ed25519:" + http::client()->device_id()] = olm::client()->identity_keys().ed25519; + + // send our master key, if we trust it + if (!this->our_trusted_master_key.empty()) + key_list["ed25519:" + our_trusted_master_key] = our_trusted_master_key; + + mtx::events::msg::KeyVerificationMac req = key_verification_mac(sas.get(), + http::client()->user_id(), + http::client()->device_id(), + this->toClient, + this->deviceId.toStdString(), + this->transaction_id, + key_list); + + send(req); + + setState(WaitingForMac); + acceptDevice(); +} +//! Completes the verification flow +void +DeviceVerificationFlow::acceptDevice() +{ + if (!isMacVerified) { + setState(WaitingForMac); + } else if (state_ == WaitingForMac) { + cache::markDeviceVerified(this->toClient.to_string(), this->deviceId.toStdString()); + this->sendVerificationDone(); + setState(Success); + + // Request secrets. We should probably check somehow, if a device knowns about the + // secrets. + if (utils::localUser().toStdString() == this->toClient.to_string() && + (!cache::secret(mtx::secret_storage::secrets::cross_signing_self_signing) || + !cache::secret(mtx::secret_storage::secrets::cross_signing_user_signing))) { + olm::request_cross_signing_keys(); + } + } +} + +void +DeviceVerificationFlow::unverify() +{ + cache::markDeviceUnverified(this->toClient.to_string(), this->deviceId.toStdString()); + + emit refreshProfile(); +} + +QSharedPointer +DeviceVerificationFlow::NewInRoomVerification(QObject *parent_, + TimelineModel *timelineModel_, + const mtx::events::msg::KeyVerificationRequest &msg, + QString other_user_, + QString event_id_) +{ + QSharedPointer flow( + new DeviceVerificationFlow(parent_, + Type::RoomMsg, + timelineModel_, + other_user_, + QString::fromStdString(msg.from_device))); + + flow->setEventId(event_id_.toStdString()); + + if (std::find(msg.methods.begin(), + msg.methods.end(), + mtx::events::msg::VerificationMethods::SASv1) == msg.methods.end()) { + flow->cancelVerification(UnknownMethod); + } + + return flow; +} +QSharedPointer +DeviceVerificationFlow::NewToDeviceVerification(QObject *parent_, + const mtx::events::msg::KeyVerificationRequest &msg, + QString other_user_, + QString txn_id_) +{ + QSharedPointer flow(new DeviceVerificationFlow( + parent_, Type::ToDevice, nullptr, other_user_, QString::fromStdString(msg.from_device))); + flow->transaction_id = txn_id_.toStdString(); + + if (std::find(msg.methods.begin(), + msg.methods.end(), + mtx::events::msg::VerificationMethods::SASv1) == msg.methods.end()) { + flow->cancelVerification(UnknownMethod); + } + + return flow; +} +QSharedPointer +DeviceVerificationFlow::NewToDeviceVerification(QObject *parent_, + const mtx::events::msg::KeyVerificationStart &msg, + QString other_user_, + QString txn_id_) +{ + QSharedPointer flow(new DeviceVerificationFlow( + parent_, Type::ToDevice, nullptr, other_user_, QString::fromStdString(msg.from_device))); + flow->transaction_id = txn_id_.toStdString(); + + flow->handleStartMessage(msg, ""); + + return flow; +} +QSharedPointer +DeviceVerificationFlow::InitiateUserVerification(QObject *parent_, + TimelineModel *timelineModel_, + QString userid) +{ + QSharedPointer flow( + new DeviceVerificationFlow(parent_, Type::RoomMsg, timelineModel_, userid, "")); + flow->sender = true; + return flow; +} +QSharedPointer +DeviceVerificationFlow::InitiateDeviceVerification(QObject *parent_, QString userid, QString device) +{ + QSharedPointer flow( + new DeviceVerificationFlow(parent_, Type::ToDevice, nullptr, userid, device)); + + flow->sender = true; + flow->transaction_id = http::client()->generate_txn_id(); + + return flow; +} diff --git a/src/encryption/DeviceVerificationFlow.h b/src/encryption/DeviceVerificationFlow.h new file mode 100644 index 00000000..f71fa337 --- /dev/null +++ b/src/encryption/DeviceVerificationFlow.h @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include +#include + +#include "CacheCryptoStructs.h" +#include "Logging.h" +#include "MatrixClient.h" +#include "Olm.h" +#include "timeline/TimelineModel.h" + +class QTimer; + +using sas_ptr = std::unique_ptr; + +// clang-format off +/* + * Stolen from fluffy chat :D + * + * State | +-------------+ +-----------+ | + * | | AliceDevice | | BobDevice | | + * | | (sender) | | | | + * | +-------------+ +-----------+ | + * promptStartVerify | | | | + * | o | (m.key.verification.request) | | + * | p |-------------------------------->| (ASK FOR VERIFICATION REQUEST) | + * waitForOtherAccept | t | | | promptStartVerify + * && | i | (m.key.verification.ready) | | + * no commitment | o |<--------------------------------| | + * && | n | | | + * no canonical_json | a | (m.key.verification.start) | | waitingForKeys + * | l |<--------------------------------| Not sending to prevent the glare resolve| && no commitment + * | | | | && no canonical_json + * | | m.key.verification.start | | + * waitForOtherAccept | |-------------------------------->| (IF NOT ALREADY ASKED, | + * && | | | ASK FOR VERIFICATION REQUEST) | promptStartVerify, if not accepted + * canonical_json | | m.key.verification.accept | | + * | |<--------------------------------| | + * waitForOtherAccept | | | | waitingForKeys + * && | | m.key.verification.key | | && canonical_json + * commitment | |-------------------------------->| | && commitment + * | | | | + * | | m.key.verification.key | | + * | |<--------------------------------| | + * compareEmoji/Number| | | | compareEmoji/Number + * | | COMPARE EMOJI / NUMBERS | | + * | | | | + * waitingForMac | | m.key.verification.mac | | waitingForMac + * | success |<------------------------------->| success | + * | | | | + * success/fail | | m.key.verification.done | | success/fail + * | |<------------------------------->| | + */ +// clang-format on +class DeviceVerificationFlow : public QObject +{ + Q_OBJECT + Q_PROPERTY(QString state READ state NOTIFY stateChanged) + Q_PROPERTY(Error error READ error NOTIFY errorChanged) + Q_PROPERTY(QString userId READ getUserId CONSTANT) + Q_PROPERTY(QString deviceId READ getDeviceId CONSTANT) + Q_PROPERTY(bool sender READ getSender CONSTANT) + Q_PROPERTY(std::vector sasList READ getSasList CONSTANT) + Q_PROPERTY(bool isDeviceVerification READ isDeviceVerification CONSTANT) + Q_PROPERTY(bool isSelfVerification READ isSelfVerification CONSTANT) + +public: + enum State + { + PromptStartVerification, + WaitingForOtherToAccept, + WaitingForKeys, + CompareEmoji, + CompareNumber, + WaitingForMac, + Success, + Failed, + }; + Q_ENUM(State) + + enum Type + { + ToDevice, + RoomMsg + }; + + enum Error + { + UnknownMethod, + MismatchedCommitment, + MismatchedSAS, + KeyMismatch, + Timeout, + User, + OutOfOrder, + }; + Q_ENUM(Error) + + static QSharedPointer NewInRoomVerification( + QObject *parent_, + TimelineModel *timelineModel_, + const mtx::events::msg::KeyVerificationRequest &msg, + QString other_user_, + QString event_id_); + static QSharedPointer NewToDeviceVerification( + QObject *parent_, + const mtx::events::msg::KeyVerificationRequest &msg, + QString other_user_, + QString txn_id_); + static QSharedPointer NewToDeviceVerification( + QObject *parent_, + const mtx::events::msg::KeyVerificationStart &msg, + QString other_user_, + QString txn_id_); + static QSharedPointer + InitiateUserVerification(QObject *parent_, TimelineModel *timelineModel_, QString userid); + static QSharedPointer InitiateDeviceVerification(QObject *parent, + QString userid, + QString device); + + // getters + QString state(); + Error error() { return error_; } + QString getUserId(); + QString getDeviceId(); + bool getSender(); + std::vector getSasList(); + QString transactionId() { return QString::fromStdString(this->transaction_id); } + // setters + void setDeviceId(QString deviceID); + void setEventId(std::string event_id); + bool isDeviceVerification() const + { + return this->type == DeviceVerificationFlow::Type::ToDevice; + } + bool isSelfVerification() const; + + void callback_fn(const UserKeyCache &res, mtx::http::RequestErr err, std::string user_id); + +public slots: + //! unverifies a device + void unverify(); + //! Continues the flow + void next(); + //! Cancel the flow + void cancel() { cancelVerification(User); } + +signals: + void refreshProfile(); + void stateChanged(); + void errorChanged(); + +private: + DeviceVerificationFlow(QObject *, + DeviceVerificationFlow::Type flow_type, + TimelineModel *model, + QString userID, + QString deviceId_); + void setState(State state) + { + if (state != state_) { + state_ = state; + emit stateChanged(); + } + } + + void handleStartMessage(const mtx::events::msg::KeyVerificationStart &msg, std::string); + //! sends a verification request + void sendVerificationRequest(); + //! accepts a verification request + void sendVerificationReady(); + //! completes the verification flow(); + void sendVerificationDone(); + //! accepts a verification + void acceptVerificationRequest(); + //! starts the verification flow + void startVerificationRequest(); + //! cancels a verification flow + void cancelVerification(DeviceVerificationFlow::Error error_code); + //! sends the verification key + void sendVerificationKey(); + //! sends the mac of the keys + void sendVerificationMac(); + //! Completes the verification flow + void acceptDevice(); + + std::string transaction_id; + + bool sender; + Type type; + mtx::identifiers::User toClient; + QString deviceId; + + // public part of our master key, when trusted or empty + std::string our_trusted_master_key; + + mtx::events::msg::SASMethods method = mtx::events::msg::SASMethods::Emoji; + QTimer *timeout = nullptr; + sas_ptr sas; + std::string mac_method; + std::string commitment; + nlohmann::json canonical_json; + + std::vector sasList; + UserKeyCache their_keys; + TimelineModel *model_; + mtx::common::Relation relation; + + State state_ = PromptStartVerification; + Error error_ = UnknownMethod; + + bool isMacVerified = false; + + template + void send(T msg) + { + if (this->type == DeviceVerificationFlow::Type::ToDevice) { + mtx::requests::ToDeviceMessages body; + msg.transaction_id = this->transaction_id; + body[this->toClient][deviceId.toStdString()] = msg; + + http::client()->send_to_device( + this->transaction_id, body, [](mtx::http::RequestErr err) { + if (err) + nhlog::net()->warn("failed to send verification to_device message: {} {}", + err->matrix_error.error, + static_cast(err->status_code)); + }); + } else if (this->type == DeviceVerificationFlow::Type::RoomMsg && model_) { + if constexpr (!std::is_same_v) { + msg.relations.relations.push_back(this->relation); + // Set synthesized to surpress the nheko relation extensions + msg.relations.synthesized = true; + } + (model_)->sendMessageEvent(msg, mtx::events::to_device_content_to_type); + } + + nhlog::net()->debug("Sent verification step: {} in state: {}", + mtx::events::to_string(mtx::events::to_device_content_to_type), + state().toStdString()); + } +}; diff --git a/src/encryption/Olm.cpp b/src/encryption/Olm.cpp new file mode 100644 index 00000000..14c97984 --- /dev/null +++ b/src/encryption/Olm.cpp @@ -0,0 +1,1612 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "Olm.h" + +#include +#include + +#include +#include + +#include +#include + +#include "Cache.h" +#include "Cache_p.h" +#include "ChatPage.h" +#include "DeviceVerificationFlow.h" +#include "Logging.h" +#include "MatrixClient.h" +#include "UserSettingsPage.h" +#include "Utils.h" + +namespace { +auto client_ = std::make_unique(); + +std::map request_id_to_secret_name; + +constexpr auto MEGOLM_ALGO = "m.megolm.v1.aes-sha2"; +} + +namespace olm { +static void +backup_session_key(const MegolmSessionIndex &idx, + const GroupSessionData &data, + mtx::crypto::InboundGroupSessionPtr &session); + +void +from_json(const nlohmann::json &obj, OlmMessage &msg) +{ + if (obj.at("type") != "m.room.encrypted") + throw std::invalid_argument("invalid type for olm message"); + + if (obj.at("content").at("algorithm") != OLM_ALGO) + throw std::invalid_argument("invalid algorithm for olm message"); + + msg.sender = obj.at("sender"); + msg.sender_key = obj.at("content").at("sender_key"); + msg.ciphertext = obj.at("content") + .at("ciphertext") + .get>(); +} + +mtx::crypto::OlmClient * +client() +{ + return client_.get(); +} + +static void +handle_secret_request(const mtx::events::DeviceEvent *e, + const std::string &sender) +{ + using namespace mtx::events; + + if (e->content.action != mtx::events::msg::RequestAction::Request) + return; + + auto local_user = http::client()->user_id(); + + if (sender != local_user.to_string()) + return; + + auto verificationStatus = cache::verificationStatus(local_user.to_string()); + + if (!verificationStatus) + return; + + auto deviceKeys = cache::userKeys(local_user.to_string()); + if (!deviceKeys) + return; + + if (std::find(verificationStatus->verified_devices.begin(), + verificationStatus->verified_devices.end(), + e->content.requesting_device_id) == verificationStatus->verified_devices.end()) + return; + + // this is a verified device + mtx::events::DeviceEvent secretSend; + secretSend.type = EventType::SecretSend; + secretSend.content.request_id = e->content.request_id; + + auto secret = cache::client()->secret(e->content.name); + if (!secret) + return; + secretSend.content.secret = secret.value(); + + send_encrypted_to_device_messages( + {{local_user.to_string(), {{e->content.requesting_device_id}}}}, secretSend); + + nhlog::net()->info("Sent secret '{}' to ({},{})", + e->content.name, + local_user.to_string(), + e->content.requesting_device_id); +} + +void +handle_to_device_messages(const std::vector &msgs) +{ + if (msgs.empty()) + return; + nhlog::crypto()->info("received {} to_device messages", msgs.size()); + nlohmann::json j_msg; + + for (const auto &msg : msgs) { + j_msg = std::visit([](auto &e) { return json(e); }, std::move(msg)); + if (j_msg.count("type") == 0) { + nhlog::crypto()->warn("received message with no type field: {}", j_msg.dump(2)); + continue; + } + + std::string msg_type = j_msg.at("type"); + + if (msg_type == to_string(mtx::events::EventType::RoomEncrypted)) { + try { + olm::OlmMessage olm_msg = j_msg; + cache::client()->query_keys( + olm_msg.sender, [olm_msg](const UserKeyCache &userKeys, mtx::http::RequestErr e) { + if (e) { + nhlog::crypto()->error("Failed to query user keys, dropping olm " + "message"); + return; + } + handle_olm_message(std::move(olm_msg), userKeys); + }); + } catch (const nlohmann::json::exception &e) { + nhlog::crypto()->warn( + "parsing error for olm message: {} {}", e.what(), j_msg.dump(2)); + } catch (const std::invalid_argument &e) { + nhlog::crypto()->warn( + "validation error for olm message: {} {}", e.what(), j_msg.dump(2)); + } + + } else if (msg_type == to_string(mtx::events::EventType::RoomKeyRequest)) { + nhlog::crypto()->warn("handling key request event: {}", j_msg.dump(2)); + try { + mtx::events::DeviceEvent req = j_msg; + if (req.content.action == mtx::events::msg::RequestAction::Request) + handle_key_request_message(req); + else + nhlog::crypto()->warn("ignore key request (unhandled action): {}", + req.content.request_id); + } catch (const nlohmann::json::exception &e) { + nhlog::crypto()->warn( + "parsing error for key_request message: {} {}", e.what(), j_msg.dump(2)); + } + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationAccept)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationAccept(message.content); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationRequest)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationRequest(message.content, + message.sender); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationCancel)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationCancel(message.content); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationKey)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationKey(message.content); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationMac)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationMac(message.content); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationStart)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationStart(message.content, message.sender); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationReady)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationReady(message.content); + } else if (msg_type == to_string(mtx::events::EventType::KeyVerificationDone)) { + auto message = + std::get>(msg); + ChatPage::instance()->receivedDeviceVerificationDone(message.content); + } else if (auto e = + std::get_if>(&msg)) { + handle_secret_request(e, e->sender); + } else { + nhlog::crypto()->warn("unhandled event: {}", j_msg.dump(2)); + } + } +} + +void +handle_olm_message(const OlmMessage &msg, const UserKeyCache &otherUserDeviceKeys) +{ + nhlog::crypto()->info("sender : {}", msg.sender); + nhlog::crypto()->info("sender_key: {}", msg.sender_key); + + if (msg.sender_key == olm::client()->identity_keys().ed25519) { + nhlog::crypto()->warn("Ignoring olm message from ourselves!"); + return; + } + + const auto my_key = olm::client()->identity_keys().curve25519; + + bool failed_decryption = false; + + for (const auto &cipher : msg.ciphertext) { + // We skip messages not meant for the current device. + if (cipher.first != my_key) { + nhlog::crypto()->debug( + "Skipping message for {} since we are {}.", cipher.first, my_key); + continue; + } + + const auto type = cipher.second.type; + nhlog::crypto()->info("type: {}", type == 0 ? "OLM_PRE_KEY" : "OLM_MESSAGE"); + + auto payload = try_olm_decryption(msg.sender_key, cipher.second); + + if (payload.is_null()) { + // Check for PRE_KEY message + if (cipher.second.type == 0) { + payload = handle_pre_key_olm_message(msg.sender, msg.sender_key, cipher.second); + } else { + nhlog::crypto()->error("Undecryptable olm message!"); + failed_decryption = true; + continue; + } + } + + if (!payload.is_null()) { + mtx::events::collections::DeviceEvents device_event; + + // Other properties are included in order to prevent an attacker from + // publishing someone else's curve25519 keys as their own and subsequently + // claiming to have sent messages which they didn't. sender must correspond + // to the user who sent the event, recipient to the local user, and + // recipient_keys to the local ed25519 key. + std::string receiver_ed25519 = payload["recipient_keys"]["ed25519"]; + if (receiver_ed25519.empty() || + receiver_ed25519 != olm::client()->identity_keys().ed25519) { + nhlog::crypto()->warn("Decrypted event doesn't include our ed25519: {}", + payload.dump()); + return; + } + std::string receiver = payload["recipient"]; + if (receiver.empty() || receiver != http::client()->user_id().to_string()) { + nhlog::crypto()->warn("Decrypted event doesn't include our user_id: {}", + payload.dump()); + return; + } + + // Clients must confirm that the sender_key and the ed25519 field value + // under the keys property match the keys returned by /keys/query for the + // given user, and must also verify the signature of the payload. Without + // this check, a client cannot be sure that the sender device owns the + // private part of the ed25519 key it claims to have in the Olm payload. + // This is crucial when the ed25519 key corresponds to a verified device. + std::string sender_ed25519 = payload["keys"]["ed25519"]; + if (sender_ed25519.empty()) { + nhlog::crypto()->warn("Decrypted event doesn't include sender ed25519: {}", + payload.dump()); + return; + } + + bool from_their_device = false; + for (auto [device_id, key] : otherUserDeviceKeys.device_keys) { + auto c_key = key.keys.find("curve25519:" + device_id); + auto e_key = key.keys.find("ed25519:" + device_id); + + if (c_key == key.keys.end() || e_key == key.keys.end()) { + nhlog::crypto()->warn("Skipping device {} as we have no keys for it.", + device_id); + } else if (c_key->second == msg.sender_key && e_key->second == sender_ed25519) { + from_their_device = true; + break; + } + } + if (!from_their_device) { + nhlog::crypto()->warn("Decrypted event isn't sent from a device " + "listed by that user! {}", + payload.dump()); + return; + } + + { + std::string msg_type = payload["type"]; + json event_array = json::array(); + event_array.push_back(payload); + + std::vector temp_events; + mtx::responses::utils::parse_device_events(event_array, temp_events); + if (temp_events.empty()) { + nhlog::crypto()->warn("Decrypted unknown event: {}", payload.dump()); + return; + } + device_event = temp_events.at(0); + } + + using namespace mtx::events; + if (auto e1 = std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationAccept(e1->content); + } else if (auto e2 = + std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationRequest(e2->content, e2->sender); + } else if (auto e3 = + std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationCancel(e3->content); + } else if (auto e4 = std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationKey(e4->content); + } else if (auto e5 = std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationMac(e5->content); + } else if (auto e6 = + std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationStart(e6->content, e6->sender); + } else if (auto e7 = + std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationReady(e7->content); + } else if (auto e8 = + std::get_if>(&device_event)) { + ChatPage::instance()->receivedDeviceVerificationDone(e8->content); + } else if (auto roomKey = std::get_if>(&device_event)) { + create_inbound_megolm_session(*roomKey, msg.sender_key, sender_ed25519); + } else if (auto forwardedRoomKey = + std::get_if>(&device_event)) { + forwardedRoomKey->content.forwarding_curve25519_key_chain.push_back(msg.sender_key); + import_inbound_megolm_session(*forwardedRoomKey); + } else if (auto e = std::get_if>(&device_event)) { + auto local_user = http::client()->user_id(); + + if (msg.sender != local_user.to_string()) + return; + + auto secret_name = request_id_to_secret_name.find(e->content.request_id); + + if (secret_name != request_id_to_secret_name.end()) { + nhlog::crypto()->info("Received secret: {}", secret_name->second); + + mtx::events::msg::SecretRequest secretRequest{}; + secretRequest.action = mtx::events::msg::RequestAction::Cancellation; + secretRequest.requesting_device_id = http::client()->device_id(); + secretRequest.request_id = e->content.request_id; + + auto verificationStatus = cache::verificationStatus(local_user.to_string()); + + if (!verificationStatus) + return; + + auto deviceKeys = cache::userKeys(local_user.to_string()); + std::string sender_device_id; + if (deviceKeys) { + for (auto &[dev, key] : deviceKeys->device_keys) { + if (key.keys["curve25519:" + dev] == msg.sender_key) { + sender_device_id = dev; + break; + } + } + } + + std::map> + body; + + for (const auto &dev : verificationStatus->verified_devices) { + if (dev != secretRequest.requesting_device_id && dev != sender_device_id) + body[local_user][dev] = secretRequest; + } + + http::client()->send_to_device( + http::client()->generate_txn_id(), + body, + [name = secret_name->second](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("Failed to send request cancellation " + "for secrect " + "'{}'", + name); + } + }); + + nhlog::crypto()->info("Storing secret {}", secret_name->second); + cache::client()->storeSecret(secret_name->second, e->content.secret); + + request_id_to_secret_name.erase(secret_name); + } + + } else if (auto sec_req = std::get_if>(&device_event)) { + handle_secret_request(sec_req, msg.sender); + } + + return; + } else { + failed_decryption = true; + } + } + + if (failed_decryption) { + try { + std::map> targets; + for (auto [device_id, key] : otherUserDeviceKeys.device_keys) { + if (key.keys.at("curve25519:" + device_id) == msg.sender_key) + targets[msg.sender].push_back(device_id); + } + + send_encrypted_to_device_messages( + targets, mtx::events::DeviceEvent{}, true); + nhlog::crypto()->info( + "Recovering from broken olm channel with {}:{}", msg.sender, msg.sender_key); + } catch (std::exception &e) { + nhlog::crypto()->error("Failed to recover from broken olm sessions: {}", e.what()); + } + } +} + +nlohmann::json +handle_pre_key_olm_message(const std::string &sender, + const std::string &sender_key, + const mtx::events::msg::OlmCipherContent &content) +{ + nhlog::crypto()->info("opening olm session with {}", sender); + + mtx::crypto::OlmSessionPtr inbound_session = nullptr; + try { + inbound_session = olm::client()->create_inbound_session_from(sender_key, content.body); + + // We also remove the one time key used to establish that + // session so we'll have to update our copy of the account object. + cache::saveOlmAccount(olm::client()->save(cache::client()->pickleSecret())); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to create inbound session with {}: {}", sender, e.what()); + return {}; + } + + if (!mtx::crypto::matches_inbound_session_from( + inbound_session.get(), sender_key, content.body)) { + nhlog::crypto()->warn("inbound olm session doesn't match sender's key ({})", sender); + return {}; + } + + mtx::crypto::BinaryBuf output; + try { + output = olm::client()->decrypt_message(inbound_session.get(), content.type, content.body); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to decrypt olm message {}: {}", content.body, e.what()); + return {}; + } + + auto plaintext = json::parse(std::string((char *)output.data(), output.size())); + nhlog::crypto()->debug("decrypted message: \n {}", plaintext.dump(2)); + + try { + nhlog::crypto()->debug("New olm session: {}", + mtx::crypto::session_id(inbound_session.get())); + cache::saveOlmSession( + sender_key, std::move(inbound_session), QDateTime::currentMSecsSinceEpoch()); + } catch (const lmdb::error &e) { + nhlog::db()->warn("failed to save inbound olm session from {}: {}", sender, e.what()); + } + + return plaintext; +} + +mtx::events::msg::Encrypted +encrypt_group_message(const std::string &room_id, const std::string &device_id, nlohmann::json body) +{ + using namespace mtx::events; + using namespace mtx::identifiers; + + auto own_user_id = http::client()->user_id().to_string(); + + auto members = cache::client()->getMembersWithKeys( + room_id, UserSettings::instance()->onlyShareKeysWithVerifiedUsers()); + + std::map> sendSessionTo; + mtx::crypto::OutboundGroupSessionPtr session = nullptr; + GroupSessionData group_session_data; + + if (cache::outboundMegolmSessionExists(room_id)) { + auto res = cache::getOutboundMegolmSession(room_id); + auto encryptionSettings = cache::client()->roomEncryptionSettings(room_id); + mtx::events::state::Encryption defaultSettings; + + // rotate if we crossed the limits for this key + if (res.data.message_index < + encryptionSettings.value_or(defaultSettings).rotation_period_msgs && + (QDateTime::currentMSecsSinceEpoch() - res.data.timestamp) < + encryptionSettings.value_or(defaultSettings).rotation_period_ms) { + auto member_it = members.begin(); + auto session_member_it = res.data.currently.keys.begin(); + auto session_member_it_end = res.data.currently.keys.end(); + + while (member_it != members.end() || session_member_it != session_member_it_end) { + if (member_it == members.end()) { + // a member left, purge session! + nhlog::crypto()->debug("Rotating megolm session because of left member"); + break; + } + + if (session_member_it == session_member_it_end) { + // share with all remaining members + while (member_it != members.end()) { + sendSessionTo[member_it->first] = {}; + + if (member_it->second) + for (const auto &dev : member_it->second->device_keys) + if (member_it->first != own_user_id || dev.first != device_id) + sendSessionTo[member_it->first].push_back(dev.first); + + ++member_it; + } + + session = std::move(res.session); + break; + } + + if (member_it->first > session_member_it->first) { + // a member left, purge session + nhlog::crypto()->debug("Rotating megolm session because of left member"); + break; + } else if (member_it->first < session_member_it->first) { + // new member, send them the session at this index + sendSessionTo[member_it->first] = {}; + + if (member_it->second) { + for (const auto &dev : member_it->second->device_keys) + if (member_it->first != own_user_id || dev.first != device_id) + sendSessionTo[member_it->first].push_back(dev.first); + } + + ++member_it; + } else { + // compare devices + bool device_removed = false; + for (const auto &dev : session_member_it->second.deviceids) { + if (!member_it->second || + !member_it->second->device_keys.count(dev.first)) { + device_removed = true; + break; + } + } + + if (device_removed) { + // device removed, rotate session! + nhlog::crypto()->debug("Rotating megolm session because of removed " + "device of {}", + member_it->first); + break; + } + + // check for new devices to share with + if (member_it->second) + for (const auto &dev : member_it->second->device_keys) + if (!session_member_it->second.deviceids.count(dev.first) && + (member_it->first != own_user_id || dev.first != device_id)) + sendSessionTo[member_it->first].push_back(dev.first); + + ++member_it; + ++session_member_it; + if (member_it == members.end() && session_member_it == session_member_it_end) { + // all devices match or are newly added + session = std::move(res.session); + } + } + } + } + + group_session_data = std::move(res.data); + } + + if (!session) { + nhlog::ui()->debug("creating new outbound megolm session"); + + // Create a new outbound megolm session. + session = olm::client()->init_outbound_group_session(); + const auto session_id = mtx::crypto::session_id(session.get()); + const auto session_key = mtx::crypto::session_key(session.get()); + + // Saving the new megolm session. + GroupSessionData session_data{}; + session_data.message_index = 0; + session_data.timestamp = QDateTime::currentMSecsSinceEpoch(); + session_data.sender_claimed_ed25519_key = olm::client()->identity_keys().ed25519; + + sendSessionTo.clear(); + + for (const auto &[user, devices] : members) { + sendSessionTo[user] = {}; + session_data.currently.keys[user] = {}; + if (devices) { + for (const auto &[device_id_, key] : devices->device_keys) { + (void)key; + if (device_id != device_id_ || user != own_user_id) { + sendSessionTo[user].push_back(device_id_); + session_data.currently.keys[user].deviceids[device_id_] = 0; + } + } + } + } + + { + MegolmSessionIndex index; + index.room_id = room_id; + index.session_id = session_id; + index.sender_key = olm::client()->identity_keys().curve25519; + auto megolm_session = olm::client()->init_inbound_group_session(session_key); + backup_session_key(index, session_data, megolm_session); + cache::saveInboundMegolmSession(index, std::move(megolm_session), session_data); + } + + cache::saveOutboundMegolmSession(room_id, session_data, session); + group_session_data = std::move(session_data); + } + + mtx::events::DeviceEvent megolm_payload{}; + megolm_payload.content.algorithm = MEGOLM_ALGO; + megolm_payload.content.room_id = room_id; + megolm_payload.content.session_id = mtx::crypto::session_id(session.get()); + megolm_payload.content.session_key = mtx::crypto::session_key(session.get()); + megolm_payload.type = mtx::events::EventType::RoomKey; + + if (!sendSessionTo.empty()) + olm::send_encrypted_to_device_messages(sendSessionTo, megolm_payload); + + // relations shouldn't be encrypted... + mtx::common::Relations relations = mtx::common::parse_relations(body["content"]); + + auto payload = olm::client()->encrypt_group_message(session.get(), body.dump()); + + // Prepare the m.room.encrypted event. + msg::Encrypted data; + data.ciphertext = std::string((char *)payload.data(), payload.size()); + data.sender_key = olm::client()->identity_keys().curve25519; + data.session_id = mtx::crypto::session_id(session.get()); + data.device_id = device_id; + data.algorithm = MEGOLM_ALGO; + data.relations = relations; + + group_session_data.message_index = olm_outbound_group_session_message_index(session.get()); + nhlog::crypto()->debug("next message_index {}", group_session_data.message_index); + + // update current set of members for the session with the new members and that message_index + for (const auto &[user, devices] : sendSessionTo) { + if (!group_session_data.currently.keys.count(user)) + group_session_data.currently.keys[user] = {}; + + for (const auto &device_id_ : devices) { + if (!group_session_data.currently.keys[user].deviceids.count(device_id_)) + group_session_data.currently.keys[user].deviceids[device_id_] = + group_session_data.message_index; + } + } + + // We need to re-pickle the session after we send a message to save the new message_index. + cache::updateOutboundMegolmSession(room_id, group_session_data, session); + + return data; +} + +nlohmann::json +try_olm_decryption(const std::string &sender_key, const mtx::events::msg::OlmCipherContent &msg) +{ + auto session_ids = cache::getOlmSessions(sender_key); + + nhlog::crypto()->info("attempt to decrypt message with {} known session_ids", + session_ids.size()); + + for (const auto &id : session_ids) { + auto session = cache::getOlmSession(sender_key, id); + + if (!session) { + nhlog::crypto()->warn("Unknown olm session: {}:{}", sender_key, id); + continue; + } + + mtx::crypto::BinaryBuf text; + + try { + text = olm::client()->decrypt_message(session->get(), msg.type, msg.body); + nhlog::crypto()->debug("Updated olm session: {}", + mtx::crypto::session_id(session->get())); + cache::saveOlmSession( + id, std::move(session.value()), QDateTime::currentMSecsSinceEpoch()); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->debug("failed to decrypt olm message ({}, {}) with {}: {}", + msg.type, + sender_key, + id, + e.what()); + continue; + } catch (const lmdb::error &e) { + nhlog::crypto()->critical("failed to save session: {}", e.what()); + return {}; + } + + try { + return json::parse(std::string_view((char *)text.data(), text.size())); + } catch (const json::exception &e) { + nhlog::crypto()->critical("failed to parse the decrypted session msg: {} {}", + e.what(), + std::string_view((char *)text.data(), text.size())); + } + } + + return {}; +} + +void +create_inbound_megolm_session(const mtx::events::DeviceEvent &roomKey, + const std::string &sender_key, + const std::string &sender_ed25519) +{ + MegolmSessionIndex index; + index.room_id = roomKey.content.room_id; + index.session_id = roomKey.content.session_id; + index.sender_key = sender_key; + + try { + GroupSessionData data{}; + data.forwarding_curve25519_key_chain = {sender_key}; + data.sender_claimed_ed25519_key = sender_ed25519; + + auto megolm_session = + olm::client()->init_inbound_group_session(roomKey.content.session_key); + backup_session_key(index, data, megolm_session); + cache::saveInboundMegolmSession(index, std::move(megolm_session), data); + } catch (const lmdb::error &e) { + nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); + return; + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to create inbound megolm session: {}", e.what()); + return; + } + + nhlog::crypto()->info( + "established inbound megolm session ({}, {})", roomKey.content.room_id, roomKey.sender); + + ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); +} + +void +import_inbound_megolm_session( + const mtx::events::DeviceEvent &roomKey) +{ + MegolmSessionIndex index; + index.room_id = roomKey.content.room_id; + index.session_id = roomKey.content.session_id; + index.sender_key = roomKey.content.sender_key; + + try { + auto megolm_session = + olm::client()->import_inbound_group_session(roomKey.content.session_key); + + GroupSessionData data{}; + data.forwarding_curve25519_key_chain = roomKey.content.forwarding_curve25519_key_chain; + data.sender_claimed_ed25519_key = roomKey.content.sender_claimed_ed25519_key; + // may have come from online key backup, so we can't trust it... + data.trusted = false; + // if we got it forwarded from the sender, assume it is trusted. They may still have + // used key backup, but it is unlikely. + if (roomKey.content.forwarding_curve25519_key_chain.size() == 1 && + roomKey.content.forwarding_curve25519_key_chain.back() == roomKey.content.sender_key) { + data.trusted = true; + } + + backup_session_key(index, data, megolm_session); + cache::saveInboundMegolmSession(index, std::move(megolm_session), data); + } catch (const lmdb::error &e) { + nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); + return; + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to import inbound megolm session: {}", e.what()); + return; + } + + nhlog::crypto()->info( + "established inbound megolm session ({}, {})", roomKey.content.room_id, roomKey.sender); + + ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); +} + +void +backup_session_key(const MegolmSessionIndex &idx, + const GroupSessionData &data, + mtx::crypto::InboundGroupSessionPtr &session) +{ + try { + if (!UserSettings::instance()->useOnlineKeyBackup()) { + // Online key backup disabled + return; + } + + auto backupVersion = cache::client()->backupVersion(); + if (!backupVersion) { + // no trusted OKB + return; + } + + using namespace mtx::crypto; + + auto decryptedSecret = cache::secret(mtx::secret_storage::secrets::megolm_backup_v1); + if (!decryptedSecret) { + // no backup key available + return; + } + auto sessionDecryptionKey = to_binary_buf(base642bin(*decryptedSecret)); + + auto public_key = mtx::crypto::CURVE25519_public_key_from_private(sessionDecryptionKey); + + mtx::responses::backup::SessionData sessionData; + sessionData.algorithm = mtx::crypto::MEGOLM_ALGO; + sessionData.forwarding_curve25519_key_chain = data.forwarding_curve25519_key_chain; + sessionData.sender_claimed_keys["ed25519"] = data.sender_claimed_ed25519_key; + sessionData.sender_key = idx.sender_key; + sessionData.session_key = mtx::crypto::export_session(session.get(), -1); + + auto encrypt_session = mtx::crypto::encrypt_session(sessionData, public_key); + + mtx::responses::backup::SessionBackup bk; + bk.first_message_index = olm_inbound_group_session_first_known_index(session.get()); + bk.forwarded_count = data.forwarding_curve25519_key_chain.size(); + bk.is_verified = false; + bk.session_data = std::move(encrypt_session); + + http::client()->put_room_keys( + backupVersion->version, + idx.room_id, + idx.session_id, + bk, + [idx](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to backup session key ({}:{}): {} ({})", + idx.room_id, + idx.session_id, + err->matrix_error.error, + static_cast(err->status_code)); + } else { + nhlog::crypto()->debug( + "backed up session key ({}:{})", idx.room_id, idx.session_id); + } + }); + } catch (std::exception &e) { + nhlog::net()->warn("failed to backup session key: {}", e.what()); + } +} + +void +mark_keys_as_published() +{ + olm::client()->mark_keys_as_published(); + cache::saveOlmAccount(olm::client()->save(cache::client()->pickleSecret())); +} + +void +lookup_keybackup(const std::string room, const std::string session_id) +{ + if (!UserSettings::instance()->useOnlineKeyBackup()) { + // Online key backup disabled + return; + } + + auto backupVersion = cache::client()->backupVersion(); + if (!backupVersion) { + // no trusted OKB + return; + } + + using namespace mtx::crypto; + + auto decryptedSecret = cache::secret(mtx::secret_storage::secrets::megolm_backup_v1); + if (!decryptedSecret) { + // no backup key available + return; + } + auto sessionDecryptionKey = to_binary_buf(base642bin(*decryptedSecret)); + + http::client()->room_keys( + backupVersion->version, + room, + session_id, + [room, session_id, sessionDecryptionKey](const mtx::responses::backup::SessionBackup &bk, + mtx::http::RequestErr err) { + if (err) { + if (err->status_code != 404) + nhlog::crypto()->error("Failed to dowload key {}:{}: {} - {}", + room, + session_id, + mtx::errors::to_string(err->matrix_error.errcode), + err->matrix_error.error); + return; + } + try { + auto session = decrypt_session(bk.session_data, sessionDecryptionKey); + + if (session.algorithm != mtx::crypto::MEGOLM_ALGO) + // don't know this algorithm + return; + + MegolmSessionIndex index; + index.room_id = room; + index.session_id = session_id; + index.sender_key = session.sender_key; + + GroupSessionData data{}; + data.forwarding_curve25519_key_chain = session.forwarding_curve25519_key_chain; + data.sender_claimed_ed25519_key = session.sender_claimed_keys["ed25519"]; + // online key backup can't be trusted, because anyone can upload to it. + data.trusted = false; + + auto megolm_session = + olm::client()->import_inbound_group_session(session.session_key); + + if (!cache::inboundMegolmSessionExists(index) || + olm_inbound_group_session_first_known_index(megolm_session.get()) < + olm_inbound_group_session_first_known_index( + cache::getInboundMegolmSession(index).get())) { + cache::saveInboundMegolmSession(index, std::move(megolm_session), data); + + nhlog::crypto()->info("imported inbound megolm session " + "from key backup ({}, {})", + room, + session_id); + + // call on UI thread + QTimer::singleShot(0, ChatPage::instance(), [index] { + ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id); + }); + } + } catch (const lmdb::error &e) { + nhlog::crypto()->critical("failed to save inbound megolm session: {}", e.what()); + return; + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to import inbound megolm session: {}", e.what()); + return; + } + }); +} + +void +send_key_request_for(mtx::events::EncryptedEvent e, + const std::string &request_id, + bool cancel) +{ + using namespace mtx::events; + + nhlog::crypto()->debug("sending key request: sender_key {}, session_id {}", + e.content.sender_key, + e.content.session_id); + + mtx::events::msg::KeyRequest request; + request.action = cancel ? mtx::events::msg::RequestAction::Cancellation + : mtx::events::msg::RequestAction::Request; + + request.algorithm = MEGOLM_ALGO; + request.room_id = e.room_id; + request.sender_key = e.content.sender_key; + request.session_id = e.content.session_id; + request.request_id = request_id; + request.requesting_device_id = http::client()->device_id(); + + nhlog::crypto()->debug("m.room_key_request: {}", json(request).dump(2)); + + std::map> body; + body[mtx::identifiers::parse(e.sender)][e.content.device_id] = request; + body[http::client()->user_id()]["*"] = request; + + http::client()->send_to_device( + http::client()->generate_txn_id(), body, [e](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to send " + "send_to_device " + "message: {}", + err->matrix_error.error); + } + + nhlog::net()->info( + "m.room_key_request sent to {}:{} and your own devices", e.sender, e.content.device_id); + }); + + // http::client()->room_keys +} + +void +handle_key_request_message(const mtx::events::DeviceEvent &req) +{ + if (req.content.algorithm != MEGOLM_ALGO) { + nhlog::crypto()->debug("ignoring key request {} with invalid algorithm: {}", + req.content.request_id, + req.content.algorithm); + return; + } + + // Check if we were the sender of the session being requested (unless it is actually us + // requesting the session). + if (req.sender != http::client()->user_id().to_string() && + req.content.sender_key != olm::client()->identity_keys().curve25519) { + nhlog::crypto()->debug( + "ignoring key request {} because we did not create the requested session: " + "\nrequested({}) ours({})", + req.content.request_id, + req.content.sender_key, + olm::client()->identity_keys().curve25519); + return; + } + + // Check that the requested session_id and the one we have saved match. + MegolmSessionIndex index{}; + index.room_id = req.content.room_id; + index.session_id = req.content.session_id; + index.sender_key = req.content.sender_key; + + // Check if we have the keys for the requested session. + auto sessionData = cache::getMegolmSessionData(index); + if (!sessionData) { + nhlog::crypto()->warn("requested session not found in room: {}", req.content.room_id); + return; + } + + const auto session = cache::getInboundMegolmSession(index); + if (!session) { + nhlog::crypto()->warn("No session with id {} in db", req.content.session_id); + return; + } + + if (!cache::isRoomMember(req.sender, req.content.room_id)) { + nhlog::crypto()->warn("user {} that requested the session key is not member of the room {}", + req.sender, + req.content.room_id); + return; + } + + // check if device is verified + auto verificationStatus = cache::verificationStatus(req.sender); + bool verifiedDevice = false; + if (verificationStatus && + // Share keys, if the option to share with trusted users is enabled or with yourself + (ChatPage::instance()->userSettings()->shareKeysWithTrustedUsers() || + req.sender == http::client()->user_id().to_string())) { + for (const auto &dev : verificationStatus->verified_devices) { + if (dev == req.content.requesting_device_id) { + verifiedDevice = true; + nhlog::crypto()->debug("Verified device: {}", dev); + break; + } + } + } + + bool shouldSeeKeys = false; + uint64_t minimumIndex = -1; + if (sessionData->currently.keys.count(req.sender)) { + if (sessionData->currently.keys.at(req.sender) + .deviceids.count(req.content.requesting_device_id)) { + shouldSeeKeys = true; + minimumIndex = sessionData->currently.keys.at(req.sender) + .deviceids.at(req.content.requesting_device_id); + } + } + + if (!verifiedDevice && !shouldSeeKeys) { + nhlog::crypto()->debug("ignoring key request for room {}", req.content.room_id); + return; + } + + if (verifiedDevice) { + // share the minimum index we have + minimumIndex = -1; + } + + try { + auto session_key = mtx::crypto::export_session(session.get(), minimumIndex); + + // + // Prepare the m.room_key event. + // + mtx::events::msg::ForwardedRoomKey forward_key{}; + forward_key.algorithm = MEGOLM_ALGO; + forward_key.room_id = index.room_id; + forward_key.session_id = index.session_id; + forward_key.session_key = session_key; + forward_key.sender_key = index.sender_key; + + // TODO(Nico): Figure out if this is correct + forward_key.sender_claimed_ed25519_key = sessionData->sender_claimed_ed25519_key; + forward_key.forwarding_curve25519_key_chain = sessionData->forwarding_curve25519_key_chain; + + send_megolm_key_to_device(req.sender, req.content.requesting_device_id, forward_key); + } catch (std::exception &e) { + nhlog::crypto()->error("Failed to forward session key: {}", e.what()); + } +} + +void +send_megolm_key_to_device(const std::string &user_id, + const std::string &device_id, + const mtx::events::msg::ForwardedRoomKey &payload) +{ + mtx::events::DeviceEvent room_key; + room_key.content = payload; + room_key.type = mtx::events::EventType::ForwardedRoomKey; + + std::map> targets; + targets[user_id] = {device_id}; + send_encrypted_to_device_messages(targets, room_key); + nhlog::crypto()->debug("Forwarded key to {}:{}", user_id, device_id); +} + +DecryptionResult +decryptEvent(const MegolmSessionIndex &index, + const mtx::events::EncryptedEvent &event, + bool dont_write_db) +{ + try { + if (!cache::client()->inboundMegolmSessionExists(index)) { + return {DecryptionErrorCode::MissingSession, std::nullopt, std::nullopt}; + } + } catch (const lmdb::error &e) { + return {DecryptionErrorCode::DbError, e.what(), std::nullopt}; + } + + // TODO: Lookup index,event_id,origin_server_ts tuple for replay attack errors + + std::string msg_str; + try { + auto session = cache::client()->getInboundMegolmSession(index); + if (!session) { + return {DecryptionErrorCode::MissingSession, std::nullopt, std::nullopt}; + } + + auto sessionData = + cache::client()->getMegolmSessionData(index).value_or(GroupSessionData{}); + + auto res = olm::client()->decrypt_group_message(session.get(), event.content.ciphertext); + msg_str = std::string((char *)res.data.data(), res.data.size()); + + if (!event.event_id.empty() && event.event_id[0] == '$') { + auto oldIdx = sessionData.indices.find(res.message_index); + if (oldIdx != sessionData.indices.end()) { + if (oldIdx->second != event.event_id) + return {DecryptionErrorCode::ReplayAttack, std::nullopt, std::nullopt}; + } else if (!dont_write_db) { + sessionData.indices[res.message_index] = event.event_id; + cache::client()->saveInboundMegolmSession(index, std::move(session), sessionData); + } + } + } catch (const lmdb::error &e) { + return {DecryptionErrorCode::DbError, e.what(), std::nullopt}; + } catch (const mtx::crypto::olm_exception &e) { + if (e.error_code() == mtx::crypto::OlmErrorCode::UNKNOWN_MESSAGE_INDEX) + return {DecryptionErrorCode::MissingSessionIndex, e.what(), std::nullopt}; + return {DecryptionErrorCode::DecryptionFailed, e.what(), std::nullopt}; + } + + try { + // Add missing fields for the event. + json body = json::parse(msg_str); + body["event_id"] = event.event_id; + body["sender"] = event.sender; + body["origin_server_ts"] = event.origin_server_ts; + body["unsigned"] = event.unsigned_data; + + // relations are unencrypted in content... + mtx::common::add_relations(body["content"], event.content.relations); + + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json(body, te); + + return {DecryptionErrorCode::NoError, std::nullopt, std::move(te.data)}; + } catch (std::exception &e) { + return {DecryptionErrorCode::ParsingFailed, e.what(), std::nullopt}; + } +} + +crypto::Trust +calculate_trust(const std::string &user_id, const MegolmSessionIndex &index) +{ + auto status = cache::client()->verificationStatus(user_id); + auto megolmData = cache::client()->getMegolmSessionData(index); + crypto::Trust trustlevel = crypto::Trust::Unverified; + + if (megolmData && megolmData->trusted && status.verified_device_keys.count(index.sender_key)) + trustlevel = status.verified_device_keys.at(index.sender_key); + + return trustlevel; +} + +//! Send encrypted to device messages, targets is a map from userid to device ids or {} for all +//! devices +void +send_encrypted_to_device_messages(const std::map> targets, + const mtx::events::collections::DeviceEvents &event, + bool force_new_session) +{ + static QMap, qint64> rateLimit; + + nlohmann::json ev_json = std::visit([](const auto &e) { return json(e); }, event); + + std::map> keysToQuery; + mtx::requests::ClaimKeys claims; + std::map> + messages; + std::map> pks; + + auto our_curve = olm::client()->identity_keys().curve25519; + + for (const auto &[user, devices] : targets) { + auto deviceKeys = cache::client()->userKeys(user); + + // no keys for user, query them + if (!deviceKeys) { + keysToQuery[user] = devices; + continue; + } + + auto deviceTargets = devices; + if (devices.empty()) { + deviceTargets.clear(); + for (const auto &[device, keys] : deviceKeys->device_keys) { + (void)keys; + deviceTargets.push_back(device); + } + } + + for (const auto &device : deviceTargets) { + if (!deviceKeys->device_keys.count(device)) { + keysToQuery[user] = {}; + break; + } + + auto d = deviceKeys->device_keys.at(device); + + if (!d.keys.count("curve25519:" + device) || !d.keys.count("ed25519:" + device)) { + nhlog::crypto()->warn("Skipping device {} since it has no keys!", device); + continue; + } + + auto device_curve = d.keys.at("curve25519:" + device); + if (device_curve == our_curve) { + nhlog::crypto()->warn("Skipping our own device, since sending " + "ourselves olm messages makes no sense."); + continue; + } + + auto session = cache::getLatestOlmSession(device_curve); + if (!session || force_new_session) { + auto currentTime = QDateTime::currentSecsSinceEpoch(); + if (rateLimit.value(QPair(user, device)) + 60 * 60 * 10 < currentTime) { + claims.one_time_keys[user][device] = mtx::crypto::SIGNED_CURVE25519; + pks[user][device].ed25519 = d.keys.at("ed25519:" + device); + pks[user][device].curve25519 = d.keys.at("curve25519:" + device); + + rateLimit.insert(QPair(user, device), currentTime); + } else { + nhlog::crypto()->warn("Not creating new session with {}:{} " + "because of rate limit", + user, + device); + } + continue; + } + + messages[mtx::identifiers::parse(user)][device] = + olm::client() + ->create_olm_encrypted_content(session->get(), + ev_json, + UserId(user), + d.keys.at("ed25519:" + device), + device_curve) + .get(); + + try { + nhlog::crypto()->debug("Updated olm session: {}", + mtx::crypto::session_id(session->get())); + cache::saveOlmSession(d.keys.at("curve25519:" + device), + std::move(*session), + QDateTime::currentMSecsSinceEpoch()); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to save outbound olm session: {}", e.what()); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to pickle outbound olm session: {}", e.what()); + } + } + } + + if (!messages.empty()) + http::client()->send_to_device( + http::client()->generate_txn_id(), messages, [](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to send " + "send_to_device " + "message: {}", + err->matrix_error.error); + } + }); + + auto BindPks = [ev_json](decltype(pks) pks_temp) { + return [pks = pks_temp, ev_json](const mtx::responses::ClaimKeys &res, + mtx::http::RequestErr) { + std::map> + messages; + for (const auto &[user_id, retrieved_devices] : res.one_time_keys) { + nhlog::net()->debug("claimed keys for {}", user_id); + if (retrieved_devices.size() == 0) { + nhlog::net()->debug("no one-time keys found for user_id: {}", user_id); + continue; + } + + for (const auto &rd : retrieved_devices) { + const auto device_id = rd.first; + + nhlog::net()->debug("{} : \n {}", device_id, rd.second.dump(2)); + + if (rd.second.empty() || !rd.second.begin()->contains("key")) { + nhlog::net()->warn("Skipping device {} as it has no key.", device_id); + continue; + } + + auto otk = rd.second.begin()->at("key"); + + auto sign_key = pks.at(user_id).at(device_id).ed25519; + auto id_key = pks.at(user_id).at(device_id).curve25519; + + // Verify signature + { + auto signedKey = *rd.second.begin(); + std::string signature = + signedKey["signatures"][user_id].value("ed25519:" + device_id, ""); + + if (signature.empty() || !mtx::crypto::ed25519_verify_signature( + sign_key, signedKey, signature)) { + nhlog::net()->warn("Skipping device {} as its one time key " + "has an invalid signature.", + device_id); + continue; + } + } + + auto session = olm::client()->create_outbound_session(id_key, otk); + + messages[mtx::identifiers::parse(user_id)][device_id] = + olm::client() + ->create_olm_encrypted_content( + session.get(), ev_json, UserId(user_id), sign_key, id_key) + .get(); + + try { + nhlog::crypto()->debug("Updated olm session: {}", + mtx::crypto::session_id(session.get())); + cache::saveOlmSession( + id_key, std::move(session), QDateTime::currentMSecsSinceEpoch()); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to save outbound olm session: {}", e.what()); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to pickle outbound olm session: {}", + e.what()); + } + } + nhlog::net()->info("send_to_device: {}", user_id); + } + + if (!messages.empty()) + http::client()->send_to_device( + http::client()->generate_txn_id(), messages, [](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to send " + "send_to_device " + "message: {}", + err->matrix_error.error); + } + }); + }; + }; + + if (!claims.one_time_keys.empty()) + http::client()->claim_keys(claims, BindPks(pks)); + + if (!keysToQuery.empty()) { + mtx::requests::QueryKeys req; + req.device_keys = keysToQuery; + http::client()->query_keys( + req, + [ev_json, BindPks, our_curve](const mtx::responses::QueryKeys &res, + mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to query device keys: {} {}", + err->matrix_error.error, + static_cast(err->status_code)); + return; + } + + nhlog::net()->info("queried keys"); + + cache::client()->updateUserKeys(cache::nextBatchToken(), res); + + mtx::requests::ClaimKeys claim_keys; + + std::map> deviceKeys; + + for (const auto &user : res.device_keys) { + for (const auto &dev : user.second) { + const auto user_id = ::UserId(dev.second.user_id); + const auto device_id = DeviceId(dev.second.device_id); + + if (user_id.get() == http::client()->user_id().to_string() && + device_id.get() == http::client()->device_id()) + continue; + + const auto device_keys = dev.second.keys; + const auto curveKey = "curve25519:" + device_id.get(); + const auto edKey = "ed25519:" + device_id.get(); + + if ((device_keys.find(curveKey) == device_keys.end()) || + (device_keys.find(edKey) == device_keys.end())) { + nhlog::net()->debug("ignoring malformed keys for device {}", + device_id.get()); + continue; + } + + DevicePublicKeys pks; + pks.ed25519 = device_keys.at(edKey); + pks.curve25519 = device_keys.at(curveKey); + + if (pks.curve25519 == our_curve) { + nhlog::crypto()->warn("Skipping our own device, since sending " + "ourselves olm messages makes no sense."); + continue; + } + + try { + if (!mtx::crypto::verify_identity_signature( + dev.second, device_id, user_id)) { + nhlog::crypto()->warn("failed to verify identity keys: {}", + json(dev.second).dump(2)); + continue; + } + } catch (const json::exception &e) { + nhlog::crypto()->warn("failed to parse device key json: {}", e.what()); + continue; + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->warn("failed to verify device key json: {}", e.what()); + continue; + } + + auto currentTime = QDateTime::currentSecsSinceEpoch(); + if (rateLimit.value(QPair(user.first, device_id.get())) + 60 * 60 * 10 < + currentTime) { + deviceKeys[user_id].emplace(device_id, pks); + claim_keys.one_time_keys[user.first][device_id] = + mtx::crypto::SIGNED_CURVE25519; + + rateLimit.insert(QPair(user.first, device_id.get()), currentTime); + } else { + nhlog::crypto()->warn("Not creating new session with {}:{} " + "because of rate limit", + user.first, + device_id.get()); + continue; + } + + nhlog::net()->info("{}", device_id.get()); + nhlog::net()->info(" curve25519 {}", pks.curve25519); + nhlog::net()->info(" ed25519 {}", pks.ed25519); + } + } + + if (!claim_keys.one_time_keys.empty()) + http::client()->claim_keys(claim_keys, BindPks(deviceKeys)); + }); + } +} + +void +request_cross_signing_keys() +{ + mtx::events::msg::SecretRequest secretRequest{}; + secretRequest.action = mtx::events::msg::RequestAction::Request; + secretRequest.requesting_device_id = http::client()->device_id(); + + auto local_user = http::client()->user_id(); + + auto verificationStatus = cache::verificationStatus(local_user.to_string()); + + if (!verificationStatus) + return; + + auto request = [&](std::string secretName) { + secretRequest.name = secretName; + secretRequest.request_id = "ss." + http::client()->generate_txn_id(); + + request_id_to_secret_name[secretRequest.request_id] = secretRequest.name; + + std::map> + body; + + for (const auto &dev : verificationStatus->verified_devices) { + if (dev != secretRequest.requesting_device_id) + body[local_user][dev] = secretRequest; + } + + http::client()->send_to_device( + http::client()->generate_txn_id(), + body, + [request_id = secretRequest.request_id, secretName](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("Failed to send request for secrect '{}'", secretName); + // Cancel request on UI thread + QTimer::singleShot(1, cache::client(), [request_id]() { + request_id_to_secret_name.erase(request_id); + }); + return; + } + }); + + for (const auto &dev : verificationStatus->verified_devices) { + if (dev != secretRequest.requesting_device_id) + body[local_user][dev].action = mtx::events::msg::RequestAction::Cancellation; + } + + // timeout after 15 min + QTimer::singleShot(15 * 60 * 1000, [secretRequest, body]() { + if (request_id_to_secret_name.count(secretRequest.request_id)) { + request_id_to_secret_name.erase(secretRequest.request_id); + http::client()->send_to_device( + http::client()->generate_txn_id(), + body, + [secretRequest](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("Failed to cancel request for secrect '{}'", + secretRequest.name); + return; + } + }); + } + }); + }; + + request(mtx::secret_storage::secrets::cross_signing_self_signing); + request(mtx::secret_storage::secrets::cross_signing_user_signing); + request(mtx::secret_storage::secrets::megolm_backup_v1); +} + +namespace { +void +unlock_secrets(const std::string &key, + const std::map &secrets) +{ + http::client()->secret_storage_key( + key, + [secrets](mtx::secret_storage::AesHmacSha2KeyDescription keyDesc, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("Failed to download secret storage key"); + return; + } + + emit ChatPage::instance()->downloadedSecrets(keyDesc, secrets); + }); +} +} + +void +download_cross_signing_keys() +{ + using namespace mtx::secret_storage; + http::client()->secret_storage_secret( + secrets::megolm_backup_v1, [](Secret secret, mtx::http::RequestErr err) { + std::optional backup_key; + if (!err) + backup_key = secret; + + http::client()->secret_storage_secret( + secrets::cross_signing_self_signing, + [backup_key](Secret secret, mtx::http::RequestErr err) { + std::optional self_signing_key; + if (!err) + self_signing_key = secret; + + http::client()->secret_storage_secret( + secrets::cross_signing_user_signing, + [backup_key, self_signing_key](Secret secret, mtx::http::RequestErr err) { + std::optional user_signing_key; + if (!err) + user_signing_key = secret; + + std::map> + secrets; + + if (backup_key && !backup_key->encrypted.empty()) + secrets[backup_key->encrypted.begin()->first][secrets::megolm_backup_v1] = + backup_key->encrypted.begin()->second; + if (self_signing_key && !self_signing_key->encrypted.empty()) + secrets[self_signing_key->encrypted.begin()->first] + [secrets::cross_signing_self_signing] = + self_signing_key->encrypted.begin()->second; + if (user_signing_key && !user_signing_key->encrypted.empty()) + secrets[user_signing_key->encrypted.begin()->first] + [secrets::cross_signing_user_signing] = + user_signing_key->encrypted.begin()->second; + + for (const auto &[key, secrets] : secrets) + unlock_secrets(key, secrets); + }); + }); + }); +} + +} // namespace olm diff --git a/src/encryption/Olm.h b/src/encryption/Olm.h new file mode 100644 index 00000000..44e2b8ed --- /dev/null +++ b/src/encryption/Olm.h @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include + +#include + +constexpr auto OLM_ALGO = "m.olm.v1.curve25519-aes-sha2"; + +namespace olm { +Q_NAMESPACE + +enum DecryptionErrorCode +{ + NoError, + MissingSession, // Session was not found, retrieve from backup or request from other devices + // and try again + MissingSessionIndex, // Session was found, but it does not reach back enough to this index, + // retrieve from backup or request from other devices and try again + DbError, // DB read failed + DecryptionFailed, // libolm error + ParsingFailed, // Failed to parse the actual event + ReplayAttack, // Megolm index reused +}; +Q_ENUM_NS(DecryptionErrorCode) + +struct DecryptionResult +{ + DecryptionErrorCode error; + std::optional error_message; + std::optional event; +}; + +struct OlmMessage +{ + std::string sender_key; + std::string sender; + + using RecipientKey = std::string; + std::map ciphertext; +}; + +void +from_json(const nlohmann::json &obj, OlmMessage &msg); + +mtx::crypto::OlmClient * +client(); + +void +handle_to_device_messages(const std::vector &msgs); + +nlohmann::json +try_olm_decryption(const std::string &sender_key, + const mtx::events::msg::OlmCipherContent &content); + +void +handle_olm_message(const OlmMessage &msg, const UserKeyCache &otherUserDeviceKeys); + +//! Establish a new inbound megolm session with the decrypted payload from olm. +void +create_inbound_megolm_session(const mtx::events::DeviceEvent &roomKey, + const std::string &sender_key, + const std::string &sender_ed25519); +void +import_inbound_megolm_session( + const mtx::events::DeviceEvent &roomKey); +void +lookup_keybackup(const std::string room, const std::string session_id); + +nlohmann::json +handle_pre_key_olm_message(const std::string &sender, + const std::string &sender_key, + const mtx::events::msg::OlmCipherContent &content); + +mtx::events::msg::Encrypted +encrypt_group_message(const std::string &room_id, + const std::string &device_id, + nlohmann::json body); + +//! Decrypt an event. Use dont_write_db to prevent db writes when already in a write transaction. +DecryptionResult +decryptEvent(const MegolmSessionIndex &index, + const mtx::events::EncryptedEvent &event, + bool dont_write_db = false); +crypto::Trust +calculate_trust(const std::string &user_id, const MegolmSessionIndex &index); + +void +mark_keys_as_published(); + +//! Request the encryption keys from sender's device for the given event. +void +send_key_request_for(mtx::events::EncryptedEvent e, + const std::string &request_id, + bool cancel = false); + +void +handle_key_request_message(const mtx::events::DeviceEvent &); + +void +send_megolm_key_to_device(const std::string &user_id, + const std::string &device_id, + const mtx::events::msg::ForwardedRoomKey &payload); + +//! Send encrypted to device messages, targets is a map from userid to device ids or {} for all +//! devices +void +send_encrypted_to_device_messages(const std::map> targets, + const mtx::events::collections::DeviceEvents &event, + bool force_new_session = false); + +//! Request backup and signing keys and cache them locally +void +request_cross_signing_keys(); +//! Download backup and signing keys and cache them locally +void +download_cross_signing_keys(); + +} // namespace olm diff --git a/src/encryption/SelfVerificationStatus.cpp b/src/encryption/SelfVerificationStatus.cpp new file mode 100644 index 00000000..d75a2109 --- /dev/null +++ b/src/encryption/SelfVerificationStatus.cpp @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "SelfVerificationStatus.h" + +#include "Cache_p.h" +#include "Logging.h" +#include "MainWindow.h" +#include "MatrixClient.h" +#include "Olm.h" +#include "ui/UIA.h" + +#include + +SelfVerificationStatus::SelfVerificationStatus(QObject *o) + : QObject(o) +{ + connect(MainWindow::instance(), &MainWindow::reload, this, [this] { + connect(cache::client(), + &Cache::selfUnverified, + this, + &SelfVerificationStatus::invalidate, + Qt::UniqueConnection); + invalidate(); + }); +} + +void +SelfVerificationStatus::setupCrosssigning(bool useSSSS, QString password, bool useOnlineKeyBackup) +{ + nhlog::db()->info("Clicked setup crossigning"); + + auto xsign_keys = olm::client()->create_crosssigning_keys(); + + if (!xsign_keys) { + nhlog::crypto()->critical("Failed to setup cross-signing keys!"); + emit setupFailed(tr("Failed to create keys for cross-signing!")); + return; + } + + cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_master, + xsign_keys->private_master_key); + cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_self_signing, + xsign_keys->private_self_signing_key); + cache::client()->storeSecret(mtx::secret_storage::secrets::cross_signing_user_signing, + xsign_keys->private_user_signing_key); + + std::optional okb; + if (useOnlineKeyBackup) { + okb = olm::client()->create_online_key_backup(xsign_keys->private_master_key); + if (!okb) { + nhlog::crypto()->critical("Failed to setup online key backup!"); + emit setupFailed(tr("Failed to create keys for online key backup!")); + return; + } + + cache::client()->storeSecret( + mtx::secret_storage::secrets::megolm_backup_v1, + mtx::crypto::bin2base64(mtx::crypto::to_string(okb->privateKey))); + + http::client()->post_backup_version( + okb->backupVersion.algorithm, + okb->backupVersion.auth_data, + [](const mtx::responses::Version &v, mtx::http::RequestErr e) { + if (e) { + nhlog::net()->error("error setting up online key backup: {} {} {} {}", + e->parse_error, + e->status_code, + e->error_code, + e->matrix_error.error); + } else { + nhlog::crypto()->info("Set up online key backup: '{}'", v.version); + } + }); + } + + std::optional ssss; + if (useSSSS) { + ssss = olm::client()->create_ssss_key(password.toStdString()); + if (!ssss) { + nhlog::crypto()->critical("Failed to setup secure server side secret storage!"); + emit setupFailed(tr("Failed to create keys secure server side secret storage!")); + return; + } + + auto master = mtx::crypto::PkSigning::from_seed(xsign_keys->private_master_key); + nlohmann::json j = ssss->keyDescription; + j.erase("signatures"); + ssss->keyDescription + .signatures[http::client()->user_id().to_string()]["ed25519:" + master.public_key()] = + master.sign(j.dump()); + + http::client()->upload_secret_storage_key( + ssss->keyDescription.name, ssss->keyDescription, [](mtx::http::RequestErr) {}); + http::client()->set_secret_storage_default_key(ssss->keyDescription.name, + [](mtx::http::RequestErr) {}); + + auto uploadSecret = [ssss](const std::string &key_name, const std::string &secret) { + mtx::secret_storage::Secret s; + s.encrypted[ssss->keyDescription.name] = + mtx::crypto::encrypt(secret, ssss->privateKey, key_name); + http::client()->upload_secret_storage_secret( + key_name, s, [key_name](mtx::http::RequestErr) { + nhlog::crypto()->info("Uploaded secret: {}", key_name); + }); + }; + + uploadSecret(mtx::secret_storage::secrets::cross_signing_master, + xsign_keys->private_master_key); + uploadSecret(mtx::secret_storage::secrets::cross_signing_self_signing, + xsign_keys->private_self_signing_key); + uploadSecret(mtx::secret_storage::secrets::cross_signing_user_signing, + xsign_keys->private_user_signing_key); + + if (okb) + uploadSecret(mtx::secret_storage::secrets::megolm_backup_v1, + mtx::crypto::bin2base64(mtx::crypto::to_string(okb->privateKey))); + } + + mtx::requests::DeviceSigningUpload device_sign{}; + device_sign.master_key = xsign_keys->master_key; + device_sign.self_signing_key = xsign_keys->self_signing_key; + device_sign.user_signing_key = xsign_keys->user_signing_key; + http::client()->device_signing_upload( + device_sign, + UIA::instance()->genericHandler(tr("Encryption Setup")), + [this, ssss, xsign_keys](mtx::http::RequestErr e) { + if (e) { + nhlog::crypto()->critical("Failed to upload cross signing keys: {}", + e->matrix_error.error); + + emit setupFailed(tr("Encryption setup failed: %1") + .arg(QString::fromStdString(e->matrix_error.error))); + return; + } + nhlog::crypto()->info("Crosssigning keys uploaded!"); + + auto deviceKeys = cache::client()->userKeys(http::client()->user_id().to_string()); + if (deviceKeys) { + auto myKey = deviceKeys->device_keys.at(http::client()->device_id()); + if (myKey.user_id == http::client()->user_id().to_string() && + myKey.device_id == http::client()->device_id() && + myKey.keys["ed25519:" + http::client()->device_id()] == + olm::client()->identity_keys().ed25519 && + myKey.keys["curve25519:" + http::client()->device_id()] == + olm::client()->identity_keys().curve25519) { + json j = myKey; + j.erase("signatures"); + j.erase("unsigned"); + + auto ssk = + mtx::crypto::PkSigning::from_seed(xsign_keys->private_self_signing_key); + myKey.signatures[http::client()->user_id().to_string()] + ["ed25519:" + ssk.public_key()] = ssk.sign(j.dump()); + mtx::requests::KeySignaturesUpload req; + req.signatures[http::client()->user_id().to_string()] + [http::client()->device_id()] = myKey; + + http::client()->keys_signatures_upload( + req, + [](const mtx::responses::KeySignaturesUpload &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("failed to upload signatures: {},{}", + mtx::errors::to_string(err->matrix_error.errcode), + static_cast(err->status_code)); + } + + for (const auto &[user_id, tmp] : res.errors) + for (const auto &[key_id, e] : tmp) + nhlog::net()->error("signature error for user {} and key " + "id {}: {}, {}", + user_id, + key_id, + mtx::errors::to_string(e.errcode), + e.error); + }); + } + } + + if (ssss) { + auto k = QString::fromStdString(mtx::crypto::key_to_recoverykey(ssss->privateKey)); + + QString r; + for (int i = 0; i < k.size(); i += 4) + r += k.mid(i, 4) + " "; + + emit showRecoveryKey(r.trimmed()); + } else { + emit setupCompleted(); + } + }); +} + +void +SelfVerificationStatus::verifyMasterKey() +{ + nhlog::db()->info("Clicked verify master key"); +} + +void +SelfVerificationStatus::verifyUnverifiedDevices() +{ + nhlog::db()->info("Clicked verify unverified devices"); +} + +void +SelfVerificationStatus::invalidate() +{ + nhlog::db()->info("Invalidating self verification status"); + auto keys = cache::client()->userKeys(http::client()->user_id().to_string()); + if (!keys) { + cache::client()->query_keys(http::client()->user_id().to_string(), + [](const UserKeyCache &, mtx::http::RequestErr) {}); + return; + } + + if (keys->master_keys.keys.empty()) { + if (status_ != SelfVerificationStatus::NoMasterKey) { + this->status_ = SelfVerificationStatus::NoMasterKey; + emit statusChanged(); + } + return; + } + + auto verifStatus = cache::client()->verificationStatus(http::client()->user_id().to_string()); + + if (!verifStatus.user_verified) { + if (status_ != SelfVerificationStatus::UnverifiedMasterKey) { + this->status_ = SelfVerificationStatus::UnverifiedMasterKey; + emit statusChanged(); + } + return; + } + + if (verifStatus.unverified_device_count > 0) { + if (status_ != SelfVerificationStatus::UnverifiedDevices) { + this->status_ = SelfVerificationStatus::UnverifiedDevices; + emit statusChanged(); + } + return; + } + + if (status_ != SelfVerificationStatus::AllVerified) { + this->status_ = SelfVerificationStatus::AllVerified; + emit statusChanged(); + return; + } +} diff --git a/src/encryption/SelfVerificationStatus.h b/src/encryption/SelfVerificationStatus.h new file mode 100644 index 00000000..8cb54df6 --- /dev/null +++ b/src/encryption/SelfVerificationStatus.h @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +class SelfVerificationStatus : public QObject +{ + Q_OBJECT + + Q_PROPERTY(Status status READ status NOTIFY statusChanged) + +public: + SelfVerificationStatus(QObject *o = nullptr); + enum Status + { + AllVerified, + NoMasterKey, + UnverifiedMasterKey, + UnverifiedDevices, + }; + Q_ENUM(Status) + + Q_INVOKABLE void setupCrosssigning(bool useSSSS, QString password, bool useOnlineKeyBackup); + Q_INVOKABLE void verifyMasterKey(); + Q_INVOKABLE void verifyUnverifiedDevices(); + + Status status() const { return status_; } + +signals: + void statusChanged(); + void setupCompleted(); + void showRecoveryKey(QString key); + void setupFailed(QString message); + +public slots: + void invalidate(); + +private: + Status status_ = AllVerified; +}; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 7144424a..d7296a7c 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -15,7 +15,6 @@ #include "EventAccessors.h" #include "Logging.h" #include "MatrixClient.h" -#include "Olm.h" #include "Utils.h" Q_DECLARE_METATYPE(Reaction) diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 53dbaff4..9b857dcf 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -15,8 +15,8 @@ #include #include -#include "Olm.h" #include "Reaction.h" +#include "encryption/Olm.h" class EventStore : public QObject { diff --git a/src/timeline/InputBar.cpp b/src/timeline/InputBar.cpp index f33d1dfd..ed97a2ca 100644 --- a/src/timeline/InputBar.cpp +++ b/src/timeline/InputBar.cpp @@ -26,7 +26,6 @@ #include "Logging.h" #include "MainWindow.h" #include "MatrixClient.h" -#include "Olm.h" #include "RoomsModel.h" #include "TimelineModel.h" #include "TimelineViewManager.h" diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 720a78fe..0e5ce510 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -27,10 +27,10 @@ #include "MatrixClient.h" #include "MemberList.h" #include "MxcImageProvider.h" -#include "Olm.h" #include "ReadReceiptsModel.h" #include "TimelineViewManager.h" #include "Utils.h" +#include "encryption/Olm.h" Q_DECLARE_METATYPE(QModelIndex) diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index a30a145d..86f59c52 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -18,7 +18,6 @@ #include "CombinedImagePackModel.h" #include "CompletionProxyModel.h" #include "DelegateChooser.h" -#include "DeviceVerificationFlow.h" #include "EventAccessors.h" #include "ImagePackListModel.h" #include "InviteesModel.h" @@ -29,13 +28,14 @@ #include "ReadReceiptsModel.h" #include "RoomDirectoryModel.h" #include "RoomsModel.h" -#include "SelfVerificationStatus.h" #include "SingleImagePackModel.h" #include "UserSettingsPage.h" #include "UsersModel.h" #include "dialogs/ImageOverlay.h" #include "emoji/EmojiModel.h" #include "emoji/Provider.h" +#include "encryption/DeviceVerificationFlow.h" +#include "encryption/SelfVerificationStatus.h" #include "ui/MxcAnimatedImage.h" #include "ui/MxcMediaProxy.h" #include "ui/NhekoCursorShape.h" diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h index ab078aa7..723282d6 100644 --- a/src/timeline/TimelineViewManager.h +++ b/src/timeline/TimelineViewManager.h @@ -17,16 +17,16 @@ #include #include "Cache.h" -#include "CallManager.h" #include "JdenticonProvider.h" #include "Logging.h" #include "TimelineModel.h" #include "Utils.h" -#include "WebRTCSession.h" #include "emoji/EmojiModel.h" #include "emoji/Provider.h" #include "timeline/CommunitiesModel.h" #include "timeline/RoomlistModel.h" +#include "voip/CallManager.h" +#include "voip/WebRTCSession.h" class MxcImageProvider; class BlurhashProvider; diff --git a/src/ui/NhekoGlobalObject.cpp b/src/ui/NhekoGlobalObject.cpp index 11fc5681..15f2a5af 100644 --- a/src/ui/NhekoGlobalObject.cpp +++ b/src/ui/NhekoGlobalObject.cpp @@ -14,7 +14,7 @@ #include "MainWindow.h" #include "UserSettingsPage.h" #include "Utils.h" -#include "WebRTCSession.h" +#include "voip/WebRTCSession.h" Nheko::Nheko() { diff --git a/src/ui/UserProfile.cpp b/src/ui/UserProfile.cpp index 591110af..d62e3248 100644 --- a/src/ui/UserProfile.cpp +++ b/src/ui/UserProfile.cpp @@ -9,10 +9,10 @@ #include "Cache_p.h" #include "ChatPage.h" -#include "DeviceVerificationFlow.h" #include "Logging.h" #include "UserProfile.h" #include "Utils.h" +#include "encryption/DeviceVerificationFlow.h" #include "mtx/responses/crypto.hpp" #include "timeline/TimelineModel.h" #include "timeline/TimelineViewManager.h" diff --git a/src/voip/CallDevices.cpp b/src/voip/CallDevices.cpp new file mode 100644 index 00000000..be185470 --- /dev/null +++ b/src/voip/CallDevices.cpp @@ -0,0 +1,385 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include + +#include "CallDevices.h" +#include "ChatPage.h" +#include "Logging.h" +#include "UserSettingsPage.h" + +#ifdef GSTREAMER_AVAILABLE +extern "C" +{ +#include "gst/gst.h" +} +#endif + +CallDevices::CallDevices() + : QObject() +{} + +#ifdef GSTREAMER_AVAILABLE +namespace { + +struct AudioSource +{ + std::string name; + GstDevice *device; +}; + +struct VideoSource +{ + struct Caps + { + std::string resolution; + std::vector frameRates; + }; + std::string name; + GstDevice *device; + std::vector caps; +}; + +std::vector audioSources_; +std::vector videoSources_; + +using FrameRate = std::pair; +std::optional +getFrameRate(const GValue *value) +{ + if (GST_VALUE_HOLDS_FRACTION(value)) { + gint num = gst_value_get_fraction_numerator(value); + gint den = gst_value_get_fraction_denominator(value); + return FrameRate{num, den}; + } + return std::nullopt; +} + +void +addFrameRate(std::vector &rates, const FrameRate &rate) +{ + constexpr double minimumFrameRate = 15.0; + if (static_cast(rate.first) / rate.second >= minimumFrameRate) + rates.push_back(std::to_string(rate.first) + "/" + std::to_string(rate.second)); +} + +void +setDefaultDevice(bool isVideo) +{ + auto settings = ChatPage::instance()->userSettings(); + if (isVideo && settings->camera().isEmpty()) { + const VideoSource &camera = videoSources_.front(); + settings->setCamera(QString::fromStdString(camera.name)); + settings->setCameraResolution(QString::fromStdString(camera.caps.front().resolution)); + settings->setCameraFrameRate( + QString::fromStdString(camera.caps.front().frameRates.front())); + } else if (!isVideo && settings->microphone().isEmpty()) { + settings->setMicrophone(QString::fromStdString(audioSources_.front().name)); + } +} + +void +addDevice(GstDevice *device) +{ + if (!device) + return; + + gchar *name = gst_device_get_display_name(device); + gchar *type = gst_device_get_device_class(device); + bool isVideo = !std::strncmp(type, "Video", 5); + g_free(type); + nhlog::ui()->debug("WebRTC: {} device added: {}", isVideo ? "video" : "audio", name); + if (!isVideo) { + audioSources_.push_back({name, device}); + g_free(name); + setDefaultDevice(false); + return; + } + + GstCaps *gstcaps = gst_device_get_caps(device); + if (!gstcaps) { + nhlog::ui()->debug("WebRTC: unable to get caps for {}", name); + g_free(name); + return; + } + + VideoSource source{name, device, {}}; + g_free(name); + guint nCaps = gst_caps_get_size(gstcaps); + for (guint i = 0; i < nCaps; ++i) { + GstStructure *structure = gst_caps_get_structure(gstcaps, i); + const gchar *struct_name = gst_structure_get_name(structure); + if (!std::strcmp(struct_name, "video/x-raw")) { + gint widthpx, heightpx; + if (gst_structure_get(structure, + "width", + G_TYPE_INT, + &widthpx, + "height", + G_TYPE_INT, + &heightpx, + nullptr)) { + VideoSource::Caps caps; + caps.resolution = std::to_string(widthpx) + "x" + std::to_string(heightpx); + const GValue *value = gst_structure_get_value(structure, "framerate"); + if (auto fr = getFrameRate(value); fr) + addFrameRate(caps.frameRates, *fr); + else if (GST_VALUE_HOLDS_FRACTION_RANGE(value)) { + addFrameRate(caps.frameRates, + *getFrameRate(gst_value_get_fraction_range_min(value))); + addFrameRate(caps.frameRates, + *getFrameRate(gst_value_get_fraction_range_max(value))); + } else if (GST_VALUE_HOLDS_LIST(value)) { + guint nRates = gst_value_list_get_size(value); + for (guint j = 0; j < nRates; ++j) { + const GValue *rate = gst_value_list_get_value(value, j); + if (auto frate = getFrameRate(rate); frate) + addFrameRate(caps.frameRates, *frate); + } + } + if (!caps.frameRates.empty()) + source.caps.push_back(std::move(caps)); + } + } + } + gst_caps_unref(gstcaps); + videoSources_.push_back(std::move(source)); + setDefaultDevice(true); +} + +template +bool +removeDevice(T &sources, GstDevice *device, bool changed) +{ + if (auto it = std::find_if( + sources.begin(), sources.end(), [device](const auto &s) { return s.device == device; }); + it != sources.end()) { + nhlog::ui()->debug( + std::string("WebRTC: device ") + (changed ? "changed: " : "removed: ") + "{}", it->name); + gst_object_unref(device); + sources.erase(it); + return true; + } + return false; +} + +void +removeDevice(GstDevice *device, bool changed) +{ + if (device) { + if (removeDevice(audioSources_, device, changed) || + removeDevice(videoSources_, device, changed)) + return; + } +} + +gboolean +newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data G_GNUC_UNUSED) +{ + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_DEVICE_ADDED: { + GstDevice *device; + gst_message_parse_device_added(msg, &device); + addDevice(device); + emit CallDevices::instance().devicesChanged(); + break; + } + case GST_MESSAGE_DEVICE_REMOVED: { + GstDevice *device; + gst_message_parse_device_removed(msg, &device); + removeDevice(device, false); + emit CallDevices::instance().devicesChanged(); + break; + } + case GST_MESSAGE_DEVICE_CHANGED: { + GstDevice *device; + GstDevice *oldDevice; + gst_message_parse_device_changed(msg, &device, &oldDevice); + removeDevice(oldDevice, true); + addDevice(device); + break; + } + default: + break; + } + return TRUE; +} + +template +std::vector +deviceNames(T &sources, const std::string &defaultDevice) +{ + std::vector ret; + ret.reserve(sources.size()); + for (const auto &s : sources) + ret.push_back(s.name); + + // move default device to top of the list + if (auto it = std::find(ret.begin(), ret.end(), defaultDevice); it != ret.end()) + std::swap(ret.front(), *it); + + return ret; +} + +std::optional +getVideoSource(const std::string &cameraName) +{ + if (auto it = std::find_if(videoSources_.cbegin(), + videoSources_.cend(), + [&cameraName](const auto &s) { return s.name == cameraName; }); + it != videoSources_.cend()) { + return *it; + } + return std::nullopt; +} + +std::pair +tokenise(std::string_view str, char delim) +{ + std::pair ret; + ret.first = std::atoi(str.data()); + auto pos = str.find_first_of(delim); + ret.second = std::atoi(str.data() + pos + 1); + return ret; +} +} + +void +CallDevices::init() +{ + static GstDeviceMonitor *monitor = nullptr; + if (!monitor) { + monitor = gst_device_monitor_new(); + GstCaps *caps = gst_caps_new_empty_simple("audio/x-raw"); + gst_device_monitor_add_filter(monitor, "Audio/Source", caps); + gst_device_monitor_add_filter(monitor, "Audio/Duplex", caps); + gst_caps_unref(caps); + caps = gst_caps_new_empty_simple("video/x-raw"); + gst_device_monitor_add_filter(monitor, "Video/Source", caps); + gst_device_monitor_add_filter(monitor, "Video/Duplex", caps); + gst_caps_unref(caps); + + GstBus *bus = gst_device_monitor_get_bus(monitor); + gst_bus_add_watch(bus, newBusMessage, nullptr); + gst_object_unref(bus); + if (!gst_device_monitor_start(monitor)) { + nhlog::ui()->error("WebRTC: failed to start device monitor"); + return; + } + } +} + +bool +CallDevices::haveMic() const +{ + return !audioSources_.empty(); +} + +bool +CallDevices::haveCamera() const +{ + return !videoSources_.empty(); +} + +std::vector +CallDevices::names(bool isVideo, const std::string &defaultDevice) const +{ + return isVideo ? deviceNames(videoSources_, defaultDevice) + : deviceNames(audioSources_, defaultDevice); +} + +std::vector +CallDevices::resolutions(const std::string &cameraName) const +{ + std::vector ret; + if (auto s = getVideoSource(cameraName); s) { + ret.reserve(s->caps.size()); + for (const auto &c : s->caps) + ret.push_back(c.resolution); + } + return ret; +} + +std::vector +CallDevices::frameRates(const std::string &cameraName, const std::string &resolution) const +{ + if (auto s = getVideoSource(cameraName); s) { + if (auto it = std::find_if(s->caps.cbegin(), + s->caps.cend(), + [&](const auto &c) { return c.resolution == resolution; }); + it != s->caps.cend()) + return it->frameRates; + } + return {}; +} + +GstDevice * +CallDevices::audioDevice() const +{ + std::string name = ChatPage::instance()->userSettings()->microphone().toStdString(); + if (auto it = std::find_if(audioSources_.cbegin(), + audioSources_.cend(), + [&name](const auto &s) { return s.name == name; }); + it != audioSources_.cend()) { + nhlog::ui()->debug("WebRTC: microphone: {}", name); + return it->device; + } else { + nhlog::ui()->error("WebRTC: unknown microphone: {}", name); + return nullptr; + } +} + +GstDevice * +CallDevices::videoDevice(std::pair &resolution, std::pair &frameRate) const +{ + auto settings = ChatPage::instance()->userSettings(); + std::string name = settings->camera().toStdString(); + if (auto s = getVideoSource(name); s) { + nhlog::ui()->debug("WebRTC: camera: {}", name); + resolution = tokenise(settings->cameraResolution().toStdString(), 'x'); + frameRate = tokenise(settings->cameraFrameRate().toStdString(), '/'); + nhlog::ui()->debug("WebRTC: camera resolution: {}x{}", resolution.first, resolution.second); + nhlog::ui()->debug("WebRTC: camera frame rate: {}/{}", frameRate.first, frameRate.second); + return s->device; + } else { + nhlog::ui()->error("WebRTC: unknown camera: {}", name); + return nullptr; + } +} + +#else + +bool +CallDevices::haveMic() const +{ + return false; +} + +bool +CallDevices::haveCamera() const +{ + return false; +} + +std::vector +CallDevices::names(bool, const std::string &) const +{ + return {}; +} + +std::vector +CallDevices::resolutions(const std::string &) const +{ + return {}; +} + +std::vector +CallDevices::frameRates(const std::string &, const std::string &) const +{ + return {}; +} + +#endif diff --git a/src/voip/CallDevices.h b/src/voip/CallDevices.h new file mode 100644 index 00000000..d30ce644 --- /dev/null +++ b/src/voip/CallDevices.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include + +typedef struct _GstDevice GstDevice; + +class CallDevices : public QObject +{ + Q_OBJECT + +public: + static CallDevices &instance() + { + static CallDevices instance; + return instance; + } + + bool haveMic() const; + bool haveCamera() const; + std::vector names(bool isVideo, const std::string &defaultDevice) const; + std::vector resolutions(const std::string &cameraName) const; + std::vector frameRates(const std::string &cameraName, + const std::string &resolution) const; + +signals: + void devicesChanged(); + +private: + CallDevices(); + + friend class WebRTCSession; + void init(); + GstDevice *audioDevice() const; + GstDevice *videoDevice(std::pair &resolution, std::pair &frameRate) const; + +public: + CallDevices(CallDevices const &) = delete; + void operator=(CallDevices const &) = delete; +}; diff --git a/src/voip/CallManager.cpp b/src/voip/CallManager.cpp new file mode 100644 index 00000000..0f701b0d --- /dev/null +++ b/src/voip/CallManager.cpp @@ -0,0 +1,680 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "Cache.h" +#include "CallDevices.h" +#include "CallManager.h" +#include "ChatPage.h" +#include "Logging.h" +#include "MatrixClient.h" +#include "UserSettingsPage.h" +#include "Utils.h" + +#include "mtx/responses/turn_server.hpp" + +#ifdef XCB_AVAILABLE +#include +#include +#endif + +#ifdef GSTREAMER_AVAILABLE +extern "C" +{ +#include "gst/gst.h" +} +#endif + +Q_DECLARE_METATYPE(std::vector) +Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) +Q_DECLARE_METATYPE(mtx::responses::TurnServer) + +using namespace mtx::events; +using namespace mtx::events::msg; + +using webrtc::CallType; + +namespace { +std::vector +getTurnURIs(const mtx::responses::TurnServer &turnServer); +} + +CallManager::CallManager(QObject *parent) + : QObject(parent) + , session_(WebRTCSession::instance()) + , turnServerTimer_(this) +{ + qRegisterMetaType>(); + qRegisterMetaType(); + qRegisterMetaType(); + + connect( + &session_, + &WebRTCSession::offerCreated, + this, + [this](const std::string &sdp, const std::vector &candidates) { + nhlog::ui()->debug("WebRTC: call id: {} - sending offer", callid_); + emit newMessage(roomid_, CallInvite{callid_, sdp, "0", timeoutms_}); + emit newMessage(roomid_, CallCandidates{callid_, candidates, "0"}); + std::string callid(callid_); + QTimer::singleShot(timeoutms_, this, [this, callid]() { + if (session_.state() == webrtc::State::OFFERSENT && callid == callid_) { + hangUp(CallHangUp::Reason::InviteTimeOut); + emit ChatPage::instance()->showNotification("The remote side failed to pick up."); + } + }); + }); + + connect( + &session_, + &WebRTCSession::answerCreated, + this, + [this](const std::string &sdp, const std::vector &candidates) { + nhlog::ui()->debug("WebRTC: call id: {} - sending answer", callid_); + emit newMessage(roomid_, CallAnswer{callid_, sdp, "0"}); + emit newMessage(roomid_, CallCandidates{callid_, candidates, "0"}); + }); + + connect(&session_, + &WebRTCSession::newICECandidate, + this, + [this](const CallCandidates::Candidate &candidate) { + nhlog::ui()->debug("WebRTC: call id: {} - sending ice candidate", callid_); + emit newMessage(roomid_, CallCandidates{callid_, {candidate}, "0"}); + }); + + connect(&turnServerTimer_, &QTimer::timeout, this, &CallManager::retrieveTurnServer); + + connect( + this, &CallManager::turnServerRetrieved, this, [this](const mtx::responses::TurnServer &res) { + nhlog::net()->info("TURN server(s) retrieved from homeserver:"); + nhlog::net()->info("username: {}", res.username); + nhlog::net()->info("ttl: {} seconds", res.ttl); + for (const auto &u : res.uris) + nhlog::net()->info("uri: {}", u); + + // Request new credentials close to expiry + // See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00 + turnURIs_ = getTurnURIs(res); + uint32_t ttl = std::max(res.ttl, UINT32_C(3600)); + if (res.ttl < 3600) + nhlog::net()->warn("Setting ttl to 1 hour"); + turnServerTimer_.setInterval(ttl * 1000 * 0.9); + }); + + connect(&session_, &WebRTCSession::stateChanged, this, [this](webrtc::State state) { + switch (state) { + case webrtc::State::DISCONNECTED: + playRingtone(QUrl("qrc:/media/media/callend.ogg"), false); + clear(); + break; + case webrtc::State::ICEFAILED: { + QString error("Call connection failed."); + if (turnURIs_.empty()) + error += " Your homeserver has no configured TURN server."; + emit ChatPage::instance()->showNotification(error); + hangUp(CallHangUp::Reason::ICEFailed); + break; + } + default: + break; + } + emit newCallState(); + }); + + connect( + &CallDevices::instance(), &CallDevices::devicesChanged, this, &CallManager::devicesChanged); + + connect( + &player_, &QMediaPlayer::mediaStatusChanged, this, [this](QMediaPlayer::MediaStatus status) { + if (status == QMediaPlayer::LoadedMedia) + player_.play(); + }); + + connect(&player_, + QOverload::of(&QMediaPlayer::error), + [this](QMediaPlayer::Error error) { + stopRingtone(); + switch (error) { + case QMediaPlayer::FormatError: + case QMediaPlayer::ResourceError: + nhlog::ui()->error("WebRTC: valid ringtone file not found"); + break; + case QMediaPlayer::AccessDeniedError: + nhlog::ui()->error("WebRTC: access to ringtone file denied"); + break; + default: + nhlog::ui()->error("WebRTC: unable to play ringtone"); + break; + } + }); +} + +void +CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int windowIndex) +{ + if (isOnCall()) + return; + if (callType == CallType::SCREEN) { + if (!screenShareSupported()) + return; + if (windows_.empty() || windowIndex >= windows_.size()) { + nhlog::ui()->error("WebRTC: window index out of range"); + return; + } + } + + auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); + if (roomInfo.member_count != 2) { + emit ChatPage::instance()->showNotification("Calls are limited to 1:1 rooms."); + return; + } + + std::string errorMessage; + if (!session_.havePlugins(false, &errorMessage) || + ((callType == CallType::VIDEO || callType == CallType::SCREEN) && + !session_.havePlugins(true, &errorMessage))) { + emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); + return; + } + + callType_ = callType; + roomid_ = roomid; + session_.setTurnServers(turnURIs_); + generateCallID(); + std::string strCallType = + callType_ == CallType::VOICE ? "voice" : (callType_ == CallType::VIDEO ? "video" : "screen"); + nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType); + std::vector members(cache::getMembers(roomid.toStdString())); + const RoomMember &callee = + members.front().user_id == utils::localUser() ? members.back() : members.front(); + callParty_ = callee.user_id; + callPartyDisplayName_ = callee.display_name.isEmpty() ? callee.user_id : callee.display_name; + callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); + emit newInviteState(); + playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); + if (!session_.createOffer(callType, + callType == CallType::SCREEN ? windows_[windowIndex].second : 0)) { + emit ChatPage::instance()->showNotification("Problem setting up call."); + endCall(); + } +} + +namespace { +std::string +callHangUpReasonString(CallHangUp::Reason reason) +{ + switch (reason) { + case CallHangUp::Reason::ICEFailed: + return "ICE failed"; + case CallHangUp::Reason::InviteTimeOut: + return "Invite time out"; + default: + return "User"; + } +} +} + +void +CallManager::hangUp(CallHangUp::Reason reason) +{ + if (!callid_.empty()) { + nhlog::ui()->debug( + "WebRTC: call id: {} - hanging up ({})", callid_, callHangUpReasonString(reason)); + emit newMessage(roomid_, CallHangUp{callid_, "0", reason}); + endCall(); + } +} + +void +CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) +{ +#ifdef GSTREAMER_AVAILABLE + if (handleEvent(event) || handleEvent(event) || + handleEvent(event) || handleEvent(event)) + return; +#else + (void)event; +#endif +} + +template +bool +CallManager::handleEvent(const mtx::events::collections::TimelineEvents &event) +{ + if (std::holds_alternative>(event)) { + handleEvent(std::get>(event)); + return true; + } + return false; +} + +void +CallManager::handleEvent(const RoomEvent &callInviteEvent) +{ + const char video[] = "m=video"; + const std::string &sdp = callInviteEvent.content.sdp; + bool isVideo = std::search(sdp.cbegin(), + sdp.cend(), + std::cbegin(video), + std::cend(video) - 1, + [](unsigned char c1, unsigned char c2) { + return std::tolower(c1) == std::tolower(c2); + }) != sdp.cend(); + + nhlog::ui()->debug("WebRTC: call id: {} - incoming {} CallInvite from {}", + callInviteEvent.content.call_id, + (isVideo ? "video" : "voice"), + callInviteEvent.sender); + + if (callInviteEvent.content.call_id.empty()) + return; + + auto roomInfo = cache::singleRoomInfo(callInviteEvent.room_id); + if (isOnCall() || roomInfo.member_count != 2) { + emit newMessage( + QString::fromStdString(callInviteEvent.room_id), + CallHangUp{callInviteEvent.content.call_id, "0", CallHangUp::Reason::InviteTimeOut}); + return; + } + + const QString &ringtone = ChatPage::instance()->userSettings()->ringtone(); + if (ringtone != "Mute") + playRingtone(ringtone == "Default" ? QUrl("qrc:/media/media/ring.ogg") + : QUrl::fromLocalFile(ringtone), + true); + roomid_ = QString::fromStdString(callInviteEvent.room_id); + callid_ = callInviteEvent.content.call_id; + remoteICECandidates_.clear(); + + std::vector members(cache::getMembers(callInviteEvent.room_id)); + const RoomMember &caller = + members.front().user_id == utils::localUser() ? members.back() : members.front(); + callParty_ = caller.user_id; + callPartyDisplayName_ = caller.display_name.isEmpty() ? caller.user_id : caller.display_name; + callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); + + haveCallInvite_ = true; + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; + inviteSDP_ = callInviteEvent.content.sdp; + emit newInviteState(); +} + +void +CallManager::acceptInvite() +{ + if (!haveCallInvite_) + return; + + stopRingtone(); + std::string errorMessage; + if (!session_.havePlugins(false, &errorMessage) || + (callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) { + emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); + hangUp(); + return; + } + + session_.setTurnServers(turnURIs_); + if (!session_.acceptOffer(inviteSDP_)) { + emit ChatPage::instance()->showNotification("Problem setting up call."); + hangUp(); + return; + } + session_.acceptICECandidates(remoteICECandidates_); + remoteICECandidates_.clear(); + haveCallInvite_ = false; + emit newInviteState(); +} + +void +CallManager::handleEvent(const RoomEvent &callCandidatesEvent) +{ + if (callCandidatesEvent.sender == utils::localUser().toStdString()) + return; + + nhlog::ui()->debug("WebRTC: call id: {} - incoming CallCandidates from {}", + callCandidatesEvent.content.call_id, + callCandidatesEvent.sender); + + if (callid_ == callCandidatesEvent.content.call_id) { + if (isOnCall()) + session_.acceptICECandidates(callCandidatesEvent.content.candidates); + else { + // CallInvite has been received and we're awaiting localUser to accept or + // reject the call + for (const auto &c : callCandidatesEvent.content.candidates) + remoteICECandidates_.push_back(c); + } + } +} + +void +CallManager::handleEvent(const RoomEvent &callAnswerEvent) +{ + nhlog::ui()->debug("WebRTC: call id: {} - incoming CallAnswer from {}", + callAnswerEvent.content.call_id, + callAnswerEvent.sender); + + if (callAnswerEvent.sender == utils::localUser().toStdString() && + callid_ == callAnswerEvent.content.call_id) { + if (!isOnCall()) { + emit ChatPage::instance()->showNotification("Call answered on another device."); + stopRingtone(); + haveCallInvite_ = false; + emit newInviteState(); + } + return; + } + + if (isOnCall() && callid_ == callAnswerEvent.content.call_id) { + stopRingtone(); + if (!session_.acceptAnswer(callAnswerEvent.content.sdp)) { + emit ChatPage::instance()->showNotification("Problem setting up call."); + hangUp(); + } + } +} + +void +CallManager::handleEvent(const RoomEvent &callHangUpEvent) +{ + nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}", + callHangUpEvent.content.call_id, + callHangUpReasonString(callHangUpEvent.content.reason), + callHangUpEvent.sender); + + if (callid_ == callHangUpEvent.content.call_id) + endCall(); +} + +void +CallManager::toggleMicMute() +{ + session_.toggleMicMute(); + emit micMuteChanged(); +} + +bool +CallManager::callsSupported() +{ +#ifdef GSTREAMER_AVAILABLE + return true; +#else + return false; +#endif +} + +bool +CallManager::screenShareSupported() +{ + return std::getenv("DISPLAY") && !std::getenv("WAYLAND_DISPLAY"); +} + +QStringList +CallManager::devices(bool isVideo) const +{ + QStringList ret; + const QString &defaultDevice = isVideo ? ChatPage::instance()->userSettings()->camera() + : ChatPage::instance()->userSettings()->microphone(); + std::vector devices = + CallDevices::instance().names(isVideo, defaultDevice.toStdString()); + ret.reserve(devices.size()); + std::transform(devices.cbegin(), devices.cend(), std::back_inserter(ret), [](const auto &d) { + return QString::fromStdString(d); + }); + + return ret; +} + +void +CallManager::generateCallID() +{ + using namespace std::chrono; + uint64_t ms = duration_cast(system_clock::now().time_since_epoch()).count(); + callid_ = "c" + std::to_string(ms); +} + +void +CallManager::clear() +{ + roomid_.clear(); + callParty_.clear(); + callPartyDisplayName_.clear(); + callPartyAvatarUrl_.clear(); + callid_.clear(); + callType_ = CallType::VOICE; + haveCallInvite_ = false; + emit newInviteState(); + inviteSDP_.clear(); + remoteICECandidates_.clear(); +} + +void +CallManager::endCall() +{ + stopRingtone(); + session_.end(); + clear(); +} + +void +CallManager::refreshTurnServer() +{ + turnURIs_.clear(); + turnServerTimer_.start(2000); +} + +void +CallManager::retrieveTurnServer() +{ + http::client()->get_turn_server( + [this](const mtx::responses::TurnServer &res, mtx::http::RequestErr err) { + if (err) { + turnServerTimer_.setInterval(5000); + return; + } + emit turnServerRetrieved(res); + }); +} + +void +CallManager::playRingtone(const QUrl &ringtone, bool repeat) +{ + static QMediaPlaylist playlist; + playlist.clear(); + playlist.setPlaybackMode(repeat ? QMediaPlaylist::CurrentItemInLoop + : QMediaPlaylist::CurrentItemOnce); + playlist.addMedia(ringtone); + player_.setVolume(100); + player_.setPlaylist(&playlist); +} + +void +CallManager::stopRingtone() +{ + player_.setPlaylist(nullptr); +} + +QStringList +CallManager::windowList() +{ + windows_.clear(); + windows_.push_back({tr("Entire screen"), 0}); + +#ifdef XCB_AVAILABLE + std::unique_ptr> connection( + xcb_connect(nullptr, nullptr), [](xcb_connection_t *c) { xcb_disconnect(c); }); + if (xcb_connection_has_error(connection.get())) { + nhlog::ui()->error("Failed to connect to X server"); + return {}; + } + + xcb_ewmh_connection_t ewmh; + if (!xcb_ewmh_init_atoms_replies( + &ewmh, xcb_ewmh_init_atoms(connection.get(), &ewmh), nullptr)) { + nhlog::ui()->error("Failed to connect to EWMH server"); + return {}; + } + std::unique_ptr> + ewmhconnection(&ewmh, [](xcb_ewmh_connection_t *c) { xcb_ewmh_connection_wipe(c); }); + + for (int i = 0; i < ewmh.nb_screens; i++) { + xcb_ewmh_get_windows_reply_t clients; + if (!xcb_ewmh_get_client_list_reply( + &ewmh, xcb_ewmh_get_client_list(&ewmh, i), &clients, nullptr)) { + nhlog::ui()->error("Failed to request window list"); + return {}; + } + + for (uint32_t w = 0; w < clients.windows_len; w++) { + xcb_window_t window = clients.windows[w]; + + std::string name; + xcb_ewmh_get_utf8_strings_reply_t data; + auto getName = [](xcb_ewmh_get_utf8_strings_reply_t *r) { + std::string name(r->strings, r->strings_len); + xcb_ewmh_get_utf8_strings_reply_wipe(r); + return name; + }; + + xcb_get_property_cookie_t cookie = xcb_ewmh_get_wm_name(&ewmh, window); + if (xcb_ewmh_get_wm_name_reply(&ewmh, cookie, &data, nullptr)) + name = getName(&data); + + cookie = xcb_ewmh_get_wm_visible_name(&ewmh, window); + if (xcb_ewmh_get_wm_visible_name_reply(&ewmh, cookie, &data, nullptr)) + name = getName(&data); + + windows_.push_back({QString::fromStdString(name), window}); + } + xcb_ewmh_get_windows_reply_wipe(&clients); + } +#endif + QStringList ret; + ret.reserve(windows_.size()); + for (const auto &w : windows_) + ret.append(w.first); + + return ret; +} + +#ifdef GSTREAMER_AVAILABLE +namespace { + +GstElement *pipe_ = nullptr; +unsigned int busWatchId_ = 0; + +gboolean +newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer G_GNUC_UNUSED) +{ + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + if (pipe_) { + gst_element_set_state(GST_ELEMENT(pipe_), GST_STATE_NULL); + gst_object_unref(pipe_); + pipe_ = nullptr; + } + if (busWatchId_) { + g_source_remove(busWatchId_); + busWatchId_ = 0; + } + break; + default: + break; + } + return TRUE; +} +} +#endif + +void +CallManager::previewWindow(unsigned int index) const +{ +#ifdef GSTREAMER_AVAILABLE + if (windows_.empty() || index >= windows_.size() || !gst_is_initialized()) + return; + + GstElement *ximagesrc = gst_element_factory_make("ximagesrc", nullptr); + if (!ximagesrc) { + nhlog::ui()->error("Failed to create ximagesrc"); + return; + } + GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); + GstElement *videoscale = gst_element_factory_make("videoscale", nullptr); + GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); + GstElement *ximagesink = gst_element_factory_make("ximagesink", nullptr); + + g_object_set(ximagesrc, "use-damage", FALSE, nullptr); + g_object_set(ximagesrc, "show-pointer", FALSE, nullptr); + g_object_set(ximagesrc, "xid", windows_[index].second, nullptr); + + GstCaps *caps = gst_caps_new_simple( + "video/x-raw", "width", G_TYPE_INT, 480, "height", G_TYPE_INT, 360, nullptr); + g_object_set(capsfilter, "caps", caps, nullptr); + gst_caps_unref(caps); + + pipe_ = gst_pipeline_new(nullptr); + gst_bin_add_many( + GST_BIN(pipe_), ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr); + if (!gst_element_link_many( + ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr)) { + nhlog::ui()->error("Failed to link preview window elements"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return; + } + if (gst_element_set_state(pipe_, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + nhlog::ui()->error("Unable to start preview pipeline"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return; + } + + GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_)); + busWatchId_ = gst_bus_add_watch(bus, newBusMessage, nullptr); + gst_object_unref(bus); +#else + (void)index; +#endif +} + +namespace { +std::vector +getTurnURIs(const mtx::responses::TurnServer &turnServer) +{ + // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp) + // where username and password are percent-encoded + std::vector ret; + for (const auto &uri : turnServer.uris) { + if (auto c = uri.find(':'); c == std::string::npos) { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } else { + std::string scheme = std::string(uri, 0, c); + if (scheme != "turn" && scheme != "turns") { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } + + QString encodedUri = + QString::fromStdString(scheme) + "://" + + QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" + + QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" + + QString::fromStdString(std::string(uri, ++c)); + ret.push_back(encodedUri.toStdString()); + } + } + return ret; +} +} diff --git a/src/voip/CallManager.h b/src/voip/CallManager.h new file mode 100644 index 00000000..22f31814 --- /dev/null +++ b/src/voip/CallManager.h @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "CallDevices.h" +#include "WebRTCSession.h" +#include "mtx/events/collections.hpp" +#include "mtx/events/voip.hpp" + +namespace mtx::responses { +struct TurnServer; +} + +class QStringList; +class QUrl; + +class CallManager : public QObject +{ + Q_OBJECT + Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState) + Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState) + Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState) + Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState) + Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) + Q_PROPERTY(QString callPartyDisplayName READ callPartyDisplayName NOTIFY newInviteState) + Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) + Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) + Q_PROPERTY(bool haveLocalPiP READ haveLocalPiP NOTIFY newCallState) + Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) + Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) + Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) + Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT) + +public: + CallManager(QObject *); + + bool haveCallInvite() const { return haveCallInvite_; } + bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; } + webrtc::CallType callType() const { return callType_; } + webrtc::State callState() const { return session_.state(); } + QString callParty() const { return callParty_; } + QString callPartyDisplayName() const { return callPartyDisplayName_; } + QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } + bool isMicMuted() const { return session_.isMicMuted(); } + bool haveLocalPiP() const { return session_.haveLocalPiP(); } + QStringList mics() const { return devices(false); } + QStringList cameras() const { return devices(true); } + void refreshTurnServer(); + + static bool callsSupported(); + static bool screenShareSupported(); + +public slots: + void sendInvite(const QString &roomid, webrtc::CallType, unsigned int windowIndex = 0); + void syncEvent(const mtx::events::collections::TimelineEvents &event); + void toggleMicMute(); + void toggleLocalPiP() { session_.toggleLocalPiP(); } + void acceptInvite(); + void hangUp(mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); + QStringList windowList(); + void previewWindow(unsigned int windowIndex) const; + +signals: + void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &); + void newMessage(const QString &roomid, const mtx::events::msg::CallCandidates &); + void newMessage(const QString &roomid, const mtx::events::msg::CallAnswer &); + void newMessage(const QString &roomid, const mtx::events::msg::CallHangUp &); + void newInviteState(); + void newCallState(); + void micMuteChanged(); + void devicesChanged(); + void turnServerRetrieved(const mtx::responses::TurnServer &); + +private slots: + void retrieveTurnServer(); + +private: + WebRTCSession &session_; + QString roomid_; + QString callParty_; + QString callPartyDisplayName_; + QString callPartyAvatarUrl_; + std::string callid_; + const uint32_t timeoutms_ = 120000; + webrtc::CallType callType_ = webrtc::CallType::VOICE; + bool haveCallInvite_ = false; + std::string inviteSDP_; + std::vector remoteICECandidates_; + std::vector turnURIs_; + QTimer turnServerTimer_; + QMediaPlayer player_; + std::vector> windows_; + + template + bool handleEvent(const mtx::events::collections::TimelineEvents &event); + void handleEvent(const mtx::events::RoomEvent &); + void handleEvent(const mtx::events::RoomEvent &); + void handleEvent(const mtx::events::RoomEvent &); + void handleEvent(const mtx::events::RoomEvent &); + void answerInvite(const mtx::events::msg::CallInvite &, bool isVideo); + void generateCallID(); + QStringList devices(bool isVideo) const; + void clear(); + void endCall(); + void playRingtone(const QUrl &ringtone, bool repeat); + void stopRingtone(); +}; diff --git a/src/voip/WebRTCSession.cpp b/src/voip/WebRTCSession.cpp new file mode 100644 index 00000000..801a365c --- /dev/null +++ b/src/voip/WebRTCSession.cpp @@ -0,0 +1,1155 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CallDevices.h" +#include "ChatPage.h" +#include "Logging.h" +#include "UserSettingsPage.h" +#include "WebRTCSession.h" + +#ifdef GSTREAMER_AVAILABLE +extern "C" +{ +#include "gst/gst.h" +#include "gst/sdp/sdp.h" + +#define GST_USE_UNSTABLE_API +#include "gst/webrtc/webrtc.h" +} +#endif + +// https://github.com/vector-im/riot-web/issues/10173 +#define STUN_SERVER "stun://turn.matrix.org:3478" + +Q_DECLARE_METATYPE(webrtc::CallType) +Q_DECLARE_METATYPE(webrtc::State) + +using webrtc::CallType; +using webrtc::State; + +WebRTCSession::WebRTCSession() + : devices_(CallDevices::instance()) +{ + qRegisterMetaType(); + qmlRegisterUncreatableMetaObject( + webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum"); + + qRegisterMetaType(); + qmlRegisterUncreatableMetaObject( + webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum"); + + connect(this, &WebRTCSession::stateChanged, this, &WebRTCSession::setState); + init(); +} + +bool +WebRTCSession::init(std::string *errorMessage) +{ +#ifdef GSTREAMER_AVAILABLE + if (initialised_) + return true; + + GError *error = nullptr; + if (!gst_init_check(nullptr, nullptr, &error)) { + std::string strError("WebRTC: failed to initialise GStreamer: "); + if (error) { + strError += error->message; + g_error_free(error); + } + nhlog::ui()->error(strError); + if (errorMessage) + *errorMessage = strError; + return false; + } + + initialised_ = true; + gchar *version = gst_version_string(); + nhlog::ui()->info("WebRTC: initialised {}", version); + g_free(version); + devices_.init(); + return true; +#else + (void)errorMessage; + return false; +#endif +} + +#ifdef GSTREAMER_AVAILABLE +namespace { + +std::string localsdp_; +std::vector localcandidates_; +bool haveAudioStream_ = false; +bool haveVideoStream_ = false; +GstPad *localPiPSinkPad_ = nullptr; +GstPad *remotePiPSinkPad_ = nullptr; + +gboolean +newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data) +{ + WebRTCSession *session = static_cast(user_data); + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + nhlog::ui()->error("WebRTC: end of stream"); + session->end(); + break; + case GST_MESSAGE_ERROR: + GError *error; + gchar *debug; + gst_message_parse_error(msg, &error, &debug); + nhlog::ui()->error( + "WebRTC: error from element {}: {}", GST_OBJECT_NAME(msg->src), error->message); + g_clear_error(&error); + g_free(debug); + session->end(); + break; + default: + break; + } + return TRUE; +} + +GstWebRTCSessionDescription * +parseSDP(const std::string &sdp, GstWebRTCSDPType type) +{ + GstSDPMessage *msg; + gst_sdp_message_new(&msg); + if (gst_sdp_message_parse_buffer((guint8 *)sdp.c_str(), sdp.size(), msg) == GST_SDP_OK) { + return gst_webrtc_session_description_new(type, msg); + } else { + nhlog::ui()->error("WebRTC: failed to parse remote session description"); + gst_sdp_message_free(msg); + return nullptr; + } +} + +void +setLocalDescription(GstPromise *promise, gpointer webrtc) +{ + const GstStructure *reply = gst_promise_get_reply(promise); + gboolean isAnswer = gst_structure_id_has_field(reply, g_quark_from_string("answer")); + GstWebRTCSessionDescription *gstsdp = nullptr; + gst_structure_get( + reply, isAnswer ? "answer" : "offer", GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &gstsdp, nullptr); + gst_promise_unref(promise); + g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr); + + gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp); + localsdp_ = std::string(sdp); + g_free(sdp); + gst_webrtc_session_description_free(gstsdp); + + nhlog::ui()->debug( + "WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_); +} + +void +createOffer(GstElement *webrtc) +{ + // create-offer first, then set-local-description + GstPromise *promise = gst_promise_new_with_change_func(setLocalDescription, webrtc, nullptr); + g_signal_emit_by_name(webrtc, "create-offer", nullptr, promise); +} + +void +createAnswer(GstPromise *promise, gpointer webrtc) +{ + // create-answer first, then set-local-description + gst_promise_unref(promise); + promise = gst_promise_new_with_change_func(setLocalDescription, webrtc, nullptr); + g_signal_emit_by_name(webrtc, "create-answer", nullptr, promise); +} + +void +iceGatheringStateChanged(GstElement *webrtc, + GParamSpec *pspec G_GNUC_UNUSED, + gpointer user_data G_GNUC_UNUSED) +{ + GstWebRTCICEGatheringState newState; + g_object_get(webrtc, "ice-gathering-state", &newState, nullptr); + if (newState == GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE) { + nhlog::ui()->debug("WebRTC: GstWebRTCICEGatheringState -> Complete"); + if (WebRTCSession::instance().isOffering()) { + emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_); + emit WebRTCSession::instance().stateChanged(State::OFFERSENT); + } else { + emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_); + emit WebRTCSession::instance().stateChanged(State::ANSWERSENT); + } + } +} + +void +addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, + guint mlineIndex, + gchar *candidate, + gpointer G_GNUC_UNUSED) +{ + nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate); + localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); +} + +void +iceConnectionStateChanged(GstElement *webrtc, + GParamSpec *pspec G_GNUC_UNUSED, + gpointer user_data G_GNUC_UNUSED) +{ + GstWebRTCICEConnectionState newState; + g_object_get(webrtc, "ice-connection-state", &newState, nullptr); + switch (newState) { + case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING: + nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking"); + emit WebRTCSession::instance().stateChanged(State::CONNECTING); + break; + case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED: + nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed"); + emit WebRTCSession::instance().stateChanged(State::ICEFAILED); + break; + default: + break; + } +} + +// https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1164 +struct KeyFrameRequestData +{ + GstElement *pipe = nullptr; + GstElement *decodebin = nullptr; + gint packetsLost = 0; + guint timerid = 0; + std::string statsField; +} keyFrameRequestData_; + +void +sendKeyFrameRequest() +{ + GstPad *sinkpad = gst_element_get_static_pad(keyFrameRequestData_.decodebin, "sink"); + if (!gst_pad_push_event(sinkpad, + gst_event_new_custom(GST_EVENT_CUSTOM_UPSTREAM, + gst_structure_new_empty("GstForceKeyUnit")))) + nhlog::ui()->error("WebRTC: key frame request failed"); + else + nhlog::ui()->debug("WebRTC: sent key frame request"); + + gst_object_unref(sinkpad); +} + +void +testPacketLoss_(GstPromise *promise, gpointer G_GNUC_UNUSED) +{ + const GstStructure *reply = gst_promise_get_reply(promise); + gint packetsLost = 0; + GstStructure *rtpStats; + if (!gst_structure_get( + reply, keyFrameRequestData_.statsField.c_str(), GST_TYPE_STRUCTURE, &rtpStats, nullptr)) { + nhlog::ui()->error("WebRTC: get-stats: no field: {}", keyFrameRequestData_.statsField); + gst_promise_unref(promise); + return; + } + gst_structure_get_int(rtpStats, "packets-lost", &packetsLost); + gst_structure_free(rtpStats); + gst_promise_unref(promise); + if (packetsLost > keyFrameRequestData_.packetsLost) { + nhlog::ui()->debug("WebRTC: inbound video lost packet count: {}", packetsLost); + keyFrameRequestData_.packetsLost = packetsLost; + sendKeyFrameRequest(); + } +} + +gboolean +testPacketLoss(gpointer G_GNUC_UNUSED) +{ + if (keyFrameRequestData_.pipe) { + GstElement *webrtc = gst_bin_get_by_name(GST_BIN(keyFrameRequestData_.pipe), "webrtcbin"); + GstPromise *promise = gst_promise_new_with_change_func(testPacketLoss_, nullptr, nullptr); + g_signal_emit_by_name(webrtc, "get-stats", nullptr, promise); + gst_object_unref(webrtc); + return TRUE; + } + return FALSE; +} + +void +setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointer G_GNUC_UNUSED) +{ + if (!std::strcmp( + gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(gst_element_get_factory(element))), + "rtpvp8depay")) + g_object_set(element, "wait-for-keyframe", TRUE, nullptr); +} + +GstElement * +newAudioSinkChain(GstElement *pipe) +{ + GstElement *queue = gst_element_factory_make("queue", nullptr); + GstElement *convert = gst_element_factory_make("audioconvert", nullptr); + GstElement *resample = gst_element_factory_make("audioresample", nullptr); + GstElement *sink = gst_element_factory_make("autoaudiosink", nullptr); + gst_bin_add_many(GST_BIN(pipe), queue, convert, resample, sink, nullptr); + gst_element_link_many(queue, convert, resample, sink, nullptr); + gst_element_sync_state_with_parent(queue); + gst_element_sync_state_with_parent(convert); + gst_element_sync_state_with_parent(resample); + gst_element_sync_state_with_parent(sink); + return queue; +} + +GstElement * +newVideoSinkChain(GstElement *pipe) +{ + // use compositor for now; acceleration needs investigation + GstElement *queue = gst_element_factory_make("queue", nullptr); + GstElement *compositor = gst_element_factory_make("compositor", "compositor"); + GstElement *glupload = gst_element_factory_make("glupload", nullptr); + GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr); + GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); + GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); + g_object_set(compositor, "background", 1, nullptr); + g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); + g_object_set(glsinkbin, "sink", qmlglsink, nullptr); + gst_bin_add_many( + GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); + gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); + gst_element_sync_state_with_parent(queue); + gst_element_sync_state_with_parent(compositor); + gst_element_sync_state_with_parent(glupload); + gst_element_sync_state_with_parent(glcolorconvert); + gst_element_sync_state_with_parent(glsinkbin); + return queue; +} + +std::pair +getResolution(GstPad *pad) +{ + std::pair ret; + GstCaps *caps = gst_pad_get_current_caps(pad); + const GstStructure *s = gst_caps_get_structure(caps, 0); + gst_structure_get_int(s, "width", &ret.first); + gst_structure_get_int(s, "height", &ret.second); + gst_caps_unref(caps); + return ret; +} + +std::pair +getResolution(GstElement *pipe, const gchar *elementName, const gchar *padName) +{ + GstElement *element = gst_bin_get_by_name(GST_BIN(pipe), elementName); + GstPad *pad = gst_element_get_static_pad(element, padName); + auto ret = getResolution(pad); + gst_object_unref(pad); + gst_object_unref(element); + return ret; +} + +std::pair +getPiPDimensions(const std::pair &resolution, int fullWidth, double scaleFactor) +{ + int pipWidth = fullWidth * scaleFactor; + int pipHeight = static_cast(resolution.second) / resolution.first * pipWidth; + return {pipWidth, pipHeight}; +} + +void +addLocalPiP(GstElement *pipe, const std::pair &videoCallSize) +{ + // embed localUser's camera into received video (CallType::VIDEO) + // OR embed screen share into received video (CallType::SCREEN) + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); + if (!tee) + return; + + GstElement *queue = gst_element_factory_make("queue", nullptr); + gst_bin_add(GST_BIN(pipe), queue); + gst_element_link(tee, queue); + gst_element_sync_state_with_parent(queue); + gst_object_unref(tee); + + GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor"); + localPiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); + g_object_set(localPiPSinkPad_, "zorder", 2, nullptr); + + bool isVideo = WebRTCSession::instance().callType() == CallType::VIDEO; + const gchar *element = isVideo ? "camerafilter" : "screenshare"; + const gchar *pad = isVideo ? "sink" : "src"; + auto resolution = getResolution(pipe, element, pad); + auto pipSize = getPiPDimensions(resolution, videoCallSize.first, 0.25); + nhlog::ui()->debug("WebRTC: local picture-in-picture: {}x{}", pipSize.first, pipSize.second); + g_object_set(localPiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); + gint offset = videoCallSize.first / 80; + g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr); + + GstPad *srcpad = gst_element_get_static_pad(queue, "src"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, localPiPSinkPad_))) + nhlog::ui()->error("WebRTC: failed to link local PiP elements"); + gst_object_unref(srcpad); + gst_object_unref(compositor); +} + +void +addRemotePiP(GstElement *pipe) +{ + // embed localUser's camera into screen image being shared + if (remotePiPSinkPad_) { + auto camRes = getResolution(pipe, "camerafilter", "sink"); + auto shareRes = getResolution(pipe, "screenshare", "src"); + auto pipSize = getPiPDimensions(camRes, shareRes.first, 0.2); + nhlog::ui()->debug( + "WebRTC: screen share picture-in-picture: {}x{}", pipSize.first, pipSize.second); + + gint offset = shareRes.first / 100; + g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr); + g_object_set(remotePiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); + g_object_set(remotePiPSinkPad_, + "xpos", + shareRes.first - pipSize.first - offset, + "ypos", + shareRes.second - pipSize.second - offset, + nullptr); + } +} + +void +addLocalVideo(GstElement *pipe) +{ + GstElement *queue = newVideoSinkChain(pipe); + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); + GstPad *srcpad = gst_element_get_request_pad(tee, "src_%u"); + GstPad *sinkpad = gst_element_get_static_pad(queue, "sink"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, sinkpad))) + nhlog::ui()->error("WebRTC: failed to link videosrctee -> video sink chain"); + gst_object_unref(srcpad); +} + +void +linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) +{ + GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); + GstCaps *sinkcaps = gst_pad_get_current_caps(sinkpad); + const GstStructure *structure = gst_caps_get_structure(sinkcaps, 0); + + gchar *mediaType = nullptr; + guint ssrc = 0; + gst_structure_get( + structure, "media", G_TYPE_STRING, &mediaType, "ssrc", G_TYPE_UINT, &ssrc, nullptr); + gst_caps_unref(sinkcaps); + gst_object_unref(sinkpad); + + WebRTCSession *session = &WebRTCSession::instance(); + GstElement *queue = nullptr; + if (!std::strcmp(mediaType, "audio")) { + nhlog::ui()->debug("WebRTC: received incoming audio stream"); + haveAudioStream_ = true; + queue = newAudioSinkChain(pipe); + } else if (!std::strcmp(mediaType, "video")) { + nhlog::ui()->debug("WebRTC: received incoming video stream"); + if (!session->getVideoItem()) { + g_free(mediaType); + nhlog::ui()->error("WebRTC: video call item not set"); + return; + } + haveVideoStream_ = true; + keyFrameRequestData_.statsField = + std::string("rtp-inbound-stream-stats_") + std::to_string(ssrc); + queue = newVideoSinkChain(pipe); + auto videoCallSize = getResolution(newpad); + nhlog::ui()->info( + "WebRTC: incoming video resolution: {}x{}", videoCallSize.first, videoCallSize.second); + addLocalPiP(pipe, videoCallSize); + } else { + g_free(mediaType); + nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); + return; + } + + GstPad *queuepad = gst_element_get_static_pad(queue, "sink"); + if (queuepad) { + if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad))) + nhlog::ui()->error("WebRTC: unable to link new pad"); + else { + if (session->callType() == CallType::VOICE || + (haveAudioStream_ && (haveVideoStream_ || session->isRemoteVideoRecvOnly()))) { + emit session->stateChanged(State::CONNECTED); + if (haveVideoStream_) { + keyFrameRequestData_.pipe = pipe; + keyFrameRequestData_.decodebin = decodebin; + keyFrameRequestData_.timerid = + g_timeout_add_seconds(3, testPacketLoss, nullptr); + } + addRemotePiP(pipe); + if (session->isRemoteVideoRecvOnly()) + addLocalVideo(pipe); + } + } + gst_object_unref(queuepad); + } + g_free(mediaType); +} + +void +addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe) +{ + if (GST_PAD_DIRECTION(newpad) != GST_PAD_SRC) + return; + + nhlog::ui()->debug("WebRTC: received incoming stream"); + GstElement *decodebin = gst_element_factory_make("decodebin", nullptr); + // hardware decoding needs investigation; eg rendering fails if vaapi plugin installed + g_object_set(decodebin, "force-sw-decoders", TRUE, nullptr); + g_signal_connect(decodebin, "pad-added", G_CALLBACK(linkNewPad), pipe); + g_signal_connect(decodebin, "element-added", G_CALLBACK(setWaitForKeyFrame), nullptr); + gst_bin_add(GST_BIN(pipe), decodebin); + gst_element_sync_state_with_parent(decodebin); + GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); + if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, sinkpad))) + nhlog::ui()->error("WebRTC: unable to link decodebin"); + gst_object_unref(sinkpad); +} + +bool +contains(std::string_view str1, std::string_view str2) +{ + return std::search(str1.cbegin(), + str1.cend(), + str2.cbegin(), + str2.cend(), + [](unsigned char c1, unsigned char c2) { + return std::tolower(c1) == std::tolower(c2); + }) != str1.cend(); +} + +bool +getMediaAttributes(const GstSDPMessage *sdp, + const char *mediaType, + const char *encoding, + int &payloadType, + bool &recvOnly, + bool &sendOnly) +{ + payloadType = -1; + recvOnly = false; + sendOnly = false; + for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) { + const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex); + if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) { + recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr; + sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr; + const gchar *rtpval = nullptr; + for (guint n = 0; n == 0 || rtpval; ++n) { + rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n); + if (rtpval && contains(rtpval, encoding)) { + payloadType = std::atoi(rtpval); + break; + } + } + return true; + } + } + return false; +} +} + +bool +WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) +{ + if (!initialised_ && !init(errorMessage)) + return false; + if (!isVideo && haveVoicePlugins_) + return true; + if (isVideo && haveVideoPlugins_) + return true; + + const gchar *voicePlugins[] = {"audioconvert", + "audioresample", + "autodetect", + "dtls", + "nice", + "opus", + "playback", + "rtpmanager", + "srtp", + "volume", + "webrtc", + nullptr}; + + const gchar *videoPlugins[] = { + "compositor", "opengl", "qmlgl", "rtp", "videoconvert", "vpx", nullptr}; + + std::string strError("Missing GStreamer plugins: "); + const gchar **needed = isVideo ? videoPlugins : voicePlugins; + bool &havePlugins = isVideo ? haveVideoPlugins_ : haveVoicePlugins_; + havePlugins = true; + GstRegistry *registry = gst_registry_get(); + for (guint i = 0; i < g_strv_length((gchar **)needed); i++) { + GstPlugin *plugin = gst_registry_find_plugin(registry, needed[i]); + if (!plugin) { + havePlugins = false; + strError += std::string(needed[i]) + " "; + continue; + } + gst_object_unref(plugin); + } + if (!havePlugins) { + nhlog::ui()->error(strError); + if (errorMessage) + *errorMessage = strError; + return false; + } + + if (isVideo) { + // load qmlglsink to register GStreamer's GstGLVideoItem QML type + GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); + gst_object_unref(qmlglsink); + } + return true; +} + +bool +WebRTCSession::createOffer(CallType callType, uint32_t shareWindowId) +{ + clear(); + isOffering_ = true; + callType_ = callType; + shareWindowId_ = shareWindowId; + + // opus and vp8 rtp payload types must be defined dynamically + // therefore from the range [96-127] + // see for example https://tools.ietf.org/html/rfc7587 + constexpr int opusPayloadType = 111; + constexpr int vp8PayloadType = 96; + return startPipeline(opusPayloadType, vp8PayloadType); +} + +bool +WebRTCSession::acceptOffer(const std::string &sdp) +{ + nhlog::ui()->debug("WebRTC: received offer:\n{}", sdp); + if (state_ != State::DISCONNECTED) + return false; + + clear(); + GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER); + if (!offer) + return false; + + int opusPayloadType; + bool recvOnly; + bool sendOnly; + if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) { + if (opusPayloadType == -1) { + nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding"); + gst_webrtc_session_description_free(offer); + return false; + } + } else { + nhlog::ui()->error("WebRTC: remote offer - no audio media"); + gst_webrtc_session_description_free(offer); + return false; + } + + int vp8PayloadType; + bool isVideo = getMediaAttributes( + offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_, isRemoteVideoSendOnly_); + if (isVideo && vp8PayloadType == -1) { + nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding"); + gst_webrtc_session_description_free(offer); + return false; + } + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; + + if (!startPipeline(opusPayloadType, vp8PayloadType)) { + gst_webrtc_session_description_free(offer); + return false; + } + + // avoid a race that sometimes leaves the generated answer without media tracks (a=ssrc + // lines) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // set-remote-description first, then create-answer + GstPromise *promise = gst_promise_new_with_change_func(createAnswer, webrtc_, nullptr); + g_signal_emit_by_name(webrtc_, "set-remote-description", offer, promise); + gst_webrtc_session_description_free(offer); + return true; +} + +bool +WebRTCSession::acceptAnswer(const std::string &sdp) +{ + nhlog::ui()->debug("WebRTC: received answer:\n{}", sdp); + if (state_ != State::OFFERSENT) + return false; + + GstWebRTCSessionDescription *answer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_ANSWER); + if (!answer) { + end(); + return false; + } + + if (callType_ != CallType::VOICE) { + int unused; + if (!getMediaAttributes( + answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_, isRemoteVideoSendOnly_)) + isRemoteVideoRecvOnly_ = true; + } + + g_signal_emit_by_name(webrtc_, "set-remote-description", answer, nullptr); + gst_webrtc_session_description_free(answer); + return true; +} + +void +WebRTCSession::acceptICECandidates( + const std::vector &candidates) +{ + if (state_ >= State::INITIATED) { + for (const auto &c : candidates) { + nhlog::ui()->debug( + "WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate); + if (!c.candidate.empty()) { + g_signal_emit_by_name( + webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str()); + } + } + } +} + +bool +WebRTCSession::startPipeline(int opusPayloadType, int vp8PayloadType) +{ + if (state_ != State::DISCONNECTED) + return false; + + emit stateChanged(State::INITIATING); + + if (!createPipeline(opusPayloadType, vp8PayloadType)) { + end(); + return false; + } + + webrtc_ = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); + + if (ChatPage::instance()->userSettings()->useStunServer()) { + nhlog::ui()->info("WebRTC: setting STUN server: {}", STUN_SERVER); + g_object_set(webrtc_, "stun-server", STUN_SERVER, nullptr); + } + + for (const auto &uri : turnServers_) { + nhlog::ui()->info("WebRTC: setting TURN server: {}", uri); + gboolean udata; + g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata)); + } + if (turnServers_.empty()) + nhlog::ui()->warn("WebRTC: no TURN server provided"); + + // generate the offer when the pipeline goes to PLAYING + if (isOffering_) + g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(::createOffer), nullptr); + + // on-ice-candidate is emitted when a local ICE candidate has been gathered + g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr); + + // capture ICE failure + g_signal_connect( + webrtc_, "notify::ice-connection-state", G_CALLBACK(iceConnectionStateChanged), nullptr); + + // incoming streams trigger pad-added + gst_element_set_state(pipe_, GST_STATE_READY); + g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_); + + // capture ICE gathering completion + g_signal_connect( + webrtc_, "notify::ice-gathering-state", G_CALLBACK(iceGatheringStateChanged), nullptr); + + // webrtcbin lifetime is the same as that of the pipeline + gst_object_unref(webrtc_); + + // start the pipeline + GstStateChangeReturn ret = gst_element_set_state(pipe_, GST_STATE_PLAYING); + if (ret == GST_STATE_CHANGE_FAILURE) { + nhlog::ui()->error("WebRTC: unable to start pipeline"); + end(); + return false; + } + + GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_)); + busWatchId_ = gst_bus_add_watch(bus, newBusMessage, this); + gst_object_unref(bus); + emit stateChanged(State::INITIATED); + return true; +} + +bool +WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType) +{ + GstDevice *device = devices_.audioDevice(); + if (!device) + return false; + + GstElement *source = gst_device_create_element(device, nullptr); + GstElement *volume = gst_element_factory_make("volume", "srclevel"); + GstElement *convert = gst_element_factory_make("audioconvert", nullptr); + GstElement *resample = gst_element_factory_make("audioresample", nullptr); + GstElement *queue1 = gst_element_factory_make("queue", nullptr); + GstElement *opusenc = gst_element_factory_make("opusenc", nullptr); + GstElement *rtp = gst_element_factory_make("rtpopuspay", nullptr); + GstElement *queue2 = gst_element_factory_make("queue", nullptr); + GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); + + GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp", + "media", + G_TYPE_STRING, + "audio", + "encoding-name", + G_TYPE_STRING, + "OPUS", + "payload", + G_TYPE_INT, + opusPayloadType, + nullptr); + g_object_set(capsfilter, "caps", rtpcaps, nullptr); + gst_caps_unref(rtpcaps); + + GstElement *webrtcbin = gst_element_factory_make("webrtcbin", "webrtcbin"); + g_object_set(webrtcbin, "bundle-policy", GST_WEBRTC_BUNDLE_POLICY_MAX_BUNDLE, nullptr); + + pipe_ = gst_pipeline_new(nullptr); + gst_bin_add_many(GST_BIN(pipe_), + source, + volume, + convert, + resample, + queue1, + opusenc, + rtp, + queue2, + capsfilter, + webrtcbin, + nullptr); + + if (!gst_element_link_many(source, + volume, + convert, + resample, + queue1, + opusenc, + rtp, + queue2, + capsfilter, + webrtcbin, + nullptr)) { + nhlog::ui()->error("WebRTC: failed to link audio pipeline elements"); + return false; + } + + return callType_ == CallType::VOICE || isRemoteVideoSendOnly_ + ? true + : addVideoPipeline(vp8PayloadType); +} + +bool +WebRTCSession::addVideoPipeline(int vp8PayloadType) +{ + // allow incoming video calls despite localUser having no webcam + if (callType_ == CallType::VIDEO && !devices_.haveCamera()) + return !isOffering_; + + auto settings = ChatPage::instance()->userSettings(); + GstElement *camerafilter = nullptr; + GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); + GstElement *tee = gst_element_factory_make("tee", "videosrctee"); + gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr); + if (callType_ == CallType::VIDEO || (settings->screenSharePiP() && devices_.haveCamera())) { + std::pair resolution; + std::pair frameRate; + GstDevice *device = devices_.videoDevice(resolution, frameRate); + if (!device) + return false; + + GstElement *camera = gst_device_create_element(device, nullptr); + GstCaps *caps = gst_caps_new_simple("video/x-raw", + "width", + G_TYPE_INT, + resolution.first, + "height", + G_TYPE_INT, + resolution.second, + "framerate", + GST_TYPE_FRACTION, + frameRate.first, + frameRate.second, + nullptr); + camerafilter = gst_element_factory_make("capsfilter", "camerafilter"); + g_object_set(camerafilter, "caps", caps, nullptr); + gst_caps_unref(caps); + + gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr); + if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link camera elements"); + return false; + } + if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) { + nhlog::ui()->error("WebRTC: failed to link camerafilter -> tee"); + return false; + } + } + + if (callType_ == CallType::SCREEN) { + nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", + settings->screenShareFrameRate()); + nhlog::ui()->debug("WebRTC: screen share picture-in-picture: {}", + settings->screenSharePiP()); + nhlog::ui()->debug("WebRTC: screen share request remote camera: {}", + settings->screenShareRemoteVideo()); + nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}", + settings->screenShareHideCursor()); + + GstElement *ximagesrc = gst_element_factory_make("ximagesrc", "screenshare"); + if (!ximagesrc) { + nhlog::ui()->error("WebRTC: failed to create ximagesrc"); + return false; + } + g_object_set(ximagesrc, "use-damage", FALSE, nullptr); + g_object_set(ximagesrc, "xid", shareWindowId_, nullptr); + g_object_set(ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr); + + GstCaps *caps = gst_caps_new_simple("video/x-raw", + "framerate", + GST_TYPE_FRACTION, + settings->screenShareFrameRate(), + 1, + nullptr); + GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); + g_object_set(capsfilter, "caps", caps, nullptr); + gst_caps_unref(caps); + gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr); + + if (settings->screenSharePiP() && devices_.haveCamera()) { + GstElement *compositor = gst_element_factory_make("compositor", nullptr); + g_object_set(compositor, "background", 1, nullptr); + gst_bin_add(GST_BIN(pipe_), compositor); + if (!gst_element_link_many(ximagesrc, compositor, capsfilter, tee, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link screen share elements"); + return false; + } + + GstPad *srcpad = gst_element_get_static_pad(camerafilter, "src"); + remotePiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, remotePiPSinkPad_))) { + nhlog::ui()->error("WebRTC: failed to link camerafilter -> compositor"); + gst_object_unref(srcpad); + return false; + } + gst_object_unref(srcpad); + } else if (!gst_element_link_many(ximagesrc, videoconvert, capsfilter, tee, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link screen share elements"); + return false; + } + } + + GstElement *queue = gst_element_factory_make("queue", nullptr); + GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); + g_object_set(vp8enc, "deadline", 1, nullptr); + g_object_set(vp8enc, "error-resilient", 1, nullptr); + GstElement *rtpvp8pay = gst_element_factory_make("rtpvp8pay", nullptr); + GstElement *rtpqueue = gst_element_factory_make("queue", nullptr); + GstElement *rtpcapsfilter = gst_element_factory_make("capsfilter", nullptr); + GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp", + "media", + G_TYPE_STRING, + "video", + "encoding-name", + G_TYPE_STRING, + "VP8", + "payload", + G_TYPE_INT, + vp8PayloadType, + nullptr); + g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr); + gst_caps_unref(rtpcaps); + + gst_bin_add_many(GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr); + + GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); + if (!gst_element_link_many( + tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link rtp video elements"); + gst_object_unref(webrtcbin); + return false; + } + + if (callType_ == CallType::SCREEN && + !ChatPage::instance()->userSettings()->screenShareRemoteVideo()) { + GArray *transceivers; + g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers); + GstWebRTCRTPTransceiver *transceiver = + g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1); + transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + g_array_unref(transceivers); + } + + gst_object_unref(webrtcbin); + return true; +} + +bool +WebRTCSession::haveLocalPiP() const +{ + if (state_ >= State::INITIATED) { + if (callType_ == CallType::VOICE || isRemoteVideoRecvOnly_) + return false; + else if (callType_ == CallType::SCREEN) + return true; + else { + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); + if (tee) { + gst_object_unref(tee); + return true; + } + } + } + return false; +} + +bool +WebRTCSession::isMicMuted() const +{ + if (state_ < State::INITIATED) + return false; + + GstElement *srclevel = gst_bin_get_by_name(GST_BIN(pipe_), "srclevel"); + gboolean muted; + g_object_get(srclevel, "mute", &muted, nullptr); + gst_object_unref(srclevel); + return muted; +} + +bool +WebRTCSession::toggleMicMute() +{ + if (state_ < State::INITIATED) + return false; + + GstElement *srclevel = gst_bin_get_by_name(GST_BIN(pipe_), "srclevel"); + gboolean muted; + g_object_get(srclevel, "mute", &muted, nullptr); + g_object_set(srclevel, "mute", !muted, nullptr); + gst_object_unref(srclevel); + return !muted; +} + +void +WebRTCSession::toggleLocalPiP() +{ + if (localPiPSinkPad_) { + guint zorder; + g_object_get(localPiPSinkPad_, "zorder", &zorder, nullptr); + g_object_set(localPiPSinkPad_, "zorder", zorder ? 0 : 2, nullptr); + } +} + +void +WebRTCSession::clear() +{ + callType_ = webrtc::CallType::VOICE; + isOffering_ = false; + isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; + videoItem_ = nullptr; + pipe_ = nullptr; + webrtc_ = nullptr; + busWatchId_ = 0; + shareWindowId_ = 0; + haveAudioStream_ = false; + haveVideoStream_ = false; + localPiPSinkPad_ = nullptr; + remotePiPSinkPad_ = nullptr; + localsdp_.clear(); + localcandidates_.clear(); +} + +void +WebRTCSession::end() +{ + nhlog::ui()->debug("WebRTC: ending session"); + keyFrameRequestData_ = KeyFrameRequestData{}; + if (pipe_) { + gst_element_set_state(pipe_, GST_STATE_NULL); + gst_object_unref(pipe_); + pipe_ = nullptr; + if (busWatchId_) { + g_source_remove(busWatchId_); + busWatchId_ = 0; + } + } + + clear(); + if (state_ != State::DISCONNECTED) + emit stateChanged(State::DISCONNECTED); +} + +#else + +bool +WebRTCSession::havePlugins(bool, std::string *) +{ + return false; +} + +bool +WebRTCSession::haveLocalPiP() const +{ + return false; +} + +bool WebRTCSession::createOffer(webrtc::CallType, uint32_t) { return false; } + +bool +WebRTCSession::acceptOffer(const std::string &) +{ + return false; +} + +bool +WebRTCSession::acceptAnswer(const std::string &) +{ + return false; +} + +void +WebRTCSession::acceptICECandidates(const std::vector &) +{} + +bool +WebRTCSession::isMicMuted() const +{ + return false; +} + +bool +WebRTCSession::toggleMicMute() +{ + return false; +} + +void +WebRTCSession::toggleLocalPiP() +{} + +void +WebRTCSession::end() +{} + +#endif diff --git a/src/voip/WebRTCSession.h b/src/voip/WebRTCSession.h new file mode 100644 index 00000000..56c0a295 --- /dev/null +++ b/src/voip/WebRTCSession.h @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include + +#include "mtx/events/voip.hpp" + +typedef struct _GstElement GstElement; +class CallDevices; +class QQuickItem; + +namespace webrtc { +Q_NAMESPACE + +enum class CallType +{ + VOICE, + VIDEO, + SCREEN // localUser is sharing screen +}; +Q_ENUM_NS(CallType) + +enum class State +{ + DISCONNECTED, + ICEFAILED, + INITIATING, + INITIATED, + OFFERSENT, + ANSWERSENT, + CONNECTING, + CONNECTED + +}; +Q_ENUM_NS(State) +} + +class WebRTCSession : public QObject +{ + Q_OBJECT + +public: + static WebRTCSession &instance() + { + static WebRTCSession instance; + return instance; + } + + bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); + webrtc::CallType callType() const { return callType_; } + webrtc::State state() const { return state_; } + bool haveLocalPiP() const; + bool isOffering() const { return isOffering_; } + bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } + bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } + + bool createOffer(webrtc::CallType, uint32_t shareWindowId); + bool acceptOffer(const std::string &sdp); + bool acceptAnswer(const std::string &sdp); + void acceptICECandidates(const std::vector &); + + bool isMicMuted() const; + bool toggleMicMute(); + void toggleLocalPiP(); + void end(); + + void setTurnServers(const std::vector &uris) { turnServers_ = uris; } + + void setVideoItem(QQuickItem *item) { videoItem_ = item; } + QQuickItem *getVideoItem() const { return videoItem_; } + +signals: + void offerCreated(const std::string &sdp, + const std::vector &); + void answerCreated(const std::string &sdp, + const std::vector &); + void newICECandidate(const mtx::events::msg::CallCandidates::Candidate &); + void stateChanged(webrtc::State); + +private slots: + void setState(webrtc::State state) { state_ = state; } + +private: + WebRTCSession(); + + CallDevices &devices_; + bool initialised_ = false; + bool haveVoicePlugins_ = false; + bool haveVideoPlugins_ = false; + webrtc::CallType callType_ = webrtc::CallType::VOICE; + webrtc::State state_ = webrtc::State::DISCONNECTED; + bool isOffering_ = false; + bool isRemoteVideoRecvOnly_ = false; + bool isRemoteVideoSendOnly_ = false; + QQuickItem *videoItem_ = nullptr; + GstElement *pipe_ = nullptr; + GstElement *webrtc_ = nullptr; + unsigned int busWatchId_ = 0; + std::vector turnServers_; + uint32_t shareWindowId_ = 0; + + bool init(std::string *errorMessage = nullptr); + bool startPipeline(int opusPayloadType, int vp8PayloadType); + bool createPipeline(int opusPayloadType, int vp8PayloadType); + bool addVideoPipeline(int vp8PayloadType); + void clear(); + +public: + WebRTCSession(WebRTCSession const &) = delete; + void operator=(WebRTCSession const &) = delete; +}; -- cgit 1.5.1