From 8d3ab300b650dcef3d12fe1e5a4bb14855e8495a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 9 Jul 2020 00:45:37 +0200 Subject: Open url using QDesktopServices from Qml This fixes some weirdness on some platforms, where Qt.openUrlExternally does weird stuff. --- resources/qml/MatrixText.qml | 2 +- src/timeline/TimelineViewManager.cpp | 7 +++++++ src/timeline/TimelineViewManager.h | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/resources/qml/MatrixText.qml b/resources/qml/MatrixText.qml index 9a4f7348..d56143dd 100644 --- a/resources/qml/MatrixText.qml +++ b/resources/qml/MatrixText.qml @@ -16,7 +16,7 @@ TextEdit { timelineManager.setHistoryView(match[1]) chat.positionViewAtIndex(chat.model.idToIndex(match[2]), ListView.Contain) } - else Qt.openUrlExternally(link) + else timelineManager.openLink(link) } MouseArea { diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index 64af8afb..060f52fe 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -1,5 +1,6 @@ #include "TimelineViewManager.h" +#include #include #include #include @@ -219,6 +220,12 @@ TimelineViewManager::openImageOverlay(QString mxcUrl, QString eventId) const }); } +void +TimelineViewManager::openLink(QString link) const +{ + QDesktopServices::openUrl(link); +} + void TimelineViewManager::updateReadReceipts(const QString &room_id, const std::vector &event_ids) diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h index ed095058..81a8b9b4 100644 --- a/src/timeline/TimelineViewManager.h +++ b/src/timeline/TimelineViewManager.h @@ -47,6 +47,8 @@ public: Q_INVOKABLE QString userPresence(QString id) const; Q_INVOKABLE QString userStatus(QString id) const; + Q_INVOKABLE void openLink(QString link) const; + signals: void clearRoomMessageCount(QString roomid); void updateRoomsLastMessage(QString roomid, const DescInfo &info); -- cgit 1.4.1 From 7650e6ced60aad073d5c5f4bf64b34be0655759f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 9 Jul 2020 13:26:01 +0200 Subject: Fix autolinking breaking on single quotes in href attribute Fixes #234 --- src/Config.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Config.h b/src/Config.h index f99cf36b..c0624709 100644 --- a/src/Config.h +++ b/src/Config.h @@ -53,9 +53,9 @@ namespace strings { const QString url_html = "\\1"; const QRegularExpression url_regex( // match an URL, that is not quoted, i.e. - // vvvvvv match quote via negative lookahead/lookbehind vv - // vvvv atomic match url -> fail if there is a " before or after vvv - R"((?((www\.(?!\.)|[a-z][a-z0-9+.-]*://)[^\s<>'"]+[^!,\.\s<>'"\]\)\:]))(?!"))"); + // vvvvvv match quote via negative lookahead/lookbehind vv + // vvvv atomic match url -> fail if there is a " before or after vvv + R"((?((www\.(?!\.)|[a-z][a-z0-9+.-]*://)[^\s<>'"]+[^!,\.\s<>'"\]\)\:]))(?!["']))"); } // Window geometry. -- cgit 1.4.1 From 7a206441c86cd2aa84cbbbc6be803f03b2f355ab Mon Sep 17 00:00:00 2001 From: trilene Date: Fri, 10 Jul 2020 19:19:48 -0400 Subject: Support voice calls --- CMakeLists.txt | 16 + resources/langs/nheko_en.ts | 45 +++ resources/media/callend.mp3 | Bin 0 -> 12971 bytes resources/media/callend.ogg | Bin 0 -> 13932 bytes resources/media/ring.mp3 | Bin 0 -> 36119 bytes resources/media/ring.ogg | Bin 0 -> 66715 bytes resources/media/ringback.mp3 | Bin 0 -> 18398 bytes resources/media/ringback.ogg | Bin 0 -> 8352 bytes resources/qml/delegates/MessageDelegate.qml | 18 ++ resources/res.qrc | 5 + src/ActiveCallBar.cpp | 74 +++++ src/ActiveCallBar.h | 26 ++ src/Cache.cpp | 3 + src/CallManager.cpp | 315 ++++++++++++++++++++ src/CallManager.h | 67 +++++ src/ChatPage.cpp | 61 +++- src/ChatPage.h | 8 +- src/TextInputWidget.cpp | 26 ++ src/TextInputWidget.h | 3 + src/UserSettingsPage.cpp | 31 ++ src/UserSettingsPage.h | 7 + src/Utils.cpp | 40 ++- src/Utils.h | 45 ++- src/WebRTCSession.cpp | 438 ++++++++++++++++++++++++++++ src/WebRTCSession.h | 58 ++++ src/dialogs/AcceptCall.cpp | 53 ++++ src/dialogs/AcceptCall.h | 26 ++ src/dialogs/PlaceCall.cpp | 60 ++++ src/dialogs/PlaceCall.h | 28 ++ src/timeline/TimelineModel.cpp | 205 +++++++++---- src/timeline/TimelineModel.h | 21 +- src/timeline/TimelineViewManager.cpp | 60 +++- src/timeline/TimelineViewManager.h | 13 +- 33 files changed, 1653 insertions(+), 99 deletions(-) create mode 100644 resources/media/callend.mp3 create mode 100644 resources/media/callend.ogg create mode 100644 resources/media/ring.mp3 create mode 100644 resources/media/ring.ogg create mode 100644 resources/media/ringback.mp3 create mode 100644 resources/media/ringback.ogg create mode 100644 src/ActiveCallBar.cpp create mode 100644 src/ActiveCallBar.h create mode 100644 src/CallManager.cpp create mode 100644 src/CallManager.h create mode 100644 src/WebRTCSession.cpp create mode 100644 src/WebRTCSession.h create mode 100644 src/dialogs/AcceptCall.cpp create mode 100644 src/dialogs/AcceptCall.h create mode 100644 src/dialogs/PlaceCall.cpp create mode 100644 src/dialogs/PlaceCall.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 04baf360..5b47b0af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,6 +227,7 @@ configure_file(cmake/nheko.h config/nheko.h) # set(SRC_FILES # Dialogs + src/dialogs/AcceptCall.cpp src/dialogs/CreateRoom.cpp src/dialogs/FallbackAuth.cpp src/dialogs/ImageOverlay.cpp @@ -235,6 +236,7 @@ set(SRC_FILES src/dialogs/LeaveRoom.cpp src/dialogs/Logout.cpp src/dialogs/MemberList.cpp + src/dialogs/PlaceCall.cpp src/dialogs/PreviewUploadOverlay.cpp src/dialogs/ReCaptcha.cpp src/dialogs/ReadReceipts.cpp @@ -278,9 +280,11 @@ set(SRC_FILES src/ui/Theme.cpp src/ui/ThemeManager.cpp + src/ActiveCallBar.cpp src/AvatarProvider.cpp src/BlurhashProvider.cpp src/Cache.cpp + src/CallManager.cpp src/ChatPage.cpp src/ColorImageProvider.cpp src/CommunitiesList.cpp @@ -306,6 +310,7 @@ set(SRC_FILES src/UserInfoWidget.cpp src/UserSettingsPage.cpp src/Utils.cpp + src/WebRTCSession.cpp src/WelcomePage.cpp src/popups/PopupItem.cpp src/popups/SuggestionsPopup.cpp @@ -423,6 +428,10 @@ else() find_package(Tweeny REQUIRED) endif() +include(FindPkgConfig) +pkg_check_modules(GST_SDP REQUIRED IMPORTED_TARGET gstreamer-sdp-1.0>=1.14) +pkg_check_modules(GST_WEBRTC REQUIRED IMPORTED_TARGET gstreamer-webrtc-1.0>=1.14) + # single instance functionality set(QAPPLICATION_CLASS QApplication CACHE STRING "Inheritance class for SingleApplication") add_subdirectory(third_party/SingleApplication-3.1.3.1/) @@ -431,6 +440,7 @@ feature_summary(WHAT ALL INCLUDE_QUIET_PACKAGES FATAL_ON_MISSING_REQUIRED_PACKAG qt5_wrap_cpp(MOC_HEADERS # Dialogs + src/dialogs/AcceptCall.h src/dialogs/CreateRoom.h src/dialogs/FallbackAuth.h src/dialogs/ImageOverlay.h @@ -439,6 +449,7 @@ qt5_wrap_cpp(MOC_HEADERS src/dialogs/LeaveRoom.h src/dialogs/Logout.h src/dialogs/MemberList.h + src/dialogs/PlaceCall.h src/dialogs/PreviewUploadOverlay.h src/dialogs/RawMessage.h src/dialogs/ReCaptcha.h @@ -482,9 +493,11 @@ qt5_wrap_cpp(MOC_HEADERS src/notifications/Manager.h + src/ActiveCallBar.h src/AvatarProvider.h src/BlurhashProvider.h src/Cache_p.h + src/CallManager.h src/ChatPage.h src/CommunitiesList.h src/CommunitiesListItem.h @@ -504,6 +517,7 @@ qt5_wrap_cpp(MOC_HEADERS src/TrayIcon.h src/UserInfoWidget.h src/UserSettingsPage.h + src/WebRTCSession.h src/WelcomePage.h src/popups/PopupItem.h src/popups/SuggestionsPopup.h @@ -583,6 +597,8 @@ target_link_libraries(nheko PRIVATE lmdbxx::lmdbxx liblmdb::lmdb tweeny + PkgConfig::GST_SDP + PkgConfig::GST_WEBRTC SingleApplication::SingleApplication) if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.16.0") diff --git a/resources/langs/nheko_en.ts b/resources/langs/nheko_en.ts index db24f1fe..27d739f2 100644 --- a/resources/langs/nheko_en.ts +++ b/resources/langs/nheko_en.ts @@ -404,6 +404,21 @@ Example: https://server.my:8787 %1 created and configured room: %2 %1 created and configured room: %2 + + + %1 placed a voice call. + %1 placed a voice call. + + + + %1 answered the call. + %1 answered the call. + + + + %1 ended the call. + %1 ended the call. + Placeholder @@ -1796,6 +1811,36 @@ Media size: %2 %1 sent an encrypted message %1 sent an encrypted message + + + You placed a call + You placed a call + + + + %1 placed a call + %1 placed a call + + + + You answered a call + You answered a call + + + + %1 answered a call + %1 answered a call + + + + You ended a call + You ended a call + + + + %1 ended a call + %1 ended a call + popups::UserMentions diff --git a/resources/media/callend.mp3 b/resources/media/callend.mp3 new file mode 100644 index 00000000..50c34e56 Binary files /dev/null and b/resources/media/callend.mp3 differ diff --git a/resources/media/callend.ogg b/resources/media/callend.ogg new file mode 100644 index 00000000..927ce1f6 Binary files /dev/null and b/resources/media/callend.ogg differ diff --git a/resources/media/ring.mp3 b/resources/media/ring.mp3 new file mode 100644 index 00000000..36200cd8 Binary files /dev/null and b/resources/media/ring.mp3 differ diff --git a/resources/media/ring.ogg b/resources/media/ring.ogg new file mode 100644 index 00000000..708213bf Binary files /dev/null and b/resources/media/ring.ogg differ diff --git a/resources/media/ringback.mp3 b/resources/media/ringback.mp3 new file mode 100644 index 00000000..6ee34bf3 Binary files /dev/null and b/resources/media/ringback.mp3 differ diff --git a/resources/media/ringback.ogg b/resources/media/ringback.ogg new file mode 100644 index 00000000..7dbfdcd0 Binary files /dev/null and b/resources/media/ringback.ogg differ diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index 17fe7360..52e628be 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -90,6 +90,24 @@ Item { text: qsTr("%1 created and configured room: %2").arg(model.data.userName).arg(model.data.roomId) } } + DelegateChoice { + roleValue: MtxEvent.CallInvite + NoticeMessage { + text: qsTr("%1 placed a voice call.").arg(model.data.userName) + } + } + DelegateChoice { + roleValue: MtxEvent.CallAnswer + NoticeMessage { + text: qsTr("%1 answered the call.").arg(model.data.userName) + } + } + DelegateChoice { + roleValue: MtxEvent.CallHangUp + NoticeMessage { + text: qsTr("%1 ended the call.").arg(model.data.userName) + } + } DelegateChoice { // TODO: make a more complex formatter for the power levels. roleValue: MtxEvent.PowerLevels diff --git a/resources/res.qrc b/resources/res.qrc index 439ed97b..3fd3fc96 100644 --- a/resources/res.qrc +++ b/resources/res.qrc @@ -136,4 +136,9 @@ qml/delegates/Placeholder.qml qml/delegates/Reply.qml + + media/ring.ogg + media/ringback.ogg + media/callend.ogg + diff --git a/src/ActiveCallBar.cpp b/src/ActiveCallBar.cpp new file mode 100644 index 00000000..a5ef754d --- /dev/null +++ b/src/ActiveCallBar.cpp @@ -0,0 +1,74 @@ +#include +#include +#include +#include + +#include "ActiveCallBar.h" +#include "WebRTCSession.h" +#include "ui/FlatButton.h" + +ActiveCallBar::ActiveCallBar(QWidget *parent) + : QWidget(parent) +{ + setAutoFillBackground(true); + auto p = palette(); + p.setColor(backgroundRole(), Qt::green); + setPalette(p); + + QFont f; + f.setPointSizeF(f.pointSizeF()); + + const int fontHeight = QFontMetrics(f).height(); + const int widgetMargin = fontHeight / 3; + const int contentHeight = fontHeight * 3; + + setFixedHeight(contentHeight + widgetMargin); + + topLayout_ = new QHBoxLayout(this); + topLayout_->setSpacing(widgetMargin); + topLayout_->setContentsMargins( + 2 * widgetMargin, widgetMargin, 2 * widgetMargin, widgetMargin); + topLayout_->setSizeConstraint(QLayout::SetMinimumSize); + + QFont labelFont; + labelFont.setPointSizeF(labelFont.pointSizeF() * 1.2); + labelFont.setWeight(QFont::Medium); + + callPartyLabel_ = new QLabel(this); + callPartyLabel_->setFont(labelFont); + + // TODO microphone mute/unmute icons + muteBtn_ = new FlatButton(this); + QIcon muteIcon; + muteIcon.addFile(":/icons/icons/ui/do-not-disturb-rounded-sign.png"); + muteBtn_->setIcon(muteIcon); + muteBtn_->setIconSize(QSize(buttonSize_ / 2, buttonSize_ / 2)); + muteBtn_->setToolTip(tr("Mute Mic")); + muteBtn_->setFixedSize(buttonSize_, buttonSize_); + muteBtn_->setCornerRadius(buttonSize_ / 2); + connect(muteBtn_, &FlatButton::clicked, this, [this]() { + if (WebRTCSession::instance().toggleMuteAudioSrc(muted_)) { + QIcon icon; + if (muted_) { + muteBtn_->setToolTip("Unmute Mic"); + icon.addFile(":/icons/icons/ui/round-remove-button.png"); + } else { + muteBtn_->setToolTip("Mute Mic"); + icon.addFile(":/icons/icons/ui/do-not-disturb-rounded-sign.png"); + } + muteBtn_->setIcon(icon); + } + }); + + topLayout_->addWidget(callPartyLabel_, 0, Qt::AlignLeft); + topLayout_->addWidget(muteBtn_, 0, Qt::AlignRight); +} + +void +ActiveCallBar::setCallParty(const QString &userid, const QString &displayName) +{ + if (!displayName.isEmpty() && displayName != userid) + callPartyLabel_->setText("Active Call: " + displayName + " (" + userid + ")"); + else + callPartyLabel_->setText("Active Call: " + userid); +} diff --git a/src/ActiveCallBar.h b/src/ActiveCallBar.h new file mode 100644 index 00000000..dd01e2ad --- /dev/null +++ b/src/ActiveCallBar.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +class QHBoxLayout; +class QLabel; +class QString; +class FlatButton; + +class ActiveCallBar : public QWidget +{ + Q_OBJECT + +public: + ActiveCallBar(QWidget *parent = nullptr); + +public slots: + void setCallParty(const QString &userid, const QString &displayName); + +private: + QHBoxLayout *topLayout_ = nullptr; + QLabel *callPartyLabel_ = nullptr; + FlatButton *muteBtn_ = nullptr; + int buttonSize_ = 32; + bool muted_ = false; +}; diff --git a/src/Cache.cpp b/src/Cache.cpp index d9d1134e..d435dc56 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1364,6 +1364,9 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) if (!(obj["event"]["type"] == "m.room.message" || obj["event"]["type"] == "m.sticker" || + obj["event"]["type"] == "m.call.invite" || + obj["event"]["type"] == "m.call.answer" || + obj["event"]["type"] == "m.call.hangup" || obj["event"]["type"] == "m.room.encrypted")) continue; diff --git a/src/CallManager.cpp b/src/CallManager.cpp new file mode 100644 index 00000000..67aabced --- /dev/null +++ b/src/CallManager.cpp @@ -0,0 +1,315 @@ +#include + +#include +#include + +#include "CallManager.h" +#include "Cache.h" +#include "ChatPage.h" +#include "Logging.h" +#include "MainWindow.h" +#include "MatrixClient.h" +#include "UserSettingsPage.h" +#include "WebRTCSession.h" + +#include "dialogs/AcceptCall.h" + +Q_DECLARE_METATYPE(std::vector) +Q_DECLARE_METATYPE(mtx::responses::TurnServer) + +using namespace mtx::events; +using namespace mtx::events::msg; + +// TODO Allow altenative in settings +#define STUN_SERVER "stun://turn.matrix.org:3478" + +CallManager::CallManager(QSharedPointer userSettings) + : QObject(), + session_(WebRTCSession::instance()), + turnServerTimer_(this), + settings_(userSettings) +{ + qRegisterMetaType>(); + qRegisterMetaType(); + + connect(&session_, &WebRTCSession::offerCreated, this, + [this](const std::string &sdp, + const std::vector& candidates) + { + nhlog::ui()->debug("Offer created with callid_ and roomid_: {} {}", callid_, roomid_.toStdString()); + emit newMessage(roomid_, CallInvite{callid_, sdp, 0, timeoutms_}); + emit newMessage(roomid_, CallCandidates{callid_, candidates, 0}); + }); + + connect(&session_, &WebRTCSession::answerCreated, this, + [this](const std::string &sdp, + const std::vector& candidates) + { + nhlog::ui()->debug("Answer created with callid_ and roomid_: {} {}", callid_, roomid_.toStdString()); + emit newMessage(roomid_, CallAnswer{callid_, sdp, 0}); + emit newMessage(roomid_, CallCandidates{callid_, candidates, 0}); + }); + + connect(&turnServerTimer_, &QTimer::timeout, this, &CallManager::retrieveTurnServer); + turnServerTimer_.start(2000); + + connect(this, &CallManager::turnServerRetrieved, this, + [this](const mtx::responses::TurnServer &res) + { + nhlog::net()->info("TURN server(s) retrieved from homeserver:"); + nhlog::net()->info("username: {}", res.username); + nhlog::net()->info("ttl: {}", res.ttl); + for (const auto &u : res.uris) + nhlog::net()->info("uri: {}", u); + + turnServer_ = res; + turnServerTimer_.setInterval(res.ttl * 1000 * 0.9); + }); + + connect(&session_, &WebRTCSession::pipelineChanged, this, + [this](bool started) { + if (!started) + playRingtone("qrc:/media/media/callend.ogg", false); + }); + + connect(&player_, &QMediaPlayer::mediaStatusChanged, this, + [this](QMediaPlayer::MediaStatus status) { + if (status == QMediaPlayer::LoadedMedia) + player_.play(); + }); +} + +void +CallManager::sendInvite(const QString &roomid) +{ + if (onActiveCall()) + return; + + std::vector members(cache::getMembers(roomid.toStdString())); + if (members.size() != 2) { + emit ChatPage::instance()->showNotification("Voice/Video calls are limited to 1:1 rooms"); + return; + } + + std::string errorMessage; + if (!session_.init(&errorMessage)) { + emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); + return; + } + + roomid_ = roomid; + setTurnServers(); + session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : ""); + + // TODO Add invite timeout + generateCallID(); + const RoomMember &callee = members.front().user_id == utils::localUser() ? members.back() : members.front(); + emit newCallParty(callee.user_id, callee.display_name); + playRingtone("qrc:/media/media/ringback.ogg", true); + if (!session_.createOffer()) { + emit ChatPage::instance()->showNotification("Problem setting up call"); + endCall(); + } +} + +void +CallManager::hangUp() +{ + nhlog::ui()->debug("CallManager::hangUp: roomid_: {}", roomid_.toStdString()); + if (!callid_.empty()) { + emit newMessage(roomid_, CallHangUp{callid_, 0, CallHangUp::Reason::User}); + endCall(); + } +} + +bool +CallManager::onActiveCall() +{ + return session_.isActive(); +} + +void CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) +{ + if (handleEvent_(event) || handleEvent_(event) + || handleEvent_(event) || handleEvent_(event)) + return; +} + +template +bool +CallManager::handleEvent_(const mtx::events::collections::TimelineEvents &event) +{ + if (std::holds_alternative>(event)) { + handleEvent(std::get>(event)); + return true; + } + return false; +} + +void +CallManager::handleEvent(const RoomEvent &callInviteEvent) +{ + nhlog::ui()->debug("CallManager::incoming CallInvite from {} with id {}", callInviteEvent.sender, callInviteEvent.content.call_id); + + if (callInviteEvent.content.call_id.empty()) + return; + + std::vector members(cache::getMembers(callInviteEvent.room_id)); + if (onActiveCall() || members.size() != 2) { + emit newMessage(QString::fromStdString(callInviteEvent.room_id), + CallHangUp{callInviteEvent.content.call_id, 0, CallHangUp::Reason::InviteTimeOut}); + return; + } + + playRingtone("qrc:/media/media/ring.ogg", true); + roomid_ = QString::fromStdString(callInviteEvent.room_id); + callid_ = callInviteEvent.content.call_id; + remoteICECandidates_.clear(); + + const RoomMember &caller = members.front().user_id == utils::localUser() ? members.back() : members.front(); + emit newCallParty(caller.user_id, caller.display_name); + + auto dialog = new dialogs::AcceptCall(caller.user_id, caller.display_name, MainWindow::instance()); + connect(dialog, &dialogs::AcceptCall::accept, this, + [this, callInviteEvent](){ + MainWindow::instance()->hideOverlay(); + answerInvite(callInviteEvent.content);}); + connect(dialog, &dialogs::AcceptCall::reject, this, + [this](){ + MainWindow::instance()->hideOverlay(); + hangUp();}); + MainWindow::instance()->showSolidOverlayModal(dialog); +} + +void +CallManager::answerInvite(const CallInvite &invite) +{ + stopRingtone(); + std::string errorMessage; + if (!session_.init(&errorMessage)) { + emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); + hangUp(); + return; + } + + setTurnServers(); + session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : ""); + + if (!session_.acceptOffer(invite.sdp)) { + emit ChatPage::instance()->showNotification("Problem setting up call"); + hangUp(); + return; + } + session_.acceptICECandidates(remoteICECandidates_); + remoteICECandidates_.clear(); +} + +void +CallManager::handleEvent(const RoomEvent &callCandidatesEvent) +{ + nhlog::ui()->debug("CallManager::incoming CallCandidates from {} with id {}", callCandidatesEvent.sender, callCandidatesEvent.content.call_id); + if (callid_ == callCandidatesEvent.content.call_id) { + if (onActiveCall()) + session_.acceptICECandidates(callCandidatesEvent.content.candidates); + else { + // CallInvite has been received and we're awaiting localUser to accept or reject the call + for (const auto &c : callCandidatesEvent.content.candidates) + remoteICECandidates_.push_back(c); + } + } +} + +void +CallManager::handleEvent(const RoomEvent &callAnswerEvent) +{ + nhlog::ui()->debug("CallManager::incoming CallAnswer from {} with id {}", callAnswerEvent.sender, callAnswerEvent.content.call_id); + if (onActiveCall() && callid_ == callAnswerEvent.content.call_id) { + stopRingtone(); + if (!session_.acceptAnswer(callAnswerEvent.content.sdp)) { + emit ChatPage::instance()->showNotification("Problem setting up call"); + hangUp(); + } + } +} + +void +CallManager::handleEvent(const RoomEvent &callHangUpEvent) +{ + nhlog::ui()->debug("CallManager::incoming CallHangUp from {} with id {}", callHangUpEvent.sender, callHangUpEvent.content.call_id); + if (onActiveCall() && callid_ == callHangUpEvent.content.call_id) + endCall(); +} + +void +CallManager::generateCallID() +{ + using namespace std::chrono; + uint64_t ms = duration_cast(system_clock::now().time_since_epoch()).count(); + callid_ = "c" + std::to_string(ms); +} + +void +CallManager::endCall() +{ + stopRingtone(); + session_.end(); + roomid_.clear(); + callid_.clear(); + remoteICECandidates_.clear(); +} + +void +CallManager::retrieveTurnServer() +{ + http::client()->get_turn_server( + [this](const mtx::responses::TurnServer &res, mtx::http::RequestErr err) { + if (err) { + turnServerTimer_.setInterval(5000); + return; + } + emit turnServerRetrieved(res); + }); +} + +void +CallManager::setTurnServers() +{ + // gstreamer expects (percent-encoded): turn(s)://username:password@host:port?transport=udp(tcp) + std::vector uris; + for (const auto &uri : turnServer_.uris) { + if (auto c = uri.find(':'); c == std::string::npos) { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } + else { + std::string scheme = std::string(uri, 0, c); + if (scheme != "turn" && scheme != "turns") { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } + std::string res = scheme + "://" + turnServer_.username + ":" + turnServer_.password + + "@" + std::string(uri, ++c); + QString encodedUri = QUrl::toPercentEncoding(QString::fromStdString(res)); + uris.push_back(encodedUri.toStdString()); + } + } + if (!uris.empty()) + session_.setTurnServers(uris); +} + +void +CallManager::playRingtone(const QString &ringtone, bool repeat) +{ + static QMediaPlaylist playlist; + playlist.clear(); + playlist.setPlaybackMode(repeat ? QMediaPlaylist::CurrentItemInLoop : QMediaPlaylist::CurrentItemOnce); + playlist.addMedia(QUrl(ringtone)); + player_.setVolume(100); + player_.setPlaylist(&playlist); +} + +void +CallManager::stopRingtone() +{ + player_.setPlaylist(nullptr); +} diff --git a/src/CallManager.h b/src/CallManager.h new file mode 100644 index 00000000..8a93241f --- /dev/null +++ b/src/CallManager.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "mtx/events/collections.hpp" +#include "mtx/events/voip.hpp" +#include "mtx/responses/turn_server.hpp" + +class UserSettings; +class WebRTCSession; + +class CallManager : public QObject +{ + Q_OBJECT + +public: + CallManager(QSharedPointer); + + void sendInvite(const QString &roomid); + void hangUp(); + bool onActiveCall(); + +public slots: + void syncEvent(const mtx::events::collections::TimelineEvents &event); + +signals: + void newMessage(const QString &roomid, const mtx::events::msg::CallInvite&); + void newMessage(const QString &roomid, const mtx::events::msg::CallCandidates&); + void newMessage(const QString &roomid, const mtx::events::msg::CallAnswer&); + void newMessage(const QString &roomid, const mtx::events::msg::CallHangUp&); + void turnServerRetrieved(const mtx::responses::TurnServer&); + void newCallParty(const QString &userid, const QString& displayName); + +private slots: + void retrieveTurnServer(); + +private: + WebRTCSession& session_; + QString roomid_; + std::string callid_; + const uint32_t timeoutms_ = 120000; + std::vector remoteICECandidates_; + mtx::responses::TurnServer turnServer_; + QTimer turnServerTimer_; + QSharedPointer settings_; + QMediaPlayer player_; + + template + bool handleEvent_(const mtx::events::collections::TimelineEvents &event); + void handleEvent(const mtx::events::RoomEvent&); + void handleEvent(const mtx::events::RoomEvent&); + void handleEvent(const mtx::events::RoomEvent&); + void handleEvent(const mtx::events::RoomEvent&); + void answerInvite(const mtx::events::msg::CallInvite&); + void generateCallID(); + void endCall(); + void setTurnServers(); + void playRingtone(const QString &ringtone, bool repeat); + void stopRingtone(); +}; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 3b8af33a..1bea8564 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -22,6 +22,7 @@ #include #include +#include "ActiveCallBar.h" #include "AvatarProvider.h" #include "Cache.h" #include "Cache_p.h" @@ -40,11 +41,13 @@ #include "UserInfoWidget.h" #include "UserSettingsPage.h" #include "Utils.h" +#include "WebRTCSession.h" #include "ui/OverlayModal.h" #include "ui/Theme.h" #include "notifications/Manager.h" +#include "dialogs/PlaceCall.h" #include "dialogs/ReadReceipts.h" #include "popups/UserMentions.h" #include "timeline/TimelineViewManager.h" @@ -68,6 +71,7 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) , isConnected_(true) , userSettings_{userSettings} , notificationsManager(this) + , callManager_(userSettings) { setObjectName("chatPage"); @@ -123,11 +127,26 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) contentLayout_->setMargin(0); top_bar_ = new TopRoomBar(this); - view_manager_ = new TimelineViewManager(userSettings_, this); + view_manager_ = new TimelineViewManager(userSettings_, &callManager_, this); contentLayout_->addWidget(top_bar_); contentLayout_->addWidget(view_manager_->getWidget()); + activeCallBar_ = new ActiveCallBar(this); + contentLayout_->addWidget(activeCallBar_); + activeCallBar_->hide(); + connect( + &callManager_, &CallManager::newCallParty, activeCallBar_, &ActiveCallBar::setCallParty); + connect(&WebRTCSession::instance(), + &WebRTCSession::pipelineChanged, + this, + [this](bool callStarted) { + if (callStarted) + activeCallBar_->show(); + else + activeCallBar_->hide(); + }); + // Splitter splitter->addWidget(sideBar_); splitter->addWidget(content_); @@ -446,6 +465,31 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) roomid, filename, encryptedFile, url, mime, dsize); }); + connect(text_input_, &TextInputWidget::callButtonPress, this, [this]() { + if (callManager_.onActiveCall()) { + callManager_.hangUp(); + } else { + if (cache::singleRoomInfo(current_room_.toStdString()).member_count != 2) { + showNotification("Voice/Video calls are limited to 1:1 rooms"); + } else { + std::vector members( + cache::getMembers(current_room_.toStdString())); + const RoomMember &callee = + members.front().user_id == utils::localUser() ? members.back() + : members.front(); + auto dialog = + new dialogs::PlaceCall(callee.user_id, callee.display_name, this); + connect(dialog, &dialogs::PlaceCall::voice, this, [this]() { + callManager_.sendInvite(current_room_); + }); + connect(dialog, &dialogs::PlaceCall::video, this, [this]() { + showNotification("Video calls not yet implemented"); + }); + dialog->show(); + } + } + }); + connect(room_list_, &RoomList::roomAvatarChanged, this, &ChatPage::updateTopBarAvatar); connect( @@ -569,6 +613,11 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) connect(this, &ChatPage::dropToLoginPageCb, this, &ChatPage::dropToLoginPage); + connectCallMessage(); + connectCallMessage(); + connectCallMessage(); + connectCallMessage(); + instance_ = this; } @@ -1430,3 +1479,13 @@ ChatPage::initiateLogout() emit showOverlayProgressBar(); } + +template +void +ChatPage::connectCallMessage() +{ + connect(&callManager_, + qOverload(&CallManager::newMessage), + view_manager_, + qOverload(&TimelineViewManager::queueCallMessage)); +} diff --git a/src/ChatPage.h b/src/ChatPage.h index c38d7717..fe63c9d9 100644 --- a/src/ChatPage.h +++ b/src/ChatPage.h @@ -35,11 +35,13 @@ #include #include "CacheStructs.h" +#include "CallManager.h" #include "CommunitiesList.h" #include "Utils.h" #include "notifications/Manager.h" #include "popups/UserMentions.h" +class ActiveCallBar; class OverlayModal; class QuickSwitcher; class RoomList; @@ -50,7 +52,6 @@ class TimelineViewManager; class TopRoomBar; class UserInfoWidget; class UserSettings; -class NotificationsManager; constexpr int CONSENSUS_TIMEOUT = 1000; constexpr int SHOW_CONTENT_TIMEOUT = 3000; @@ -216,6 +217,9 @@ private: void showNotificationsDialog(const QPoint &point); + template + void connectCallMessage(); + QHBoxLayout *topLayout_; Splitter *splitter; @@ -235,6 +239,7 @@ private: TopRoomBar *top_bar_; TextInputWidget *text_input_; + ActiveCallBar *activeCallBar_; QTimer connectivityTimer_; std::atomic_bool isConnected_; @@ -252,6 +257,7 @@ private: QSharedPointer userSettings_; NotificationsManager notificationsManager; + CallManager callManager_; }; template diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp index 3e3915bb..2be0b404 100644 --- a/src/TextInputWidget.cpp +++ b/src/TextInputWidget.cpp @@ -31,6 +31,7 @@ #include "Logging.h" #include "TextInputWidget.h" #include "Utils.h" +#include "WebRTCSession.h" #include "ui/FlatButton.h" #include "ui/LoadingIndicator.h" @@ -453,6 +454,13 @@ TextInputWidget::TextInputWidget(QWidget *parent) topLayout_->setSpacing(0); topLayout_->setContentsMargins(13, 1, 13, 0); + callBtn_ = new FlatButton(this); + changeCallButtonState(false); + connect(&WebRTCSession::instance(), + &WebRTCSession::pipelineChanged, + this, + &TextInputWidget::changeCallButtonState); + QIcon send_file_icon; send_file_icon.addFile(":/icons/icons/ui/paper-clip-outline.png"); @@ -521,6 +529,7 @@ TextInputWidget::TextInputWidget(QWidget *parent) emojiBtn_->setIcon(emoji_icon); emojiBtn_->setIconSize(QSize(ButtonHeight, ButtonHeight)); + topLayout_->addWidget(callBtn_); topLayout_->addWidget(sendFileBtn_); topLayout_->addWidget(input_); topLayout_->addWidget(emojiBtn_); @@ -528,6 +537,7 @@ TextInputWidget::TextInputWidget(QWidget *parent) setLayout(topLayout_); + connect(callBtn_, &FlatButton::clicked, this, &TextInputWidget::callButtonPress); connect(sendMessageBtn_, &FlatButton::clicked, input_, &FilteredTextEdit::submit); connect(sendFileBtn_, SIGNAL(clicked()), this, SLOT(openFileSelection())); connect(input_, &FilteredTextEdit::message, this, &TextInputWidget::sendTextMessage); @@ -652,3 +662,19 @@ TextInputWidget::paintEvent(QPaintEvent *) style()->drawPrimitive(QStyle::PE_Widget, &opt, &p, this); } + +void +TextInputWidget::changeCallButtonState(bool callStarted) +{ + // TODO Telephone and HangUp icons - co-opt the ones below for now + QIcon icon; + if (callStarted) { + callBtn_->setToolTip(tr("Hang up")); + icon.addFile(":/icons/icons/ui/remove-symbol.png"); + } else { + callBtn_->setToolTip(tr("Place a call")); + icon.addFile(":/icons/icons/ui/speech-bubbles-comment-option.png"); + } + callBtn_->setIcon(icon); + callBtn_->setIconSize(QSize(ButtonHeight, ButtonHeight)); +} diff --git a/src/TextInputWidget.h b/src/TextInputWidget.h index a0105eb0..ae58f4e3 100644 --- a/src/TextInputWidget.h +++ b/src/TextInputWidget.h @@ -149,6 +149,7 @@ public slots: void openFileSelection(); void hideUploadSpinner(); void focusLineEdit() { input_->setFocus(); } + void changeCallButtonState(bool callStarted); private slots: void addSelectedEmoji(const QString &emoji); @@ -161,6 +162,7 @@ signals: void uploadMedia(const QSharedPointer data, QString mimeClass, const QString &filename); + void callButtonPress(); void sendJoinRoomRequest(const QString &room); void sendInviteRoomRequest(const QString &userid, const QString &reason); @@ -185,6 +187,7 @@ private: LoadingIndicator *spinner_; + FlatButton *callBtn_; FlatButton *sendFileBtn_; FlatButton *sendMessageBtn_; emoji::PickButton *emojiBtn_; diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index 05ff6d38..e67da997 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -77,6 +77,7 @@ UserSettings::load() presence_ = settings.value("user/presence", QVariant::fromValue(Presence::AutomaticPresence)) .value(); + useStunServer_ = settings.value("user/use_stun_server", false).toBool(); applyTheme(); } @@ -279,6 +280,16 @@ UserSettings::setTheme(QString theme) emit themeChanged(theme); } +void +UserSettings::setUseStunServer(bool useStunServer) +{ + if (useStunServer == useStunServer_) + return; + useStunServer_ = useStunServer; + emit useStunServerChanged(useStunServer); + save(); +} + void UserSettings::applyTheme() { @@ -364,6 +375,7 @@ UserSettings::save() settings.setValue("font_family", font_); settings.setValue("emoji_font_family", emojiFont_); settings.setValue("presence", QVariant::fromValue(presence_)); + settings.setValue("use_stun_server", useStunServer_); settings.endGroup(); @@ -429,6 +441,7 @@ UserSettingsPage::UserSettingsPage(QSharedPointer settings, QWidge markdown_ = new Toggle{this}; desktopNotifications_ = new Toggle{this}; alertOnNotification_ = new Toggle{this}; + useStunServer_ = new Toggle{this}; scaleFactorCombo_ = new QComboBox{this}; fontSizeCombo_ = new QComboBox{this}; fontSelectionCombo_ = new QComboBox{this}; @@ -482,6 +495,12 @@ UserSettingsPage::UserSettingsPage(QSharedPointer settings, QWidge timelineMaxWidthSpin_->setMaximum(100'000'000); timelineMaxWidthSpin_->setSingleStep(10); + auto callsLabel = new QLabel{tr("CALLS"), this}; + callsLabel->setFixedHeight(callsLabel->minimumHeight() + LayoutTopMargin); + callsLabel->setAlignment(Qt::AlignBottom); + callsLabel->setFont(font); + useStunServer_ = new Toggle{this}; + auto encryptionLabel_ = new QLabel{tr("ENCRYPTION"), this}; encryptionLabel_->setFixedHeight(encryptionLabel_->minimumHeight() + LayoutTopMargin); encryptionLabel_->setAlignment(Qt::AlignBottom); @@ -612,6 +631,13 @@ UserSettingsPage::UserSettingsPage(QSharedPointer settings, QWidge #endif boxWrap(tr("Theme"), themeCombo_); + + formLayout_->addRow(callsLabel); + formLayout_->addRow(new HorizontalLine{this}); + boxWrap(tr("Allow Fallback Call Assist Server"), + useStunServer_, + tr("Will use turn.matrix.org as assist when your home server does not offer one.")); + formLayout_->addRow(encryptionLabel_); formLayout_->addRow(new HorizontalLine{this}); boxWrap(tr("Device ID"), deviceIdValue_); @@ -724,6 +750,10 @@ UserSettingsPage::UserSettingsPage(QSharedPointer settings, QWidge settings_->setEnlargeEmojiOnlyMessages(!disabled); }); + connect(useStunServer_, &Toggle::toggled, this, [this](bool disabled) { + settings_->setUseStunServer(!disabled); + }); + connect(timelineMaxWidthSpin_, qOverload(&QSpinBox::valueChanged), this, @@ -766,6 +796,7 @@ UserSettingsPage::showEvent(QShowEvent *) enlargeEmojiOnlyMessages_->setState(!settings_->enlargeEmojiOnlyMessages()); deviceIdValue_->setText(QString::fromStdString(http::client()->device_id())); timelineMaxWidthSpin_->setValue(settings_->timelineMaxWidth()); + useStunServer_->setState(!settings_->useStunServer()); deviceFingerprintValue_->setText( utils::humanReadableFingerprint(olm::client()->identity_keys().ed25519)); diff --git a/src/UserSettingsPage.h b/src/UserSettingsPage.h index d2a1c641..567a7520 100644 --- a/src/UserSettingsPage.h +++ b/src/UserSettingsPage.h @@ -71,6 +71,8 @@ class UserSettings : public QObject Q_PROPERTY( QString emojiFont READ emojiFont WRITE setEmojiFontFamily NOTIFY emojiFontChanged) Q_PROPERTY(Presence presence READ presence WRITE setPresence NOTIFY presenceChanged) + Q_PROPERTY( + bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged) public: UserSettings(); @@ -107,6 +109,7 @@ public: void setAvatarCircles(bool state); void setDecryptSidebar(bool state); void setPresence(Presence state); + void setUseStunServer(bool state); QString theme() const { return !theme_.isEmpty() ? theme_ : defaultTheme_; } bool messageHoverHighlight() const { return messageHoverHighlight_; } @@ -132,6 +135,7 @@ public: QString font() const { return font_; } QString emojiFont() const { return emojiFont_; } Presence presence() const { return presence_; } + bool useStunServer() const { return useStunServer_; } signals: void groupViewStateChanged(bool state); @@ -154,6 +158,7 @@ signals: void fontChanged(QString state); void emojiFontChanged(QString state); void presenceChanged(Presence state); + void useStunServerChanged(bool state); private: // Default to system theme if QT_QPA_PLATFORMTHEME var is set. @@ -181,6 +186,7 @@ private: QString font_; QString emojiFont_; Presence presence_; + bool useStunServer_; }; class HorizontalLine : public QFrame @@ -234,6 +240,7 @@ private: Toggle *desktopNotifications_; Toggle *alertOnNotification_; Toggle *avatarCircles_; + Toggle *useStunServer_; Toggle *decryptSidebar_; QLabel *deviceFingerprintValue_; QLabel *deviceIdValue_; diff --git a/src/Utils.cpp b/src/Utils.cpp index 26ea124c..0bfc82c3 100644 --- a/src/Utils.cpp +++ b/src/Utils.cpp @@ -35,14 +35,13 @@ createDescriptionInfo(const Event &event, const QString &localUser, const QStrin const auto username = cache::displayName(room_id, sender); const auto ts = QDateTime::fromMSecsSinceEpoch(msg.origin_server_ts); - return DescInfo{ - QString::fromStdString(msg.event_id), - sender, - utils::messageDescription( - username, QString::fromStdString(msg.content.body).trimmed(), sender == localUser), - utils::descriptiveTime(ts), - msg.origin_server_ts, - ts}; + return DescInfo{QString::fromStdString(msg.event_id), + sender, + utils::messageDescription( + username, utils::event_body(event).trimmed(), sender == localUser), + utils::descriptiveTime(ts), + msg.origin_server_ts, + ts}; } QString @@ -156,14 +155,17 @@ utils::getMessageDescription(const TimelineEvent &event, const QString &localUser, const QString &room_id) { - using Audio = mtx::events::RoomEvent; - using Emote = mtx::events::RoomEvent; - using File = mtx::events::RoomEvent; - using Image = mtx::events::RoomEvent; - using Notice = mtx::events::RoomEvent; - using Text = mtx::events::RoomEvent; - using Video = mtx::events::RoomEvent; - using Encrypted = mtx::events::EncryptedEvent; + using Audio = mtx::events::RoomEvent; + using Emote = mtx::events::RoomEvent; + using File = mtx::events::RoomEvent; + using Image = mtx::events::RoomEvent; + using Notice = mtx::events::RoomEvent; + using Text = mtx::events::RoomEvent; + using Video = mtx::events::RoomEvent; + using CallInvite = mtx::events::RoomEvent; + using CallAnswer = mtx::events::RoomEvent; + using CallHangUp = mtx::events::RoomEvent; + using Encrypted = mtx::events::EncryptedEvent; if (std::holds_alternative