From 2e77a1554f1572b7c7e59f8177a48e5dffa16c23 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Tue, 26 Jan 2021 22:36:35 +0100 Subject: Switch to new relations format --- src/Cache.cpp | 72 +++++++++++++++++++++++++---------------------------------- 1 file changed, 31 insertions(+), 41 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 3f2bf73a..94b9a6a6 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2713,23 +2713,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put(txn, evToOrderDb, event_id, txn_order); lmdb::dbi_del(txn, evToOrderDb, lmdb::val(txn_id)); - if (event.contains("content") && - event["content"].contains("m.relates_to")) { - auto temp = event["content"]["m.relates_to"]; - json relates_to_j = temp.contains("m.in_reply_to") && - temp["m.in_reply_to"].is_object() - ? temp["m.in_reply_to"]["event_id"] - : temp["event_id"]; - std::string relates_to = - relates_to_j.is_string() ? relates_to_j.get() : ""; - - if (!relates_to.empty()) { - lmdb::dbi_del(txn, - relationsDb, - lmdb::val(relates_to), - lmdb::val(txn_id)); - lmdb::dbi_put( - txn, relationsDb, lmdb::val(relates_to), event_id); + auto relations = mtx::accessors::relations(e); + if (!relations.relations.empty()) { + for (const auto &r : relations.relations) { + if (!r.event_id.empty()) { + lmdb::dbi_del(txn, + relationsDb, + lmdb::val(r.event_id), + lmdb::val(txn_id)); + lmdb::dbi_put(txn, + relationsDb, + lmdb::val(r.event_id), + event_id); + } } } @@ -2808,19 +2804,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val(&msgIndex, sizeof(msgIndex))); } - if (event.contains("content") && - event["content"].contains("m.relates_to")) { - auto temp = event["content"]["m.relates_to"]; - json relates_to_j = temp.contains("m.in_reply_to") && - temp["m.in_reply_to"].is_object() - ? temp["m.in_reply_to"]["event_id"] - : temp["event_id"]; - std::string relates_to = - relates_to_j.is_string() ? relates_to_j.get() : ""; - - if (!relates_to.empty()) - lmdb::dbi_put( - txn, relationsDb, lmdb::val(relates_to), event_id); + auto relations = mtx::accessors::relations(e); + if (!relations.relations.empty()) { + for (const auto &r : relations.relations) { + if (!r.event_id.empty()) { + lmdb::dbi_put(txn, + relationsDb, + lmdb::val(r.event_id), + event_id); + } + } } } } @@ -2901,17 +2894,14 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex))); } - if (event.contains("content") && event["content"].contains("m.relates_to")) { - auto temp = event["content"]["m.relates_to"]; - json relates_to_j = - temp.contains("m.in_reply_to") && temp["m.in_reply_to"].is_object() - ? temp["m.in_reply_to"]["event_id"] - : temp["event_id"]; - std::string relates_to = - relates_to_j.is_string() ? relates_to_j.get() : ""; - - if (!relates_to.empty()) - lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id); + auto relations = mtx::accessors::relations(e); + if (!relations.relations.empty()) { + for (const auto &r : relations.relations) { + if (!r.event_id.empty()) { + lmdb::dbi_put( + txn, relationsDb, lmdb::val(r.event_id), event_id); + } + } } } -- cgit 1.5.1 From d6504812c71ff7251a5319113c580ab322469eb3 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 27 Jan 2021 02:45:33 +0100 Subject: Render edits --- src/Cache.cpp | 30 +++++++++++++++++++++++++ src/Cache_p.h | 2 ++ src/timeline/EventStore.cpp | 54 ++++++++++++++++++++++++++++++++++++++++++--- src/timeline/EventStore.h | 4 +++- 4 files changed, 86 insertions(+), 4 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 94b9a6a6..49861a9a 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -108,6 +108,11 @@ Cache::isHiddenEvent(lmdb::txn &txn, const std::string &room_id) { using namespace mtx::events; + + // Always hide edits + if (mtx::accessors::relations(e).replaces()) + return true; + if (auto encryptedEvent = std::get_if>(&e)) { MegolmSessionIndex index; index.room_id = room_id; @@ -1891,6 +1896,31 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) return *val.data(); } +std::optional +Cache::getArrivalIndex(const std::string &room_id, std::string_view event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + + lmdb::dbi orderDb{0}; + try { + orderDb = getEventToOrderDb(txn, room_id); + } catch (lmdb::runtime_error &e) { + nhlog::db()->error("Can't open db for room '{}', probably doesn't exist yet. ({})", + room_id, + e.what()); + return {}; + } + + lmdb::val indexVal{event_id.data(), event_id.size()}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + return {}; + } + + return *val.data(); +} + std::optional Cache::getTimelineEventId(const std::string &room_id, uint64_t index) { diff --git a/src/Cache_p.h b/src/Cache_p.h index e2ce1668..c96a3f30 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -205,6 +205,8 @@ public: std::optional getTimelineIndex(const std::string &room_id, std::string_view event_id); std::optional getTimelineEventId(const std::string &room_id, uint64_t index); + std::optional getArrivalIndex(const std::string &room_id, + std::string_view event_id); std::string previousBatchToken(const std::string &room_id); uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 4a90222f..ebf2f024 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -405,6 +405,41 @@ EventStore::handle_room_verification(mtx::events::collections::TimelineEvents ev event); } +std::vector +EventStore::edits(const std::string &event_id) +{ + auto event_ids = cache::client()->relatedEvents(room_id_, event_id); + + auto original_event = get(event_id, "", false, false); + if (!original_event) + return {}; + + auto original_sender = mtx::accessors::sender(*original_event); + + std::vector edits; + for (const auto &id : event_ids) { + auto related_event = get(id, event_id, false, false); + if (!related_event) + continue; + + auto edit_rel = mtx::accessors::relations(*related_event); + if (edit_rel.replaces() == event_id && + original_sender == mtx::accessors::sender(*related_event)) + edits.push_back(*related_event); + } + + auto c = cache::client(); + std::sort(edits.begin(), + edits.end(), + [this, c](const mtx::events::collections::TimelineEvents &a, + const mtx::events::collections::TimelineEvents &b) { + return c->getArrivalIndex(this->room_id_, mtx::accessors::event_id(a)) < + c->getArrivalIndex(this->room_id_, mtx::accessors::event_id(b)); + }); + + return edits; +} + QVariantList EventStore::reactions(const std::string &event_id) { @@ -487,7 +522,13 @@ EventStore::get(int idx, bool decrypt) if (!event_id) return nullptr; - auto event = cache::client()->getEvent(room_id_, *event_id); + std::optional event; + auto edits_ = edits(*event_id); + if (edits_.empty()) + event = cache::client()->getEvent(room_id_, *event_id); + else + event = {edits_.back()}; + if (!event) return nullptr; else @@ -714,7 +755,7 @@ EventStore::decryptEvent(const IdIndex &idx, } mtx::events::collections::TimelineEvents * -EventStore::get(std::string_view id, std::string_view related_to, bool decrypt) +EventStore::get(std::string_view id, std::string_view related_to, bool decrypt, bool resolve_edits) { if (this->thread() != QThread::currentThread()) nhlog::db()->warn("{} called from a different thread!", __func__); @@ -722,7 +763,14 @@ EventStore::get(std::string_view id, std::string_view related_to, bool decrypt) if (id.empty()) return nullptr; - IdIndex index{room_id_, std::string(id.data(), id.size())}; + std::string id_ = std::string(id); + if (resolve_edits) { + auto edits_ = edits(id_); + if (!edits_.empty()) + id_ = mtx::accessors::event_id(edits_.back()); + } + + IdIndex index{room_id_, id_}; auto event_ptr = events_by_id_.object(index); if (!event_ptr) { diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index f8eff9a9..ced7bdc0 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -66,7 +66,8 @@ public: // relatedFetched event mtx::events::collections::TimelineEvents *get(std::string_view id, std::string_view related_to, - bool decrypt = true); + bool decrypt = true, + bool resolve_edits = true); // always returns a proper event as long as the idx is valid mtx::events::collections::TimelineEvents *get(int idx, bool decrypt = true); @@ -110,6 +111,7 @@ public slots: void clearTimeline(); private: + std::vector edits(const std::string &event_id); mtx::events::collections::TimelineEvents *decryptEvent( const IdIndex &idx, const mtx::events::EncryptedEvent &e); -- cgit 1.5.1 From bdb6e6b79e9beeaabfbde99cd760de77247d11a4 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 10 Feb 2021 01:03:20 +0100 Subject: Fix stuck notifications because of edits Does not fix the read status yet, for that we need to compare read receipts for all events after the last visible event. --- src/Cache.cpp | 90 ++++++++++++++++++++++++++++++++++++++++++ src/Cache.h | 6 +++ src/Cache_p.h | 5 +++ src/timeline/TimelineModel.cpp | 23 +++++++++-- src/timeline/TimelineModel.h | 2 +- 5 files changed, 121 insertions(+), 5 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 49861a9a..109fc60d 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1896,6 +1896,84 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) return *val.data(); } +std::optional +Cache::getEventIndex(const std::string &room_id, std::string_view event_id) +{ + if (event_id.empty()) + return {}; + + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + + lmdb::dbi orderDb{0}; + try { + orderDb = getEventToOrderDb(txn, room_id); + } catch (lmdb::runtime_error &e) { + nhlog::db()->error("Can't open db for room '{}', probably doesn't exist yet. ({})", + room_id, + e.what()); + return {}; + } + + lmdb::val indexVal{event_id.data(), event_id.size()}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + nhlog::db()->critical("Could not find event id: {}", event_id); + return {}; + } + + return *val.data(); +} + +std::optional> +Cache::lastInvisibleEventAfter(const std::string &room_id, std::string_view event_id) +{ + if (event_id.empty()) + return {}; + + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + + lmdb::dbi orderDb{0}; + lmdb::dbi eventOrderDb{0}; + lmdb::dbi timelineDb{0}; + try { + orderDb = getEventToOrderDb(txn, room_id); + eventOrderDb = getEventOrderDb(txn, room_id); + timelineDb = getMessageToOrderDb(txn, room_id); + } catch (lmdb::runtime_error &e) { + nhlog::db()->error("Can't open db for room '{}', probably doesn't exist yet. ({})", + room_id, + e.what()); + return {}; + } + + lmdb::val eventIdVal{event_id.data(), event_id.size()}, indexVal; + + bool success = lmdb::dbi_get(txn, orderDb, eventIdVal, indexVal); + if (!success) { + return {}; + } + uint64_t prevIdx = *indexVal.data(); + std::string prevId{eventIdVal.data(), eventIdVal.size()}; + + auto cursor = lmdb::cursor::open(txn, eventOrderDb); + cursor.get(indexVal, MDB_SET); + while (cursor.get(indexVal, eventIdVal, MDB_NEXT)) { + std::string evId = + json::parse(std::string_view(eventIdVal.data(), eventIdVal.size()))["event_id"] + .get(); + lmdb::val temp; + if (lmdb::dbi_get(txn, timelineDb, lmdb::val(evId.data(), evId.size()), temp)) { + return std::pair{prevIdx, std::string(prevId)}; + } else { + prevIdx = *indexVal.data(); + prevId = std::move(evId); + } + } + + return std::pair{prevIdx, std::string(prevId)}; +} + std::optional Cache::getArrivalIndex(const std::string &room_id, std::string_view event_id) { @@ -4253,6 +4331,18 @@ readReceipts(const QString &event_id, const QString &room_id) return instance_->readReceipts(event_id, room_id); } +std::optional +getEventIndex(const std::string &room_id, std::string_view event_id) +{ + return instance_->getEventIndex(room_id, event_id); +} + +std::optional> +lastInvisibleEventAfter(const std::string &room_id, std::string_view event_id) +{ + return instance_->lastInvisibleEventAfter(room_id, event_id); +} + QByteArray image(const QString &url) { diff --git a/src/Cache.h b/src/Cache.h index 91956725..e60fc970 100644 --- a/src/Cache.h +++ b/src/Cache.h @@ -168,6 +168,12 @@ using UserReceipts = std::multimap UserReceipts readReceipts(const QString &event_id, const QString &room_id); +//! get index of the event in the event db, not representing the visual index +std::optional +getEventIndex(const std::string &room_id, std::string_view event_id); +std::optional> +lastInvisibleEventAfter(const std::string &room_id, std::string_view event_id); + QByteArray image(const QString &url); QByteArray diff --git a/src/Cache_p.h b/src/Cache_p.h index c96a3f30..431e7bc3 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -204,6 +204,11 @@ public: std::optional getTimelineRange(const std::string &room_id); std::optional getTimelineIndex(const std::string &room_id, std::string_view event_id); + std::optional getEventIndex(const std::string &room_id, + std::string_view event_id); + std::optional> lastInvisibleEventAfter( + const std::string &room_id, + std::string_view event_id); std::optional getTimelineEventId(const std::string &room_id, uint64_t index); std::optional getArrivalIndex(const std::string &room_id, std::string_view event_id); diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index de43d5ea..1163d931 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -740,10 +740,25 @@ TimelineModel::setCurrentIndex(int index) auto oldIndex = idToIndex(currentId); currentId = indexToId(index); - emit currentIndexChanged(index); - - if ((oldIndex > index || oldIndex == -1) && !currentId.startsWith("m")) { - readEvent(currentId.toStdString()); + if (index != oldIndex) + emit currentIndexChanged(index); + + if (!currentId.startsWith("m")) { + auto oldReadIndex = + cache::getEventIndex(roomId().toStdString(), currentReadId.toStdString()); + auto nextEventIndexAndId = + cache::lastInvisibleEventAfter(roomId().toStdString(), currentId.toStdString()); + + if (nextEventIndexAndId && + (!oldReadIndex || *oldReadIndex < nextEventIndexAndId->first)) { + readEvent(nextEventIndexAndId->second); + currentReadId = QString::fromStdString(nextEventIndexAndId->second); + + nhlog::net()->info("Marked as read {}, index {}, oldReadIndex {}", + nextEventIndexAndId->second, + nextEventIndexAndId->first, + *oldReadIndex); + } } } diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index 0aec27a1..017b6589 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -329,7 +329,7 @@ private: bool decryptDescription = true; bool m_paginationInProgress = false; - QString currentId; + QString currentId, currentReadId; QString reply_, edit_; std::vector typingUsers_; -- cgit 1.5.1 From 6d678a108f0f1c551565c124de478f366dbe4ee2 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 10 Feb 2021 02:37:47 +0100 Subject: Use fully read marker and fix stuck read marker with edits --- src/Cache.cpp | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 109fc60d..8cf66d21 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1202,25 +1202,24 @@ Cache::calculateRoomReadStatus(const std::string &room_id) const auto last_event_id = getLastEventId(txn, room_id); const auto localUser = utils::localUser().toStdString(); + std::string fullyReadEventId; + if (auto ev = getAccountData(txn, mtx::events::EventType::FullyRead, room_id)) { + if (auto fr = std::get_if< + mtx::events::AccountDataEvent>( + &ev.value())) { + fullyReadEventId = fr->content.event_id; + } + } txn.commit(); - if (last_event_id.empty()) - return false; - - // Retrieve all read receipts for that event. - const auto receipts = - readReceipts(QString::fromStdString(last_event_id), QString::fromStdString(room_id)); - - if (receipts.size() == 0) + if (last_event_id.empty() || fullyReadEventId.empty()) return true; - // Check if the local user has a read receipt for it. - for (auto it = receipts.cbegin(); it != receipts.cend(); it++) { - if (it->second == localUser) - return false; - } + if (last_event_id == fullyReadEventId) + return false; - return true; + // Retrieve all read receipts for that event. + return getEventIndex(room_id, last_event_id) > getEventIndex(room_id, fullyReadEventId); } void @@ -1918,7 +1917,6 @@ Cache::getEventIndex(const std::string &room_id, std::string_view event_id) bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); if (!success) { - nhlog::db()->critical("Could not find event id: {}", event_id); return {}; } @@ -3320,9 +3318,12 @@ Cache::getAccountData(lmdb::txn &txn, mtx::events::EventType type, const std::st lmdb::val data; if (lmdb::dbi_get(txn, db, lmdb::val(to_string(type)), data)) { mtx::responses::utils::RoomAccountDataEvents events; - mtx::responses::utils::parse_room_account_data_events( - std::string_view(data.data(), data.size()), events); - return events.front(); + json j = json::array({ + json::parse(std::string_view(data.data(), data.size())), + }); + mtx::responses::utils::parse_room_account_data_events(j, events); + if (events.size() == 1) + return events.front(); } } catch (...) { } -- cgit 1.5.1