From 233b3c06ce79ee1c4445863430c0ebbee1f8faf1 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 1 Jul 2020 20:15:39 +0200 Subject: Store events in room specific db --- src/Cache.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index d9d1134e..27f4e694 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1955,24 +1955,34 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const mtx::responses::Timeline &res) { auto db = getMessagesDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); using namespace mtx::events; using namespace mtx::events::state; for (const auto &e : res.events) { - if (std::holds_alternative>(e)) - continue; + auto event = mtx::accessors::serialize_event(e); + if (auto redaction = + std::get_if>(&e)) { + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + } else { + json obj = json::object(); - json obj = json::object(); + obj["event"] = event; + obj["token"] = res.prev_batch; - obj["event"] = mtx::accessors::serialize_event(e); - obj["token"] = res.prev_batch; + lmdb::dbi_put( + txn, + db, + lmdb::val(std::to_string(event["origin_server_ts"].get())), + lmdb::val(obj.dump())); - lmdb::dbi_put( - txn, - db, - lmdb::val(std::to_string(obj["event"]["origin_server_ts"].get())), - lmdb::val(obj.dump())); + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(event["event_id"].get()), + lmdb::val(event.dump())); + } } } -- cgit 1.5.1 From 79a29953dd92b9c025a8b9915aeb56d1d78e5607 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 4 Jul 2020 02:09:12 +0200 Subject: Persist event order --- src/Cache.cpp | 20 ++++++++++++++++++++ src/Cache_p.h | 6 ++++++ 2 files changed, 26 insertions(+) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 27f4e694..2824960b 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1957,9 +1957,21 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto db = getMessagesDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); + auto orderDb = getEventOrderDb(txn, room_id); + if (res.limited) + lmdb::dbi_drop(txn, orderDb, false); + using namespace mtx::events; using namespace mtx::events::state; + lmdb::val indexVal, val; + int64_t index = 0; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_LAST)) { + index = *indexVal.data(); + } + + bool first = true; for (const auto &e : res.events) { auto event = mtx::accessors::serialize_event(e); if (auto redaction = @@ -1982,6 +1994,14 @@ Cache::saveTimelineMessages(lmdb::txn &txn, eventsDb, lmdb::val(event["event_id"].get()), lmdb::val(event.dump())); + + ++index; + + lmdb::cursor_put(cursor.handle(), + lmdb::val(&index, sizeof(index)), + lmdb::val(first ? res.prev_batch : ""), + MDB_APPEND); + first = false; } } } diff --git a/src/Cache_p.h b/src/Cache_p.h index e657447b..5f01f736 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -416,6 +416,12 @@ private: return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); } + lmdb::dbi getEventOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( -- cgit 1.5.1 From c79205c26a77df9086bd6294ae6285a7346e6656 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 5 Jul 2020 05:29:07 +0200 Subject: Use new timeline cache structure --- src/Cache.cpp | 246 ++++++++++++++++++++++++++++++++++------------------------ src/Cache_p.h | 24 +++--- 2 files changed, 154 insertions(+), 116 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 2824960b..26291cfd 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -37,7 +37,7 @@ //! Should be changed when a breaking change occurs in the cache format. //! This will reset client's data. -static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.05.01"); +static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.07.05"); static const std::string SECRET("secret"); static lmdb::val NEXT_BATCH_KEY("next_batch"); @@ -46,8 +46,9 @@ static lmdb::val CACHE_FORMAT_VERSION_KEY("cache_format_version"); constexpr size_t MAX_RESTORED_MESSAGES = 30'000; -constexpr auto DB_SIZE = 32ULL * 1024ULL * 1024ULL * 1024ULL; // 32 GB -constexpr auto MAX_DBS = 8092UL; +constexpr auto DB_SIZE = 32ULL * 1024ULL * 1024ULL * 1024ULL; // 32 GB +constexpr auto MAX_DBS = 8092UL; +constexpr auto BATCH_SIZE = 100; //! Cache databases and their format. //! @@ -63,7 +64,6 @@ constexpr auto SYNC_STATE_DB("sync_state"); //! Read receipts per room/event. constexpr auto READ_RECEIPTS_DB("read_receipts"); constexpr auto NOTIFICATIONS_DB("sent_notifications"); -//! TODO: delete pending_receipts database on old cache versions //! Encryption related databases. @@ -93,20 +93,6 @@ namespace { std::unique_ptr instance_ = nullptr; } -int -numeric_key_comparison(const MDB_val *a, const MDB_val *b) -{ - auto lhs = std::stoull(std::string((char *)a->mv_data, a->mv_size)); - auto rhs = std::stoull(std::string((char *)b->mv_data, b->mv_size)); - - if (lhs < rhs) - return 1; - else if (lhs == rhs) - return 0; - - return -1; -} - Cache::Cache(const QString &userId, QObject *parent) : QObject{parent} , env_{nullptr} @@ -697,6 +683,27 @@ Cache::runMigrations() return false; } + nhlog::db()->info("Successfully deleted pending receipts database."); + return true; + }}, + {"2020.07.05", + [this]() { + try { + auto txn = lmdb::txn::begin(env_, nullptr); + auto room_ids = getRoomIds(txn); + + for (const auto &room_id : room_ids) { + auto messagesDb = lmdb::dbi::open( + txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); + lmdb::dbi_drop(txn, messagesDb, true); + } + txn.commit(); + } catch (const lmdb::error &) { + nhlog::db()->critical( + "Failed to delete messages database in migration!"); + return false; + } + nhlog::db()->info("Successfully deleted pending receipts database."); return true; }}, @@ -1232,38 +1239,64 @@ Cache::getTimelineMentions() return notifs; } -mtx::responses::Timeline -Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id) +Cache::Messages +Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? - auto db = getMessagesDb(txn, room_id); + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); - mtx::responses::Timeline timeline; - std::string timestamp, msg; + Messages messages{}; - auto cursor = lmdb::cursor::open(txn, db); + lmdb::val indexVal, val; + + auto cursor = lmdb::cursor::open(txn, orderDb); + if (index == std::numeric_limits::max()) { + if (cursor.get(indexVal, val, forward ? MDB_FIRST : MDB_LAST)) { + index = *indexVal.data(); + } else { + messages.end_of_cache = true; + return messages; + } + } else { + if (cursor.get(indexVal, val, MDB_SET)) { + index = *indexVal.data(); + } else { + messages.end_of_cache = true; + return messages; + } + } - size_t index = 0; + int counter = 0; - while (cursor.get(timestamp, msg, MDB_NEXT) && index < MAX_RESTORED_MESSAGES) { - auto obj = json::parse(msg); + bool ret; + while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && + counter++ < BATCH_SIZE) { + auto obj = json::parse(std::string(val.data(), val.size())); - if (obj.count("event") == 0 || obj.count("token") == 0) - continue; + if (obj.count("event_id") == 0) + break; - mtx::events::collections::TimelineEvent event; - mtx::events::collections::from_json(obj.at("event"), event); + lmdb::val event; + bool success = lmdb::dbi_get( + txn, eventsDb, lmdb::val(obj["event_id"].get()), event); + if (!success) + continue; - index += 1; + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string(event.data(), event.size())), te); - timeline.events.push_back(event.data); - timeline.prev_batch = obj.at("token").get(); + messages.timeline.events.push_back(std::move(te.data)); + // timeline.prev_batch = obj.at("token").get(); } cursor.close(); - std::reverse(timeline.events.begin(), timeline.events.end()); + // std::reverse(timeline.events.begin(), timeline.events.end()); + messages.next_index = *indexVal.data(); + messages.end_of_cache = !ret; - return timeline; + return messages; } QMap @@ -1306,55 +1339,59 @@ Cache::roomInfo(bool withInvites) std::string Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) { - auto db = getMessagesDb(txn, room_id); - - if (db.size(txn) == 0) - return {}; - - std::string timestamp, msg; - - auto cursor = lmdb::cursor::open(txn, db); - while (cursor.get(timestamp, msg, MDB_NEXT)) { - auto obj = json::parse(msg); + auto orderDb = getEventOrderDb(txn, room_id); - if (obj.count("event") == 0) - continue; + lmdb::val indexVal, val; - cursor.close(); - return obj["event"]["event_id"]; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (!cursor.get(indexVal, val, MDB_LAST)) { + return {}; } - cursor.close(); - return {}; + auto obj = json::parse(std::string(val.data(), val.size())); + + if (obj.count("event_id") == 0) + return {}; + else + return obj["event_id"]; } DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { - auto db = getMessagesDb(txn, room_id); - - if (db.size(txn) == 0) + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + if (orderDb.size(txn) == 0) return DescInfo{}; - std::string timestamp, msg; - const auto local_user = utils::localUser(); DescInfo fallbackDesc{}; - auto cursor = lmdb::cursor::open(txn, db); - while (cursor.get(timestamp, msg, MDB_NEXT)) { - auto obj = json::parse(msg); + lmdb::val indexVal, val; - if (obj.count("event") == 0) + auto cursor = lmdb::cursor::open(txn, orderDb); + cursor.get(indexVal, val, MDB_LAST); + while (cursor.get(indexVal, val, MDB_PREV)) { + auto temp = json::parse(std::string(val.data(), val.size())); + + if (temp.count("event_id") == 0) + break; + + lmdb::val event; + bool success = lmdb::dbi_get( + txn, eventsDb, lmdb::val(temp["event_id"].get()), event); + if (!success) continue; - if (fallbackDesc.event_id.isEmpty() && obj["event"]["type"] == "m.room.member" && - obj["event"]["state_key"] == local_user.toStdString() && - obj["event"]["content"]["membership"] == "join") { - uint64_t ts = obj["event"]["origin_server_ts"]; + auto obj = json::parse(std::string(event.data(), event.size())); + + if (fallbackDesc.event_id.isEmpty() && obj["type"] == "m.room.member" && + obj["state_key"] == local_user.toStdString() && + obj["content"]["membership"] == "join") { + uint64_t ts = obj["origin_server_ts"]; auto time = QDateTime::fromMSecsSinceEpoch(ts); - fallbackDesc = DescInfo{QString::fromStdString(obj["event"]["event_id"]), + fallbackDesc = DescInfo{QString::fromStdString(obj["event_id"]), local_user, tr("You joined this room."), utils::descriptiveTime(time), @@ -1362,17 +1399,16 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) time}; } - if (!(obj["event"]["type"] == "m.room.message" || - obj["event"]["type"] == "m.sticker" || - obj["event"]["type"] == "m.room.encrypted")) + if (!(obj["type"] == "m.room.message" || obj["type"] == "m.sticker" || + obj["type"] == "m.room.encrypted")) continue; - mtx::events::collections::TimelineEvent event; - mtx::events::collections::from_json(obj.at("event"), event); + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json(obj, te); cursor.close(); return utils::getMessageDescription( - event.data, local_user, QString::fromStdString(room_id)); + te.data, local_user, QString::fromStdString(room_id)); } cursor.close(); @@ -1954,7 +1990,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { - auto db = getMessagesDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); auto orderDb = getEventOrderDb(txn, room_id); @@ -1966,7 +2001,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val indexVal, val; int64_t index = 0; - auto cursor = lmdb::cursor::open(txn, orderDb); + auto cursor = lmdb::cursor::open(txn, orderDb); if (cursor.get(indexVal, val, MDB_LAST)) { index = *indexVal.data(); } @@ -1979,17 +2014,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put( txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); } else { - json obj = json::object(); - - obj["event"] = event; - obj["token"] = res.prev_batch; - - lmdb::dbi_put( - txn, - db, - lmdb::val(std::to_string(event["origin_server_ts"].get())), - lmdb::val(obj.dump())); - lmdb::dbi_put(txn, eventsDb, lmdb::val(event["event_id"].get()), @@ -1997,9 +2021,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn, ++index; + json orderEntry = json::object(); + orderEntry["event_id"] = event["event_id"]; + if (first) + orderEntry["prev_batch"] = res.prev_batch; + + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + lmdb::cursor_put(cursor.handle(), lmdb::val(&index, sizeof(index)), - lmdb::val(first ? res.prev_batch : ""), + lmdb::val(orderEntry.dump()), MDB_APPEND); first = false; } @@ -2138,34 +2169,43 @@ Cache::getRoomIds(lmdb::txn &txn) void Cache::deleteOldMessages() { + lmdb::val indexVal, val; + auto txn = lmdb::txn::begin(env_); auto room_ids = getRoomIds(txn); - for (const auto &id : room_ids) { - auto msg_db = getMessagesDb(txn, id); - - std::string ts, event; - uint64_t idx = 0; + for (const auto &room_id : room_ids) { + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + auto cursor = lmdb::cursor::open(txn, orderDb); - const auto db_size = msg_db.size(txn); - if (db_size <= 3 * MAX_RESTORED_MESSAGES) + int64_t first, last; + if (cursor.get(indexVal, val, MDB_LAST)) { + last = *indexVal.data(); + } else { continue; + } + if (cursor.get(indexVal, val, MDB_FIRST)) { + first = *indexVal.data(); + } else { + continue; + } - nhlog::db()->info("[{}] message count: {}", id, db_size); + size_t message_count = static_cast(last - first); + if (message_count < MAX_RESTORED_MESSAGES) + continue; - auto cursor = lmdb::cursor::open(txn, msg_db); - while (cursor.get(ts, event, MDB_NEXT)) { - idx += 1; + while (cursor.get(indexVal, val, MDB_NEXT) && + message_count-- < MAX_RESTORED_MESSAGES) { + auto obj = json::parse(std::string(val.data(), val.size())); - if (idx > MAX_RESTORED_MESSAGES) - lmdb::cursor_del(cursor); + if (obj.count("event_id") != 0) + lmdb::dbi_del( + txn, eventsDb, lmdb::val(obj["event_id"].get())); + lmdb::cursor_del(cursor); } - cursor.close(); - - nhlog::db()->info("[{}] updated message count: {}", id, msg_db.size(txn)); } - txn.commit(); } diff --git a/src/Cache_p.h b/src/Cache_p.h index 5f01f736..37486ca0 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -38,9 +39,6 @@ #include "CacheCryptoStructs.h" #include "CacheStructs.h" -int -numeric_key_comparison(const MDB_val *a, const MDB_val *b); - class Cache : public QObject { Q_OBJECT @@ -250,7 +248,16 @@ private: const std::string &room_id, const mtx::responses::Timeline &res); - mtx::responses::Timeline getTimelineMessages(lmdb::txn &txn, const std::string &room_id); + struct Messages + { + mtx::responses::Timeline timeline; + uint64_t next_index; + bool end_of_cache = false; + }; + Messages getTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + int64_t index = std::numeric_limits::max(), + bool forward = false); //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); @@ -402,15 +409,6 @@ private: return lmdb::dbi::open(txn, "pending_receipts", MDB_CREATE); } - lmdb::dbi getMessagesDb(lmdb::txn &txn, const std::string &room_id) - { - auto db = - lmdb::dbi::open(txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); - lmdb::dbi_set_compare(txn, db, numeric_key_comparison); - - return db; - } - lmdb::dbi getEventsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); -- cgit 1.5.1 From 82eff09062c51a3136d169e97c70bbed2f439f26 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 6 Jul 2020 03:43:14 +0200 Subject: Fetch event from db and use string_view where possible --- src/Cache.cpp | 83 ++++++++++++++++++++++++++++++++++++++--------------------- src/Cache_p.h | 4 +++ 2 files changed, 58 insertions(+), 29 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 26291cfd..f07c3855 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -778,8 +778,9 @@ Cache::readReceipts(const QString &event_id, const QString &room_id) txn.commit(); if (res) { - auto json_response = json::parse(std::string(value.data(), value.size())); - auto values = json_response.get>(); + auto json_response = + json::parse(std::string_view(value.data(), value.size())); + auto values = json_response.get>(); for (const auto &v : values) // timestamp, user_id @@ -817,8 +818,8 @@ Cache::updateReadReceipt(lmdb::txn &txn, const std::string &room_id, const Recei // If an entry for the event id already exists, we would // merge the existing receipts with the new ones. if (exists) { - auto json_value = - json::parse(std::string(prev_value.data(), prev_value.size())); + auto json_value = json::parse( + std::string_view(prev_value.data(), prev_value.size())); // Retrieve the saved receipts. saved_receipts = json_value.get>(); @@ -937,7 +938,7 @@ Cache::saveState(const mtx::responses::Sync &res) if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room.first), data)) { try { RoomInfo tmp = - json::parse(std::string(data.data(), data.size())); + json::parse(std::string_view(data.data(), data.size())); updatedInfo.tags = tmp.tags; } catch (const json::exception &e) { nhlog::db()->warn( @@ -1129,7 +1130,7 @@ Cache::singleRoomInfo(const std::string &room_id) // Check if the room is joined. if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room_id), data)) { try { - RoomInfo tmp = json::parse(std::string(data.data(), data.size())); + RoomInfo tmp = json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getMembersDb(txn, room_id).size(txn); tmp.join_rule = getRoomJoinRule(txn, statesdb); tmp.guest_access = getRoomGuestAccess(txn, statesdb); @@ -1164,7 +1165,8 @@ Cache::getRoomInfo(const std::vector &rooms) // Check if the room is joined. if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room), data)) { try { - RoomInfo tmp = json::parse(std::string(data.data(), data.size())); + RoomInfo tmp = + json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getMembersDb(txn, room).size(txn); tmp.join_rule = getRoomJoinRule(txn, statesdb); tmp.guest_access = getRoomGuestAccess(txn, statesdb); @@ -1180,7 +1182,7 @@ Cache::getRoomInfo(const std::vector &rooms) if (lmdb::dbi_get(txn, invitesDb_, lmdb::val(room), data)) { try { RoomInfo tmp = - json::parse(std::string(data.data(), data.size())); + json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getInviteMembersDb(txn, room).size(txn); room_info.emplace(QString::fromStdString(room), @@ -1272,7 +1274,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i bool ret; while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && counter++ < BATCH_SIZE) { - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") == 0) break; @@ -1285,10 +1287,14 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i mtx::events::collections::TimelineEvent te; mtx::events::collections::from_json( - json::parse(std::string(event.data(), event.size())), te); + json::parse(std::string_view(event.data(), event.size())), te); messages.timeline.events.push_back(std::move(te.data)); - // timeline.prev_batch = obj.at("token").get(); + + if (forward && messages.timeline.prev_batch.empty() && obj.contains("prev_batch")) + messages.timeline.prev_batch = obj["prev_batch"]; + else if (!forward && obj.contains("prev_batch")) + messages.timeline.prev_batch = obj["prev_batch"]; } cursor.close(); @@ -1299,6 +1305,24 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i return messages; } +std::optional +Cache::getEvent(const std::string &room_id, const std::string &event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto eventsDb = getEventsDb(txn, room_id); + + lmdb::val event{}; + bool success = lmdb::dbi_get(txn, eventsDb, lmdb::val(event_id), event); + if (!success) + return {}; + + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + + return te; +} + QMap Cache::roomInfo(bool withInvites) { @@ -1348,7 +1372,7 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return {}; } - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") == 0) return {}; @@ -1373,7 +1397,7 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) auto cursor = lmdb::cursor::open(txn, orderDb); cursor.get(indexVal, val, MDB_LAST); while (cursor.get(indexVal, val, MDB_PREV)) { - auto temp = json::parse(std::string(val.data(), val.size())); + auto temp = json::parse(std::string_view(val.data(), val.size())); if (temp.count("event_id") == 0) break; @@ -1384,7 +1408,7 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) if (!success) continue; - auto obj = json::parse(std::string(event.data(), event.size())); + auto obj = json::parse(std::string_view(event.data(), event.size())); if (fallbackDesc.event_id.isEmpty() && obj["type"] == "m.room.member" && obj["state_key"] == local_user.toStdString() && @@ -1450,7 +1474,7 @@ Cache::getRoomAvatarUrl(lmdb::txn &txn, if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.url.empty()) return QString::fromStdString(msg.content.url); @@ -1500,7 +1524,8 @@ Cache::getRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb) if (res) { try { - StateEvent msg = json::parse(std::string(event.data(), event.size())); + StateEvent msg = + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.name.empty()) return QString::fromStdString(msg.content.name); @@ -1515,7 +1540,7 @@ Cache::getRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.alias.empty()) return QString::fromStdString(msg.content.alias); @@ -1578,7 +1603,7 @@ Cache::getRoomJoinRule(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return msg.content.join_rule; } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.join_rule event: {}", e.what()); @@ -1600,7 +1625,7 @@ Cache::getRoomGuestAccess(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return msg.content.guest_access == AccessState::CanJoin; } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.guest_access event: {}", @@ -1623,7 +1648,7 @@ Cache::getRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.topic.empty()) return QString::fromStdString(msg.content.topic); @@ -1648,7 +1673,7 @@ Cache::getRoomVersion(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.room_version.empty()) return QString::fromStdString(msg.content.room_version); @@ -1674,7 +1699,7 @@ Cache::getInviteRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &members if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.name); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.name event: {}", e.what()); @@ -1716,7 +1741,7 @@ Cache::getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &me if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.url); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.avatar event: {}", e.what()); @@ -1758,7 +1783,7 @@ Cache::getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &db) if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.topic); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.topic event: {}", e.what()); @@ -1789,7 +1814,7 @@ Cache::getRoomAvatar(const std::string &room_id) std::string media_url; try { - RoomInfo info = json::parse(std::string(response.data(), response.size())); + RoomInfo info = json::parse(std::string_view(response.data(), response.size())); media_url = std::move(info.avatar_url); if (media_url.empty()) { @@ -2197,7 +2222,7 @@ Cache::deleteOldMessages() while (cursor.get(indexVal, val, MDB_NEXT) && message_count-- < MAX_RESTORED_MESSAGES) { - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") != 0) lmdb::dbi_del( @@ -2239,7 +2264,7 @@ Cache::hasEnoughPowerLevel(const std::vector &eventTypes if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); user_level = msg.content.user_level(user_id); @@ -2354,7 +2379,7 @@ Cache::presenceState(const std::string &user_id) if (res) { mtx::events::presence::Presence presence = - json::parse(std::string(presenceVal.data(), presenceVal.size())); + json::parse(std::string_view(presenceVal.data(), presenceVal.size())); state = presence.presence; } @@ -2376,7 +2401,7 @@ Cache::statusMessage(const std::string &user_id) if (res) { mtx::events::presence::Presence presence = - json::parse(std::string(presenceVal.data(), presenceVal.size())); + json::parse(std::string_view(presenceVal.data(), presenceVal.size())); status_msg = presence.status_msg; } diff --git a/src/Cache_p.h b/src/Cache_p.h index 37486ca0..10839967 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -259,6 +259,10 @@ private: int64_t index = std::numeric_limits::max(), bool forward = false); + std::optional getEvent( + const std::string &room_id, + const std::string &event_id); + //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template -- cgit 1.5.1 From 0da1a6d5fce0f25162abfb9bcfd4041fb167ebf4 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 8 Jul 2020 02:02:14 +0200 Subject: Add relations and order without hidden events to db --- src/Cache.cpp | 119 +++++++++++++++++++++++++++++++++++----------------------- src/Cache_p.h | 18 +++++++++ 2 files changed, 90 insertions(+), 47 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index f07c3855..852d45ec 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1245,23 +1245,23 @@ Cache::Messages Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); Messages messages{}; - lmdb::val indexVal, val; + lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); if (index == std::numeric_limits::max()) { - if (cursor.get(indexVal, val, forward ? MDB_FIRST : MDB_LAST)) { + if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) { index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; } } else { - if (cursor.get(indexVal, val, MDB_SET)) { + if (cursor.get(indexVal, event_id, MDB_SET)) { index = *indexVal.data(); } else { messages.end_of_cache = true; @@ -1272,16 +1272,10 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i int counter = 0; bool ret; - while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && + while ((ret = cursor.get(indexVal, event_id, forward ? MDB_NEXT : MDB_LAST)) && counter++ < BATCH_SIZE) { - auto obj = json::parse(std::string_view(val.data(), val.size())); - - if (obj.count("event_id") == 0) - break; - lmdb::val event; - bool success = lmdb::dbi_get( - txn, eventsDb, lmdb::val(obj["event_id"].get()), event); + bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) continue; @@ -1290,11 +1284,6 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i json::parse(std::string_view(event.data(), event.size())), te); messages.timeline.events.push_back(std::move(te.data)); - - if (forward && messages.timeline.prev_batch.empty() && obj.contains("prev_batch")) - messages.timeline.prev_batch = obj["prev_batch"]; - else if (!forward && obj.contains("prev_batch")) - messages.timeline.prev_batch = obj["prev_batch"]; } cursor.close(); @@ -1363,7 +1352,7 @@ Cache::roomInfo(bool withInvites) std::string Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) { - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); lmdb::val indexVal, val; @@ -1372,18 +1361,13 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return {}; } - auto obj = json::parse(std::string_view(val.data(), val.size())); - - if (obj.count("event_id") == 0) - return {}; - else - return obj["event_id"]; + return std::string(val.data(), val.size()); } DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); if (orderDb.size(txn) == 0) return DescInfo{}; @@ -1392,19 +1376,13 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) DescInfo fallbackDesc{}; - lmdb::val indexVal, val; + lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - cursor.get(indexVal, val, MDB_LAST); - while (cursor.get(indexVal, val, MDB_PREV)) { - auto temp = json::parse(std::string_view(val.data(), val.size())); - - if (temp.count("event_id") == 0) - break; - + cursor.get(indexVal, event_id, MDB_LAST); + while (cursor.get(indexVal, event_id, MDB_PREV)) { lmdb::val event; - bool success = lmdb::dbi_get( - txn, eventsDb, lmdb::val(temp["event_id"].get()), event); + bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) continue; @@ -2015,11 +1993,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { - auto eventsDb = getEventsDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); - auto orderDb = getEventOrderDb(txn, room_id); - if (res.limited) + auto orderDb = getEventOrderDb(txn, room_id); + auto msg2orderDb = getMessageToOrderDb(txn, room_id); + auto order2msgDb = getOrderToMessageDb(txn, room_id); + if (res.limited) { lmdb::dbi_drop(txn, orderDb, false); + lmdb::dbi_drop(txn, msg2orderDb, false); + lmdb::dbi_drop(txn, order2msgDb, false); + } using namespace mtx::events; using namespace mtx::events::state; @@ -2031,6 +2015,12 @@ Cache::saveTimelineMessages(lmdb::txn &txn, index = *indexVal.data(); } + int64_t msgIndex = 0; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_LAST)) { + msgIndex = *indexVal.data(); + } + bool first = true; for (const auto &e : res.events) { auto event = mtx::accessors::serialize_event(e); @@ -2039,17 +2029,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put( txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); } else { - lmdb::dbi_put(txn, - eventsDb, - lmdb::val(event["event_id"].get()), - lmdb::val(event.dump())); + std::string event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; json orderEntry = json::object(); - orderEntry["event_id"] = event["event_id"]; + orderEntry["event_id"] = event_id_val; if (first) orderEntry["prev_batch"] = res.prev_batch; + first = false; nhlog::db()->debug("saving '{}'", orderEntry.dump()); @@ -2057,7 +2047,32 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()), MDB_APPEND); - first = false; + + // TODO(Nico): Allow blacklisting more event types in UI + if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + ++msgIndex; + lmdb::cursor_put(msgCursor.handle(), + lmdb::val(&msgIndex, sizeof(msgIndex)), + event_id, + MDB_APPEND); + + lmdb::dbi_put(txn, + msg2orderDb, + event_id, + lmdb::val(&msgIndex, sizeof(msgIndex))); + } + + if (event.contains("content") && + event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) + lmdb::dbi_put( + txn, relationsDb, lmdb::val(relates_to), event_id); + } } } } @@ -2201,6 +2216,8 @@ Cache::deleteOldMessages() for (const auto &room_id : room_ids) { auto orderDb = getEventOrderDb(txn, room_id); + auto o2m = getOrderToMessageDb(txn, room_id); + auto m2o = getMessageToOrderDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); auto cursor = lmdb::cursor::open(txn, orderDb); @@ -2224,9 +2241,17 @@ Cache::deleteOldMessages() message_count-- < MAX_RESTORED_MESSAGES) { auto obj = json::parse(std::string_view(val.data(), val.size())); - if (obj.count("event_id") != 0) - lmdb::dbi_del( - txn, eventsDb, lmdb::val(obj["event_id"].get())); + if (obj.count("event_id") != 0) { + lmdb::val event_id = obj["event_id"].get(); + lmdb::dbi_del(txn, eventsDb, event_id); + + lmdb::val order{}; + bool exists = lmdb::dbi_get(txn, m2o, event_id, order); + if (exists) { + lmdb::dbi_del(txn, o2m, order); + lmdb::dbi_del(txn, m2o, event_id); + } + } lmdb::cursor_del(cursor); } cursor.close(); diff --git a/src/Cache_p.h b/src/Cache_p.h index 10839967..3f7b592d 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -424,6 +424,24 @@ private: txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + lmdb::dbi getMessageToOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/msg2order").c_str(), MDB_CREATE); + } + + lmdb::dbi getOrderToMessageDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/order2msg").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + + lmdb::dbi getRelationsDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/related").c_str(), MDB_CREATE | MDB_DUPSORT); + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( -- cgit 1.5.1 From 530c531c4b447a0d3599a74731441f2656374f3f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 9 Jul 2020 23:15:22 +0200 Subject: WIP: Event Store split out --- CMakeLists.txt | 2 + resources/qml/TimelineRow.qml | 2 +- resources/qml/TimelineView.qml | 2 +- src/Cache.cpp | 133 +++++++- src/Cache_p.h | 39 ++- src/timeline/EventStore.cpp | 259 ++++++++++++++++ src/timeline/EventStore.h | 98 ++++++ src/timeline/TimelineModel.cpp | 669 ++++++++++++++++------------------------- src/timeline/TimelineModel.h | 13 +- 9 files changed, 767 insertions(+), 450 deletions(-) create mode 100644 src/timeline/EventStore.cpp create mode 100644 src/timeline/EventStore.h (limited to 'src/Cache.cpp') diff --git a/CMakeLists.txt b/CMakeLists.txt index f1ccde5f..8d1441c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -250,6 +250,7 @@ set(SRC_FILES # Timeline + src/timeline/EventStore.cpp src/timeline/ReactionsModel.cpp src/timeline/TimelineViewManager.cpp src/timeline/TimelineModel.cpp @@ -453,6 +454,7 @@ qt5_wrap_cpp(MOC_HEADERS src/emoji/Provider.h # Timeline + src/timeline/EventStore.h src/timeline/ReactionsModel.h src/timeline/TimelineViewManager.h src/timeline/TimelineModel.h diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml index dfee62dc..e87590f1 100644 --- a/resources/qml/TimelineRow.qml +++ b/resources/qml/TimelineRow.qml @@ -45,7 +45,7 @@ MouseArea { // fancy reply, if this is a reply Reply { visible: model.replyTo - modelData: chat.model.getDump(model.replyTo) + modelData: chat.model.getDump(model.replyTo, model.id) userColor: timelineManager.userColor(modelData.userId, colors.window) } diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index f81f5986..fd185bd9 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -353,7 +353,7 @@ Page { anchors.rightMargin: 20 anchors.bottom: parent.bottom - modelData: chat.model ? chat.model.getDump(chat.model.reply) : {} + modelData: chat.model ? chat.model.getDump(chat.model.reply, chat.model.id) : {} userColor: timelineManager.userColor(modelData.userId, colors.window) } diff --git a/src/Cache.cpp b/src/Cache.cpp index 852d45ec..d2c790dd 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1272,7 +1272,10 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i int counter = 0; bool ret; - while ((ret = cursor.get(indexVal, event_id, forward ? MDB_NEXT : MDB_LAST)) && + while ((ret = cursor.get(indexVal, + event_id, + counter == 0 ? (forward ? MDB_FIRST : MDB_LAST) + : (forward ? MDB_NEXT : MDB_PREV))) && counter++ < BATCH_SIZE) { lmdb::val event; bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); @@ -1280,8 +1283,13 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i continue; mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json( - json::parse(std::string_view(event.data(), event.size())), te); + try { + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + continue; + } messages.timeline.events.push_back(std::move(te.data)); } @@ -1306,8 +1314,13 @@ Cache::getEvent(const std::string &room_id, const std::string &event_id) return {}; mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json( - json::parse(std::string_view(event.data(), event.size())), te); + try { + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + return std::nullopt; + } return te; } @@ -1364,6 +1377,61 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return std::string(val.data(), val.size()); } +std::optional +Cache::getTimelineRange(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal, val; + + auto cursor = lmdb::cursor::open(txn, orderDb); + if (!cursor.get(indexVal, val, MDB_LAST)) { + return {}; + } + + TimelineRange range{}; + range.last = *indexVal.data(); + + if (!cursor.get(indexVal, val, MDB_FIRST)) { + return {}; + } + range.first = *indexVal.data(); + + return range; +} +std::optional +Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getMessageToOrderDb(txn, room_id); + + lmdb::val indexVal{event_id.data(), event_id.size()}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + return {}; + } + + return *val.data(); +} + +std::optional +Cache::getTimelineEventId(const std::string &room_id, int64_t index) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal{&index, sizeof(index)}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + return {}; + } + + return std::string(val.data(), val.size()); +} + DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { @@ -1379,8 +1447,10 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - cursor.get(indexVal, event_id, MDB_LAST); - while (cursor.get(indexVal, event_id, MDB_PREV)) { + bool first = true; + while (cursor.get(indexVal, event_id, first ? MDB_LAST : MDB_PREV)) { + first = false; + lmdb::val event; bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) @@ -2026,8 +2096,43 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto event = mtx::accessors::serialize_event(e); if (auto redaction = std::get_if>(&e)) { - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + if (redaction->redacts.empty()) + continue; + + lmdb::val ev{}; + bool success = + lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev); + if (!success) + continue; + + mtx::events::collections::TimelineEvent te; + + try { + mtx::events::collections::from_json( + json::parse(std::string_view(ev.data(), ev.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", + e.what()); + continue; + } + + auto redactedEvent = std::visit( + [](const auto &ev) -> mtx::events::RoomEvent { + mtx::events::RoomEvent replacement = + {}; + replacement.event_id = ev.event_id; + replacement.room_id = ev.room_id; + replacement.sender = ev.sender; + replacement.origin_server_ts = ev.origin_server_ts; + replacement.type = ev.type; + return replacement; + }, + te.data); + + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(redaction->redacts), + lmdb::val(json(redactedEvent).dump())); } else { std::string event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; @@ -2237,8 +2342,10 @@ Cache::deleteOldMessages() if (message_count < MAX_RESTORED_MESSAGES) continue; - while (cursor.get(indexVal, val, MDB_NEXT) && + bool start = true; + while (cursor.get(indexVal, val, start ? MDB_FIRST : MDB_NEXT) && message_count-- < MAX_RESTORED_MESSAGES) { + start = false; auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") != 0) { @@ -2394,6 +2501,9 @@ Cache::removeAvatarUrl(const QString &room_id, const QString &user_id) mtx::presence::PresenceState Cache::presenceState(const std::string &user_id) { + if (user_id.empty()) + return {}; + lmdb::val presenceVal; auto txn = lmdb::txn::begin(env_); @@ -2416,6 +2526,9 @@ Cache::presenceState(const std::string &user_id) std::string Cache::statusMessage(const std::string &user_id) { + if (user_id.empty()) + return {}; + lmdb::val presenceVal; auto txn = lmdb::txn::begin(env_); diff --git a/src/Cache_p.h b/src/Cache_p.h index 3f7b592d..40c8e98b 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -170,6 +170,30 @@ public: //! Add all notifications containing a user mention to the db. void saveTimelineMentions(const mtx::responses::Notifications &res); + //! retrieve events in timeline and related functions + struct Messages + { + mtx::responses::Timeline timeline; + uint64_t next_index; + bool end_of_cache = false; + }; + Messages getTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + int64_t index = std::numeric_limits::max(), + bool forward = false); + + std::optional getEvent( + const std::string &room_id, + const std::string &event_id); + struct TimelineRange + { + int64_t first, last; + }; + std::optional getTimelineRange(const std::string &room_id); + std::optional getTimelineIndex(const std::string &room_id, + std::string_view event_id); + std::optional getTimelineEventId(const std::string &room_id, int64_t index); + //! Remove old unused data. void deleteOldMessages(); void deleteOldData() noexcept; @@ -248,21 +272,6 @@ private: const std::string &room_id, const mtx::responses::Timeline &res); - struct Messages - { - mtx::responses::Timeline timeline; - uint64_t next_index; - bool end_of_cache = false; - }; - Messages getTimelineMessages(lmdb::txn &txn, - const std::string &room_id, - int64_t index = std::numeric_limits::max(), - bool forward = false); - - std::optional getEvent( - const std::string &room_id, - const std::string &event_id); - //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp new file mode 100644 index 00000000..eb54d475 --- /dev/null +++ b/src/timeline/EventStore.cpp @@ -0,0 +1,259 @@ +#include "EventStore.h" + +#include + +#include "Cache_p.h" +#include "EventAccessors.h" +#include "Logging.h" +#include "Olm.h" + +QCache EventStore::decryptedEvents_{ + 1000}; +QCache EventStore::events_by_id_{ + 1000}; +QCache EventStore::events_{1000}; + +EventStore::EventStore(std::string room_id, QObject *) + : room_id_(std::move(room_id)) +{ + auto range = cache::client()->getTimelineRange(room_id_); + + if (range) { + this->first = range->first; + this->last = range->last; + } +} + +void +EventStore::handleSync(const mtx::responses::Timeline &events) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + auto range = cache::client()->getTimelineRange(room_id_); + + if (range && range->last > this->last) { + emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last)); + this->last = range->last; + emit endInsertRows(); + } + + for (const auto &event : events.events) { + std::string relates_to; + if (auto redaction = + std::get_if>( + &event)) { + relates_to = redaction->redacts; + } else if (auto reaction = + std::get_if>( + &event)) { + relates_to = reaction->content.relates_to.event_id; + } else { + relates_to = mtx::accessors::in_reply_to_event(event); + } + + if (!relates_to.empty()) { + auto idx = cache::client()->getTimelineIndex(room_id_, relates_to); + if (idx) + emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } + } +} + +mtx::events::collections::TimelineEvents * +EventStore::event(int idx, bool decrypt) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + Index index{room_id_, toInternalIdx(idx)}; + if (index.idx > last || index.idx < first) + return nullptr; + + auto event_ptr = events_.object(index); + if (!event_ptr) { + auto event_id = cache::client()->getTimelineEventId(room_id_, index.idx); + if (!event_id) + return nullptr; + + auto event = cache::client()->getEvent(room_id_, *event_id); + if (!event) + return nullptr; + else + event_ptr = + new mtx::events::collections::TimelineEvents(std::move(event->data)); + events_.insert(index, event_ptr); + } + + if (decrypt) + if (auto encrypted = + std::get_if>( + event_ptr)) + return decryptEvent({room_id_, encrypted->event_id}, *encrypted); + + return event_ptr; +} + +std::optional +EventStore::idToIndex(std::string_view id) const +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + auto idx = cache::client()->getTimelineIndex(room_id_, id); + if (idx) + return toExternalIdx(*idx); + else + return std::nullopt; +} +std::optional +EventStore::indexToId(int idx) const +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + return cache::client()->getTimelineEventId(room_id_, toInternalIdx(idx)); +} + +mtx::events::collections::TimelineEvents * +EventStore::decryptEvent(const IdIndex &idx, + const mtx::events::EncryptedEvent &e) +{ + if (auto cachedEvent = decryptedEvents_.object(idx)) + return cachedEvent; + + MegolmSessionIndex index; + index.room_id = room_id_; + index.session_id = e.content.session_id; + index.sender_key = e.content.sender_key; + + mtx::events::RoomEvent dummy; + dummy.origin_server_ts = e.origin_server_ts; + dummy.event_id = e.event_id; + dummy.sender = e.sender; + dummy.content.body = + tr("-- Encrypted Event (No keys found for decryption) --", + "Placeholder, when the message was not decrypted yet or can't be decrypted.") + .toStdString(); + + auto asCacheEntry = [&idx](mtx::events::collections::TimelineEvents &&event) { + auto event_ptr = new mtx::events::collections::TimelineEvents(std::move(event)); + decryptedEvents_.insert(idx, event_ptr); + return event_ptr; + }; + + try { + if (!cache::client()->inboundMegolmSessionExists(index)) { + nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})", + index.room_id, + index.session_id, + e.sender); + // TODO: request megolm session_id & session_key from the sender. + return asCacheEntry(std::move(dummy)); + } + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to check megolm session's existence: {}", e.what()); + dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --", + "Placeholder, when the message can't be decrypted, because " + "the DB access failed when trying to lookup the session.") + .toStdString(); + return asCacheEntry(std::move(dummy)); + } + + std::string msg_str; + try { + auto session = cache::client()->getInboundMegolmSession(index); + auto res = olm::client()->decrypt_group_message(session, e.content.ciphertext); + msg_str = std::string((char *)res.data.data(), res.data.size()); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})", + index.room_id, + index.session_id, + index.sender_key, + e.what()); + dummy.content.body = + tr("-- Decryption Error (failed to retrieve megolm keys from db) --", + "Placeholder, when the message can't be decrypted, because the DB access " + "failed.") + .toStdString(); + return asCacheEntry(std::move(dummy)); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}", + index.room_id, + index.session_id, + index.sender_key, + e.what()); + dummy.content.body = + tr("-- Decryption Error (%1) --", + "Placeholder, when the message can't be decrypted. In this case, the Olm " + "decrytion returned an error, which is passed as %1.") + .arg(e.what()) + .toStdString(); + return asCacheEntry(std::move(dummy)); + } + + // Add missing fields for the event. + json body = json::parse(msg_str); + body["event_id"] = e.event_id; + body["sender"] = e.sender; + body["origin_server_ts"] = e.origin_server_ts; + body["unsigned"] = e.unsigned_data; + + // relations are unencrypted in content... + if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0) + body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"]; + + json event_array = json::array(); + event_array.push_back(body); + + std::vector temp_events; + mtx::responses::utils::parse_timeline_events(event_array, temp_events); + + if (temp_events.size() == 1) { + auto encInfo = mtx::accessors::file(temp_events[0]); + + if (encInfo) + emit newEncryptedImage(encInfo.value()); + + return asCacheEntry(std::move(temp_events[0])); + } + + dummy.content.body = + tr("-- Encrypted Event (Unknown event type) --", + "Placeholder, when the message was decrypted, but we couldn't parse it, because " + "Nheko/mtxclient don't support that event type yet.") + .toStdString(); + return asCacheEntry(std::move(dummy)); +} + +mtx::events::collections::TimelineEvents * +EventStore::event(std::string_view id, std::string_view related_to, bool decrypt) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + if (id.empty()) + return nullptr; + + IdIndex index{room_id_, std::string(id.data(), id.size())}; + + auto event_ptr = events_by_id_.object(index); + if (!event_ptr) { + auto event = cache::client()->getEvent(room_id_, index.id); + if (!event) { + // fetch + (void)related_to; + return nullptr; + } + event_ptr = new mtx::events::collections::TimelineEvents(std::move(event->data)); + events_by_id_.insert(index, event_ptr); + } + + if (decrypt) + if (auto encrypted = + std::get_if>( + event_ptr)) + return decryptEvent(index, *encrypted); + + return event_ptr; +} diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h new file mode 100644 index 00000000..77d73536 --- /dev/null +++ b/src/timeline/EventStore.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +class EventStore : public QObject +{ + Q_OBJECT + +public: + EventStore(std::string room_id, QObject *parent); + + struct Index + { + std::string room; + int64_t idx; + + friend uint qHash(const Index &i, uint seed = 0) noexcept + { + QtPrivate::QHashCombine hash; + seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size())); + seed = hash(seed, i.idx); + return seed; + } + + friend bool operator==(const Index &a, const Index &b) noexcept + { + return a.idx == b.idx && a.room == b.room; + } + }; + struct IdIndex + { + std::string room, id; + + friend uint qHash(const IdIndex &i, uint seed = 0) noexcept + { + QtPrivate::QHashCombine hash; + seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size())); + seed = hash(seed, QByteArray::fromRawData(i.id.data(), i.id.size())); + return seed; + } + + friend bool operator==(const IdIndex &a, const IdIndex &b) noexcept + { + return a.id == b.id && a.room == b.room; + } + }; + + void fetchMore(); + void handleSync(const mtx::responses::Timeline &events); + + // optionally returns the event or nullptr and fetches it, after which it emits a + // relatedFetched event + mtx::events::collections::TimelineEvents *event(std::string_view id, + std::string_view related_to, + bool decrypt = true); + // always returns a proper event as long as the idx is valid + mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true); + + int size() const + { + return last != std::numeric_limits::max() + ? static_cast(last - first) + 1 + : 0; + } + int toExternalIdx(int64_t idx) const { return static_cast(idx - first); } + int64_t toInternalIdx(int idx) const { return first + idx; } + + std::optional idToIndex(std::string_view id) const; + std::optional indexToId(int idx) const; + +signals: + void beginInsertRows(int from, int to); + void endInsertRows(); + void dataChanged(int from, int to); + void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); + +private: + mtx::events::collections::TimelineEvents *decryptEvent( + const IdIndex &idx, + const mtx::events::EncryptedEvent &e); + + std::string room_id_; + + int64_t first = std::numeric_limits::max(), + last = std::numeric_limits::max(); + + static QCache decryptedEvents_; + static QCache events_; + static QCache events_by_id_; +}; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 504c6dcf..492d4e0a 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -141,6 +141,7 @@ toRoomEventTypeString(const mtx::events::collections::TimelineEvents &event) TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObject *parent) : QAbstractListModel(parent) + , events(room_id.toStdString(), this) , room_id_(room_id) , manager_(manager) { @@ -165,41 +166,41 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj this, [this](QString txn_id, QString event_id) { pending.removeOne(txn_id); - - auto ev = events.value(txn_id); - - if (auto reaction = - std::get_if>(&ev)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - auto &rModel = reactions[reactedTo]; - rModel.removeReaction(*reaction); - auto rCopy = *reaction; - rCopy.event_id = event_id.toStdString(); - rModel.addReaction(room_id_.toStdString(), rCopy); - } - - int idx = idToIndex(txn_id); - if (idx < 0) { - // transaction already received via sync - return; - } - eventOrder[idx] = event_id; - ev = std::visit( - [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { - auto eventCopy = e; - eventCopy.event_id = event_id.toStdString(); - return eventCopy; - }, - ev); - - events.remove(txn_id); - events.insert(event_id, ev); - - // mark our messages as read - readEvent(event_id.toStdString()); - - emit dataChanged(index(idx, 0), index(idx, 0)); + (void)event_id; + // auto ev = events.value(txn_id); + + // if (auto reaction = + // std::get_if>(&ev)) { + // QString reactedTo = + // QString::fromStdString(reaction->content.relates_to.event_id); + // auto &rModel = reactions[reactedTo]; + // rModel.removeReaction(*reaction); + // auto rCopy = *reaction; + // rCopy.event_id = event_id.toStdString(); + // rModel.addReaction(room_id_.toStdString(), rCopy); + //} + + // int idx = idToIndex(txn_id); + // if (idx < 0) { + // // transaction already received via sync + // return; + //} + // eventOrder[idx] = event_id; + // ev = std::visit( + // [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { + // auto eventCopy = e; + // eventCopy.event_id = event_id.toStdString(); + // return eventCopy; + // }, + // ev); + + // events.remove(txn_id); + // events.insert(event_id, ev); + + //// mark our messages as read + // readEvent(event_id.toStdString()); + + // emit dataChanged(index(idx, 0), index(idx, 0)); if (pending.size() > 0) emit nextPendingMessage(); @@ -224,16 +225,24 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj Qt::QueuedConnection); connect( + &events, + &EventStore::dataChanged, this, - &TimelineModel::eventFetched, - this, - [this](QString requestingEvent, mtx::events::collections::TimelineEvents event) { - events.insert(QString::fromStdString(mtx::accessors::event_id(event)), event); - auto idx = idToIndex(requestingEvent); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); + [this](int from, int to) { + emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0)); }, Qt::QueuedConnection); + + connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) { + nhlog::ui()->info("begin insert from {} to {}", + events.size() - to + (to - from), + events.size() - from + (to - from)); + beginInsertRows(QModelIndex(), + events.size() - to + (to - from), + events.size() - from + (to - from)); + }); + connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); }); + connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage); } QHash @@ -274,28 +283,22 @@ int TimelineModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent); - return (int)this->eventOrder.size(); + return this->events.size() + static_cast(pending.size()); } QVariantMap -TimelineModel::getDump(QString eventId) const +TimelineModel::getDump(QString eventId, QString relatedTo) const { - if (events.contains(eventId)) - return data(eventId, Dump).toMap(); + if (auto event = events.event(eventId.toStdString(), relatedTo.toStdString())) + return data(*event, Dump).toMap(); return {}; } QVariant -TimelineModel::data(const QString &id, int role) const +TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int role) const { using namespace mtx::accessors; - namespace acc = mtx::accessors; - mtx::events::collections::TimelineEvents event = events.value(id); - - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + namespace acc = mtx::accessors; switch (role) { case UserId: @@ -381,8 +384,9 @@ TimelineModel::data(const QString &id, int role) const return QVariant(prop > 0 ? prop : 1.); } case Id: - return id; + return QVariant(QString::fromStdString(event_id(event))); case State: { + auto id = QString::fromStdString(event_id(event)); auto containsOthers = [](const auto &vec) { for (const auto &e : vec) if (e.second != http::client()->user_id().to_string()) @@ -401,19 +405,22 @@ TimelineModel::data(const QString &id, int role) const return qml_mtx_events::Received; } case IsEncrypted: { - return std::holds_alternative< - mtx::events::EncryptedEvent>(events[id]); + // return std::holds_alternative< + // mtx::events::EncryptedEvent>(events[id]); + return false; } case IsRoomEncrypted: { return cache::isRoomEncrypted(room_id_.toStdString()); } case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); - case Reactions: + case Reactions: { + auto id = QString::fromStdString(event_id(event)); if (reactions.count(id)) return QVariant::fromValue((QObject *)&reactions.at(id)); else return {}; + } case RoomId: return QVariant(room_id_); case RoomName: @@ -425,30 +432,31 @@ TimelineModel::data(const QString &id, int role) const auto names = roleNames(); // m.insert(names[Section], data(id, static_cast(Section))); - m.insert(names[Type], data(id, static_cast(Type))); - m.insert(names[TypeString], data(id, static_cast(TypeString))); - m.insert(names[IsOnlyEmoji], data(id, static_cast(IsOnlyEmoji))); - m.insert(names[Body], data(id, static_cast(Body))); - m.insert(names[FormattedBody], data(id, static_cast(FormattedBody))); - m.insert(names[UserId], data(id, static_cast(UserId))); - m.insert(names[UserName], data(id, static_cast(UserName))); - m.insert(names[Timestamp], data(id, static_cast(Timestamp))); - m.insert(names[Url], data(id, static_cast(Url))); - m.insert(names[ThumbnailUrl], data(id, static_cast(ThumbnailUrl))); - m.insert(names[Blurhash], data(id, static_cast(Blurhash))); - m.insert(names[Filename], data(id, static_cast(Filename))); - m.insert(names[Filesize], data(id, static_cast(Filesize))); - m.insert(names[MimeType], data(id, static_cast(MimeType))); - m.insert(names[Height], data(id, static_cast(Height))); - m.insert(names[Width], data(id, static_cast(Width))); - m.insert(names[ProportionalHeight], data(id, static_cast(ProportionalHeight))); - m.insert(names[Id], data(id, static_cast(Id))); - m.insert(names[State], data(id, static_cast(State))); - m.insert(names[IsEncrypted], data(id, static_cast(IsEncrypted))); - m.insert(names[IsRoomEncrypted], data(id, static_cast(IsRoomEncrypted))); - m.insert(names[ReplyTo], data(id, static_cast(ReplyTo))); - m.insert(names[RoomName], data(id, static_cast(RoomName))); - m.insert(names[RoomTopic], data(id, static_cast(RoomTopic))); + m.insert(names[Type], data(event, static_cast(Type))); + m.insert(names[TypeString], data(event, static_cast(TypeString))); + m.insert(names[IsOnlyEmoji], data(event, static_cast(IsOnlyEmoji))); + m.insert(names[Body], data(event, static_cast(Body))); + m.insert(names[FormattedBody], data(event, static_cast(FormattedBody))); + m.insert(names[UserId], data(event, static_cast(UserId))); + m.insert(names[UserName], data(event, static_cast(UserName))); + m.insert(names[Timestamp], data(event, static_cast(Timestamp))); + m.insert(names[Url], data(event, static_cast(Url))); + m.insert(names[ThumbnailUrl], data(event, static_cast(ThumbnailUrl))); + m.insert(names[Blurhash], data(event, static_cast(Blurhash))); + m.insert(names[Filename], data(event, static_cast(Filename))); + m.insert(names[Filesize], data(event, static_cast(Filesize))); + m.insert(names[MimeType], data(event, static_cast(MimeType))); + m.insert(names[Height], data(event, static_cast(Height))); + m.insert(names[Width], data(event, static_cast(Width))); + m.insert(names[ProportionalHeight], + data(event, static_cast(ProportionalHeight))); + m.insert(names[Id], data(event, static_cast(Id))); + m.insert(names[State], data(event, static_cast(State))); + m.insert(names[IsEncrypted], data(event, static_cast(IsEncrypted))); + m.insert(names[IsRoomEncrypted], data(event, static_cast(IsRoomEncrypted))); + m.insert(names[ReplyTo], data(event, static_cast(ReplyTo))); + m.insert(names[RoomName], data(event, static_cast(RoomName))); + m.insert(names[RoomTopic], data(event, static_cast(RoomTopic))); return QVariant(m); } @@ -462,29 +470,33 @@ TimelineModel::data(const QModelIndex &index, int role) const { using namespace mtx::accessors; namespace acc = mtx::accessors; - if (index.row() < 0 && index.row() >= (int)eventOrder.size()) + if (index.row() < 0 && index.row() >= rowCount()) return QVariant(); - QString id = eventOrder[index.row()]; + auto event = events.event(rowCount() - index.row() - 1); - mtx::events::collections::TimelineEvents event = events.value(id); + if (!event) + return ""; if (role == Section) { - QDateTime date = origin_server_ts(event); + QDateTime date = origin_server_ts(*event); date.setTime(QTime()); - std::string userId = acc::sender(event); + std::string userId = acc::sender(*event); + + for (int r = rowCount() - index.row(); r < events.size(); r++) { + auto tempEv = events.event(r); + if (!tempEv) + break; - for (size_t r = index.row() + 1; r < eventOrder.size(); r++) { - auto tempEv = events.value(eventOrder[r]); - QDateTime prevDate = origin_server_ts(tempEv); + QDateTime prevDate = origin_server_ts(*tempEv); prevDate.setTime(QTime()); if (prevDate != date) return QString("%2 %1") .arg(date.toMSecsSinceEpoch()) .arg(QString::fromStdString(userId)); - std::string prevUserId = acc::sender(tempEv); + std::string prevUserId = acc::sender(*tempEv); if (userId != prevUserId) break; } @@ -492,16 +504,16 @@ TimelineModel::data(const QModelIndex &index, int role) const return QString("%1").arg(QString::fromStdString(userId)); } - return data(id, role); + return data(*event, role); } bool TimelineModel::canFetchMore(const QModelIndex &) const { - if (eventOrder.empty()) + if (!events.size()) return true; if (!std::holds_alternative>( - events[eventOrder.back()])) + *events.event(0))) return true; else @@ -562,13 +574,9 @@ TimelineModel::addEvents(const mtx::responses::Timeline &timeline) if (timeline.events.empty()) return; - std::vector ids = internalAddEvents(timeline.events); + internalAddEvents(timeline.events); - if (!ids.empty()) { - beginInsertRows(QModelIndex(), 0, static_cast(ids.size() - 1)); - this->eventOrder.insert(this->eventOrder.begin(), ids.rbegin(), ids.rend()); - endInsertRows(); - } + events.handleSync(timeline); if (!timeline.events.empty()) updateLastMessage(); @@ -613,21 +621,17 @@ isYourJoin(const mtx::events::Event &) void TimelineModel::updateLastMessage() { - for (auto it = eventOrder.begin(); it != eventOrder.end(); ++it) { - auto event = events.value(*it); - if (auto e = std::get_if>( - &event)) { - if (decryptDescription) { - event = decryptEvent(*e).event; - } - } + for (auto it = events.size() - 1; it >= 0; --it) { + auto event = events.event(it, decryptDescription); + if (!event) + continue; - if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, event)) { - auto time = mtx::accessors::origin_server_ts(event); + if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, *event)) { + auto time = mtx::accessors::origin_server_ts(*event); uint64_t ts = time.toMSecsSinceEpoch(); emit manager_->updateRoomsLastMessage( room_id_, - DescInfo{QString::fromStdString(mtx::accessors::event_id(event)), + DescInfo{QString::fromStdString(mtx::accessors::event_id(*event)), QString::fromStdString(http::client()->user_id().to_string()), tr("You joined this room."), utils::descriptiveTime(time), @@ -635,54 +639,34 @@ TimelineModel::updateLastMessage() time}); return; } - if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, event)) + if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, *event)) continue; auto description = utils::getMessageDescription( - event, QString::fromStdString(http::client()->user_id().to_string()), room_id_); + *event, QString::fromStdString(http::client()->user_id().to_string()), room_id_); emit manager_->updateRoomsLastMessage(room_id_, description); return; } } -std::vector +void TimelineModel::internalAddEvents( const std::vector &timeline) { - std::vector ids; for (auto e : timeline) { QString id = QString::fromStdString(mtx::accessors::event_id(e)); - if (this->events.contains(id)) { - this->events.insert(id, e); - int idx = idToIndex(id); - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; - } - - QString txid = QString::fromStdString(mtx::accessors::transaction_id(e)); - if (this->pending.removeOne(txid)) { - this->events.insert(id, e); - this->events.remove(txid); - int idx = idToIndex(txid); - if (idx < 0) { - nhlog::ui()->warn("Received index out of range"); - continue; - } - eventOrder[idx] = id; - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; - } - if (auto redaction = std::get_if>(&e)) { QString redacts = QString::fromStdString(redaction->redacts); - auto redacted = std::find(eventOrder.begin(), eventOrder.end(), redacts); - auto event = events.value(redacts); + auto event = events.event(redaction->redacts, redaction->event_id); + if (!event) + continue; + if (auto reaction = std::get_if>( - &event)) { + event)) { QString reactedTo = QString::fromStdString(reaction->content.relates_to.event_id); reactions[reactedTo].removeReaction(*reaction); @@ -691,26 +675,6 @@ TimelineModel::internalAddEvents( emit dataChanged(index(idx, 0), index(idx, 0)); } - if (redacted != eventOrder.end()) { - auto redactedEvent = std::visit( - [](const auto &ev) - -> mtx::events::RoomEvent { - mtx::events::RoomEvent - replacement = {}; - replacement.event_id = ev.event_id; - replacement.room_id = ev.room_id; - replacement.sender = ev.sender; - replacement.origin_server_ts = ev.origin_server_ts; - replacement.type = ev.type; - return replacement; - }, - e); - events.insert(redacts, redactedEvent); - - int row = (int)std::distance(eventOrder.begin(), redacted); - emit dataChanged(index(row, 0), index(row, 0)); - } - continue; // don't insert redaction into timeline } @@ -718,14 +682,13 @@ TimelineModel::internalAddEvents( std::get_if>(&e)) { QString reactedTo = QString::fromStdString(reaction->content.relates_to.event_id); - events.insert(id, e); - // remove local echo - if (!txid.isEmpty()) { - auto rCopy = *reaction; - rCopy.event_id = txid.toStdString(); - reactions[reactedTo].removeReaction(rCopy); - } + // // remove local echo + // if (!txid.isEmpty()) { + // auto rCopy = *reaction; + // rCopy.event_id = txid.toStdString(); + // reactions[reactedTo].removeReaction(rCopy); + // } reactions[reactedTo].addReaction(room_id_.toStdString(), *reaction); int idx = idToIndex(reactedTo); @@ -734,40 +697,27 @@ TimelineModel::internalAddEvents( continue; // don't insert reaction into timeline } - if (auto event = - std::get_if>(&e)) { - auto e_ = decryptEvent(*event).event; - auto encInfo = mtx::accessors::file(e_); - - if (encInfo) - emit newEncryptedImage(encInfo.value()); - } - - this->events.insert(id, e); - ids.push_back(id); - - auto replyTo = mtx::accessors::in_reply_to_event(e); - auto qReplyTo = QString::fromStdString(replyTo); - if (!replyTo.empty() && !events.contains(qReplyTo)) { - http::client()->get_event( - this->room_id_.toStdString(), - replyTo, - [this, id, replyTo]( - const mtx::events::collections::TimelineEvents &timeline, - mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error( - "Failed to retrieve event with id {}, which was " - "requested to show the replyTo for event {}", - replyTo, - id.toStdString()); - return; - } - emit eventFetched(id, timeline); - }); - } + // auto replyTo = mtx::accessors::in_reply_to_event(e); + // auto qReplyTo = QString::fromStdString(replyTo); + // if (!replyTo.empty() && !events.contains(qReplyTo)) { + // http::client()->get_event( + // this->room_id_.toStdString(), + // replyTo, + // [this, id, replyTo]( + // const mtx::events::collections::TimelineEvents &timeline, + // mtx::http::RequestErr err) { + // if (err) { + // nhlog::net()->error( + // "Failed to retrieve event with id {}, which was " + // "requested to show the replyTo for event {}", + // replyTo, + // id.toStdString()); + // return; + // } + // emit eventFetched(id, timeline); + // }); + //} } - return ids; } void @@ -798,22 +748,23 @@ TimelineModel::readEvent(const std::string &id) void TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs) { - std::vector ids = internalAddEvents(msgs.chunk); - - if (!ids.empty()) { - beginInsertRows(QModelIndex(), - static_cast(this->eventOrder.size()), - static_cast(this->eventOrder.size() + ids.size() - 1)); - this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); - endInsertRows(); - } - - prev_batch_token_ = QString::fromStdString(msgs.end); - - if (ids.empty() && !msgs.chunk.empty()) { - // no visible events fetched, prevent loading from stopping - fetchMore(QModelIndex()); - } + (void)msgs; + // std::vector ids = internalAddEvents(msgs.chunk); + + // if (!ids.empty()) { + // beginInsertRows(QModelIndex(), + // static_cast(this->eventOrder.size()), + // static_cast(this->eventOrder.size() + ids.size() - 1)); + // this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); + // endInsertRows(); + //} + + // prev_batch_token_ = QString::fromStdString(msgs.end); + + // if (ids.empty() && !msgs.chunk.empty()) { + // // no visible events fetched, prevent loading from stopping + // fetchMore(QModelIndex()); + //} } QString @@ -852,7 +803,10 @@ TimelineModel::escapeEmoji(QString str) const void TimelineModel::viewRawMessage(QString id) const { - std::string ev = mtx::accessors::serialize_event(events.value(id)).dump(4); + auto e = events.event(id.toStdString(), "", false); + if (!e) + return; + std::string ev = mtx::accessors::serialize_event(*e).dump(4); auto dialog = new dialogs::RawMessage(QString::fromStdString(ev)); Q_UNUSED(dialog); } @@ -860,13 +814,11 @@ TimelineModel::viewRawMessage(QString id) const void TimelineModel::viewDecryptedRawMessage(QString id) const { - auto event = events.value(id); - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + auto e = events.event(id.toStdString(), ""); + if (!e) + return; - std::string ev = mtx::accessors::serialize_event(event).dump(4); + std::string ev = mtx::accessors::serialize_event(*e).dump(4); auto dialog = new dialogs::RawMessage(QString::fromStdString(ev)); Q_UNUSED(dialog); } @@ -877,114 +829,6 @@ TimelineModel::openUserProfile(QString userid) const MainWindow::instance()->openUserProfile(userid, room_id_); } -DecryptionResult -TimelineModel::decryptEvent(const mtx::events::EncryptedEvent &e) const -{ - static QCache decryptedEvents{300}; - - if (auto cachedEvent = decryptedEvents.object(e.event_id)) - return *cachedEvent; - - MegolmSessionIndex index; - index.room_id = room_id_.toStdString(); - index.session_id = e.content.session_id; - index.sender_key = e.content.sender_key; - - mtx::events::RoomEvent dummy; - dummy.origin_server_ts = e.origin_server_ts; - dummy.event_id = e.event_id; - dummy.sender = e.sender; - dummy.content.body = - tr("-- Encrypted Event (No keys found for decryption) --", - "Placeholder, when the message was not decrypted yet or can't be decrypted.") - .toStdString(); - - try { - if (!cache::inboundMegolmSessionExists(index)) { - nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})", - index.room_id, - index.session_id, - e.sender); - // TODO: request megolm session_id & session_key from the sender. - decryptedEvents.insert( - dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to check megolm session's existence: {}", e.what()); - dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --", - "Placeholder, when the message can't be decrypted, because " - "the DB access failed when trying to lookup the session.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - - std::string msg_str; - try { - auto session = cache::getInboundMegolmSession(index); - auto res = olm::client()->decrypt_group_message(session, e.content.ciphertext); - msg_str = std::string((char *)res.data.data(), res.data.size()); - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})", - index.room_id, - index.session_id, - index.sender_key, - e.what()); - dummy.content.body = - tr("-- Decryption Error (failed to retrieve megolm keys from db) --", - "Placeholder, when the message can't be decrypted, because the DB access " - "failed.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}", - index.room_id, - index.session_id, - index.sender_key, - e.what()); - dummy.content.body = - tr("-- Decryption Error (%1) --", - "Placeholder, when the message can't be decrypted. In this case, the Olm " - "decrytion returned an error, which is passed ad %1.") - .arg(e.what()) - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - - // Add missing fields for the event. - json body = json::parse(msg_str); - body["event_id"] = e.event_id; - body["sender"] = e.sender; - body["origin_server_ts"] = e.origin_server_ts; - body["unsigned"] = e.unsigned_data; - - // relations are unencrypted in content... - if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0) - body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"]; - - json event_array = json::array(); - event_array.push_back(body); - - std::vector temp_events; - mtx::responses::utils::parse_timeline_events(event_array, temp_events); - - if (temp_events.size() == 1) { - decryptedEvents.insert(e.event_id, new DecryptionResult{temp_events[0], true}, 1); - return {temp_events[0], true}; - } - - dummy.content.body = - tr("-- Encrypted Event (Unknown event type) --", - "Placeholder, when the message was decrypted, but we couldn't parse it, because " - "Nheko/mtxclient don't support that event type yet.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; -} - void TimelineModel::replyAction(QString id) { @@ -995,23 +839,18 @@ TimelineModel::replyAction(QString id) RelatedInfo TimelineModel::relatedInfo(QString id) { - if (!events.contains(id)) + auto event = events.event(id.toStdString(), ""); + if (!event) return {}; - auto event = events.value(id); - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } - RelatedInfo related = {}; - related.quoted_user = QString::fromStdString(mtx::accessors::sender(event)); - related.related_event = mtx::accessors::event_id(event); - related.type = mtx::accessors::msg_type(event); + related.quoted_user = QString::fromStdString(mtx::accessors::sender(*event)); + related.related_event = mtx::accessors::event_id(*event); + related.type = mtx::accessors::msg_type(*event); // get body, strip reply fallback, then transform the event to text, if it is a media event // etc - related.quoted_body = QString::fromStdString(mtx::accessors::body(event)); + related.quoted_body = QString::fromStdString(mtx::accessors::body(*event)); QRegularExpression plainQuote("^>.*?$\n?", QRegularExpression::MultilineOption); while (related.quoted_body.startsWith(">")) related.quoted_body.remove(plainQuote); @@ -1020,7 +859,7 @@ TimelineModel::relatedInfo(QString id) related.quoted_body = utils::getQuoteBody(related); // get quoted body and strip reply fallback - related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(event); + related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(*event); related.quoted_formatted_body.remove(QRegularExpression( ".*", QRegularExpression::DotMatchesEverythingOption)); related.room = room_id_; @@ -1058,18 +897,19 @@ TimelineModel::idToIndex(QString id) const { if (id.isEmpty()) return -1; - for (int i = 0; i < (int)eventOrder.size(); i++) - if (id == eventOrder[i]) - return i; - return -1; + + auto idx = events.idToIndex(id.toStdString()); + if (idx) + return events.size() - *idx; + else + return -1; } QString TimelineModel::indexToId(int index) const { - if (index < 0 || index >= (int)eventOrder.size()) - return ""; - return eventOrder[index]; + auto id = events.indexToId(events.size() - index); + return id ? QString::fromStdString(*id) : ""; } // Note: this will only be called for our messages @@ -1477,58 +1317,56 @@ struct SendMessageVisitor void TimelineModel::processOnePendingMessage() { - if (pending.isEmpty()) - return; + // if (pending.isEmpty()) + // return; - QString txn_id_qstr = pending.first(); + // QString txn_id_qstr = pending.first(); - auto event = events.value(txn_id_qstr); - std::visit(SendMessageVisitor{txn_id_qstr, this}, event); + // auto event = events.value(txn_id_qstr); + // std::visit(SendMessageVisitor{txn_id_qstr, this}, event); } void TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event) { - std::visit( - [](auto &msg) { - msg.type = mtx::events::EventType::RoomMessage; - msg.event_id = http::client()->generate_txn_id(); - msg.sender = http::client()->user_id().to_string(); - msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); - }, - event); - - internalAddEvents({event}); - - QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); - pending.push_back(txn_id_qstr); - if (!std::get_if>(&event)) { - beginInsertRows(QModelIndex(), 0, 0); - this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); - endInsertRows(); - } - updateLastMessage(); - - emit nextPendingMessage(); + (void)event; + // std::visit( + // [](auto &msg) { + // msg.type = mtx::events::EventType::RoomMessage; + // msg.event_id = http::client()->generate_txn_id(); + // msg.sender = http::client()->user_id().to_string(); + // msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); + // }, + // event); + + // internalAddEvents({event}); + + // QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); + // pending.push_back(txn_id_qstr); + // if (!std::get_if>(&event)) { + // beginInsertRows(QModelIndex(), 0, 0); + // this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); + // endInsertRows(); + //} + // updateLastMessage(); + + // emit nextPendingMessage(); } bool TimelineModel::saveMedia(QString eventId) const { - mtx::events::collections::TimelineEvents event = events.value(eventId); - - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), ""); + if (!event) + return false; - QString mxcUrl = QString::fromStdString(mtx::accessors::url(event)); - QString originalFilename = QString::fromStdString(mtx::accessors::filename(event)); - QString mimeType = QString::fromStdString(mtx::accessors::mimetype(event)); + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)); - auto encryptionInfo = mtx::accessors::file(event); + auto encryptionInfo = mtx::accessors::file(*event); - qml_mtx_events::EventType eventType = toRoomEventType(event); + qml_mtx_events::EventType eventType = toRoomEventType(*event); QString dialogTitle; if (eventType == qml_mtx_events::EventType::ImageMessage) { @@ -1593,18 +1431,15 @@ TimelineModel::saveMedia(QString eventId) const void TimelineModel::cacheMedia(QString eventId) { - mtx::events::collections::TimelineEvents event = events.value(eventId); - - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), ""); + if (!event) + return; - QString mxcUrl = QString::fromStdString(mtx::accessors::url(event)); - QString originalFilename = QString::fromStdString(mtx::accessors::filename(event)); - QString mimeType = QString::fromStdString(mtx::accessors::mimetype(event)); + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)); - auto encryptionInfo = mtx::accessors::file(event); + auto encryptionInfo = mtx::accessors::file(*event); // If the message is a link to a non mxcUrl, don't download it if (!mxcUrl.startsWith("mxc://")) { @@ -1725,11 +1560,11 @@ TimelineModel::formatTypingUsers(const std::vector &users, QColor bg) QString TimelineModel::formatJoinRuleEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1750,11 +1585,11 @@ TimelineModel::formatJoinRuleEvent(QString id) QString TimelineModel::formatGuestAccessEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1774,11 +1609,11 @@ TimelineModel::formatGuestAccessEvent(QString id) QString TimelineModel::formatHistoryVisibilityEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1808,11 +1643,11 @@ TimelineModel::formatHistoryVisibilityEvent(QString id) QString TimelineModel::formatPowerLevelEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1826,28 +1661,30 @@ TimelineModel::formatPowerLevelEvent(QString id) QString TimelineModel::formatMemberEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; mtx::events::StateEvent *prevEvent = nullptr; - QString prevEventId = QString::fromStdString(event->unsigned_data.replaces_state); - if (!prevEventId.isEmpty()) { - if (!events.contains(prevEventId)) { + if (!event->unsigned_data.replaces_state.empty()) { + auto tempPrevEvent = + events.event(event->unsigned_data.replaces_state, event->event_id); + if (!tempPrevEvent) { http::client()->get_event( this->room_id_.toStdString(), event->unsigned_data.replaces_state, - [this, id, prevEventId]( + [this, id, prevEventId = event->unsigned_data.replaces_state]( const mtx::events::collections::TimelineEvents &timeline, mtx::http::RequestErr err) { if (err) { nhlog::net()->error( "Failed to retrieve event with id {}, which was " "requested to show the membership for event {}", - prevEventId.toStdString(), + prevEventId, id.toStdString()); return; } @@ -1856,7 +1693,7 @@ TimelineModel::formatMemberEvent(QString id) } else { prevEvent = std::get_if>( - &events[prevEventId]); + tempPrevEvent); } } diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index a3b92f83..f322b482 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -9,6 +9,7 @@ #include #include "CacheCryptoStructs.h" +#include "EventStore.h" #include "ReactionsModel.h" namespace mtx::http { @@ -170,7 +171,7 @@ public: QHash roleNames() const override; int rowCount(const QModelIndex &parent = QModelIndex()) const override; QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; - QVariant data(const QString &id, int role) const; + QVariant data(const mtx::events::collections::TimelineEvents &event, int role) const; bool canFetchMore(const QModelIndex &) const override; void fetchMore(const QModelIndex &) override; @@ -207,7 +208,7 @@ public slots: void setCurrentIndex(int index); int currentIndex() const { return idToIndex(currentId); } void markEventsAsRead(const std::vector &event_ids); - QVariantMap getDump(QString eventId) const; + QVariantMap getDump(QString eventId, QString relatedTo) const; void updateTypingUsers(const std::vector &users) { if (this->typingUsers_ != users) { @@ -257,9 +258,7 @@ signals: void paginationInProgressChanged(const bool); private: - DecryptionResult decryptEvent( - const mtx::events::EncryptedEvent &e) const; - std::vector internalAddEvents( + void internalAddEvents( const std::vector &timeline); void sendEncryptedMessage(const std::string &txn_id, nlohmann::json content); void handleClaimedKeys(std::shared_ptr keeper, @@ -272,12 +271,12 @@ private: void setPaginationInProgress(const bool paginationInProgress); - QHash events; QSet read; QList pending; - std::vector eventOrder; std::map reactions; + mutable EventStore events; + QString room_id_; QString prev_batch_token_; -- cgit 1.5.1 From 3421728898cd12a39d541ae7bedee4f0e58f47b5 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Fri, 10 Jul 2020 01:37:55 +0200 Subject: Fetch missing events --- src/Cache.cpp | 11 +++++++++++ src/Cache_p.h | 3 +++ src/timeline/EventStore.cpp | 37 +++++++++++++++++++++++++++++++++++-- src/timeline/EventStore.h | 3 +++ src/timeline/TimelineModel.cpp | 21 --------------------- 5 files changed, 52 insertions(+), 23 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index d2c790dd..173b2c70 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1324,6 +1324,17 @@ Cache::getEvent(const std::string &room_id, const std::string &event_id) return te; } +void +Cache::storeEvent(const std::string &room_id, + const std::string &event_id, + const mtx::events::collections::TimelineEvent &event) +{ + auto txn = lmdb::txn::begin(env_); + auto eventsDb = getEventsDb(txn, room_id); + auto event_json = mtx::accessors::serialize_event(event.data); + lmdb::dbi_put(txn, eventsDb, lmdb::val(event_id), lmdb::val(event_json.dump())); + txn.commit(); +} QMap Cache::roomInfo(bool withInvites) diff --git a/src/Cache_p.h b/src/Cache_p.h index 40c8e98b..6b4b260e 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -185,6 +185,9 @@ public: std::optional getEvent( const std::string &room_id, const std::string &event_id); + void storeEvent(const std::string &room_id, + const std::string &event_id, + const mtx::events::collections::TimelineEvent &event); struct TimelineRange { int64_t first, last; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index eb54d475..719743fb 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -5,6 +5,7 @@ #include "Cache_p.h" #include "EventAccessors.h" #include "Logging.h" +#include "MatrixClient.h" #include "Olm.h" QCache EventStore::decryptedEvents_{ @@ -22,6 +23,23 @@ EventStore::EventStore(std::string room_id, QObject *) this->first = range->first; this->last = range->last; } + + connect( + this, + &EventStore::eventFetched, + this, + [this](std::string id, + std::string relatedTo, + mtx::events::collections::TimelineEvents timeline) { + cache::client()->storeEvent(room_id_, id, {timeline}); + + if (!relatedTo.empty()) { + auto idx = idToIndex(id); + if (idx) + emit dataChanged(*idx, *idx); + } + }, + Qt::QueuedConnection); } void @@ -241,8 +259,23 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt if (!event_ptr) { auto event = cache::client()->getEvent(room_id_, index.id); if (!event) { - // fetch - (void)related_to; + http::client()->get_event( + room_id_, + index.id, + [this, + relatedTo = std::string(related_to.data(), related_to.size()), + id = index.id](const mtx::events::collections::TimelineEvents &timeline, + mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error( + "Failed to retrieve event with id {}, which was " + "requested to show the replyTo for event {}", + relatedTo, + id); + return; + } + emit eventFetched(id, relatedTo, timeline); + }); return nullptr; } event_ptr = new mtx::events::collections::TimelineEvents(std::move(event->data)); diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 77d73536..83c8f7a4 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -81,6 +81,9 @@ signals: void endInsertRows(); void dataChanged(int from, int to); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); + void eventFetched(std::string id, + std::string relatedTo, + mtx::events::collections::TimelineEvents timeline); private: mtx::events::collections::TimelineEvents *decryptEvent( diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 492d4e0a..6df92d7a 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -696,27 +696,6 @@ TimelineModel::internalAddEvents( emit dataChanged(index(idx, 0), index(idx, 0)); continue; // don't insert reaction into timeline } - - // auto replyTo = mtx::accessors::in_reply_to_event(e); - // auto qReplyTo = QString::fromStdString(replyTo); - // if (!replyTo.empty() && !events.contains(qReplyTo)) { - // http::client()->get_event( - // this->room_id_.toStdString(), - // replyTo, - // [this, id, replyTo]( - // const mtx::events::collections::TimelineEvents &timeline, - // mtx::http::RequestErr err) { - // if (err) { - // nhlog::net()->error( - // "Failed to retrieve event with id {}, which was " - // "requested to show the replyTo for event {}", - // replyTo, - // id.toStdString()); - // return; - // } - // emit eventFetched(id, timeline); - // }); - //} } } -- cgit 1.5.1 From 9ae7d0dce3d78cefc0498e2322117ef00c6ec2e8 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 13 Jul 2020 00:08:58 +0200 Subject: Readd pagination and fix redactions --- resources/qml/delegates/MessageDelegate.qml | 6 + src/Cache.cpp | 177 ++++++++++++++++++++-------- src/Cache_p.h | 15 ++- src/ChatPage.cpp | 56 ++++----- src/timeline/EventStore.cpp | 55 ++++++++- src/timeline/EventStore.h | 17 ++- src/timeline/TimelineModel.cpp | 53 ++++----- 7 files changed, 256 insertions(+), 123 deletions(-) (limited to 'src/Cache.cpp') diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index 17fe7360..9630ae3a 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -66,6 +66,12 @@ Item { text: qsTr("redacted") } } + DelegateChoice { + roleValue: MtxEvent.Redaction + Pill { + text: qsTr("redacted") + } + } DelegateChoice { roleValue: MtxEvent.Encryption Pill { diff --git a/src/Cache.cpp b/src/Cache.cpp index 173b2c70..233ef2b4 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1241,8 +1241,25 @@ Cache::getTimelineMentions() return notifs; } +std::string +Cache::previousBatchToken(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr); + auto orderDb = getEventOrderDb(txn, room_id); + + auto cursor = lmdb::cursor::open(txn, orderDb); + lmdb::val indexVal, val; + if (!cursor.get(indexVal, val, MDB_FIRST)) { + return ""; + } + + auto j = json::parse(std::string_view(val.data(), val.size())); + + return j.value("prev_batch", ""); +} + Cache::Messages -Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) +Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? auto orderDb = getOrderToMessageDb(txn, room_id); @@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - if (index == std::numeric_limits::max()) { + if (index == std::numeric_limits::max()) { if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) { - index = *indexVal.data(); + index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; } } else { if (cursor.get(indexVal, event_id, MDB_SET)) { - index = *indexVal.data(); + index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; @@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i cursor.close(); // std::reverse(timeline.events.begin(), timeline.events.end()); - messages.next_index = *indexVal.data(); + messages.next_index = *indexVal.data(); messages.end_of_cache = !ret; return messages; @@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id) } TimelineRange range{}; - range.last = *indexVal.data(); + range.last = *indexVal.data(); if (!cursor.get(indexVal, val, MDB_FIRST)) { return {}; } - range.first = *indexVal.data(); + range.first = *indexVal.data(); return range; } -std::optional +std::optional Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); @@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) return {}; } - return *val.data(); + return *val.data(); } std::optional -Cache::getTimelineEventId(const std::string &room_id, int64_t index) +Cache::getTimelineEventId(const std::string &room_id, uint64_t index) { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); auto orderDb = getOrderToMessageDb(txn, room_id); @@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { + if (res.events.empty()) + return; + auto eventsDb = getEventsDb(txn, room_id); auto relationsDb = getRelationsDb(txn, room_id); @@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn, using namespace mtx::events::state; lmdb::val indexVal, val; - int64_t index = 0; - auto cursor = lmdb::cursor::open(txn, orderDb); + uint64_t index = std::numeric_limits::max() / 2; + auto cursor = lmdb::cursor::open(txn, orderDb); if (cursor.get(indexVal, val, MDB_LAST)) { index = *indexVal.data(); } - int64_t msgIndex = 0; - auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + uint64_t msgIndex = std::numeric_limits::max() / 2; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); if (msgCursor.get(indexVal, val, MDB_LAST)) { - msgIndex = *indexVal.data(); + msgIndex = *indexVal.data(); } bool first = true; @@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn, continue; lmdb::val ev{}; - bool success = - lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev); - if (!success) - continue; - - mtx::events::collections::TimelineEvent te; - - try { - mtx::events::collections::from_json( - json::parse(std::string_view(ev.data(), ev.size())), te); - } catch (std::exception &e) { - nhlog::db()->error("Failed to parse message from cache {}", - e.what()); - continue; + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump())); + + lmdb::val oldIndex{}; + if (lmdb::dbi_get( + txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) { + lmdb::dbi_put( + txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id)); + lmdb::dbi_put( + txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); } - - auto redactedEvent = std::visit( - [](const auto &ev) -> mtx::events::RoomEvent { - mtx::events::RoomEvent replacement = - {}; - replacement.event_id = ev.event_id; - replacement.room_id = ev.room_id; - replacement.sender = ev.sender; - replacement.origin_server_ts = ev.origin_server_ts; - replacement.type = ev.type; - return replacement; - }, - te.data); - - lmdb::dbi_put(txn, - eventsDb, - lmdb::val(redaction->redacts), - lmdb::val(json(redactedEvent).dump())); } else { std::string event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; @@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn, } } +uint64_t +Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res) +{ + auto txn = lmdb::txn::begin(env_); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); + + auto orderDb = getEventOrderDb(txn, room_id); + auto msg2orderDb = getMessageToOrderDb(txn, room_id); + auto order2msgDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal, val; + uint64_t index = std::numeric_limits::max() / 2; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_FIRST)) { + index = *indexVal.data(); + } + + uint64_t msgIndex = std::numeric_limits::max() / 2; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_FIRST)) { + msgIndex = *indexVal.data(); + } + + if (res.chunk.empty()) + return index; + + std::string event_id_val; + for (const auto &e : res.chunk) { + auto event = mtx::accessors::serialize_event(e); + event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); + + --index; + + json orderEntry = json::object(); + orderEntry["event_id"] = event_id_val; + + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + + lmdb::dbi_put( + txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + + // TODO(Nico): Allow blacklisting more event types in UI + if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + --msgIndex; + lmdb::dbi_put( + txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id); + + lmdb::dbi_put( + txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex))); + } + + if (event.contains("content") && event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) + lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id); + } + } + + json orderEntry = json::object(); + orderEntry["event_id"] = event_id_val; + orderEntry["prev_batch"] = res.end; + lmdb::cursor_put( + cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + + txn.commit(); + + return msgIndex; +} + mtx::responses::Notifications Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id) { @@ -2337,14 +2414,14 @@ Cache::deleteOldMessages() auto eventsDb = getEventsDb(txn, room_id); auto cursor = lmdb::cursor::open(txn, orderDb); - int64_t first, last; + uint64_t first, last; if (cursor.get(indexVal, val, MDB_LAST)) { - last = *indexVal.data(); + last = *indexVal.data(); } else { continue; } if (cursor.get(indexVal, val, MDB_FIRST)) { - first = *indexVal.data(); + first = *indexVal.data(); } else { continue; } diff --git a/src/Cache_p.h b/src/Cache_p.h index 6b4b260e..1d6d62dd 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -179,8 +179,8 @@ public: }; Messages getTimelineMessages(lmdb::txn &txn, const std::string &room_id, - int64_t index = std::numeric_limits::max(), - bool forward = false); + uint64_t index = std::numeric_limits::max(), + bool forward = false); std::optional getEvent( const std::string &room_id, @@ -190,12 +190,15 @@ public: const mtx::events::collections::TimelineEvent &event); struct TimelineRange { - int64_t first, last; + uint64_t first, last; }; std::optional getTimelineRange(const std::string &room_id); - std::optional getTimelineIndex(const std::string &room_id, - std::string_view event_id); - std::optional getTimelineEventId(const std::string &room_id, int64_t index); + std::optional getTimelineIndex(const std::string &room_id, + std::string_view event_id); + std::optional getTimelineEventId(const std::string &room_id, uint64_t index); + + std::string previousBatchToken(const std::string &room_id); + uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); //! Remove old unused data. void deleteOldMessages(); diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 3ef28c86..666912ee 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -795,43 +795,39 @@ ChatPage::loadStateFromCache() nhlog::db()->info("restoring state from cache"); - QtConcurrent::run([this]() { - try { - cache::restoreSessions(); - olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY); + try { + cache::restoreSessions(); + olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY); - cache::populateMembers(); + cache::populateMembers(); - emit initializeEmptyViews(cache::roomMessages()); - emit initializeRoomList(cache::roomInfo()); - emit initializeMentions(cache::getTimelineMentions()); - emit syncTags(cache::roomInfo().toStdMap()); + emit initializeEmptyViews(cache::roomMessages()); + emit initializeRoomList(cache::roomInfo()); + emit initializeMentions(cache::getTimelineMentions()); + emit syncTags(cache::roomInfo().toStdMap()); - cache::calculateRoomReadStatus(); + cache::calculateRoomReadStatus(); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to restore olm account: {}", e.what()); - emit dropToLoginPageCb( - tr("Failed to restore OLM account. Please login again.")); - return; - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to restore cache: {}", e.what()); - emit dropToLoginPageCb( - tr("Failed to restore save data. Please login again.")); - return; - } catch (const json::exception &e) { - nhlog::db()->critical("failed to parse cache data: {}", e.what()); - return; - } + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to restore olm account: {}", e.what()); + emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again.")); + return; + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to restore cache: {}", e.what()); + emit dropToLoginPageCb(tr("Failed to restore save data. Please login again.")); + return; + } catch (const json::exception &e) { + nhlog::db()->critical("failed to parse cache data: {}", e.what()); + return; + } - nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); - nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519); + nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); + nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519); - getProfileInfo(); + getProfileInfo(); - // Start receiving events. - emit trySyncCb(); - }); + // Start receiving events. + emit trySyncCb(); } void diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 719743fb..7f21e1ed 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *) cache::client()->storeEvent(room_id_, id, {timeline}); if (!relatedTo.empty()) { - auto idx = idToIndex(id); + auto idx = idToIndex(relatedTo); if (idx) emit dataChanged(*idx, *idx); } }, Qt::QueuedConnection); + + connect( + this, + &EventStore::oldMessagesRetrieved, + this, + [this](const mtx::responses::Messages &res) { + // + uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res); + if (newFirst == first) + fetchMore(); + else { + emit beginInsertRows(toExternalIdx(newFirst), + toExternalIdx(this->first - 1)); + this->first = newFirst; + emit endInsertRows(); + emit fetchedMore(); + } + }, + Qt::QueuedConnection); } void @@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events) nhlog::db()->warn("{} called from a different thread!", __func__); auto range = cache::client()->getTimelineRange(room_id_); + if (!range) + return; + + if (events.limited) { + emit beginResetModel(); + this->last = range->last; + this->first = range->first; + emit endResetModel(); - if (range && range->last > this->last) { + } else if (range->last > this->last) { emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last)); this->last = range->last; emit endInsertRows(); @@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt return event_ptr; } + +void +EventStore::fetchMore() +{ + mtx::http::MessagesOpts opts; + opts.room_id = room_id_; + opts.from = cache::client()->previousBatchToken(room_id_); + + nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from); + + http::client()->messages( + opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("failed to call /messages ({}): {} - {} - {}", + opts.room_id, + mtx::errors::to_string(err->matrix_error.errcode), + err->matrix_error.error, + err->parse_error); + return; + } + + emit oldMessagesRetrieved(std::move(res)); + }); +} diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 83c8f7a4..f2997245 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -8,6 +8,7 @@ #include #include +#include #include class EventStore : public QObject @@ -20,7 +21,7 @@ public: struct Index { std::string room; - int64_t idx; + uint64_t idx; friend uint qHash(const Index &i, uint seed = 0) noexcept { @@ -66,12 +67,12 @@ public: int size() const { - return last != std::numeric_limits::max() + return last != std::numeric_limits::max() ? static_cast(last - first) + 1 : 0; } - int toExternalIdx(int64_t idx) const { return static_cast(idx - first); } - int64_t toInternalIdx(int idx) const { return first + idx; } + int toExternalIdx(uint64_t idx) const { return static_cast(idx - first); } + uint64_t toInternalIdx(int idx) const { return first + idx; } std::optional idToIndex(std::string_view id) const; std::optional indexToId(int idx) const; @@ -79,11 +80,15 @@ public: signals: void beginInsertRows(int from, int to); void endInsertRows(); + void beginResetModel(); + void endResetModel(); void dataChanged(int from, int to); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); void eventFetched(std::string id, std::string relatedTo, mtx::events::collections::TimelineEvents timeline); + void oldMessagesRetrieved(const mtx::responses::Messages &); + void fetchedMore(); private: mtx::events::collections::TimelineEvents *decryptEvent( @@ -92,8 +97,8 @@ private: std::string room_id_; - int64_t first = std::numeric_limits::max(), - last = std::numeric_limits::max(); + uint64_t first = std::numeric_limits::max(), + last = std::numeric_limits::max(); static QCache decryptedEvents_; static QCache events_; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 6df92d7a..60264e86 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj &EventStore::dataChanged, this, [this](int from, int to) { - emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0)); + nhlog::ui()->debug( + "data changed {} to {}", events.size() - to - 1, events.size() - from - 1); + emit dataChanged(index(events.size() - to - 1, 0), + index(events.size() - from - 1, 0)); }, Qt::QueuedConnection); connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) { - nhlog::ui()->info("begin insert from {} to {}", - events.size() - to + (to - from), - events.size() - from + (to - from)); - beginInsertRows(QModelIndex(), - events.size() - to + (to - from), - events.size() - from + (to - from)); + int first = events.size() - to; + int last = events.size() - from; + if (from >= events.size()) { + int batch_size = to - from; + first += batch_size; + last += batch_size; + } else { + first -= 1; + last -= 1; + } + nhlog::ui()->debug("begin insert from {} to {}", first, last); + beginInsertRows(QModelIndex(), first, last); }); connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); }); + connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); }); + connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); }); connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage); + connect( + &events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); }); } QHash @@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const { if (!events.size()) return true; - if (!std::holds_alternative>( - *events.event(0))) + if (auto first = events.event(0); + first && + !std::holds_alternative>(*first)) return true; else @@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &) } setPaginationInProgress(true); - mtx::http::MessagesOpts opts; - opts.room_id = room_id_.toStdString(); - opts.from = prev_batch_token_.toStdString(); - nhlog::ui()->debug("Paginating room {}", opts.room_id); - - http::client()->messages( - opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("failed to call /messages ({}): {} - {} - {}", - opts.room_id, - mtx::errors::to_string(err->matrix_error.errcode), - err->matrix_error.error, - err->parse_error); - setPaginationInProgress(false); - return; - } - - emit oldMessagesRetrieved(std::move(res)); - setPaginationInProgress(false); - }); + events.fetchMore(); } void -- cgit 1.5.1 From 56ea89aa1133f01e356b1e7dce4322b883600e53 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 17:43:49 +0200 Subject: Reenable sending messages --- src/Cache.cpp | 140 +++++++++++++++- src/Cache_p.h | 18 ++ src/ChatPage.cpp | 20 +-- src/dialogs/RoomSettings.cpp | 10 +- src/timeline/EventStore.cpp | 109 ++++++++++++ src/timeline/EventStore.h | 10 ++ src/timeline/TimelineModel.cpp | 364 ++++++----------------------------------- src/timeline/TimelineModel.h | 20 +-- 8 files changed, 342 insertions(+), 349 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 233ef2b4..8fa94d1e 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2086,6 +2086,77 @@ Cache::isRoomMember(const std::string &user_id, const std::string &room_id) return res; } +void +Cache::savePendingMessage(const std::string &room_id, + const mtx::events::collections::TimelineEvent &message) +{ + auto txn = lmdb::txn::begin(env_); + + mtx::responses::Timeline timeline; + timeline.events.push_back(message.data); + saveTimelineMessages(txn, room_id, timeline); + + auto pending = getPendingMessagesDb(txn, room_id); + + int64_t now = QDateTime::currentMSecsSinceEpoch(); + lmdb::dbi_put(txn, + pending, + lmdb::val(&now, sizeof(now)), + lmdb::val(mtx::accessors::event_id(message.data))); + + txn.commit(); +} + +std::optional +Cache::firstPendingMessage(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_); + auto pending = getPendingMessagesDb(txn, room_id); + + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + auto eventsDb = getEventsDb(txn, room_id); + lmdb::val event; + if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) { + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } + + try { + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + + txn.commit(); + return te; + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } + } + + txn.commit(); + + return std::nullopt; +} + +void +Cache::removePendingStatus(const std::string &room_id, const std::string &txn_id) +{ + auto txn = lmdb::txn::begin(env_); + auto pending = getPendingMessagesDb(txn, room_id); + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id) + lmdb::cursor_del(pendingCursor); + } + + txn.commit(); +} + void Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, @@ -2098,12 +2169,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto relationsDb = getRelationsDb(txn, room_id); auto orderDb = getEventOrderDb(txn, room_id); + auto evToOrderDb = getEventToOrderDb(txn, room_id); auto msg2orderDb = getMessageToOrderDb(txn, room_id); auto order2msgDb = getOrderToMessageDb(txn, room_id); + auto pending = getPendingMessagesDb(txn, room_id); + if (res.limited) { lmdb::dbi_drop(txn, orderDb, false); + lmdb::dbi_drop(txn, evToOrderDb, false); lmdb::dbi_drop(txn, msg2orderDb, false); lmdb::dbi_drop(txn, order2msgDb, false); + lmdb::dbi_drop(txn, pending, true); } using namespace mtx::events; @@ -2124,9 +2200,55 @@ Cache::saveTimelineMessages(lmdb::txn &txn, bool first = true; for (const auto &e : res.events) { - auto event = mtx::accessors::serialize_event(e); - if (auto redaction = - std::get_if>(&e)) { + auto event = mtx::accessors::serialize_event(e); + auto txn_id = mtx::accessors::transaction_id(e); + + lmdb::val txn_order; + if (!txn_id.empty() && + lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) { + std::string event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); + lmdb::dbi_del(txn, eventsDb, lmdb::val(txn_id)); + + lmdb::val msg_txn_order; + if (lmdb::dbi_get(txn, msg2orderDb, lmdb::val(txn_id), msg_txn_order)) { + lmdb::dbi_put(txn, order2msgDb, msg_txn_order, event_id); + lmdb::dbi_put(txn, msg2orderDb, event_id, msg_txn_order); + lmdb::dbi_del(txn, msg2orderDb, lmdb::val(txn_id)); + } + + lmdb::dbi_put(txn, orderDb, txn_order, event_id); + lmdb::dbi_put(txn, evToOrderDb, event_id, txn_order); + lmdb::dbi_del(txn, evToOrderDb, lmdb::val(txn_id)); + + if (event.contains("content") && + event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) { + lmdb::dbi_del(txn, + relationsDb, + lmdb::val(relates_to), + lmdb::val(txn_id)); + lmdb::dbi_put( + txn, relationsDb, lmdb::val(relates_to), event_id); + } + } + + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + if (std::string_view(pendingTxn.data(), pendingTxn.size()) == + txn_id) + lmdb::cursor_del(pendingCursor); + } + } else if (auto redaction = + std::get_if>( + &e)) { if (redaction->redacts.empty()) continue; @@ -2145,15 +2267,20 @@ Cache::saveTimelineMessages(lmdb::txn &txn, txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); } } else { - std::string event_id_val = event["event_id"].get(); - lmdb::val event_id = event_id_val; + std::string event_id_val = event.value("event_id", ""); + if (event_id_val.empty()) { + nhlog::db()->error("Event without id!"); + continue; + } + + lmdb::val event_id = event_id_val; lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; json orderEntry = json::object(); orderEntry["event_id"] = event_id_val; - if (first) + if (first && !res.prev_batch.empty()) orderEntry["prev_batch"] = res.prev_batch; first = false; @@ -2163,6 +2290,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()), MDB_APPEND); + lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index))); // TODO(Nico): Allow blacklisting more event types in UI if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { diff --git a/src/Cache_p.h b/src/Cache_p.h index 1d6d62dd..88308e45 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -199,6 +199,11 @@ public: std::string previousBatchToken(const std::string &room_id); uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); + void savePendingMessage(const std::string &room_id, + const mtx::events::collections::TimelineEvent &message); + std::optional firstPendingMessage( + const std::string &room_id); + void removePendingStatus(const std::string &room_id, const std::string &txn_id); //! Remove old unused data. void deleteOldMessages(); @@ -439,6 +444,13 @@ private: txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + // inverse of EventOrderDb + lmdb::dbi getEventToOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/event2order").c_str(), MDB_CREATE); + } + lmdb::dbi getMessageToOrderDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( @@ -451,6 +463,12 @@ private: txn, std::string(room_id + "/order2msg").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + lmdb::dbi getPendingMessagesDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/pending").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + lmdb::dbi getRelationsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 666912ee..813b0c2a 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -313,17 +313,15 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) .toStdString(); member.membership = mtx::events::state::Membership::Join; - http::client() - ->send_state_event( - currentRoom().toStdString(), - http::client()->user_id().to_string(), - member, - [](mtx::responses::EventId, mtx::http::RequestErr err) { - if (err) - nhlog::net()->error("Failed to set room displayname: {}", - err->matrix_error.error); - }); + http::client()->send_state_event( + currentRoom().toStdString(), + http::client()->user_id().to_string(), + member, + [](mtx::responses::EventId, mtx::http::RequestErr err) { + if (err) + nhlog::net()->error("Failed to set room displayname: {}", + err->matrix_error.error); + }); }); connect( diff --git a/src/dialogs/RoomSettings.cpp b/src/dialogs/RoomSettings.cpp index 26aece32..822b7218 100644 --- a/src/dialogs/RoomSettings.cpp +++ b/src/dialogs/RoomSettings.cpp @@ -151,7 +151,7 @@ EditModal::applyClicked() state::Name body; body.name = newName.toStdString(); - http::client()->send_state_event( + http::client()->send_state_event( roomId_.toStdString(), body, [proxy, newName](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -169,7 +169,7 @@ EditModal::applyClicked() state::Topic body; body.topic = newTopic.toStdString(); - http::client()->send_state_event( + http::client()->send_state_event( roomId_.toStdString(), body, [proxy](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -694,7 +694,7 @@ RoomSettings::updateAccessRules(const std::string &room_id, startLoadingSpinner(); resetErrorLabel(); - http::client()->send_state_event( + http::client()->send_state_event( room_id, join_rule, [this, room_id, guest_access](const mtx::responses::EventId &, @@ -708,7 +708,7 @@ RoomSettings::updateAccessRules(const std::string &room_id, return; } - http::client()->send_state_event( + http::client()->send_state_event( room_id, guest_access, [this](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -843,7 +843,7 @@ RoomSettings::updateAvatar() avatar_event.image_info.size = size; avatar_event.url = res.content_uri; - http::client()->send_state_event( + http::client()->send_state_event( room_id, avatar_event, [content = std::move(content), proxy = std::move(proxy)]( diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 7f21e1ed..b7cf4f96 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -1,6 +1,7 @@ #include "EventStore.h" #include +#include #include "Cache_p.h" #include "EventAccessors.h" @@ -59,6 +60,104 @@ EventStore::EventStore(std::string room_id, QObject *) } }, Qt::QueuedConnection); + + connect(this, &EventStore::processPending, this, [this]() { + if (!current_txn.empty()) { + nhlog::ui()->debug("Already processing {}", current_txn); + return; + } + + auto event = cache::client()->firstPendingMessage(room_id_); + + if (!event) { + nhlog::ui()->debug("No event to send"); + return; + } + + std::visit( + [this](auto e) { + auto txn_id = e.event_id; + this->current_txn = txn_id; + + if (txn_id.empty() || txn_id[0] != 'm') { + nhlog::ui()->debug("Invalid txn id '{}'", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + return; + } + + if constexpr (mtx::events::message_content_to_type != + mtx::events::EventType::Unsupported) + http::client()->send_room_message( + room_id_, + txn_id, + e.content, + [this, txn_id](const mtx::responses::EventId &, + mtx::http::RequestErr err) { + if (err) { + const int status_code = + static_cast(err->status_code); + nhlog::net()->warn( + "[{}] failed to send message: {} {}", + txn_id, + err->matrix_error.error, + status_code); + emit messageFailed(txn_id); + return; + } + emit messageSent(txn_id); + }); + }, + event->data); + }); + + connect( + this, + &EventStore::messageFailed, + this, + [this](std::string txn_id) { + if (current_txn == txn_id) { + current_txn_error_count++; + if (current_txn_error_count > 10) { + nhlog::ui()->debug("failing txn id '{}'", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + current_txn_error_count = 0; + } + } + QTimer::singleShot(1000, this, [this]() { + nhlog::ui()->debug("timeout"); + this->current_txn = ""; + emit processPending(); + }); + }, + Qt::QueuedConnection); + + connect( + this, + &EventStore::messageSent, + this, + [this](std::string txn_id) { + nhlog::ui()->debug("sent {}", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + this->current_txn = ""; + this->current_txn_error_count = 0; + emit processPending(); + }, + Qt::QueuedConnection); +} + +void +EventStore::addPending(mtx::events::collections::TimelineEvents event) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + cache::client()->savePendingMessage(this->room_id_, {event}); + mtx::responses::Timeline events; + events.limited = false; + events.events.emplace_back(event); + handleSync(events); + + emit processPending(); } void @@ -102,6 +201,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (idx) emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); } + + if (auto txn_id = mtx::accessors::transaction_id(event); !txn_id.empty()) { + auto idx = cache::client()->getTimelineIndex( + room_id_, mtx::accessors::event_id(event)); + if (idx) { + Index index{room_id_, *idx}; + events_.remove(index); + emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } + } } } diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index f2997245..b4d5bb23 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -90,6 +90,13 @@ signals: void oldMessagesRetrieved(const mtx::responses::Messages &); void fetchedMore(); + void processPending(); + void messageSent(std::string txn_id); + void messageFailed(std::string txn_id); + +public slots: + void addPending(mtx::events::collections::TimelineEvents event); + private: mtx::events::collections::TimelineEvents *decryptEvent( const IdIndex &idx, @@ -103,4 +110,7 @@ private: static QCache decryptedEvents_; static QCache events_; static QCache events_by_id_; + + std::string current_txn; + int current_txn_error_count = 0; }; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 60264e86..aa6cea4f 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -145,67 +145,6 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj , room_id_(room_id) , manager_(manager) { - connect(this, - &TimelineModel::oldMessagesRetrieved, - this, - &TimelineModel::addBackwardsEvents, - Qt::QueuedConnection); - connect( - this, - &TimelineModel::messageFailed, - this, - [this](QString txn_id) { - nhlog::ui()->error("Failed to send {}, retrying", txn_id.toStdString()); - - QTimer::singleShot(5000, this, [this]() { emit nextPendingMessage(); }); - }, - Qt::QueuedConnection); - connect( - this, - &TimelineModel::messageSent, - this, - [this](QString txn_id, QString event_id) { - pending.removeOne(txn_id); - (void)event_id; - // auto ev = events.value(txn_id); - - // if (auto reaction = - // std::get_if>(&ev)) { - // QString reactedTo = - // QString::fromStdString(reaction->content.relates_to.event_id); - // auto &rModel = reactions[reactedTo]; - // rModel.removeReaction(*reaction); - // auto rCopy = *reaction; - // rCopy.event_id = event_id.toStdString(); - // rModel.addReaction(room_id_.toStdString(), rCopy); - //} - - // int idx = idToIndex(txn_id); - // if (idx < 0) { - // // transaction already received via sync - // return; - //} - // eventOrder[idx] = event_id; - // ev = std::visit( - // [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { - // auto eventCopy = e; - // eventCopy.event_id = event_id.toStdString(); - // return eventCopy; - // }, - // ev); - - // events.remove(txn_id); - // events.insert(event_id, ev); - - //// mark our messages as read - // readEvent(event_id.toStdString()); - - // emit dataChanged(index(idx, 0), index(idx, 0)); - - if (pending.size() > 0) - emit nextPendingMessage(); - }, - Qt::QueuedConnection); connect( this, &TimelineModel::redactionFailed, @@ -213,16 +152,12 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj [](const QString &msg) { emit ChatPage::instance()->showNotification(msg); }, Qt::QueuedConnection); - connect(this, - &TimelineModel::nextPendingMessage, - this, - &TimelineModel::processOnePendingMessage, - Qt::QueuedConnection); connect(this, &TimelineModel::newMessageToSend, this, &TimelineModel::addPendingMessage, Qt::QueuedConnection); + connect(this, &TimelineModel::addPendingMessageToStore, &events, &EventStore::addPending); connect( &events, @@ -296,7 +231,7 @@ int TimelineModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent); - return this->events.size() + static_cast(pending.size()); + return this->events.size(); } QVariantMap @@ -410,7 +345,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r // only show read receipts for messages not from us if (acc::sender(event) != http::client()->user_id().to_string()) return qml_mtx_events::Empty; - else if (pending.contains(id)) + else if (!id.isEmpty() && id[0] == "m") return qml_mtx_events::Sent; else if (read.contains(id) || containsOthers(cache::readReceipts(id, room_id_))) return qml_mtx_events::Read; @@ -428,11 +363,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); case Reactions: { - auto id = QString::fromStdString(event_id(event)); - if (reactions.count(id)) - return QVariant::fromValue((QObject *)&reactions.at(id)); - else - return {}; + return {}; } case RoomId: return QVariant(room_id_); @@ -561,16 +492,9 @@ TimelineModel::fetchMore(const QModelIndex &) void TimelineModel::addEvents(const mtx::responses::Timeline &timeline) { - if (isInitialSync) { - prev_batch_token_ = QString::fromStdString(timeline.prev_batch); - isInitialSync = false; - } - if (timeline.events.empty()) return; - internalAddEvents(timeline.events); - events.handleSync(timeline); if (!timeline.events.empty()) @@ -644,56 +568,6 @@ TimelineModel::updateLastMessage() } } -void -TimelineModel::internalAddEvents( - const std::vector &timeline) -{ - for (auto e : timeline) { - QString id = QString::fromStdString(mtx::accessors::event_id(e)); - - if (auto redaction = - std::get_if>(&e)) { - QString redacts = QString::fromStdString(redaction->redacts); - - auto event = events.event(redaction->redacts, redaction->event_id); - if (!event) - continue; - - if (auto reaction = - std::get_if>( - event)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - reactions[reactedTo].removeReaction(*reaction); - int idx = idToIndex(reactedTo); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); - } - - continue; // don't insert redaction into timeline - } - - if (auto reaction = - std::get_if>(&e)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - - // // remove local echo - // if (!txid.isEmpty()) { - // auto rCopy = *reaction; - // rCopy.event_id = txid.toStdString(); - // reactions[reactedTo].removeReaction(rCopy); - // } - - reactions[reactedTo].addReaction(room_id_.toStdString(), *reaction); - int idx = idToIndex(reactedTo); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; // don't insert reaction into timeline - } - } -} - void TimelineModel::setCurrentIndex(int index) { @@ -701,7 +575,7 @@ TimelineModel::setCurrentIndex(int index) currentId = indexToId(index); emit currentIndexChanged(index); - if ((oldIndex > index || oldIndex == -1) && !pending.contains(currentId) && + if ((oldIndex > index || oldIndex == -1) && !currentId.startsWith("m") && ChatPage::instance()->isActiveWindow()) { readEvent(currentId.toStdString()); } @@ -719,28 +593,6 @@ TimelineModel::readEvent(const std::string &id) }); } -void -TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs) -{ - (void)msgs; - // std::vector ids = internalAddEvents(msgs.chunk); - - // if (!ids.empty()) { - // beginInsertRows(QModelIndex(), - // static_cast(this->eventOrder.size()), - // static_cast(this->eventOrder.size() + ids.size() - 1)); - // this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); - // endInsertRows(); - //} - - // prev_batch_token_ = QString::fromStdString(msgs.end); - - // if (ids.empty() && !msgs.chunk.empty()) { - // // no visible events fetched, prevent loading from stopping - // fetchMore(QModelIndex()); - //} -} - QString TimelineModel::displayName(QString id) const { @@ -902,7 +754,7 @@ TimelineModel::markEventsAsRead(const std::vector &event_ids) } void -TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json content) +TimelineModel::sendEncryptedMessage(const std::string txn_id, nlohmann::json content) { const auto room_id = room_id_.toStdString(); @@ -914,28 +766,15 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co try { // Check if we have already an outbound megolm session then we can use. if (cache::outboundMegolmSessionExists(room_id)) { - auto data = + mtx::events::EncryptedEvent event; + event.content = olm::encrypt_group_message(room_id, http::client()->device_id(), doc); + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; - http::client()->send_room_message( - room_id, - txn_id, - data, - [this, txn_id](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id, - err->matrix_error.error, - status_code); - emit messageFailed(QString::fromStdString(txn_id)); - } - emit messageSent( - QString::fromStdString(txn_id), - QString::fromStdString(res.event_id.to_string())); - }); + emit this->addPendingMessageToStore(event); return; } @@ -964,40 +803,24 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co const auto members = cache::roomMembers(room_id); nhlog::ui()->info("retrieved {} members for {}", members.size(), room_id); - auto keeper = - std::make_shared([megolm_payload, room_id, doc, txn_id, this]() { - try { - auto data = olm::encrypt_group_message( - room_id, http::client()->device_id(), doc); - - http::client() - ->send_room_message( - room_id, - txn_id, - data, - [this, txn_id](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn( - "[{}] failed to send message: {} {}", - txn_id, - err->matrix_error.error, - status_code); - emit messageFailed( - QString::fromStdString(txn_id)); - } - emit messageSent( - QString::fromStdString(txn_id), - QString::fromStdString(res.event_id.to_string())); - }); - } catch (const lmdb::error &e) { - nhlog::db()->critical( - "failed to save megolm outbound session: {}", e.what()); - emit messageFailed(QString::fromStdString(txn_id)); - } - }); + auto keeper = std::make_shared([room_id, doc, txn_id, this]() { + try { + mtx::events::EncryptedEvent event; + event.content = olm::encrypt_group_message( + room_id, http::client()->device_id(), doc); + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; + + emit this->addPendingMessageToStore(event); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to save megolm outbound session: {}", + e.what()); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); + } + }); mtx::requests::QueryKeys req; for (const auto &member : members) @@ -1011,8 +834,8 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co nhlog::net()->warn("failed to query device keys: {} {}", err->matrix_error.error, static_cast(err->status_code)); - // TODO: Mark the event as failed. Communicate with the UI. - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); return; } @@ -1112,11 +935,13 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co } catch (const lmdb::error &e) { nhlog::db()->critical( "failed to open outbound megolm session ({}): {}", room_id, e.what()); - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); } catch (const mtx::crypto::olm_exception &e) { nhlog::crypto()->critical( "failed to open outbound megolm session ({}): {}", room_id, e.what()); - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); } } @@ -1208,9 +1033,8 @@ TimelineModel::handleClaimedKeys(std::shared_ptr keeper, struct SendMessageVisitor { - SendMessageVisitor(const QString &txn_id, TimelineModel *model) - : txn_id_qstr_(txn_id) - , model_(model) + explicit SendMessageVisitor(TimelineModel *model) + : model_(model) {} // Do-nothing operator for all unhandled events @@ -1228,29 +1052,9 @@ struct SendMessageVisitor if (encInfo) emit model_->newEncryptedImage(encInfo.value()); - model_->sendEncryptedMessage(txn_id_qstr_.toStdString(), - nlohmann::json(msg.content)); + model_->sendEncryptedMessage(msg.event_id, nlohmann::json(msg.content)); } else { - QString txn_id_qstr = txn_id_qstr_; - TimelineModel *model = model_; - http::client()->send_room_message( - model->room_id_.toStdString(), - txn_id_qstr.toStdString(), - msg.content, - [txn_id_qstr, model](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id_qstr.toStdString(), - err->matrix_error.error, - status_code); - emit model->messageFailed(txn_id_qstr); - } - emit model->messageSent( - txn_id_qstr, QString::fromStdString(res.event_id.to_string())); - }); + emit model_->addPendingMessageToStore(msg); } } @@ -1260,71 +1064,26 @@ struct SendMessageVisitor // cannot handle it correctly. See the MSC for more details: // https://github.com/matrix-org/matrix-doc/blob/matthew/msc1849/proposals/1849-aggregations.md#end-to-end-encryption void operator()(const mtx::events::RoomEvent &msg) - { - QString txn_id_qstr = txn_id_qstr_; - TimelineModel *model = model_; - http::client() - ->send_room_message( - model->room_id_.toStdString(), - txn_id_qstr.toStdString(), - msg.content, - [txn_id_qstr, model](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id_qstr.toStdString(), - err->matrix_error.error, - status_code); - emit model->messageFailed(txn_id_qstr); - } - emit model->messageSent( - txn_id_qstr, QString::fromStdString(res.event_id.to_string())); - }); + emit model_->addPendingMessageToStore(msg); } - QString txn_id_qstr_; TimelineModel *model_; }; -void -TimelineModel::processOnePendingMessage() -{ - // if (pending.isEmpty()) - // return; - - // QString txn_id_qstr = pending.first(); - - // auto event = events.value(txn_id_qstr); - // std::visit(SendMessageVisitor{txn_id_qstr, this}, event); -} - void TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event) { - (void)event; - // std::visit( - // [](auto &msg) { - // msg.type = mtx::events::EventType::RoomMessage; - // msg.event_id = http::client()->generate_txn_id(); - // msg.sender = http::client()->user_id().to_string(); - // msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); - // }, - // event); - - // internalAddEvents({event}); - - // QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); - // pending.push_back(txn_id_qstr); - // if (!std::get_if>(&event)) { - // beginInsertRows(QModelIndex(), 0, 0); - // this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); - // endInsertRows(); - //} - // updateLastMessage(); - - // emit nextPendingMessage(); + std::visit( + [](auto &msg) { + msg.type = mtx::events::EventType::RoomMessage; + msg.event_id = "m" + http::client()->generate_txn_id(); + msg.sender = http::client()->user_id().to_string(); + msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); + }, + event); + + std::visit(SendMessageVisitor{this}, event); } bool @@ -1647,24 +1406,7 @@ TimelineModel::formatMemberEvent(QString id) if (!event->unsigned_data.replaces_state.empty()) { auto tempPrevEvent = events.event(event->unsigned_data.replaces_state, event->event_id); - if (!tempPrevEvent) { - http::client()->get_event( - this->room_id_.toStdString(), - event->unsigned_data.replaces_state, - [this, id, prevEventId = event->unsigned_data.replaces_state]( - const mtx::events::collections::TimelineEvents &timeline, - mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error( - "Failed to retrieve event with id {}, which was " - "requested to show the membership for event {}", - prevEventId, - id.toStdString()); - return; - } - emit eventFetched(id, timeline); - }); - } else { + if (tempPrevEvent) { prevEvent = std::get_if>( tempPrevEvent); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index f322b482..9f9717df 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -236,31 +236,23 @@ public slots: void setDecryptDescription(bool decrypt) { decryptDescription = decrypt; } private slots: - // Add old events at the top of the timeline. - void addBackwardsEvents(const mtx::responses::Messages &msgs); - void processOnePendingMessage(); void addPendingMessage(mtx::events::collections::TimelineEvents event); signals: - void oldMessagesRetrieved(const mtx::responses::Messages &res); - void messageFailed(QString txn_id); - void messageSent(QString txn_id, QString event_id); void currentIndexChanged(int index); void redactionFailed(QString id); void eventRedacted(QString id); - void nextPendingMessage(); - void newMessageToSend(mtx::events::collections::TimelineEvents event); void mediaCached(QString mxcUrl, QString cacheUrl); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); - void eventFetched(QString requestingEvent, mtx::events::collections::TimelineEvents event); void typingUsersChanged(std::vector users); void replyChanged(QString reply); void paginationInProgressChanged(const bool); + void newMessageToSend(mtx::events::collections::TimelineEvents event); + void addPendingMessageToStore(mtx::events::collections::TimelineEvents event); + private: - void internalAddEvents( - const std::vector &timeline); - void sendEncryptedMessage(const std::string &txn_id, nlohmann::json content); + void sendEncryptedMessage(const std::string txn_id, nlohmann::json content); void handleClaimedKeys(std::shared_ptr keeper, const std::map &room_key, const std::map &pks, @@ -272,15 +264,11 @@ private: void setPaginationInProgress(const bool paginationInProgress); QSet read; - QList pending; - std::map reactions; mutable EventStore events; QString room_id_; - QString prev_batch_token_; - bool isInitialSync = true; bool decryptDescription = true; bool m_paginationInProgress = false; -- cgit 1.5.1 From d467568a65235093c5ea4591486818ab5de42119 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 22:59:03 +0200 Subject: Close cursor we don't need and where we overwrite the contents --- src/Cache.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 8fa94d1e..9464a546 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2334,15 +2334,19 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message lmdb::val indexVal, val; uint64_t index = std::numeric_limits::max() / 2; - auto cursor = lmdb::cursor::open(txn, orderDb); - if (cursor.get(indexVal, val, MDB_FIRST)) { - index = *indexVal.data(); + { + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_FIRST)) { + index = *indexVal.data(); + } } uint64_t msgIndex = std::numeric_limits::max() / 2; - auto msgCursor = lmdb::cursor::open(txn, order2msgDb); - if (msgCursor.get(indexVal, val, MDB_FIRST)) { - msgIndex = *indexVal.data(); + { + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_FIRST)) { + msgIndex = *indexVal.data(); + } } if (res.chunk.empty()) @@ -2389,8 +2393,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message json orderEntry = json::object(); orderEntry["event_id"] = event_id_val; orderEntry["prev_batch"] = res.end; - lmdb::cursor_put( - cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + lmdb::dbi_put(txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); nhlog::db()->debug("saving '{}'", orderEntry.dump()); txn.commit(); -- cgit 1.5.1 From 6f2bc908badc207754ff55d543d41d9e2b847c97 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 19 Jul 2020 12:22:54 +0200 Subject: Fix reaction display --- CMakeLists.txt | 4 +- resources/qml/Reactions.qml | 18 ++++---- src/Cache.cpp | 35 +++++++++++++++ src/Cache_p.h | 3 ++ src/timeline/EventStore.cpp | 70 +++++++++++++++++++++++++++++ src/timeline/EventStore.h | 5 +++ src/timeline/Reaction.cpp | 1 + src/timeline/Reaction.h | 24 ++++++++++ src/timeline/ReactionsModel.cpp | 98 ----------------------------------------- src/timeline/ReactionsModel.h | 41 ----------------- src/timeline/TimelineModel.cpp | 3 +- src/timeline/TimelineModel.h | 1 - 12 files changed, 151 insertions(+), 152 deletions(-) create mode 100644 src/timeline/Reaction.cpp create mode 100644 src/timeline/Reaction.h delete mode 100644 src/timeline/ReactionsModel.cpp delete mode 100644 src/timeline/ReactionsModel.h (limited to 'src/Cache.cpp') diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d1441c1..658232e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,7 +251,7 @@ set(SRC_FILES # Timeline src/timeline/EventStore.cpp - src/timeline/ReactionsModel.cpp + src/timeline/Reaction.cpp src/timeline/TimelineViewManager.cpp src/timeline/TimelineModel.cpp src/timeline/DelegateChooser.cpp @@ -455,7 +455,7 @@ qt5_wrap_cpp(MOC_HEADERS # Timeline src/timeline/EventStore.h - src/timeline/ReactionsModel.h + src/timeline/Reaction.h src/timeline/TimelineViewManager.h src/timeline/TimelineModel.h src/timeline/DelegateChooser.h diff --git a/resources/qml/Reactions.qml b/resources/qml/Reactions.qml index c06dc826..5b3bbc20 100644 --- a/resources/qml/Reactions.qml +++ b/resources/qml/Reactions.qml @@ -30,11 +30,11 @@ Flow { implicitHeight: contentItem.childrenRect.height ToolTip.visible: hovered - ToolTip.text: model.users + ToolTip.text: modelData.users onClicked: { - console.debug("Picked " + model.key + "in response to " + reactionFlow.eventId + " in room " + reactionFlow.roomId + ". selfReactedEvent: " + model.selfReactedEvent) - timelineManager.reactToMessage(reactionFlow.roomId, reactionFlow.eventId, model.key, model.selfReactedEvent) + console.debug("Picked " + modelData.key + "in response to " + reactionFlow.eventId + " in room " + reactionFlow.roomId + ". selfReactedEvent: " + modelData.selfReactedEvent) + timelineManager.reactToMessage(reactionFlow.roomId, reactionFlow.eventId, modelData.key, modelData.selfReactedEvent) } @@ -49,13 +49,13 @@ Flow { font.family: settings.emojiFont elide: Text.ElideRight elideWidth: 150 - text: model.key + text: modelData.key } Text { anchors.baseline: reactionCounter.baseline id: reactionText - text: textMetrics.elidedText + (textMetrics.elidedText == model.key ? "" : "…") + text: textMetrics.elidedText + (textMetrics.elidedText == modelData.key ? "" : "…") font.family: settings.emojiFont color: reaction.hovered ? colors.highlight : colors.text maximumLineCount: 1 @@ -65,13 +65,13 @@ Flow { id: divider height: Math.floor(reactionCounter.implicitHeight * 1.4) width: 1 - color: (reaction.hovered || model.selfReactedEvent !== '') ? colors.highlight : colors.text + color: (reaction.hovered || modelData.selfReactedEvent !== '') ? colors.highlight : colors.text } Text { anchors.verticalCenter: divider.verticalCenter id: reactionCounter - text: model.counter + text: modelData.count font: reaction.font color: reaction.hovered ? colors.highlight : colors.text } @@ -82,8 +82,8 @@ Flow { implicitWidth: reaction.implicitWidth implicitHeight: reaction.implicitHeight - border.color: (reaction.hovered || model.selfReactedEvent !== '') ? colors.highlight : colors.text - color: model.selfReactedEvent !== '' ? Qt.hsla(highlightHue, highlightSat, highlightLight, 0.20) : colors.base + border.color: (reaction.hovered || modelData.selfReactedEvent !== '') ? colors.highlight : colors.text + color: modelData.selfReactedEvent !== '' ? Qt.hsla(highlightHue, highlightSat, highlightLight, 0.20) : colors.base border.width: 1 radius: reaction.height / 2.0 } diff --git a/src/Cache.cpp b/src/Cache.cpp index 9464a546..0307bee1 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1353,6 +1353,37 @@ Cache::storeEvent(const std::string &room_id, txn.commit(); } +std::vector +Cache::relatedEvents(const std::string &room_id, const std::string &event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto relationsDb = getRelationsDb(txn, room_id); + + std::vector related_ids; + + auto related_cursor = lmdb::cursor::open(txn, relationsDb); + lmdb::val related_to = event_id, related_event; + bool first = true; + + try { + if (!related_cursor.get(related_to, related_event, MDB_SET)) + return {}; + + while (related_cursor.get( + related_to, related_event, first ? MDB_FIRST_DUP : MDB_NEXT_DUP)) { + first = false; + if (event_id != std::string_view(related_to.data(), related_to.size())) + break; + + related_ids.emplace_back(related_event.data(), related_event.size()); + } + } catch (const lmdb::error &e) { + nhlog::db()->error("related events error: {}", e.what()); + } + + return related_ids; +} + QMap Cache::roomInfo(bool withInvites) { @@ -2354,6 +2385,10 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message std::string event_id_val; for (const auto &e : res.chunk) { + if (std::holds_alternative< + mtx::events::RedactionEvent>(e)) + continue; + auto event = mtx::accessors::serialize_event(e); event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; diff --git a/src/Cache_p.h b/src/Cache_p.h index 88308e45..61d91b0c 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -188,6 +188,9 @@ public: void storeEvent(const std::string &room_id, const std::string &event_id, const mtx::events::collections::TimelineEvent &event); + std::vector relatedEvents(const std::string &room_id, + const std::string &event_id); + struct TimelineRange { uint64_t first, last; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 80e8d474..0bd7a97e 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -3,12 +3,15 @@ #include #include +#include "Cache.h" #include "Cache_p.h" #include "EventAccessors.h" #include "Logging.h" #include "MatrixClient.h" #include "Olm.h" +Q_DECLARE_METATYPE(Reaction) + QCache EventStore::decryptedEvents_{ 1000}; QCache EventStore::events_by_id_{ @@ -18,6 +21,9 @@ QCache EventStore:: EventStore::EventStore(std::string room_id, QObject *) : room_id_(std::move(room_id)) { + static auto reactionType = qRegisterMetaType(); + (void)reactionType; + auto range = cache::client()->getTimelineRange(room_id_); if (range) { @@ -223,6 +229,70 @@ EventStore::handleSync(const mtx::responses::Timeline &events) } } +QVariantList +EventStore::reactions(const std::string &event_id) +{ + auto event_ids = cache::client()->relatedEvents(room_id_, event_id); + + struct TempReaction + { + int count = 0; + std::vector users; + std::string reactedBySelf; + }; + std::map aggregation; + std::vector reactions; + + auto self = http::client()->user_id().to_string(); + for (const auto &id : event_ids) { + auto related_event = event(id, event_id); + if (!related_event) + continue; + + if (auto reaction = std::get_if>( + related_event)) { + auto &agg = aggregation[reaction->content.relates_to.key]; + + if (agg.count == 0) { + Reaction temp{}; + temp.key_ = + QString::fromStdString(reaction->content.relates_to.key); + reactions.push_back(temp); + } + + agg.count++; + agg.users.push_back(cache::displayName(room_id_, reaction->sender)); + if (reaction->sender == self) + agg.reactedBySelf = reaction->event_id; + } + } + + QVariantList temp; + for (auto &reaction : reactions) { + const auto &agg = aggregation[reaction.key_.toStdString()]; + reaction.count_ = agg.count; + reaction.selfReactedEvent_ = QString::fromStdString(agg.reactedBySelf); + + bool first = true; + for (const auto &user : agg.users) { + if (first) + first = false; + else + reaction.users_ += ", "; + + reaction.users_ += QString::fromStdString(user); + } + + nhlog::db()->debug("key: {}, count: {}, users: {}", + reaction.key_.toStdString(), + reaction.count_, + reaction.users_.toStdString()); + temp.append(QVariant::fromValue(reaction)); + } + + return temp; +} + mtx::events::collections::TimelineEvents * EventStore::event(int idx, bool decrypt) { diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 3a78cba8..5a792040 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -5,12 +5,15 @@ #include #include +#include #include #include #include #include +#include "Reaction.h" + class EventStore : public QObject { Q_OBJECT @@ -65,6 +68,8 @@ public: // always returns a proper event as long as the idx is valid mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true); + QVariantList reactions(const std::string &event_id); + int size() const { return last != std::numeric_limits::max() diff --git a/src/timeline/Reaction.cpp b/src/timeline/Reaction.cpp new file mode 100644 index 00000000..343c4649 --- /dev/null +++ b/src/timeline/Reaction.cpp @@ -0,0 +1 @@ +#include "Reaction.h" diff --git a/src/timeline/Reaction.h b/src/timeline/Reaction.h new file mode 100644 index 00000000..5f122e0a --- /dev/null +++ b/src/timeline/Reaction.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +struct Reaction +{ + Q_GADGET + Q_PROPERTY(QString key READ key) + Q_PROPERTY(QString users READ users) + Q_PROPERTY(QString selfReactedEvent READ selfReactedEvent) + Q_PROPERTY(int count READ count) + +public: + QString key() const { return key_; } + QString users() const { return users_; } + QString selfReactedEvent() const { return selfReactedEvent_; } + int count() const { return count_; } + + QString key_; + QString users_; + QString selfReactedEvent_; + int count_; +}; diff --git a/src/timeline/ReactionsModel.cpp b/src/timeline/ReactionsModel.cpp deleted file mode 100644 index 1200e2ba..00000000 --- a/src/timeline/ReactionsModel.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "ReactionsModel.h" - -#include -#include - -QHash -ReactionsModel::roleNames() const -{ - return { - {Key, "key"}, - {Count, "counter"}, - {Users, "users"}, - {SelfReactedEvent, "selfReactedEvent"}, - }; -} - -int -ReactionsModel::rowCount(const QModelIndex &) const -{ - return static_cast(reactions.size()); -} - -QVariant -ReactionsModel::data(const QModelIndex &index, int role) const -{ - const int i = index.row(); - if (i < 0 || i >= static_cast(reactions.size())) - return {}; - - switch (role) { - case Key: - return QString::fromStdString(reactions[i].key); - case Count: - return static_cast(reactions[i].reactions.size()); - case Users: { - QString users; - bool first = true; - for (const auto &reaction : reactions[i].reactions) { - if (!first) - users += ", "; - else - first = false; - users += QString::fromStdString( - cache::displayName(room_id_, reaction.second.sender)); - } - return users; - } - case SelfReactedEvent: - for (const auto &reaction : reactions[i].reactions) - if (reaction.second.sender == http::client()->user_id().to_string()) - return QString::fromStdString(reaction.second.event_id); - return QStringLiteral(""); - default: - return {}; - } -} - -void -ReactionsModel::addReaction(const std::string &room_id, - const mtx::events::RoomEvent &reaction) -{ - room_id_ = room_id; - - int idx = 0; - for (auto &storedReactions : reactions) { - if (storedReactions.key == reaction.content.relates_to.key) { - storedReactions.reactions[reaction.event_id] = reaction; - emit dataChanged(index(idx, 0), index(idx, 0)); - return; - } - idx++; - } - - beginInsertRows(QModelIndex(), idx, idx); - reactions.push_back( - KeyReaction{reaction.content.relates_to.key, {{reaction.event_id, reaction}}}); - endInsertRows(); -} - -void -ReactionsModel::removeReaction(const mtx::events::RoomEvent &reaction) -{ - int idx = 0; - for (auto &storedReactions : reactions) { - if (storedReactions.key == reaction.content.relates_to.key) { - storedReactions.reactions.erase(reaction.event_id); - - if (storedReactions.reactions.size() == 0) { - beginRemoveRows(QModelIndex(), idx, idx); - reactions.erase(reactions.begin() + idx); - endRemoveRows(); - } else - emit dataChanged(index(idx, 0), index(idx, 0)); - return; - } - idx++; - } -} diff --git a/src/timeline/ReactionsModel.h b/src/timeline/ReactionsModel.h deleted file mode 100644 index c839afc8..00000000 --- a/src/timeline/ReactionsModel.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - -class ReactionsModel : public QAbstractListModel -{ - Q_OBJECT -public: - explicit ReactionsModel(QObject *parent = nullptr) { Q_UNUSED(parent); } - enum Roles - { - Key, - Count, - Users, - SelfReactedEvent, - }; - - QHash roleNames() const override; - int rowCount(const QModelIndex &parent = QModelIndex()) const override; - QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; - -public slots: - void addReaction(const std::string &room_id, - const mtx::events::RoomEvent &reaction); - void removeReaction(const mtx::events::RoomEvent &reaction); - -private: - struct KeyReaction - { - std::string key; - std::map> reactions; - }; - std::string room_id_; - std::vector reactions; -}; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 470e3988..85d2eb4e 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -366,7 +366,8 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); case Reactions: { - return {}; + auto id = event_id(event); + return QVariant::fromValue(events.reactions(id)); } case RoomId: return QVariant(room_id_); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index 9f9717df..cbe88fd2 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -10,7 +10,6 @@ #include "CacheCryptoStructs.h" #include "EventStore.h" -#include "ReactionsModel.h" namespace mtx::http { using RequestErr = const std::optional &; -- cgit 1.5.1 From f23d733cffc9cfbc806631fe670d3ca28c40417a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 20 Jul 2020 18:25:22 +0200 Subject: Fix room joins --- src/Cache.cpp | 12 ++++++++++-- src/ChatPage.cpp | 9 ++------- 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 0307bee1..3aec445a 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1439,8 +1439,16 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) std::optional Cache::getTimelineRange(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); - auto orderDb = getOrderToMessageDb(txn, room_id); + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + lmdb::dbi orderDb{0}; + try { + orderDb = getOrderToMessageDb(txn, room_id); + } catch (lmdb::runtime_error &e) { + nhlog::db()->error("Can't open db for room '{}', probably doesn't exist yet. ({})", + room_id, + e.what()); + return {}; + } lmdb::val indexVal, val; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index c4376905..012f1e69 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -244,7 +244,6 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) room_list_, &RoomList::roomChanged, view_manager_, &TimelineViewManager::setHistoryView); connect(room_list_, &RoomList::acceptInvite, this, [this](const QString &room_id) { - view_manager_->addRoom(room_id); joinRoom(room_id); room_list_->removeRoom(room_id, currentRoom() == room_id); }); @@ -543,12 +542,8 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) emit notificationsRetrieved(std::move(res)); }); }); - connect(this, &ChatPage::syncRoomlist, room_list_, &RoomList::sync, Qt::QueuedConnection); - connect(this, - &ChatPage::syncTags, - communitiesList_, - &CommunitiesList::syncTags, - Qt::QueuedConnection); + connect(this, &ChatPage::syncRoomlist, room_list_, &RoomList::sync); + connect(this, &ChatPage::syncTags, communitiesList_, &CommunitiesList::syncTags); connect( this, &ChatPage::syncTopBar, this, [this](const std::map &updates) { if (updates.find(currentRoom()) != updates.end()) -- cgit 1.5.1 From b294430fe5814d56dcfb32b9db22a8d5f52bfdce Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 25 Jul 2020 14:08:13 +0200 Subject: Return to redacted messages instead of just storing the redaction --- src/Cache.cpp | 60 +++++++++++++++++++++++++++++---------------- src/timeline/EventStore.cpp | 7 +++++- 2 files changed, 45 insertions(+), 22 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 3aec445a..59755e1e 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2242,11 +2242,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto event = mtx::accessors::serialize_event(e); auto txn_id = mtx::accessors::transaction_id(e); + std::string event_id_val = event.value("event_id", ""); + if (event_id_val.empty()) { + nhlog::db()->error("Event without id!"); + continue; + } + + lmdb::val event_id = event_id_val; + lmdb::val txn_order; if (!txn_id.empty() && lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) { - std::string event_id_val = event["event_id"].get(); - lmdb::val event_id = event_id_val; lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); lmdb::dbi_del(txn, eventsDb, lmdb::val(txn_id)); @@ -2291,28 +2297,40 @@ Cache::saveTimelineMessages(lmdb::txn &txn, if (redaction->redacts.empty()) continue; - lmdb::val ev{}; - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump())); - - lmdb::val oldIndex{}; - if (lmdb::dbi_get( - txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) { - lmdb::dbi_put( - txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id)); - lmdb::dbi_put( - txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); - } - } else { - std::string event_id_val = event.value("event_id", ""); - if (event_id_val.empty()) { - nhlog::db()->error("Event without id!"); + lmdb::val oldEvent; + bool success = + lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), oldEvent); + if (!success) + continue; + + mtx::events::collections::TimelineEvent te; + try { + mtx::events::collections::from_json( + json::parse(std::string_view(oldEvent.data(), oldEvent.size())), + te); + // overwrite the content and add redation data + std::visit( + [redaction](auto &ev) { + ev.unsigned_data.redacted_because = *redaction; + ev.unsigned_data.redacted_by = redaction->event_id; + }, + te.data); + event = mtx::accessors::serialize_event(te.data); + event["content"].clear(); + + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", + e.what()); continue; } - lmdb::val event_id = event_id_val; + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(redaction->event_id), + lmdb::val(json(*redaction).dump())); + } else { lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index eb1162cc..704402c8 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -211,6 +211,7 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (idx) { events_by_id_.remove( {room_id_, redaction->redacts}); + events_.remove({room_id_, toInternalIdx(*idx)}); emit dataChanged(*idx, *idx); } } @@ -227,8 +228,12 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (!relates_to.empty()) { auto idx = cache::client()->getTimelineIndex(room_id_, relates_to); - if (idx) + if (idx) { + events_by_id_.remove({room_id_, relates_to}); + decryptedEvents_.remove({room_id_, relates_to}); + events_.remove({room_id_, *idx}); emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } } if (auto txn_id = mtx::accessors::transaction_id(event); !txn_id.empty()) { -- cgit 1.5.1 From 720bb164f7051949f9f47b33d2793bd6e5c13f19 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 26 Jul 2020 19:04:36 +0200 Subject: Fix migration (hopefully) --- src/Cache.cpp | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 59755e1e..628062a1 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -693,9 +693,56 @@ Cache::runMigrations() auto room_ids = getRoomIds(txn); for (const auto &room_id : room_ids) { - auto messagesDb = lmdb::dbi::open( - txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); - lmdb::dbi_drop(txn, messagesDb, true); + try { + auto messagesDb = lmdb::dbi::open( + txn, std::string(room_id + "/messages").c_str()); + + // keep some old messages and batch token + { + auto roomsCursor = + lmdb::cursor::open(txn, messagesDb); + lmdb::val ts, stored_message; + bool start = true; + mtx::responses::Timeline oldMessages; + while (roomsCursor.get(ts, + stored_message, + start ? MDB_FIRST + : MDB_NEXT)) { + start = false; + + auto j = json::parse(std::string_view( + stored_message.data(), + stored_message.size())); + + if (oldMessages.prev_batch.empty()) + oldMessages.prev_batch = + j["token"].get(); + else if (j["token"] != + oldMessages.prev_batch) + break; + + mtx::events::collections::TimelineEvent + te; + mtx::events::collections::from_json( + j["event"], te); + oldMessages.events.push_back(te.data); + } + // messages were stored in reverse order, so we + // need to reverse them + std::reverse(oldMessages.events.begin(), + oldMessages.events.end()); + // save messages using the new method + saveTimelineMessages(txn, room_id, oldMessages); + } + + // delete old messages db + lmdb::dbi_drop(txn, messagesDb, true); + } catch (std::exception &e) { + nhlog::db()->error( + "While migrating messages from {}, ignoring error {}", + room_id, + e.what()); + } } txn.commit(); } catch (const lmdb::error &) { -- cgit 1.5.1 From 7f3d97517f334cbc9b07100d20acb612a3293bfd Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 30 Jul 2020 18:13:19 +0200 Subject: Fix double free by closing cursor at the right time --- src/Cache.cpp | 59 +++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 26 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 628062a1..0c692d07 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2199,27 +2199,31 @@ Cache::firstPendingMessage(const std::string &room_id) auto txn = lmdb::txn::begin(env_); auto pending = getPendingMessagesDb(txn, room_id); - auto pendingCursor = lmdb::cursor::open(txn, pending); - lmdb::val tsIgnored, pendingTxn; - while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { - auto eventsDb = getEventsDb(txn, room_id); - lmdb::val event; - if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) { - lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); - continue; - } + { + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + auto eventsDb = getEventsDb(txn, room_id); + lmdb::val event; + if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) { + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } - try { - mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json( - json::parse(std::string_view(event.data(), event.size())), te); + try { + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); - txn.commit(); - return te; - } catch (std::exception &e) { - nhlog::db()->error("Failed to parse message from cache {}", e.what()); - lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); - continue; + pendingCursor.close(); + txn.commit(); + return te; + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", + e.what()); + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } } } @@ -2231,13 +2235,16 @@ Cache::firstPendingMessage(const std::string &room_id) void Cache::removePendingStatus(const std::string &room_id, const std::string &txn_id) { - auto txn = lmdb::txn::begin(env_); - auto pending = getPendingMessagesDb(txn, room_id); - auto pendingCursor = lmdb::cursor::open(txn, pending); - lmdb::val tsIgnored, pendingTxn; - while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { - if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id) - lmdb::cursor_del(pendingCursor); + auto txn = lmdb::txn::begin(env_); + auto pending = getPendingMessagesDb(txn, room_id); + + { + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id) + lmdb::cursor_del(pendingCursor); + } } txn.commit(); -- cgit 1.5.1 From 14a0aac74873c27c0454d206848f27b4eec123ae Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 9 Aug 2020 23:36:47 +0200 Subject: Add /clear-timeline command --- src/Cache.cpp | 118 +++++++++++++++++++++++++++++++++---- src/Cache_p.h | 3 + src/ChatPage.cpp | 5 ++ src/TextInputWidget.cpp | 24 ++++---- src/TextInputWidget.h | 1 + src/timeline/EventStore.cpp | 20 +++++++ src/timeline/EventStore.h | 1 + src/timeline/TimelineModel.h | 1 + src/timeline/TimelineViewManager.h | 6 ++ 9 files changed, 157 insertions(+), 22 deletions(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 0c692d07..0d879584 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2304,6 +2304,11 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val event_id = event_id_val; + json orderEntry = json::object(); + orderEntry["event_id"] = event_id_val; + if (first && !res.prev_batch.empty()) + orderEntry["prev_batch"] = res.prev_batch; + lmdb::val txn_order; if (!txn_id.empty() && lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) { @@ -2317,7 +2322,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_del(txn, msg2orderDb, lmdb::val(txn_id)); } - lmdb::dbi_put(txn, orderDb, txn_order, event_id); + lmdb::dbi_put(txn, orderDb, txn_order, lmdb::val(orderEntry.dump())); lmdb::dbi_put(txn, evToOrderDb, event_id, txn_order); lmdb::dbi_del(txn, evToOrderDb, lmdb::val(txn_id)); @@ -2389,10 +2394,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn, ++index; - json orderEntry = json::object(); - orderEntry["event_id"] = event_id_val; - if (first && !res.prev_batch.empty()) - orderEntry["prev_batch"] = res.prev_batch; first = false; nhlog::db()->debug("saving '{}'", orderEntry.dump()); @@ -2440,6 +2441,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message auto relationsDb = getRelationsDb(txn, room_id); auto orderDb = getEventOrderDb(txn, room_id); + auto evToOrderDb = getEventToOrderDb(txn, room_id); auto msg2orderDb = getMessageToOrderDb(txn, room_id); auto order2msgDb = getOrderToMessageDb(txn, room_id); @@ -2483,6 +2485,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message lmdb::dbi_put( txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index))); // TODO(Nico): Allow blacklisting more event types in UI if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { @@ -2516,6 +2519,94 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message return msgIndex; } +void +Cache::clearTimeline(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); + + auto orderDb = getEventOrderDb(txn, room_id); + auto evToOrderDb = getEventToOrderDb(txn, room_id); + auto msg2orderDb = getMessageToOrderDb(txn, room_id); + auto order2msgDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal, val; + auto cursor = lmdb::cursor::open(txn, orderDb); + + bool start = true; + bool passed_pagination_token = false; + while (cursor.get(indexVal, val, start ? MDB_LAST : MDB_PREV)) { + start = false; + json obj; + + try { + obj = json::parse(std::string_view(val.data(), val.size())); + } catch (std::exception &) { + // workaround bug in the initial db format, where we sometimes didn't store + // json... + obj = {{"event_id", std::string(val.data(), val.size())}}; + } + + if (passed_pagination_token) { + if (obj.count("event_id") != 0) { + lmdb::val event_id = obj["event_id"].get(); + lmdb::dbi_del(txn, evToOrderDb, event_id); + lmdb::dbi_del(txn, eventsDb, event_id); + + lmdb::dbi_del(txn, relationsDb, event_id); + + lmdb::val order{}; + bool exists = lmdb::dbi_get(txn, msg2orderDb, event_id, order); + if (exists) { + lmdb::dbi_del(txn, order2msgDb, order); + lmdb::dbi_del(txn, msg2orderDb, event_id); + } + } + lmdb::cursor_del(cursor); + } else { + if (obj.count("prev_batch") != 0) + passed_pagination_token = true; + } + } + + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + start = true; + while (msgCursor.get(indexVal, val, start ? MDB_LAST : MDB_PREV)) { + start = false; + + lmdb::val eventId; + bool innerStart = true; + bool found = false; + while (cursor.get(indexVal, eventId, innerStart ? MDB_LAST : MDB_PREV)) { + innerStart = false; + + json obj; + try { + obj = json::parse(std::string_view(eventId.data(), eventId.size())); + } catch (std::exception &) { + obj = {{"event_id", std::string(eventId.data(), eventId.size())}}; + } + + if (obj["event_id"] == std::string(val.data(), val.size())) { + found = true; + break; + } + } + + if (!found) + break; + } + + do { + lmdb::cursor_del(msgCursor); + } while (msgCursor.get(indexVal, val, MDB_PREV)); + + cursor.close(); + msgCursor.close(); + txn.commit(); +} + mtx::responses::Notifications Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id) { @@ -2654,11 +2745,13 @@ Cache::deleteOldMessages() auto room_ids = getRoomIds(txn); for (const auto &room_id : room_ids) { - auto orderDb = getEventOrderDb(txn, room_id); - auto o2m = getOrderToMessageDb(txn, room_id); - auto m2o = getMessageToOrderDb(txn, room_id); - auto eventsDb = getEventsDb(txn, room_id); - auto cursor = lmdb::cursor::open(txn, orderDb); + auto orderDb = getEventOrderDb(txn, room_id); + auto evToOrderDb = getEventToOrderDb(txn, room_id); + auto o2m = getOrderToMessageDb(txn, room_id); + auto m2o = getMessageToOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); + auto cursor = lmdb::cursor::open(txn, orderDb); uint64_t first, last; if (cursor.get(indexVal, val, MDB_LAST)) { @@ -2678,14 +2771,17 @@ Cache::deleteOldMessages() bool start = true; while (cursor.get(indexVal, val, start ? MDB_FIRST : MDB_NEXT) && - message_count-- < MAX_RESTORED_MESSAGES) { + message_count-- > MAX_RESTORED_MESSAGES) { start = false; auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") != 0) { lmdb::val event_id = obj["event_id"].get(); + lmdb::dbi_del(txn, evToOrderDb, event_id); lmdb::dbi_del(txn, eventsDb, event_id); + lmdb::dbi_del(txn, relationsDb, event_id); + lmdb::val order{}; bool exists = lmdb::dbi_get(txn, m2o, event_id, order); if (exists) { diff --git a/src/Cache_p.h b/src/Cache_p.h index 61d91b0c..d3ec6ee0 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -208,6 +208,9 @@ public: const std::string &room_id); void removePendingStatus(const std::string &room_id, const std::string &txn_id); + //! clear timeline keeping only the latest batch + void clearTimeline(const std::string &room_id); + //! Remove old unused data. void deleteOldMessages(); void deleteOldData() noexcept; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 518be31c..63d13fb9 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -155,6 +155,11 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) trySync(); }); + connect(text_input_, + &TextInputWidget::clearRoomTimeline, + view_manager_, + &TimelineViewManager::clearCurrentRoomTimeline); + connect( new QShortcut(QKeySequence("Ctrl+Down"), this), &QShortcut::activated, this, [this]() { if (isVisible()) diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp index 3e3915bb..91846230 100644 --- a/src/TextInputWidget.cpp +++ b/src/TextInputWidget.cpp @@ -566,27 +566,29 @@ void TextInputWidget::command(QString command, QString args) { if (command == "me") { - sendEmoteMessage(args); + emit sendEmoteMessage(args); } else if (command == "join") { - sendJoinRoomRequest(args); + emit sendJoinRoomRequest(args); } else if (command == "invite") { - sendInviteRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); + emit sendInviteRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); } else if (command == "kick") { - sendKickRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); + emit sendKickRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); } else if (command == "ban") { - sendBanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); + emit sendBanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); } else if (command == "unban") { - sendUnbanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); + emit sendUnbanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1)); } else if (command == "roomnick") { - changeRoomNick(args); + emit changeRoomNick(args); } else if (command == "shrug") { - sendTextMessage("¯\\_(ツ)_/¯"); + emit sendTextMessage("¯\\_(ツ)_/¯"); } else if (command == "fliptable") { - sendTextMessage("(╯°□°)╯︵ ┻━┻"); + emit sendTextMessage("(╯°□°)╯︵ ┻━┻"); } else if (command == "unfliptable") { - sendTextMessage(" ┯━┯╭( º _ º╭)"); + emit sendTextMessage(" ┯━┯╭( º _ º╭)"); } else if (command == "sovietflip") { - sendTextMessage("ノ┬─┬ノ ︵ ( \\o°o)\\"); + emit sendTextMessage("ノ┬─┬ノ ︵ ( \\o°o)\\"); + } else if (command == "clear-timeline") { + emit clearRoomTimeline(); } } diff --git a/src/TextInputWidget.h b/src/TextInputWidget.h index a0105eb0..cbb6ea95 100644 --- a/src/TextInputWidget.h +++ b/src/TextInputWidget.h @@ -156,6 +156,7 @@ private slots: signals: void sendTextMessage(const QString &msg); void sendEmoteMessage(QString msg); + void clearRoomTimeline(); void heightChanged(int height); void uploadMedia(const QSharedPointer data, diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index a983fe01..fca1d31d 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -175,6 +175,26 @@ EventStore::addPending(mtx::events::collections::TimelineEvents event) emit processPending(); } +void +EventStore::clearTimeline() +{ + emit beginResetModel(); + + cache::client()->clearTimeline(room_id_); + auto range = cache::client()->getTimelineRange(room_id_); + if (range) { + nhlog::db()->info("Range {} {}", range->last, range->first); + this->last = range->last; + this->first = range->first; + } else { + this->first = std::numeric_limits::max(); + this->last = std::numeric_limits::max(); + } + nhlog::ui()->info("Range {} {}", this->last, this->first); + + emit endResetModel(); +} + void EventStore::handleSync(const mtx::responses::Timeline &events) { diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index b5c17d10..d4353a18 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -101,6 +101,7 @@ signals: public slots: void addPending(mtx::events::collections::TimelineEvents event); + void clearTimeline(); private: mtx::events::collections::TimelineEvents *decryptEvent( diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index f8a84f17..0bcf42b7 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -242,6 +242,7 @@ public slots: } } void setDecryptDescription(bool decrypt) { decryptDescription = decrypt; } + void clearTimeline() { events.clearTimeline(); } private slots: void addPendingMessage(mtx::events::collections::TimelineEvents event); diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h index 63106916..20dbc3bb 100644 --- a/src/timeline/TimelineViewManager.h +++ b/src/timeline/TimelineViewManager.h @@ -92,6 +92,12 @@ public slots: uint64_t dsize); void updateEncryptedDescriptions(); + void clearCurrentRoomTimeline() + { + if (timeline_) + timeline_->clearTimeline(); + } + private: #ifdef USE_QUICK_VIEW QQuickView *view; -- cgit 1.5.1 From 7f7108161e87df272aefb9a14aec708ff427839f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 17 Aug 2020 23:30:36 +0200 Subject: Hide CallCandidates again in new store --- resources/qml/delegates/MessageDelegate.qml | 6 ++++++ src/Cache.cpp | 32 +++++++++++++++++++++++++++-- src/timeline/TimelineModel.cpp | 6 +++++- src/timeline/TimelineModel.h | 2 ++ 4 files changed, 43 insertions(+), 3 deletions(-) (limited to 'src/Cache.cpp') diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index 7b6e0703..56b8040e 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -114,6 +114,12 @@ Item { text: qsTr("%1 ended the call.").arg(model.data.userName) } } + DelegateChoice { + roleValue: MtxEvent.CallCandidates + NoticeMessage { + text: qsTr("Negotiating call...") + } + } DelegateChoice { // TODO: make a more complex formatter for the power levels. roleValue: MtxEvent.PowerLevels diff --git a/src/Cache.cpp b/src/Cache.cpp index fd26f63e..e41ad7ca 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -33,6 +33,7 @@ #include "Cache_p.h" #include "EventAccessors.h" #include "Logging.h" +#include "Olm.h" #include "Utils.h" //! Should be changed when a breaking change occurs in the cache format. @@ -93,6 +94,33 @@ namespace { std::unique_ptr instance_ = nullptr; } +static bool +isHiddenEvent(mtx::events::collections::TimelineEvents e, const std::string &room_id) +{ + using namespace mtx::events; + if (auto encryptedEvent = std::get_if>(&e)) { + MegolmSessionIndex index; + index.room_id = room_id; + index.session_id = encryptedEvent->content.session_id; + index.sender_key = encryptedEvent->content.sender_key; + + auto result = olm::decryptEvent(index, *encryptedEvent); + if (!result.error) + e = result.event.value(); + } + + static constexpr std::initializer_list hiddenEvents = { + EventType::Reaction, EventType::CallCandidates, EventType::Unsupported}; + + return std::visit( + [](const auto &ev) { + return std::any_of(hiddenEvents.begin(), + hiddenEvents.end(), + [ev](EventType type) { return type == ev.type; }); + }, + e); +} + Cache::Cache(const QString &userId, QObject *parent) : QObject{parent} , env_{nullptr} @@ -2406,7 +2434,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index))); // TODO(Nico): Allow blacklisting more event types in UI - if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + if (!isHiddenEvent(e, room_id)) { ++msgIndex; lmdb::cursor_put(msgCursor.handle(), lmdb::val(&msgIndex, sizeof(msgIndex)), @@ -2489,7 +2517,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index))); // TODO(Nico): Allow blacklisting more event types in UI - if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + if (!isHiddenEvent(e, room_id)) { --msgIndex; lmdb::dbi_put( txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id); diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 9695f850..b6c2d4bb 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -136,6 +136,11 @@ struct RoomEventType { return qml_mtx_events::EventType::CallHangUp; } + qml_mtx_events::EventType operator()( + const mtx::events::Event &) + { + return qml_mtx_events::EventType::CallCandidates; + } // ::EventType::Type operator()(const Event &e) { return // ::EventType::LocationMessage; } }; @@ -1122,7 +1127,6 @@ struct SendMessageVisitor } } - // Do-nothing operator for all unhandled events template void operator()(const mtx::events::Event &) diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index 034ae31a..156606e6 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -42,6 +42,8 @@ enum EventType CallAnswer, /// m.call.hangup CallHangUp, + /// m.call.candidates + CallCandidates, /// m.room.canonical_alias CanonicalAlias, /// m.room.create -- cgit 1.5.1 From 9f79b855799e8b11f971d2481621a3de344fac4a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Tue, 25 Aug 2020 23:05:20 +0200 Subject: Speedup db a bit, but loose some crash resiliency The loss in durability shouldn't matter, if we can just receive the same events again after a restart --- src/Cache.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index e41ad7ca..2231aaac 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -168,7 +168,10 @@ Cache::setup() } try { - env_.open(statePath.toStdString().c_str()); + // NOTE(Nico): We may want to use (MDB_MAPASYNC | MDB_WRITEMAP) in the future, but + // it can really mess up our database, so we shouldn't. For now, hopefully + // NOMETASYNC is fast enough. + env_.open(statePath.toStdString().c_str(), MDB_NOMETASYNC); } catch (const lmdb::error &e) { if (e.code() != MDB_VERSION_MISMATCH && e.code() != MDB_INVALID) { throw std::runtime_error("LMDB initialization failed" + -- cgit 1.5.1 From 3df4bde0324f9389a59749847cb5b79dff4bea1f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Tue, 25 Aug 2020 23:12:01 +0200 Subject: Add some log messages, that migrations are in progress --- src/Cache.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp index 2231aaac..91cde9e7 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -787,6 +787,7 @@ Cache::runMigrations() }}, }; + nhlog::db()->info("Running migrations, this may take a while!"); for (const auto &[target_version, migration] : migrations) { if (target_version > stored_version) if (!migration()) { @@ -794,6 +795,7 @@ Cache::runMigrations() return false; } } + nhlog::db()->info("Migrations finished."); setCurrentFormat(); return true; -- cgit 1.5.1