// SPDX-FileCopyrightText: 2017 Konstantinos Sideris // SPDX-FileCopyrightText: 2019 The nheko authors // SPDX-FileCopyrightText: 2021 Nheko Contributors // SPDX-FileCopyrightText: 2022 Nheko Contributors // SPDX-FileCopyrightText: 2023 Nheko Contributors // // SPDX-License-Identifier: GPL-3.0-or-later #pragma once #include #include #include #include #if __has_include() #include #else #include #endif #include #include #include #include #include #include "CacheCryptoStructs.h" #include "CacheStructs.h" #include "Logging.h" namespace mtx::responses { struct Messages; } class Cache final : public QObject { Q_OBJECT public: Cache(const QString &userId, QObject *parent = nullptr); std::string displayName(const std::string &room_id, const std::string &user_id); QString displayName(const QString &room_id, const QString &user_id); QString avatarUrl(const QString &room_id, const QString &user_id); // presence mtx::events::presence::Presence presence(const std::string &user_id); // user cache stores user keys std::map> getMembersWithKeys(const std::string &room_id, bool verified_only); void updateUserKeys(const std::string &sync_token, const mtx::responses::QueryKeys &keyQuery); void markUserKeysOutOfDate(const std::vector &user_ids); void markUserKeysOutOfDate(lmdb::txn &txn, lmdb::dbi &db, const std::vector &user_ids, const std::string &sync_token); void query_keys(const std::string &user_id, std::function cb); // device & user verification cache std::optional userKeys(const std::string &user_id); VerificationStatus verificationStatus(const std::string &user_id); void markDeviceVerified(const std::string &user_id, const std::string &device); void markDeviceUnverified(const std::string &user_id, const std::string &device); crypto::Trust roomVerificationStatus(const std::string &room_id); std::vector joinedRooms(); QMap roomInfo(bool withInvites = true); QHash invites(); std::optional invite(std::string_view roomid); QMap> spaces(); //! Calculate & return the name of the room. QString getRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb); //! Get room join rules mtx::events::state::JoinRule getRoomJoinRule(lmdb::txn &txn, lmdb::dbi &statesdb); bool getRoomGuestAccess(lmdb::txn &txn, lmdb::dbi &statesdb); //! Retrieve the topic of the room if any. QString getRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb); //! Retrieve the room avatar's url if any. QString getRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb); //! Retrieve the version of the room if any. QString getRoomVersion(lmdb::txn &txn, lmdb::dbi &statesdb); //! Retrieve if the room is a space bool getRoomIsSpace(lmdb::txn &txn, lmdb::dbi &statesdb); //! Get a specific state event template std::optional> getStateEvent(const std::string &room_id, std::string_view state_key = "") { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); return getStateEvent(txn, room_id, state_key); } template std::vector> getStateEventsWithType(const std::string &room_id, mtx::events::EventType type = mtx::events::state_content_to_type) { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); return getStateEventsWithType(txn, room_id, type); } //! retrieve a specific event from account data //! pass empty room_id for global account data std::optional getAccountData(mtx::events::EventType type, const std::string &room_id = ""); //! Retrieve member info from a room. std::vector getMembers(const std::string &room_id, std::size_t startIndex = 0, std::size_t len = 30); std::vector getMembersFromInvite(const std::string &room_id, std::size_t startIndex = 0, std::size_t len = 30); size_t memberCount(const std::string &room_id); void updateState(const std::string &room, const mtx::responses::StateEvents &state, bool wipe = false); void saveState(const mtx::responses::Sync &res); bool isInitialized(); bool isDatabaseReady() { return databaseReady_ && isInitialized(); } std::string nextBatchToken(); void deleteData(); void removeInvite(lmdb::txn &txn, const std::string &room_id); void removeInvite(const std::string &room_id); void removeRoom(lmdb::txn &txn, const std::string &roomid); void removeRoom(const std::string &roomid); void setup(); cache::CacheVersion formatVersion(); void setCurrentFormat(); bool runMigrations(); std::vector roomIds(); //! Retrieve all the user ids from a room. std::vector roomMembers(const std::string &room_id); //! Check if the given user has power leve greater than than //! lowest power level of the given events. bool hasEnoughPowerLevel(const std::vector &eventTypes, const std::string &room_id, const std::string &user_id); //! Adds a user to the read list for the given event. //! //! There should be only one user id present in a receipt list per room. //! The user id should be removed from any other lists. using Receipts = std::map>; void updateReadReceipt(lmdb::txn &txn, const std::string &room_id, const Receipts &receipts); //! Retrieve all the read receipts for the given event id and room. //! //! Returns a map of user ids and the time of the read receipt in milliseconds. using UserReceipts = std::multimap>; UserReceipts readReceipts(const QString &event_id, const QString &room_id); RoomInfo singleRoomInfo(const std::string &room_id); std::vector roomsWithStateUpdates(const mtx::responses::Sync &res); std::map getRoomInfo(const std::vector &rooms); std::vector roomNamesAndAliases(); void updateLastMessageTimestamp(const std::string &room_id, uint64_t ts); //! Calculates which the read status of a room. //! Whether all the events in the timeline have been read. std::string getFullyReadEventId(const std::string &room_id); bool calculateRoomReadStatus(const std::string &room_id); void calculateRoomReadStatus(); void markSentNotification(const std::string &event_id); //! Removes an event from the sent notifications. void removeReadNotification(const std::string &event_id); //! Check if we have sent a desktop notification for the given event id. bool isNotificationSent(const std::string &event_id); //! retrieve events in timeline and related functions struct Messages { mtx::responses::Timeline timeline; uint64_t next_index; bool end_of_cache = false; }; Messages getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index = std::numeric_limits::max(), bool forward = false); std::optional getEvent(const std::string &room_id, std::string_view event_id); void storeEvent(const std::string &room_id, const std::string &event_id, const mtx::events::collections::TimelineEvent &event); void replaceEvent(const std::string &room_id, const std::string &event_id, const mtx::events::collections::TimelineEvent &event); std::vector relatedEvents(const std::string &room_id, const std::string &event_id); struct TimelineRange { uint64_t first, last; }; std::optional getTimelineRange(const std::string &room_id); std::optional getTimelineIndex(const std::string &room_id, std::string_view event_id); std::optional getEventIndex(const std::string &room_id, std::string_view event_id); std::optional> lastInvisibleEventAfter(const std::string &room_id, std::string_view event_id); std::optional> lastVisibleEvent(const std::string &room_id, std::string_view event_id); std::optional getTimelineEventId(const std::string &room_id, uint64_t index); std::string previousBatchToken(const std::string &room_id); uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); void savePendingMessage(const std::string &room_id, const mtx::events::collections::TimelineEvent &message); std::vector pendingEvents(const std::string &room_id); std::optional firstPendingMessage(const std::string &room_id); void removePendingStatus(const std::string &room_id, const std::string &txn_id); //! clear timeline keeping only the latest batch void clearTimeline(const std::string &room_id); //! Remove old unused data. void deleteOldMessages(); void deleteOldData() noexcept; //! Retrieve all saved room ids. std::vector getRoomIds(lmdb::txn &txn); std::vector getParentRoomIds(const std::string &room_id); std::vector getChildRoomIds(const std::string &room_id); std::vector getImagePacks(const std::string &room_id, std::optional stickers); //! Mark a room that uses e2e encryption. void setEncryptedRoom(lmdb::txn &txn, const std::string &room_id); bool isRoomEncrypted(const std::string &room_id); std::optional roomEncryptionSettings(const std::string &room_id); //! Check if a user is a member of the room. bool isRoomMember(const std::string &user_id, const std::string &room_id); std::optional getInviteMember(const std::string &room_id, const std::string &user_id); // // Outbound Megolm Sessions // void saveOutboundMegolmSession(const std::string &room_id, const GroupSessionData &data, mtx::crypto::OutboundGroupSessionPtr &session); OutboundGroupSessionDataRef getOutboundMegolmSession(const std::string &room_id); bool outboundMegolmSessionExists(const std::string &room_id) noexcept; void updateOutboundMegolmSession(const std::string &room_id, const GroupSessionData &data, mtx::crypto::OutboundGroupSessionPtr &session); void dropOutboundMegolmSession(const std::string &room_id); void importSessionKeys(const mtx::crypto::ExportedSessionKeys &keys); mtx::crypto::ExportedSessionKeys exportSessionKeys(); // // Inbound Megolm Sessions // void saveInboundMegolmSession(const MegolmSessionIndex &index, mtx::crypto::InboundGroupSessionPtr session, const GroupSessionData &data); mtx::crypto::InboundGroupSessionPtr getInboundMegolmSession(const MegolmSessionIndex &index); bool inboundMegolmSessionExists(const MegolmSessionIndex &index); std::optional getMegolmSessionData(const MegolmSessionIndex &index); // // Olm Sessions // void saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session, uint64_t timestamp); void saveOlmSessions(std::vector> sessions, uint64_t timestamp); std::vector getOlmSessions(const std::string &curve25519); std::optional getOlmSession(const std::string &curve25519, const std::string &session_id); std::optional getLatestOlmSession(const std::string &curve25519); void saveOlmAccount(const std::string &pickled); std::string restoreOlmAccount(); void saveBackupVersion(const OnlineBackupVersion &data); void deleteBackupVersion(); std::optional backupVersion(); void storeSecret(const std::string &name, const std::string &secret, bool internal = false); void deleteSecret(const std::string &name, bool internal = false); std::optional secret(const std::string &name, bool internal = false); std::string pickleSecret(); template constexpr static bool isStateEvent_ = std::is_same_v>, mtx::events::StateEvent().content)>>; static int compare_state_key(const MDB_val *a, const MDB_val *b) { auto get_skey = [](const MDB_val *v) { return nlohmann::json::parse( std::string_view(static_cast(v->mv_data), v->mv_size)) .value("key", ""); }; return get_skey(a).compare(get_skey(b)); } signals: void newReadReceipts(const QString &room_id, const std::vector &event_ids); void roomReadStatus(const std::map &status); void userKeysUpdate(const std::string &sync_token, const mtx::responses::QueryKeys &keyQuery); void userKeysUpdateFinalize(const std::string &user_id); void verificationStatusChanged(const std::string &userid); void selfVerificationStatusChanged(); void secretChanged(const std::string name); void databaseReady(); private: void loadSecretsFromStore( std::vector> toLoad, std::function callback, bool databaseReadyOnFinished = false); void storeSecretInStore(const std::string name, const std::string secret); void deleteSecretFromStore(const std::string name, bool internal); //! Save an invited room. void saveInvite(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb, const mtx::responses::InvitedRoom &room); QString getInviteRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb); QString getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb); QString getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb); bool getInviteRoomIsSpace(lmdb::txn &txn, lmdb::dbi &db); std::optional getMember(const std::string &room_id, const std::string &user_id); std::string getLastEventId(lmdb::txn &txn, const std::string &room_id); void saveTimelineMessages(lmdb::txn &txn, lmdb::dbi &eventsDb, const std::string &room_id, const mtx::responses::Timeline &res); //! retrieve a specific event from account data //! pass empty room_id for global account data std::optional getAccountData(lmdb::txn &txn, mtx::events::EventType type, const std::string &room_id); bool isHiddenEvent(lmdb::txn &txn, mtx::events::collections::TimelineEvents e, const std::string &room_id); //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template void saveStateEvents(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &stateskeydb, lmdb::dbi &membersdb, lmdb::dbi &eventsDb, const std::string &room_id, const std::vector &events) { for (const auto &e : events) saveStateEvent(txn, statesdb, stateskeydb, membersdb, eventsDb, room_id, e); } template void saveStateEvent(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &stateskeydb, lmdb::dbi &membersdb, lmdb::dbi &eventsDb, const std::string &room_id, const T &event) { using namespace mtx::events; using namespace mtx::events::state; if (auto e = std::get_if>(&event); e != nullptr) { switch (e->content.membership) { // // We only keep users with invite or join membership. // case Membership::Invite: case Membership::Join: { auto display_name = e->content.display_name.empty() ? e->state_key : e->content.display_name; // Lightweight representation of a member. MemberInfo tmp{display_name, e->content.avatar_url, e->content.reason}; membersdb.put(txn, e->state_key, nlohmann::json(tmp).dump()); break; } default: { membersdb.del(txn, e->state_key, ""); break; } } // BUG(Nico): Ideally we would fall through and store this in the database, but it seems // to currently corrupt the db sometimes, so... let's find that bug first! return; } else if (std::holds_alternative>(event)) { setEncryptedRoom(txn, room_id); return; } std::visit( [&txn, &statesdb, &stateskeydb, &eventsDb, &membersdb](const auto &e) { if constexpr (isStateEvent_) { eventsDb.put(txn, e.event_id, nlohmann::json(e).dump()); if (e.type != EventType::Unsupported) { if (std::is_same_v>, StateEvent>) { if (e.type == EventType::RoomMember) membersdb.del(txn, e.state_key, ""); else if (e.state_key.empty()) statesdb.del(txn, to_string(e.type)); else stateskeydb.del(txn, to_string(e.type), nlohmann::json::object({ {"key", e.state_key}, {"id", e.event_id}, }) .dump()); } else if (e.state_key.empty()) statesdb.put(txn, to_string(e.type), nlohmann::json(e).dump()); else stateskeydb.put(txn, to_string(e.type), nlohmann::json::object({ {"key", e.state_key}, {"id", e.event_id}, }) .dump()); } } }, event); } template std::optional> getStateEvent(lmdb::txn &txn, const std::string &room_id, std::string_view state_key = "") { try { constexpr auto type = mtx::events::state_content_to_type; static_assert(type != mtx::events::EventType::Unsupported, "Not a supported type in state events."); if (room_id.empty()) return std::nullopt; const auto typeStr = to_string(type); std::string_view value; if (state_key.empty()) { auto db = getStatesDb(txn, room_id); if (!db.get(txn, typeStr, value)) { return std::nullopt; } } else { auto db = getStatesKeyDb(txn, room_id); std::string d = nlohmann::json::object({{"key", state_key}}).dump(); std::string_view data = d; std::string_view typeStrV = typeStr; auto cursor = lmdb::cursor::open(txn, db); if (!cursor.get(typeStrV, data, MDB_GET_BOTH)) return std::nullopt; try { auto eventsDb = getEventsDb(txn, room_id); if (!eventsDb.get( txn, nlohmann::json::parse(data)["id"].get(), value)) return std::nullopt; } catch (std::exception &) { return std::nullopt; } } return nlohmann::json::parse(value).get>(); } catch (std::exception &) { return std::nullopt; } } template std::vector> getStateEventsWithType(lmdb::txn &txn, const std::string &room_id, mtx::events::EventType type = mtx::events::state_content_to_type) { if (room_id.empty()) return {}; std::vector> events; { auto db = getStatesKeyDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); const auto typeStr = to_string(type); std::string_view typeStrV = typeStr; std::string_view data; std::string_view value; auto cursor = lmdb::cursor::open(txn, db); bool first = true; if (cursor.get(typeStrV, data, MDB_SET)) { while (cursor.get(typeStrV, data, first ? MDB_FIRST_DUP : MDB_NEXT_DUP)) { first = false; try { if (eventsDb.get( txn, nlohmann::json::parse(data)["id"].get(), value)) events.push_back( nlohmann::json::parse(value).get>()); } catch (std::exception &e) { nhlog::db()->warn("Failed to parse state event: {}", e.what()); } } } } return events; } void saveInvites(lmdb::txn &txn, const std::map &rooms); void savePresence( lmdb::txn &txn, const std::vector> &presenceUpdates); //! Sends signals for the rooms that are removed. void removeLeftRooms(lmdb::txn &txn, const std::map &rooms) { for (const auto &room : rooms) { removeRoom(txn, room.first); // Clean up leftover invites. removeInvite(txn, room.first); } } void updateSpaces(lmdb::txn &txn, const std::set &spaces_with_updates, std::set rooms_with_updates); lmdb::dbi getPendingReceiptsDb(lmdb::txn &txn) { return lmdb::dbi::open(txn, "pending_receipts", MDB_CREATE); } lmdb::dbi getEventsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); } lmdb::dbi getEventOrderDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); } // inverse of EventOrderDb lmdb::dbi getEventToOrderDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/event2order").c_str(), MDB_CREATE); } lmdb::dbi getMessageToOrderDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/msg2order").c_str(), MDB_CREATE); } lmdb::dbi getOrderToMessageDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( txn, std::string(room_id + "/order2msg").c_str(), MDB_CREATE | MDB_INTEGERKEY); } lmdb::dbi getPendingMessagesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( txn, std::string(room_id + "/pending").c_str(), MDB_CREATE | MDB_INTEGERKEY); } lmdb::dbi getRelationsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( txn, std::string(room_id + "/related").c_str(), MDB_CREATE | MDB_DUPSORT); } lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/invite_state").c_str(), MDB_CREATE); } lmdb::dbi getInviteMembersDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/invite_members").c_str(), MDB_CREATE); } lmdb::dbi getStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/state").c_str(), MDB_CREATE); } lmdb::dbi getStatesKeyDb(lmdb::txn &txn, const std::string &room_id) { auto db = lmdb::dbi::open( txn, std::string(room_id + "/state_by_key").c_str(), MDB_CREATE | MDB_DUPSORT); lmdb::dbi_set_dupsort(txn, db, compare_state_key); return db; } lmdb::dbi getAccountDataDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/account_data").c_str(), MDB_CREATE); } lmdb::dbi getMembersDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/members").c_str(), MDB_CREATE); } lmdb::dbi getUserKeysDb(lmdb::txn &txn) { return lmdb::dbi::open(txn, "user_key", MDB_CREATE); } lmdb::dbi getVerificationDb(lmdb::txn &txn) { return lmdb::dbi::open(txn, "verified", MDB_CREATE); } //! Retrieves or creates the database that stores the open OLM sessions between our device //! and the given curve25519 key which represents another device. //! //! Each entry is a map from the session_id to the pickled representation of the session. lmdb::dbi getOlmSessionsDb(lmdb::txn &txn, const std::string &curve25519_key) { return lmdb::dbi::open( txn, std::string("olm_sessions.v2/" + curve25519_key).c_str(), MDB_CREATE); } QString getDisplayName(const mtx::events::StateEvent &event) { if (!event.content.display_name.empty()) return QString::fromStdString(event.content.display_name); return QString::fromStdString(event.state_key); } std::optional verificationCache(const std::string &user_id, lmdb::txn &txn); VerificationStatus verificationStatus_(const std::string &user_id, lmdb::txn &txn); std::optional userKeys_(const std::string &user_id, lmdb::txn &txn); void setNextBatchToken(lmdb::txn &txn, const std::string &token); lmdb::env env_; lmdb::dbi syncStateDb_; lmdb::dbi roomsDb_; lmdb::dbi spacesChildrenDb_, spacesParentsDb_; lmdb::dbi invitesDb_; lmdb::dbi readReceiptsDb_; lmdb::dbi notificationsDb_; lmdb::dbi presenceDb_; lmdb::dbi devicesDb_; lmdb::dbi deviceKeysDb_; lmdb::dbi inboundMegolmSessionDb_; lmdb::dbi outboundMegolmSessionDb_; lmdb::dbi megolmSessionDataDb_; lmdb::dbi encryptedRooms_; QString localUserId_; QString cacheDirectory_; std::string pickle_secret_; VerificationStorage verification_storage; bool databaseReady_ = false; }; namespace cache { Cache * client(); }