diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml
index 17fe7360..9630ae3a 100644
--- a/resources/qml/delegates/MessageDelegate.qml
+++ b/resources/qml/delegates/MessageDelegate.qml
@@ -67,6 +67,12 @@ Item {
}
}
DelegateChoice {
+ roleValue: MtxEvent.Redaction
+ Pill {
+ text: qsTr("redacted")
+ }
+ }
+ DelegateChoice {
roleValue: MtxEvent.Encryption
Pill {
text: qsTr("Encryption enabled")
diff --git a/src/Cache.cpp b/src/Cache.cpp
index 173b2c70..233ef2b4 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -1241,8 +1241,25 @@ Cache::getTimelineMentions()
return notifs;
}
+std::string
+Cache::previousBatchToken(const std::string &room_id)
+{
+ auto txn = lmdb::txn::begin(env_, nullptr);
+ auto orderDb = getEventOrderDb(txn, room_id);
+
+ auto cursor = lmdb::cursor::open(txn, orderDb);
+ lmdb::val indexVal, val;
+ if (!cursor.get(indexVal, val, MDB_FIRST)) {
+ return "";
+ }
+
+ auto j = json::parse(std::string_view(val.data(), val.size()));
+
+ return j.value("prev_batch", "");
+}
+
Cache::Messages
-Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward)
+Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward)
{
// TODO(nico): Limit the messages returned by this maybe?
auto orderDb = getOrderToMessageDb(txn, room_id);
@@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
lmdb::val indexVal, event_id;
auto cursor = lmdb::cursor::open(txn, orderDb);
- if (index == std::numeric_limits<int64_t>::max()) {
+ if (index == std::numeric_limits<uint64_t>::max()) {
if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) {
- index = *indexVal.data<int64_t>();
+ index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
}
} else {
if (cursor.get(indexVal, event_id, MDB_SET)) {
- index = *indexVal.data<int64_t>();
+ index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
@@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
cursor.close();
// std::reverse(timeline.events.begin(), timeline.events.end());
- messages.next_index = *indexVal.data<int64_t>();
+ messages.next_index = *indexVal.data<uint64_t>();
messages.end_of_cache = !ret;
return messages;
@@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id)
}
TimelineRange range{};
- range.last = *indexVal.data<int64_t>();
+ range.last = *indexVal.data<uint64_t>();
if (!cursor.get(indexVal, val, MDB_FIRST)) {
return {};
}
- range.first = *indexVal.data<int64_t>();
+ range.first = *indexVal.data<uint64_t>();
return range;
}
-std::optional<int64_t>
+std::optional<uint64_t>
Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
@@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
return {};
}
- return *val.data<int64_t>();
+ return *val.data<uint64_t>();
}
std::optional<std::string>
-Cache::getTimelineEventId(const std::string &room_id, int64_t index)
+Cache::getTimelineEventId(const std::string &room_id, uint64_t index)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
auto orderDb = getOrderToMessageDb(txn, room_id);
@@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res)
{
+ if (res.events.empty())
+ return;
+
auto eventsDb = getEventsDb(txn, room_id);
auto relationsDb = getRelationsDb(txn, room_id);
@@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
using namespace mtx::events::state;
lmdb::val indexVal, val;
- int64_t index = 0;
- auto cursor = lmdb::cursor::open(txn, orderDb);
+ uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
+ auto cursor = lmdb::cursor::open(txn, orderDb);
if (cursor.get(indexVal, val, MDB_LAST)) {
index = *indexVal.data<int64_t>();
}
- int64_t msgIndex = 0;
- auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
+ uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
+ auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
if (msgCursor.get(indexVal, val, MDB_LAST)) {
- msgIndex = *indexVal.data<int64_t>();
+ msgIndex = *indexVal.data<uint64_t>();
}
bool first = true;
@@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
continue;
lmdb::val ev{};
- bool success =
- lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev);
- if (!success)
- continue;
-
- mtx::events::collections::TimelineEvent te;
-
- try {
- mtx::events::collections::from_json(
- json::parse(std::string_view(ev.data(), ev.size())), te);
- } catch (std::exception &e) {
- nhlog::db()->error("Failed to parse message from cache {}",
- e.what());
- continue;
+ lmdb::dbi_put(
+ txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump()));
+ lmdb::dbi_put(
+ txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump()));
+
+ lmdb::val oldIndex{};
+ if (lmdb::dbi_get(
+ txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) {
+ lmdb::dbi_put(
+ txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id));
+ lmdb::dbi_put(
+ txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex);
}
-
- auto redactedEvent = std::visit(
- [](const auto &ev) -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
- mtx::events::RoomEvent<mtx::events::msg::Redacted> replacement =
- {};
- replacement.event_id = ev.event_id;
- replacement.room_id = ev.room_id;
- replacement.sender = ev.sender;
- replacement.origin_server_ts = ev.origin_server_ts;
- replacement.type = ev.type;
- return replacement;
- },
- te.data);
-
- lmdb::dbi_put(txn,
- eventsDb,
- lmdb::val(redaction->redacts),
- lmdb::val(json(redactedEvent).dump()));
} else {
std::string event_id_val = event["event_id"].get<std::string>();
lmdb::val event_id = event_id_val;
@@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
}
}
+uint64_t
+Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res)
+{
+ auto txn = lmdb::txn::begin(env_);
+ auto eventsDb = getEventsDb(txn, room_id);
+ auto relationsDb = getRelationsDb(txn, room_id);
+
+ auto orderDb = getEventOrderDb(txn, room_id);
+ auto msg2orderDb = getMessageToOrderDb(txn, room_id);
+ auto order2msgDb = getOrderToMessageDb(txn, room_id);
+
+ lmdb::val indexVal, val;
+ uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
+ auto cursor = lmdb::cursor::open(txn, orderDb);
+ if (cursor.get(indexVal, val, MDB_FIRST)) {
+ index = *indexVal.data<uint64_t>();
+ }
+
+ uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
+ auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
+ if (msgCursor.get(indexVal, val, MDB_FIRST)) {
+ msgIndex = *indexVal.data<uint64_t>();
+ }
+
+ if (res.chunk.empty())
+ return index;
+
+ std::string event_id_val;
+ for (const auto &e : res.chunk) {
+ auto event = mtx::accessors::serialize_event(e);
+ event_id_val = event["event_id"].get<std::string>();
+ lmdb::val event_id = event_id_val;
+ lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump()));
+
+ --index;
+
+ json orderEntry = json::object();
+ orderEntry["event_id"] = event_id_val;
+
+ nhlog::db()->debug("saving '{}'", orderEntry.dump());
+
+ lmdb::dbi_put(
+ txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
+
+ // TODO(Nico): Allow blacklisting more event types in UI
+ if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
+ --msgIndex;
+ lmdb::dbi_put(
+ txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id);
+
+ lmdb::dbi_put(
+ txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex)));
+ }
+
+ if (event.contains("content") && event["content"].contains("m.relates_to")) {
+ auto temp = event["content"]["m.relates_to"];
+ std::string relates_to = temp.contains("m.in_reply_to")
+ ? temp["m.in_reply_to"]["event_id"]
+ : temp["event_id"];
+
+ if (!relates_to.empty())
+ lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id);
+ }
+ }
+
+ json orderEntry = json::object();
+ orderEntry["event_id"] = event_id_val;
+ orderEntry["prev_batch"] = res.end;
+ lmdb::cursor_put(
+ cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
+ nhlog::db()->debug("saving '{}'", orderEntry.dump());
+
+ txn.commit();
+
+ return msgIndex;
+}
+
mtx::responses::Notifications
Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id)
{
@@ -2337,14 +2414,14 @@ Cache::deleteOldMessages()
auto eventsDb = getEventsDb(txn, room_id);
auto cursor = lmdb::cursor::open(txn, orderDb);
- int64_t first, last;
+ uint64_t first, last;
if (cursor.get(indexVal, val, MDB_LAST)) {
- last = *indexVal.data<int64_t>();
+ last = *indexVal.data<uint64_t>();
} else {
continue;
}
if (cursor.get(indexVal, val, MDB_FIRST)) {
- first = *indexVal.data<int64_t>();
+ first = *indexVal.data<uint64_t>();
} else {
continue;
}
diff --git a/src/Cache_p.h b/src/Cache_p.h
index 6b4b260e..1d6d62dd 100644
--- a/src/Cache_p.h
+++ b/src/Cache_p.h
@@ -179,8 +179,8 @@ public:
};
Messages getTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
- int64_t index = std::numeric_limits<int64_t>::max(),
- bool forward = false);
+ uint64_t index = std::numeric_limits<uint64_t>::max(),
+ bool forward = false);
std::optional<mtx::events::collections::TimelineEvent> getEvent(
const std::string &room_id,
@@ -190,12 +190,15 @@ public:
const mtx::events::collections::TimelineEvent &event);
struct TimelineRange
{
- int64_t first, last;
+ uint64_t first, last;
};
std::optional<TimelineRange> getTimelineRange(const std::string &room_id);
- std::optional<int64_t> getTimelineIndex(const std::string &room_id,
- std::string_view event_id);
- std::optional<std::string> getTimelineEventId(const std::string &room_id, int64_t index);
+ std::optional<uint64_t> getTimelineIndex(const std::string &room_id,
+ std::string_view event_id);
+ std::optional<std::string> getTimelineEventId(const std::string &room_id, uint64_t index);
+
+ std::string previousBatchToken(const std::string &room_id);
+ uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res);
//! Remove old unused data.
void deleteOldMessages();
diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp
index 3ef28c86..666912ee 100644
--- a/src/ChatPage.cpp
+++ b/src/ChatPage.cpp
@@ -795,43 +795,39 @@ ChatPage::loadStateFromCache()
nhlog::db()->info("restoring state from cache");
- QtConcurrent::run([this]() {
- try {
- cache::restoreSessions();
- olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
+ try {
+ cache::restoreSessions();
+ olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
- cache::populateMembers();
+ cache::populateMembers();
- emit initializeEmptyViews(cache::roomMessages());
- emit initializeRoomList(cache::roomInfo());
- emit initializeMentions(cache::getTimelineMentions());
- emit syncTags(cache::roomInfo().toStdMap());
+ emit initializeEmptyViews(cache::roomMessages());
+ emit initializeRoomList(cache::roomInfo());
+ emit initializeMentions(cache::getTimelineMentions());
+ emit syncTags(cache::roomInfo().toStdMap());
- cache::calculateRoomReadStatus();
+ cache::calculateRoomReadStatus();
- } catch (const mtx::crypto::olm_exception &e) {
- nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
- emit dropToLoginPageCb(
- tr("Failed to restore OLM account. Please login again."));
- return;
- } catch (const lmdb::error &e) {
- nhlog::db()->critical("failed to restore cache: {}", e.what());
- emit dropToLoginPageCb(
- tr("Failed to restore save data. Please login again."));
- return;
- } catch (const json::exception &e) {
- nhlog::db()->critical("failed to parse cache data: {}", e.what());
- return;
- }
+ } catch (const mtx::crypto::olm_exception &e) {
+ nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
+ emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again."));
+ return;
+ } catch (const lmdb::error &e) {
+ nhlog::db()->critical("failed to restore cache: {}", e.what());
+ emit dropToLoginPageCb(tr("Failed to restore save data. Please login again."));
+ return;
+ } catch (const json::exception &e) {
+ nhlog::db()->critical("failed to parse cache data: {}", e.what());
+ return;
+ }
- nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
- nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519);
+ nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
+ nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519);
- getProfileInfo();
+ getProfileInfo();
- // Start receiving events.
- emit trySyncCb();
- });
+ // Start receiving events.
+ emit trySyncCb();
}
void
diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index 719743fb..7f21e1ed 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *)
cache::client()->storeEvent(room_id_, id, {timeline});
if (!relatedTo.empty()) {
- auto idx = idToIndex(id);
+ auto idx = idToIndex(relatedTo);
if (idx)
emit dataChanged(*idx, *idx);
}
},
Qt::QueuedConnection);
+
+ connect(
+ this,
+ &EventStore::oldMessagesRetrieved,
+ this,
+ [this](const mtx::responses::Messages &res) {
+ //
+ uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res);
+ if (newFirst == first)
+ fetchMore();
+ else {
+ emit beginInsertRows(toExternalIdx(newFirst),
+ toExternalIdx(this->first - 1));
+ this->first = newFirst;
+ emit endInsertRows();
+ emit fetchedMore();
+ }
+ },
+ Qt::QueuedConnection);
}
void
@@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events)
nhlog::db()->warn("{} called from a different thread!", __func__);
auto range = cache::client()->getTimelineRange(room_id_);
+ if (!range)
+ return;
+
+ if (events.limited) {
+ emit beginResetModel();
+ this->last = range->last;
+ this->first = range->first;
+ emit endResetModel();
- if (range && range->last > this->last) {
+ } else if (range->last > this->last) {
emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last));
this->last = range->last;
emit endInsertRows();
@@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt
return event_ptr;
}
+
+void
+EventStore::fetchMore()
+{
+ mtx::http::MessagesOpts opts;
+ opts.room_id = room_id_;
+ opts.from = cache::client()->previousBatchToken(room_id_);
+
+ nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from);
+
+ http::client()->messages(
+ opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
+ if (err) {
+ nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
+ opts.room_id,
+ mtx::errors::to_string(err->matrix_error.errcode),
+ err->matrix_error.error,
+ err->parse_error);
+ return;
+ }
+
+ emit oldMessagesRetrieved(std::move(res));
+ });
+}
diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h
index 83c8f7a4..f2997245 100644
--- a/src/timeline/EventStore.h
+++ b/src/timeline/EventStore.h
@@ -8,6 +8,7 @@
#include <qhashfunctions.h>
#include <mtx/events/collections.hpp>
+#include <mtx/responses/messages.hpp>
#include <mtx/responses/sync.hpp>
class EventStore : public QObject
@@ -20,7 +21,7 @@ public:
struct Index
{
std::string room;
- int64_t idx;
+ uint64_t idx;
friend uint qHash(const Index &i, uint seed = 0) noexcept
{
@@ -66,12 +67,12 @@ public:
int size() const
{
- return last != std::numeric_limits<int64_t>::max()
+ return last != std::numeric_limits<uint64_t>::max()
? static_cast<int>(last - first) + 1
: 0;
}
- int toExternalIdx(int64_t idx) const { return static_cast<int>(idx - first); }
- int64_t toInternalIdx(int idx) const { return first + idx; }
+ int toExternalIdx(uint64_t idx) const { return static_cast<int>(idx - first); }
+ uint64_t toInternalIdx(int idx) const { return first + idx; }
std::optional<int> idToIndex(std::string_view id) const;
std::optional<std::string> indexToId(int idx) const;
@@ -79,11 +80,15 @@ public:
signals:
void beginInsertRows(int from, int to);
void endInsertRows();
+ void beginResetModel();
+ void endResetModel();
void dataChanged(int from, int to);
void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo);
void eventFetched(std::string id,
std::string relatedTo,
mtx::events::collections::TimelineEvents timeline);
+ void oldMessagesRetrieved(const mtx::responses::Messages &);
+ void fetchedMore();
private:
mtx::events::collections::TimelineEvents *decryptEvent(
@@ -92,8 +97,8 @@ private:
std::string room_id_;
- int64_t first = std::numeric_limits<int64_t>::max(),
- last = std::numeric_limits<int64_t>::max();
+ uint64_t first = std::numeric_limits<uint64_t>::max(),
+ last = std::numeric_limits<uint64_t>::max();
static QCache<IdIndex, mtx::events::collections::TimelineEvents> decryptedEvents_;
static QCache<Index, mtx::events::collections::TimelineEvents> events_;
diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp
index 6df92d7a..60264e86 100644
--- a/src/timeline/TimelineModel.cpp
+++ b/src/timeline/TimelineModel.cpp
@@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
&EventStore::dataChanged,
this,
[this](int from, int to) {
- emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0));
+ nhlog::ui()->debug(
+ "data changed {} to {}", events.size() - to - 1, events.size() - from - 1);
+ emit dataChanged(index(events.size() - to - 1, 0),
+ index(events.size() - from - 1, 0));
},
Qt::QueuedConnection);
connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) {
- nhlog::ui()->info("begin insert from {} to {}",
- events.size() - to + (to - from),
- events.size() - from + (to - from));
- beginInsertRows(QModelIndex(),
- events.size() - to + (to - from),
- events.size() - from + (to - from));
+ int first = events.size() - to;
+ int last = events.size() - from;
+ if (from >= events.size()) {
+ int batch_size = to - from;
+ first += batch_size;
+ last += batch_size;
+ } else {
+ first -= 1;
+ last -= 1;
+ }
+ nhlog::ui()->debug("begin insert from {} to {}", first, last);
+ beginInsertRows(QModelIndex(), first, last);
});
connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); });
+ connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); });
+ connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); });
connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage);
+ connect(
+ &events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); });
}
QHash<int, QByteArray>
@@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const
{
if (!events.size())
return true;
- if (!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(
- *events.event(0)))
+ if (auto first = events.event(0);
+ first &&
+ !std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(*first))
return true;
else
@@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &)
}
setPaginationInProgress(true);
- mtx::http::MessagesOpts opts;
- opts.room_id = room_id_.toStdString();
- opts.from = prev_batch_token_.toStdString();
- nhlog::ui()->debug("Paginating room {}", opts.room_id);
-
- http::client()->messages(
- opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
- if (err) {
- nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
- opts.room_id,
- mtx::errors::to_string(err->matrix_error.errcode),
- err->matrix_error.error,
- err->parse_error);
- setPaginationInProgress(false);
- return;
- }
-
- emit oldMessagesRetrieved(std::move(res));
- setPaginationInProgress(false);
- });
+ events.fetchMore();
}
void
|
