diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs
index 0529e50..fcb23c2 100644
--- a/LibMatrix/Helpers/SyncStateResolver.cs
+++ b/LibMatrix/Helpers/SyncStateResolver.cs
@@ -1,12 +1,14 @@
+using ArcaneLibs.Extensions;
using LibMatrix.Extensions;
using LibMatrix.Filters;
using LibMatrix.Homeservers;
+using LibMatrix.Interfaces.Services;
using LibMatrix.Responses;
using Microsoft.Extensions.Logging;
namespace LibMatrix.Helpers;
-public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null) {
+public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null, IStorageProvider? storageProvider = null) {
public string? Since { get; set; }
public int Timeout { get; set; } = 30000;
public string? SetPresence { get; set; } = "online";
@@ -24,57 +26,100 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
_syncHelper.SetPresence = SetPresence;
_syncHelper.Filter = Filter;
_syncHelper.FullState = FullState;
- // run sync
- var sync = await _syncHelper.SyncAsync(cancellationToken);
+ // run sync or grab from storage if available
+ var sync = storageProvider != null && await storageProvider.ObjectExistsAsync(Since ?? "init")
+ ? await storageProvider.LoadObjectAsync<SyncResponse>(Since ?? "init")
+ : await _syncHelper.SyncAsync(cancellationToken);
if (sync is null) return await ContinueAsync(cancellationToken);
+
+ if (storageProvider != null && !await storageProvider.ObjectExistsAsync(Since ?? "init"))
+ await storageProvider.SaveObjectAsync(Since ?? "init", sync);
+
if (MergedState is null) MergedState = sync;
else MergedState = MergeSyncs(MergedState, sync);
Since = sync.NextBatch;
+
return (sync, MergedState);
}
- private SyncResponse MergeSyncs(SyncResponse oldState, SyncResponse newState) {
- oldState.NextBatch = newState.NextBatch ?? oldState.NextBatch;
-
- oldState.AccountData ??= new EventList();
- oldState.AccountData.Events ??= [];
- if (newState.AccountData?.Events is not null)
- oldState.AccountData.Events.MergeStateEventLists(newState.AccountData?.Events ?? new List<StateEventResponse>());
-
- oldState.Presence ??= new SyncResponse.PresenceDataStructure();
- if (newState.Presence?.Events is not null)
- oldState.Presence.Events.MergeStateEventLists(newState.Presence?.Events ?? new List<StateEventResponse>());
-
- oldState.DeviceOneTimeKeysCount ??= new Dictionary<string, int>();
- if (newState.DeviceOneTimeKeysCount is not null)
- foreach (var (key, value) in newState.DeviceOneTimeKeysCount)
- oldState.DeviceOneTimeKeysCount[key] = value;
-
- oldState.Rooms ??= new SyncResponse.RoomsDataStructure();
- if (newState.Rooms is not null)
- oldState.Rooms = MergeRoomsDataStructure(oldState.Rooms, newState.Rooms);
-
- oldState.ToDevice ??= new EventList();
- oldState.ToDevice.Events ??= [];
- if (newState.ToDevice?.Events is not null)
- oldState.ToDevice.Events.MergeStateEventLists(newState.ToDevice?.Events ?? new List<StateEventResponse>());
-
- oldState.DeviceLists ??= new SyncResponse.DeviceListsDataStructure();
- oldState.DeviceLists.Changed ??= [];
- oldState.DeviceLists.Left ??= [];
- if (newState.DeviceLists?.Changed is not null)
- foreach (var s in newState.DeviceLists.Changed!)
- oldState.DeviceLists.Changed.Add(s);
- if (newState.DeviceLists?.Left is not null)
- foreach (var s in newState.DeviceLists.Left!)
- oldState.DeviceLists.Left.Add(s);
+ public async Task OptimiseStore() {
+ if (storageProvider is null) return;
- return oldState;
+ var keys = await storageProvider.GetAllKeysAsync();
+ var count = keys.Count - 2;
+ var merged = await storageProvider.LoadObjectAsync<SyncResponse>("init");
+ if (merged is null) return;
+
+ while (keys.Contains(merged.NextBatch)) {
+ var next = await storageProvider.LoadObjectAsync<SyncResponse>(merged.NextBatch);
+ if (next is null) break;
+ merged = MergeSyncs(merged, next);
+ Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining...");
+ }
+
+ await storageProvider.SaveObjectAsync("merged", merged);
+
+ Environment.Exit(0);
+ }
+
+ private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync) {
+ oldSync.NextBatch = newSync.NextBatch ?? oldSync.NextBatch;
+
+ oldSync.AccountData ??= new EventList();
+ oldSync.AccountData.Events ??= [];
+ if (newSync.AccountData?.Events is not null)
+ oldSync.AccountData.Events.MergeStateEventLists(newSync.AccountData?.Events ?? []);
+
+ oldSync.Presence ??= new();
+ oldSync.Presence.Events?.ReplaceBy(newSync.Presence?.Events ?? [], (oldState, newState) => oldState.Sender == newState.Sender && oldState.Type == newState.Type);
+
+ oldSync.DeviceOneTimeKeysCount ??= new();
+ if (newSync.DeviceOneTimeKeysCount is not null)
+ foreach (var (key, value) in newSync.DeviceOneTimeKeysCount)
+ oldSync.DeviceOneTimeKeysCount[key] = value;
+
+ if (newSync.Rooms is not null)
+ oldSync.Rooms = MergeRoomsDataStructure(oldSync.Rooms, newSync.Rooms);
+
+ oldSync.ToDevice ??= new EventList();
+ oldSync.ToDevice.Events ??= [];
+ if (newSync.ToDevice?.Events is not null)
+ oldSync.ToDevice.Events.MergeStateEventLists(newSync.ToDevice?.Events ?? []);
+
+ oldSync.DeviceLists ??= new SyncResponse.DeviceListsDataStructure();
+ oldSync.DeviceLists.Changed ??= [];
+ oldSync.DeviceLists.Left ??= [];
+ if (newSync.DeviceLists?.Changed is not null)
+ foreach (var s in newSync.DeviceLists.Changed!) {
+ oldSync.DeviceLists.Left.Remove(s);
+ oldSync.DeviceLists.Changed.Add(s);
+ }
+
+ if (newSync.DeviceLists?.Left is not null)
+ foreach (var s in newSync.DeviceLists.Left!) {
+ oldSync.DeviceLists.Changed.Remove(s);
+ oldSync.DeviceLists.Left.Add(s);
+ }
+
+ return oldSync;
+ }
+
+ private List<StateEventResponse>? MergePresenceEvents(List<StateEventResponse>? oldEvents, List<StateEventResponse>? newEvents) {
+ if (oldEvents is null) return newEvents;
+ if (newEvents is null) return oldEvents;
+
+ foreach (var newEvent in newEvents) {
+ oldEvents.RemoveAll(x => x.Sender == newEvent.Sender && x.Type == newEvent.Type);
+ oldEvents.Add(newEvent);
+ }
+
+ return oldEvents;
}
#region Merge rooms
- private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure oldState, SyncResponse.RoomsDataStructure newState) {
+ private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure? oldState, SyncResponse.RoomsDataStructure newState) {
+ if (oldState is null) return newState;
oldState.Join ??= new Dictionary<string, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure>();
foreach (var (key, value) in newState.Join ?? new Dictionary<string, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure>())
if (!oldState.Join.ContainsKey(key)) oldState.Join[key] = value;
@@ -99,22 +144,22 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
private SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData,
SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData) {
oldData.AccountData ??= new EventList();
- oldData.AccountData.Events ??= new List<StateEventResponse>();
+ oldData.AccountData.Events ??= [];
oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure();
- oldData.Timeline.Events ??= new List<StateEventResponse>();
+ oldData.Timeline.Events ??= [];
oldData.State ??= new EventList();
- oldData.State.Events ??= new List<StateEventResponse>();
+ oldData.State.Events ??= [];
if (newData.AccountData?.Events is not null)
- oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List<StateEventResponse>());
+ oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []);
if (newData.Timeline?.Events is not null)
- oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List<StateEventResponse>());
+ oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []);
oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited;
oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch;
if (newData.State?.Events is not null)
- oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List<StateEventResponse>());
+ oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []);
return oldData;
}
@@ -122,9 +167,9 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
private SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData,
SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData) {
oldData.InviteState ??= new EventList();
- oldData.InviteState.Events ??= new List<StateEventResponse>();
+ oldData.InviteState.Events ??= [];
if (newData.InviteState?.Events is not null)
- oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? new List<StateEventResponse>());
+ oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? []);
return oldData;
}
@@ -132,27 +177,27 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
private SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData,
SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData) {
oldData.AccountData ??= new EventList();
- oldData.AccountData.Events ??= new List<StateEventResponse>();
+ oldData.AccountData.Events ??= [];
oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure();
- oldData.Timeline.Events ??= new List<StateEventResponse>();
+ oldData.Timeline.Events ??= [];
oldData.State ??= new EventList();
- oldData.State.Events ??= new List<StateEventResponse>();
+ oldData.State.Events ??= [];
oldData.Ephemeral ??= new EventList();
- oldData.Ephemeral.Events ??= new List<StateEventResponse>();
+ oldData.Ephemeral.Events ??= [];
if (newData.AccountData?.Events is not null)
- oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List<StateEventResponse>());
+ oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []);
if (newData.Timeline?.Events is not null)
- oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List<StateEventResponse>());
+ oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []);
oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited;
oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch;
if (newData.State?.Events is not null)
- oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List<StateEventResponse>());
+ oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []);
if (newData.Ephemeral?.Events is not null)
- oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? new List<StateEventResponse>());
+ oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? []);
oldData.UnreadNotifications ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.UnreadNotificationsDataStructure();
oldData.UnreadNotifications.HighlightCount = newData.UnreadNotifications?.HighlightCount ?? oldData.UnreadNotifications.HighlightCount;
diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs
index b2308c5..e7fe109 100644
--- a/LibMatrix/Responses/SyncResponse.cs
+++ b/LibMatrix/Responses/SyncResponse.cs
@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
+using LibMatrix.EventTypes.Spec.State;
namespace LibMatrix.Responses;
@@ -14,7 +15,7 @@ public class SyncResponse {
public EventList? AccountData { get; set; }
[JsonPropertyName("presence")]
- public PresenceDataStructure? Presence { get; set; }
+ public EventList? Presence { get; set; }
[JsonPropertyName("device_one_time_keys_count")]
public Dictionary<string, int>? DeviceOneTimeKeysCount { get; set; } = null!;
@@ -61,6 +62,13 @@ public class SyncResponse {
[JsonPropertyName("state")]
public EventList? State { get; set; }
+
+ public override string ToString() {
+ var lastEvent = Timeline?.Events?.LastOrDefault(x=>x.Type == "m.room.member");
+ var membership = (lastEvent?.TypedContent as RoomMemberEventContent);
+ return $"LeftRoomDataStructure: {lastEvent?.Sender} {membership?.Membership} ({membership?.Reason})";
+
+ }
}
public class JoinedRoomDataStructure {
@@ -125,4 +133,4 @@ public class SyncResponse {
public EventList? InviteState { get; set; }
}
}
-}
\ No newline at end of file
+}
|
