Sync storage

3 files changed, 110 insertions, 61 deletions

diff --git a/ArcaneLibs b/ArcaneLibs
-Subproject 26b02bc5459f33d3b9b6bd2e4dda558cb8ac2e9
+Subproject 4d32676b97001f6988d1e62e53fbc837ef3fefc
diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs
index 0529e50..fcb23c2 100644
--- a/LibMatrix/Helpers/SyncStateResolver.cs
+++ b/LibMatrix/Helpers/SyncStateResolver.cs
@@ -1,12 +1,14 @@
+using ArcaneLibs.Extensions;
 using LibMatrix.Extensions;
 using LibMatrix.Filters;
 using LibMatrix.Homeservers;
+using LibMatrix.Interfaces.Services;
 using LibMatrix.Responses;
 using Microsoft.Extensions.Logging;
 
 namespace LibMatrix.Helpers;
 
-public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null) {
+public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null, IStorageProvider? storageProvider = null) {
     public string? Since { get; set; }
     public int Timeout { get; set; } = 30000;
     public string? SetPresence { get; set; } = "online";
@@ -24,57 +26,100 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
         _syncHelper.SetPresence = SetPresence;
         _syncHelper.Filter = Filter;
         _syncHelper.FullState = FullState;
-        // run sync
-        var sync = await _syncHelper.SyncAsync(cancellationToken);
+        // run sync or grab from storage if available
+        var sync = storageProvider != null && await storageProvider.ObjectExistsAsync(Since ?? "init")
+            ? await storageProvider.LoadObjectAsync<SyncResponse>(Since ?? "init")
+            : await _syncHelper.SyncAsync(cancellationToken);
         if (sync is null) return await ContinueAsync(cancellationToken);
+
+        if (storageProvider != null && !await storageProvider.ObjectExistsAsync(Since ?? "init"))
+            await storageProvider.SaveObjectAsync(Since ?? "init", sync);
+
         if (MergedState is null) MergedState = sync;
         else MergedState = MergeSyncs(MergedState, sync);
         Since = sync.NextBatch;
+
         return (sync, MergedState);
     }
 
-    private SyncResponse MergeSyncs(SyncResponse oldState, SyncResponse newState) {
-        oldState.NextBatch = newState.NextBatch ?? oldState.NextBatch;
-
-        oldState.AccountData ??= new EventList();
-        oldState.AccountData.Events ??= [];
-        if (newState.AccountData?.Events is not null)
-            oldState.AccountData.Events.MergeStateEventLists(newState.AccountData?.Events ?? new List<StateEventResponse>());
-
-        oldState.Presence ??= new SyncResponse.PresenceDataStructure();
-        if (newState.Presence?.Events is not null)
-            oldState.Presence.Events.MergeStateEventLists(newState.Presence?.Events ?? new List<StateEventResponse>());
-
-        oldState.DeviceOneTimeKeysCount ??= new Dictionary<string, int>();
-        if (newState.DeviceOneTimeKeysCount is not null)
-            foreach (var (key, value) in newState.DeviceOneTimeKeysCount)
-                oldState.DeviceOneTimeKeysCount[key] = value;
-
-        oldState.Rooms ??= new SyncResponse.RoomsDataStructure();
-        if (newState.Rooms is not null)
-            oldState.Rooms = MergeRoomsDataStructure(oldState.Rooms, newState.Rooms);
-
-        oldState.ToDevice ??= new EventList();
-        oldState.ToDevice.Events ??= [];
-        if (newState.ToDevice?.Events is not null)
-            oldState.ToDevice.Events.MergeStateEventLists(newState.ToDevice?.Events ?? new List<StateEventResponse>());
-
-        oldState.DeviceLists ??= new SyncResponse.DeviceListsDataStructure();
-        oldState.DeviceLists.Changed ??= [];
-        oldState.DeviceLists.Left ??= [];
-        if (newState.DeviceLists?.Changed is not null)
-            foreach (var s in newState.DeviceLists.Changed!)
-                oldState.DeviceLists.Changed.Add(s);
-        if (newState.DeviceLists?.Left is not null)
-            foreach (var s in newState.DeviceLists.Left!)
-                oldState.DeviceLists.Left.Add(s);
+    public async Task OptimiseStore() {
+        if (storageProvider is null) return;
 
-        return oldState;
+        var keys = await storageProvider.GetAllKeysAsync();
+        var count = keys.Count - 2;
+        var merged = await storageProvider.LoadObjectAsync<SyncResponse>("init");
+        if (merged is null) return;
+
+        while (keys.Contains(merged.NextBatch)) {
+            var next = await storageProvider.LoadObjectAsync<SyncResponse>(merged.NextBatch);
+            if (next is null) break;
+            merged = MergeSyncs(merged, next);
+            Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining...");
+        }
+
+        await storageProvider.SaveObjectAsync("merged", merged);
+
+        Environment.Exit(0);
+    }
+
+    private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync) {
+        oldSync.NextBatch = newSync.NextBatch ?? oldSync.NextBatch;
+
+        oldSync.AccountData ??= new EventList();
+        oldSync.AccountData.Events ??= [];
+        if (newSync.AccountData?.Events is not null)
+            oldSync.AccountData.Events.MergeStateEventLists(newSync.AccountData?.Events ?? []);
+
+        oldSync.Presence ??= new();
+        oldSync.Presence.Events?.ReplaceBy(newSync.Presence?.Events ?? [], (oldState, newState) => oldState.Sender == newState.Sender && oldState.Type == newState.Type);
+
+        oldSync.DeviceOneTimeKeysCount ??= new();
+        if (newSync.DeviceOneTimeKeysCount is not null)
+            foreach (var (key, value) in newSync.DeviceOneTimeKeysCount)
+                oldSync.DeviceOneTimeKeysCount[key] = value;
+
+        if (newSync.Rooms is not null)
+            oldSync.Rooms = MergeRoomsDataStructure(oldSync.Rooms, newSync.Rooms);
+
+        oldSync.ToDevice ??= new EventList();
+        oldSync.ToDevice.Events ??= [];
+        if (newSync.ToDevice?.Events is not null)
+            oldSync.ToDevice.Events.MergeStateEventLists(newSync.ToDevice?.Events ?? []);
+
+        oldSync.DeviceLists ??= new SyncResponse.DeviceListsDataStructure();
+        oldSync.DeviceLists.Changed ??= [];
+        oldSync.DeviceLists.Left ??= [];
+        if (newSync.DeviceLists?.Changed is not null)
+            foreach (var s in newSync.DeviceLists.Changed!) {
+                oldSync.DeviceLists.Left.Remove(s);
+                oldSync.DeviceLists.Changed.Add(s);
+            }
+
+        if (newSync.DeviceLists?.Left is not null)
+            foreach (var s in newSync.DeviceLists.Left!) {
+                oldSync.DeviceLists.Changed.Remove(s);
+                oldSync.DeviceLists.Left.Add(s);
+            }
+
+        return oldSync;
+    }
+
+    private List<StateEventResponse>? MergePresenceEvents(List<StateEventResponse>? oldEvents, List<StateEventResponse>? newEvents) {
+        if (oldEvents is null) return newEvents;
+        if (newEvents is null) return oldEvents;
+
+        foreach (var newEvent in newEvents) {
+            oldEvents.RemoveAll(x => x.Sender == newEvent.Sender && x.Type == newEvent.Type);
+            oldEvents.Add(newEvent);
+        }
+
+        return oldEvents;
     }
 
 #region Merge rooms
 
-    private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure oldState, SyncResponse.RoomsDataStructure newState) {
+    private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure? oldState, SyncResponse.RoomsDataStructure newState) {
+        if (oldState is null) return newState;
         oldState.Join ??= new Dictionary<string, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure>();
         foreach (var (key, value) in newState.Join ?? new Dictionary<string, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure>())
             if (!oldState.Join.ContainsKey(key)) oldState.Join[key] = value;
@@ -99,22 +144,22 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
     private SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData,
         SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData) {
         oldData.AccountData ??= new EventList();
-        oldData.AccountData.Events ??= new List<StateEventResponse>();
+        oldData.AccountData.Events ??= [];
         oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure();
-        oldData.Timeline.Events ??= new List<StateEventResponse>();
+        oldData.Timeline.Events ??= [];
         oldData.State ??= new EventList();
-        oldData.State.Events ??= new List<StateEventResponse>();
+        oldData.State.Events ??= [];
 
         if (newData.AccountData?.Events is not null)
-            oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List<StateEventResponse>());
+            oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []);
 
         if (newData.Timeline?.Events is not null)
-            oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List<StateEventResponse>());
+            oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []);
         oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited;
         oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch;
 
         if (newData.State?.Events is not null)
-            oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List<StateEventResponse>());
+            oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []);
 
         return oldData;
     }
@@ -122,9 +167,9 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
     private SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData,
         SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData) {
         oldData.InviteState ??= new EventList();
-        oldData.InviteState.Events ??= new List<StateEventResponse>();
+        oldData.InviteState.Events ??= [];
         if (newData.InviteState?.Events is not null)
-            oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? new List<StateEventResponse>());
+            oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? []);
 
         return oldData;
     }
@@ -132,27 +177,27 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge
     private SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData,
         SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData) {
         oldData.AccountData ??= new EventList();
-        oldData.AccountData.Events ??= new List<StateEventResponse>();
+        oldData.AccountData.Events ??= [];
         oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure();
-        oldData.Timeline.Events ??= new List<StateEventResponse>();
+        oldData.Timeline.Events ??= [];
         oldData.State ??= new EventList();
-        oldData.State.Events ??= new List<StateEventResponse>();
+        oldData.State.Events ??= [];
         oldData.Ephemeral ??= new EventList();
-        oldData.Ephemeral.Events ??= new List<StateEventResponse>();
+        oldData.Ephemeral.Events ??= [];
 
         if (newData.AccountData?.Events is not null)
-            oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List<StateEventResponse>());
+            oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []);
 
         if (newData.Timeline?.Events is not null)
-            oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List<StateEventResponse>());
+            oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []);
         oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited;
         oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch;
 
         if (newData.State?.Events is not null)
-            oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List<StateEventResponse>());
+            oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []);
 
         if (newData.Ephemeral?.Events is not null)
-            oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? new List<StateEventResponse>());
+            oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? []);
 
         oldData.UnreadNotifications ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.UnreadNotificationsDataStructure();
         oldData.UnreadNotifications.HighlightCount = newData.UnreadNotifications?.HighlightCount ?? oldData.UnreadNotifications.HighlightCount;
diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs
index e4addb6..d807ecb 100644
--- a/LibMatrix/Responses/SyncResponse.cs
+++ b/LibMatrix/Responses/SyncResponse.cs
@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using LibMatrix.EventTypes.Spec.State;
 
 namespace LibMatrix.Responses;
 
@@ -14,7 +15,7 @@ public class SyncResponse {
     public EventList? AccountData { get; set; }
 
     [JsonPropertyName("presence")]
-    public PresenceDataStructure? Presence { get; set; }
+    public EventList? Presence { get; set; }
 
     [JsonPropertyName("device_one_time_keys_count")]
     public Dictionary<string, int>? DeviceOneTimeKeysCount { get; set; } = null!;
@@ -37,10 +38,6 @@ public class SyncResponse {
     }
 
     // supporting classes
-    public class PresenceDataStructure {
-        [JsonPropertyName("events")]
-        public List<StateEventResponse> Events { get; set; } = new();
-    }
 
     public class RoomsDataStructure {
         [JsonPropertyName("join")]
@@ -61,6 +58,13 @@ public class SyncResponse {
 
             [JsonPropertyName("state")]
             public EventList? State { get; set; }
+
+            public override string ToString() {
+                var lastEvent = Timeline?.Events?.LastOrDefault(x=>x.Type == "m.room.member");
+                var membership = (lastEvent?.TypedContent as RoomMemberEventContent);
+                return $"LeftRoomDataStructure: {lastEvent?.Sender} {membership?.Membership} ({membership?.Reason})";
+                
+            }
         }
 
         public class JoinedRoomDataStructure {