From 3b488242050bbc0521d846bd31cb6ea59b8d4e38 Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 5 Aug 2024 06:49:58 +0200 Subject: Sync storage --- ArcaneLibs | 2 +- LibMatrix/Helpers/SyncStateResolver.cs | 157 +++++++++++++++++++++------------ LibMatrix/Responses/SyncResponse.cs | 14 +-- 3 files changed, 111 insertions(+), 62 deletions(-) diff --git a/ArcaneLibs b/ArcaneLibs index 26b02bc..4d32676 160000 --- a/ArcaneLibs +++ b/ArcaneLibs @@ -1 +1 @@ -Subproject commit 26b02bc5459f33d3b9b6bd2e4dda558cb8ac2e9b +Subproject commit 4d32676b97001f6988d1e62e53fbc837ef3fefc3 diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs index 0529e50..fcb23c2 100644 --- a/LibMatrix/Helpers/SyncStateResolver.cs +++ b/LibMatrix/Helpers/SyncStateResolver.cs @@ -1,12 +1,14 @@ +using ArcaneLibs.Extensions; using LibMatrix.Extensions; using LibMatrix.Filters; using LibMatrix.Homeservers; +using LibMatrix.Interfaces.Services; using LibMatrix.Responses; using Microsoft.Extensions.Logging; namespace LibMatrix.Helpers; -public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null) { +public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null, IStorageProvider? storageProvider = null) { public string? Since { get; set; } public int Timeout { get; set; } = 30000; public string? SetPresence { get; set; } = "online"; @@ -24,57 +26,100 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge _syncHelper.SetPresence = SetPresence; _syncHelper.Filter = Filter; _syncHelper.FullState = FullState; - // run sync - var sync = await _syncHelper.SyncAsync(cancellationToken); + // run sync or grab from storage if available + var sync = storageProvider != null && await storageProvider.ObjectExistsAsync(Since ?? "init") + ? await storageProvider.LoadObjectAsync(Since ?? "init") + : await _syncHelper.SyncAsync(cancellationToken); if (sync is null) return await ContinueAsync(cancellationToken); + + if (storageProvider != null && !await storageProvider.ObjectExistsAsync(Since ?? "init")) + await storageProvider.SaveObjectAsync(Since ?? "init", sync); + if (MergedState is null) MergedState = sync; else MergedState = MergeSyncs(MergedState, sync); Since = sync.NextBatch; + return (sync, MergedState); } - private SyncResponse MergeSyncs(SyncResponse oldState, SyncResponse newState) { - oldState.NextBatch = newState.NextBatch ?? oldState.NextBatch; - - oldState.AccountData ??= new EventList(); - oldState.AccountData.Events ??= []; - if (newState.AccountData?.Events is not null) - oldState.AccountData.Events.MergeStateEventLists(newState.AccountData?.Events ?? new List()); - - oldState.Presence ??= new SyncResponse.PresenceDataStructure(); - if (newState.Presence?.Events is not null) - oldState.Presence.Events.MergeStateEventLists(newState.Presence?.Events ?? new List()); - - oldState.DeviceOneTimeKeysCount ??= new Dictionary(); - if (newState.DeviceOneTimeKeysCount is not null) - foreach (var (key, value) in newState.DeviceOneTimeKeysCount) - oldState.DeviceOneTimeKeysCount[key] = value; - - oldState.Rooms ??= new SyncResponse.RoomsDataStructure(); - if (newState.Rooms is not null) - oldState.Rooms = MergeRoomsDataStructure(oldState.Rooms, newState.Rooms); - - oldState.ToDevice ??= new EventList(); - oldState.ToDevice.Events ??= []; - if (newState.ToDevice?.Events is not null) - oldState.ToDevice.Events.MergeStateEventLists(newState.ToDevice?.Events ?? new List()); - - oldState.DeviceLists ??= new SyncResponse.DeviceListsDataStructure(); - oldState.DeviceLists.Changed ??= []; - oldState.DeviceLists.Left ??= []; - if (newState.DeviceLists?.Changed is not null) - foreach (var s in newState.DeviceLists.Changed!) - oldState.DeviceLists.Changed.Add(s); - if (newState.DeviceLists?.Left is not null) - foreach (var s in newState.DeviceLists.Left!) - oldState.DeviceLists.Left.Add(s); + public async Task OptimiseStore() { + if (storageProvider is null) return; - return oldState; + var keys = await storageProvider.GetAllKeysAsync(); + var count = keys.Count - 2; + var merged = await storageProvider.LoadObjectAsync("init"); + if (merged is null) return; + + while (keys.Contains(merged.NextBatch)) { + var next = await storageProvider.LoadObjectAsync(merged.NextBatch); + if (next is null) break; + merged = MergeSyncs(merged, next); + Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining..."); + } + + await storageProvider.SaveObjectAsync("merged", merged); + + Environment.Exit(0); + } + + private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync) { + oldSync.NextBatch = newSync.NextBatch ?? oldSync.NextBatch; + + oldSync.AccountData ??= new EventList(); + oldSync.AccountData.Events ??= []; + if (newSync.AccountData?.Events is not null) + oldSync.AccountData.Events.MergeStateEventLists(newSync.AccountData?.Events ?? []); + + oldSync.Presence ??= new(); + oldSync.Presence.Events?.ReplaceBy(newSync.Presence?.Events ?? [], (oldState, newState) => oldState.Sender == newState.Sender && oldState.Type == newState.Type); + + oldSync.DeviceOneTimeKeysCount ??= new(); + if (newSync.DeviceOneTimeKeysCount is not null) + foreach (var (key, value) in newSync.DeviceOneTimeKeysCount) + oldSync.DeviceOneTimeKeysCount[key] = value; + + if (newSync.Rooms is not null) + oldSync.Rooms = MergeRoomsDataStructure(oldSync.Rooms, newSync.Rooms); + + oldSync.ToDevice ??= new EventList(); + oldSync.ToDevice.Events ??= []; + if (newSync.ToDevice?.Events is not null) + oldSync.ToDevice.Events.MergeStateEventLists(newSync.ToDevice?.Events ?? []); + + oldSync.DeviceLists ??= new SyncResponse.DeviceListsDataStructure(); + oldSync.DeviceLists.Changed ??= []; + oldSync.DeviceLists.Left ??= []; + if (newSync.DeviceLists?.Changed is not null) + foreach (var s in newSync.DeviceLists.Changed!) { + oldSync.DeviceLists.Left.Remove(s); + oldSync.DeviceLists.Changed.Add(s); + } + + if (newSync.DeviceLists?.Left is not null) + foreach (var s in newSync.DeviceLists.Left!) { + oldSync.DeviceLists.Changed.Remove(s); + oldSync.DeviceLists.Left.Add(s); + } + + return oldSync; + } + + private List? MergePresenceEvents(List? oldEvents, List? newEvents) { + if (oldEvents is null) return newEvents; + if (newEvents is null) return oldEvents; + + foreach (var newEvent in newEvents) { + oldEvents.RemoveAll(x => x.Sender == newEvent.Sender && x.Type == newEvent.Type); + oldEvents.Add(newEvent); + } + + return oldEvents; } #region Merge rooms - private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure oldState, SyncResponse.RoomsDataStructure newState) { + private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure? oldState, SyncResponse.RoomsDataStructure newState) { + if (oldState is null) return newState; oldState.Join ??= new Dictionary(); foreach (var (key, value) in newState.Join ?? new Dictionary()) if (!oldState.Join.ContainsKey(key)) oldState.Join[key] = value; @@ -99,22 +144,22 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge private SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData, SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData) { oldData.AccountData ??= new EventList(); - oldData.AccountData.Events ??= new List(); + oldData.AccountData.Events ??= []; oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure(); - oldData.Timeline.Events ??= new List(); + oldData.Timeline.Events ??= []; oldData.State ??= new EventList(); - oldData.State.Events ??= new List(); + oldData.State.Events ??= []; if (newData.AccountData?.Events is not null) - oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List()); + oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []); if (newData.Timeline?.Events is not null) - oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List()); + oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []); oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited; oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch; if (newData.State?.Events is not null) - oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List()); + oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []); return oldData; } @@ -122,9 +167,9 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge private SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData, SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData) { oldData.InviteState ??= new EventList(); - oldData.InviteState.Events ??= new List(); + oldData.InviteState.Events ??= []; if (newData.InviteState?.Events is not null) - oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? new List()); + oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? []); return oldData; } @@ -132,27 +177,27 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge private SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData) { oldData.AccountData ??= new EventList(); - oldData.AccountData.Events ??= new List(); + oldData.AccountData.Events ??= []; oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure(); - oldData.Timeline.Events ??= new List(); + oldData.Timeline.Events ??= []; oldData.State ??= new EventList(); - oldData.State.Events ??= new List(); + oldData.State.Events ??= []; oldData.Ephemeral ??= new EventList(); - oldData.Ephemeral.Events ??= new List(); + oldData.Ephemeral.Events ??= []; if (newData.AccountData?.Events is not null) - oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? new List()); + oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []); if (newData.Timeline?.Events is not null) - oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? new List()); + oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []); oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited; oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch; if (newData.State?.Events is not null) - oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? new List()); + oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []); if (newData.Ephemeral?.Events is not null) - oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? new List()); + oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? []); oldData.UnreadNotifications ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.UnreadNotificationsDataStructure(); oldData.UnreadNotifications.HighlightCount = newData.UnreadNotifications?.HighlightCount ?? oldData.UnreadNotifications.HighlightCount; diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs index e4addb6..d807ecb 100644 --- a/LibMatrix/Responses/SyncResponse.cs +++ b/LibMatrix/Responses/SyncResponse.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using LibMatrix.EventTypes.Spec.State; namespace LibMatrix.Responses; @@ -14,7 +15,7 @@ public class SyncResponse { public EventList? AccountData { get; set; } [JsonPropertyName("presence")] - public PresenceDataStructure? Presence { get; set; } + public EventList? Presence { get; set; } [JsonPropertyName("device_one_time_keys_count")] public Dictionary? DeviceOneTimeKeysCount { get; set; } = null!; @@ -37,10 +38,6 @@ public class SyncResponse { } // supporting classes - public class PresenceDataStructure { - [JsonPropertyName("events")] - public List Events { get; set; } = new(); - } public class RoomsDataStructure { [JsonPropertyName("join")] @@ -61,6 +58,13 @@ public class SyncResponse { [JsonPropertyName("state")] public EventList? State { get; set; } + + public override string ToString() { + var lastEvent = Timeline?.Events?.LastOrDefault(x=>x.Type == "m.room.member"); + var membership = (lastEvent?.TypedContent as RoomMemberEventContent); + return $"LeftRoomDataStructure: {lastEvent?.Sender} {membership?.Membership} ({membership?.Reason})"; + + } } public class JoinedRoomDataStructure { -- cgit 1.4.1