From 18c95766748432d3f5729072a3e8dd1495d8c1e0 Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 29 Jul 2024 22:44:44 +0200 Subject: Some cleanup, fixes --- LibMatrix/Extensions/MatrixHttpClient.Single.cs | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'LibMatrix/Extensions') diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs index 4145a16..b15804d 100644 --- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs +++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs @@ -1,5 +1,5 @@ #define SINGLE_HTTPCLIENT // Use a single HttpClient instance for all MatrixHttpClient instances -// #define SYNC_HTTPCLIENT // Only allow one request as a time, for debugging +// #define SYNC_HTTPCLIENT // Only allow one request as a time, for debugging using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Net; @@ -15,7 +15,7 @@ using ArcaneLibs.Extensions; namespace LibMatrix.Extensions; #if SINGLE_HTTPCLIENT -// TODO: Add URI wrapper for +// TODO: Add URI wrapper for public class MatrixHttpClient { private static readonly HttpClient Client; @@ -70,6 +70,10 @@ public class MatrixHttpClient { } public async Task SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) { + if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); + // if (!request.RequestUri.IsAbsoluteUri) + request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!); + var swWait = Stopwatch.StartNew(); #if SYNC_HTTPCLIENT await _rateLimitSemaphore.WaitAsync(cancellationToken); #endif @@ -86,13 +90,15 @@ public class MatrixHttpClient { request.Options.Set(new HttpRequestOptionsKey("WebAssemblyEnableStreamingResponse"), true); + Console.WriteLine("Sending " + request.Summarise(includeHeaders:true, includeQuery: true, includeContentIfText: true)); + HttpResponseMessage? responseMessage; try { responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); } catch (Exception e) { Console.WriteLine( - $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}"); + $"Failed to send request {request.Method} {request.RequestUri} ({Util.BytesToString(request.GetContentLength())}):\n{e}"); throw; } #if SYNC_HTTPCLIENT @@ -101,8 +107,20 @@ public class MatrixHttpClient { } #endif - Console.WriteLine( - $"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.Content.Headers.ContentLength ?? 0)})"); + // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)"); + Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [ + "Server", + "Date", + "Transfer-Encoding", + "Connection", + "Vary", + "Content-Length", + "Access-Control-Allow-Origin", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", + "Access-Control-Expose-Headers", + "Cache-Control" + ])); return responseMessage; } @@ -110,13 +128,13 @@ public class MatrixHttpClient { public async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) { var responseMessage = await SendUnhandledAsync(request, cancellationToken); if (responseMessage.IsSuccessStatusCode) return responseMessage; - + //retry on gateway timeout if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) { request.ResetSendStatus(); return await SendAsync(request, cancellationToken); } - + //error handling var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken); if (content.Length == 0) @@ -238,4 +256,4 @@ public class MatrixHttpClient { return await SendAsync(request, cancellationToken); } } -#endif \ No newline at end of file +#endif -- cgit 1.5.1 From 8130c84784cc4e696100fdda57c0d3b987ab4f80 Mon Sep 17 00:00:00 2001 From: Rory& Date: Fri, 9 Aug 2024 21:12:11 +0200 Subject: Minor cleanup --- LibMatrix/Extensions/MatrixHttpClient.Single.cs | 4 ++-- LibMatrix/Helpers/MessageBuilder.cs | 2 +- LibMatrix/Helpers/SyncStateResolver.cs | 6 +++--- LibMatrix/Services/HomeserverResolverService.cs | 10 +++++----- LibMatrix/StateEvent.cs | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) (limited to 'LibMatrix/Extensions') diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs index b15804d..771f41e 100644 --- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs +++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs @@ -60,7 +60,7 @@ public class MatrixHttpClient { typeof(HttpRequestHeaders).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, [], null)?.Invoke([]) as HttpRequestHeaders ?? throw new InvalidOperationException("Failed to create HttpRequestHeaders"); - private JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) { + private static JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) { options ??= new JsonSerializerOptions(); options.Converters.Add(new JsonFloatStringConverter()); options.Converters.Add(new JsonDoubleStringConverter()); @@ -237,7 +237,7 @@ public class MatrixHttpClient { await foreach (var resp in result) yield return resp; } - public async Task CheckSuccessStatus(string url) { + public static async Task CheckSuccessStatus(string url) { //cors causes failure, try to catch try { var resp = await Client.GetAsync(url); diff --git a/LibMatrix/Helpers/MessageBuilder.cs b/LibMatrix/Helpers/MessageBuilder.cs index b639e1f..d3bd6a5 100644 --- a/LibMatrix/Helpers/MessageBuilder.cs +++ b/LibMatrix/Helpers/MessageBuilder.cs @@ -105,7 +105,7 @@ public class MessageBuilder(string msgType = "m.text", string format = "org.matr public MessageBuilder WithTable(Action tableBuilder) { var tb = new TableBuilder(this); - this.WithHtmlTag("table", msb => tableBuilder(tb)); + WithHtmlTag("table", msb => tableBuilder(tb)); return this; } diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs index 0daccec..e2dbdee 100644 --- a/LibMatrix/Helpers/SyncStateResolver.cs +++ b/LibMatrix/Helpers/SyncStateResolver.cs @@ -142,7 +142,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge return oldState; } - private SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData, + private static SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData, SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData) { oldData.AccountData ??= new EventList(); oldData.AccountData.Events ??= []; @@ -165,7 +165,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge return oldData; } - private SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData, + private static SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData, SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData) { oldData.InviteState ??= new EventList(); oldData.InviteState.Events ??= []; @@ -175,7 +175,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge return oldData; } - private SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData, + private static SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData, SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData) { oldData.AccountData ??= new EventList(); oldData.AccountData.Events ??= []; diff --git a/LibMatrix/Services/HomeserverResolverService.cs b/LibMatrix/Services/HomeserverResolverService.cs index f899230..dc8e047 100644 --- a/LibMatrix/Services/HomeserverResolverService.cs +++ b/LibMatrix/Services/HomeserverResolverService.cs @@ -85,9 +85,9 @@ public class HomeserverResolverService { clientWellKnown ??= await _httpClient.TryGetFromJsonAsync($"http://{homeserver}/.well-known/matrix/client"); if (clientWellKnown is null) { - if (await _httpClient.CheckSuccessStatus($"https://{homeserver}/_matrix/client/versions")) + if (await MatrixHttpClient.CheckSuccessStatus($"https://{homeserver}/_matrix/client/versions")) return $"https://{homeserver}"; - if (await _httpClient.CheckSuccessStatus($"http://{homeserver}/_matrix/client/versions")) + if (await MatrixHttpClient.CheckSuccessStatus($"http://{homeserver}/_matrix/client/versions")) return $"http://{homeserver}"; } } @@ -122,16 +122,16 @@ public class HomeserverResolverService { var resolved = serverWellKnown.Homeserver; if (resolved.StartsWith("https://") || resolved.StartsWith("http://")) return resolved; - if (await _httpClient.CheckSuccessStatus($"https://{resolved}/_matrix/federation/v1/version")) + if (await MatrixHttpClient.CheckSuccessStatus($"https://{resolved}/_matrix/federation/v1/version")) return $"https://{resolved}"; - if (await _httpClient.CheckSuccessStatus($"http://{resolved}/_matrix/federation/v1/version")) + if (await MatrixHttpClient.CheckSuccessStatus($"http://{resolved}/_matrix/federation/v1/version")) return $"http://{resolved}"; _logger.LogWarning("Server well-known points to invalid server: {resolved}", resolved); } // fallback: most servers host C2S and S2S on the same domain var clientUrl = await _tryResolveClientEndpoint(homeserver); - if (clientUrl is not null && await _httpClient.CheckSuccessStatus($"{clientUrl}/_matrix/federation/v1/version")) + if (clientUrl is not null && await MatrixHttpClient.CheckSuccessStatus($"{clientUrl}/_matrix/federation/v1/version")) return clientUrl; _logger.LogInformation("No server well-known for {server}...", homeserver); diff --git a/LibMatrix/StateEvent.cs b/LibMatrix/StateEvent.cs index 0cd5662..8f99e10 100644 --- a/LibMatrix/StateEvent.cs +++ b/LibMatrix/StateEvent.cs @@ -283,4 +283,4 @@ public class ForgivingObjectConverter : JsonConverter where T : new() { public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) => JsonSerializer.Serialize(writer, value, options); -}*/ \ No newline at end of file +}*/ -- cgit 1.5.1 From db965b8bc3528814467713cb3467ac98da4e2df6 Mon Sep 17 00:00:00 2001 From: Rory& Date: Fri, 23 Aug 2024 02:55:07 +0200 Subject: Synapse admin API stuff, a mass of other changes --- LibMatrix/Extensions/MatrixHttpClient.Single.cs | 5 + LibMatrix/Helpers/SyncHelper.cs | 6 + LibMatrix/Helpers/SyncStateResolver.cs | 100 +++++++-- .../Models/Requests/AdminRoomDeleteRequest.cs | 23 -- .../SynapseAdminRegistrationTokenCreateRequest.cs | 31 +++ .../Requests/SynapseAdminRoomDeleteRequest.cs | 23 ++ .../Models/Responses/AdminRoomListResult.cs | 64 ------ .../Models/Responses/AdminUserListResult.cs | 58 ----- .../Synapse/Models/Responses/BackgroundUpdates.cs | 28 +++ .../Synapse/Models/Responses/Destinations.cs | 56 +++++ .../Models/Responses/EventReportListResult.cs | 169 ++++++++++++++ .../Responses/RegistrationTokenListResult.cs | 31 +++ .../Synapse/Models/Responses/RoomListResult.cs | 64 ++++++ .../Models/Responses/RoomMediaListResult.cs | 11 + .../Responses/SynapseAdminEventReportListResult.cs | 58 ----- .../Models/Responses/SynapseCollectionResult.cs | 250 +++++++++++++++++++++ .../Synapse/Models/Responses/UserListResult.cs | 71 ++++++ .../Synapse/SynapseAdminApiClient.cs | 195 +++++++++++++++- LibMatrix/Interfaces/Services/IStorageProvider.cs | 16 +- LibMatrix/Responses/SyncResponse.cs | 15 +- LibMatrix/RoomTypes/GenericRoom.cs | 11 +- LibMatrix/StateEvent.cs | 49 ++-- 22 files changed, 1072 insertions(+), 262 deletions(-) delete mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/AdminRoomDeleteRequest.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRegistrationTokenCreateRequest.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRoomDeleteRequest.cs delete mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminRoomListResult.cs delete mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminUserListResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/BackgroundUpdates.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/Destinations.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/EventReportListResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RegistrationTokenListResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomListResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomMediaListResult.cs delete mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseAdminEventReportListResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseCollectionResult.cs create mode 100644 LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/UserListResult.cs (limited to 'LibMatrix/Extensions') diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs index 771f41e..cdc0dca 100644 --- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs +++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs @@ -255,5 +255,10 @@ public class MatrixHttpClient { }; return await SendAsync(request, cancellationToken); } + + public async Task DeleteAsync(string url) { + var request = new HttpRequestMessage(HttpMethod.Delete, url); + await SendAsync(request); + } } #endif diff --git a/LibMatrix/Helpers/SyncHelper.cs b/LibMatrix/Helpers/SyncHelper.cs index ae033f1..a7010ee 100644 --- a/LibMatrix/Helpers/SyncHelper.cs +++ b/LibMatrix/Helpers/SyncHelper.cs @@ -56,6 +56,12 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg public TimeSpan MinimumDelay { get; set; } = new(0); + public async Task GetUnoptimisedStoreCount() { + if (storageProvider is null) return -1; + var keys = await storageProvider.GetAllKeysAsync(); + return keys.Count(x => !x.StartsWith("old/")) - 1; + } + private async Task UpdateFilterAsync() { if (!string.IsNullOrWhiteSpace(NamedFilterName)) { _filterId = await homeserver.NamedCaches.FilterCache.GetOrSetValueAsync(NamedFilterName); diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs index e2dbdee..e9c5938 100644 --- a/LibMatrix/Helpers/SyncStateResolver.cs +++ b/LibMatrix/Helpers/SyncStateResolver.cs @@ -1,3 +1,5 @@ +using System.Collections.Frozen; +using System.Diagnostics; using ArcaneLibs.Extensions; using LibMatrix.Extensions; using LibMatrix.Filters; @@ -26,16 +28,10 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge _syncHelper.SetPresence = SetPresence; _syncHelper.Filter = Filter; _syncHelper.FullState = FullState; - // run sync or grab from storage if available - // var sync = storageProvider != null && await storageProvider.ObjectExistsAsync(Since ?? "init") - // ? await storageProvider.LoadObjectAsync(Since ?? "init") - // : await _syncHelper.SyncAsync(cancellationToken); + var sync = await _syncHelper.SyncAsync(cancellationToken); if (sync is null) return await ContinueAsync(cancellationToken); - // if (storageProvider != null && !await storageProvider.ObjectExistsAsync(Since ?? "init")) - // await storageProvider.SaveObjectAsync(Since ?? "init", sync); - if (MergedState is null) MergedState = sync; else MergedState = MergeSyncs(MergedState, sync); Since = sync.NextBatch; @@ -45,22 +41,98 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge public async Task OptimiseStore() { if (storageProvider is null) return; + if (!await storageProvider.ObjectExistsAsync("init")) return; + + Console.Write("Optimising sync store..."); + var initLoadTask = storageProvider.LoadObjectAsync("init"); + var keys = (await storageProvider.GetAllKeysAsync()).ToFrozenSet(); + var count = keys.Count(x => !x.StartsWith("old/")) - 1; + Console.WriteLine($"Found {count} entries to optimise."); - var keys = await storageProvider.GetAllKeysAsync(); - var count = keys.Count - 2; - var merged = await storageProvider.LoadObjectAsync("init"); + var merged = await initLoadTask; if (merged is null) return; + if (!keys.Contains(merged.NextBatch)) { + Console.WriteLine("Next response after initial sync is not present, not checkpointing!"); + return; + } + + // We back up old entries + var oldPath = $"old/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; + await storageProvider.MoveObjectAsync("init", $"{oldPath}/init"); + + var moveTasks = new List(); while (keys.Contains(merged.NextBatch)) { + Console.Write($"Merging {merged.NextBatch}, {--count} remaining... "); + var sw = Stopwatch.StartNew(); + var swt = Stopwatch.StartNew(); var next = await storageProvider.LoadObjectAsync(merged.NextBatch); - if (next is null) break; + Console.Write($"Load {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); + if (next is null || merged.NextBatch == next.NextBatch) break; + + Console.Write($"Check {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); + // back up old entry + moveTasks.Add(storageProvider.MoveObjectAsync(merged.NextBatch, $"{oldPath}/{merged.NextBatch}")); + Console.Write($"Move {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); + merged = MergeSyncs(merged, next); - Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining..."); + Console.Write($"Merge {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); + Console.WriteLine($"Total {swt.Elapsed.TotalMilliseconds}ms"); + // Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining..."); + } + + await storageProvider.SaveObjectAsync("init", merged); + await Task.WhenAll(moveTasks); + } + + public async Task UnrollOptimisedStore() { + if (storageProvider is null) return; + Console.WriteLine("WARNING: Unrolling sync store!"); + } + + public async Task SquashOptimisedStore(int targetCountPerCheckpoint) { + Console.Write($"Balancing optimised store to {targetCountPerCheckpoint} per checkpoint..."); + var checkpoints = await GetCheckpointMap(); + if (checkpoints is null) return; + + Console.WriteLine( + $" Stats: {checkpoints.Count} checkpoints with [{checkpoints.Min(x => x.Value.Count)} < ~{checkpoints.Average(x => x.Value.Count)} < {checkpoints.Max(x => x.Value.Count)}] entries"); + Console.WriteLine($"Found {checkpoints?.Count ?? 0} checkpoints."); + } + + public async Task dev() { + var keys = (await storageProvider?.GetAllKeysAsync()).ToFrozenSet(); + var times = new Dictionary>(); + var values = keys.Select(async x => Task.Run(async () => (x, await storageProvider?.LoadObjectAsync(x)))).ToAsyncEnumerable(); + await foreach (var task in values) { + var (key, data) = await task; + if (data is null) continue; + var derivTime = data.GetDerivedSyncTime(); + if (!times.ContainsKey(derivTime)) times[derivTime] = new(); + times[derivTime].Add(key); } - await storageProvider.SaveObjectAsync("merged", merged); + foreach (var (time, ckeys) in times.OrderBy(x => x.Key)) { + Console.WriteLine($"{time}: {ckeys.Count} keys"); + } + } + + private async Task>?> GetCheckpointMap() { + if (storageProvider is null) return null; + var keys = (await storageProvider.GetAllKeysAsync()).ToFrozenSet(); + var map = new Dictionary>(); + foreach (var key in keys) { + if (!key.StartsWith("old/")) continue; + var parts = key.Split('/'); + if (parts.Length < 3) continue; + // if (!map.ContainsKey(parts[1])) map[parts[1]] = new(); + // map[parts[1]].Add(parts[2]); + if (!ulong.TryParse(parts[1], out var checkpoint)) continue; + if (!map.ContainsKey(checkpoint)) map[checkpoint] = new(); + map[checkpoint].Add(parts[2]); + } - Environment.Exit(0); + return map.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value); } private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync) { diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/AdminRoomDeleteRequest.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/AdminRoomDeleteRequest.cs deleted file mode 100644 index f4c927a..0000000 --- a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/AdminRoomDeleteRequest.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Text.Json.Serialization; - -namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Requests; - -public class AdminRoomDeleteRequest { - [JsonPropertyName("new_room_user_id")] - public string? NewRoomUserId { get; set; } - - [JsonPropertyName("room_name")] - public string? RoomName { get; set; } - - [JsonPropertyName("block")] - public bool Block { get; set; } - - [JsonPropertyName("purge")] - public bool Purge { get; set; } - - [JsonPropertyName("message")] - public string? Message { get; set; } - - [JsonPropertyName("force_purge")] - public bool ForcePurge { get; set; } -} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRegistrationTokenCreateRequest.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRegistrationTokenCreateRequest.cs new file mode 100644 index 0000000..197fd5d --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRegistrationTokenCreateRequest.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminRegistrationTokenUpdateRequest { + [JsonPropertyName("uses_allowed")] + public int? UsesAllowed { get; set; } + + [JsonPropertyName("expiry_time")] + public long? ExpiryTime { get; set; } + + [JsonIgnore] + public DateTime? ExpiresAt { + get => ExpiryTime.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(ExpiryTime.Value).DateTime : null; + set => ExpiryTime = value.HasValue ? new DateTimeOffset(value.Value).ToUnixTimeMilliseconds() : null; + } + + [JsonIgnore] + public TimeSpan? ExpiresAfter { + get => ExpiryTime.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(ExpiryTime.Value).DateTime - DateTimeOffset.Now : null; + set => ExpiryTime = value.HasValue ? (DateTimeOffset.Now + value.Value).ToUnixTimeMilliseconds() : null; + } +} + +public class SynapseAdminRegistrationTokenCreateRequest : SynapseAdminRegistrationTokenUpdateRequest { + [JsonPropertyName("token")] + public string? Token { get; set; } + + [JsonPropertyName("length")] + public int? Length { get; set; } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRoomDeleteRequest.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRoomDeleteRequest.cs new file mode 100644 index 0000000..67a3104 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Requests/SynapseAdminRoomDeleteRequest.cs @@ -0,0 +1,23 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Requests; + +public class SynapseAdminRoomDeleteRequest { + [JsonPropertyName("new_room_user_id")] + public string? NewRoomUserId { get; set; } + + [JsonPropertyName("room_name")] + public string? RoomName { get; set; } + + [JsonPropertyName("block")] + public bool Block { get; set; } + + [JsonPropertyName("purge")] + public bool Purge { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("force_purge")] + public bool ForcePurge { get; set; } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminRoomListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminRoomListResult.cs deleted file mode 100644 index c9d7e52..0000000 --- a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminRoomListResult.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System.Text.Json.Serialization; - -namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; - -public class AdminRoomListResult { - [JsonPropertyName("offset")] - public int Offset { get; set; } - - [JsonPropertyName("total_rooms")] - public int TotalRooms { get; set; } - - [JsonPropertyName("next_batch")] - public int? NextBatch { get; set; } - - [JsonPropertyName("prev_batch")] - public int? PrevBatch { get; set; } - - [JsonPropertyName("rooms")] - public List Rooms { get; set; } = new(); - - public class AdminRoomListResultRoom { - [JsonPropertyName("room_id")] - public required string RoomId { get; set; } - - [JsonPropertyName("name")] - public string? Name { get; set; } - - [JsonPropertyName("canonical_alias")] - public string? CanonicalAlias { get; set; } - - [JsonPropertyName("joined_members")] - public int JoinedMembers { get; set; } - - [JsonPropertyName("joined_local_members")] - public int JoinedLocalMembers { get; set; } - - [JsonPropertyName("version")] - public string? Version { get; set; } - - [JsonPropertyName("creator")] - public string? Creator { get; set; } - - [JsonPropertyName("encryption")] - public string? Encryption { get; set; } - - [JsonPropertyName("federatable")] - public bool Federatable { get; set; } - - [JsonPropertyName("public")] - public bool Public { get; set; } - - [JsonPropertyName("join_rules")] - public string? JoinRules { get; set; } - - [JsonPropertyName("guest_access")] - public string? GuestAccess { get; set; } - - [JsonPropertyName("history_visibility")] - public string? HistoryVisibility { get; set; } - - [JsonPropertyName("state_events")] - public int StateEvents { get; set; } - } -} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminUserListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminUserListResult.cs deleted file mode 100644 index 9b0c481..0000000 --- a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/AdminUserListResult.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Text.Json.Serialization; - -namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; - -public class AdminUserListResult { - [JsonPropertyName("offset")] - public int Offset { get; set; } - - [JsonPropertyName("total")] - public int Total { get; set; } - - [JsonPropertyName("next_token")] - public string? NextToken { get; set; } - - [JsonPropertyName("users")] - public List Users { get; set; } = new(); - - public class AdminUserListResultUser { - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("is_guest")] - public bool? IsGuest { get; set; } - - [JsonPropertyName("admin")] - public bool? Admin { get; set; } - - [JsonPropertyName("user_type")] - public string? UserType { get; set; } - - [JsonPropertyName("deactivated")] - public bool Deactivated { get; set; } - - [JsonPropertyName("erased")] - public bool Erased { get; set; } - - [JsonPropertyName("shadow_banned")] - public bool ShadowBanned { get; set; } - - [JsonPropertyName("displayname")] - public string? DisplayName { get; set; } - - [JsonPropertyName("avatar_url")] - public string? AvatarUrl { get; set; } - - [JsonPropertyName("creation_ts")] - public long CreationTs { get; set; } - - [JsonPropertyName("last_seen_ts")] - public long? LastSeenTs { get; set; } - - [JsonPropertyName("locked")] - public bool Locked { get; set; } - - [JsonPropertyName("approved")] - public bool Approved { get; set; } - } -} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/BackgroundUpdates.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/BackgroundUpdates.cs new file mode 100644 index 0000000..2394b98 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/BackgroundUpdates.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminBackgroundUpdateStatusResponse { + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } + + [JsonPropertyName("current_updates")] + public Dictionary CurrentUpdates { get; set; } + + public class BackgroundUpdateInfo { + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("total_item_count")] + public int TotalItemCount { get; set; } + + [JsonPropertyName("total_duration_ms")] + public double TotalDurationMs { get; set; } + + [JsonPropertyName("average_items_per_ms")] + public double AverageItemsPerMs { get; set; } + + [JsonIgnore] + public TimeSpan TotalDuration => TimeSpan.FromMilliseconds(TotalDurationMs); + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/Destinations.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/Destinations.cs new file mode 100644 index 0000000..646a4b5 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/Destinations.cs @@ -0,0 +1,56 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminDestinationListResult : SynapseNextTokenTotalCollectionResult { + [JsonPropertyName("destinations")] + public List Destinations { get; set; } = new(); + + public class SynapseAdminDestinationListResultDestination { + [JsonPropertyName("destination")] + public string Destination { get; set; } + + [JsonPropertyName("retry_last_ts")] + public long RetryLastTs { get; set; } + + [JsonPropertyName("retry_interval")] + public long RetryInterval { get; set; } + + [JsonPropertyName("failure_ts")] + public long? FailureTs { get; set; } + + [JsonPropertyName("last_successful_stream_ordering")] + public long? LastSuccessfulStreamOrdering { get; set; } + + [JsonIgnore] + public DateTime? FailureTsDateTime { + get => FailureTs.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(FailureTs.Value).DateTime : null; + set => FailureTs = value.HasValue ? new DateTimeOffset(value.Value).ToUnixTimeMilliseconds() : null; + } + + [JsonIgnore] + public DateTime? RetryLastTsDateTime { + get => DateTimeOffset.FromUnixTimeMilliseconds(RetryLastTs).DateTime; + set => RetryLastTs = new DateTimeOffset(value.Value).ToUnixTimeMilliseconds(); + } + + [JsonIgnore] + public TimeSpan RetryIntervalTimeSpan { + get => TimeSpan.FromMilliseconds(RetryInterval); + set => RetryInterval = (long)value.TotalMilliseconds; + } + } +} + +public class SynapseAdminDestinationRoomListResult : SynapseNextTokenTotalCollectionResult { + [JsonPropertyName("rooms")] + public List Rooms { get; set; } = new(); + + public class SynapseAdminDestinationRoomListResultRoom { + [JsonPropertyName("room_id")] + public string RoomId { get; set; } + + [JsonPropertyName("stream_ordering")] + public int StreamOrdering { get; set; } + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/EventReportListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/EventReportListResult.cs new file mode 100644 index 0000000..10fc039 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/EventReportListResult.cs @@ -0,0 +1,169 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using ArcaneLibs; +using ArcaneLibs.Attributes; +using ArcaneLibs.Extensions; +using LibMatrix.EventTypes; +using LibMatrix.Extensions; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminEventReportListResult : SynapseNextTokenTotalCollectionResult { + [JsonPropertyName("event_reports")] + public List Reports { get; set; } = new(); + + public class SynapseAdminEventReportListResultReport { + [JsonPropertyName("event_id")] + public string EventId { get; set; } + + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("reason")] + public string? Reason { get; set; } + + [JsonPropertyName("score")] + public int? Score { get; set; } + + [JsonPropertyName("received_ts")] + public long ReceivedTs { get; set; } + + [JsonPropertyName("canonical_alias")] + public string? CanonicalAlias { get; set; } + + [JsonPropertyName("room_id")] + public string RoomId { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("sender")] + public string Sender { get; set; } + + [JsonPropertyName("user_id")] + public string UserId { get; set; } + + [JsonIgnore] + public DateTime ReceivedTsDateTime { + get => DateTimeOffset.FromUnixTimeMilliseconds(ReceivedTs).DateTime; + set => ReceivedTs = new DateTimeOffset(value).ToUnixTimeMilliseconds(); + } + } + + public class SynapseAdminEventReportListResultReportWithDetails : SynapseAdminEventReportListResultReport { + [JsonPropertyName("event_json")] + public SynapseEventJson EventJson { get; set; } + + public class SynapseEventJson { + [JsonPropertyName("auth_events")] + public List AuthEvents { get; set; } + + [JsonPropertyName("content")] + public JsonObject? RawContent { get; set; } + + [JsonPropertyName("depth")] + public int Depth { get; set; } + + [JsonPropertyName("hashes")] + public Dictionary Hashes { get; set; } + + [JsonPropertyName("origin")] + public string Origin { get; set; } + + [JsonPropertyName("origin_server_ts")] + public long OriginServerTs { get; set; } + + [JsonPropertyName("prev_events")] + public List PrevEvents { get; set; } + + [JsonPropertyName("prev_state")] + public List PrevState { get; set; } + + [JsonPropertyName("room_id")] + public string RoomId { get; set; } + + [JsonPropertyName("sender")] + public string Sender { get; set; } + + [JsonPropertyName("signatures")] + public Dictionary> Signatures { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("unsigned")] + public JsonObject? Unsigned { get; set; } + + // Extra... copied from StateEventResponse + + [JsonIgnore] + public Type MappedType => StateEvent.GetStateEventType(Type); + + [JsonIgnore] + public bool IsLegacyType => MappedType.GetCustomAttributes().FirstOrDefault(x => x.EventName == Type)?.Legacy ?? false; + + [JsonIgnore] + public string FriendlyTypeName => MappedType.GetFriendlyNameOrNull() ?? Type; + + [JsonIgnore] + public string FriendlyTypeNamePlural => MappedType.GetFriendlyNamePluralOrNull() ?? Type; + + private static readonly JsonSerializerOptions TypedContentSerializerOptions = new() { + Converters = { + new JsonFloatStringConverter(), + new JsonDoubleStringConverter(), + new JsonDecimalStringConverter() + } + }; + + [JsonIgnore] + [SuppressMessage("ReSharper", "PropertyCanBeMadeInitOnly.Global")] + public EventContent? TypedContent { + get { + ClassCollector.ResolveFromAllAccessibleAssemblies(); + // if (Type == "m.receipt") { + // return null; + // } + try { + var mappedType = StateEvent.GetStateEventType(Type); + if (mappedType == typeof(UnknownEventContent)) + Console.WriteLine($"Warning: unknown event type '{Type}'"); + var deserialisedContent = (EventContent)RawContent.Deserialize(mappedType, TypedContentSerializerOptions)!; + return deserialisedContent; + } + catch (JsonException e) { + Console.WriteLine(e); + Console.WriteLine("Content:\n" + (RawContent?.ToJson() ?? "null")); + } + + return null; + } + set { + if (value is null) + RawContent?.Clear(); + else + RawContent = JsonSerializer.Deserialize(JsonSerializer.Serialize(value, value.GetType(), + new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })); + } + } + + //debug + [JsonIgnore] + public string InternalSelfTypeName { + get { + var res = GetType().Name switch { + "StateEvent`1" => "StateEvent", + _ => GetType().Name + }; + return res; + } + } + + [JsonIgnore] + public string InternalContentTypeName => TypedContent?.GetType().Name ?? "null"; + } + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RegistrationTokenListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RegistrationTokenListResult.cs new file mode 100644 index 0000000..fa92ef9 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RegistrationTokenListResult.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminRegistrationTokenListResult { + [JsonPropertyName("registration_tokens")] + public List RegistrationTokens { get; set; } = new(); + + public class SynapseAdminRegistrationTokenListResultToken { + [JsonPropertyName("token")] + public string Token { get; set; } + + [JsonPropertyName("uses_allowed")] + public int? UsesAllowed { get; set; } + + [JsonPropertyName("pending")] + public int Pending { get; set; } + + [JsonPropertyName("completed")] + public int Completed { get; set; } + + [JsonPropertyName("expiry_time")] + public long? ExpiryTime { get; set; } + + [JsonIgnore] + public DateTime? ExpiryTimeDateTime { + get => ExpiryTime.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(ExpiryTime.Value).DateTime : null; + set => ExpiryTime = value.HasValue ? new DateTimeOffset(value.Value).ToUnixTimeMilliseconds() : null; + } + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomListResult.cs new file mode 100644 index 0000000..d84c89b --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomListResult.cs @@ -0,0 +1,64 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminRoomListResult { + [JsonPropertyName("offset")] + public int Offset { get; set; } + + [JsonPropertyName("total_rooms")] + public int TotalRooms { get; set; } + + [JsonPropertyName("next_batch")] + public int? NextBatch { get; set; } + + [JsonPropertyName("prev_batch")] + public int? PrevBatch { get; set; } + + [JsonPropertyName("rooms")] + public List Rooms { get; set; } = new(); + + public class SynapseAdminRoomListResultRoom { + [JsonPropertyName("room_id")] + public required string RoomId { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("canonical_alias")] + public string? CanonicalAlias { get; set; } + + [JsonPropertyName("joined_members")] + public int JoinedMembers { get; set; } + + [JsonPropertyName("joined_local_members")] + public int JoinedLocalMembers { get; set; } + + [JsonPropertyName("version")] + public string? Version { get; set; } + + [JsonPropertyName("creator")] + public string? Creator { get; set; } + + [JsonPropertyName("encryption")] + public string? Encryption { get; set; } + + [JsonPropertyName("federatable")] + public bool Federatable { get; set; } + + [JsonPropertyName("public")] + public bool Public { get; set; } + + [JsonPropertyName("join_rules")] + public string? JoinRules { get; set; } + + [JsonPropertyName("guest_access")] + public string? GuestAccess { get; set; } + + [JsonPropertyName("history_visibility")] + public string? HistoryVisibility { get; set; } + + [JsonPropertyName("state_events")] + public int StateEvents { get; set; } + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomMediaListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomMediaListResult.cs new file mode 100644 index 0000000..97e85ad --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/RoomMediaListResult.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminRoomMediaListResult { + [JsonPropertyName("local")] + public List Local { get; set; } = new(); + + [JsonPropertyName("remote")] + public List Remote { get; set; } = new(); +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseAdminEventReportListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseAdminEventReportListResult.cs deleted file mode 100644 index 030108a..0000000 --- a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseAdminEventReportListResult.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Text.Json.Serialization; - -namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; - -public class SynapseAdminEventReportListResult { - [JsonPropertyName("offset")] - public int Offset { get; set; } - - [JsonPropertyName("total")] - public int Total { get; set; } - - [JsonPropertyName("next_token")] - public string? NextToken { get; set; } - - [JsonPropertyName("event_reports")] - public List Reports { get; set; } = new(); - - public class SynapseAdminEventReportListResultReport { - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("is_guest")] - public bool? IsGuest { get; set; } - - [JsonPropertyName("admin")] - public bool? Admin { get; set; } - - [JsonPropertyName("user_type")] - public string? UserType { get; set; } - - [JsonPropertyName("deactivated")] - public bool Deactivated { get; set; } - - [JsonPropertyName("erased")] - public bool Erased { get; set; } - - [JsonPropertyName("shadow_banned")] - public bool ShadowBanned { get; set; } - - [JsonPropertyName("displayname")] - public string? DisplayName { get; set; } - - [JsonPropertyName("avatar_url")] - public string? AvatarUrl { get; set; } - - [JsonPropertyName("creation_ts")] - public long CreationTs { get; set; } - - [JsonPropertyName("last_seen_ts")] - public long? LastSeenTs { get; set; } - - [JsonPropertyName("locked")] - public bool Locked { get; set; } - - [JsonPropertyName("approved")] - public bool Approved { get; set; } - } -} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseCollectionResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseCollectionResult.cs new file mode 100644 index 0000000..36a5596 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/SynapseCollectionResult.cs @@ -0,0 +1,250 @@ +using System.Buffers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using ArcaneLibs.Extensions; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseNextTokenTotalCollectionResult { + [JsonPropertyName("total")] + public int Total { get; set; } + + [JsonPropertyName("next_token")] + public string? NextToken { get; set; } +} + +// [JsonConverter(typeof(SynapseCollectionJsonConverter<>))] +public class SynapseCollectionResult(string chunkKey = "chunk", string prevTokenKey = "prev_token", string nextTokenKey = "next_token", string totalKey = "total") { + public int? Total { get; set; } + public string? PrevToken { get; set; } + public string? NextToken { get; set; } + public List Chunk { get; set; } = []; + + // TODO: figure out how to provide an IAsyncEnumerable for this + // https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/use-utf8jsonreader#read-from-a-stream-using-utf8jsonreader + + // public async IAsyncEnumerable FromJsonAsync(Stream stream) { + // + // } + + public SynapseCollectionResult FromJson(Stream stream, Action action) { + byte[] buffer = new byte[4096]; + _ = stream.Read(buffer); + var reader = new Utf8JsonReader(buffer, isFinalBlock: false, state: default); + + try { + FromJsonInternal(stream, ref buffer, ref reader, action); + } + catch (JsonException e) { + Console.WriteLine($"Caught a JsonException: {e}"); + int hexdumpWidth = 64; + Console.WriteLine($"Check hexdump line {reader.BytesConsumed / hexdumpWidth} index {reader.BytesConsumed % hexdumpWidth}"); + buffer.HexDump(64); + } + finally { } + + return this; + } + + private void FromJsonInternal(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader, Action action) { + while (!reader.IsFinalBlock) { + while (!reader.Read()) { + GetMoreBytesFromStream(stream, ref buffer, ref reader); + } + + if (reader.TokenType == JsonTokenType.PropertyName) { + var propName = reader.GetString(); + Console.WriteLine($"SynapseCollectionResult: encountered property name: {propName}"); + + while (!reader.Read()) { + GetMoreBytesFromStream(stream, ref buffer, ref reader); + } + + Console.WriteLine($"{reader.BytesConsumed}/{stream.Position} {reader.TokenType}"); + + if (propName == totalKey && reader.TokenType == JsonTokenType.Number) { + Total = reader.GetInt32(); + } + else if (propName == prevTokenKey && reader.TokenType == JsonTokenType.String) { + PrevToken = reader.GetString(); + } + else if (propName == nextTokenKey && reader.TokenType == JsonTokenType.String) { + NextToken = reader.GetString(); + } + else if (propName == chunkKey) { + if (reader.TokenType == JsonTokenType.StartArray) { + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { + // if (reader.TokenType == JsonTokenType.EndArray) { + // break; + // } + // Console.WriteLine($"Encountered token in chunk: {reader.TokenType}"); + // var _buf = reader.ValueSequence.ToArray(); + // try { + // var item = JsonSerializer.Deserialize(_buf); + // action(item); + // Chunk.Add(item); + // } + // catch(JsonException e) { + // Console.WriteLine($"Caught a JsonException: {e}"); + // int hexdumpWidth = 64; + // + // // Console.WriteLine($"Check hexdump line {reader.BytesConsumed / hexdumpWidth} index {reader.BytesConsumed % hexdumpWidth}"); + // Console.WriteLine($"Buffer length: {_buf.Length}"); + // _buf.HexDump(64); + // throw; + // } + var item = ReadItem(stream, ref buffer, ref reader); + action(item); + Chunk.Add(item); + } + } + } + } + } + } + + private T ReadItem(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) { + while (!reader.Read()) { + GetMoreBytesFromStream(stream, ref buffer, ref reader); + } + + // handle nullable types + if (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(Nullable<>)) { + if (reader.TokenType == JsonTokenType.Null) { + return default(T); + } + } + + // if(typeof(T) == typeof(string)) { + // return (T)(object)reader.GetString(); + // } + // else if(typeof(T) == typeof(int)) { + // return (T)(object)reader.GetInt32(); + // } + // else { + // var _buf = reader.ValueSequence.ToArray(); + // return JsonSerializer.Deserialize(_buf); + // } + + // default branch uses "object?" cast to avoid compiler error + // add more branches here as nessesary + // reader.Read(); + var call = typeof(T) switch { + Type t when t == typeof(string) => reader.GetString(), + _ => ReadObject(stream, ref buffer, ref reader) + }; + + object ReadObject(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) { + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new JsonException(); + } + + List objBuffer = [(byte)'{', ..reader.ValueSequence.ToArray()]; + var currentDepth = reader.CurrentDepth; + while (reader.CurrentDepth >= currentDepth) { + while (!reader.Read()) { + GetMoreBytesFromStream(stream, ref buffer, ref reader); + } + + if (reader.TokenType == JsonTokenType.EndObject && reader.CurrentDepth == currentDepth) { + break; + } + + objBuffer.AddRange(reader.ValueSpan); + } + + return JsonSerializer.Deserialize(objBuffer.ToArray()); + } + + return (T)call; + + // return JsonSerializer.Deserialize(ref reader); + } + + private static void GetMoreBytesFromStream(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) { + int bytesRead; + if (reader.BytesConsumed < buffer.Length) { + ReadOnlySpan leftover = buffer.AsSpan((int)reader.BytesConsumed); + + if (leftover.Length == buffer.Length) { + Array.Resize(ref buffer, buffer.Length * 2); + Console.WriteLine($"Increased buffer size to {buffer.Length}"); + } + + leftover.CopyTo(buffer); + bytesRead = stream.Read(buffer.AsSpan(leftover.Length)); + } + else { + bytesRead = stream.Read(buffer); + } + + // Console.WriteLine($"String in buffer is: {Encoding.UTF8.GetString(buffer)}"); + reader = new Utf8JsonReader(buffer, isFinalBlock: bytesRead == 0, reader.CurrentState); + } +} + +public partial class SynapseCollectionJsonConverter : JsonConverter> { + public override SynapseCollectionResult? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.TokenType != JsonTokenType.StartObject) { + throw new JsonException(); + } + + var result = new SynapseCollectionResult(); + while (reader.Read()) { + if (reader.TokenType == JsonTokenType.EndObject) { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new JsonException(); + } + + var propName = reader.GetString(); + reader.Read(); + if (propName == "total") { + result.Total = reader.GetInt32(); + } + else if (propName == "prev_token") { + result.PrevToken = reader.GetString(); + } + else if (propName == "next_token") { + result.NextToken = reader.GetString(); + } + else if (propName == "chunk") { + if (reader.TokenType != JsonTokenType.StartArray) { + throw new JsonException(); + } + + while (reader.Read()) { + if (reader.TokenType == JsonTokenType.EndArray) { + break; + } + + var item = JsonSerializer.Deserialize(ref reader, options); + result.Chunk.Add(item); + } + } + } + + return result; + } + + public override void Write(Utf8JsonWriter writer, SynapseCollectionResult value, JsonSerializerOptions options) { + writer.WriteStartObject(); + if (value.Total is not null) + writer.WriteNumber("total", value.Total ?? 0); + if (value.PrevToken is not null) + writer.WriteString("prev_token", value.PrevToken); + if (value.NextToken is not null) + writer.WriteString("next_token", value.NextToken); + + writer.WriteStartArray("chunk"); + foreach (var item in value.Chunk) { + JsonSerializer.Serialize(writer, item, options); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/UserListResult.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/UserListResult.cs new file mode 100644 index 0000000..3132906 --- /dev/null +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/Models/Responses/UserListResult.cs @@ -0,0 +1,71 @@ +using System.Text.Json.Serialization; + +namespace LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Responses; + +public class SynapseAdminUserListResult { + [JsonPropertyName("offset")] + public int Offset { get; set; } + + [JsonPropertyName("total")] + public int Total { get; set; } + + [JsonPropertyName("next_token")] + public string? NextToken { get; set; } + + [JsonPropertyName("users")] + public List Users { get; set; } = new(); + + public class SynapseAdminUserListResultUser { + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("is_guest")] + public bool? IsGuest { get; set; } + + [JsonPropertyName("admin")] + public bool? Admin { get; set; } + + [JsonPropertyName("user_type")] + public string? UserType { get; set; } + + [JsonPropertyName("deactivated")] + public bool Deactivated { get; set; } + + [JsonPropertyName("erased")] + public bool Erased { get; set; } + + [JsonPropertyName("shadow_banned")] + public bool ShadowBanned { get; set; } + + [JsonPropertyName("displayname")] + public string? DisplayName { get; set; } + + [JsonPropertyName("avatar_url")] + public string? AvatarUrl { get; set; } + + [JsonPropertyName("creation_ts")] + public long CreationTs { get; set; } + + [JsonPropertyName("last_seen_ts")] + public long? LastSeenTs { get; set; } + + [JsonPropertyName("locked")] + public bool Locked { get; set; } + + // Requires enabling MSC3866 + [JsonPropertyName("approved")] + public bool? Approved { get; set; } + + [JsonIgnore] + public DateTime CreationTsDateTime { + get => DateTimeOffset.FromUnixTimeMilliseconds(CreationTs).DateTime; + set => CreationTs = new DateTimeOffset(value).ToUnixTimeMilliseconds(); + } + + [JsonIgnore] + public DateTime? LastSeenTsDateTime { + get => LastSeenTs.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(LastSeenTs.Value).DateTime : null; + set => LastSeenTs = value.HasValue ? new DateTimeOffset(value.Value).ToUnixTimeMilliseconds() : null; + } + } +} \ No newline at end of file diff --git a/LibMatrix/Homeservers/ImplementationDetails/Synapse/SynapseAdminApiClient.cs b/LibMatrix/Homeservers/ImplementationDetails/Synapse/SynapseAdminApiClient.cs index b3902eb..4d8a577 100644 --- a/LibMatrix/Homeservers/ImplementationDetails/Synapse/SynapseAdminApiClient.cs +++ b/LibMatrix/Homeservers/ImplementationDetails/Synapse/SynapseAdminApiClient.cs @@ -1,5 +1,7 @@ using System.Net.Http.Json; +using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using ArcaneLibs.Extensions; using LibMatrix.Filters; using LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Filters; @@ -10,12 +12,13 @@ namespace LibMatrix.Homeservers.ImplementationDetails.Synapse; public class SynapseAdminApiClient(AuthenticatedHomeserverSynapse authenticatedHomeserver) { // https://github.com/element-hq/synapse/tree/develop/docs/admin_api + // https://github.com/element-hq/synapse/tree/develop/docs/usage/administration/admin_api #region Rooms - public async IAsyncEnumerable SearchRoomsAsync(int limit = int.MaxValue, int chunkLimit = 250, string orderBy = "name", - string dir = "f", string? searchTerm = null, SynapseAdminLocalRoomQueryFilter? localFilter = null) { - AdminRoomListResult? res = null; + public async IAsyncEnumerable SearchRoomsAsync(int limit = int.MaxValue, int chunkLimit = 250, + string orderBy = "name", string dir = "f", string? searchTerm = null, SynapseAdminLocalRoomQueryFilter? localFilter = null) { + SynapseAdminRoomListResult? res = null; var i = 0; int? totalRooms = null; do { @@ -26,7 +29,7 @@ public class SynapseAdminApiClient(AuthenticatedHomeserverSynapse authenticatedH Console.WriteLine($"--- ADMIN Querying Room List with URL: {url} - Already have {i} items... ---"); - res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url); + res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url); totalRooms ??= res.TotalRooms; Console.WriteLine(res.ToJson(false)); foreach (var room in res.Rooms) { @@ -117,7 +120,7 @@ public class SynapseAdminApiClient(AuthenticatedHomeserverSynapse authenticatedH #region Users - public async IAsyncEnumerable SearchUsersAsync(int limit = int.MaxValue, int chunkLimit = 250, + public async IAsyncEnumerable SearchUsersAsync(int limit = int.MaxValue, int chunkLimit = 250, SynapseAdminLocalUserQueryFilter? localFilter = null) { // TODO: implement filters string? from = null; @@ -127,7 +130,7 @@ public class SynapseAdminApiClient(AuthenticatedHomeserverSynapse authenticatedH if (!string.IsNullOrWhiteSpace(from)) url = url.AddQuery("from", from); Console.WriteLine($"--- ADMIN Querying User List with URL: {url} ---"); // TODO: implement URI methods in http client - var res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + var res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); foreach (var user in res.Users) { limit--; yield return user; @@ -171,5 +174,185 @@ public class SynapseAdminApiClient(AuthenticatedHomeserverSynapse authenticatedH } } + public async Task GetEventReportDetailsAsync(string reportId) { + var url = new Uri($"/_synapse/admin/v1/event_reports/{reportId.UrlEncode()}", UriKind.Relative); + return await authenticatedHomeserver.ClientHttpClient + .GetFromJsonAsync(url.ToString()); + } + + // Utility function to get details straight away + public async IAsyncEnumerable GetEventReportsWithDetailsAsync(int limit = int.MaxValue, + int chunkLimit = 250, string dir = "f", SynapseAdminLocalEventReportQueryFilter? filter = null) { + Queue> tasks = []; + await foreach (var report in GetEventReportsAsync(limit, chunkLimit, dir, filter)) { + tasks.Enqueue(GetEventReportDetailsAsync(report.Id)); + while (tasks.Peek().IsCompleted) yield return await tasks.Dequeue(); // early return if possible + } + + while (tasks.Count > 0) yield return await tasks.Dequeue(); + } + + public async Task DeleteEventReportAsync(string reportId) { + var url = new Uri($"/_synapse/admin/v1/event_reports/{reportId.UrlEncode()}", UriKind.Relative); + await authenticatedHomeserver.ClientHttpClient.DeleteAsync(url.ToString()); + } + +#endregion + +#region Background Updates + + public async Task GetBackgroundUpdatesEnabledAsync() { + var url = new Uri("/_synapse/admin/v1/background_updates/enabled", UriKind.Relative); + // The return type is technically wrong, but includes the field we want. + var resp = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + return resp.Enabled; + } + + public async Task SetBackgroundUpdatesEnabledAsync(bool enabled) { + var url = new Uri("/_synapse/admin/v1/background_updates/enabled", UriKind.Relative); + // The used types are technically wrong, but include the field we want. + var resp = await authenticatedHomeserver.ClientHttpClient.PostAsJsonAsync(url.ToString(), new JsonObject { + ["enabled"] = enabled + }); + var json = await resp.Content.ReadFromJsonAsync(); + return json!.Enabled; + } + + public async Task GetBackgroundUpdatesStatusAsync() { + var url = new Uri("/_synapse/admin/v1/background_updates/status", UriKind.Relative); + return await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + } + + /// + /// Run a background job + /// + /// One of "populate_stats_process_rooms" or "regenerate_directory" + public async Task RunBackgroundJobsAsync(string jobName) { + var url = new Uri("/_synapse/admin/v1/background_updates/run", UriKind.Relative); + await authenticatedHomeserver.ClientHttpClient.PostAsJsonAsync(url.ToString(), new JsonObject() { + ["job_name"] = jobName + }); + } + +#endregion + +#region Federation + + public async IAsyncEnumerable GetFederationDestinationsAsync(int limit = int.MaxValue, + int chunkLimit = 250) { + string? from = null; + while (limit > 0) { + var url = new Uri("/_synapse/admin/v1/federation/destinations", UriKind.Relative); + url = url.AddQuery("limit", Math.Min(limit, chunkLimit).ToString()); + if (!string.IsNullOrWhiteSpace(from)) url = url.AddQuery("from", from); + Console.WriteLine($"--- ADMIN Querying Federation Destinations with URL: {url} ---"); + var res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + foreach (var dest in res.Destinations) { + limit--; + yield return dest; + } + } + } + + public async Task GetFederationDestinationDetailsAsync(string destination) { + var url = new Uri($"/_synapse/admin/v1/federation/destinations/{destination}", UriKind.Relative); + return await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + } + + public async IAsyncEnumerable GetFederationDestinationRoomsAsync(string destination, + int limit = int.MaxValue, int chunkLimit = 250) { + string? from = null; + while (limit > 0) { + var url = new Uri($"/_synapse/admin/v1/federation/destinations/{destination}/rooms", UriKind.Relative); + url = url.AddQuery("limit", Math.Min(limit, chunkLimit).ToString()); + if (!string.IsNullOrWhiteSpace(from)) url = url.AddQuery("from", from); + Console.WriteLine($"--- ADMIN Querying Federation Destination Rooms with URL: {url} ---"); + var res = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + foreach (var room in res.Rooms) { + limit--; + yield return room; + } + } + } + + public async Task ResetFederationConnectionTimeoutAsync(string destination) { + await authenticatedHomeserver.ClientHttpClient.PostAsJsonAsync($"/_synapse/admin/v1/federation/destinations/{destination}/reset_connection", new JsonObject()); + } + +#endregion + +#region Registration Tokens + + // does not support pagination + public async Task> GetRegistrationTokensAsync() { + var url = new Uri("/_synapse/admin/v1/registration_tokens", UriKind.Relative); + var resp = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + return resp.RegistrationTokens; + } + + public async Task GetRegistrationTokenAsync(string token) { + var url = new Uri($"/_synapse/admin/v1/registration_tokens/{token.UrlEncode()}", UriKind.Relative); + var resp = + await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + return resp; + } + + public async Task CreateRegistrationTokenAsync( + SynapseAdminRegistrationTokenCreateRequest request) { + var url = new Uri("/_synapse/admin/v1/", UriKind.Relative); + var resp = await authenticatedHomeserver.ClientHttpClient.PostAsJsonAsync(url.ToString(), request); + var token = await resp.Content.ReadFromJsonAsync(); + return token!; + } + + public async Task UpdateRegistrationTokenAsync(string token, + SynapseAdminRegistrationTokenUpdateRequest request) { + var url = new Uri($"/_synapse/admin/v1/registration_tokens/{token.UrlEncode()}", UriKind.Relative); + var resp = await authenticatedHomeserver.ClientHttpClient.PutAsJsonAsync(url.ToString(), request); + return await resp.Content.ReadFromJsonAsync(); + } + + public async Task DeleteRegistrationTokenAsync(string token) { + var url = new Uri($"/_synapse/admin/v1/registration_tokens/{token.UrlEncode()}", UriKind.Relative); + await authenticatedHomeserver.ClientHttpClient.DeleteAsync(url.ToString()); + } + +#endregion + +#region Account Validity + + // Does anyone even use this? + // README: https://github.com/matrix-org/synapse/issues/15271 + // -> Don't implement unless requested, if not for this feature almost never being used. + +#endregion + +#region Experimental Features + + public async Task> GetExperimentalFeaturesAsync(string userId) { + var url = new Uri($"/_synapse/admin/v1/experimental_features/{userId.UrlEncode()}", UriKind.Relative); + var resp = await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + return resp["features"]!.GetValue>(); + } + + public async Task SetExperimentalFeaturesAsync(string userId, Dictionary features) { + var url = new Uri($"/_synapse/admin/v1/experimental_features/{userId.UrlEncode()}", UriKind.Relative); + await authenticatedHomeserver.ClientHttpClient.PostAsJsonAsync(url.ToString(), new JsonObject { + ["features"] = JsonSerializer.Deserialize(features.ToJson()) + }); + } + +#endregion + +#region Media + + public async Task GetRoomMediaAsync(string roomId) { + var url = new Uri($"/_synapse/admin/v1/rooms/{roomId.UrlEncode()}/media", UriKind.Relative); + return await authenticatedHomeserver.ClientHttpClient.GetFromJsonAsync(url.ToString()); + } + + // This is in the user admin API section + // public async IAsyncEnumerable + #endregion } \ No newline at end of file diff --git a/LibMatrix/Interfaces/Services/IStorageProvider.cs b/LibMatrix/Interfaces/Services/IStorageProvider.cs index 165e7df..fb7bb6d 100644 --- a/LibMatrix/Interfaces/Services/IStorageProvider.cs +++ b/LibMatrix/Interfaces/Services/IStorageProvider.cs @@ -31,7 +31,7 @@ public interface IStorageProvider { } // get all keys - public Task> GetAllKeysAsync() { + public Task> GetAllKeysAsync() { Console.WriteLine($"StorageProvider<{GetType().Name}> does not implement GetAllKeys()!"); throw new NotImplementedException(); } @@ -53,4 +53,18 @@ public interface IStorageProvider { Console.WriteLine($"StorageProvider<{GetType().Name}> does not implement LoadStream(key)!"); throw new NotImplementedException(); } + + // copy + public async Task CopyObjectAsync(string sourceKey, string destKey) { + Console.WriteLine($"StorageProvider<{GetType().Name}> does not implement CopyObject(sourceKey, destKey), using load + save!"); + var data = await LoadObjectAsync(sourceKey); + await SaveObjectAsync(destKey, data); + } + + // move + public async Task MoveObjectAsync(string sourceKey, string destKey) { + Console.WriteLine($"StorageProvider<{GetType().Name}> does not implement MoveObject(sourceKey, destKey), using copy + delete!"); + await CopyObjectAsync(sourceKey, destKey); + await DeleteObjectAsync(sourceKey); + } } \ No newline at end of file diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs index e7fe109..9b4ce05 100644 --- a/LibMatrix/Responses/SyncResponse.cs +++ b/LibMatrix/Responses/SyncResponse.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using LibMatrix.EventTypes.Spec.Ephemeral; using LibMatrix.EventTypes.Spec.State; namespace LibMatrix.Responses; @@ -64,10 +65,9 @@ public class SyncResponse { public EventList? State { get; set; } public override string ToString() { - var lastEvent = Timeline?.Events?.LastOrDefault(x=>x.Type == "m.room.member"); + var lastEvent = Timeline?.Events?.LastOrDefault(x => x.Type == "m.room.member"); var membership = (lastEvent?.TypedContent as RoomMemberEventContent); return $"LeftRoomDataStructure: {lastEvent?.Sender} {membership?.Membership} ({membership?.Reason})"; - } } @@ -133,4 +133,15 @@ public class SyncResponse { public EventList? InviteState { get; set; } } } + + public long GetDerivedSyncTime() { + return ((long[]) [ + AccountData?.Events?.Max(x => x.OriginServerTs) ?? 0, + Presence?.Events?.Max(x => x.OriginServerTs) ?? 0, + ToDevice?.Events?.Max(x => x.OriginServerTs) ?? 0, + Rooms?.Join?.Values?.Max(x => x.Timeline?.Events?.Max(y => y.OriginServerTs)) ?? 0, + Rooms?.Invite?.Values?.Max(x => x.InviteState?.Events?.Max(y => y.OriginServerTs)) ?? 0, + Rooms?.Leave?.Values?.Max(x => x.Timeline?.Events?.Max(y => y.OriginServerTs)) ?? 0 + ]).Max(); + } } diff --git a/LibMatrix/RoomTypes/GenericRoom.cs b/LibMatrix/RoomTypes/GenericRoom.cs index 8398ab9..02bd555 100644 --- a/LibMatrix/RoomTypes/GenericRoom.cs +++ b/LibMatrix/RoomTypes/GenericRoom.cs @@ -328,8 +328,8 @@ public class GenericRoom { catch { try { var alias = await GetCanonicalAliasAsync(); - if (alias?.Alias is not null) return alias.Alias; - throw new Exception("No name or alias"); + if (!string.IsNullOrWhiteSpace(alias?.Alias)) return alias.Alias; + throw new Exception("No alias"); } catch { try { @@ -337,7 +337,8 @@ public class GenericRoom { var memberList = new List(); var memberCount = 0; await foreach (var member in members) - memberList.Add(member.RawContent?["displayname"]?.GetValue() ?? ""); + if (member.StateKey != Homeserver.UserId) + memberList.Add(member.RawContent?["displayname"]?.GetValue() ?? ""); memberCount = memberList.Count; memberList.RemoveAll(string.IsNullOrWhiteSpace); memberList = memberList.OrderBy(x => x).ToList(); @@ -431,7 +432,7 @@ public class GenericRoom { return await res.Content.ReadFromJsonAsync(); } - + public async Task GetRoomAccountDataOrNullAsync(string key) { try { return await GetRoomAccountDataAsync(key); @@ -554,4 +555,4 @@ public class GenericRoom { public class RoomIdResponse { [JsonPropertyName("room_id")] public string RoomId { get; set; } = null!; -} \ No newline at end of file +} diff --git a/LibMatrix/StateEvent.cs b/LibMatrix/StateEvent.cs index 8f99e10..87050cc 100644 --- a/LibMatrix/StateEvent.cs +++ b/LibMatrix/StateEvent.cs @@ -31,6 +31,23 @@ public class StateEvent { public static Type GetStateEventType(string? type) => string.IsNullOrWhiteSpace(type) ? typeof(UnknownEventContent) : KnownStateEventTypesByName.GetValueOrDefault(type) ?? typeof(UnknownEventContent); + [JsonPropertyName("state_key")] + public string? StateKey { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("replaces_state")] + public string? ReplacesState { get; set; } + + private JsonObject? _rawContent; + + [JsonPropertyName("content")] + public JsonObject? RawContent { + get => _rawContent; + set => _rawContent = value; + } + [JsonIgnore] public Type MappedType => GetStateEventType(Type); @@ -79,6 +96,7 @@ public class StateEvent { } } +<<<<<<< HEAD public T? ContentAs() { try { return RawContent.Deserialize(TypedContentSerializerOptions)!; @@ -107,37 +125,6 @@ public class StateEvent { get => _rawContent; set => _rawContent = value; } - // - // [JsonIgnore] - // public new Type GetType { - // get { - // var type = GetStateEventType(Type); - // - // //special handling for some types - // // if (type == typeof(RoomEmotesEventContent)) { - // // RawContent["emote"] = RawContent["emote"]?.AsObject() ?? new JsonObject(); - // // } - // // - // // if (this is StateEventResponse stateEventResponse) { - // // if (type == null || type == typeof(object)) { - // // Console.WriteLine($"Warning: unknown event type '{Type}'!"); - // // Console.WriteLine(RawContent.ToJson()); - // // Directory.CreateDirectory($"unknown_state_events/{Type}"); - // // File.WriteAllText($"unknown_state_events/{Type}/{stateEventResponse.EventId}.json", - // // RawContent.ToJson()); - // // Console.WriteLine($"Saved to unknown_state_events/{Type}/{stateEventResponse.EventId}.json"); - // // } - // // else if (RawContent is not null && RawContent.FindExtraJsonObjectFields(type)) { - // // Directory.CreateDirectory($"unknown_state_events/{Type}"); - // // File.WriteAllText($"unknown_state_events/{Type}/{stateEventResponse.EventId}.json", - // // RawContent.ToJson()); - // // Console.WriteLine($"Saved to unknown_state_events/{Type}/{stateEventResponse.EventId}.json"); - // // } - // // } - // - // return type; - // } - // } //debug [JsonIgnore] -- cgit 1.5.1 From af48e92d735a1f4d76aedd75460c8adbe1c882ad Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 16 Sep 2024 08:43:42 +0200 Subject: Sync optimisation changes --- LibMatrix/Extensions/EnumerableExtensions.cs | 90 ++++++-- LibMatrix/Extensions/MatrixHttpClient.Single.cs | 41 ++-- LibMatrix/Helpers/SyncHelper.cs | 28 ++- LibMatrix/Helpers/SyncStateResolver.cs | 251 +++++++++++++-------- LibMatrix/Responses/SyncResponse.cs | 6 +- LibMatrix/RoomTypes/GenericRoom.cs | 1 + .../Services/RoomStore.cs | 2 +- .../LibMatrix.Utilities.Bot/BotCommandInstaller.cs | 2 +- .../Services/CommandListenerHostedService.cs | 48 ++-- 9 files changed, 317 insertions(+), 152 deletions(-) (limited to 'LibMatrix/Extensions') diff --git a/LibMatrix/Extensions/EnumerableExtensions.cs b/LibMatrix/Extensions/EnumerableExtensions.cs index 42d9491..ace2c0c 100644 --- a/LibMatrix/Extensions/EnumerableExtensions.cs +++ b/LibMatrix/Extensions/EnumerableExtensions.cs @@ -1,29 +1,91 @@ +using System.Collections.Frozen; +using System.Collections.Immutable; + namespace LibMatrix.Extensions; public static class EnumerableExtensions { + public static int insertions = 0; + public static int replacements = 0; + public static void MergeStateEventLists(this IList oldState, IList newState) { - foreach (var stateEvent in newState) { - var old = oldState.FirstOrDefault(x => x.Type == stateEvent.Type && x.StateKey == stateEvent.StateKey); - if (old is null) { - oldState.Add(stateEvent); - continue; + // foreach (var stateEvent in newState) { + // var old = oldState.FirstOrDefault(x => x.Type == stateEvent.Type && x.StateKey == stateEvent.StateKey); + // if (old is null) { + // oldState.Add(stateEvent); + // continue; + // } + // + // oldState.Remove(old); + // oldState.Add(stateEvent); + // } + + foreach (var e in newState) { + switch (FindIndex(e)) { + case -1: + oldState.Add(e); + break; + case var index: + oldState[index] = e; + break; } + } - oldState.Remove(old); - oldState.Add(stateEvent); + int FindIndex(StateEvent needle) { + for (int i = 0; i < oldState.Count; i++) { + var old = oldState[i]; + if (old.Type == needle.Type && old.StateKey == needle.StateKey) + return i; + } + + return -1; } } public static void MergeStateEventLists(this IList oldState, IList newState) { - foreach (var stateEvent in newState) { - var old = oldState.FirstOrDefault(x => x.Type == stateEvent.Type && x.StateKey == stateEvent.StateKey); - if (old is null) { - oldState.Add(stateEvent); - continue; + foreach (var e in newState) { + switch (FindIndex(e)) { + case -1: + oldState.Add(e); + break; + case var index: + oldState[index] = e; + break; + } + } + + int FindIndex(StateEventResponse needle) { + for (int i = 0; i < oldState.Count; i++) { + var old = oldState[i]; + if (old.Type == needle.Type && old.StateKey == needle.StateKey) + return i; + } + + return -1; + } + } + + public static void MergeStateEventLists(this List oldState, List newState) { + foreach (var e in newState) { + switch (FindIndex(e)) { + case -1: + oldState.Add(e); + insertions++; + break; + case var index: + oldState[index] = e; + replacements++; + break; + } + } + + int FindIndex(StateEventResponse needle) { + for (int i = 0; i < oldState.Count; i++) { + var old = oldState[i]; + if (old.Type == needle.Type && old.StateKey == needle.StateKey) + return i; } - oldState.Remove(old); - oldState.Add(stateEvent); + return -1; } } } \ No newline at end of file diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs index cdc0dca..0e6d467 100644 --- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs +++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs @@ -51,6 +51,7 @@ public class MatrixHttpClient { internal SemaphoreSlim _rateLimitSemaphore { get; } = new(1, 1); #endif + private const bool LogRequests = true; public Dictionary AdditionalQueryParameters { get; set; } = new(); public Uri? BaseAddress { get; set; } @@ -72,7 +73,7 @@ public class MatrixHttpClient { public async Task SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); // if (!request.RequestUri.IsAbsoluteUri) - request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!); + request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!); var swWait = Stopwatch.StartNew(); #if SYNC_HTTPCLIENT await _rateLimitSemaphore.WaitAsync(cancellationToken); @@ -82,6 +83,9 @@ public class MatrixHttpClient { if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); if (!request.RequestUri.IsAbsoluteUri) request.RequestUri = new Uri(BaseAddress, request.RequestUri); + swWait.Stop(); + var swExec = Stopwatch.StartNew(); + foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value); foreach (var (key, value) in DefaultRequestHeaders) { if (request.Headers.Contains(key)) continue; @@ -90,7 +94,8 @@ public class MatrixHttpClient { request.Options.Set(new HttpRequestOptionsKey("WebAssemblyEnableStreamingResponse"), true); - Console.WriteLine("Sending " + request.Summarise(includeHeaders:true, includeQuery: true, includeContentIfText: true)); + if (LogRequests) + Console.WriteLine("Sending " + request.Summarise(includeHeaders: true, includeQuery: true, includeContentIfText: true, hideHeaders: ["Accept"])); HttpResponseMessage? responseMessage; try { @@ -108,19 +113,25 @@ public class MatrixHttpClient { #endif // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)"); - Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [ - "Server", - "Date", - "Transfer-Encoding", - "Connection", - "Vary", - "Content-Length", - "Access-Control-Allow-Origin", - "Access-Control-Allow-Methods", - "Access-Control-Allow-Headers", - "Access-Control-Expose-Headers", - "Cache-Control" - ])); + if (LogRequests) + Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [ + "Server", + "Date", + "Transfer-Encoding", + "Connection", + "Vary", + "Content-Length", + "Access-Control-Allow-Origin", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", + "Access-Control-Expose-Headers", + "Cache-Control", + "Cross-Origin-Resource-Policy", + "X-Content-Security-Policy", + "Referrer-Policy", + "X-Robots-Tag", + "Content-Security-Policy" + ])); return responseMessage; } diff --git a/LibMatrix/Helpers/SyncHelper.cs b/LibMatrix/Helpers/SyncHelper.cs index a7010ee..f95d6f8 100644 --- a/LibMatrix/Helpers/SyncHelper.cs +++ b/LibMatrix/Helpers/SyncHelper.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.Net.Http.Json; using System.Text.Json; using ArcaneLibs.Collections; +using System.Text.Json.Nodes; using ArcaneLibs.Extensions; using LibMatrix.Filters; using LibMatrix.Homeservers; @@ -21,6 +22,7 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg public string? Since { get; set; } public int Timeout { get; set; } = 30000; public string? SetPresence { get; set; } = "online"; + public bool UseInternalStreamingSync { get; set; } = true; public string? FilterId { get => _filterId; @@ -114,15 +116,23 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg // logger?.LogInformation("SyncHelper: Calling: {}", url); try { - var httpResp = await homeserver.ClientHttpClient.GetAsync(url, cancellationToken ?? CancellationToken.None); - if (httpResp is null) throw new NullReferenceException("Failed to send HTTP request"); - logger?.LogTrace("Got sync response: {} bytes, {} elapsed", httpResp.GetContentLength(), sw.Elapsed); - var deserializeSw = Stopwatch.StartNew(); - var stream = await httpResp.Content.ReadAsStreamAsync(); - await using var seekableStream = new SeekableStream(stream); - var resp = await JsonSerializer.DeserializeAsync(seekableStream, cancellationToken: cancellationToken ?? CancellationToken.None, - jsonTypeInfo: SyncResponseSerializerContext.Default.SyncResponse); - logger?.LogInformation("Deserialized sync response: {} bytes, {} elapsed, {} total", seekableStream.Position, deserializeSw.Elapsed, sw.Elapsed); + SyncResponse? resp = null; + if (UseInternalStreamingSync) { + resp = await homeserver.ClientHttpClient.GetFromJsonAsync(url, cancellationToken: cancellationToken ?? CancellationToken.None); + logger?.LogInformation("Got sync response: ~{} bytes, {} elapsed", resp.ToJson(false, true, true).Length, sw.Elapsed); + } + else { + var httpResp = await homeserver.ClientHttpClient.GetAsync(url, cancellationToken ?? CancellationToken.None); + if (httpResp is null) throw new NullReferenceException("Failed to send HTTP request"); + logger?.LogInformation("Got sync response: {} bytes, {} elapsed", httpResp.GetContentLength(), sw.Elapsed); + var deserializeSw = Stopwatch.StartNew(); + // var jsonResp = await httpResp.Content.ReadFromJsonAsync(cancellationToken: cancellationToken ?? CancellationToken.None); + // var resp = jsonResp.Deserialize(); + resp = await httpResp.Content.ReadFromJsonAsync(cancellationToken: cancellationToken ?? CancellationToken.None, + jsonTypeInfo: SyncResponseSerializerContext.Default.SyncResponse); + logger?.LogInformation("Deserialized sync response: {} bytes, {} elapsed, {} total", httpResp.GetContentLength(), deserializeSw.Elapsed, sw.Elapsed); + } + var timeToWait = MinimumDelay.Subtract(sw.Elapsed); if (timeToWait.TotalMilliseconds > 0) await Task.Delay(timeToWait); diff --git a/LibMatrix/Helpers/SyncStateResolver.cs b/LibMatrix/Helpers/SyncStateResolver.cs index 3be4492..282d26f 100644 --- a/LibMatrix/Helpers/SyncStateResolver.cs +++ b/LibMatrix/Helpers/SyncStateResolver.cs @@ -1,6 +1,7 @@ using System.Collections.Frozen; using System.Collections.Immutable; using System.Diagnostics; +using System.Text; using ArcaneLibs.Extensions; using LibMatrix.Extensions; using LibMatrix.Filters; @@ -40,15 +41,16 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge return (sync, MergedState); } - public async Task OptimiseStore() { + public async Task OptimiseStore(Action? progressCallback = null) { if (storageProvider is null) return; if (!await storageProvider.ObjectExistsAsync("init")) return; var totalSw = Stopwatch.StartNew(); Console.Write("Optimising sync store..."); var initLoadTask = storageProvider.LoadObjectAsync("init"); - var keys = (await storageProvider.GetAllKeysAsync()).Where(x=>!x.StartsWith("old/")).ToFrozenSet(); + var keys = (await storageProvider.GetAllKeysAsync()).Where(x => !x.StartsWith("old/")).ToFrozenSet(); var count = keys.Count - 1; + int total = count; Console.WriteLine($"Found {count} entries to optimise."); var merged = await initLoadTask; @@ -64,6 +66,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge var moveTasks = new List(); + Dictionary> traces = []; while (keys.Contains(merged.NextBatch)) { Console.Write($"Merging {merged.NextBatch}, {--count} remaining... "); var sw = Stopwatch.StartNew(); @@ -77,28 +80,36 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge moveTasks.Add(storageProvider.MoveObjectAsync(merged.NextBatch, $"{oldPath}/{merged.NextBatch}")); Console.Write($"Move {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); - merged = MergeSyncs(merged, next); + var trace = new Dictionary(); + traces[merged.NextBatch] = trace; + merged = MergeSyncs(merged, next, trace); Console.Write($"Merge {sw.GetElapsedAndRestart().TotalMilliseconds}ms... "); Console.WriteLine($"Total {swt.Elapsed.TotalMilliseconds}ms"); // Console.WriteLine($"Merged {merged.NextBatch}, {--count} remaining..."); + progressCallback?.Invoke(count, total); } + var traceString = string.Join("\n", traces.Select(x => $"{x.Key}\t{x.Value.ToJson(indent: false)}")); + var ms = new MemoryStream(Encoding.UTF8.GetBytes(traceString)); + await storageProvider.SaveStreamAsync($"traces/{oldPath}", ms); + await storageProvider.SaveObjectAsync("init", merged); await Task.WhenAll(moveTasks); - + Console.WriteLine($"Optimised store in {totalSw.Elapsed.TotalMilliseconds}ms"); + Console.WriteLine($"Insertions: {EnumerableExtensions.insertions}, replacements: {EnumerableExtensions.replacements}"); } /// /// Remove all but initial sync and last checkpoint /// public async Task RemoveOldSnapshots() { - if(storageProvider is null) return; + if (storageProvider is null) return; var sw = Stopwatch.StartNew(); var map = await GetCheckpointMap(); if (map is null) return; - if(map.Count < 3) return; + if (map.Count < 3) return; var toRemove = map.Keys.Skip(1).Take(map.Count - 2).ToList(); Console.Write("Cleaning up old snapshots: "); @@ -109,6 +120,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge await storageProvider?.DeleteObjectAsync(path); } } + Console.WriteLine("Done!"); Console.WriteLine($"Removed {toRemove.Count} old snapshots in {sw.Elapsed.TotalMilliseconds}ms"); } @@ -137,6 +149,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge // Console.WriteLine($"[{++i}] {key} -> {resp.NextBatch} ({resp.GetDerivedSyncTime()})"); i++; } + Console.WriteLine($"Iterated {i} syncResponses in {sw.Elapsed}"); Environment.Exit(0); } @@ -188,7 +201,7 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge if (resp.GetDerivedSyncTime() > unixTime) break; merged = MergeSyncs(merged, resp); } - + return merged; } @@ -208,29 +221,29 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge return map.OrderBy(x => x.Key).ToImmutableSortedDictionary(x => x.Key, x => x.Value.ToFrozenSet()); } - private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync) { + private SyncResponse MergeSyncs(SyncResponse oldSync, SyncResponse newSync, Dictionary? trace = null) { + var sw = Stopwatch.StartNew(); oldSync.NextBatch = newSync.NextBatch ?? oldSync.NextBatch; - oldSync.AccountData ??= new EventList(); - oldSync.AccountData.Events ??= []; - if (newSync.AccountData?.Events is not null) - oldSync.AccountData.Events.MergeStateEventLists(newSync.AccountData?.Events ?? []); + oldSync.AccountData = MergeEventList(oldSync.AccountData, newSync.AccountData); + trace?.Add("AccountData", sw.GetElapsedAndRestart()); - oldSync.Presence ??= new(); - oldSync.Presence.Events?.ReplaceBy(newSync.Presence?.Events ?? [], (oldState, newState) => oldState.Sender == newState.Sender && oldState.Type == newState.Type); + oldSync.Presence = MergeEventListBy(oldSync.Presence, newSync.Presence, (oldState, newState) => oldState.Sender == newState.Sender && oldState.Type == newState.Type); + trace?.Add("Presence", sw.GetElapsedAndRestart()); + // TODO: can this be cleaned up? oldSync.DeviceOneTimeKeysCount ??= new(); if (newSync.DeviceOneTimeKeysCount is not null) foreach (var (key, value) in newSync.DeviceOneTimeKeysCount) oldSync.DeviceOneTimeKeysCount[key] = value; + trace?.Add("DeviceOneTimeKeysCount", sw.GetElapsedAndRestart()); if (newSync.Rooms is not null) - oldSync.Rooms = MergeRoomsDataStructure(oldSync.Rooms, newSync.Rooms); + oldSync.Rooms = MergeRoomsDataStructure(oldSync.Rooms, newSync.Rooms, trace); + trace?.Add("Rooms", sw.GetElapsedAndRestart()); - oldSync.ToDevice ??= new EventList(); - oldSync.ToDevice.Events ??= []; - if (newSync.ToDevice?.Events is not null) - oldSync.ToDevice.Events.MergeStateEventLists(newSync.ToDevice?.Events ?? []); + oldSync.ToDevice = MergeEventList(oldSync.ToDevice, newSync.ToDevice); + trace?.Add("ToDevice", sw.GetElapsedAndRestart()); oldSync.DeviceLists ??= new SyncResponse.DeviceListsDataStructure(); oldSync.DeviceLists.Changed ??= []; @@ -241,125 +254,171 @@ public class SyncStateResolver(AuthenticatedHomeserverGeneric homeserver, ILogge oldSync.DeviceLists.Changed.Add(s); } + trace?.Add("DeviceLists.Changed", sw.GetElapsedAndRestart()); + if (newSync.DeviceLists?.Left is not null) foreach (var s in newSync.DeviceLists.Left!) { oldSync.DeviceLists.Changed.Remove(s); oldSync.DeviceLists.Left.Add(s); } - return oldSync; - } - - private List? MergePresenceEvents(List? oldEvents, List? newEvents) { - if (oldEvents is null) return newEvents; - if (newEvents is null) return oldEvents; + trace?.Add("DeviceLists.Left", sw.GetElapsedAndRestart()); - foreach (var newEvent in newEvents) { - oldEvents.RemoveAll(x => x.Sender == newEvent.Sender && x.Type == newEvent.Type); - oldEvents.Add(newEvent); - } - - return oldEvents; + return oldSync; } #region Merge rooms - private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure? oldState, SyncResponse.RoomsDataStructure newState) { + private SyncResponse.RoomsDataStructure MergeRoomsDataStructure(SyncResponse.RoomsDataStructure? oldState, SyncResponse.RoomsDataStructure newState, + Dictionary? trace) { + var sw = Stopwatch.StartNew(); if (oldState is null) return newState; - oldState.Join ??= new Dictionary(); - foreach (var (key, value) in newState.Join ?? new Dictionary()) - if (!oldState.Join.ContainsKey(key)) oldState.Join[key] = value; - else oldState.Join[key] = MergeJoinedRoomDataStructure(oldState.Join[key], value); - - oldState.Invite ??= new Dictionary(); - foreach (var (key, value) in newState.Invite ?? new Dictionary()) - if (!oldState.Invite.ContainsKey(key)) oldState.Invite[key] = value; - else oldState.Invite[key] = MergeInvitedRoomDataStructure(oldState.Invite[key], value); - - oldState.Leave ??= new Dictionary(); - foreach (var (key, value) in newState.Leave ?? new Dictionary()) { - if (!oldState.Leave.ContainsKey(key)) oldState.Leave[key] = value; - else oldState.Leave[key] = MergeLeftRoomDataStructure(oldState.Leave[key], value); - if (oldState.Invite.ContainsKey(key)) oldState.Invite.Remove(key); - if (oldState.Join.ContainsKey(key)) oldState.Join.Remove(key); - } + + if (newState.Join is { Count: > 0 }) + if (oldState.Join is null) + oldState.Join = newState.Join; + else + foreach (var (key, value) in newState.Join) + if (!oldState.Join.TryAdd(key, value)) + oldState.Join[key] = MergeJoinedRoomDataStructure(oldState.Join[key], value, trace); + trace?.Add("MergeRoomsDataStructure.Join", sw.GetElapsedAndRestart()); + + if (newState.Invite is { Count: > 0 }) + if (oldState.Invite is null) + oldState.Invite = newState.Invite; + else + foreach (var (key, value) in newState.Invite) + if (!oldState.Invite.TryAdd(key, value)) + oldState.Invite[key] = MergeInvitedRoomDataStructure(oldState.Invite[key], value, trace); + trace?.Add("MergeRoomsDataStructure.Invite", sw.GetElapsedAndRestart()); + + if (newState.Leave is { Count: > 0 }) + if (oldState.Leave is null) + oldState.Leave = newState.Leave; + else + foreach (var (key, value) in newState.Leave) { + if (!oldState.Leave.TryAdd(key, value)) + oldState.Leave[key] = MergeLeftRoomDataStructure(oldState.Leave[key], value, trace); + if (oldState.Invite?.ContainsKey(key) ?? false) oldState.Invite.Remove(key); + if (oldState.Join?.ContainsKey(key) ?? false) oldState.Join.Remove(key); + } + trace?.Add("MergeRoomsDataStructure.Leave", sw.GetElapsedAndRestart()); return oldState; } private static SyncResponse.RoomsDataStructure.LeftRoomDataStructure MergeLeftRoomDataStructure(SyncResponse.RoomsDataStructure.LeftRoomDataStructure oldData, - SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData) { - oldData.AccountData ??= new EventList(); - oldData.AccountData.Events ??= []; - oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure(); - oldData.Timeline.Events ??= []; - oldData.State ??= new EventList(); - oldData.State.Events ??= []; - - if (newData.AccountData?.Events is not null) - oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []); - - if (newData.Timeline?.Events is not null) - oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []); + SyncResponse.RoomsDataStructure.LeftRoomDataStructure newData, Dictionary? trace) { + var sw = Stopwatch.StartNew(); + + oldData.AccountData = MergeEventList(oldData.AccountData, newData.AccountData); + trace?.Add($"LeftRoomDataStructure.AccountData/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); + + oldData.Timeline = AppendEventList(oldData.Timeline, newData.Timeline) as SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure + ?? throw new InvalidOperationException("Merged room timeline was not TimelineDataStructure"); oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited; oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch; + trace?.Add($"LeftRoomDataStructure.Timeline/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); - if (newData.State?.Events is not null) - oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []); + oldData.State = MergeEventList(oldData.State, newData.State); + trace?.Add($"LeftRoomDataStructure.State/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); return oldData; } private static SyncResponse.RoomsDataStructure.InvitedRoomDataStructure MergeInvitedRoomDataStructure(SyncResponse.RoomsDataStructure.InvitedRoomDataStructure oldData, - SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData) { - oldData.InviteState ??= new EventList(); - oldData.InviteState.Events ??= []; - if (newData.InviteState?.Events is not null) - oldData.InviteState.Events.MergeStateEventLists(newData.InviteState?.Events ?? []); + SyncResponse.RoomsDataStructure.InvitedRoomDataStructure newData, Dictionary? trace) { + var sw = Stopwatch.StartNew(); + oldData.InviteState = MergeEventList(oldData.InviteState, newData.InviteState); + trace?.Add($"InvitedRoomDataStructure.InviteState/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); return oldData; } private static SyncResponse.RoomsDataStructure.JoinedRoomDataStructure MergeJoinedRoomDataStructure(SyncResponse.RoomsDataStructure.JoinedRoomDataStructure oldData, - SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData) { - oldData.AccountData ??= new EventList(); - oldData.AccountData.Events ??= []; - oldData.Timeline ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure(); - oldData.Timeline.Events ??= []; - oldData.State ??= new EventList(); - oldData.State.Events ??= []; - oldData.Ephemeral ??= new EventList(); - oldData.Ephemeral.Events ??= []; - - if (newData.AccountData?.Events is not null) - oldData.AccountData.Events.MergeStateEventLists(newData.AccountData?.Events ?? []); - - if (newData.Timeline?.Events is not null) - oldData.Timeline.Events.MergeStateEventLists(newData.Timeline?.Events ?? []); + SyncResponse.RoomsDataStructure.JoinedRoomDataStructure newData, Dictionary? trace) { + var sw = Stopwatch.StartNew(); + + oldData.AccountData = MergeEventList(oldData.AccountData, newData.AccountData); + trace?.Add($"JoinedRoomDataStructure.AccountData/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); + + oldData.Timeline = AppendEventList(oldData.Timeline, newData.Timeline) as SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.TimelineDataStructure + ?? throw new InvalidOperationException("Merged room timeline was not TimelineDataStructure"); oldData.Timeline.Limited = newData.Timeline?.Limited ?? oldData.Timeline.Limited; oldData.Timeline.PrevBatch = newData.Timeline?.PrevBatch ?? oldData.Timeline.PrevBatch; + trace?.Add($"JoinedRoomDataStructure.Timeline/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); - if (newData.State?.Events is not null) - oldData.State.Events.MergeStateEventLists(newData.State?.Events ?? []); + oldData.State = MergeEventList(oldData.State, newData.State); + trace?.Add($"JoinedRoomDataStructure.State/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); - if (newData.Ephemeral?.Events is not null) - oldData.Ephemeral.Events.MergeStateEventLists(newData.Ephemeral?.Events ?? []); + oldData.Ephemeral = MergeEventList(oldData.Ephemeral, newData.Ephemeral); + trace?.Add($"JoinedRoomDataStructure.Ephemeral/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); oldData.UnreadNotifications ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.UnreadNotificationsDataStructure(); oldData.UnreadNotifications.HighlightCount = newData.UnreadNotifications?.HighlightCount ?? oldData.UnreadNotifications.HighlightCount; oldData.UnreadNotifications.NotificationCount = newData.UnreadNotifications?.NotificationCount ?? oldData.UnreadNotifications.NotificationCount; + trace?.Add($"JoinedRoom$DataStructure.UnreadNotifications/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); + + if (oldData.Summary is null) + oldData.Summary = newData.Summary; + else { + oldData.Summary.Heroes = newData.Summary?.Heroes ?? oldData.Summary.Heroes; + oldData.Summary.JoinedMemberCount = newData.Summary?.JoinedMemberCount ?? oldData.Summary.JoinedMemberCount; + oldData.Summary.InvitedMemberCount = newData.Summary?.InvitedMemberCount ?? oldData.Summary.InvitedMemberCount; + } - oldData.Summary ??= new SyncResponse.RoomsDataStructure.JoinedRoomDataStructure.SummaryDataStructure { - Heroes = newData.Summary?.Heroes ?? oldData.Summary.Heroes, - JoinedMemberCount = newData.Summary?.JoinedMemberCount ?? oldData.Summary.JoinedMemberCount, - InvitedMemberCount = newData.Summary?.InvitedMemberCount ?? oldData.Summary.InvitedMemberCount - }; - oldData.Summary.Heroes = newData.Summary?.Heroes ?? oldData.Summary.Heroes; - oldData.Summary.JoinedMemberCount = newData.Summary?.JoinedMemberCount ?? oldData.Summary.JoinedMemberCount; - oldData.Summary.InvitedMemberCount = newData.Summary?.InvitedMemberCount ?? oldData.Summary.InvitedMemberCount; + trace?.Add($"JoinedRoomDataStructure.Summary/{oldData.GetHashCode()}", sw.GetElapsedAndRestart()); return oldData; } #endregion + + private static EventList? MergeEventList(EventList? oldState, EventList? newState) { + if (newState is null) return oldState; + if (oldState is null) { + return newState; + } + + if (newState.Events is null) return oldState; + if (oldState.Events is null) { + oldState.Events = newState.Events; + return oldState; + } + + oldState.Events.MergeStateEventLists(newState.Events); + return oldState; + } + + private static EventList? MergeEventListBy(EventList? oldState, EventList? newState, Func comparer) { + if (newState is null) return oldState; + if (oldState is null) { + return newState; + } + + if (newState.Events is null) return oldState; + if (oldState.Events is null) { + oldState.Events = newState.Events; + return oldState; + } + + oldState.Events.ReplaceBy(newState.Events, comparer); + return oldState; + } + + private static EventList? AppendEventList(EventList? oldState, EventList? newState) { + if (newState is null) return oldState; + if (oldState is null) { + return newState; + } + + if (newState.Events is null) return oldState; + if (oldState.Events is null) { + oldState.Events = newState.Events; + return oldState; + } + + oldState.Events.AddRange(newState.Events); + return oldState; + } } \ No newline at end of file diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs index 9b4ce05..a4391b7 100644 --- a/LibMatrix/Responses/SyncResponse.cs +++ b/LibMatrix/Responses/SyncResponse.cs @@ -90,7 +90,7 @@ public class SyncResponse { [JsonPropertyName("summary")] public SummaryDataStructure? Summary { get; set; } - public class TimelineDataStructure { + public class TimelineDataStructure : EventList { public TimelineDataStructure() { } public TimelineDataStructure(List? events, bool? limited) { @@ -98,8 +98,8 @@ public class SyncResponse { Limited = limited; } - [JsonPropertyName("events")] - public List? Events { get; set; } + // [JsonPropertyName("events")] + // public List? Events { get; set; } [JsonPropertyName("prev_batch")] public string? PrevBatch { get; set; } diff --git a/LibMatrix/RoomTypes/GenericRoom.cs b/LibMatrix/RoomTypes/GenericRoom.cs index 02bd555..84a3d30 100644 --- a/LibMatrix/RoomTypes/GenericRoom.cs +++ b/LibMatrix/RoomTypes/GenericRoom.cs @@ -1,6 +1,7 @@ using System.Collections.Frozen; using System.Diagnostics; using System.Net.Http.Json; +using System.Security.Cryptography; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; diff --git a/Utilities/LibMatrix.HomeserverEmulator/Services/RoomStore.cs b/Utilities/LibMatrix.HomeserverEmulator/Services/RoomStore.cs index 5cdc3ab..c798cce 100644 --- a/Utilities/LibMatrix.HomeserverEmulator/Services/RoomStore.cs +++ b/Utilities/LibMatrix.HomeserverEmulator/Services/RoomStore.cs @@ -137,7 +137,7 @@ public class RoomStore { public Room(string roomId) { if (string.IsNullOrWhiteSpace(roomId)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(roomId)); - if (roomId[0] != '!') throw new ArgumentException("Room ID must start with !", nameof(roomId)); + if (roomId[0] != '!') throw new ArgumentException($"Room ID must start with '!', provided value: {roomId ?? "null"}", nameof(roomId)); RoomId = roomId; Timeline = new(); AccountData = new(); diff --git a/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs b/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs index ca6a4d8..8501d41 100644 --- a/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs +++ b/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs @@ -18,7 +18,7 @@ public class BotInstaller(IServiceCollection services) { public BotInstaller AddMatrixBot() { services.AddSingleton(); - services.AddScoped(x => { + services.AddSingleton(x => { var config = x.GetService() ?? throw new Exception("No configuration found!"); var hsProvider = x.GetService() ?? throw new Exception("No homeserver provider found!"); diff --git a/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs b/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs index 601e598..9a7585e 100644 --- a/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs +++ b/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs @@ -59,22 +59,44 @@ public class CommandListenerHostedService : IHostedService { FilterId = filter }; - syncHelper.TimelineEventHandlers.Add(async @event => { - try { - var room = _hs.GetRoom(@event.RoomId); - // _logger.LogInformation(eventResponse.ToJson(indent: false)); - if (@event is { Type: "m.room.message", TypedContent: RoomMessageEventContent message }) - if (message is { MessageType: "m.text" }) { - var usedPrefix = await GetUsedPrefix(@event); - if (usedPrefix is null) return; - var res = await InvokeCommand(@event, usedPrefix); - await (_commandResultHandler?.Invoke(res) ?? HandleResult(res)); + syncHelper.SyncReceivedHandlers.Add(async sync => { + _logger.LogInformation("Sync received!"); + foreach (var roomResp in sync.Rooms?.Join ?? []) { + if (roomResp.Value.Timeline?.Events is null or { Count: > 5 }) continue; + foreach (var @event in roomResp.Value.Timeline.Events) { + @event.RoomId = roomResp.Key; + try { + // var room = _hs.GetRoom(@event.RoomId); + // _logger.LogInformation(eventResponse.ToJson(indent: false)); + if (@event is { Type: "m.room.message", TypedContent: RoomMessageEventContent message }) + if (message is { MessageType: "m.text" }) { + var usedPrefix = await GetUsedPrefix(@event); + if (usedPrefix is null) return; + var res = await InvokeCommand(@event, usedPrefix); + await (_commandResultHandler?.Invoke(res) ?? HandleResult(res)); + } } - } - catch (Exception e) { - _logger.LogError(e, "Error in command listener!"); + catch (Exception e) { + _logger.LogError(e, "Error in command listener!"); + Console.WriteLine(@event.ToJson(ignoreNull: false, indent: true)); + var fakeResult = new CommandResult() { + Result = CommandResult.CommandResultType.Failure_Exception, + Exception = e, + Success = false, + Context = new() { + Homeserver = _hs, + CommandName = "[CommandListener.SyncHandler]", + Room = _hs.GetRoom(roomResp.Key), + Args = [], + MessageEvent = @event + } + }; + await (_commandResultHandler?.Invoke(fakeResult) ?? HandleResult(fakeResult)); + } + } } }); + await syncHelper.RunSyncLoopAsync(cancellationToken: cancellationToken); } -- cgit 1.5.1