From 9f8d0c85c54b4715974994aea52562072d6f1751 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Wed, 31 Jan 2024 12:09:28 +0100 Subject: Better sync filter support, named filters, error handling --- LibMatrix/Filters/SyncFilter.cs | 24 ++-- LibMatrix/Helpers/HomeserverWeightEstimation.cs | 2 +- LibMatrix/Helpers/SyncHelper.cs | 56 +++++++++- .../Homeservers/AuthenticatedHomeserverGeneric.cs | 124 +++++++++++++++++++-- LibMatrix/LibMatrix.csproj | 1 - LibMatrix/RoomTypes/GenericRoom.cs | 2 +- LibMatrix/StateEvent.cs | 15 --- LibMatrix/Utilities/CommonSyncFilters.cs | 73 ++++++++++++ 8 files changed, 248 insertions(+), 49 deletions(-) create mode 100644 LibMatrix/Utilities/CommonSyncFilters.cs (limited to 'LibMatrix') diff --git a/LibMatrix/Filters/SyncFilter.cs b/LibMatrix/Filters/SyncFilter.cs index b05f6b4..5ffef4d 100644 --- a/LibMatrix/Filters/SyncFilter.cs +++ b/LibMatrix/Filters/SyncFilter.cs @@ -24,6 +24,15 @@ public class SyncFilter { [JsonPropertyName("timeline")] public StateFilter? Timeline { get; set; } + + [JsonPropertyName("rooms")] + public List? Rooms { get; set; } + + [JsonPropertyName("not_rooms")] + public List? NotRooms { get; set; } + + [JsonPropertyName("include_leave")] + public bool? IncludeLeave { get; set; } public class StateFilter(bool? containsUrl = null, bool? includeRedundantMembers = null, bool? lazyLoadMembers = null, List? rooms = null, List? notRooms = null, bool? unreadThreadNotifications = null, @@ -66,17 +75,4 @@ public class SyncFilter { [JsonPropertyName("not_senders")] public List? NotSenders { get; set; } = notSenders; } -} - -public static class ExampleFilters { - public static readonly SyncFilter Limit1Filter = new() { - Presence = new(limit: 1), - Room = new() { - AccountData = new(limit: 1), - Ephemeral = new(limit: 1), - State = new(limit: 1), - Timeline = new(limit: 1), - }, - AccountData = new(limit: 1) - }; -} +} \ No newline at end of file diff --git a/LibMatrix/Helpers/HomeserverWeightEstimation.cs b/LibMatrix/Helpers/HomeserverWeightEstimation.cs index 8f1bf3a..02f9185 100644 --- a/LibMatrix/Helpers/HomeserverWeightEstimation.cs +++ b/LibMatrix/Helpers/HomeserverWeightEstimation.cs @@ -2,7 +2,7 @@ namespace LibMatrix.Helpers; public class HomeserverWeightEstimation { public static Dictionary EstimatedSize = new() { - { "matrix.org", 84387 }, + { "matrix.org", 843870 }, { "anontier.nl", 44809 }, { "nixos.org", 8195 }, { "the-apothecary.club", 6983 }, diff --git a/LibMatrix/Helpers/SyncHelper.cs b/LibMatrix/Helpers/SyncHelper.cs index 691b964..636cfdd 100644 --- a/LibMatrix/Helpers/SyncHelper.cs +++ b/LibMatrix/Helpers/SyncHelper.cs @@ -11,16 +11,59 @@ using Microsoft.Extensions.Logging; namespace LibMatrix.Helpers; public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logger = null) { + private SyncFilter? _filter; + private string? _namedFilterName; + private bool _filterIsDirty = false; + private string? _filterId = null; + public string? Since { get; set; } public int Timeout { get; set; } = 30000; public string? SetPresence { get; set; } = "online"; - public SyncFilter? Filter { get; set; } + + public string? FilterId { + get => _filterId; + set { + _filterId = value; + _namedFilterName = null; + _filter = null; + } + } + public string? NamedFilterName { + get => _namedFilterName; + set { + _namedFilterName = value; + _filterIsDirty = true; + _filterId = null; + } + } + + public SyncFilter? Filter { + get => _filter; + set { + _filter = value; + _filterIsDirty = true; + _filterId = null; + } + } + public bool FullState { get; set; } public bool IsInitialSync { get; set; } = true; public TimeSpan MinimumDelay { get; set; } = new(0); + private async Task updateFilterAsync() { + if (!string.IsNullOrWhiteSpace(NamedFilterName)) { + _filterId = await homeserver.GetNamedFilterIdOrNullAsync(NamedFilterName); + if (_filterId is null) + if (logger is null) Console.WriteLine($"Failed to get filter ID for named filter {NamedFilterName}"); + else logger.LogWarning("Failed to get filter ID for named filter {NamedFilterName}", NamedFilterName); + } + else if (Filter is not null) + _filterId = (await homeserver.UploadFilterAsync(Filter)).FilterId; + else _filterId = null; + } + public async Task SyncAsync(CancellationToken? cancellationToken = null) { if (homeserver is null) { Console.WriteLine("Null passed as homeserver for SyncHelper!"); @@ -33,12 +76,14 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg } var sw = Stopwatch.StartNew(); + if (_filterIsDirty) await updateFilterAsync(); var url = $"/_matrix/client/v3/sync?timeout={Timeout}&set_presence={SetPresence}&full_state={(FullState ? "true" : "false")}"; if (!string.IsNullOrWhiteSpace(Since)) url += $"&since={Since}"; - if (Filter is not null) url += $"&filter={Filter.ToJson(ignoreNull: true, indent: false)}"; - // Console.WriteLine("Calling: " + url); + if (_filterId is not null) url += $"&filter={_filterId}"; + logger?.LogInformation("SyncHelper: Calling: {}", url); + try { var httpResp = await homeserver.ClientHttpClient.GetAsync(url, cancellationToken: cancellationToken ?? CancellationToken.None); if (httpResp is null) throw new NullReferenceException("Failed to send HTTP request"); @@ -99,14 +144,15 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg }, ToDevice: null or { Events: null or { Count: 0 } - } + } }) { emptyInitialSyncCount++; if (emptyInitialSyncCount >= 2) { IsInitialSync = false; Timeout = oldTimeout; } - } else if (syncCount > 15) + } + else if (syncCount > 15) Console.WriteLine(sync.ToJson(ignoreNull: true, indent: true)); await RunSyncLoopCallbacksAsync(sync, IsInitialSync && skipInitialSyncEvents); diff --git a/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs b/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs index 5db9a48..ef6fa68 100644 --- a/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs +++ b/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs @@ -12,19 +12,21 @@ using LibMatrix.Helpers; using LibMatrix.Responses; using LibMatrix.RoomTypes; using LibMatrix.Services; +using LibMatrix.Utilities; namespace LibMatrix.Homeservers; public class AuthenticatedHomeserverGeneric(string serverName, string accessToken) : RemoteHomeserver(serverName) { public static async Task Create(string serverName, string accessToken, string? proxy = null) where T : AuthenticatedHomeserverGeneric => await Create(typeof(T), serverName, accessToken, proxy) as T ?? throw new InvalidOperationException($"Failed to create instance of {typeof(T).Name}"); + public static async Task Create(Type type, string serverName, string accessToken, string? proxy = null) { if (string.IsNullOrWhiteSpace(proxy)) proxy = null; - if(!type.IsAssignableTo(typeof(AuthenticatedHomeserverGeneric))) throw new ArgumentException("Type must be a subclass of AuthenticatedHomeserverGeneric", nameof(type)); + if (!type.IsAssignableTo(typeof(AuthenticatedHomeserverGeneric))) throw new ArgumentException("Type must be a subclass of AuthenticatedHomeserverGeneric", nameof(type)); var instance = Activator.CreateInstance(type, serverName, accessToken) as AuthenticatedHomeserverGeneric ?? throw new InvalidOperationException($"Failed to create instance of {type.Name}"); - + instance.ClientHttpClient = new() { Timeout = TimeSpan.FromMinutes(15), DefaultRequestHeaders = { @@ -44,7 +46,6 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke instance.WhoAmI = await instance.ClientHttpClient.GetFromJsonAsync("/_matrix/client/v3/account/whoami"); - return instance; } @@ -127,7 +128,7 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke } } - #region Utility Functions +#region Utility Functions public virtual async IAsyncEnumerable GetJoinedRoomsByType(string type) { var rooms = await GetJoinedRooms(); @@ -145,9 +146,9 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke } } - #endregion +#endregion - #region Account Data +#region Account Data public virtual async Task GetAccountDataAsync(string key) { // var res = await _httpClient.GetAsync($"/_matrix/client/v3/user/{UserId}/account_data/{key}"); @@ -168,7 +169,7 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke } } - #endregion +#endregion public async Task UpdateProfileAsync(UserProfileResponse? newProfile, bool preserveCustomRoomProfile = true) { if (newProfile is null) return; @@ -290,17 +291,116 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke return await res.Content.ReadFromJsonAsync() ?? throw new Exception("Failed to join room?"); } - #region Room Profile Utility +#region Room Profile Utility private async Task> GetOwnRoomProfileWithIdAsync(GenericRoom room) { return new KeyValuePair(room.RoomId, await room.GetStateAsync("m.room.member", WhoAmI.UserId!)); } - #endregion - +#endregion + public async Task SetImpersonate(string mxid) { - if(ClientHttpClient.AdditionalQueryParameters.TryGetValue("user_id", out var existingMxid) && existingMxid == mxid && WhoAmI.UserId == mxid) return; + if (ClientHttpClient.AdditionalQueryParameters.TryGetValue("user_id", out var existingMxid) && existingMxid == mxid && WhoAmI.UserId == mxid) return; ClientHttpClient.AdditionalQueryParameters["user_id"] = mxid; WhoAmI = await ClientHttpClient.GetFromJsonAsync("/_matrix/client/v3/account/whoami"); } -} + + public async Task UploadFilterAsync(SyncFilter filter) { + var resp = await ClientHttpClient.PostAsJsonAsync("/_matrix/client/v3/user/" + UserId + "/filter", filter); + return await resp.Content.ReadFromJsonAsync() ?? throw new Exception("Failed to upload filter?"); + } + + public async Task GetFilterAsync(string filterId) { + if (_filterCache.TryGetValue(filterId, out var filter)) return filter; + var resp = await ClientHttpClient.GetAsync("/_matrix/client/v3/user/" + UserId + "/filter/" + filterId); + return _filterCache[filterId] = await resp.Content.ReadFromJsonAsync() ?? throw new Exception("Failed to get filter?"); + } + +#region Named filters + + private async Task?> GetNamedFilterListOrNullAsync(bool cached = true) { + if (cached && _namedFilterCache is not null) return _namedFilterCache; + try { + return _namedFilterCache = await GetAccountDataAsync>("gay.rory.libmatrix.named_filters"); + } + catch (MatrixException e) { + if (e is not { ErrorCode: "M_NOT_FOUND" }) throw; + } + + return null; + } + + /// + /// Utility function to allow avoiding serverside duplication + /// + /// Name of the filter (please properly namespace and possibly version this...) + /// The filter data + /// Filter ID response + /// + public async Task UploadNamedFilterAsync(string filterName, SyncFilter filter) { + var resp = await ClientHttpClient.PostAsJsonAsync("/_matrix/client/v3/user/" + UserId + "/filter", filter); + var idResp = await resp.Content.ReadFromJsonAsync() ?? throw new Exception("Failed to upload filter?"); + + var filterList = await GetNamedFilterListOrNullAsync() ?? new(); + filterList[filterName] = idResp.FilterId; + await SetAccountDataAsync("gay.rory.libmatrix.named_filters", filterList); + + _namedFilterCache = filterList; + + return idResp; + } + + public async Task GetNamedFilterIdOrNullAsync(string filterName) { + var filterList = await GetNamedFilterListOrNullAsync() ?? new(); + return filterList.GetValueOrDefault(filterName); //todo: validate that filter exists + } + + public async Task GetNamedFilterOrNullAsync(string filterName) { + var filterId = await GetNamedFilterIdOrNullAsync(filterName); + if (filterId is null) return null; + return await GetFilterAsync(filterId); + } + + public async Task GetOrUploadNamedFilterIdAsync(string filterName, SyncFilter? filter = null) { + var filterId = await GetNamedFilterIdOrNullAsync(filterName); + if (filterId is not null) return filterId; + if (filter is null && CommonSyncFilters.FilterMap.TryGetValue(filterName, out var commonFilter)) filter = commonFilter; + if (filter is null) throw new ArgumentException($"Filter is null and no common filter was found, filterName={filterName}", nameof(filter)); + var idResp = await UploadNamedFilterAsync(filterName, filter); + return idResp.FilterId; + } + +#endregion + + public class FilterIdResponse { + [JsonPropertyName("filter_id")] + public required string FilterId { get; set; } + } + + public async Task> EnumerateAccountDataPerRoom(bool includeGlobal = false) { + var syncHelper = new SyncHelper(this); + syncHelper.FilterId = await GetOrUploadNamedFilterIdAsync(CommonSyncFilters.GetAccountDataWithRooms); + var resp = await syncHelper.SyncAsync(); + if(resp is null) throw new Exception("Sync failed"); + var perRoomAccountData = new Dictionary(); + + if(includeGlobal) + perRoomAccountData[""] = resp.AccountData; + foreach (var (roomId, room) in resp.Rooms?.Join ?? []) { + perRoomAccountData[roomId] = room.AccountData; + } + + return perRoomAccountData; + } + + public async Task EnumerateAccountData() { + var syncHelper = new SyncHelper(this); + syncHelper.FilterId = await GetOrUploadNamedFilterIdAsync(CommonSyncFilters.GetAccountData); + var resp = await syncHelper.SyncAsync(); + if(resp is null) throw new Exception("Sync failed"); + return resp.AccountData; + } + + private Dictionary? _namedFilterCache; + private Dictionary _filterCache = new(); +} \ No newline at end of file diff --git a/LibMatrix/LibMatrix.csproj b/LibMatrix/LibMatrix.csproj index 16e43f5..e6b091f 100644 --- a/LibMatrix/LibMatrix.csproj +++ b/LibMatrix/LibMatrix.csproj @@ -11,7 +11,6 @@ - diff --git a/LibMatrix/RoomTypes/GenericRoom.cs b/LibMatrix/RoomTypes/GenericRoom.cs index f5cbc51..38ced58 100644 --- a/LibMatrix/RoomTypes/GenericRoom.cs +++ b/LibMatrix/RoomTypes/GenericRoom.cs @@ -16,7 +16,7 @@ using LibMatrix.Services; namespace LibMatrix.RoomTypes; public class GenericRoom { - internal readonly AuthenticatedHomeserverGeneric Homeserver; + public readonly AuthenticatedHomeserverGeneric Homeserver; public GenericRoom(AuthenticatedHomeserverGeneric homeserver, string roomId) { if (string.IsNullOrWhiteSpace(roomId)) diff --git a/LibMatrix/StateEvent.cs b/LibMatrix/StateEvent.cs index 1a8df11..d78939e 100644 --- a/LibMatrix/StateEvent.cs +++ b/LibMatrix/StateEvent.cs @@ -8,7 +8,6 @@ using System.Text.Json.Serialization; using ArcaneLibs; using ArcaneLibs.Attributes; using ArcaneLibs.Extensions; -using Castle.DynamicProxy; using LibMatrix.EventTypes; using LibMatrix.Extensions; @@ -49,18 +48,6 @@ public class StateEvent { new JsonDecimalStringConverter() } }; - - private class EventContentInterceptor : IInterceptor { - public void Intercept(IInvocation invocation) { - Console.WriteLine($"Intercepting {invocation.Method.Name}"); - // if (invocation.Method.Name == "ToString") { - // invocation.ReturnValue = "EventContent"; - // return; - // } - - invocation.Proceed(); - } - } [JsonIgnore] [SuppressMessage("ReSharper", "PropertyCanBeMadeInitOnly.Global")] @@ -71,8 +58,6 @@ public class StateEvent { // } try { var c= (EventContent)RawContent.Deserialize(GetStateEventType(Type), TypedContentSerializerOptions)!; - // c = (EventContent)new ProxyGenerator().CreateClassProxyWithTarget(GetStateEventType(Type), c, new EventContentInterceptor()); - // Console.WriteLine(c.GetType().Name + ": " + string.Join(", ", c.GetType().GetRuntimeProperties().Select(x=>x.Name))); return c; } catch (JsonException e) { diff --git a/LibMatrix/Utilities/CommonSyncFilters.cs b/LibMatrix/Utilities/CommonSyncFilters.cs new file mode 100644 index 0000000..7cf1b41 --- /dev/null +++ b/LibMatrix/Utilities/CommonSyncFilters.cs @@ -0,0 +1,73 @@ +using System.Collections.Frozen; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.Filters; + +namespace LibMatrix.Utilities; + +public static class CommonSyncFilters { + public const string GetAccountData = "gay.rory.libmatrix.get_account_data.v0"; + public const string GetAccountDataWithRooms = "gay.rory.libmatrix.get_account_data_with_rooms.v0"; + public const string GetBasicRoomInfo = "gay.rory.matrixutils.get_basic_room_info.v0"; + public const string GetSpaceRelations = "gay.rory.matrixutils.get_space_relations.v0"; + + public static readonly SyncFilter GetAccountDataFilter = new() { + Presence = new SyncFilter.EventFilter(notTypes: ["*"]), + Room = new SyncFilter.RoomFilter() { + Rooms = [] + } + }; + + public static readonly SyncFilter GetAccountDataWithRoomsFilter = new() { + Presence = new SyncFilter.EventFilter(notTypes: ["*"]), + Room = new SyncFilter.RoomFilter() { + State = new SyncFilter.RoomFilter.StateFilter(notTypes: ["*"]), + Ephemeral = new SyncFilter.RoomFilter.StateFilter(notTypes: ["*"]), + Timeline = new SyncFilter.RoomFilter.StateFilter(notTypes: ["*"]) + } + }; + + public static readonly SyncFilter GetBasicRoomDataFilter = new() { + AccountData = new SyncFilter.EventFilter(notTypes: ["*"], limit: 1), + Presence = new SyncFilter.EventFilter(notTypes: ["*"], limit: 1), + Room = new SyncFilter.RoomFilter { + AccountData = new SyncFilter.RoomFilter.StateFilter(rooms: []), + Ephemeral = new SyncFilter.RoomFilter.StateFilter(rooms: []), + State = new SyncFilter.RoomFilter.StateFilter { + Types = new List { + "m.room.create", + "m.room.name", + "m.room.avatar", + "org.matrix.mjolnir.shortcode", + "m.room.power_levels", + }, + LazyLoadMembers = true, IncludeRedundantMembers = false + }, + Timeline = new SyncFilter.RoomFilter.StateFilter(rooms: []), + } + }; + + public static readonly SyncFilter GetSpaceRelationsFilter = new() { + AccountData = new SyncFilter.EventFilter(notTypes: ["*"], limit: 1), + Presence = new SyncFilter.EventFilter(notTypes: ["*"], limit: 1), + Room = new SyncFilter.RoomFilter { + AccountData = new SyncFilter.RoomFilter.StateFilter(rooms: []), + Ephemeral = new SyncFilter.RoomFilter.StateFilter(rooms: []), + State = new SyncFilter.RoomFilter.StateFilter { + Types = new List { + "m.space.child", + "m.space.parent" + }, + LazyLoadMembers = true, IncludeRedundantMembers = false + }, + Timeline = new SyncFilter.RoomFilter.StateFilter(rooms: []), + } + }; + + // This must be down here, due to statics load order + public static readonly FrozenDictionary FilterMap = new Dictionary() { + [GetAccountData] = GetAccountDataFilter, + [GetAccountDataWithRooms] = GetAccountDataWithRoomsFilter, + [GetBasicRoomInfo] = GetBasicRoomDataFilter, + [GetSpaceRelations] = GetSpaceRelationsFilter + }.ToFrozenDictionary(); +} \ No newline at end of file -- cgit 1.4.1