From ede3857084bc7c6e65b7d36cbf913b09596e2787 Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 8 Jan 2024 13:55:15 +0100 Subject: Internal changes to policy list viewer (extensibility), fix duplicating change handler for room list page (performance), use /state in room list page before sync --- MatrixRoomUtils.Web/Pages/Rooms/Index.razor | 102 ++++--- MatrixRoomUtils.Web/Pages/Rooms/PolicyList.razor | 336 +++++++++++++---------- 2 files changed, 248 insertions(+), 190 deletions(-) (limited to 'MatrixRoomUtils.Web/Pages/Rooms') diff --git a/MatrixRoomUtils.Web/Pages/Rooms/Index.razor b/MatrixRoomUtils.Web/Pages/Rooms/Index.razor index 2ac4bcb..6cabe82 100644 --- a/MatrixRoomUtils.Web/Pages/Rooms/Index.razor +++ b/MatrixRoomUtils.Web/Pages/Rooms/Index.razor @@ -1,25 +1,21 @@ @page "/Rooms" @using LibMatrix.Filters @using LibMatrix.Helpers -@using LibMatrix.EventTypes.Spec.State -@using LibMatrix -@using LibMatrix.Homeservers -@using ArcaneLibs.Extensions @using LibMatrix.Extensions @using LibMatrix.Responses @using System.Collections.ObjectModel @using System.Diagnostics +@using ArcaneLibs.Extensions +@using MatrixRoomUtils.Abstractions @inject ILogger logger

Room list

@Status

@Status2

-@* @if (RenderContents) { *@ + +Create new room + -@* } *@ -@* else { *@ -@* *@ -@* } *@ @code { private ObservableCollection Rooms { get; } = new(); @@ -47,9 +43,9 @@ }, State = new SyncFilter.RoomFilter.StateFilter { Types = new List { + "m.room.create", "m.room.name", "m.room.avatar", - "m.room.create", "org.matrix.mjolnir.shortcode", "m.room.power_levels", } @@ -92,49 +88,72 @@ // } // }; + private SyncHelper syncHelper; + + // SyncHelper profileSyncHelper; + protected override async Task OnInitializedAsync() { Homeserver = await MRUStorage.GetCurrentSessionOrNavigate(); if (Homeserver is null) return; var rooms = await Homeserver.GetJoinedRooms(); - foreach (var room in rooms) { - Rooms.Add(new(){Room = room}); + // SemaphoreSlim _semaphore = new(160, 160); + + var roomTasks = rooms.Select(async room => { + RoomInfo ri; + // await _semaphore.WaitAsync(); + ri = new() { Room = room }; + await Task.WhenAll((filter.Room?.State?.Types ?? []).Select(x => ri.GetStateEvent(x))); + return ri; + }).ToAsyncEnumerable(); + + await foreach (var room in roomTasks) { + Rooms.Add(room); + StateHasChanged(); + // await Task.Delay(50); + // _semaphore.Release(); } - - GlobalProfile = await Homeserver.GetProfileAsync(Homeserver.WhoAmI.UserId); - var syncHelper = new SyncHelper(Homeserver, logger) { - Timeout = 10000, + if (rooms.Count >= 150) RenderContents = true; + + GlobalProfile = await Homeserver.GetProfileAsync(Homeserver.WhoAmI.UserId); + syncHelper = new SyncHelper(Homeserver, logger) { + Timeout = 30000, Filter = filter, MinimumDelay = TimeSpan.FromMilliseconds(5000) }; - // profileUpdateFilter.Room.State.Senders.Add(Homeserver.WhoAmI.UserId); - // var profileSyncHelper = new SyncHelper(Homeserver, logger) { + // profileSyncHelper = new SyncHelper(Homeserver, logger) { // Timeout = 10000, // Filter = profileUpdateFilter, // MinimumDelay = TimeSpan.FromMilliseconds(5000) - // }; + // }; + // profileUpdateFilter.Room.State.Senders.Add(Homeserver.WhoAmI.UserId); + RunSyncLoop(syncHelper); // RunSyncLoop(profileSyncHelper); RunQueueProcessor(); + await base.OnInitializedAsync(); } - + private async Task RunQueueProcessor() { var renderTimeSw = Stopwatch.StartNew(); + var isInitialSync = true; while (true) { try { - if (queue.Count == 0) { - while (queue.Count == 0) { - Console.WriteLine("Queue is empty, waiting..."); - await Task.Delay(2500); - } - Console.WriteLine("Queue no longer empty!"); + while (queue.Count == 0) { + Console.WriteLine("Queue is empty, waiting..."); + await Task.Delay(isInitialSync ? 100 : 2500); } - while (queue.TryDequeue(out var queueEntry)) { + + Console.WriteLine($"Queue no longer empty after {renderTimeSw.Elapsed}!"); + + int maxUpdates = 10; + isInitialSync = false; + while (maxUpdates-- > 0 && queue.TryDequeue(out var queueEntry)) { var (roomId, roomData) = queueEntry; Console.WriteLine($"Dequeued room {roomId}"); RoomInfo room; - + if (Rooms.Any(x => x.Room.RoomId == roomId)) { room = Rooms.First(x => x.Room.RoomId == roomId); Console.WriteLine($"QueueWorker: {roomId} already known with {room.StateEvents?.Count ?? 0} state events"); @@ -146,26 +165,23 @@ }; Rooms.Add(room); } - + if (room.StateEvents is null) { Console.WriteLine($"QueueWorker: {roomId} does not have state events on record?"); throw new InvalidDataException("Somehow this is null???"); } - if (roomData.State?.Events is {Count: >0 }) + + if (roomData.State?.Events is { Count: > 0 }) room.StateEvents.MergeStateEventLists(roomData.State.Events); else { Console.WriteLine($"QueueWorker: could not merge state for {room.Room.RoomId} as new data contains no state events!"); } - if (Random.Shared.Next(101) < 20 || true) { - Status = $"Got {Rooms.Count} rooms so far! {queue.Count} entries in processing queue..."; - } - RenderContents |= queue.Count == 0; - if (queue.Count > 10) RenderContents = false; - await Task.Delay(RenderContents ? 25 : 6); } - // else { - // Console.WriteLine("Failed to dequeue item"); - // } + Console.WriteLine($"QueueWorker: {queue.Count} entries left in queue, {maxUpdates} maxUpdates left, RenderContents: {RenderContents}"); + Status = $"Got {Rooms.Count} rooms so far! {queue.Count} entries in processing queue..."; + + RenderContents |= queue.Count == 0; + await Task.Delay(Rooms.Count); } catch (Exception e) { Console.WriteLine("QueueWorker exception: " + e); @@ -214,11 +230,15 @@ // We can't trust servers to give us what we ask for, and this ruins performance // Thanks, Conduit. joinedRoom.Value.State.Events.RemoveAll(x => filter.Room?.State?.Types?.Contains(x.Type) == false); - if(filter.Room?.State?.NotSenders?.Any() ?? false) + if (filter.Room?.State?.NotSenders?.Any() ?? false) joinedRoom.Value.State.Events.RemoveAll(x => filter.Room?.State?.NotSenders?.Contains(x.Sender) ?? false); - + queue.Enqueue(joinedRoom); } + if (sync.Rooms.Leave is {Count: > 0}) + foreach (var leftRoom in sync.Rooms.Leave) + if (Rooms.Any(x => x.Room.RoomId == leftRoom.Key)) + Rooms.Remove(Rooms.First(x => x.Room.RoomId == leftRoom.Key)); Status = $"Got {Rooms.Count} rooms so far! {queue.Count} entries in processing queue... " + $"{sync?.Rooms?.Join?.Count ?? 0} new updates!"; diff --git a/MatrixRoomUtils.Web/Pages/Rooms/PolicyList.razor b/MatrixRoomUtils.Web/Pages/Rooms/PolicyList.razor index 846d1cb..dbe0648 100644 --- a/MatrixRoomUtils.Web/Pages/Rooms/PolicyList.razor +++ b/MatrixRoomUtils.Web/Pages/Rooms/PolicyList.razor @@ -4,184 +4,199 @@ @using ArcaneLibs.Extensions @using LibMatrix.EventTypes.Spec.State @using LibMatrix.EventTypes.Spec.State.Policy +@using System.Diagnostics +@using System.Diagnostics.CodeAnalysis +@using LibMatrix.Extensions +@using LibMatrix.Responses

Policy list editor - Editing @RoomId

- This policy list contains @PolicyEvents.Count(x => x.Type == "m.policy.rule.server") server bans, - @PolicyEvents.Count(x => x.Type == "m.policy.rule.room") room bans and - @PolicyEvents.Count(x => x.Type == "m.policy.rule.user") user bans. + This policy list contains @GetPolicyCount(typeof(ServerPolicyRuleEventContent)) server bans, + @GetPolicyCount(typeof(RoomPolicyRuleEventContent)) room bans and + @GetPolicyCount(typeof(UserPolicyRuleEventContent)) user bans. + @foreach (var (key, value) in PolicyEventsByType) { +

@key.Name: @value.Count

+ }

- - + -@if (!PolicyEvents.Any(x => x.Type == "m.policy.rule.server")) { +

Server policies

+
+@if (!GetPolicyEventsByType(typeof(ServerPolicyRuleEventContent)).Any()) {

No server policies

} else { -

Server policies

-
- +
- - - - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.server" && (x.TypedContent as PolicyRuleEventContent).Entity is not null)) { - var policyData = policyEvent.TypedContent as PolicyRuleEventContent; - - - - + + + + - } + + + @foreach (var policyEvent in GetValidPolicyEventsByType(typeof(ServerPolicyRuleEventContent))) { + var policyData = policyEvent.TypedContent as PolicyRuleEventContent; + + + + + + + }
ServerReasonExpiresActions
Entity: @policyData.Entity
State: @policyEvent.StateKey		@policyData.Reason - @policyData.ExpiryDateTime - - - @* *@ - ServerReasonExpiresActions
+ Entity: @policyData.Entity +
State: @policyEvent.StateKey + 		@policyData.Reason + @policyData.ExpiryDateTime + + + @* *@ +
- Redacted events - + Redacted or invalid events +
- - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.server" && (x.TypedContent as PolicyRuleEventContent).Entity == null)) { - var policyData = policyEvent.TypedContent as PolicyRuleEventContent; - - + + - } + + + @foreach (var policyEvent in GetInvalidPolicyEventsByType(typeof(ServerPolicyRuleEventContent))) { + + + + + }
State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent.ToJson(false, true)State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent.ToJson(false, true)
} -@if (!PolicyEvents.Any(x => x.Type == "m.policy.rule.room")) { +

Room policies

+
+@if (!GetPolicyEventsByType(typeof(RoomPolicyRuleEventContent)).Any()) {

No room policies

} else { -

Room policies

-
- +
- - - - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.room" && (x.TypedContent as PolicyRuleEventContent).Entity is not null)) { - var policyData = policyEvent.TypedContent as PolicyRuleEventContent; - - - - + + + + - } + + + @foreach (var policyEvent in GetValidPolicyEventsByType(typeof(RoomPolicyRuleEventContent))) { + var policyData = policyEvent.TypedContent as PolicyRuleEventContent; + + + + + + + }
RoomReasonExpiresActions
Entity: @policyData.Entity
State: @policyEvent.StateKey		@policyData.Reason - @policyData.ExpiryDateTime - - - RoomReasonExpiresActions
Entity: @policyData.Entity
State: @policyEvent.StateKey		@policyData.Reason + @policyData.ExpiryDateTime + + +
- Redacted events - + Redacted or invalid events +
- - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.room" && (x.TypedContent as PolicyRuleEventContent).Entity == null)) { - - + + - } + + + @foreach (var policyEvent in GetInvalidPolicyEventsByType(typeof(RoomPolicyRuleEventContent))) { + + + + + }
State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent!.ToJson(false, true)State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent!.ToJson(false, true)
} -@if (!PolicyEvents.Any(x => x.Type == "m.policy.rule.user")) { +

User policies

+
+@if (!GetPolicyEventsByType(typeof(UserPolicyRuleEventContent)).Any()) {

No user policies

} else { -

User policies

-
- +
- - @if (_enableAvatars) { - - } - - - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.user" && (x.TypedContent as PolicyRuleEventContent).Entity is not null)) { - var policyData = policyEvent.TypedContent as PolicyRuleEventContent; - @if (_enableAvatars) { - + @if (EnableAvatars) { + } - - - - + + + + - } + + + @foreach (var policyEvent in GetValidPolicyEventsByType(typeof(UserPolicyRuleEventContent))) { + var policyData = policyEvent.TypedContent as PolicyRuleEventContent; + + @if (EnableAvatars) { + + } + + + + + + }
UserReasonExpiresActions
- - Entity: @string.Join("", policyData.Entity.Take(64))
State: @string.Join("", policyEvent.StateKey.Take(64))		@policyData.Reason - @policyData.ExpiryDateTime - - - UserReasonExpiresActions
+ @if (Avatars.ContainsKey(policyData.Entity)) { + + } + Entity: @string.Join("", policyData.Entity.Take(64))
State: @string.Join("", policyEvent.StateKey.Take(64))		@policyData.Reason + @policyData.ExpiryDateTime + + +
- Redacted events - + Redacted or invalid events +
- - - - - - - @foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.user" && (x.TypedContent as PolicyRuleEventContent).Entity == null)) { - - + + - } + + + @foreach (var policyEvent in GetInvalidPolicyEventsByType(typeof(UserPolicyRuleEventContent))) { + + + + + }
State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent.ToJson(false, true)State keySerialised Contents
@policyEvent.StateKey@policyEvent.RawContent.ToJson(false, true)
} - - @code { + +#if DEBUG + private const bool Debug = true; +#else + private const bool Debug = false; +#endif + //get room list // - sync withroom list filter // Type = support.feline.msc3784 @@ -192,18 +207,28 @@ else { private bool _enableAvatars; - static readonly Dictionary avatars = new(); - static readonly Dictionary servers = new(); + static readonly Dictionary Avatars = new(); + // static readonly Dictionary Servers = new(); - public static List PolicyEvents { get; set; } = new(); + // private static List PolicyEvents { get; set; } = new(); + private Dictionary> PolicyEventsByType { get; set; } = new(); + + public bool EnableAvatars { + get => _enableAvatars; + set { + _enableAvatars = value; + if (value) GetAllAvatars(); + } + } protected override async Task OnInitializedAsync() { + var sw = Stopwatch.StartNew(); await base.OnInitializedAsync(); var hs = await MRUStorage.GetCurrentSessionOrNavigate(); if (hs is null) return; RoomId = RoomId.Replace('~', '.'); await LoadStatesAsync(); - Console.WriteLine("Policy list editor initialized!"); + Console.WriteLine($"Policy list editor initialized in {sw.Elapsed}!"); } private async Task LoadStatesAsync() { @@ -214,39 +239,52 @@ else { var states = room.GetFullStateAsync(); await foreach (var state in states) { - if (!state.Type.StartsWith("m.policy.rule")) continue; - PolicyEvents.Add(state); + if (state is null) continue; + if (!state.MappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue; + if (!PolicyEventsByType.ContainsKey(state.MappedType)) PolicyEventsByType.Add(state.MappedType, new()); + PolicyEventsByType[state.MappedType].Add(state); } - - // var stateEventsQuery = await room.GetStateAsync(""); - // var stateEvents = stateEventsQuery.Value.Deserialize>(); - // PolicyEvents = stateEvents.Where(x => x.Type.StartsWith("m.policy.rule")) - // .Select(x => JsonSerializer.Deserialize(JsonSerializer.Serialize(x))).ToList(); StateHasChanged(); } - private async Task GetAvatar(string userId) { - try { - if (avatars.ContainsKey(userId)) return; - var hs = userId.Split(':')[1]; - var server = servers.ContainsKey(hs) ? servers[hs] : new RemoteHomeserver(userId.Split(':')[1]); - if (!servers.ContainsKey(hs)) servers.Add(hs, server); - var profile = await server.GetProfileAsync(userId); - avatars.Add(userId, await hsResolver.ResolveMediaUri(server.BaseUrl, profile.AvatarUrl)); - servers.Add(userId, server); + private async Task GetAllAvatars() { + // if (!_enableAvatars) return; + Console.WriteLine("Getting avatars..."); + var users = GetValidPolicyEventsByType(typeof(UserPolicyRuleEventContent)).Select(x => x.RawContent!["entity"]!.GetValue()).Where(x => x.Contains(':') && !x.Contains("*")).ToList(); + Console.WriteLine($"Got {users.Count} users!"); + var usersByHomeServer = users.GroupBy(x => x!.Split(':')[1]).ToDictionary(x => x.Key!, x => x.ToList()); + Console.WriteLine($"Got {usersByHomeServer.Count} homeservers!"); + var homeserverTasks = usersByHomeServer.Keys.Select(x => RemoteHomeserver.TryCreate(x)).ToAsyncEnumerable(); + await foreach (var server in homeserverTasks) { + if (server is null) continue; + var profileTasks = usersByHomeServer[server.BaseUrl].Select(x => TryGetProfile(server, x)).ToList(); + await Task.WhenAll(profileTasks); + profileTasks.RemoveAll(x => x.Result is not { Value: { AvatarUrl: not null } }); + foreach (var profile in profileTasks.Select(x => x.Result!.Value)) { + // if (profile is null) continue; + if (!string.IsNullOrWhiteSpace(profile.Value.AvatarUrl)) { + var url = await hsResolver.ResolveMediaUri(server.BaseUrl, profile.Value.AvatarUrl); + Avatars.TryAdd(profile.Key, url); + } + else Avatars.TryAdd(profile.Key, null); + } StateHasChanged(); } - catch { - // ignored - } } - private async Task GetAllAvatars() { - foreach (var policyEvent in PolicyEvents.Where(x => x.Type == "m.policy.rule.user" && (x.TypedContent as PolicyRuleEventContent).Entity is not null)) { - await GetAvatar((policyEvent.TypedContent as PolicyRuleEventContent).Entity); + private async Task?> TryGetProfile(RemoteHomeserver server, string mxid) { + try { + return new KeyValuePair(mxid, await server.GetProfileAsync(mxid)); + } + catch { + return null; } - StateHasChanged(); } -} + private List GetPolicyEventsByType(Type type) => PolicyEventsByType.ContainsKey(type) ? PolicyEventsByType[type] : []; + private List GetValidPolicyEventsByType(Type type) => GetPolicyEventsByType(type).Where(x => !string.IsNullOrWhiteSpace(x.RawContent?["entity"]?.GetValue())).ToList(); + private List GetInvalidPolicyEventsByType(Type type) => GetPolicyEventsByType(type).Where(x => string.IsNullOrWhiteSpace(x.RawContent?["entity"]?.GetValue())).ToList(); + private int GetPolicyCount(Type type) => PolicyEventsByType.ContainsKey(type) ? PolicyEventsByType[type].Count : 0; + +} \ No newline at end of file -- cgit 1.5.1