From 93db033377b5e457bd04daf0b16f3d8b8cd77b44 Mon Sep 17 00:00:00 2001 From: Rory& Date: Fri, 22 Aug 2025 17:39:57 +0200 Subject: Room upgrade CLI changes, policy list work --- .../Pages/HSAdmin/Synapse/BlockMedia.razor | 1 + MatrixUtils.Web/Pages/Rooms/PolicyList.razor | 238 +++++++++++++++------ .../PolicyListCategoryComponent.razor | 5 +- .../PolicyListRowComponent.razor | 21 +- 4 files changed, 200 insertions(+), 65 deletions(-) (limited to 'MatrixUtils.Web/Pages') diff --git a/MatrixUtils.Web/Pages/HSAdmin/Synapse/BlockMedia.razor b/MatrixUtils.Web/Pages/HSAdmin/Synapse/BlockMedia.razor index 594ff35..e9d0cd2 100644 --- a/MatrixUtils.Web/Pages/HSAdmin/Synapse/BlockMedia.razor +++ b/MatrixUtils.Web/Pages/HSAdmin/Synapse/BlockMedia.razor @@ -3,6 +3,7 @@ @using ArcaneLibs.Extensions @using LibMatrix @using LibMatrix.EventTypes.Spec +@using LibMatrix.StructuredData

Homeserver Administration - Block media

@if (Homeserver is not null) { diff --git a/MatrixUtils.Web/Pages/Rooms/PolicyList.razor b/MatrixUtils.Web/Pages/Rooms/PolicyList.razor index cdb5894..412d8c9 100644 --- a/MatrixUtils.Web/Pages/Rooms/PolicyList.razor +++ b/MatrixUtils.Web/Pages/Rooms/PolicyList.razor @@ -6,6 +6,7 @@ @using LibMatrix.RoomTypes @using System.Collections.Frozen @using System.Reflection +@using System.Text.Json @using ArcaneLibs.Attributes @using LibMatrix.EventTypes @using LibMatrix.EventTypes.Interop.Draupnir @@ -136,6 +137,9 @@ else { [Parameter] public required string RoomId { get; set; } + [Parameter, SupplyParameterFromQuery] + public bool RenderEventInfo { get; set; } + private Dictionary> PolicyEventsByType { get; set; } = new(); public StateEventResponse? ServerPolicyToMakePermanent { @@ -172,12 +176,13 @@ else { } private async Task LoadStateAsync(bool firstLoad = false) { + // preload workers in task pool + // await Task.WhenAll(Enumerable.Range(0, WebWorkerService.MaxWorkerCount).Select(async _ => (await WebWorkerService.TaskPool.GetWorkerAsync()).WhenReady).ToList()); var sw = Stopwatch.StartNew(); // Loading = true; // var states = Room.GetFullStateAsync(); var states = await Room.GetFullStateAsListAsync(); // PolicyEventsByType.Clear(); - logger.LogInformation($"LoadStatesAsync: Loaded state in {sw.Elapsed}"); foreach (var type in KnownPolicyTypes) { @@ -215,7 +220,8 @@ else { var key = (evt.Type, evt.StateKey!); var policyInfo = new PolicyCollection.PolicyInfo { Policy = evt, - MadeRedundantBy = [] + MadeRedundantBy = [], + DuplicatedBy = [] }; if (evt.RawContent is null or { Count: 0 } || string.IsNullOrWhiteSpace(evt.RawContent?["recommendation"]?.GetValue())) { collection.ActivePolicies.Remove(key); @@ -238,76 +244,187 @@ else { logger.LogInformation($"Policy collection {collection.Key.FullName} has {collection.Value.ActivePolicies.Count} active and {collection.Value.RemovedPolicies.Count} removed policies."); } + Loading = false; + StateHasChanged(); + await Task.Delay(100); + logger.LogInformation("LoadStatesAsync: Scanning for redundant policies..."); - Loading = false; - var allPolicies = PolicyCollections.Values + var scanSw = Stopwatch.StartNew(); + var allPolicyInfos = PolicyCollections.Values .SelectMany(x => x.ActivePolicies.Values) - .Select(x => (x, x.Policy.TypedContent as PolicyRuleEventContent)) .ToList(); - var wildcardPolicies = allPolicies - .Where(x => x.Item2!.IsGlobRule() || x.Item2 is ServerPolicyRuleEventContent) - .ToList(); - Console.WriteLine($"Got {allPolicies.Count} total policies, {wildcardPolicies.Count} wildcard policies."); - int i = 0; - int hits = 0; - int redundant = 0; - int duplicates = 0; - foreach (var policy in allPolicies) { - if (policy.Item2 is null) continue; - var matchingPolicies = wildcardPolicies - .Where(x => - !StateEvent.TypeKeyPairMatches(policy.x.Policy, x.x.Policy) - && x.Item2!.EntityMatches(policy.Item2.Entity!) - ) - .ToList(); - - if (matchingPolicies.Count > 0) { - logger.LogInformation($"{i} Got {matchingPolicies.Count} hits for {policy.x.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.x.Policy.RawContent).ToJson()}"); - foreach (var match in matchingPolicies) { - policy.x.MadeRedundantBy.Add(match.x.Policy); - } - - hits++; - redundant += matchingPolicies.Count; + // var allPolicies = allPolicyInfos + // .Select(x => (x, (x.Policy.TypedContent as PolicyRuleEventContent)!)) + // .ToList(); + // var hashPolicies = allPolicies + // .Where(x => x.TypedContent.IsHashedRule()) + // .ToList(); + // var wildcardPolicies = allPolicies + // .Except(hashPolicies) // hashed policies cannot be wildcards + // .Where(x => x.TypedContent.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent) + // .ToList(); + // var nonWildcardPolicies = allPolicies + // // .Except(wildcardPolicies) + // .Where(x => !x.TypedContent!.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent) + // .ToList(); + // Console.WriteLine($"Got {allPolicies.Count} total policies, {wildcardPolicies.Count} wildcard policies. Time spent: {scanSw.Elapsed}"); + // int i = 0; + // int hits = 0; + // int redundant = 0; + // int duplicates = 0; + + // foreach (var (policyInfo, policyContent) in allPolicies) { + // foreach (var (otherPolicyInfo, otherPolicyContent) in allPolicies) { + // if (policyInfo.Policy == otherPolicyInfo.Policy) continue; // same event + // if (StateEvent.TypeKeyPairMatches(policyInfo.Policy, otherPolicyInfo.Policy)) { + // logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson()); + // continue; // same type and state key + // } + // // if(!policyContent.IsHashedRule()) + // } + // + // if (++i % 100 == 0) { + // Console.WriteLine($"Processed {i} policies in {scanSw.Elapsed}"); + // await Task.Delay(1); + // } + // } - if (hits % 5 == 0) - StateHasChanged(); + Console.WriteLine($"Scanning for redundant policies in {allPolicyInfos.Count} total policies... ({scanSw.Elapsed})"); + List>> tasks = []; + // try to save some load... + var policiesJson = JsonSerializer.Serialize(allPolicyInfos.Select(x => x.Policy)); + var ranges = Enumerable.Range(0, allPolicyInfos.Count).DistributeSequentially(WebWorkerService.MaxWorkerCount); + foreach (var range in ranges) + tasks.Add(WebWorkerService.TaskPool.Invoke(CheckDuplicatePoliciesAsync, policiesJson, range.First(), range.Last())); + + // tasks.Add(CheckDuplicatePoliciesAsync(allPolicyInfos, range.First() .. range.Last())); + + await foreach (var modifiedPolicyInfos in tasks.ToAsyncEnumerable()) { + Console.WriteLine($"Main: got {modifiedPolicyInfos.Count} modified policies from worker, time: {scanSw.Elapsed}"); + foreach (var modifiedPolicyInfo in modifiedPolicyInfos) { + var original = allPolicyInfos.First(p => p.Policy.EventId == modifiedPolicyInfo.Policy.EventId); + original.DuplicatedBy = modifiedPolicyInfo.DuplicatedBy; + original.MadeRedundantBy = modifiedPolicyInfo.MadeRedundantBy; } - else logger.LogInformation("Sleeping..."); - await Task.Delay(1); - i++; + + Console.WriteLine($"Processed {modifiedPolicyInfos.Count} modified policies in {scanSw.Elapsed}"); } - i = 0; - foreach (var policy in allPolicies) { - if (policy.Item2 is null) continue; - var matchingPolicies = allPolicies - .Where(x => - !StateEvent.TypeKeyPairMatches(policy.x.Policy, x.x.Policy) - && x.Item2!.Entity == policy.Item2.Entity! - ) - .ToList(); - - if (matchingPolicies.Count > 0) { - logger.LogInformation($"{i} Got {matchingPolicies.Count} duplicates for {policy.x.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.x.Policy.RawContent).ToJson()}"); - foreach (var match in matchingPolicies) { - policy.x.MadeRedundantBy.Add(match.x.Policy); - } + Console.WriteLine($"Processed {allPolicyInfos.Count} policies in {scanSw.Elapsed}"); + + // // scan for wildcard matches + // foreach (var policy in allPolicies) { + // var matchingPolicies = wildcardPolicies + // .Where(x => + // !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy) + // && x.Item2.EntityMatches(policy.TypedContent.Entity!) + // ) + // .ToList(); + // + // if (matchingPolicies.Count > 0) { + // logger.LogInformation($"{i} Got {matchingPolicies.Count} hits for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}"); + // foreach (var match in matchingPolicies) { + // policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy); + // } + // + // hits++; + // redundant += matchingPolicies.Count; + // + // if (hits % 5 == 0) + // StateHasChanged(); + // } + // else { + // //logger.LogInformation("Sleeping..."); + // await Task.Delay(1); + // } + // + // i++; + // } + // + // i = 0; + // // scan for exact duplicates + // foreach (var policy in allPolicies) { + // var matchingPolicies = allPolicies + // .Where(x => + // !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy) + // && ( + // x.Item2.IsHashedRule() + // ? x.Item2.EntityMatches(policy.Item2.Entity) + // : x.Item2!.Entity == policy.Item2.Entity! + // ) + // ) + // .ToList(); + // + // if (matchingPolicies.Count > 0) { + // logger.LogInformation($"{i} Got {matchingPolicies.Count} duplicates for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}"); + // foreach (var match in matchingPolicies) { + // policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy); + // } + // + // hits++; + // duplicates += matchingPolicies.Count; + // + // if (hits % 5 == 0) + // StateHasChanged(); + // } + // else { + // //logger.LogInformation("Sleeping..."); + // await Task.Delay(1); + // } + // + // i++; + // } + // + // logger.LogInformation($"LoadStatesAsync: Found {hits} ({redundant} redundant, {duplicates} duplicates) redundant policies in {sw.Elapsed}"); + // StateHasChanged(); + } + + [return: WorkerTransfer] + private static async Task> CheckDuplicatePoliciesAsync(string policiesJson, int start, int end) { + Console.WriteLine($"Got request to check duplicate policies in range {start} to {end} (length: {end - start}), {policiesJson.Length} bytes of JSON"); + return await CheckDuplicatePoliciesAsync(JsonSerializer.Deserialize>(policiesJson), start .. end); + } + + [return: WorkerTransfer] + private static Task> CheckDuplicatePoliciesAsync(List policies, int start, int end) + => CheckDuplicatePoliciesAsync(policies, start .. end); - hits++; - redundant += matchingPolicies.Count; + [return: WorkerTransfer] + private static async Task> CheckDuplicatePoliciesAsync(List policies, Range range) { + Console.WriteLine($"Processing policies in range {range} ({range.GetOffsetAndLength(policies.Count).Length}) with {policies.Count} total policies"); + var allPolicies = policies + .Select(x => (Event: x, TypedContent: (x.TypedContent as PolicyRuleEventContent)!)) + .ToList(); + var toCheck = allPolicies[range]; + var modifiedPolicies = new List(); + + foreach (var (policyEvent, policyContent) in toCheck) { + List duplicatedBy = []; + List madeRedundantBy = []; + + foreach (var (otherPolicyEvent, otherPolicyContent) in allPolicies) { + if (policyEvent == otherPolicyEvent) continue; // same event + if (StateEvent.TypeKeyPairMatches(policyEvent, otherPolicyEvent)) { + // logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson()); + Console.WriteLine($"Sanity check failed: Found same type and state key for two different policies: {policyEvent.RawContent.ToJson()} and {otherPolicyEvent.RawContent.ToJson()}"); + continue; // same type and state key + } + // if(!policyContent.IsHashedRule()) + } - if (hits % 5 == 0) - StateHasChanged(); + if (duplicatedBy.Count > 0 || madeRedundantBy.Count > 0) { + modifiedPolicies.Add(new() { + Policy = policyEvent, + DuplicatedBy = duplicatedBy, + MadeRedundantBy = madeRedundantBy + }); } - else logger.LogInformation("Sleeping..."); - await Task.Delay(1); - i++; + + // await Task.Delay(1); } - logger.LogInformation($"LoadStatesAsync: Found {hits} ({redundant} redundant, {duplicates} duplicates) redundant policies in {sw.Elapsed}"); - StateHasChanged(); + return modifiedPolicies; } // the old one: @@ -415,9 +532,10 @@ else { public required Dictionary<(string Type, string StateKey), PolicyInfo> RemovedPolicies { get; set; } public required FrozenDictionary PropertiesToDisplay { get; set; } - public struct PolicyInfo { + public class PolicyInfo { public required StateEventResponse Policy { get; init; } public required List MadeRedundantBy { get; set; } + public required List DuplicatedBy { get; set; } } } diff --git a/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListCategoryComponent.razor b/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListCategoryComponent.razor index f818b62..b52e03f 100644 --- a/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListCategoryComponent.razor +++ b/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListCategoryComponent.razor @@ -18,7 +18,7 @@ @foreach (var policy in PolicyCollection.ActivePolicies.Values.OrderBy(x => x.Policy.RawContent?["entity"]?.GetValue())) { - + } @@ -56,6 +56,9 @@ [Parameter] public required GenericRoom Room { get; set; } + + [Parameter] + public bool RenderEventInfo { get; set; } protected override bool ShouldRender() { // if (PolicyCollection is null) return false; diff --git a/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListRowComponent.razor b/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListRowComponent.razor index 11de82c..9ac5077 100644 --- a/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListRowComponent.razor +++ b/MatrixUtils.Web/Pages/Rooms/PolicyListComponents/PolicyListRowComponent.razor @@ -10,9 +10,19 @@ if (prop.Name == "Entity") { @TruncateMxid(TypedContent.Entity) + @foreach (var dup in PolicyInfo.DuplicatedBy) { +
+ Duplicated by @dup.FriendlyTypeName.ToLower() @TruncateMxid(dup.RawContent["entity"]?.GetValue()) + } @foreach (var dup in PolicyInfo.MadeRedundantBy) {
- Also matched by @dup.FriendlyTypeName.ToLower() @TruncateMxid(dup.RawContent["entity"].GetValue()) + Also matched by @dup.FriendlyTypeName.ToLower() @TruncateMxid(dup.RawContent["entity"]?.GetValue()) + } + @if (RenderEventInfo) { +
+ 
+                            @PolicyInfo.Policy.Type/@PolicyInfo.Policy.StateKey by @PolicyInfo.Policy.Sender at @PolicyInfo.Policy.OriginServerTs 
+
} } @@ -25,9 +35,9 @@ @* @if (PowerLevels.UserHasStatePermission(Homeserver.WhoAmI.UserId, Policy.Type)) { *@ @if (true) { Edit + IsEditing = true; + return Task.CompletedTask; + })">Edit Remove @if (Policy.IsLegacyType) { @@ -82,6 +92,9 @@ [Parameter] public required PolicyList.PolicyCollection PolicyCollection { get; set; } + [Parameter] + public bool RenderEventInfo { get; set; } + private StateEventResponse Policy => PolicyInfo.Policy; private bool IsEditing { -- cgit 1.5.1