about summary refs log tree commit diff
path: root/MatrixUtils.Web/Pages/Rooms/PolicyList.razor
diff options
context:
space:
mode:
Diffstat (limited to 'MatrixUtils.Web/Pages/Rooms/PolicyList.razor')
-rw-r--r--MatrixUtils.Web/Pages/Rooms/PolicyList.razor877
1 files changed, 560 insertions, 317 deletions
diff --git a/MatrixUtils.Web/Pages/Rooms/PolicyList.razor b/MatrixUtils.Web/Pages/Rooms/PolicyList.razor

index 9c35673..f2ab186 100644 --- a/MatrixUtils.Web/Pages/Rooms/PolicyList.razor +++ b/MatrixUtils.Web/Pages/Rooms/PolicyList.razor
@@ -5,159 +5,148 @@ @using System.Diagnostics @using LibMatrix.RoomTypes @using System.Collections.Frozen +@using System.Collections.Immutable @using System.Reflection +@using System.Text.Json @using ArcaneLibs.Attributes +@using ArcaneLibs.Blazor.Components.Services @using LibMatrix.EventTypes -@using LibMatrix.EventTypes.Common @using LibMatrix.EventTypes.Interop.Draupnir @using LibMatrix.EventTypes.Spec.State.RoomInfo - -@using MatrixUtils.Web.Shared.PolicyEditorComponents @using SpawnDev.BlazorJS.WebWorkers +@using MatrixUtils.Web.Pages.Rooms.PolicyListComponents +@using SpawnDev.BlazorJS @inject WebWorkerService WebWorkerService +@inject ILogger<PolicyList> logger +@inject BlazorJSRuntime JsRuntime -<h3>Policy list editor - Editing @(RoomName ?? RoomId)</h3> -@if (!string.IsNullOrWhiteSpace(DraupnirShortcode)) { - <span style="margin-right: 2em;">Shortcode: @DraupnirShortcode</span> -} -@if (!string.IsNullOrWhiteSpace(RoomAlias)) { - <span>Alias: @RoomAlias</span> -} -<hr/> -@* <InputCheckbox @bind-Value="EnableAvatars"></InputCheckbox><label>Enable avatars (WILL EXPOSE YOUR IP TO TARGET HOMESERVERS!)</label> *@ -<LinkButton OnClick="@(() => { CurrentlyEditingEvent = new() { Type = "", RawContent = new() }; return Task.CompletedTask; })">Create new policy</LinkButton> -<LinkButton OnClick="@(() => { MassCreatePolicies = true; return Task.CompletedTask; })">Create many new policies</LinkButton> - -@if (Loading) { - <p>Loading...</p> -} -else if (PolicyEventsByType is not { Count: > 0 }) { - <p>No policies yet</p> +@if (!IsInitialised) { + <p>Connecting to homeserver...</p> } else { - var renderSw = Stopwatch.StartNew(); - var renderTotalSw = Stopwatch.StartNew(); - @foreach (var (type, value) in PolicyEventsByType) { - <p> - @(GetValidPolicyEventsByType(type).Count) active, - @(GetInvalidPolicyEventsByType(type).Count) invalid - (@value.Count total) - @(GetPolicyTypeName(type).ToLower()) - </p> + <PolicyListEditorHeader Room="@Room" @bind-RenderEventInfo="@RenderEventInfo" ReloadStateAsync="@(() => LoadStateAsync(true))"></PolicyListEditorHeader> + @if (Loading) { + <p>Loading...</p> } + // else if (PolicyEventsByType is not { Count: > 0 }) { + @* <p>No policies yet</p> *@ + // } + else { + var renderSw = Stopwatch.StartNew(); + var renderTotalSw = Stopwatch.StartNew(); + @foreach (var value in PolicyCollections.Values.OrderByDescending(x => x.TotalCount)) { + <p> + @value.ActivePolicies.Count active, + @value.RemovedPolicies.Count removed + (@value.TotalCount total) + @value.Name.ToLower() + </p> + } - Console.WriteLine($"Rendered hearder in {renderSw.GetElapsedAndRestart()}"); + @if (DuplicateBans?.ActivePolicies.Count > 0) { + <p style="color: orange;"> + Found @DuplicateBans.Value.ActivePolicies.Count duplicate bans + </p> + } - @foreach (var type in KnownPolicyTypes.OrderByDescending(t => GetPolicyEventsByType(t).Count)) { - <details> - <summary> - <span> - @($"{GetPolicyTypeName(type)}: {GetPolicyEventsByType(type).Count} policies") - </span> - <hr style="margin: revert;"/> - </summary> - <table class="table table-striped table-hover"> - @{ - var policies = GetValidPolicyEventsByType(type); - var invalidPolicies = GetInvalidPolicyEventsByType(type); - // enumerate all properties with friendly name - var props = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(x => (x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyNameOrNull()) is not null) - .Where(x => x.GetCustomAttribute<TableHideAttribute>() is null) - .ToFrozenSet(); - var propNames = props.Select(x => x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyName()!).ToFrozenSet(); + @if (RedundantBans?.ActivePolicies.Count > 0) { + <p style="color: orange;"> + Found @RedundantBans.Value.ActivePolicies.Count redundant bans + </p> + } - var proxySafeProps = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(x => props.Any(y => y.Name == x.Name)) - .ToFrozenSet(); - Console.WriteLine($"{proxySafeProps?.Count} proxy safe props found in {policies.FirstOrDefault()?.TypedContent?.GetType()}"); - } - <thead> - <tr> - @foreach (var name in propNames) { - <th>@name</th> - } - <th>Actions</th> - </tr> - </thead> - <tbody style="border-width: 1px;"> - @foreach (var policy in policies.OrderBy(x => x.RawContent?["entity"]?.GetValue<string>())) { - <tr> - @{ - var typedContent = policy.TypedContent!; - } - @foreach (var prop in proxySafeProps ?? Enumerable.Empty<PropertyInfo>()) { - <td>@prop.GetGetMethod()?.Invoke(typedContent, null)</td> - } - <td> - <div style="display: ruby;"> - @if (PowerLevels.UserHasStatePermission(Homeserver.WhoAmI.UserId, policy.Type)) { - <LinkButton OnClick="@(() => { CurrentlyEditingEvent = policy; return Task.CompletedTask; })">Edit</LinkButton> - <LinkButton OnClick="@(() => RemovePolicyAsync(policy))">Remove</LinkButton> - @if (policy.IsLegacyType) { - <LinkButton OnClick="@(() => RemovePolicyAsync(policy))">Update policy type</LinkButton> - } + // logger.LogInformation($"Rendered header in {renderSw.GetElapsedAndRestart()}"); - @if (PolicyTypeIds[typeof(ServerPolicyRuleEventContent)].Contains(policy.Type)) { - <LinkButton OnClick="@(() => { ServerPolicyToMakePermanent = policy; return Task.CompletedTask; })">Make permanent</LinkButton> - @if (CurrentUserIsDraupnir) { - <LinkButton Color="@(ActiveKicks.ContainsKey(policy) ? "#FF0000" : null)" OnClick="@(() => DraupnirKickMatching(policy))">Kick users @(ActiveKicks.ContainsKey(policy) ? $"({ActiveKicks[policy]})" : null)</LinkButton> - } - } - } - else { - <p>No permission to modify</p> - } - </div> - </td> - </tr> - } - </tbody> - </table> - <details> - <summary> - <u> - @("Invalid " + GetPolicyTypeName(type).ToLower()) - </u> - </summary> - <table class="table table-striped table-hover"> - <thead> - <tr> - <th>State key</th> - <th>Json contents</th> - </tr> - </thead> - <tbody> - @foreach (var policy in invalidPolicies) { - <tr> - <td>@policy.StateKey</td> - <td> - <pre>@policy.RawContent.ToJson(true, false)</pre> - </td> - </tr> - } - </tbody> - </table> - </details> - </details> - } + // var renderSw2 = Stopwatch.StartNew(); + // IOrderedEnumerable<Type> policiesByType = KnownPolicyTypes.Where(t => GetPolicyEventsByType(t).Count > 0).OrderByDescending(t => GetPolicyEventsByType(t).Count); + // logger.LogInformation($"Ordered policy types by count in {renderSw2.GetElapsedAndRestart()}"); - Console.WriteLine($"Rendered policies in {renderSw.GetElapsedAndRestart()}"); - Console.WriteLine($"Rendered in {renderTotalSw.Elapsed}"); -} + @if (DuplicateBans?.ActivePolicies.Count > 0) { + <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@DuplicateBans.Value" + Room="@Room"></PolicyListCategoryComponent> + } -@if (CurrentlyEditingEvent is not null) { - <PolicyEditorModal PolicyEvent="@CurrentlyEditingEvent" OnClose="@(() => CurrentlyEditingEvent = null)" OnSave="@(e => UpdatePolicyAsync(e))"></PolicyEditorModal> -} + @if (RedundantBans?.ActivePolicies.Count > 0) { + <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@RedundantBans.Value" + Room="@Room"></PolicyListCategoryComponent> + } -@if (ServerPolicyToMakePermanent is not null) { - <ModalWindow Title="Make policy permanent"> + foreach (var collection in PolicyCollections.Values.OrderByDescending(x => x.ActivePolicies.Count)) { + <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@collection" Room="@Room"></PolicyListCategoryComponent> + } - </ModalWindow> -} + // foreach (var type in policiesByType) { + @* foreach (var type in (List<Type>) []) { *@ + @* <details> *@ + @* <summary> *@ + @* <span> *@ + @* @($"{GetPolicyTypeName(type)}: {GetPolicyEventsByType(type).Count} policies") *@ + @* </span> *@ + @* <hr style="margin: revert;"/> *@ + @* </summary> *@ + @* <table class="table table-striped table-hover table-bordered align-middle"> *@ + @* @{ *@ + @* var renderSw3 = Stopwatch.StartNew(); *@ + @* var policies = GetValidPolicyEventsByType(type); *@ + @* var invalidPolicies = GetInvalidPolicyEventsByType(type); *@ + @* // enumerate all properties with friendly name *@ + @* var props = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) *@ + @* .Where(x => (x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyNameOrNull()) is not null) *@ + @* .Where(x => x.GetCustomAttribute<TableHideAttribute>() is null) *@ + @* .ToFrozenSet(); *@ + @* var propNames = props.Select(x => x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyName()!).ToFrozenSet(); *@ + @* *@ + @* var proxySafeProps = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) *@ + @* .Where(x => props.Any(y => y.Name == x.Name)) *@ + @* .ToFrozenSet(); *@ + @* logger.LogInformation($"{proxySafeProps?.Count} proxy safe props found in {policies.FirstOrDefault()?.TypedContent?.GetType()}"); *@ + @* logger.LogInformation($"Filtered policies and got properties in {renderSw3.GetElapsedAndRestart()}"); *@ + @* } *@ + @* <thead> *@ + @* <tr> *@ + @* @foreach (var name in propNames) { *@ + @* <th>@name</th> *@ + @* } *@ + @* <th>Actions</th> *@ + @* </tr> *@ + @* </thead> *@ + @* <tbody> *@ + @* @foreach (var policy in policies.OrderBy(x => x.RawContent?["entity"]?.GetValue<string>())) { *@ + @* <PolicyListRowComponent PolicyInfo="@policy" Room="@Room"></PolicyListRowComponent> *@ + @* } *@ + @* </tbody> *@ + @* </table> *@ + @* <details> *@ + @* <summary> *@ + @* <u> *@ + @* @("Invalid " + GetPolicyTypeName(type).ToLower()) *@ + @* </u> *@ + @* </summary> *@ + @* <table class="table table-striped table-hover"> *@ + @* <thead> *@ + @* <tr> *@ + @* <th>State key</th> *@ + @* <th>Json contents</th> *@ + @* </tr> *@ + @* </thead> *@ + @* <tbody> *@ + @* @foreach (var policy in invalidPolicies) { *@ + @* <tr> *@ + @* <td>@policy.StateKey</td> *@ + @* <td> *@ + @* <pre>@policy.RawContent.ToJson(true, false)</pre> *@ + @* </td> *@ + @* </tr> *@ + @* } *@ + @* </tbody> *@ + @* </table> *@ + @* </details> *@ + @* </details> *@ + // } -@if (MassCreatePolicies) { - <MassPolicyEditorModal Room="@Room" OnClose="@(() => MassCreatePolicies = false)" OnSaved="@(() => { MassCreatePolicies = false; LoadStatesAsync(); })"></MassPolicyEditorModal> + // logger.LogInformation($"Rendered policies in {renderSw.GetElapsedAndRestart()}"); + logger.LogInformation("Rendered in {TimeSpan}", renderTotalSw.Elapsed); + } } @code { @@ -168,50 +157,50 @@ else { private const bool Debug = false; #endif + private bool IsInitialised { get; set; } = false; private bool Loading { get; set; } = true; [Parameter] - public string RoomId { get; set; } + public required string RoomId { get; set; } - private bool _enableAvatars; - private StateEventResponse? _currentlyEditingEvent; - private bool _massCreatePolicies; - private StateEventResponse? _serverPolicyToMakePermanent; - - private Dictionary<Type, List<StateEventResponse>> PolicyEventsByType { get; set; } = new(); - - private StateEventResponse? CurrentlyEditingEvent { - get => _currentlyEditingEvent; + [Parameter, SupplyParameterFromQuery] + public bool RenderEventInfo { + get; set { - _currentlyEditingEvent = value; + field = value; StateHasChanged(); } } - public StateEventResponse? ServerPolicyToMakePermanent { - get => _serverPolicyToMakePermanent; + private Dictionary<Type, List<MatrixEventResponse>> PolicyEventsByType { get; set; } = new(); + + public MatrixEventResponse? ServerPolicyToMakePermanent { + get; set { - _serverPolicyToMakePermanent = value; + field = value; StateHasChanged(); } } - private AuthenticatedHomeserverGeneric Homeserver { get; set; } - private GenericRoom Room { get; set; } - private RoomPowerLevelEventContent PowerLevels { get; set; } + private AuthenticatedHomeserverGeneric Homeserver { get; set; } = null!; + private GenericRoom Room { get; set; } = null!; + private RoomPowerLevelEventContent PowerLevels { get; set; } = null!; public bool CurrentUserIsDraupnir { get; set; } - public string? RoomName { get; set; } - public string? RoomAlias { get; set; } - public string? DraupnirShortcode { get; set; } - public Dictionary<StateEventResponse, int> ActiveKicks { get; set; } = []; - public bool MassCreatePolicies { - get => _massCreatePolicies; - set { - _massCreatePolicies = value; - StateHasChanged(); - } - } + public Dictionary<MatrixEventResponse, int> ActiveKicks { get; set; } = []; + + private static FrozenSet<Type> KnownPolicyTypes = MatrixEvent.KnownEventTypes.Where(x => x.IsAssignableTo(typeof(PolicyRuleEventContent))).ToFrozenSet(); + + // event types, unnamed + // private static Dictionary<string, Type> PolicyTypes = KnownPolicyTypes + // .ToDictionary(x => x.GetCustomAttributes<MatrixEventAttribute>().First(y => !string.IsNullOrWhiteSpace(y.EventName)).EventName, x => x); + // + // private static Dictionary<Type, string[]> PolicyTypeIds = KnownPolicyTypes + // .ToDictionary(x => x, x => x.GetCustomAttributes<MatrixEventAttribute>().Select(y => y.EventName).ToArray()); + + Dictionary<Type, PolicyCollection> PolicyCollections { get; set; } = new(); + PolicyCollection? DuplicateBans { get; set; } + PolicyCollection? RedundantBans { get; set; } protected override async Task OnInitializedAsync() { var sw = Stopwatch.StartNew(); @@ -219,202 +208,456 @@ else { Homeserver = (await sessionStore.GetCurrentHomeserver(navigateOnFailure: true))!; if (Homeserver is null) return; Room = Homeserver.GetRoom(RoomId!); - await Task.WhenAll([ + IsInitialised = true; + StateHasChanged(); + await Task.WhenAll( Task.Run(async () => { PowerLevels = (await Room.GetPowerLevelsAsync())!; }), - Task.Run(async () => { DraupnirShortcode = (await Room.GetStateOrNullAsync<MjolnirShortcodeEventContent>(MjolnirShortcodeEventContent.EventId))?.Shortcode; }), - Task.Run(async () => { RoomAlias = (await Room.GetCanonicalAliasAsync())?.Alias; }), - Task.Run(async () => { RoomName = await Room.GetNameOrFallbackAsync(); }), - Task.Run(async () => { CurrentUserIsDraupnir = (await Homeserver.GetAccountDataOrNullAsync<object>("org.matrix.mjolnir.protected_rooms")) is not null; }), - ]); - await LoadStatesAsync(); - Console.WriteLine($"Policy list editor initialized in {sw.Elapsed}!"); + Task.Run(async () => { CurrentUserIsDraupnir = (await Homeserver.GetAccountDataOrNullAsync<object>(DraupnirProtectedRoomsData.EventId)) is not null; }) + ); + StateHasChanged(); + await LoadStateAsync(firstLoad: true); + Loading = false; + logger.LogInformation("Policy list editor initialized in {SwElapsed}!", sw.Elapsed); } - private async Task LoadStatesAsync() { - Loading = true; - var states = Room.GetFullStateAsync(); - PolicyEventsByType.Clear(); - await foreach (var state in states) { - if (state is null) continue; - if (!state.MappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue; - if (!PolicyEventsByType.ContainsKey(state.MappedType)) PolicyEventsByType.Add(state.MappedType, new()); - PolicyEventsByType[state.MappedType].Add(state); + private async Task LoadStateAsync(bool firstLoad = false) { + // preload workers in task pool + // await Task.WhenAll(Enumerable.Range(0, WebWorkerService.MaxWorkerCount).Select(async _ => (await WebWorkerService.TaskPool.GetWorkerAsync()).WhenReady).ToList()); + var taskPoolReadyTask = WebWorkerService.TaskPool.SetWorkerCount(WebWorkerService.MaxWorkerCount); + var sw = Stopwatch.StartNew(); + // Loading = true; + // var states = Room.GetFullStateAsync(); + var states = await Room.GetFullStateAsListAsync(); + // PolicyEventsByType.Clear(); + logger.LogInformation("LoadStatesAsync: Loaded state in {SwElapsed}", sw.Elapsed); + + foreach (var type in KnownPolicyTypes) { + if (!PolicyCollections.ContainsKey(type)) { + var filterPropSw = Stopwatch.StartNew(); + // enumerate all properties with friendly name + var props = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(x => (x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyNameOrNull()) is not null) + .Where(x => x.GetCustomAttribute<TableHideAttribute>() is null) + .ToFrozenSet(); + + var proxySafeProps = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(x => props.Any(y => y.Name == x.Name)) + .ToFrozenDictionary(x => x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyName(), x => x); + logger.LogInformation("{Count} proxy safe props found in {TypeFullName} ({TimeSpan})", proxySafeProps?.Count, type.FullName, filterPropSw.Elapsed); + PolicyCollections.Add(type, new() { + Name = type.GetFriendlyNamePluralOrNull() ?? type.FullName ?? type.Name, + ActivePolicies = [], + RemovedPolicies = [], + PropertiesToDisplay = proxySafeProps + }); + } } + var count = 0; + var parseSw = Stopwatch.StartNew(); + foreach (var evt in states) { + var mappedType = evt.MappedType; + if (count % 100 == 0) + logger.LogInformation("Processing state #{Count:000000} {EvtType} @ {SwElapsed} (took {ParseSwElapsed:c} so far to process)", count, evt.Type, sw.Elapsed, parseSw.Elapsed); + count++; + + if (!mappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue; + + var collection = PolicyCollections[mappedType]; + + var key = (evt.Type, evt.StateKey!); + var policyInfo = new PolicyCollection.PolicyInfo { + Policy = evt, + MadeRedundantBy = [], + DuplicatedBy = [] + }; + if (evt.RawContent is null or { Count: 0 } || string.IsNullOrWhiteSpace(evt.RawContent?["recommendation"]?.GetValue<string>())) { + collection.ActivePolicies.Remove(key); + if (!collection.RemovedPolicies.TryAdd(key, policyInfo)) { + if (MatrixEvent.Equals(collection.RemovedPolicies[key].Policy, evt)) continue; + collection.RemovedPolicies[key] = policyInfo; + } + } + else { + collection.RemovedPolicies.Remove(key); + if (!collection.ActivePolicies.TryAdd(key, policyInfo)) { + if (MatrixEvent.Equals(collection.ActivePolicies[key].Policy, evt)) continue; + collection.ActivePolicies[key] = policyInfo; + } + } + } + + logger.LogInformation("LoadStatesAsync: Processed state in {SwElapsed}", sw.Elapsed); + foreach (var collection in PolicyCollections) { + logger.LogInformation("Policy collection {KeyFullName} has {ActivePoliciesCount} active and {RemovedPoliciesCount} removed policies.", collection.Key.FullName, collection.Value.ActivePolicies.Count, collection.Value.RemovedPolicies.Count); + } + + await Task.Delay(1); + Loading = false; StateHasChanged(); - } + await Task.Delay(100); - private List<StateEventResponse> GetPolicyEventsByType(Type type) => PolicyEventsByType.ContainsKey(type) ? PolicyEventsByType[type] : []; + // return; + logger.LogInformation("LoadStatesAsync: Scanning for redundant policies..."); - private List<StateEventResponse> GetValidPolicyEventsByType(Type type) => GetPolicyEventsByType(type) - .Where(x => !string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList(); + var scanSw = Stopwatch.StartNew(); + // var allPolicyInfos = PolicyCollections.Values + // .SelectMany(x => x.ActivePolicies.Values) + // .ToArray(); + // var allPolicies = allPolicyInfos + // .Select<PolicyCollection.PolicyInfo, (PolicyCollection.PolicyInfo PolicyInfo, PolicyRuleEventContent TypedContent)>(x => (x, (x.Policy.TypedContent as PolicyRuleEventContent)!)) + // .ToList(); + // var hashPolicies = allPolicies + // .Where(x => x.TypedContent.IsHashedRule()) + // .ToList(); + // var wildcardPolicies = allPolicies + // .Except(hashPolicies) // hashed policies cannot be wildcards + // .Where(x => x.TypedContent.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent) + // .ToList(); + // var nonWildcardPolicies = allPolicies + // // .Except(wildcardPolicies) + // .Where(x => !x.TypedContent!.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent) + // .ToList(); + // Console.WriteLine($"Got {allPolicies.Count} total policies, {wildcardPolicies.Count} wildcard policies. Time spent: {scanSw.Elapsed}"); + // int i = 0; + // int hits = 0; + // int redundant = 0; + // int duplicates = 0; - private List<StateEventResponse> GetInvalidPolicyEventsByType(Type type) => GetPolicyEventsByType(type) - .Where(x => string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList(); + // foreach (var (policyInfo, policyContent) in allPolicies) { + // foreach (var (otherPolicyInfo, otherPolicyContent) in allPolicies) { + // if (policyInfo.Policy == otherPolicyInfo.Policy) continue; // same event + // if (MatrixEvent.TypeKeyPairMatches(policyInfo.Policy, otherPolicyInfo.Policy)) { + // logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson()); + // continue; // same type and state key + // } + // // if(!policyContent.IsHashedRule()) + // } + // + // if (++i % 100 == 0) { + // Console.WriteLine($"Processed {i} policies in {scanSw.Elapsed}"); + // await Task.Delay(1); + // } + // } - private string? GetPolicyTypeNameOrNull(Type type) => type.GetFriendlyNamePluralOrNull() - ?? type.GetCustomAttributes<MatrixEventAttribute>() - .FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.EventName))?.EventName; + int scanningPolicyCount = 0; + var aggregatedPolicies = PolicyCollections.Values + .Aggregate(new List<MatrixEventResponse>(), (acc, val) => { + acc.AddRange(val.ActivePolicies.Select(x => x.Value.Policy)); + return acc; + }); + Console.WriteLine($"Scanning for redundant policies in {aggregatedPolicies.Count} total policies... ({scanSw.Elapsed})"); + List<Task<List<PolicyCollection.PolicyInfo>>> tasks = []; + // try to save some load... + var policiesJson = JsonSerializer.Serialize(aggregatedPolicies); + var policiesJsonMarshalled = JsRuntime.ReturnMe<SpawnDev.BlazorJS.JSObjects.String>(policiesJson); + var ranges = Enumerable.Range(0, aggregatedPolicies.Count).DistributeSequentially(WebWorkerService.MaxWorkerCount); + await taskPoolReadyTask; + tasks.AddRange(ranges.Select(range => WebWorkerService.TaskPool.Invoke(CheckDuplicatePoliciesAsync, policiesJsonMarshalled, range.First(), range.Last()))); - private string GetPolicyTypeName(Type type) => GetPolicyTypeNameOrNull(type) ?? type.Name; + Console.WriteLine($"Main: started {tasks.Count} workers in {scanSw.Elapsed}"); + // tasks.Add(CheckDuplicatePoliciesAsync(allPolicyInfos, range.First() .. range.Last())); + + // var allPolicyEvents = aggregatedPolicies.Select(x => x.Policy).ToList(); + + DuplicateBans = new() { + Name = "Duplicate bans", + ViewType = PolicyCollection.SpecialViewType.Duplicates, + ActivePolicies = [], + RemovedPolicies = [], + PropertiesToDisplay = PolicyCollections.SelectMany(x => x.Value.PropertiesToDisplay).DistinctBy(x => x.Key).ToFrozenDictionary() + }; + + RedundantBans = new() { + Name = "Redundant bans", + ViewType = PolicyCollection.SpecialViewType.Redundant, + ActivePolicies = [], + RemovedPolicies = [], + PropertiesToDisplay = PolicyCollections.SelectMany(x => x.Value.PropertiesToDisplay).DistinctBy(x => x.Key).ToFrozenDictionary() + }; + + var allPolicyInfos = PolicyCollections.Values + .SelectMany(x => x.ActivePolicies.Values) + .ToArray(); + + await foreach (var modifiedPolicyInfos in tasks.ToAsyncResultEnumerable()) { + if (modifiedPolicyInfos.Count == 0) continue; + var applySw = Stopwatch.StartNew(); + // Console.WriteLine($"Main: got {modifiedPolicyInfos.Count} modified policies from worker, time: {scanSw.Elapsed}"); + foreach (var modifiedPolicyInfo in modifiedPolicyInfos) { + var original = allPolicyInfos.First(p => p.Policy.EventId == modifiedPolicyInfo.Policy.EventId); + original.DuplicatedBy = aggregatedPolicies.Where(x => modifiedPolicyInfo.DuplicatedBy.Any(y => MatrixEvent.Equals(x, y))).ToList(); + original.MadeRedundantBy = aggregatedPolicies.Where(x => modifiedPolicyInfo.MadeRedundantBy.Any(y => MatrixEvent.Equals(x, y))).ToList(); + modifiedPolicyInfo.DuplicatedBy = modifiedPolicyInfo.MadeRedundantBy = []; // Early dereference + if (original.DuplicatedBy.Count > 0) { + if (!DuplicateBans.Value.ActivePolicies.ContainsKey((original.Policy.Type, original.Policy.StateKey!))) + DuplicateBans.Value.ActivePolicies.Add((original.Policy.Type, original.Policy.StateKey!), original); + } - private async Task RemovePolicyAsync(StateEventResponse policyEvent) { - await Room.SendStateEventAsync(policyEvent.Type, policyEvent.StateKey, new { }); - PolicyEventsByType[policyEvent.MappedType].Remove(policyEvent); - await LoadStatesAsync(); + if (original.MadeRedundantBy.Count > 0) { + if (!RedundantBans.Value.ActivePolicies.ContainsKey((original.Policy.Type, original.Policy.StateKey!))) + RedundantBans.Value.ActivePolicies.Add((original.Policy.Type, original.Policy.StateKey!), original); + } + // Console.WriteLine($"Memory usage: {Util.BytesToString(GC.GetTotalMemory(false))}"); + } + + Console.WriteLine($"Main: Processed {modifiedPolicyInfos.Count} modified policies in {scanSw.Elapsed} (applied in {applySw.Elapsed})"); + } + + Console.WriteLine($"Processed {allPolicyInfos.Length} policies in {scanSw.Elapsed}"); + + // // scan for wildcard matches + // foreach (var policy in allPolicies) { + // var matchingPolicies = wildcardPolicies + // .Where(x => + // !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy) + // && x.Item2.EntityMatches(policy.TypedContent.Entity!) + // ) + // .ToList(); + // + // if (matchingPolicies.Count > 0) { + // logger.LogInformation($"{i} Got {matchingPolicies.Count} hits for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}"); + // foreach (var match in matchingPolicies) { + // policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy); + // } + // + // hits++; + // redundant += matchingPolicies.Count; + // + // if (hits % 5 == 0) + // StateHasChanged(); + // } + // else { + // //logger.LogInformation("Sleeping..."); + // await Task.Delay(1); + // } + // + // i++; + // } + // + // i = 0; + // // scan for exact duplicates + // foreach (var policy in allPolicies) { + // var matchingPolicies = allPolicies + // .Where(x => + // !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy) + // && ( + // x.Item2.IsHashedRule() + // ? x.Item2.EntityMatches(policy.Item2.Entity) + // : x.Item2!.Entity == policy.Item2.Entity! + // ) + // ) + // .ToList(); + // + // if (matchingPolicies.Count > 0) { + // logger.LogInformation($"{i} Got {matchingPolicies.Count} duplicates for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}"); + // foreach (var match in matchingPolicies) { + // policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy); + // } + // + // hits++; + // duplicates += matchingPolicies.Count; + // + // if (hits % 5 == 0) + // StateHasChanged(); + // } + // else { + // //logger.LogInformation("Sleeping..."); + // await Task.Delay(1); + // } + // + // i++; + // } + // + // logger.LogInformation($"LoadStatesAsync: Found {hits} ({redundant} redundant, {duplicates} duplicates) redundant policies in {sw.Elapsed}"); + // StateHasChanged(); } - private async Task UpdatePolicyAsync(StateEventResponse policyEvent) { - await Room.SendStateEventAsync(policyEvent.Type, policyEvent.StateKey, policyEvent.RawContent); - CurrentlyEditingEvent = null; - await LoadStatesAsync(); + [return: WorkerTransfer] + private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(SpawnDev.BlazorJS.JSObjects.String policiesJson, int start, int end) { + var policies = JsonSerializer.Deserialize<List<MatrixEventResponse>>(policiesJson.ValueOf()); + Console.WriteLine($"Got request to check duplicate policies in range {start} to {end} (length: {end - start}), {policiesJson.ValueOf().Length} bytes of JSON ({policies!.Count} policies)"); + return await CheckDuplicatePoliciesAsync(policies!, start .. end); } - private async Task UpgradePolicyAsync(StateEventResponse policyEvent) { - policyEvent.RawContent["upgraded_from_type"] = policyEvent.Type; - await LoadStatesAsync(); + [return: WorkerTransfer] + private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(string policiesJson, int start, int end) { + var policies = JsonSerializer.Deserialize<List<MatrixEventResponse>>(policiesJson); + Console.WriteLine($"Got request to check duplicate policies in range {start} to {end} (length: {end - start}), {policiesJson.Length} bytes of JSON ({policies!.Count} policies)"); + return await CheckDuplicatePoliciesAsync(policies!, start .. end); } - private static FrozenSet<Type> KnownPolicyTypes = StateEvent.KnownStateEventTypes.Where(x => x.IsAssignableTo(typeof(PolicyRuleEventContent))).ToFrozenSet(); + [return: WorkerTransfer] + private static Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(List<MatrixEventResponse> policies, int start, int end) + => CheckDuplicatePoliciesAsync(policies, start .. end); - // event types, unnamed - private static Dictionary<string, Type> PolicyTypes = KnownPolicyTypes - .ToDictionary(x => x.GetCustomAttributes<MatrixEventAttribute>().First(y => !string.IsNullOrWhiteSpace(y.EventName)).EventName, x => x); + [return: WorkerTransfer] + private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(List<MatrixEventResponse> policies, Range range) { + var sw = Stopwatch.StartNew(); + var jsConsole = App.Host.Services.GetService<JsConsoleService>()!; + Console.WriteLine($"Processing policies in range {range} ({range.GetOffsetAndLength(policies.Count).Length}) with {policies.Count} total policies"); + var allPolicies = policies + .Select(x => (Event: x, TypedContent: (x.TypedContent as PolicyRuleEventContent)!)) + .ToList(); + var toCheck = allPolicies[range]; + var modifiedPolicies = new List<PolicyCollection.PolicyInfo>(); - private static Dictionary<Type, string[]> PolicyTypeIds = KnownPolicyTypes - .ToDictionary(x => x, x => x.GetCustomAttributes<MatrixEventAttribute>().Select(y => y.EventName).ToArray()); + foreach (var (policyEvent, policyContent) in toCheck) { + List<MatrixEventResponse> duplicatedBy = []; + List<MatrixEventResponse> madeRedundantBy = []; -#region Draupnir interop + foreach (var (otherPolicyEvent, otherPolicyContent) in allPolicies) { + if (policyEvent == otherPolicyEvent) continue; // same event + if (MatrixEvent.TypeKeyPairMatches(policyEvent, otherPolicyEvent)) { + // logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson()); + Console.WriteLine($"Sanity check failed: Found same type and state key for two different policies: {policyEvent.RawContent.ToJson()} and {otherPolicyEvent.RawContent.ToJson()}"); + continue; // same type and state key + } - private SemaphoreSlim ss = new(16, 16); + // if(!policyContent.IsHashedRule()) + if (!string.IsNullOrWhiteSpace(policyContent.Entity) && policyContent.Entity == otherPolicyContent.Entity) { + // Console.WriteLine($"Found duplicate policy: {policyEvent.EventId} is duplicated by {otherPolicyEvent.EventId}"); + duplicatedBy.Add(otherPolicyEvent); + } + } - private async Task DraupnirKickMatching(StateEventResponse policy) { - try { - var content = policy.TypedContent! as PolicyRuleEventContent; - if (content is null) return; - if (string.IsNullOrWhiteSpace(content.Entity)) return; + if (duplicatedBy.Count > 0 || madeRedundantBy.Count > 0) { + var summary = $"Policy {policyEvent.EventId} is:"; + if (duplicatedBy.Count > 0) + summary += $"\n- Duplicated by {duplicatedBy.Count} policies: {string.Join(", ", duplicatedBy.Select(x => x.EventId))}"; + if (madeRedundantBy.Count > 0) + summary += $"\n- Made redundant by {madeRedundantBy.Count} policies: {string.Join(", ", madeRedundantBy.Select(x => x.EventId))}"; + // Console.WriteLine(summary); + await jsConsole.Info(summary); + await Task.Delay(1); + modifiedPolicies.Add(new() { + Policy = policyEvent, + DuplicatedBy = duplicatedBy, + MadeRedundantBy = madeRedundantBy + }); + } - var data = await Homeserver.GetAccountDataAsync<DraupnirProtectedRoomsData>(DraupnirProtectedRoomsData.EventId); - var rooms = data.Rooms.Select(Homeserver.GetRoom).ToList(); + // await Task.Delay(1); + } - ActiveKicks.Add(policy, rooms.Count); - StateHasChanged(); - await Task.Delay(500); + await jsConsole.Info($"Worker: Found {modifiedPolicies.Count} modified policies in range {range} (length: {range.GetOffsetAndLength(policies.Count).Length}) in {sw.Elapsed}"); - // for (int i = 0; i < 12; i++) { - // _ = WebWorkerService.TaskPool.Invoke(WasteCpu); - // } + return modifiedPolicies; + } + + // the old one: + private async Task LoadStatesAsync(bool firstLoad = false) { + await LoadStateAsync(firstLoad); + return; + var sw = Stopwatch.StartNew(); + Loading = true; + // var states = Room.GetFullStateAsync(); + var states = await Room.GetFullStateAsListAsync(); + // PolicyEventsByType.Clear(); - // static async Task runKicks(string roomId, PolicyRuleEventContent content) { - // Console.WriteLine($"Checking {roomId}..."); - // // Console.WriteLine($"Checking {room.RoomId}..."); - // // - // // try { - // // var members = await room.GetMembersListAsync(); - // // foreach (var member in members) { - // // var membership = member.ContentAs<RoomMemberEventContent>(); - // // if (member.StateKey == room.Homeserver.WhoAmI.UserId) continue; - // // if (membership?.Membership is "leave" or "ban") continue; - // // - // // if (content.EntityMatches(member.StateKey!)) - // // // await room.KickAsync(member.StateKey, content.Reason ?? "No reason given"); - // // Console.WriteLine($"Would kick {member.StateKey} from {room.RoomId} (EntityMatches)"); - // // } - // // } - // // finally { - // // Console.WriteLine($"Finished checking {room.RoomId}..."); - // // } - // } - // - // try { - // var tasks = rooms.Select(room => WebWorkerService.TaskPool.Invoke(runKicks, room.RoomId, content)).ToList(); - // - // await Task.WhenAll(tasks); - // } - // catch (Exception e) { - // Console.WriteLine(e); - // } + logger.LogInformation("LoadStatesAsync: Loaded state in {SwElapsed}", sw.Elapsed); - await NastyInternalsPleaseIgnore.ExecuteKickWithWasmWorkers(WebWorkerService, Homeserver, policy, data.Rooms); - // await Task.Run(async () => { - // foreach (var room in rooms) { - // try { - // Console.WriteLine($"Checking {room.RoomId}..."); - // var members = await room.GetMembersListAsync(); - // foreach (var member in members) { - // var membership = member.ContentAs<RoomMemberEventContent>(); - // if (member.StateKey == room.Homeserver.WhoAmI.UserId) continue; - // if (membership?.Membership is "leave" or "ban") continue; - // - // if (content.EntityMatches(member.StateKey!)) - // // await room.KickAsync(member.StateKey, content.Reason ?? "No reason given"); - // Console.WriteLine($"Would kick {member.StateKey} from {room.RoomId} (EntityMatches)"); - // } - // ActiveKicks[policy]--; - // StateHasChanged(); - // } - // finally { - // Console.WriteLine($"Finished checking {room.RoomId}..."); - // } - // } - // }); - } - finally { - ActiveKicks.Remove(policy); - StateHasChanged(); - await Task.Delay(500); + foreach (var type in KnownPolicyTypes) { + if (!PolicyEventsByType.ContainsKey(type)) + PolicyEventsByType.Add(type, new List + <MatrixEventResponse>(16000)); } - } -#region Nasty, nasty internals, please ignore! + int count = 0; + + foreach (var state in states) { + var _spsw = Stopwatch.StartNew(); + TimeSpan e1, e2, e3, e4, e5, e6, t; + if (!state.MappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue; + e1 = _spsw.Elapsed; + var targetPolicies = PolicyEventsByType[state.MappedType]; + e2 = _spsw.Elapsed; + if (!firstLoad && targetPolicies.FirstOrDefault(x => MatrixEvent.TypeKeyPairMatches(x, state)) is { } evt) { + e3 = _spsw.Elapsed; + if (MatrixEvent.Equals(evt, state)) { + if (count % 100 == 0) { + await Task.Delay(10); + await Task.Yield(); + } - private static class NastyInternalsPleaseIgnore { - public static async Task ExecuteKickWithWasmWorkers(WebWorkerService workerService, AuthenticatedHomeserverGeneric hs, StateEventResponse evt, List<string> roomIds) { - try { - // var tasks = roomIds.Select(roomId => workerService.TaskPool.Invoke(ExecuteKickInternal, hs.WellKnownUris.Client, hs.AccessToken, roomId, content.Entity)).ToList(); - var tasks = roomIds.Select(roomId => workerService.TaskPool.Invoke(ExecuteKickInternal2, hs.WellKnownUris, hs.AccessToken, roomId, evt)).ToList(); - // workerService.TaskPool.Invoke(ExecuteKickInternal, hs.BaseUrl, hs.AccessToken, roomIds, content.Entity); - await Task.WhenAll(tasks); + e4 = _spsw.Elapsed; + logger.LogInformation("[E] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={E3:c}, e4={E4:c}, e5={Zero:c},t={SpswElapsed:c})", count++, state.Type, sw.Elapsed, e1, e2, e3, e4, TimeSpan.Zero, _spsw.Elapsed); + continue; + } + + e4 = _spsw.Elapsed; + targetPolicies.Remove(evt); + e5 = _spsw.Elapsed; + targetPolicies.Add(state); + e6 = _spsw.Elapsed; + t = _spsw.Elapsed; + logger.LogInformation("[M] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={E3:c}, e4={E4:c}, e5={E5:c}, e6={E6:c},t={TimeSpan1:c})", count++, state.Type, sw.Elapsed, e1, e2, e3, e4, e5, e6, t); } - catch (Exception e) { - Console.WriteLine(e); + else { + targetPolicies.Add(state); + t = _spsw.Elapsed; + logger.LogInformation("[N] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={Zero:c}, e4={TimeSpan1:c}, e5={Zero1:c}, e6={TimeSpan2:c}, t={TimeSpan3:c})", count++, state.Type, sw.Elapsed, e1, e2, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, t); } + + // await Task.Delay(10); + // await Task.Yield(); } - private static async Task ExecuteKickInternal(string homeserverBaseUrl, string accessToken, string roomId, string entity) { - try { - Console.WriteLine("args: " + string.Join(", ", homeserverBaseUrl, accessToken, roomId, entity)); - Console.WriteLine($"Checking {roomId}..."); - var hs = new AuthenticatedHomeserverGeneric(homeserverBaseUrl, new() { Client = homeserverBaseUrl }, null, accessToken); - Console.WriteLine($"Got HS..."); - var room = hs.GetRoom(roomId); - Console.WriteLine($"Got room..."); - var members = await room.GetMembersListAsync(); - Console.WriteLine($"Got members..."); - // foreach (var member in members) { - // var membership = member.ContentAs<RoomMemberEventContent>(); - // if (member.StateKey == hs.WhoAmI.UserId) continue; - // if (membership?.Membership is "leave" or "ban") continue; - // - // if (entity == member.StateKey) - // // await room.KickAsync(member.StateKey, content.Reason ?? "No reason given"); - // Console.WriteLine($"Would kick {member.StateKey} from {room.RoomId} (EntityMatches)"); - // } - } - catch (Exception e) { - Console.WriteLine(e); - } + logger.LogInformation("LoadStatesAsync: Processed state in {SwElapsed}", sw.Elapsed); + + Loading = false; + StateHasChanged(); + await Task.Delay(10); + await Task.Yield(); + logger.LogInformation("LoadStatesAsync: yield finished in {SwElapsed}", sw.Elapsed); + } + + private List<MatrixEventResponse> GetPolicyEventsByType(Type type) => PolicyEventsByType.ContainsKey(type) ? PolicyEventsByType[type] : []; + + // private List<MatrixEventResponse> GetValidPolicyEventsByType(Type type) => GetPolicyEventsByType(type) + // .Where(x => !string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList(); + // + // private List<MatrixEventResponse> GetInvalidPolicyEventsByType(Type type) => GetPolicyEventsByType(type) + // .Where(x => x.RawContent is { Count: > 0 } && string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList(); + // + // private List<MatrixEventResponse> GetRemovedPolicyEventsByType(Type type) => GetPolicyEventsByType(type) + // .Where(x => x.RawContent is null or { Count: 0 }).ToList(); + + private string? GetPolicyTypeNameOrNull(Type type) => type.GetFriendlyNamePluralOrNull() + ?? type.GetCustomAttributes<MatrixEventAttribute>() + .FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.EventName))?.EventName; + + private string GetPolicyTypeName(Type type) => GetPolicyTypeNameOrNull(type) ?? type.Name; + + public struct PolicyCollection { + public required string Name { get; init; } + public SpecialViewType ViewType { get; init; } + public int TotalCount => ActivePolicies.Count + RemovedPolicies.Count; + + public required Dictionary<(string Type, string StateKey), PolicyInfo> ActivePolicies { get; set; } + + // public Dictionary<(string Type, string StateKey), MatrixEventResponse> InvalidPolicies { get; set; } + public required Dictionary<(string Type, string StateKey), PolicyInfo> RemovedPolicies { get; set; } + public required FrozenDictionary<string, PropertyInfo> PropertiesToDisplay { get; set; } + + public class PolicyInfo { + public required MatrixEventResponse Policy { get; init; } + public required List<MatrixEventResponse> MadeRedundantBy { get; set; } + public required List<MatrixEventResponse> DuplicatedBy { get; set; } } - private async static Task ExecuteKickInternal2(HomeserverResolverService.WellKnownUris wellKnownUris, string accessToken, string roomId, StateEventResponse policy) { - Console.WriteLine($"Checking {roomId}..."); - Console.WriteLine(policy.EventId); + public enum SpecialViewType { + None, + Duplicates, + Redundant, } } -#endregion - -#endregion + // private struct PolicyStats { + // public int Active { get; set; } + // public int Invalid { get; set; } + // public int Removed { get; set; } + // } } \ No newline at end of file