path: root/MatrixUtils.Web/Pages/Rooms/PolicyList.razor

@page "/Rooms/{RoomId}/Policies"
@using LibMatrix
@using ArcaneLibs.Extensions
@using LibMatrix.EventTypes.Spec.State.Policy
@using System.Diagnostics
@using LibMatrix.RoomTypes
@using System.Collections.Frozen
@using System.Collections.Immutable
@using System.Reflection
@using System.Text.Json
@using ArcaneLibs.Attributes
@using ArcaneLibs.Blazor.Components.Services
@using LibMatrix.EventTypes
@using LibMatrix.EventTypes.Interop.Draupnir
@using LibMatrix.EventTypes.Spec.State.RoomInfo
@using SpawnDev.BlazorJS.WebWorkers
@using MatrixUtils.Web.Pages.Rooms.PolicyListComponents
@using SpawnDev.BlazorJS
@inject WebWorkerService WebWorkerService
@inject ILogger<PolicyList> logger
@inject BlazorJSRuntime JsRuntime

@if (!IsInitialised) {
    <p>Connecting to homeserver...</p>
}
else {
    <PolicyListEditorHeader Room="@Room" @bind-RenderEventInfo="@RenderEventInfo" ReloadStateAsync="@(() => LoadStateAsync(true))"></PolicyListEditorHeader>
    @if (Loading) {
        <p>Loading...</p>
    }
    // else if (PolicyEventsByType is not { Count: > 0 }) {
    @* <p>No policies yet</p> *@
    // }
    else {
        var renderSw = Stopwatch.StartNew();
        var renderTotalSw = Stopwatch.StartNew();
        @foreach (var value in PolicyCollections.Values.OrderByDescending(x => x.TotalCount)) {
            <p>
                @value.ActivePolicies.Count active,
                @value.RemovedPolicies.Count removed
                (@value.TotalCount total)
                @value.Name.ToLower()
            </p>
        }

        @if (DuplicateBans?.ActivePolicies.Count > 0) {
            <p style="color: orange;">
                Found @DuplicateBans.Value.ActivePolicies.Count duplicate bans
            </p>
        }

        @if (RedundantBans?.ActivePolicies.Count > 0) {
            <p style="color: orange;">
                Found @RedundantBans.Value.ActivePolicies.Count redundant bans
            </p>
        }

        // logger.LogInformation($"Rendered header in {renderSw.GetElapsedAndRestart()}");

        // var renderSw2 = Stopwatch.StartNew();
        // IOrderedEnumerable<Type> policiesByType = KnownPolicyTypes.Where(t => GetPolicyEventsByType(t).Count > 0).OrderByDescending(t => GetPolicyEventsByType(t).Count);
        // logger.LogInformation($"Ordered policy types by count in {renderSw2.GetElapsedAndRestart()}");

        @if (DuplicateBans?.ActivePolicies.Count > 0) {
            <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@DuplicateBans.Value"
                                         Room="@Room"></PolicyListCategoryComponent>
        }

        @if (RedundantBans?.ActivePolicies.Count > 0) {
            <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@RedundantBans.Value"
                                         Room="@Room"></PolicyListCategoryComponent>
        }

        foreach (var collection in PolicyCollections.Values.OrderByDescending(x => x.ActivePolicies.Count)) {
            <PolicyListCategoryComponent RenderInvalidSection="false" RenderEventInfo="@RenderEventInfo" PolicyCollection="@collection" Room="@Room"></PolicyListCategoryComponent>
        }

        // foreach (var type in policiesByType) {
        @* foreach (var type in (List<Type>) []) { *@
        @*     <details> *@
        @*         <summary> *@
        @*             <span> *@
        @*                 @($"{GetPolicyTypeName(type)}: {GetPolicyEventsByType(type).Count} policies") *@
        @*             </span> *@
        @*             <hr style="margin: revert;"/> *@
        @*         </summary> *@
        @*         <table class="table table-striped table-hover table-bordered align-middle"> *@
        @*             @{ *@
        @*                 var renderSw3 = Stopwatch.StartNew(); *@
        @*                 var policies = GetValidPolicyEventsByType(type); *@
        @*                 var invalidPolicies = GetInvalidPolicyEventsByType(type); *@
        @*                 // enumerate all properties with friendly name *@
        @*                 var props = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) *@
        @*                     .Where(x => (x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyNameOrNull()) is not null) *@
        @*                     .Where(x => x.GetCustomAttribute<TableHideAttribute>() is null) *@
        @*                     .ToFrozenSet(); *@
        @*                 var propNames = props.Select(x => x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyName()!).ToFrozenSet(); *@
        @* *@
        @*                 var proxySafeProps = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) *@
        @*                     .Where(x => props.Any(y => y.Name == x.Name)) *@
        @*                     .ToFrozenSet(); *@
        @*                 logger.LogInformation($"{proxySafeProps?.Count} proxy safe props found in {policies.FirstOrDefault()?.TypedContent?.GetType()}"); *@
        @*                 logger.LogInformation($"Filtered policies and got properties in {renderSw3.GetElapsedAndRestart()}"); *@
        @*             } *@
        @*             <thead> *@
        @*                 <tr> *@
        @*                     @foreach (var name in propNames) { *@
        @*                         <th>@name</th> *@
        @*                     } *@
        @*                     <th>Actions</th> *@
        @*                 </tr> *@
        @*             </thead> *@
        @*             <tbody> *@
        @*                 @foreach (var policy in policies.OrderBy(x => x.RawContent?["entity"]?.GetValue<string>())) { *@
        @*                     <PolicyListRowComponent PolicyInfo="@policy" Room="@Room"></PolicyListRowComponent> *@
        @*                 } *@
        @*             </tbody> *@
        @*         </table> *@
        @*         <details> *@
        @*             <summary> *@
        @*                 <u> *@
        @*                     @("Invalid " + GetPolicyTypeName(type).ToLower()) *@
        @*                 </u> *@
        @*             </summary> *@
        @*             <table class="table table-striped table-hover"> *@
        @*                 <thead> *@
        @*                     <tr> *@
        @*                         <th>State key</th> *@
        @*                         <th>Json contents</th> *@
        @*                     </tr> *@
        @*                 </thead> *@
        @*                 <tbody> *@
        @*                     @foreach (var policy in invalidPolicies) { *@
        @*                         <tr> *@
        @*                             <td>@policy.StateKey</td> *@
        @*                             <td> *@
        @*                                 <pre>@policy.RawContent.ToJson(true, false)</pre> *@
        @*                             </td> *@
        @*                         </tr> *@
        @*                     } *@
        @*                 </tbody> *@
        @*             </table> *@
        @*         </details> *@
        @*     </details> *@
        // }

        // logger.LogInformation($"Rendered policies in {renderSw.GetElapsedAndRestart()}");
        logger.LogInformation("Rendered in {TimeSpan}", renderTotalSw.Elapsed);
    }
}

@code {

#if DEBUG
    private const bool Debug = true;
#else
    private const bool Debug = false;
#endif

    private bool IsInitialised { get; set; } = false;
    private bool Loading { get; set; } = true;

    [Parameter]
    public required string RoomId { get; set; }

    [Parameter, SupplyParameterFromQuery]
    public bool RenderEventInfo {
        get;
        set {
            field = value;
            StateHasChanged();
        }
    }

    private Dictionary<Type, List<StateEventResponse>> PolicyEventsByType { get; set; } = new();

    public StateEventResponse? ServerPolicyToMakePermanent {
        get;
        set {
            field = value;
            StateHasChanged();
        }
    }

    private AuthenticatedHomeserverGeneric Homeserver { get; set; } = null!;
    private GenericRoom Room { get; set; } = null!;
    private RoomPowerLevelEventContent PowerLevels { get; set; } = null!;
    public bool CurrentUserIsDraupnir { get; set; }

    public Dictionary<StateEventResponse, int> ActiveKicks { get; set; } = [];

    private static FrozenSet<Type> KnownPolicyTypes = StateEvent.KnownStateEventTypes.Where(x => x.IsAssignableTo(typeof(PolicyRuleEventContent))).ToFrozenSet();

    // event types, unnamed
    // private static Dictionary<string, Type> PolicyTypes = KnownPolicyTypes
    //     .ToDictionary(x => x.GetCustomAttributes<MatrixEventAttribute>().First(y => !string.IsNullOrWhiteSpace(y.EventName)).EventName, x => x);
    //
    // private static Dictionary<Type, string[]> PolicyTypeIds = KnownPolicyTypes
    //     .ToDictionary(x => x, x => x.GetCustomAttributes<MatrixEventAttribute>().Select(y => y.EventName).ToArray());

    Dictionary<Type, PolicyCollection> PolicyCollections { get; set; } = new();
    PolicyCollection? DuplicateBans { get; set; }
    PolicyCollection? RedundantBans { get; set; }

    protected override async Task OnInitializedAsync() {
        var sw = Stopwatch.StartNew();
        await base.OnInitializedAsync();
        Homeserver = (await sessionStore.GetCurrentHomeserver(navigateOnFailure: true))!;
        if (Homeserver is null) return;
        Room = Homeserver.GetRoom(RoomId!);
        IsInitialised = true;
        StateHasChanged();
        await Task.WhenAll(
            Task.Run(async () => { PowerLevels = (await Room.GetPowerLevelsAsync())!; }),
            Task.Run(async () => { CurrentUserIsDraupnir = (await Homeserver.GetAccountDataOrNullAsync<object>(DraupnirProtectedRoomsData.EventId)) is not null; })
        );
        StateHasChanged();
        await LoadStateAsync(firstLoad: true);
        Loading = false;
        logger.LogInformation("Policy list editor initialized in {SwElapsed}!", sw.Elapsed);
    }

    private async Task LoadStateAsync(bool firstLoad = false) {
        // preload workers in task pool
        // await Task.WhenAll(Enumerable.Range(0, WebWorkerService.MaxWorkerCount).Select(async _ => (await WebWorkerService.TaskPool.GetWorkerAsync()).WhenReady).ToList());
        var taskPoolReadyTask = WebWorkerService.TaskPool.SetWorkerCount(WebWorkerService.MaxWorkerCount);
        var sw = Stopwatch.StartNew();
        // Loading = true;
        // var states = Room.GetFullStateAsync();
        var states = await Room.GetFullStateAsListAsync();
        // PolicyEventsByType.Clear();
        logger.LogInformation("LoadStatesAsync: Loaded state in {SwElapsed}", sw.Elapsed);

        foreach (var type in KnownPolicyTypes) {
            if (!PolicyCollections.ContainsKey(type)) {
                var filterPropSw = Stopwatch.StartNew();
                // enumerate all properties with friendly name
                var props = type.GetProperties(BindingFlags.Public | BindingFlags.Instance)
                    .Where(x => (x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyNameOrNull()) is not null)
                    .Where(x => x.GetCustomAttribute<TableHideAttribute>() is null)
                    .ToFrozenSet();

                var proxySafeProps = type.GetProperties(BindingFlags.Public | BindingFlags.Instance)
                    .Where(x => props.Any(y => y.Name == x.Name))
                    .ToFrozenDictionary(x => x.GetFriendlyNameOrNull() ?? x.GetJsonPropertyName(), x => x);
                logger.LogInformation("{Count} proxy safe props found in {TypeFullName} ({TimeSpan})", proxySafeProps?.Count, type.FullName, filterPropSw.Elapsed);
                PolicyCollections.Add(type, new() {
                    Name = type.GetFriendlyNamePluralOrNull() ?? type.FullName ?? type.Name,
                    ActivePolicies = [],
                    RemovedPolicies = [],
                    PropertiesToDisplay = proxySafeProps
                });
            }
        }

        var count = 0;
        var parseSw = Stopwatch.StartNew();
        foreach (var evt in states) {
            var mappedType = evt.MappedType;
            if (count % 100 == 0)
                logger.LogInformation("Processing state #{Count:000000} {EvtType} @ {SwElapsed} (took {ParseSwElapsed:c} so far to process)", count, evt.Type, sw.Elapsed, parseSw.Elapsed);
            count++;

            if (!mappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue;

            var collection = PolicyCollections[mappedType];

            var key = (evt.Type, evt.StateKey!);
            var policyInfo = new PolicyCollection.PolicyInfo {
                Policy = evt,
                MadeRedundantBy = [],
                DuplicatedBy = []
            };
            if (evt.RawContent is null or { Count: 0 } || string.IsNullOrWhiteSpace(evt.RawContent?["recommendation"]?.GetValue<string>())) {
                collection.ActivePolicies.Remove(key);
                if (!collection.RemovedPolicies.TryAdd(key, policyInfo)) {
                    if (StateEvent.Equals(collection.RemovedPolicies[key].Policy, evt)) continue;
                    collection.RemovedPolicies[key] = policyInfo;
                }
            }
            else {
                collection.RemovedPolicies.Remove(key);
                if (!collection.ActivePolicies.TryAdd(key, policyInfo)) {
                    if (StateEvent.Equals(collection.ActivePolicies[key].Policy, evt)) continue;
                    collection.ActivePolicies[key] = policyInfo;
                }
            }
        }

        logger.LogInformation("LoadStatesAsync: Processed state in {SwElapsed}", sw.Elapsed);
        foreach (var collection in PolicyCollections) {
            logger.LogInformation("Policy collection {KeyFullName} has {ActivePoliciesCount} active and {RemovedPoliciesCount} removed policies.", collection.Key.FullName, collection.Value.ActivePolicies.Count, collection.Value.RemovedPolicies.Count);
        }

        await Task.Delay(1);

        Loading = false;
        StateHasChanged();
        await Task.Delay(100);

        // return;
        logger.LogInformation("LoadStatesAsync: Scanning for redundant policies...");

        var scanSw = Stopwatch.StartNew();
        // var allPolicyInfos = PolicyCollections.Values
        //     .SelectMany(x => x.ActivePolicies.Values)
        //     .ToArray();
        // var allPolicies = allPolicyInfos
        //     .Select<PolicyCollection.PolicyInfo, (PolicyCollection.PolicyInfo PolicyInfo, PolicyRuleEventContent TypedContent)>(x => (x, (x.Policy.TypedContent as PolicyRuleEventContent)!))
        //     .ToList();
        // var hashPolicies = allPolicies
        //     .Where(x => x.TypedContent.IsHashedRule())
        //     .ToList();
        // var wildcardPolicies = allPolicies
        //     .Except(hashPolicies) // hashed policies cannot be wildcards
        //     .Where(x => x.TypedContent.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent)
        //     .ToList();
        // var nonWildcardPolicies = allPolicies
        //     // .Except(wildcardPolicies)
        //     .Where(x => !x.TypedContent!.IsGlobRule() || x.TypedContent is ServerPolicyRuleEventContent)
        //     .ToList();
        // Console.WriteLine($"Got {allPolicies.Count} total policies, {wildcardPolicies.Count} wildcard policies. Time spent: {scanSw.Elapsed}");
        // int i = 0;
        // int hits = 0;
        // int redundant = 0;
        // int duplicates = 0;

        // foreach (var (policyInfo, policyContent) in allPolicies) {
        //     foreach (var (otherPolicyInfo, otherPolicyContent) in allPolicies) {
        //         if (policyInfo.Policy == otherPolicyInfo.Policy) continue; // same event
        //         if (StateEvent.TypeKeyPairMatches(policyInfo.Policy, otherPolicyInfo.Policy)) {
        //             logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson());
        //             continue; // same type and state key
        //         }
        //         // if(!policyContent.IsHashedRule())
        //     }
        //
        //     if (++i % 100 == 0) {
        //         Console.WriteLine($"Processed {i} policies in {scanSw.Elapsed}");
        //         await Task.Delay(1);
        //     }
        // }

        int scanningPolicyCount = 0;
        var aggregatedPolicies = PolicyCollections.Values
            .Aggregate(new List<StateEventResponse>(), (acc, val) => {
                acc.AddRange(val.ActivePolicies.Select(x => x.Value.Policy));
                return acc;
            });
        Console.WriteLine($"Scanning for redundant policies in {aggregatedPolicies.Count} total policies... ({scanSw.Elapsed})");
        List<Task<List<PolicyCollection.PolicyInfo>>> tasks = [];
        // try to save some load...
        var policiesJson = JsonSerializer.Serialize(aggregatedPolicies);
        var policiesJsonMarshalled = JsRuntime.ReturnMe<SpawnDev.BlazorJS.JSObjects.String>(policiesJson);
        var ranges = Enumerable.Range(0, aggregatedPolicies.Count).DistributeSequentially(WebWorkerService.MaxWorkerCount);
        await taskPoolReadyTask;
        tasks.AddRange(ranges.Select(range => WebWorkerService.TaskPool.Invoke(CheckDuplicatePoliciesAsync, policiesJsonMarshalled, range.First(), range.Last())));

        Console.WriteLine($"Main: started {tasks.Count} workers in {scanSw.Elapsed}");
        // tasks.Add(CheckDuplicatePoliciesAsync(allPolicyInfos, range.First() .. range.Last()));

        // var allPolicyEvents = aggregatedPolicies.Select(x => x.Policy).ToList();

        DuplicateBans = new() {
            Name = "Duplicate bans",
            ViewType = PolicyCollection.SpecialViewType.Duplicates,
            ActivePolicies = [],
            RemovedPolicies = [],
            PropertiesToDisplay = PolicyCollections.SelectMany(x => x.Value.PropertiesToDisplay).DistinctBy(x => x.Key).ToFrozenDictionary()
        };

        RedundantBans = new() {
            Name = "Redundant bans",
            ViewType = PolicyCollection.SpecialViewType.Redundant,
            ActivePolicies = [],
            RemovedPolicies = [],
            PropertiesToDisplay = PolicyCollections.SelectMany(x => x.Value.PropertiesToDisplay).DistinctBy(x => x.Key).ToFrozenDictionary()
        };

        var allPolicyInfos = PolicyCollections.Values
            .SelectMany(x => x.ActivePolicies.Values)
            .ToArray();

        await foreach (var modifiedPolicyInfos in tasks.ToAsyncEnumerable()) {
            if (modifiedPolicyInfos.Count == 0) continue;
            var applySw = Stopwatch.StartNew();
            // Console.WriteLine($"Main: got {modifiedPolicyInfos.Count} modified policies from worker, time: {scanSw.Elapsed}");
            foreach (var modifiedPolicyInfo in modifiedPolicyInfos) {
                var original = allPolicyInfos.First(p => p.Policy.EventId == modifiedPolicyInfo.Policy.EventId);
                original.DuplicatedBy = aggregatedPolicies.Where(x => modifiedPolicyInfo.DuplicatedBy.Any(y => StateEvent.Equals(x, y))).ToList();
                original.MadeRedundantBy = aggregatedPolicies.Where(x => modifiedPolicyInfo.MadeRedundantBy.Any(y => StateEvent.Equals(x, y))).ToList();
                modifiedPolicyInfo.DuplicatedBy = modifiedPolicyInfo.MadeRedundantBy = []; // Early dereference
                if (original.DuplicatedBy.Count > 0) {
                    if (!DuplicateBans.Value.ActivePolicies.ContainsKey((original.Policy.Type, original.Policy.StateKey!)))
                        DuplicateBans.Value.ActivePolicies.Add((original.Policy.Type, original.Policy.StateKey!), original);
                }

                if (original.MadeRedundantBy.Count > 0) {
                    if (!RedundantBans.Value.ActivePolicies.ContainsKey((original.Policy.Type, original.Policy.StateKey!)))
                        RedundantBans.Value.ActivePolicies.Add((original.Policy.Type, original.Policy.StateKey!), original);
                }
                // Console.WriteLine($"Memory usage: {Util.BytesToString(GC.GetTotalMemory(false))}");
            }

            Console.WriteLine($"Main: Processed {modifiedPolicyInfos.Count} modified policies in {scanSw.Elapsed} (applied in {applySw.Elapsed})");
        }

        Console.WriteLine($"Processed {allPolicyInfos.Length} policies in {scanSw.Elapsed}");

        // // scan for wildcard matches
        // foreach (var policy in allPolicies) {
        //     var matchingPolicies = wildcardPolicies
        //         .Where(x =>
        //             !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy)
        //             && x.Item2.EntityMatches(policy.TypedContent.Entity!)
        //         )
        //         .ToList();
        //
        //     if (matchingPolicies.Count > 0) {
        //         logger.LogInformation($"{i} Got {matchingPolicies.Count} hits for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}");
        //         foreach (var match in matchingPolicies) {
        //             policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy);
        //         }
        //
        //         hits++;
        //         redundant += matchingPolicies.Count;
        //
        //         if (hits % 5 == 0)
        //             StateHasChanged();
        //     }
        //     else {
        //         //logger.LogInformation("Sleeping...");
        //         await Task.Delay(1);
        //     }
        //
        //     i++;
        // }
        //
        // i = 0;
        // // scan for exact duplicates
        // foreach (var policy in allPolicies) {
        //     var matchingPolicies = allPolicies
        //         .Where(x =>
        //             !StateEvent.TypeKeyPairMatches(policy.PolicyInfo.Policy, x.PolicyInfo.Policy)
        //             && (
        //                 x.Item2.IsHashedRule()
        //                     ? x.Item2.EntityMatches(policy.Item2.Entity)
        //                     : x.Item2!.Entity == policy.Item2.Entity!
        //             )
        //         )
        //         .ToList();
        //
        //     if (matchingPolicies.Count > 0) {
        //         logger.LogInformation($"{i} Got {matchingPolicies.Count} duplicates for {policy.PolicyInfo.Policy.RawContent.ToJson()}: {matchingPolicies.Select(x => x.PolicyInfo.Policy.RawContent).ToJson()}");
        //         foreach (var match in matchingPolicies) {
        //             policy.PolicyInfo.MadeRedundantBy.Add(match.PolicyInfo.Policy);
        //         }
        //
        //         hits++;
        //         duplicates += matchingPolicies.Count;
        //
        //         if (hits % 5 == 0)
        //             StateHasChanged();
        //     }
        //     else {
        //         //logger.LogInformation("Sleeping...");
        //         await Task.Delay(1);
        //     }
        //
        //     i++;
        // }
        //
        // logger.LogInformation($"LoadStatesAsync: Found {hits} ({redundant} redundant, {duplicates} duplicates) redundant policies in {sw.Elapsed}");
        // StateHasChanged();
    }

    [return: WorkerTransfer]
    private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(SpawnDev.BlazorJS.JSObjects.String policiesJson, int start, int end) {
        var policies = JsonSerializer.Deserialize<List<StateEventResponse>>(policiesJson.ValueOf());
        Console.WriteLine($"Got request to check duplicate policies in range {start} to {end} (length: {end - start}), {policiesJson.ValueOf().Length} bytes of JSON ({policies!.Count} policies)");
        return await CheckDuplicatePoliciesAsync(policies!, start .. end);
    }

    [return: WorkerTransfer]
    private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(string policiesJson, int start, int end) {
        var policies = JsonSerializer.Deserialize<List<StateEventResponse>>(policiesJson);
        Console.WriteLine($"Got request to check duplicate policies in range {start} to {end} (length: {end - start}), {policiesJson.Length} bytes of JSON ({policies!.Count} policies)");
        return await CheckDuplicatePoliciesAsync(policies!, start .. end);
    }

    [return: WorkerTransfer]
    private static Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(List<StateEventResponse> policies, int start, int end)
        => CheckDuplicatePoliciesAsync(policies, start .. end);

    [return: WorkerTransfer]
    private static async Task<List<PolicyCollection.PolicyInfo>> CheckDuplicatePoliciesAsync(List<StateEventResponse> policies, Range range) {
        var sw = Stopwatch.StartNew();
        var jsConsole = App.Host.Services.GetService<JsConsoleService>()!;
        Console.WriteLine($"Processing policies in range {range} ({range.GetOffsetAndLength(policies.Count).Length}) with {policies.Count} total policies");
        var allPolicies = policies
            .Select(x => (Event: x, TypedContent: (x.TypedContent as PolicyRuleEventContent)!))
            .ToList();
        var toCheck = allPolicies[range];
        var modifiedPolicies = new List<PolicyCollection.PolicyInfo>();

        foreach (var (policyEvent, policyContent) in toCheck) {
            List<StateEventResponse> duplicatedBy = [];
            List<StateEventResponse> madeRedundantBy = [];

            foreach (var (otherPolicyEvent, otherPolicyContent) in allPolicies) {
                if (policyEvent == otherPolicyEvent) continue; // same event
                if (StateEvent.TypeKeyPairMatches(policyEvent, otherPolicyEvent)) {
                    // logger.LogWarning("Sanity check failed: Found same type and state key for two different policies: {Policy1} and {Policy2}", policyInfo.Policy.RawContent.ToJson(), otherPolicyInfo.Policy.RawContent.ToJson());
                    Console.WriteLine($"Sanity check failed: Found same type and state key for two different policies: {policyEvent.RawContent.ToJson()} and {otherPolicyEvent.RawContent.ToJson()}");
                    continue; // same type and state key
                }

                // if(!policyContent.IsHashedRule())
                if (!string.IsNullOrWhiteSpace(policyContent.Entity) && policyContent.Entity == otherPolicyContent.Entity) {
                    // Console.WriteLine($"Found duplicate policy: {policyEvent.EventId} is duplicated by {otherPolicyEvent.EventId}");
                    duplicatedBy.Add(otherPolicyEvent);
                }
            }

            if (duplicatedBy.Count > 0 || madeRedundantBy.Count > 0) {
                var summary = $"Policy {policyEvent.EventId} is:";
                if (duplicatedBy.Count > 0)
                    summary += $"\n- Duplicated by {duplicatedBy.Count} policies: {string.Join(", ", duplicatedBy.Select(x => x.EventId))}";
                if (madeRedundantBy.Count > 0)
                    summary += $"\n- Made redundant by {madeRedundantBy.Count} policies: {string.Join(", ", madeRedundantBy.Select(x => x.EventId))}";
                // Console.WriteLine(summary);
                await jsConsole.Info(summary);
                await Task.Delay(1);
                modifiedPolicies.Add(new() {
                    Policy = policyEvent,
                    DuplicatedBy = duplicatedBy,
                    MadeRedundantBy = madeRedundantBy
                });
            }

            // await Task.Delay(1);
        }

        await jsConsole.Info($"Worker: Found {modifiedPolicies.Count} modified policies in range {range} (length: {range.GetOffsetAndLength(policies.Count).Length}) in {sw.Elapsed}");

        return modifiedPolicies;
    }

    // the old one:
    private async Task LoadStatesAsync(bool firstLoad = false) {
        await LoadStateAsync(firstLoad);
        return;
        var sw = Stopwatch.StartNew();
        Loading = true;
        // var states = Room.GetFullStateAsync();
        var states = await Room.GetFullStateAsListAsync();
        // PolicyEventsByType.Clear();

        logger.LogInformation("LoadStatesAsync: Loaded state in {SwElapsed}", sw.Elapsed);

        foreach (var type in KnownPolicyTypes) {
            if (!PolicyEventsByType.ContainsKey(type))
                PolicyEventsByType.Add(type, new List
                    <StateEventResponse>(16000));
        }

        int count = 0;

        foreach (var state in states) {
            var _spsw = Stopwatch.StartNew();
            TimeSpan e1, e2, e3, e4, e5, e6, t;
            if (!state.MappedType.IsAssignableTo(typeof(PolicyRuleEventContent))) continue;
            e1 = _spsw.Elapsed;
            var targetPolicies = PolicyEventsByType[state.MappedType];
            e2 = _spsw.Elapsed;
            if (!firstLoad && targetPolicies.FirstOrDefault(x => StateEvent.TypeKeyPairMatches(x, state)) is { } evt) {
                e3 = _spsw.Elapsed;
                if (StateEvent.Equals(evt, state)) {
                    if (count % 100 == 0) {
                        await Task.Delay(10);
                        await Task.Yield();
                    }

                    e4 = _spsw.Elapsed;
                    logger.LogInformation("[E] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={E3:c}, e4={E4:c}, e5={Zero:c},t={SpswElapsed:c})", count++, state.Type, sw.Elapsed, e1, e2, e3, e4, TimeSpan.Zero, _spsw.Elapsed);
                    continue;
                }

                e4 = _spsw.Elapsed;
                targetPolicies.Remove(evt);
                e5 = _spsw.Elapsed;
                targetPolicies.Add(state);
                e6 = _spsw.Elapsed;
                t = _spsw.Elapsed;
                logger.LogInformation("[M] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={E3:c}, e4={E4:c}, e5={E5:c}, e6={E6:c},t={TimeSpan1:c})", count++, state.Type, sw.Elapsed, e1, e2, e3, e4, e5, e6, t);
            }
            else {
                targetPolicies.Add(state);
                t = _spsw.Elapsed;
                logger.LogInformation("[N] LoadStatesAsync: Processed state #{I:000000} {StateType} @ {SwElapsed} (e1={TimeSpan:c}, e2={E2:c}, e3={Zero:c}, e4={TimeSpan1:c}, e5={Zero1:c}, e6={TimeSpan2:c}, t={TimeSpan3:c})", count++, state.Type, sw.Elapsed, e1, e2, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, t);
            }

            // await Task.Delay(10);
            // await Task.Yield();
        }

        logger.LogInformation("LoadStatesAsync: Processed state in {SwElapsed}", sw.Elapsed);

        Loading = false;
        StateHasChanged();
        await Task.Delay(10);
        await Task.Yield();
        logger.LogInformation("LoadStatesAsync: yield finished in {SwElapsed}", sw.Elapsed);
    }

    private List<StateEventResponse> GetPolicyEventsByType(Type type) => PolicyEventsByType.ContainsKey(type) ? PolicyEventsByType[type] : [];

    // private List<StateEventResponse> GetValidPolicyEventsByType(Type type) => GetPolicyEventsByType(type)
    //     .Where(x => !string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList();
    //
    // private List<StateEventResponse> GetInvalidPolicyEventsByType(Type type) => GetPolicyEventsByType(type)
    //     .Where(x => x.RawContent is { Count: > 0 } && string.IsNullOrWhiteSpace(x.RawContent?["recommendation"]?.GetValue<string>())).ToList();
    //
    // private List<StateEventResponse> GetRemovedPolicyEventsByType(Type type) => GetPolicyEventsByType(type)
    //     .Where(x => x.RawContent is null or { Count: 0 }).ToList();

    private string? GetPolicyTypeNameOrNull(Type type) => type.GetFriendlyNamePluralOrNull()
                                                          ?? type.GetCustomAttributes<MatrixEventAttribute>()
                                                              .FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.EventName))?.EventName;

    private string GetPolicyTypeName(Type type) => GetPolicyTypeNameOrNull(type) ?? type.Name;

    public struct PolicyCollection {
        public required string Name { get; init; }
        public SpecialViewType ViewType { get; init; }
        public int TotalCount => ActivePolicies.Count + RemovedPolicies.Count;

        public required Dictionary<(string Type, string StateKey), PolicyInfo> ActivePolicies { get; set; }

        // public Dictionary<(string Type, string StateKey), StateEventResponse> InvalidPolicies { get; set; }
        public required Dictionary<(string Type, string StateKey), PolicyInfo> RemovedPolicies { get; set; }
        public required FrozenDictionary<string, PropertyInfo> PropertiesToDisplay { get; set; }

        public class PolicyInfo {
            public required StateEventResponse Policy { get; init; }
            public required List<StateEventResponse> MadeRedundantBy { get; set; }
            public required List<StateEventResponse> DuplicatedBy { get; set; }
        }

        public enum SpecialViewType {
            None,
            Duplicates,
            Redundant,
        }
    }

    // private struct PolicyStats {
    // public int Active { get; set; }
    // public int Invalid { get; set; }
    // public int Removed { get; set; }
    // }

}