From e3e4b10cc85decc832117717af8a4f4d9f6f728d Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Tue, 23 May 2023 11:25:30 +0200 Subject: Prevent deploy without commit --- .../Pages/KnownHomeserverList.razor | 118 +++++++++++++++++++-- deploy.sh | 8 ++ 2 files changed, 116 insertions(+), 10 deletions(-) diff --git a/MatrixRoomUtils.Web/Pages/KnownHomeserverList.razor b/MatrixRoomUtils.Web/Pages/KnownHomeserverList.razor index f396025..882dd1e 100644 --- a/MatrixRoomUtils.Web/Pages/KnownHomeserverList.razor +++ b/MatrixRoomUtils.Web/Pages/KnownHomeserverList.razor @@ -1,12 +1,32 @@ @page "/KnownHomeserverList" @using System.Text.Json @using MatrixRoomUtils.Core.Extensions +@using System.Diagnostics

Known Homeserver List

@if (!IsFinished) {

Loading... Please wait...

+ +

@QueryProgress.ProcessedRooms / @QueryProgress.TotalRooms

+ @foreach (var (room, state) in QueryProgress.ProcessedUsers.Where(x => !x.Value.IsFinished).OrderByDescending(x => x.Value.Total).ToList()) + { + @if (state.Blocked) + { +

🔒 @room.RoomId - @state.Processed / @state.Total, @state.Timing.Elapsed elapsed...

+ } + else if (state.Slowmode) + { + +

🐢 @room.RoomId - @state.Processed / @state.Total, @state.Timing.Elapsed elapsed...

+ } + else + { +

@room.RoomId - @state.Processed / @state.Total, @state.Timing.Elapsed elapsed...

+ } + + } } else { @@ -20,12 +40,28 @@ else @code { List HomeServers = new(); bool IsFinished { get; set; } + HomeServerInfoQueryProgress QueryProgress { get; set; } = new(); protected override async Task OnInitializedAsync() { await LocalStorageWrapper.LoadFromLocalStorage(LocalStorage); - HomeServers = await GetHomeservers(); + var sw = Stopwatch.StartNew(); + HomeServers = await GetHomeservers(progressCallback: async progress => + { + if (sw.ElapsedMilliseconds > 1000) + { + Console.WriteLine("Progress updated..."); + QueryProgress = progress; + StateHasChanged(); + Console.WriteLine("Progress rendered!"); + sw.Restart(); + await Task.Delay(100); + return true; + } + Console.WriteLine($"Progress updated, but not rendering because only {sw.ElapsedMilliseconds}ms elapsed since last call..."); + return false; + }); IsFinished = true; StateHasChanged(); @@ -34,37 +70,80 @@ else } - private async Task> GetHomeservers() + private async Task> GetHomeservers(int memberLimit = 1000, Func>? progressCallback = null) { + HomeServerInfoQueryProgress progress = new(); List homeServers = new(); var rooms = await RuntimeCache.CurrentHomeServer.GetJoinedRooms(); - // Dictionary roomMembers = new(); - //start a task for each room + progress.TotalRooms = rooms.Count; + + var semaphore = new SemaphoreSlim(4); + var semLock = new SemaphoreSlim(1); var tasks = rooms.Select(async room => { + await semaphore.WaitAsync(); + progress.ProcessedUsers.Add(room, new()); Console.WriteLine($"Fetching states for room ({rooms.IndexOf(room)}/{rooms.Count}) ({room.RoomId})"); - StateHasChanged(); - var states = (await room.GetStateAsync("")).Value.Deserialize>(); - states.RemoveAll(x => x.type != "m.room.member"); + states.RemoveAll(x => x.type != "m.room.member" || x.content.GetProperty("membership").GetString() != "join"); Console.WriteLine($"Room {room.RoomId} has {states.Count} members"); + if (states.Count > memberLimit) + { + Console.WriteLine("Skipping!"); + semaphore.Release(); + progress.ProcessedUsers.Remove(room); + progress.TotalRooms--; + return; + } + progress.ProcessedUsers[room].Total = states.Count; + var updateInterval = progress.ProcessedUsers[room].Total >= 1000 ? 1000 : 100; + while (progress.ProcessedUsers.Any(x => x.Value.Total == 0) && progress.ProcessedUsers[room].Total >= 1000) + { + progress.ProcessedUsers[room].Blocked = true; + await Task.Delay(1000); + // if(progressCallback != null) + // await progressCallback.Invoke(progress); + } + progress.ProcessedUsers[room].Blocked = false; + int processedStates = 0; foreach (var state in states) { + await semLock.WaitAsync(); + semLock.Release(); + if (progress.ProcessedUsers.Count(x => x.Value.Total == 0) > 5 && progress.ProcessedUsers[room].Total >= 200) + { + progress.ProcessedUsers[room].Slowmode = true; + await Task.Delay(progress.ProcessedUsers[room].Total >= 500 ? 1000 : 100); + } + else + { + progress.ProcessedUsers[room].Slowmode = false; + } if (!homeServers.Any(x => x.Server == state.state_key.Split(':')[1])) { homeServers.Add(new HomeServerInfo() { Server = state.state_key.Split(':')[1] }); } var hs = homeServers.First(x => x.Server == state.state_key.Split(':')[1]); - if(!hs.KnownUsers.Contains(state.state_key.Split(':')[0])) + if (!hs.KnownUsers.Contains(state.state_key.Split(':')[0])) hs.KnownUsers.Add(state.state_key.Split(':')[0]); + if (++progress.ProcessedUsers[room].Processed % updateInterval == 0 && progressCallback != null) + { + await semLock.WaitAsync(); + var _ = await progressCallback.Invoke(progress); + semLock.Release(); + } } Console.WriteLine("Collected states!"); + progress.ProcessedRooms++; + progress.ProcessedUsers[room].IsFinished = true; + progressCallback?.Invoke(progress); + semaphore.Release(); }); await Task.WhenAll(tasks); - + Console.WriteLine("Calculating member counts..."); homeServers.ForEach(x => x.KnownUserCount = x.KnownUsers.Count); - Console.WriteLine(homeServers.First(x=>x.Server=="rory.gay").ToJson()); + Console.WriteLine(homeServers.First(x => x.Server == "rory.gay").ToJson()); Console.WriteLine("Recalculated!"); return homeServers; } @@ -76,4 +155,23 @@ else public List KnownUsers { get; set; } = new(); } + class HomeServerInfoQueryProgress + { + public int ProcessedRooms { get; set; } + public int TotalRooms { get; set; } + public Dictionary ProcessedUsers { get; set; } = new(); + public List CurrentState { get; set; } = new(); + + public class State + { + public int Processed { get; set; } + public int Total { get; set; } + public bool Blocked { get; set; } + public bool Slowmode { get; set; } + public float Progress => (float)Processed / Total; + public bool IsFinished { get; set; } + public Stopwatch Timing { get; set; } = Stopwatch.StartNew(); + } + } + } \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 9762db5..66182a4 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,4 +1,12 @@ #!/bin/sh +if [[ -z $(git status -s) ]] +then + echo "tree is clean" +else + echo "tree is dirty, please commit changes before running this" + exit +fi + BASE_DIR=`pwd` rm -rf **/bin/Release cd MatrixRoomUtils.Web -- cgit 1.4.1