From c04719d87abbe9feb94d9b3e8cf1812d3f7356e0 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 27 Dec 2023 19:45:51 +0100 Subject: Room query --- LibMatrix | 2 +- ModAS.Classes/ModAS.Classes.csproj | 9 ++ ModAS.Classes/RoomQueryFilter.cs | 26 ++++ .../AppService/TransactionsController.cs | 80 ++++++++++++ ModAS.Server/Controllers/DebugController.cs | 94 +++++++++----- ModAS.Server/Controllers/HomeController.cs | 8 +- ModAS.Server/Controllers/RoomQueryController.cs | 135 +++++++++++++++++++++ ModAS.Server/Controllers/TransactionsController.cs | 15 --- ModAS.Server/ModAS.Server.csproj | 8 ++ ModAS.Server/Program.cs | 21 +++- .../AuthenticatedHomeserverProviderService.cs | 51 -------- ModAS.Server/Services/ModASConfiguration.cs | 2 +- ModAS.Server/Services/RoomContextService.cs | 111 +++++++++++++++++ ModAS.Server/Services/RoomStateCacheService.cs | 66 ++++++++++ ModAS.Server/Services/UserProviderService.cs | 95 +++++++++++++++ ModAS.sln | 13 ++ 16 files changed, 631 insertions(+), 105 deletions(-) create mode 100644 ModAS.Classes/ModAS.Classes.csproj create mode 100644 ModAS.Classes/RoomQueryFilter.cs create mode 100644 ModAS.Server/Controllers/AppService/TransactionsController.cs create mode 100644 ModAS.Server/Controllers/RoomQueryController.cs delete mode 100644 ModAS.Server/Controllers/TransactionsController.cs delete mode 100644 ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs create mode 100644 ModAS.Server/Services/RoomContextService.cs create mode 100644 ModAS.Server/Services/RoomStateCacheService.cs create mode 100644 ModAS.Server/Services/UserProviderService.cs diff --git a/LibMatrix b/LibMatrix index 314f704..94b83d4 160000 --- a/LibMatrix +++ b/LibMatrix @@ -1 +1 @@ -Subproject commit 314f7044f62b92c49abe2d5c7422c6cf3430b021 +Subproject commit 94b83d4de5e435796da9cc14667c1023a09df8eb diff --git a/ModAS.Classes/ModAS.Classes.csproj b/ModAS.Classes/ModAS.Classes.csproj new file mode 100644 index 0000000..3a63532 --- /dev/null +++ b/ModAS.Classes/ModAS.Classes.csproj @@ -0,0 +1,9 @@ + + + + net8.0 + enable + enable + + + diff --git a/ModAS.Classes/RoomQueryFilter.cs b/ModAS.Classes/RoomQueryFilter.cs new file mode 100644 index 0000000..0a6430e --- /dev/null +++ b/ModAS.Classes/RoomQueryFilter.cs @@ -0,0 +1,26 @@ +namespace ModAS.Classes; + +public class RoomQueryFilter { + public string? RoomIdContains { get; set; } + public string? NameContains { get; set; } + public string? CanonicalAliasContains { get; set; } + public string? VersionContains { get; set; } + public string? CreatorContains { get; set; } + public string? EncryptionContains { get; set; } + public string? JoinRulesContains { get; set; } + public string? GuestAccessContains { get; set; } + public string? HistoryVisibilityContains { get; set; } + public string? AvatarUrlContains { get; set; } + public string? RoomTopicContains { get; set; } + + public bool? IsFederatable { get; set; } = true; + public bool? IsPublic { get; set; } = true; + + public uint? JoinedMembersMin { get; set; } + public uint? JoinedMembersMax { get; set; } + public uint? JoinedLocalMembersMin { get; set; } + public uint? JoinedLocalMembersMax { get; set; } + public uint? StateEventsMin { get; set; } + public uint? StateEventsMax { get; set; } + +} \ No newline at end of file diff --git a/ModAS.Server/Controllers/AppService/TransactionsController.cs b/ModAS.Server/Controllers/AppService/TransactionsController.cs new file mode 100644 index 0000000..b74e1e1 --- /dev/null +++ b/ModAS.Server/Controllers/AppService/TransactionsController.cs @@ -0,0 +1,80 @@ +using System.IO.Pipelines; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using ArcaneLibs; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration.Json; +using ModAS.Server; +using ModAS.Server.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Controllers.AppService; + +[ApiController] +public class TransactionsController( + AppServiceRegistration asr, + ModASConfiguration config, + UserProviderService userProvider, + RoomContextService roomContextService, + RoomStateCacheService stateCacheService) : ControllerBase { + private static List _ignoredInvalidationEvents { get; set; } = [ + RoomMessageEventContent.EventId, + RoomMessageReactionEventContent.EventId + ]; + + [HttpPut("/_matrix/app/v1/transactions/{txnId}")] + public async Task PutTransactions(string txnId) { + if (!Request.Headers.ContainsKey("Authorization")) { + Console.WriteLine("PutTransaction: missing authorization header"); + return Unauthorized(); + } + + if (Request.GetTypedHeaders().Get("Authorization")?.Parameter != asr.HomeserverToken) { + Console.WriteLine($"PutTransaction: invalid authorization header: {Request.Headers["Authorization"]}"); + return Unauthorized(); + } + + var data = await JsonSerializer.DeserializeAsync(Request.Body); + Console.WriteLine( + $"PutTransaction: {txnId}: {data.Events.Count} events, {Util.BytesToString(Request.Headers.ContentLength ?? Request.ContentLength ?? Request.Body.Length)}"); + + if (!Directory.Exists("data")) + Directory.CreateDirectory("data"); + Directory.CreateDirectory($"data/{txnId}"); + // var pipe = PipeReader.Create(Request.Body); + // await using var file = System.IO.File.OpenWrite($"data/{txnId}"); + // await pipe.CopyToAsync(file); + // await pipe.CompleteAsync(); + // + // Console.WriteLine($"PutTransaction: {txnId}: {Util.BytesToString(file.Length)}"); + for (var i = 0; i < data.Events.Count; i++) { + var evt = data.Events[i]; + Console.WriteLine($"PutTransaction: {txnId}/{i}: {evt.Type} {evt.StateKey} {evt.Sender}"); + await System.IO.File.WriteAllTextAsync($"data/{txnId}/{i}-{evt.Type}-{evt.StateKey}-{evt.Sender}.json", JsonSerializer.Serialize(evt)); + + if (evt.Sender.EndsWith(':' + config.ServerName)) { + Console.WriteLine("PutTransaction: sender is local user, updating data..."); + try { + var user = await userProvider.GetImpersonatedHomeserver(evt.Sender); + var rooms = await user.GetJoinedRooms(); + foreach (var room in rooms) { + await roomContextService.GetRoomContext(room); + } + } + catch (Exception e) { + Console.WriteLine($"PutTransaction: failed to update data: {e}"); + } + } + else + Console.WriteLine("PutTransaction: sender is remote user"); + + if (!string.IsNullOrWhiteSpace(evt.RoomId) && !_ignoredInvalidationEvents.Contains(evt.Type)) + await stateCacheService.InvalidateRoomState(evt.RoomId); + } + + return Ok(new { }); + } +} \ No newline at end of file diff --git a/ModAS.Server/Controllers/DebugController.cs b/ModAS.Server/Controllers/DebugController.cs index d3c7ad0..f0fe91e 100644 --- a/ModAS.Server/Controllers/DebugController.cs +++ b/ModAS.Server/Controllers/DebugController.cs @@ -1,14 +1,17 @@ +using System.Collections.Frozen; using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; using LibMatrix; using LibMatrix.Homeservers; using Microsoft.AspNetCore.Mvc; using ModAS.Server.Services; using MxApiExtensions.Services; -namespace WebApplication1.Controllers; +namespace ModAS.Server.Controllers; [ApiController] -public class DebugController(ModASConfiguration config, AuthenticatedHomeserverProviderService authHsProvider) : ControllerBase { +public class DebugController(ModASConfiguration config, UserProviderService authHsProvider, RoomContextService roomContextService) : ControllerBase { [HttpGet("/_matrix/_modas/debug")] public IActionResult Index() { return Ok(new { @@ -29,40 +32,73 @@ public class DebugController(ModASConfiguration config, AuthenticatedHomeserverP [HttpGet("/_matrix/_modas/debug/test_locate_users")] public async IAsyncEnumerable TestLocateUsers([FromQuery] string startUser) { - List foundUsers = [startUser], processedRooms = new List(); + List foundUsers = (await authHsProvider.GetValidUsers()).Select(x=>x.Value).ToList(); + if(!foundUsers.Any(x=>x.WhoAmI.UserId == startUser)) { + foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(startUser)); + } + + List processedRooms = [], processedUsers = []; var foundNew = true; while (foundNew) { + var span1 = currentTransaction.StartSpan("iterateUsers", ApiConstants.TypeApp); foundNew = false; - foreach (var user in foundUsers.ToList()) { - AuthenticatedHomeserverGeneric? ahs = null; - try { - ahs = await authHsProvider.GetImpersonatedHomeserver(user); - await ahs.GetJoinedRooms(); - } - catch (MatrixException e) { - if (e is { ErrorCode: "M_FORBIDDEN" }) continue; - throw; - } + var usersToProcess = foundUsers.Where(x => !processedUsers.Any(y=>x.WhoAmI.UserId == y)).ToFrozenSet(); + Console.WriteLine($"Got {usersToProcess.Count} users: {string.Join(", ", usersToProcess)}"); - if(ahs is null) continue; - var rooms = await ahs.GetJoinedRooms(); - Console.WriteLine($"Got {rooms.Count} rooms"); - rooms.RemoveAll(r => processedRooms.Contains(r.RoomId)); - processedRooms.AddRange(rooms.Select(r => r.RoomId)); - foundNew = rooms.Count > 0; - Console.WriteLine($"Found {rooms.Count} new rooms"); - - var roomMemberTasks = rooms.Select(r => r.GetMembersListAsync(false)).ToAsyncEnumerable(); - await foreach (var roomMembers in roomMemberTasks) { - Console.WriteLine($"Got {roomMembers.Count} members"); - foreach (var member in roomMembers) { - if (!member.StateKey.EndsWith(':' + config.ServerName)) continue; - if (foundUsers.Contains(member.StateKey)) continue; - foundUsers.Add(member.StateKey); - yield return member.StateKey; + var rooms = usersToProcess.Select(async x => await x.GetJoinedRooms()); + var roomLists = rooms.ToAsyncEnumerable(); + await foreach (var roomList in roomLists) { + if (roomList is null) continue; + foreach (var room in roomList) { + if (processedRooms.Contains(room.RoomId)) continue; + processedRooms.Add(room.RoomId); + var roomMembers = await room.GetMembersListAsync(false); + foreach (var roomMember in roomMembers) { + if (roomMember.StateKey.EndsWith(':' + config.ServerName) && !foundUsers.Any(x=>x.WhoAmI.UserId == roomMember.StateKey)) { + foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(roomMember.StateKey)); + foundNew = true; + yield return roomMember.StateKey; + } } } } + + // await foreach (var task in tasks) { + // if (task is null) continue; + // foreach (var user in task) { + // if (foundUsers.Contains(user)) continue; + // foundUsers.Add(user); + // foundNew = true; + // yield return user; + // } + // } + + span1.End(); } } + + [HttpGet("/_matrix/_modas/debug/room_contexts")] + public IActionResult RoomContexts() { + return Ok(roomContextService.RoomContexts.Values); + } + + [HttpGet("/_matrix/_modas/debug/room_contexts/{roomId}")] + public async Task RoomContext(string roomId) { + var roomContext = await roomContextService.GetRoomContext(roomId); + if (roomContext is null) return NotFound("Room not found"); + return Ok(roomContext); + } + + [HttpGet("/_matrix/_modas/debug/room_contexts/by_user/{userId}")] + public async IAsyncEnumerable RoomContextByUser(string userId) { + var user = await authHsProvider.GetImpersonatedHomeserver(userId); + var rooms = await user.GetJoinedRooms(); + var contexts = rooms.Select(x => roomContextService.GetRoomContext(x.RoomId)).ToAsyncEnumerable(); + await foreach (var context in contexts) { + if (context is null) continue; + yield return context; + } + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; } \ No newline at end of file diff --git a/ModAS.Server/Controllers/HomeController.cs b/ModAS.Server/Controllers/HomeController.cs index 290441e..5fd309f 100644 --- a/ModAS.Server/Controllers/HomeController.cs +++ b/ModAS.Server/Controllers/HomeController.cs @@ -1,15 +1,13 @@ using System.Diagnostics; using Microsoft.AspNetCore.Mvc; -namespace WebApplication1.Controllers; +namespace ModAS.Server.Controllers; [ApiController] -public class HomeController : Controller -{ +public class HomeController : Controller { private readonly ILogger _logger; - public HomeController(ILogger logger) - { + public HomeController(ILogger logger) { _logger = logger; } diff --git a/ModAS.Server/Controllers/RoomQueryController.cs b/ModAS.Server/Controllers/RoomQueryController.cs new file mode 100644 index 0000000..a49e5c0 --- /dev/null +++ b/ModAS.Server/Controllers/RoomQueryController.cs @@ -0,0 +1,135 @@ +using System.Collections.Frozen; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.EventTypes.Spec.State.RoomInfo; +using LibMatrix.Responses.ModAS; +using LibMatrix.RoomTypes; +using Microsoft.AspNetCore.Mvc; +using ModAS.Classes; +using ModAS.Server.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Controllers; + +[ApiController] +public class RoomQueryController(UserProviderService ahsProvider, ModASConfiguration config, RoomStateCacheService stateCacheService, UserProviderService userProviderService) : ControllerBase { + [HttpGet("/_matrix/_modas/room_query")] + public async IAsyncEnumerable RoomQuery() { + var fetchRoomQueryDataSpan = currentTransaction.StartSpan("fetchRoomQueryData", ApiConstants.TypeApp); + List processedRooms = new(); + var getUsersSpan = currentTransaction.StartSpan("getUsers", ApiConstants.TypeApp); + var validUsers = await ahsProvider.GetValidUsers(); + getUsersSpan.End(); + + var collectRoomsSpan = currentTransaction.StartSpan("collectRooms", ApiConstants.TypeApp); + // var userRoomLists = validUsers.Values.Select(ahs => ahs.GetJoinedRooms()).ToList(); + // await Task.WhenAll(userRoomLists); + // var rooms = userRoomLists.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet(); + var roomsTasks = validUsers.Values.Select(u => userProviderService.GetUserRoomsCached(u.WhoAmI.UserId)).ToList(); + await Task.WhenAll(roomsTasks); + var rooms = roomsTasks.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet(); + collectRoomsSpan.End(); + + var collectRoomStateSpan = currentTransaction.StartSpan("collectRoomState", ApiConstants.TypeApp); + //make sure we lock!!!! + await stateCacheService.EnsureCachedFromRoomList(rooms); + var roomStateTasks = rooms.Select(GetRoomState).ToAsyncEnumerable(); + var awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp); + await foreach (var (room, roomState) in roomStateTasks) { + awaitRoomStateSpan.Name = $"awaitRoomState {room.RoomId}"; + awaitRoomStateSpan.End(); + + var filterStateEventsSpan = currentTransaction.StartSpan($"filterStateEvents {room.RoomId}", ApiConstants.TypeApp); + var roomMembers = roomState.Where(r => r.Type == RoomMemberEventContent.EventId).ToFrozenSet(); + var localRoomMembers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName)).ToFrozenSet(); + var nonMemberState = roomState.Where(x => !roomMembers.Contains(x)).ToFrozenSet(); + filterStateEventsSpan.End(); + var buildResultSpan = currentTransaction.StartSpan($"buildResult {room.RoomId}", ApiConstants.TypeApp); + + // forgive me for raw json access... attempt at optimisation -emma + yield return new ModASRoomQueryResult { + RoomId = room.RoomId, + StateEvents = roomState.Count, + //members + TotalMembers = roomMembers.Count, + TotalLocalMembers = localRoomMembers.Count, + JoinedMembers = roomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"), + JoinedLocalMembers = localRoomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"), + //-members + //creation event + Creator = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.Sender, + Version = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["room_version"]?.GetValue(), + Type = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["type"]?.GetValue(), + Federatable = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["m.federate"]?.GetValue() ?? true, + //-creation event + Name = nonMemberState.FirstOrDefault(r => r.Type == RoomNameEventContent.EventId)?.RawContent?["name"]?.GetValue(), + CanonicalAlias = nonMemberState.FirstOrDefault(r => r.Type == RoomCanonicalAliasEventContent.EventId)?.RawContent?["alias"]?.GetValue(), + JoinRules = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue(), + GuestAccess = nonMemberState.FirstOrDefault(r => r.Type == RoomGuestAccessEventContent.EventId)?.RawContent?["guest_access"]?.GetValue(), + HistoryVisibility = + nonMemberState.FirstOrDefault(r => r.Type == RoomHistoryVisibilityEventContent.EventId)?.RawContent?["history_visibility"]?.GetValue(), + Public = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue() == "public", + Encryption = nonMemberState.FirstOrDefault(r => r.Type == RoomEncryptionEventContent.EventId)?.RawContent?["algorithm"]?.GetValue(), + AvatarUrl = nonMemberState.FirstOrDefault(r => r.Type == RoomAvatarEventContent.EventId)?.RawContent?["url"]?.GetValue(), + RoomTopic = nonMemberState.FirstOrDefault(r => r.Type == RoomTopicEventContent.EventId)?.RawContent?["topic"]?.GetValue() + }; + buildResultSpan.End(); + awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp); + } + + collectRoomStateSpan.End(); + fetchRoomQueryDataSpan.End(); + } + + [HttpPost("/_matrix/_modas/room_query")] + public async IAsyncEnumerable RoomQuery([FromBody] RoomQueryFilter request) { + await foreach (var room in RoomQuery()) { + if (!string.IsNullOrWhiteSpace(request.RoomIdContains) && !room.RoomId.Contains(request.RoomIdContains)) continue; + + if (!string.IsNullOrWhiteSpace(request.NameContains) && room.Name?.Contains(request.NameContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.CanonicalAliasContains) && room.CanonicalAlias?.Contains(request.CanonicalAliasContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.VersionContains) && room.Version?.Contains(request.VersionContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.CreatorContains) && room.Creator?.Contains(request.CreatorContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.EncryptionContains) && room.Encryption?.Contains(request.EncryptionContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.JoinRulesContains) && room.JoinRules?.Contains(request.JoinRulesContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.GuestAccessContains) && room.GuestAccess?.Contains(request.GuestAccessContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.HistoryVisibilityContains) && room.HistoryVisibility?.Contains(request.HistoryVisibilityContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.AvatarUrlContains) && room.AvatarUrl?.Contains(request.AvatarUrlContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.RoomTopicContains) && room.RoomTopic?.Contains(request.RoomTopicContains) != true) continue; + + if (request.JoinedMembersMin.HasValue && room.JoinedMembers < request.JoinedMembersMin.Value) continue; + if (request.JoinedMembersMax.HasValue && room.JoinedMembers > request.JoinedMembersMax.Value) continue; + + if (request.JoinedLocalMembersMin.HasValue && room.JoinedLocalMembers < request.JoinedLocalMembersMin.Value) continue; + if (request.JoinedLocalMembersMax.HasValue && room.JoinedLocalMembers > request.JoinedLocalMembersMax.Value) continue; + + if (request.StateEventsMin.HasValue && room.StateEvents < request.StateEventsMin.Value) continue; + if (request.StateEventsMax.HasValue && room.StateEvents > request.StateEventsMax.Value) continue; + + if (request.IsFederatable.HasValue && room.Federatable != request.IsFederatable.Value) continue; + + if (request.IsPublic.HasValue && room.Public != request.IsPublic.Value) continue; + + yield return room; + } + } + + private async Task>> GetRoomState(GenericRoom room) { + var roomState = await stateCacheService.GetRoomState(room.RoomId, room); + return new(room, roomState); + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Controllers/TransactionsController.cs b/ModAS.Server/Controllers/TransactionsController.cs deleted file mode 100644 index 8e4e018..0000000 --- a/ModAS.Server/Controllers/TransactionsController.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.IO.Pipelines; -using Microsoft.AspNetCore.Mvc; -using ModAS.Server; - -namespace WebApplication1.Controllers; - -[ApiController] -public class TransactionsController(AppServiceRegistration asr) : ControllerBase { - [HttpPut(" /_matrix/app/v1/transactions/{txnId}")] - public async Task PutTransactions(string txnId) { - if(!Request.Headers.ContainsKey("Authorization") || Request.Headers["Authorization"] != asr.HomeserverToken) return Unauthorized(); - await Request.Body.CopyToAsync(Console.OpenStandardOutput()); - return Ok(new{}); - } -} \ No newline at end of file diff --git a/ModAS.Server/ModAS.Server.csproj b/ModAS.Server/ModAS.Server.csproj index 8b48b8a..07cc67c 100644 --- a/ModAS.Server/ModAS.Server.csproj +++ b/ModAS.Server/ModAS.Server.csproj @@ -7,9 +7,16 @@ true true preview + false + true + true + Speed + true + + @@ -17,6 +24,7 @@ + diff --git a/ModAS.Server/Program.cs b/ModAS.Server/Program.cs index da44ca8..0b3d121 100644 --- a/ModAS.Server/Program.cs +++ b/ModAS.Server/Program.cs @@ -3,12 +3,17 @@ using Microsoft.OpenApi.Models; using ModAS.Server; using System.Diagnostics; using System.Text.Json; +using Elastic.Apm; +using Elastic.Apm.Api; +using Elastic.Apm.AspNetCore; +using Elastic.Apm.NetCoreAll; using LibMatrix; using LibMatrix.Services; using ModAS.Server.Services; using MxApiExtensions.Services; -var builder = WebApplication.CreateBuilder(args); +// var builder = WebApplication.CreateBuilder(args); +var builder = WebApplication.CreateSlimBuilder(args); builder.Services.AddControllers().AddJsonOptions(options => { options.JsonSerializerOptions.WriteIndented = true; }); ///add wwwroot @@ -30,7 +35,9 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); // builder.Services.AddScoped(); builder.Services.AddSingleton(x => { @@ -82,7 +89,6 @@ builder.Services.AddCors(options => { var app = builder.Build(); -// Configure the HTTP request pipeline. // if (app.Environment.IsDevelopment()) { app.UseSwagger(); app.UseSwaggerUI(c => { @@ -99,6 +105,15 @@ app.UseReDoc(c => { }); // } +app.UseAllElasticApm(builder.Configuration); +Agent.AddFilter((ISpan span) => { + if (span.Context.Http is not null && span.Name.StartsWith($"{span.Context.Http.Method} ")) { + span.Name = $"{span.Context.Http.Method} {span.Context.Http.Url}"; + } + + return span; +}); + ///wwwroot app.UseFileServer(); // app.UseStaticFiles(); diff --git a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs b/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs deleted file mode 100644 index 1ceb095..0000000 --- a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.Collections.Concurrent; -using ArcaneLibs.Extensions; -using LibMatrix; -using LibMatrix.Homeservers; -using LibMatrix.Services; -using MxApiExtensions.Services; - -namespace ModAS.Server.Services; - -public class AuthenticatedHomeserverProviderService( - AuthenticationService authenticationService, - HomeserverProviderService homeserverProviderService, - IHttpContextAccessor request, - ModASConfiguration config, - AppServiceRegistration asRegistration - ) { - public HttpContext? _context = request.HttpContext; - public ConcurrentDictionary KnownUsers { get; set; } = new(); - - public async Task GetImpersonatedHomeserver(string mxid) { - if (!KnownUsers.TryGetValue(mxid, out var homeserver)) { - homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl); - KnownUsers.TryAdd(mxid, homeserver); - } - //var hs = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AsToken, config.HomeserverUrl); - await homeserver.SetImpersonate(mxid); - // KnownUsers.TryAdd(mxid, homeserver); - return homeserver; - } - - public async Task GetHomeserver() { - var token = authenticationService.GetToken(); - if (token == null) { - throw new MatrixException { - ErrorCode = "M_MISSING_TOKEN", - Error = "Missing access token" - }; - } - - var mxid = await authenticationService.GetMxidFromToken(token); - if (mxid == "@anonymous:*") { - throw new MatrixException { - ErrorCode = "M_MISSING_TOKEN", - Error = "Missing access token" - }; - } - - var hsCanonical = string.Join(":", mxid.Split(':').Skip(1)); - return await homeserverProviderService.GetAuthenticatedWithToken(hsCanonical, token); - } -} \ No newline at end of file diff --git a/ModAS.Server/Services/ModASConfiguration.cs b/ModAS.Server/Services/ModASConfiguration.cs index dc7c552..063e838 100644 --- a/ModAS.Server/Services/ModASConfiguration.cs +++ b/ModAS.Server/Services/ModASConfiguration.cs @@ -1,10 +1,10 @@ namespace MxApiExtensions.Services; - public class ModASConfiguration { public ModASConfiguration(IConfiguration configuration) { configuration.GetRequiredSection("ModAS").Bind(this); } + public string ServerName { get; set; } public string HomeserverUrl { get; set; } } \ No newline at end of file diff --git a/ModAS.Server/Services/RoomContextService.cs b/ModAS.Server/Services/RoomContextService.cs new file mode 100644 index 0000000..b3673c1 --- /dev/null +++ b/ModAS.Server/Services/RoomContextService.cs @@ -0,0 +1,111 @@ +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.RoomTypes; +using MxApiExtensions.Services; + +namespace ModAS.Server.Services; + +public class RoomContextService(UserProviderService userProvider, ModASConfiguration config) { + public Dictionary RoomContexts { get; } = new(); + public Dictionary> LocalUsersByRoom { get; private set; } = new(); + + public async Task GetRoomContext(GenericRoom room) { + if (RoomContexts.TryGetValue(room.RoomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext; + + var newRoomContext = await FetchRoomContext(room.RoomId); + if (newRoomContext is not null) RoomContexts[room.RoomId] = newRoomContext; + return roomContext; + } + + public async Task GetRoomContext(string roomId) { + if (RoomContexts.TryGetValue(roomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext; + + var newRoomContext = await FetchRoomContext(roomId); + if (newRoomContext is not null) RoomContexts[roomId] = newRoomContext; + return roomContext; + } + + public async Task FetchRoomContext(string roomId) { + var span = currentTransaction.StartSpan($"FetchRoomContext - {roomId}", ApiConstants.TypeApp); + if (!LocalUsersByRoom.ContainsKey(roomId)) + await UpdateLocalUserRoomLists(); + if (!LocalUsersByRoom.TryGetValue(roomId, out var localUsers)) return null; + if (localUsers.Count == 0) return null; + + var roomContext = new RoomContext { + RoomId = roomId + }; + + var room = (await userProvider.GetImpersonatedHomeserver(localUsers.First())).GetRoom(roomId); + var roomMembers = await room.GetMembersListAsync(false); + + roomContext.UserCountByMembership = roomMembers.GroupBy(x => (x.TypedContent as RoomMemberEventContent)?.Membership) + .ToDictionary(x => x.Key, x => x.Count()); + roomContext.LocalUsers = roomMembers.Select(x => x.StateKey).Where(x => x.EndsWith(':' + config.ServerName)).ToList(); + roomContext.CurrentLocalUsers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName) && (x.TypedContent as RoomMemberEventContent)?.Membership == "join") + .Select(x => x.StateKey).ToList(); + + var powerLevels = await room.GetPowerLevelsAsync(); + roomContext.LocalUsersByStatePermission = powerLevels?.Events? + .Select(@event => (@event.Key, roomContext.CurrentLocalUsers.Where(clu => powerLevels.UserHasStatePermission(clu, @event.Key)))) + .Where(x => x.Item2.Any()) + .ToDictionary(x => x.Key, x => x.Item2.ToList()); + + roomContext.LastUpdate = DateTime.Now; + + span.End(); + return roomContext; + } + + private async Task UpdateLocalUserRoomLists() { + var span = currentTransaction.StartSpan("UpdateLocalUserRoomLists", ApiConstants.TypeApp); + var newLocalUsersByRoom = new Dictionary>(); + var users = await userProvider.GetValidUsers(); + var getRoomsSpan = currentTransaction.StartSpan("GetRooms", ApiConstants.TypeApp); + var userRoomLists = users.Values.Select(ahs => (ahs, ahs.GetJoinedRooms())).ToList(); + await Task.WhenAll(userRoomLists.Select(x => x.Item2)); + getRoomsSpan.End(); + foreach (var (ahs, rooms) in userRoomLists) { + foreach (var room in rooms.Result) { + newLocalUsersByRoom.TryAdd(room.RoomId, new List()); + newLocalUsersByRoom[room.RoomId].Add(ahs.UserId); + } + } + span.End(); + LocalUsersByRoom = newLocalUsersByRoom; + } + + public class RoomContext { + public string RoomId { get; set; } + public List LocalUsers { get; set; } + public List CurrentLocalUsers { get; set; } + public Dictionary UserCountByMembership { get; set; } + public Dictionary> LocalUsersByStatePermission { get; set; } + + public DateTime LastUpdate { get; set; } + public bool NeedsUpdate => DateTime.Now - LastUpdate > TimeSpan.FromMinutes(5); + } + + public async Task GetRoomReferenceById(string roomId) { + var roomContext = await GetRoomContext(roomId); + var localUsers = roomContext.LocalUsers.Select(userProvider.GetImpersonatedHomeserver).ToAsyncEnumerable(); + await foreach (var localUser in localUsers) { + var room = localUser.GetRoom(roomId); + try { + if (await room.GetCreateEventAsync() is not null) + return room; + } + catch (MatrixException e) { + if (e is { ErrorCode: "M_UNAUTHORIZED" }) continue; + throw; + } + } + + return null; + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Services/RoomStateCacheService.cs b/ModAS.Server/Services/RoomStateCacheService.cs new file mode 100644 index 0000000..3fdf29a --- /dev/null +++ b/ModAS.Server/Services/RoomStateCacheService.cs @@ -0,0 +1,66 @@ +using System.Collections.Frozen; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.RoomTypes; + +namespace ModAS.Server.Services; + +public class RoomStateCacheService(RoomContextService roomContextService) { + public FrozenDictionary> RoomStateCache { get; private set; } = FrozenDictionary>.Empty; + private SemaphoreSlim updateLock = new(1, 1); + public async Task> GetRoomState(string roomId, GenericRoom? roomReference = null) { + if (RoomStateCache.TryGetValue(roomId, out var roomState)) return roomState; + return await InvalidateRoomState(roomId, roomReference); + } + + public async Task> InvalidateRoomState(string roomId, GenericRoom? roomReference = null) { + var invalidateSpan = currentTransaction.StartSpan($"invalidateRoomState - {roomId}", ApiConstants.TypeApp); + var getRoomReferenceSpan = currentTransaction.StartSpan($"getRoomReference - {roomId}", ApiConstants.TypeApp); + if (roomReference is null) { + var rc = await roomContextService.GetRoomContext(roomId); + if (rc is null) return FrozenSet.Empty; + roomReference = await roomContextService.GetRoomReferenceById(roomId); + } + + if (roomReference is null) { + currentTransaction.CaptureException(new Exception("Could not get room reference for room state invalidation"), roomId, true); + return FrozenSet.Empty; + } + + getRoomReferenceSpan.End(); + + var updateSpan = currentTransaction.StartSpan($"updateRoomState - {roomId}", ApiConstants.TypeApp); + await updateLock.WaitAsync(); + var unfrozen = RoomStateCache.ToDictionary(); + unfrozen[roomId] = (await roomReference.GetFullStateAsListAsync()).ToFrozenSet(); + RoomStateCache = unfrozen.ToFrozenDictionary(); + updateSpan.End(); + updateLock.Release(); + + invalidateSpan.End(); + if (!RoomStateCache.ContainsKey(roomId)) { + currentTransaction.CaptureException(new Exception("Room state cache does not contain room after invalidation"), roomId, false); + if (!unfrozen.ContainsKey(roomId)) + currentTransaction.CaptureException(new Exception("Unfrozen room state cache does not contain room after invalidation either..."), roomId, false); + } + + return RoomStateCache[roomId]; + } + + public async Task EnsureCachedFromRoomList(IEnumerable rooms) { + await updateLock.WaitAsync(); + var unfrozen = RoomStateCache.ToDictionary(); + + var tasks = rooms.Select(async room => { + if (RoomStateCache.ContainsKey(room.RoomId)) return; + unfrozen[room.RoomId] = (await room.GetFullStateAsListAsync()).ToFrozenSet(); + }).ToList(); + await Task.WhenAll(tasks); + RoomStateCache = unfrozen.ToFrozenDictionary(); + updateLock.Release(); + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Services/UserProviderService.cs b/ModAS.Server/Services/UserProviderService.cs new file mode 100644 index 0000000..bb281c4 --- /dev/null +++ b/ModAS.Server/Services/UserProviderService.cs @@ -0,0 +1,95 @@ +using System.Collections.Concurrent; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.Homeservers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Services; + +public class UserProviderService( + AuthenticationService authenticationService, + HomeserverProviderService homeserverProviderService, + IHttpContextAccessor request, + ModASConfiguration config, + AppServiceRegistration asRegistration +) { + public HttpContext? _context = request.HttpContext; + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; + + private SemaphoreSlim updateLock = new(1,1); + + public ConcurrentDictionary KnownUsers { get; set; } = new(); + public ConcurrentDictionary UserValidationExpiry { get; set; } = new(); + public ConcurrentDictionary> CachedUserRooms { get; set; } = new(); + public ConcurrentDictionary CachedUserRoomsExpiry { get; set; } = new(); + + + public async Task GetImpersonatedHomeserver(string mxid) { + var span = currentTransaction.StartSpan("GetImpersonatedHomeserver", ApiConstants.TypeApp); + if (!KnownUsers.TryGetValue(mxid, out var homeserver)) { + var getUserSpan = currentTransaction.StartSpan($"GetUser - {mxid}", ApiConstants.TypeApp); + homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl, mxid); + KnownUsers.TryAdd(mxid, homeserver); + getUserSpan.End(); + } + + await homeserver.SetImpersonate(mxid); + span.End(); + return homeserver; + } + + public async Task> GetValidUsers() { + var span = currentTransaction.StartSpan("GetValidUsers", ApiConstants.TypeApp); + var tasks = KnownUsers.Select(kvp => ValidateUser(kvp.Key, kvp.Value)).ToList(); + var results = await Task.WhenAll(tasks); + var validUsers = results.Where(r => r.Value is not null).ToDictionary(r => r.Key, r => r.Value!); + span.End(); + return validUsers; + } + + public async Task> GetUserRoomsCached(string mxid) { + var span = currentTransaction.StartSpan($"GetUserRoomsCached - {mxid}", ApiConstants.TypeApp); + var hs = await GetImpersonatedHomeserver(mxid); + if (CachedUserRoomsExpiry.TryGetValue(mxid, out var expiry) && expiry > DateTime.Now) { + if (CachedUserRooms.TryGetValue(mxid, out var rooms)) { + span.End(); + return rooms.Select(hs.GetRoom).ToList(); + } + } + + var userRooms = await hs.GetJoinedRooms(); + await updateLock.WaitAsync(); + CachedUserRooms[mxid] = userRooms.Select(r => r.RoomId).ToList(); + CachedUserRoomsExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5); + updateLock.Release(); + span.End(); + return userRooms; + } + + private async Task> ValidateUser(string mxid, AuthenticatedHomeserverGeneric hs) { + if(UserValidationExpiry.TryGetValue(mxid, out var expires)) + if(DateTime.Now < expires) return new KeyValuePair(mxid, hs); + var span = currentTransaction.StartSpan($"ValidateUser - {mxid}", ApiConstants.TypeApp); + try { + await hs.GetJoinedRooms(); + await updateLock.WaitAsync(); + UserValidationExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5); + updateLock.Release(); + return new KeyValuePair(mxid, hs); + } + catch (MatrixException e) { + if (e.ErrorCode == "M_FORBIDDEN") { + return new KeyValuePair(mxid, null); + } + throw; + } + finally { + span.End(); + } + } +} \ No newline at end of file diff --git a/ModAS.sln b/ModAS.sln index dbaf8ce..b4af365 100644 --- a/ModAS.sln +++ b/ModAS.sln @@ -6,6 +6,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LibMatrix", "LibMatrix", "{ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix", "LibMatrix\LibMatrix\LibMatrix.csproj", "{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.EventTypes", "LibMatrix\LibMatrix.EventTypes\LibMatrix.EventTypes.csproj", "{D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModAS.Classes", "ModAS.Classes\ModAS.Classes.csproj", "{70CA5BEC-8902-4CDC-A684-50D93B354643}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -20,8 +24,17 @@ Global {5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Debug|Any CPU.Build.0 = Debug|Any CPU {5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Release|Any CPU.ActiveCfg = Release|Any CPU {5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Release|Any CPU.Build.0 = Release|Any CPU + {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Release|Any CPU.Build.0 = Release|Any CPU + {70CA5BEC-8902-4CDC-A684-50D93B354643}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70CA5BEC-8902-4CDC-A684-50D93B354643}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70CA5BEC-8902-4CDC-A684-50D93B354643}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70CA5BEC-8902-4CDC-A684-50D93B354643}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {5C0EFE64-FCFF-474E-B55E-4DC30E8921D5} = {45E81953-A24E-40B8-AD1E-CD2C51020BE8} + {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335} = {45E81953-A24E-40B8-AD1E-CD2C51020BE8} EndGlobalSection EndGlobal -- cgit 1.4.1