diff options
Diffstat (limited to 'ModAS.Server')
| -rw-r--r-- | ModAS.Server/Controllers/AppService/TransactionsController.cs | 80 | ||||
| -rw-r--r-- | ModAS.Server/Controllers/DebugController.cs | 94 | ||||
| -rw-r--r-- | ModAS.Server/Controllers/HomeController.cs | 8 | ||||
| -rw-r--r-- | ModAS.Server/Controllers/RoomQueryController.cs | 135 | ||||
| -rw-r--r-- | ModAS.Server/Controllers/TransactionsController.cs | 15 | ||||
| -rw-r--r-- | ModAS.Server/ModAS.Server.csproj | 8 | ||||
| -rw-r--r-- | ModAS.Server/Program.cs | 21 | ||||
| -rw-r--r-- | ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs | 51 | ||||
| -rw-r--r-- | ModAS.Server/Services/ModASConfiguration.cs | 2 | ||||
| -rw-r--r-- | ModAS.Server/Services/RoomContextService.cs | 111 | ||||
| -rw-r--r-- | ModAS.Server/Services/RoomStateCacheService.cs | 66 | ||||
| -rw-r--r-- | ModAS.Server/Services/UserProviderService.cs | 95 |
12 files changed, 582 insertions, 104 deletions
diff --git a/ModAS.Server/Controllers/AppService/TransactionsController.cs b/ModAS.Server/Controllers/AppService/TransactionsController.cs new file mode 100644 index 0000000..b74e1e1 --- /dev/null +++ b/ModAS.Server/Controllers/AppService/TransactionsController.cs @@ -0,0 +1,80 @@ +using System.IO.Pipelines; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using ArcaneLibs; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration.Json; +using ModAS.Server; +using ModAS.Server.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Controllers.AppService; + +[ApiController] +public class TransactionsController( + AppServiceRegistration asr, + ModASConfiguration config, + UserProviderService userProvider, + RoomContextService roomContextService, + RoomStateCacheService stateCacheService) : ControllerBase { + private static List<string> _ignoredInvalidationEvents { get; set; } = [ + RoomMessageEventContent.EventId, + RoomMessageReactionEventContent.EventId + ]; + + [HttpPut("/_matrix/app/v1/transactions/{txnId}")] + public async Task<IActionResult> PutTransactions(string txnId) { + if (!Request.Headers.ContainsKey("Authorization")) { + Console.WriteLine("PutTransaction: missing authorization header"); + return Unauthorized(); + } + + if (Request.GetTypedHeaders().Get<AuthenticationHeaderValue>("Authorization")?.Parameter != asr.HomeserverToken) { + Console.WriteLine($"PutTransaction: invalid authorization header: {Request.Headers["Authorization"]}"); + return Unauthorized(); + } + + var data = await JsonSerializer.DeserializeAsync<EventList>(Request.Body); + Console.WriteLine( + $"PutTransaction: {txnId}: {data.Events.Count} events, {Util.BytesToString(Request.Headers.ContentLength ?? Request.ContentLength ?? Request.Body.Length)}"); + + if (!Directory.Exists("data")) + Directory.CreateDirectory("data"); + Directory.CreateDirectory($"data/{txnId}"); + // var pipe = PipeReader.Create(Request.Body); + // await using var file = System.IO.File.OpenWrite($"data/{txnId}"); + // await pipe.CopyToAsync(file); + // await pipe.CompleteAsync(); + // + // Console.WriteLine($"PutTransaction: {txnId}: {Util.BytesToString(file.Length)}"); + for (var i = 0; i < data.Events.Count; i++) { + var evt = data.Events[i]; + Console.WriteLine($"PutTransaction: {txnId}/{i}: {evt.Type} {evt.StateKey} {evt.Sender}"); + await System.IO.File.WriteAllTextAsync($"data/{txnId}/{i}-{evt.Type}-{evt.StateKey}-{evt.Sender}.json", JsonSerializer.Serialize(evt)); + + if (evt.Sender.EndsWith(':' + config.ServerName)) { + Console.WriteLine("PutTransaction: sender is local user, updating data..."); + try { + var user = await userProvider.GetImpersonatedHomeserver(evt.Sender); + var rooms = await user.GetJoinedRooms(); + foreach (var room in rooms) { + await roomContextService.GetRoomContext(room); + } + } + catch (Exception e) { + Console.WriteLine($"PutTransaction: failed to update data: {e}"); + } + } + else + Console.WriteLine("PutTransaction: sender is remote user"); + + if (!string.IsNullOrWhiteSpace(evt.RoomId) && !_ignoredInvalidationEvents.Contains(evt.Type)) + await stateCacheService.InvalidateRoomState(evt.RoomId); + } + + return Ok(new { }); + } +} \ No newline at end of file diff --git a/ModAS.Server/Controllers/DebugController.cs b/ModAS.Server/Controllers/DebugController.cs index d3c7ad0..f0fe91e 100644 --- a/ModAS.Server/Controllers/DebugController.cs +++ b/ModAS.Server/Controllers/DebugController.cs @@ -1,14 +1,17 @@ +using System.Collections.Frozen; using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; using LibMatrix; using LibMatrix.Homeservers; using Microsoft.AspNetCore.Mvc; using ModAS.Server.Services; using MxApiExtensions.Services; -namespace WebApplication1.Controllers; +namespace ModAS.Server.Controllers; [ApiController] -public class DebugController(ModASConfiguration config, AuthenticatedHomeserverProviderService authHsProvider) : ControllerBase { +public class DebugController(ModASConfiguration config, UserProviderService authHsProvider, RoomContextService roomContextService) : ControllerBase { [HttpGet("/_matrix/_modas/debug")] public IActionResult Index() { return Ok(new { @@ -29,40 +32,73 @@ public class DebugController(ModASConfiguration config, AuthenticatedHomeserverP [HttpGet("/_matrix/_modas/debug/test_locate_users")] public async IAsyncEnumerable<string> TestLocateUsers([FromQuery] string startUser) { - List<string> foundUsers = [startUser], processedRooms = new List<string>(); + List<AuthenticatedHomeserverGeneric> foundUsers = (await authHsProvider.GetValidUsers()).Select(x=>x.Value).ToList(); + if(!foundUsers.Any(x=>x.WhoAmI.UserId == startUser)) { + foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(startUser)); + } + + List<string> processedRooms = [], processedUsers = []; var foundNew = true; while (foundNew) { + var span1 = currentTransaction.StartSpan("iterateUsers", ApiConstants.TypeApp); foundNew = false; - foreach (var user in foundUsers.ToList()) { - AuthenticatedHomeserverGeneric? ahs = null; - try { - ahs = await authHsProvider.GetImpersonatedHomeserver(user); - await ahs.GetJoinedRooms(); - } - catch (MatrixException e) { - if (e is { ErrorCode: "M_FORBIDDEN" }) continue; - throw; - } + var usersToProcess = foundUsers.Where(x => !processedUsers.Any(y=>x.WhoAmI.UserId == y)).ToFrozenSet(); + Console.WriteLine($"Got {usersToProcess.Count} users: {string.Join(", ", usersToProcess)}"); - if(ahs is null) continue; - var rooms = await ahs.GetJoinedRooms(); - Console.WriteLine($"Got {rooms.Count} rooms"); - rooms.RemoveAll(r => processedRooms.Contains(r.RoomId)); - processedRooms.AddRange(rooms.Select(r => r.RoomId)); - foundNew = rooms.Count > 0; - Console.WriteLine($"Found {rooms.Count} new rooms"); - - var roomMemberTasks = rooms.Select(r => r.GetMembersListAsync(false)).ToAsyncEnumerable(); - await foreach (var roomMembers in roomMemberTasks) { - Console.WriteLine($"Got {roomMembers.Count} members"); - foreach (var member in roomMembers) { - if (!member.StateKey.EndsWith(':' + config.ServerName)) continue; - if (foundUsers.Contains(member.StateKey)) continue; - foundUsers.Add(member.StateKey); - yield return member.StateKey; + var rooms = usersToProcess.Select(async x => await x.GetJoinedRooms()); + var roomLists = rooms.ToAsyncEnumerable(); + await foreach (var roomList in roomLists) { + if (roomList is null) continue; + foreach (var room in roomList) { + if (processedRooms.Contains(room.RoomId)) continue; + processedRooms.Add(room.RoomId); + var roomMembers = await room.GetMembersListAsync(false); + foreach (var roomMember in roomMembers) { + if (roomMember.StateKey.EndsWith(':' + config.ServerName) && !foundUsers.Any(x=>x.WhoAmI.UserId == roomMember.StateKey)) { + foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(roomMember.StateKey)); + foundNew = true; + yield return roomMember.StateKey; + } } } } + + // await foreach (var task in tasks) { + // if (task is null) continue; + // foreach (var user in task) { + // if (foundUsers.Contains(user)) continue; + // foundUsers.Add(user); + // foundNew = true; + // yield return user; + // } + // } + + span1.End(); } } + + [HttpGet("/_matrix/_modas/debug/room_contexts")] + public IActionResult RoomContexts() { + return Ok(roomContextService.RoomContexts.Values); + } + + [HttpGet("/_matrix/_modas/debug/room_contexts/{roomId}")] + public async Task<IActionResult> RoomContext(string roomId) { + var roomContext = await roomContextService.GetRoomContext(roomId); + if (roomContext is null) return NotFound("Room not found"); + return Ok(roomContext); + } + + [HttpGet("/_matrix/_modas/debug/room_contexts/by_user/{userId}")] + public async IAsyncEnumerable<RoomContextService.RoomContext> RoomContextByUser(string userId) { + var user = await authHsProvider.GetImpersonatedHomeserver(userId); + var rooms = await user.GetJoinedRooms(); + var contexts = rooms.Select(x => roomContextService.GetRoomContext(x.RoomId)).ToAsyncEnumerable(); + await foreach (var context in contexts) { + if (context is null) continue; + yield return context; + } + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; } \ No newline at end of file diff --git a/ModAS.Server/Controllers/HomeController.cs b/ModAS.Server/Controllers/HomeController.cs index 290441e..5fd309f 100644 --- a/ModAS.Server/Controllers/HomeController.cs +++ b/ModAS.Server/Controllers/HomeController.cs @@ -1,15 +1,13 @@ using System.Diagnostics; using Microsoft.AspNetCore.Mvc; -namespace WebApplication1.Controllers; +namespace ModAS.Server.Controllers; [ApiController] -public class HomeController : Controller -{ +public class HomeController : Controller { private readonly ILogger<HomeController> _logger; - public HomeController(ILogger<HomeController> logger) - { + public HomeController(ILogger<HomeController> logger) { _logger = logger; } diff --git a/ModAS.Server/Controllers/RoomQueryController.cs b/ModAS.Server/Controllers/RoomQueryController.cs new file mode 100644 index 0000000..a49e5c0 --- /dev/null +++ b/ModAS.Server/Controllers/RoomQueryController.cs @@ -0,0 +1,135 @@ +using System.Collections.Frozen; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.EventTypes.Spec.State.RoomInfo; +using LibMatrix.Responses.ModAS; +using LibMatrix.RoomTypes; +using Microsoft.AspNetCore.Mvc; +using ModAS.Classes; +using ModAS.Server.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Controllers; + +[ApiController] +public class RoomQueryController(UserProviderService ahsProvider, ModASConfiguration config, RoomStateCacheService stateCacheService, UserProviderService userProviderService) : ControllerBase { + [HttpGet("/_matrix/_modas/room_query")] + public async IAsyncEnumerable<ModASRoomQueryResult> RoomQuery() { + var fetchRoomQueryDataSpan = currentTransaction.StartSpan("fetchRoomQueryData", ApiConstants.TypeApp); + List<string> processedRooms = new(); + var getUsersSpan = currentTransaction.StartSpan("getUsers", ApiConstants.TypeApp); + var validUsers = await ahsProvider.GetValidUsers(); + getUsersSpan.End(); + + var collectRoomsSpan = currentTransaction.StartSpan("collectRooms", ApiConstants.TypeApp); + // var userRoomLists = validUsers.Values.Select(ahs => ahs.GetJoinedRooms()).ToList(); + // await Task.WhenAll(userRoomLists); + // var rooms = userRoomLists.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet(); + var roomsTasks = validUsers.Values.Select(u => userProviderService.GetUserRoomsCached(u.WhoAmI.UserId)).ToList(); + await Task.WhenAll(roomsTasks); + var rooms = roomsTasks.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet(); + collectRoomsSpan.End(); + + var collectRoomStateSpan = currentTransaction.StartSpan("collectRoomState", ApiConstants.TypeApp); + //make sure we lock!!!! + await stateCacheService.EnsureCachedFromRoomList(rooms); + var roomStateTasks = rooms.Select(GetRoomState).ToAsyncEnumerable(); + var awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp); + await foreach (var (room, roomState) in roomStateTasks) { + awaitRoomStateSpan.Name = $"awaitRoomState {room.RoomId}"; + awaitRoomStateSpan.End(); + + var filterStateEventsSpan = currentTransaction.StartSpan($"filterStateEvents {room.RoomId}", ApiConstants.TypeApp); + var roomMembers = roomState.Where(r => r.Type == RoomMemberEventContent.EventId).ToFrozenSet(); + var localRoomMembers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName)).ToFrozenSet(); + var nonMemberState = roomState.Where(x => !roomMembers.Contains(x)).ToFrozenSet(); + filterStateEventsSpan.End(); + var buildResultSpan = currentTransaction.StartSpan($"buildResult {room.RoomId}", ApiConstants.TypeApp); + + // forgive me for raw json access... attempt at optimisation -emma + yield return new ModASRoomQueryResult { + RoomId = room.RoomId, + StateEvents = roomState.Count, + //members + TotalMembers = roomMembers.Count, + TotalLocalMembers = localRoomMembers.Count, + JoinedMembers = roomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"), + JoinedLocalMembers = localRoomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"), + //-members + //creation event + Creator = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.Sender, + Version = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["room_version"]?.GetValue<string>(), + Type = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["type"]?.GetValue<string>(), + Federatable = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["m.federate"]?.GetValue<bool>() ?? true, + //-creation event + Name = nonMemberState.FirstOrDefault(r => r.Type == RoomNameEventContent.EventId)?.RawContent?["name"]?.GetValue<string>(), + CanonicalAlias = nonMemberState.FirstOrDefault(r => r.Type == RoomCanonicalAliasEventContent.EventId)?.RawContent?["alias"]?.GetValue<string>(), + JoinRules = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue<string>(), + GuestAccess = nonMemberState.FirstOrDefault(r => r.Type == RoomGuestAccessEventContent.EventId)?.RawContent?["guest_access"]?.GetValue<string>(), + HistoryVisibility = + nonMemberState.FirstOrDefault(r => r.Type == RoomHistoryVisibilityEventContent.EventId)?.RawContent?["history_visibility"]?.GetValue<string>(), + Public = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue<string>() == "public", + Encryption = nonMemberState.FirstOrDefault(r => r.Type == RoomEncryptionEventContent.EventId)?.RawContent?["algorithm"]?.GetValue<string>(), + AvatarUrl = nonMemberState.FirstOrDefault(r => r.Type == RoomAvatarEventContent.EventId)?.RawContent?["url"]?.GetValue<string>(), + RoomTopic = nonMemberState.FirstOrDefault(r => r.Type == RoomTopicEventContent.EventId)?.RawContent?["topic"]?.GetValue<string>() + }; + buildResultSpan.End(); + awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp); + } + + collectRoomStateSpan.End(); + fetchRoomQueryDataSpan.End(); + } + + [HttpPost("/_matrix/_modas/room_query")] + public async IAsyncEnumerable<ModASRoomQueryResult> RoomQuery([FromBody] RoomQueryFilter request) { + await foreach (var room in RoomQuery()) { + if (!string.IsNullOrWhiteSpace(request.RoomIdContains) && !room.RoomId.Contains(request.RoomIdContains)) continue; + + if (!string.IsNullOrWhiteSpace(request.NameContains) && room.Name?.Contains(request.NameContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.CanonicalAliasContains) && room.CanonicalAlias?.Contains(request.CanonicalAliasContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.VersionContains) && room.Version?.Contains(request.VersionContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.CreatorContains) && room.Creator?.Contains(request.CreatorContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.EncryptionContains) && room.Encryption?.Contains(request.EncryptionContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.JoinRulesContains) && room.JoinRules?.Contains(request.JoinRulesContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.GuestAccessContains) && room.GuestAccess?.Contains(request.GuestAccessContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.HistoryVisibilityContains) && room.HistoryVisibility?.Contains(request.HistoryVisibilityContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.AvatarUrlContains) && room.AvatarUrl?.Contains(request.AvatarUrlContains) != true) continue; + + if (!string.IsNullOrWhiteSpace(request.RoomTopicContains) && room.RoomTopic?.Contains(request.RoomTopicContains) != true) continue; + + if (request.JoinedMembersMin.HasValue && room.JoinedMembers < request.JoinedMembersMin.Value) continue; + if (request.JoinedMembersMax.HasValue && room.JoinedMembers > request.JoinedMembersMax.Value) continue; + + if (request.JoinedLocalMembersMin.HasValue && room.JoinedLocalMembers < request.JoinedLocalMembersMin.Value) continue; + if (request.JoinedLocalMembersMax.HasValue && room.JoinedLocalMembers > request.JoinedLocalMembersMax.Value) continue; + + if (request.StateEventsMin.HasValue && room.StateEvents < request.StateEventsMin.Value) continue; + if (request.StateEventsMax.HasValue && room.StateEvents > request.StateEventsMax.Value) continue; + + if (request.IsFederatable.HasValue && room.Federatable != request.IsFederatable.Value) continue; + + if (request.IsPublic.HasValue && room.Public != request.IsPublic.Value) continue; + + yield return room; + } + } + + private async Task<KeyValuePair<GenericRoom, FrozenSet<StateEventResponse>>> GetRoomState(GenericRoom room) { + var roomState = await stateCacheService.GetRoomState(room.RoomId, room); + return new(room, roomState); + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Controllers/TransactionsController.cs b/ModAS.Server/Controllers/TransactionsController.cs deleted file mode 100644 index 8e4e018..0000000 --- a/ModAS.Server/Controllers/TransactionsController.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.IO.Pipelines; -using Microsoft.AspNetCore.Mvc; -using ModAS.Server; - -namespace WebApplication1.Controllers; - -[ApiController] -public class TransactionsController(AppServiceRegistration asr) : ControllerBase { - [HttpPut(" /_matrix/app/v1/transactions/{txnId}")] - public async Task<IActionResult> PutTransactions(string txnId) { - if(!Request.Headers.ContainsKey("Authorization") || Request.Headers["Authorization"] != asr.HomeserverToken) return Unauthorized(); - await Request.Body.CopyToAsync(Console.OpenStandardOutput()); - return Ok(new{}); - } -} \ No newline at end of file diff --git a/ModAS.Server/ModAS.Server.csproj b/ModAS.Server/ModAS.Server.csproj index 8b48b8a..07cc67c 100644 --- a/ModAS.Server/ModAS.Server.csproj +++ b/ModAS.Server/ModAS.Server.csproj @@ -7,9 +7,16 @@ <InvariantGlobalization>true</InvariantGlobalization> <GenerateDocumentationFile>true</GenerateDocumentationFile> <LangVersion>preview</LangVersion> + <CheckForOverflowUnderflow>false</CheckForOverflowUnderflow> + <EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator> + <StripSymbols>true</StripSymbols> + <OptimizationPreference>Speed</OptimizationPreference> + <TieredPGO>true</TieredPGO> </PropertyGroup> <ItemGroup> + <PackageReference Include="Elastic.Apm.AspNetCore" Version="1.25.2" /> + <PackageReference Include="Elastic.Apm.NetCoreAll" Version="1.25.2" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.0"/> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" /> <PackageReference Include="Swashbuckle.AspNetCore.ReDoc" Version="6.5.0" /> @@ -17,6 +24,7 @@ <ItemGroup> <ProjectReference Include="..\LibMatrix\LibMatrix\LibMatrix.csproj" /> + <ProjectReference Include="..\ModAS.Classes\ModAS.Classes.csproj" /> </ItemGroup> </Project> diff --git a/ModAS.Server/Program.cs b/ModAS.Server/Program.cs index da44ca8..0b3d121 100644 --- a/ModAS.Server/Program.cs +++ b/ModAS.Server/Program.cs @@ -3,12 +3,17 @@ using Microsoft.OpenApi.Models; using ModAS.Server; using System.Diagnostics; using System.Text.Json; +using Elastic.Apm; +using Elastic.Apm.Api; +using Elastic.Apm.AspNetCore; +using Elastic.Apm.NetCoreAll; using LibMatrix; using LibMatrix.Services; using ModAS.Server.Services; using MxApiExtensions.Services; -var builder = WebApplication.CreateBuilder(args); +// var builder = WebApplication.CreateBuilder(args); +var builder = WebApplication.CreateSlimBuilder(args); builder.Services.AddControllers().AddJsonOptions(options => { options.JsonSerializerOptions.WriteIndented = true; }); ///add wwwroot @@ -30,7 +35,9 @@ builder.Services.AddSingleton<IHttpContextAccessor, HttpContextAccessor>(); builder.Services.AddSingleton<ModASConfiguration>(); builder.Services.AddSingleton<AuthenticationService>(); -builder.Services.AddSingleton<AuthenticatedHomeserverProviderService>(); +builder.Services.AddSingleton<UserProviderService>(); +builder.Services.AddSingleton<RoomContextService>(); +builder.Services.AddSingleton<RoomStateCacheService>(); // builder.Services.AddScoped<UserContextService>(); builder.Services.AddSingleton<TieredStorageService>(x => { @@ -82,7 +89,6 @@ builder.Services.AddCors(options => { var app = builder.Build(); -// Configure the HTTP request pipeline. // if (app.Environment.IsDevelopment()) { app.UseSwagger(); app.UseSwaggerUI(c => { @@ -99,6 +105,15 @@ app.UseReDoc(c => { }); // } +app.UseAllElasticApm(builder.Configuration); +Agent.AddFilter((ISpan span) => { + if (span.Context.Http is not null && span.Name.StartsWith($"{span.Context.Http.Method} ")) { + span.Name = $"{span.Context.Http.Method} {span.Context.Http.Url}"; + } + + return span; +}); + ///wwwroot app.UseFileServer(); // app.UseStaticFiles(); diff --git a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs b/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs deleted file mode 100644 index 1ceb095..0000000 --- a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.Collections.Concurrent; -using ArcaneLibs.Extensions; -using LibMatrix; -using LibMatrix.Homeservers; -using LibMatrix.Services; -using MxApiExtensions.Services; - -namespace ModAS.Server.Services; - -public class AuthenticatedHomeserverProviderService( - AuthenticationService authenticationService, - HomeserverProviderService homeserverProviderService, - IHttpContextAccessor request, - ModASConfiguration config, - AppServiceRegistration asRegistration - ) { - public HttpContext? _context = request.HttpContext; - public ConcurrentDictionary<string, AuthenticatedHomeserverGeneric> KnownUsers { get; set; } = new(); - - public async Task<AuthenticatedHomeserverGeneric> GetImpersonatedHomeserver(string mxid) { - if (!KnownUsers.TryGetValue(mxid, out var homeserver)) { - homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl); - KnownUsers.TryAdd(mxid, homeserver); - } - //var hs = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AsToken, config.HomeserverUrl); - await homeserver.SetImpersonate(mxid); - // KnownUsers.TryAdd(mxid, homeserver); - return homeserver; - } - - public async Task<AuthenticatedHomeserverGeneric> GetHomeserver() { - var token = authenticationService.GetToken(); - if (token == null) { - throw new MatrixException { - ErrorCode = "M_MISSING_TOKEN", - Error = "Missing access token" - }; - } - - var mxid = await authenticationService.GetMxidFromToken(token); - if (mxid == "@anonymous:*") { - throw new MatrixException { - ErrorCode = "M_MISSING_TOKEN", - Error = "Missing access token" - }; - } - - var hsCanonical = string.Join(":", mxid.Split(':').Skip(1)); - return await homeserverProviderService.GetAuthenticatedWithToken(hsCanonical, token); - } -} \ No newline at end of file diff --git a/ModAS.Server/Services/ModASConfiguration.cs b/ModAS.Server/Services/ModASConfiguration.cs index dc7c552..063e838 100644 --- a/ModAS.Server/Services/ModASConfiguration.cs +++ b/ModAS.Server/Services/ModASConfiguration.cs @@ -1,10 +1,10 @@ namespace MxApiExtensions.Services; - public class ModASConfiguration { public ModASConfiguration(IConfiguration configuration) { configuration.GetRequiredSection("ModAS").Bind(this); } + public string ServerName { get; set; } public string HomeserverUrl { get; set; } } \ No newline at end of file diff --git a/ModAS.Server/Services/RoomContextService.cs b/ModAS.Server/Services/RoomContextService.cs new file mode 100644 index 0000000..b3673c1 --- /dev/null +++ b/ModAS.Server/Services/RoomContextService.cs @@ -0,0 +1,111 @@ +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.RoomTypes; +using MxApiExtensions.Services; + +namespace ModAS.Server.Services; + +public class RoomContextService(UserProviderService userProvider, ModASConfiguration config) { + public Dictionary<string, RoomContext> RoomContexts { get; } = new(); + public Dictionary<string, List<string>> LocalUsersByRoom { get; private set; } = new(); + + public async Task<RoomContext?> GetRoomContext(GenericRoom room) { + if (RoomContexts.TryGetValue(room.RoomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext; + + var newRoomContext = await FetchRoomContext(room.RoomId); + if (newRoomContext is not null) RoomContexts[room.RoomId] = newRoomContext; + return roomContext; + } + + public async Task<RoomContext?> GetRoomContext(string roomId) { + if (RoomContexts.TryGetValue(roomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext; + + var newRoomContext = await FetchRoomContext(roomId); + if (newRoomContext is not null) RoomContexts[roomId] = newRoomContext; + return roomContext; + } + + public async Task<RoomContext?> FetchRoomContext(string roomId) { + var span = currentTransaction.StartSpan($"FetchRoomContext - {roomId}", ApiConstants.TypeApp); + if (!LocalUsersByRoom.ContainsKey(roomId)) + await UpdateLocalUserRoomLists(); + if (!LocalUsersByRoom.TryGetValue(roomId, out var localUsers)) return null; + if (localUsers.Count == 0) return null; + + var roomContext = new RoomContext { + RoomId = roomId + }; + + var room = (await userProvider.GetImpersonatedHomeserver(localUsers.First())).GetRoom(roomId); + var roomMembers = await room.GetMembersListAsync(false); + + roomContext.UserCountByMembership = roomMembers.GroupBy(x => (x.TypedContent as RoomMemberEventContent)?.Membership) + .ToDictionary(x => x.Key, x => x.Count()); + roomContext.LocalUsers = roomMembers.Select(x => x.StateKey).Where(x => x.EndsWith(':' + config.ServerName)).ToList(); + roomContext.CurrentLocalUsers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName) && (x.TypedContent as RoomMemberEventContent)?.Membership == "join") + .Select(x => x.StateKey).ToList(); + + var powerLevels = await room.GetPowerLevelsAsync(); + roomContext.LocalUsersByStatePermission = powerLevels?.Events? + .Select(@event => (@event.Key, roomContext.CurrentLocalUsers.Where(clu => powerLevels.UserHasStatePermission(clu, @event.Key)))) + .Where(x => x.Item2.Any()) + .ToDictionary(x => x.Key, x => x.Item2.ToList()); + + roomContext.LastUpdate = DateTime.Now; + + span.End(); + return roomContext; + } + + private async Task UpdateLocalUserRoomLists() { + var span = currentTransaction.StartSpan("UpdateLocalUserRoomLists", ApiConstants.TypeApp); + var newLocalUsersByRoom = new Dictionary<string, List<string>>(); + var users = await userProvider.GetValidUsers(); + var getRoomsSpan = currentTransaction.StartSpan("GetRooms", ApiConstants.TypeApp); + var userRoomLists = users.Values.Select(ahs => (ahs, ahs.GetJoinedRooms())).ToList(); + await Task.WhenAll(userRoomLists.Select(x => x.Item2)); + getRoomsSpan.End(); + foreach (var (ahs, rooms) in userRoomLists) { + foreach (var room in rooms.Result) { + newLocalUsersByRoom.TryAdd(room.RoomId, new List<string>()); + newLocalUsersByRoom[room.RoomId].Add(ahs.UserId); + } + } + span.End(); + LocalUsersByRoom = newLocalUsersByRoom; + } + + public class RoomContext { + public string RoomId { get; set; } + public List<string> LocalUsers { get; set; } + public List<string> CurrentLocalUsers { get; set; } + public Dictionary<string, int> UserCountByMembership { get; set; } + public Dictionary<string, List<string>> LocalUsersByStatePermission { get; set; } + + public DateTime LastUpdate { get; set; } + public bool NeedsUpdate => DateTime.Now - LastUpdate > TimeSpan.FromMinutes(5); + } + + public async Task<GenericRoom?> GetRoomReferenceById(string roomId) { + var roomContext = await GetRoomContext(roomId); + var localUsers = roomContext.LocalUsers.Select(userProvider.GetImpersonatedHomeserver).ToAsyncEnumerable(); + await foreach (var localUser in localUsers) { + var room = localUser.GetRoom(roomId); + try { + if (await room.GetCreateEventAsync() is not null) + return room; + } + catch (MatrixException e) { + if (e is { ErrorCode: "M_UNAUTHORIZED" }) continue; + throw; + } + } + + return null; + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Services/RoomStateCacheService.cs b/ModAS.Server/Services/RoomStateCacheService.cs new file mode 100644 index 0000000..3fdf29a --- /dev/null +++ b/ModAS.Server/Services/RoomStateCacheService.cs @@ -0,0 +1,66 @@ +using System.Collections.Frozen; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.RoomTypes; + +namespace ModAS.Server.Services; + +public class RoomStateCacheService(RoomContextService roomContextService) { + public FrozenDictionary<string, FrozenSet<StateEventResponse>> RoomStateCache { get; private set; } = FrozenDictionary<string, FrozenSet<StateEventResponse>>.Empty; + private SemaphoreSlim updateLock = new(1, 1); + public async Task<FrozenSet<StateEventResponse>> GetRoomState(string roomId, GenericRoom? roomReference = null) { + if (RoomStateCache.TryGetValue(roomId, out var roomState)) return roomState; + return await InvalidateRoomState(roomId, roomReference); + } + + public async Task<FrozenSet<StateEventResponse>> InvalidateRoomState(string roomId, GenericRoom? roomReference = null) { + var invalidateSpan = currentTransaction.StartSpan($"invalidateRoomState - {roomId}", ApiConstants.TypeApp); + var getRoomReferenceSpan = currentTransaction.StartSpan($"getRoomReference - {roomId}", ApiConstants.TypeApp); + if (roomReference is null) { + var rc = await roomContextService.GetRoomContext(roomId); + if (rc is null) return FrozenSet<StateEventResponse>.Empty; + roomReference = await roomContextService.GetRoomReferenceById(roomId); + } + + if (roomReference is null) { + currentTransaction.CaptureException(new Exception("Could not get room reference for room state invalidation"), roomId, true); + return FrozenSet<StateEventResponse>.Empty; + } + + getRoomReferenceSpan.End(); + + var updateSpan = currentTransaction.StartSpan($"updateRoomState - {roomId}", ApiConstants.TypeApp); + await updateLock.WaitAsync(); + var unfrozen = RoomStateCache.ToDictionary(); + unfrozen[roomId] = (await roomReference.GetFullStateAsListAsync()).ToFrozenSet(); + RoomStateCache = unfrozen.ToFrozenDictionary(); + updateSpan.End(); + updateLock.Release(); + + invalidateSpan.End(); + if (!RoomStateCache.ContainsKey(roomId)) { + currentTransaction.CaptureException(new Exception("Room state cache does not contain room after invalidation"), roomId, false); + if (!unfrozen.ContainsKey(roomId)) + currentTransaction.CaptureException(new Exception("Unfrozen room state cache does not contain room after invalidation either..."), roomId, false); + } + + return RoomStateCache[roomId]; + } + + public async Task EnsureCachedFromRoomList(IEnumerable<GenericRoom> rooms) { + await updateLock.WaitAsync(); + var unfrozen = RoomStateCache.ToDictionary(); + + var tasks = rooms.Select(async room => { + if (RoomStateCache.ContainsKey(room.RoomId)) return; + unfrozen[room.RoomId] = (await room.GetFullStateAsListAsync()).ToFrozenSet(); + }).ToList(); + await Task.WhenAll(tasks); + RoomStateCache = unfrozen.ToFrozenDictionary(); + updateLock.Release(); + } + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; +} \ No newline at end of file diff --git a/ModAS.Server/Services/UserProviderService.cs b/ModAS.Server/Services/UserProviderService.cs new file mode 100644 index 0000000..bb281c4 --- /dev/null +++ b/ModAS.Server/Services/UserProviderService.cs @@ -0,0 +1,95 @@ +using System.Collections.Concurrent; +using ArcaneLibs.Extensions; +using Elastic.Apm; +using Elastic.Apm.Api; +using LibMatrix; +using LibMatrix.Homeservers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using MxApiExtensions.Services; + +namespace ModAS.Server.Services; + +public class UserProviderService( + AuthenticationService authenticationService, + HomeserverProviderService homeserverProviderService, + IHttpContextAccessor request, + ModASConfiguration config, + AppServiceRegistration asRegistration +) { + public HttpContext? _context = request.HttpContext; + + private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction; + + private SemaphoreSlim updateLock = new(1,1); + + public ConcurrentDictionary<string, AuthenticatedHomeserverGeneric> KnownUsers { get; set; } = new(); + public ConcurrentDictionary<string, DateTime> UserValidationExpiry { get; set; } = new(); + public ConcurrentDictionary<string, List<string>> CachedUserRooms { get; set; } = new(); + public ConcurrentDictionary<string, DateTime> CachedUserRoomsExpiry { get; set; } = new(); + + + public async Task<AuthenticatedHomeserverGeneric> GetImpersonatedHomeserver(string mxid) { + var span = currentTransaction.StartSpan("GetImpersonatedHomeserver", ApiConstants.TypeApp); + if (!KnownUsers.TryGetValue(mxid, out var homeserver)) { + var getUserSpan = currentTransaction.StartSpan($"GetUser - {mxid}", ApiConstants.TypeApp); + homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl, mxid); + KnownUsers.TryAdd(mxid, homeserver); + getUserSpan.End(); + } + + await homeserver.SetImpersonate(mxid); + span.End(); + return homeserver; + } + + public async Task<Dictionary<string, AuthenticatedHomeserverGeneric>> GetValidUsers() { + var span = currentTransaction.StartSpan("GetValidUsers", ApiConstants.TypeApp); + var tasks = KnownUsers.Select(kvp => ValidateUser(kvp.Key, kvp.Value)).ToList(); + var results = await Task.WhenAll(tasks); + var validUsers = results.Where(r => r.Value is not null).ToDictionary(r => r.Key, r => r.Value!); + span.End(); + return validUsers; + } + + public async Task<List<GenericRoom>> GetUserRoomsCached(string mxid) { + var span = currentTransaction.StartSpan($"GetUserRoomsCached - {mxid}", ApiConstants.TypeApp); + var hs = await GetImpersonatedHomeserver(mxid); + if (CachedUserRoomsExpiry.TryGetValue(mxid, out var expiry) && expiry > DateTime.Now) { + if (CachedUserRooms.TryGetValue(mxid, out var rooms)) { + span.End(); + return rooms.Select(hs.GetRoom).ToList(); + } + } + + var userRooms = await hs.GetJoinedRooms(); + await updateLock.WaitAsync(); + CachedUserRooms[mxid] = userRooms.Select(r => r.RoomId).ToList(); + CachedUserRoomsExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5); + updateLock.Release(); + span.End(); + return userRooms; + } + + private async Task<KeyValuePair<string, AuthenticatedHomeserverGeneric?>> ValidateUser(string mxid, AuthenticatedHomeserverGeneric hs) { + if(UserValidationExpiry.TryGetValue(mxid, out var expires)) + if(DateTime.Now < expires) return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, hs); + var span = currentTransaction.StartSpan($"ValidateUser - {mxid}", ApiConstants.TypeApp); + try { + await hs.GetJoinedRooms(); + await updateLock.WaitAsync(); + UserValidationExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5); + updateLock.Release(); + return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, hs); + } + catch (MatrixException e) { + if (e.ErrorCode == "M_FORBIDDEN") { + return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, null); + } + throw; + } + finally { + span.End(); + } + } +} \ No newline at end of file |