diff --git a/LibMatrix b/LibMatrix
-Subproject 314f7044f62b92c49abe2d5c7422c6cf3430b02
+Subproject 94b83d4de5e435796da9cc14667c1023a09df8e
diff --git a/ModAS.Classes/ModAS.Classes.csproj b/ModAS.Classes/ModAS.Classes.csproj
new file mode 100644
index 0000000..3a63532
--- /dev/null
+++ b/ModAS.Classes/ModAS.Classes.csproj
@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFramework>net8.0</TargetFramework>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ </PropertyGroup>
+
+</Project>
diff --git a/ModAS.Classes/RoomQueryFilter.cs b/ModAS.Classes/RoomQueryFilter.cs
new file mode 100644
index 0000000..0a6430e
--- /dev/null
+++ b/ModAS.Classes/RoomQueryFilter.cs
@@ -0,0 +1,26 @@
+namespace ModAS.Classes;
+
+public class RoomQueryFilter {
+ public string? RoomIdContains { get; set; }
+ public string? NameContains { get; set; }
+ public string? CanonicalAliasContains { get; set; }
+ public string? VersionContains { get; set; }
+ public string? CreatorContains { get; set; }
+ public string? EncryptionContains { get; set; }
+ public string? JoinRulesContains { get; set; }
+ public string? GuestAccessContains { get; set; }
+ public string? HistoryVisibilityContains { get; set; }
+ public string? AvatarUrlContains { get; set; }
+ public string? RoomTopicContains { get; set; }
+
+ public bool? IsFederatable { get; set; } = true;
+ public bool? IsPublic { get; set; } = true;
+
+ public uint? JoinedMembersMin { get; set; }
+ public uint? JoinedMembersMax { get; set; }
+ public uint? JoinedLocalMembersMin { get; set; }
+ public uint? JoinedLocalMembersMax { get; set; }
+ public uint? StateEventsMin { get; set; }
+ public uint? StateEventsMax { get; set; }
+
+}
\ No newline at end of file
diff --git a/ModAS.Server/Controllers/AppService/TransactionsController.cs b/ModAS.Server/Controllers/AppService/TransactionsController.cs
new file mode 100644
index 0000000..b74e1e1
--- /dev/null
+++ b/ModAS.Server/Controllers/AppService/TransactionsController.cs
@@ -0,0 +1,80 @@
+using System.IO.Pipelines;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using ArcaneLibs;
+using LibMatrix;
+using LibMatrix.EventTypes.Spec;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Configuration.Json;
+using ModAS.Server;
+using ModAS.Server.Services;
+using MxApiExtensions.Services;
+
+namespace ModAS.Server.Controllers.AppService;
+
+[ApiController]
+public class TransactionsController(
+ AppServiceRegistration asr,
+ ModASConfiguration config,
+ UserProviderService userProvider,
+ RoomContextService roomContextService,
+ RoomStateCacheService stateCacheService) : ControllerBase {
+ private static List<string> _ignoredInvalidationEvents { get; set; } = [
+ RoomMessageEventContent.EventId,
+ RoomMessageReactionEventContent.EventId
+ ];
+
+ [HttpPut("/_matrix/app/v1/transactions/{txnId}")]
+ public async Task<IActionResult> PutTransactions(string txnId) {
+ if (!Request.Headers.ContainsKey("Authorization")) {
+ Console.WriteLine("PutTransaction: missing authorization header");
+ return Unauthorized();
+ }
+
+ if (Request.GetTypedHeaders().Get<AuthenticationHeaderValue>("Authorization")?.Parameter != asr.HomeserverToken) {
+ Console.WriteLine($"PutTransaction: invalid authorization header: {Request.Headers["Authorization"]}");
+ return Unauthorized();
+ }
+
+ var data = await JsonSerializer.DeserializeAsync<EventList>(Request.Body);
+ Console.WriteLine(
+ $"PutTransaction: {txnId}: {data.Events.Count} events, {Util.BytesToString(Request.Headers.ContentLength ?? Request.ContentLength ?? Request.Body.Length)}");
+
+ if (!Directory.Exists("data"))
+ Directory.CreateDirectory("data");
+ Directory.CreateDirectory($"data/{txnId}");
+ // var pipe = PipeReader.Create(Request.Body);
+ // await using var file = System.IO.File.OpenWrite($"data/{txnId}");
+ // await pipe.CopyToAsync(file);
+ // await pipe.CompleteAsync();
+ //
+ // Console.WriteLine($"PutTransaction: {txnId}: {Util.BytesToString(file.Length)}");
+ for (var i = 0; i < data.Events.Count; i++) {
+ var evt = data.Events[i];
+ Console.WriteLine($"PutTransaction: {txnId}/{i}: {evt.Type} {evt.StateKey} {evt.Sender}");
+ await System.IO.File.WriteAllTextAsync($"data/{txnId}/{i}-{evt.Type}-{evt.StateKey}-{evt.Sender}.json", JsonSerializer.Serialize(evt));
+
+ if (evt.Sender.EndsWith(':' + config.ServerName)) {
+ Console.WriteLine("PutTransaction: sender is local user, updating data...");
+ try {
+ var user = await userProvider.GetImpersonatedHomeserver(evt.Sender);
+ var rooms = await user.GetJoinedRooms();
+ foreach (var room in rooms) {
+ await roomContextService.GetRoomContext(room);
+ }
+ }
+ catch (Exception e) {
+ Console.WriteLine($"PutTransaction: failed to update data: {e}");
+ }
+ }
+ else
+ Console.WriteLine("PutTransaction: sender is remote user");
+
+ if (!string.IsNullOrWhiteSpace(evt.RoomId) && !_ignoredInvalidationEvents.Contains(evt.Type))
+ await stateCacheService.InvalidateRoomState(evt.RoomId);
+ }
+
+ return Ok(new { });
+ }
+}
\ No newline at end of file
diff --git a/ModAS.Server/Controllers/DebugController.cs b/ModAS.Server/Controllers/DebugController.cs
index d3c7ad0..f0fe91e 100644
--- a/ModAS.Server/Controllers/DebugController.cs
+++ b/ModAS.Server/Controllers/DebugController.cs
@@ -1,14 +1,17 @@
+using System.Collections.Frozen;
using ArcaneLibs.Extensions;
+using Elastic.Apm;
+using Elastic.Apm.Api;
using LibMatrix;
using LibMatrix.Homeservers;
using Microsoft.AspNetCore.Mvc;
using ModAS.Server.Services;
using MxApiExtensions.Services;
-namespace WebApplication1.Controllers;
+namespace ModAS.Server.Controllers;
[ApiController]
-public class DebugController(ModASConfiguration config, AuthenticatedHomeserverProviderService authHsProvider) : ControllerBase {
+public class DebugController(ModASConfiguration config, UserProviderService authHsProvider, RoomContextService roomContextService) : ControllerBase {
[HttpGet("/_matrix/_modas/debug")]
public IActionResult Index() {
return Ok(new {
@@ -29,40 +32,73 @@ public class DebugController(ModASConfiguration config, AuthenticatedHomeserverP
[HttpGet("/_matrix/_modas/debug/test_locate_users")]
public async IAsyncEnumerable<string> TestLocateUsers([FromQuery] string startUser) {
- List<string> foundUsers = [startUser], processedRooms = new List<string>();
+ List<AuthenticatedHomeserverGeneric> foundUsers = (await authHsProvider.GetValidUsers()).Select(x=>x.Value).ToList();
+ if(!foundUsers.Any(x=>x.WhoAmI.UserId == startUser)) {
+ foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(startUser));
+ }
+
+ List<string> processedRooms = [], processedUsers = [];
var foundNew = true;
while (foundNew) {
+ var span1 = currentTransaction.StartSpan("iterateUsers", ApiConstants.TypeApp);
foundNew = false;
- foreach (var user in foundUsers.ToList()) {
- AuthenticatedHomeserverGeneric? ahs = null;
- try {
- ahs = await authHsProvider.GetImpersonatedHomeserver(user);
- await ahs.GetJoinedRooms();
- }
- catch (MatrixException e) {
- if (e is { ErrorCode: "M_FORBIDDEN" }) continue;
- throw;
- }
+ var usersToProcess = foundUsers.Where(x => !processedUsers.Any(y=>x.WhoAmI.UserId == y)).ToFrozenSet();
+ Console.WriteLine($"Got {usersToProcess.Count} users: {string.Join(", ", usersToProcess)}");
- if(ahs is null) continue;
- var rooms = await ahs.GetJoinedRooms();
- Console.WriteLine($"Got {rooms.Count} rooms");
- rooms.RemoveAll(r => processedRooms.Contains(r.RoomId));
- processedRooms.AddRange(rooms.Select(r => r.RoomId));
- foundNew = rooms.Count > 0;
- Console.WriteLine($"Found {rooms.Count} new rooms");
-
- var roomMemberTasks = rooms.Select(r => r.GetMembersListAsync(false)).ToAsyncEnumerable();
- await foreach (var roomMembers in roomMemberTasks) {
- Console.WriteLine($"Got {roomMembers.Count} members");
- foreach (var member in roomMembers) {
- if (!member.StateKey.EndsWith(':' + config.ServerName)) continue;
- if (foundUsers.Contains(member.StateKey)) continue;
- foundUsers.Add(member.StateKey);
- yield return member.StateKey;
+ var rooms = usersToProcess.Select(async x => await x.GetJoinedRooms());
+ var roomLists = rooms.ToAsyncEnumerable();
+ await foreach (var roomList in roomLists) {
+ if (roomList is null) continue;
+ foreach (var room in roomList) {
+ if (processedRooms.Contains(room.RoomId)) continue;
+ processedRooms.Add(room.RoomId);
+ var roomMembers = await room.GetMembersListAsync(false);
+ foreach (var roomMember in roomMembers) {
+ if (roomMember.StateKey.EndsWith(':' + config.ServerName) && !foundUsers.Any(x=>x.WhoAmI.UserId == roomMember.StateKey)) {
+ foundUsers.Add(await authHsProvider.GetImpersonatedHomeserver(roomMember.StateKey));
+ foundNew = true;
+ yield return roomMember.StateKey;
+ }
}
}
}
+
+ // await foreach (var task in tasks) {
+ // if (task is null) continue;
+ // foreach (var user in task) {
+ // if (foundUsers.Contains(user)) continue;
+ // foundUsers.Add(user);
+ // foundNew = true;
+ // yield return user;
+ // }
+ // }
+
+ span1.End();
}
}
+
+ [HttpGet("/_matrix/_modas/debug/room_contexts")]
+ public IActionResult RoomContexts() {
+ return Ok(roomContextService.RoomContexts.Values);
+ }
+
+ [HttpGet("/_matrix/_modas/debug/room_contexts/{roomId}")]
+ public async Task<IActionResult> RoomContext(string roomId) {
+ var roomContext = await roomContextService.GetRoomContext(roomId);
+ if (roomContext is null) return NotFound("Room not found");
+ return Ok(roomContext);
+ }
+
+ [HttpGet("/_matrix/_modas/debug/room_contexts/by_user/{userId}")]
+ public async IAsyncEnumerable<RoomContextService.RoomContext> RoomContextByUser(string userId) {
+ var user = await authHsProvider.GetImpersonatedHomeserver(userId);
+ var rooms = await user.GetJoinedRooms();
+ var contexts = rooms.Select(x => roomContextService.GetRoomContext(x.RoomId)).ToAsyncEnumerable();
+ await foreach (var context in contexts) {
+ if (context is null) continue;
+ yield return context;
+ }
+ }
+
+ private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction;
}
\ No newline at end of file
diff --git a/ModAS.Server/Controllers/HomeController.cs b/ModAS.Server/Controllers/HomeController.cs
index 290441e..5fd309f 100644
--- a/ModAS.Server/Controllers/HomeController.cs
+++ b/ModAS.Server/Controllers/HomeController.cs
@@ -1,15 +1,13 @@
using System.Diagnostics;
using Microsoft.AspNetCore.Mvc;
-namespace WebApplication1.Controllers;
+namespace ModAS.Server.Controllers;
[ApiController]
-public class HomeController : Controller
-{
+public class HomeController : Controller {
private readonly ILogger<HomeController> _logger;
- public HomeController(ILogger<HomeController> logger)
- {
+ public HomeController(ILogger<HomeController> logger) {
_logger = logger;
}
diff --git a/ModAS.Server/Controllers/RoomQueryController.cs b/ModAS.Server/Controllers/RoomQueryController.cs
new file mode 100644
index 0000000..a49e5c0
--- /dev/null
+++ b/ModAS.Server/Controllers/RoomQueryController.cs
@@ -0,0 +1,135 @@
+using System.Collections.Frozen;
+using ArcaneLibs.Extensions;
+using Elastic.Apm;
+using Elastic.Apm.Api;
+using LibMatrix;
+using LibMatrix.EventTypes.Spec.State;
+using LibMatrix.EventTypes.Spec.State.RoomInfo;
+using LibMatrix.Responses.ModAS;
+using LibMatrix.RoomTypes;
+using Microsoft.AspNetCore.Mvc;
+using ModAS.Classes;
+using ModAS.Server.Services;
+using MxApiExtensions.Services;
+
+namespace ModAS.Server.Controllers;
+
+[ApiController]
+public class RoomQueryController(UserProviderService ahsProvider, ModASConfiguration config, RoomStateCacheService stateCacheService, UserProviderService userProviderService) : ControllerBase {
+ [HttpGet("/_matrix/_modas/room_query")]
+ public async IAsyncEnumerable<ModASRoomQueryResult> RoomQuery() {
+ var fetchRoomQueryDataSpan = currentTransaction.StartSpan("fetchRoomQueryData", ApiConstants.TypeApp);
+ List<string> processedRooms = new();
+ var getUsersSpan = currentTransaction.StartSpan("getUsers", ApiConstants.TypeApp);
+ var validUsers = await ahsProvider.GetValidUsers();
+ getUsersSpan.End();
+
+ var collectRoomsSpan = currentTransaction.StartSpan("collectRooms", ApiConstants.TypeApp);
+ // var userRoomLists = validUsers.Values.Select(ahs => ahs.GetJoinedRooms()).ToList();
+ // await Task.WhenAll(userRoomLists);
+ // var rooms = userRoomLists.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet();
+ var roomsTasks = validUsers.Values.Select(u => userProviderService.GetUserRoomsCached(u.WhoAmI.UserId)).ToList();
+ await Task.WhenAll(roomsTasks);
+ var rooms = roomsTasks.SelectMany(r => r.Result).DistinctBy(x => x.RoomId).ToFrozenSet();
+ collectRoomsSpan.End();
+
+ var collectRoomStateSpan = currentTransaction.StartSpan("collectRoomState", ApiConstants.TypeApp);
+ //make sure we lock!!!!
+ await stateCacheService.EnsureCachedFromRoomList(rooms);
+ var roomStateTasks = rooms.Select(GetRoomState).ToAsyncEnumerable();
+ var awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp);
+ await foreach (var (room, roomState) in roomStateTasks) {
+ awaitRoomStateSpan.Name = $"awaitRoomState {room.RoomId}";
+ awaitRoomStateSpan.End();
+
+ var filterStateEventsSpan = currentTransaction.StartSpan($"filterStateEvents {room.RoomId}", ApiConstants.TypeApp);
+ var roomMembers = roomState.Where(r => r.Type == RoomMemberEventContent.EventId).ToFrozenSet();
+ var localRoomMembers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName)).ToFrozenSet();
+ var nonMemberState = roomState.Where(x => !roomMembers.Contains(x)).ToFrozenSet();
+ filterStateEventsSpan.End();
+ var buildResultSpan = currentTransaction.StartSpan($"buildResult {room.RoomId}", ApiConstants.TypeApp);
+
+ // forgive me for raw json access... attempt at optimisation -emma
+ yield return new ModASRoomQueryResult {
+ RoomId = room.RoomId,
+ StateEvents = roomState.Count,
+ //members
+ TotalMembers = roomMembers.Count,
+ TotalLocalMembers = localRoomMembers.Count,
+ JoinedMembers = roomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"),
+ JoinedLocalMembers = localRoomMembers.Count(x => (x.TypedContent as RoomMemberEventContent)?.Membership == "join"),
+ //-members
+ //creation event
+ Creator = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.Sender,
+ Version = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["room_version"]?.GetValue<string>(),
+ Type = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["type"]?.GetValue<string>(),
+ Federatable = nonMemberState.FirstOrDefault(r => r.Type == RoomCreateEventContent.EventId)?.RawContent?["m.federate"]?.GetValue<bool>() ?? true,
+ //-creation event
+ Name = nonMemberState.FirstOrDefault(r => r.Type == RoomNameEventContent.EventId)?.RawContent?["name"]?.GetValue<string>(),
+ CanonicalAlias = nonMemberState.FirstOrDefault(r => r.Type == RoomCanonicalAliasEventContent.EventId)?.RawContent?["alias"]?.GetValue<string>(),
+ JoinRules = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue<string>(),
+ GuestAccess = nonMemberState.FirstOrDefault(r => r.Type == RoomGuestAccessEventContent.EventId)?.RawContent?["guest_access"]?.GetValue<string>(),
+ HistoryVisibility =
+ nonMemberState.FirstOrDefault(r => r.Type == RoomHistoryVisibilityEventContent.EventId)?.RawContent?["history_visibility"]?.GetValue<string>(),
+ Public = nonMemberState.FirstOrDefault(r => r.Type == RoomJoinRulesEventContent.EventId)?.RawContent?["join_rule"]?.GetValue<string>() == "public",
+ Encryption = nonMemberState.FirstOrDefault(r => r.Type == RoomEncryptionEventContent.EventId)?.RawContent?["algorithm"]?.GetValue<string>(),
+ AvatarUrl = nonMemberState.FirstOrDefault(r => r.Type == RoomAvatarEventContent.EventId)?.RawContent?["url"]?.GetValue<string>(),
+ RoomTopic = nonMemberState.FirstOrDefault(r => r.Type == RoomTopicEventContent.EventId)?.RawContent?["topic"]?.GetValue<string>()
+ };
+ buildResultSpan.End();
+ awaitRoomStateSpan = currentTransaction.StartSpan("fetchRoomState", ApiConstants.TypeApp);
+ }
+
+ collectRoomStateSpan.End();
+ fetchRoomQueryDataSpan.End();
+ }
+
+ [HttpPost("/_matrix/_modas/room_query")]
+ public async IAsyncEnumerable<ModASRoomQueryResult> RoomQuery([FromBody] RoomQueryFilter request) {
+ await foreach (var room in RoomQuery()) {
+ if (!string.IsNullOrWhiteSpace(request.RoomIdContains) && !room.RoomId.Contains(request.RoomIdContains)) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.NameContains) && room.Name?.Contains(request.NameContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.CanonicalAliasContains) && room.CanonicalAlias?.Contains(request.CanonicalAliasContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.VersionContains) && room.Version?.Contains(request.VersionContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.CreatorContains) && room.Creator?.Contains(request.CreatorContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.EncryptionContains) && room.Encryption?.Contains(request.EncryptionContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.JoinRulesContains) && room.JoinRules?.Contains(request.JoinRulesContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.GuestAccessContains) && room.GuestAccess?.Contains(request.GuestAccessContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.HistoryVisibilityContains) && room.HistoryVisibility?.Contains(request.HistoryVisibilityContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.AvatarUrlContains) && room.AvatarUrl?.Contains(request.AvatarUrlContains) != true) continue;
+
+ if (!string.IsNullOrWhiteSpace(request.RoomTopicContains) && room.RoomTopic?.Contains(request.RoomTopicContains) != true) continue;
+
+ if (request.JoinedMembersMin.HasValue && room.JoinedMembers < request.JoinedMembersMin.Value) continue;
+ if (request.JoinedMembersMax.HasValue && room.JoinedMembers > request.JoinedMembersMax.Value) continue;
+
+ if (request.JoinedLocalMembersMin.HasValue && room.JoinedLocalMembers < request.JoinedLocalMembersMin.Value) continue;
+ if (request.JoinedLocalMembersMax.HasValue && room.JoinedLocalMembers > request.JoinedLocalMembersMax.Value) continue;
+
+ if (request.StateEventsMin.HasValue && room.StateEvents < request.StateEventsMin.Value) continue;
+ if (request.StateEventsMax.HasValue && room.StateEvents > request.StateEventsMax.Value) continue;
+
+ if (request.IsFederatable.HasValue && room.Federatable != request.IsFederatable.Value) continue;
+
+ if (request.IsPublic.HasValue && room.Public != request.IsPublic.Value) continue;
+
+ yield return room;
+ }
+ }
+
+ private async Task<KeyValuePair<GenericRoom, FrozenSet<StateEventResponse>>> GetRoomState(GenericRoom room) {
+ var roomState = await stateCacheService.GetRoomState(room.RoomId, room);
+ return new(room, roomState);
+ }
+
+ private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction;
+}
\ No newline at end of file
diff --git a/ModAS.Server/Controllers/TransactionsController.cs b/ModAS.Server/Controllers/TransactionsController.cs
deleted file mode 100644
index 8e4e018..0000000
--- a/ModAS.Server/Controllers/TransactionsController.cs
+++ /dev/null
@@ -1,15 +0,0 @@
-using System.IO.Pipelines;
-using Microsoft.AspNetCore.Mvc;
-using ModAS.Server;
-
-namespace WebApplication1.Controllers;
-
-[ApiController]
-public class TransactionsController(AppServiceRegistration asr) : ControllerBase {
- [HttpPut(" /_matrix/app/v1/transactions/{txnId}")]
- public async Task<IActionResult> PutTransactions(string txnId) {
- if(!Request.Headers.ContainsKey("Authorization") || Request.Headers["Authorization"] != asr.HomeserverToken) return Unauthorized();
- await Request.Body.CopyToAsync(Console.OpenStandardOutput());
- return Ok(new{});
- }
-}
\ No newline at end of file
diff --git a/ModAS.Server/ModAS.Server.csproj b/ModAS.Server/ModAS.Server.csproj
index 8b48b8a..07cc67c 100644
--- a/ModAS.Server/ModAS.Server.csproj
+++ b/ModAS.Server/ModAS.Server.csproj
@@ -7,9 +7,16 @@
<InvariantGlobalization>true</InvariantGlobalization>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<LangVersion>preview</LangVersion>
+ <CheckForOverflowUnderflow>false</CheckForOverflowUnderflow>
+ <EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
+ <StripSymbols>true</StripSymbols>
+ <OptimizationPreference>Speed</OptimizationPreference>
+ <TieredPGO>true</TieredPGO>
</PropertyGroup>
<ItemGroup>
+ <PackageReference Include="Elastic.Apm.AspNetCore" Version="1.25.2" />
+ <PackageReference Include="Elastic.Apm.NetCoreAll" Version="1.25.2" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.0"/>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Swashbuckle.AspNetCore.ReDoc" Version="6.5.0" />
@@ -17,6 +24,7 @@
<ItemGroup>
<ProjectReference Include="..\LibMatrix\LibMatrix\LibMatrix.csproj" />
+ <ProjectReference Include="..\ModAS.Classes\ModAS.Classes.csproj" />
</ItemGroup>
</Project>
diff --git a/ModAS.Server/Program.cs b/ModAS.Server/Program.cs
index da44ca8..0b3d121 100644
--- a/ModAS.Server/Program.cs
+++ b/ModAS.Server/Program.cs
@@ -3,12 +3,17 @@ using Microsoft.OpenApi.Models;
using ModAS.Server;
using System.Diagnostics;
using System.Text.Json;
+using Elastic.Apm;
+using Elastic.Apm.Api;
+using Elastic.Apm.AspNetCore;
+using Elastic.Apm.NetCoreAll;
using LibMatrix;
using LibMatrix.Services;
using ModAS.Server.Services;
using MxApiExtensions.Services;
-var builder = WebApplication.CreateBuilder(args);
+// var builder = WebApplication.CreateBuilder(args);
+var builder = WebApplication.CreateSlimBuilder(args);
builder.Services.AddControllers().AddJsonOptions(options => { options.JsonSerializerOptions.WriteIndented = true; });
///add wwwroot
@@ -30,7 +35,9 @@ builder.Services.AddSingleton<IHttpContextAccessor, HttpContextAccessor>();
builder.Services.AddSingleton<ModASConfiguration>();
builder.Services.AddSingleton<AuthenticationService>();
-builder.Services.AddSingleton<AuthenticatedHomeserverProviderService>();
+builder.Services.AddSingleton<UserProviderService>();
+builder.Services.AddSingleton<RoomContextService>();
+builder.Services.AddSingleton<RoomStateCacheService>();
// builder.Services.AddScoped<UserContextService>();
builder.Services.AddSingleton<TieredStorageService>(x => {
@@ -82,7 +89,6 @@ builder.Services.AddCors(options => {
var app = builder.Build();
-// Configure the HTTP request pipeline.
// if (app.Environment.IsDevelopment()) {
app.UseSwagger();
app.UseSwaggerUI(c => {
@@ -99,6 +105,15 @@ app.UseReDoc(c => {
});
// }
+app.UseAllElasticApm(builder.Configuration);
+Agent.AddFilter((ISpan span) => {
+ if (span.Context.Http is not null && span.Name.StartsWith($"{span.Context.Http.Method} ")) {
+ span.Name = $"{span.Context.Http.Method} {span.Context.Http.Url}";
+ }
+
+ return span;
+});
+
///wwwroot
app.UseFileServer();
// app.UseStaticFiles();
diff --git a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs b/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs
deleted file mode 100644
index 1ceb095..0000000
--- a/ModAS.Server/Services/AuthenticatedHomeserverProviderService.cs
+++ /dev/null
@@ -1,51 +0,0 @@
-using System.Collections.Concurrent;
-using ArcaneLibs.Extensions;
-using LibMatrix;
-using LibMatrix.Homeservers;
-using LibMatrix.Services;
-using MxApiExtensions.Services;
-
-namespace ModAS.Server.Services;
-
-public class AuthenticatedHomeserverProviderService(
- AuthenticationService authenticationService,
- HomeserverProviderService homeserverProviderService,
- IHttpContextAccessor request,
- ModASConfiguration config,
- AppServiceRegistration asRegistration
- ) {
- public HttpContext? _context = request.HttpContext;
- public ConcurrentDictionary<string, AuthenticatedHomeserverGeneric> KnownUsers { get; set; } = new();
-
- public async Task<AuthenticatedHomeserverGeneric> GetImpersonatedHomeserver(string mxid) {
- if (!KnownUsers.TryGetValue(mxid, out var homeserver)) {
- homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl);
- KnownUsers.TryAdd(mxid, homeserver);
- }
- //var hs = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AsToken, config.HomeserverUrl);
- await homeserver.SetImpersonate(mxid);
- // KnownUsers.TryAdd(mxid, homeserver);
- return homeserver;
- }
-
- public async Task<AuthenticatedHomeserverGeneric> GetHomeserver() {
- var token = authenticationService.GetToken();
- if (token == null) {
- throw new MatrixException {
- ErrorCode = "M_MISSING_TOKEN",
- Error = "Missing access token"
- };
- }
-
- var mxid = await authenticationService.GetMxidFromToken(token);
- if (mxid == "@anonymous:*") {
- throw new MatrixException {
- ErrorCode = "M_MISSING_TOKEN",
- Error = "Missing access token"
- };
- }
-
- var hsCanonical = string.Join(":", mxid.Split(':').Skip(1));
- return await homeserverProviderService.GetAuthenticatedWithToken(hsCanonical, token);
- }
-}
\ No newline at end of file
diff --git a/ModAS.Server/Services/ModASConfiguration.cs b/ModAS.Server/Services/ModASConfiguration.cs
index dc7c552..063e838 100644
--- a/ModAS.Server/Services/ModASConfiguration.cs
+++ b/ModAS.Server/Services/ModASConfiguration.cs
@@ -1,10 +1,10 @@
namespace MxApiExtensions.Services;
-
public class ModASConfiguration {
public ModASConfiguration(IConfiguration configuration) {
configuration.GetRequiredSection("ModAS").Bind(this);
}
+
public string ServerName { get; set; }
public string HomeserverUrl { get; set; }
}
\ No newline at end of file
diff --git a/ModAS.Server/Services/RoomContextService.cs b/ModAS.Server/Services/RoomContextService.cs
new file mode 100644
index 0000000..b3673c1
--- /dev/null
+++ b/ModAS.Server/Services/RoomContextService.cs
@@ -0,0 +1,111 @@
+using ArcaneLibs.Extensions;
+using Elastic.Apm;
+using Elastic.Apm.Api;
+using LibMatrix;
+using LibMatrix.EventTypes.Spec.State;
+using LibMatrix.RoomTypes;
+using MxApiExtensions.Services;
+
+namespace ModAS.Server.Services;
+
+public class RoomContextService(UserProviderService userProvider, ModASConfiguration config) {
+ public Dictionary<string, RoomContext> RoomContexts { get; } = new();
+ public Dictionary<string, List<string>> LocalUsersByRoom { get; private set; } = new();
+
+ public async Task<RoomContext?> GetRoomContext(GenericRoom room) {
+ if (RoomContexts.TryGetValue(room.RoomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext;
+
+ var newRoomContext = await FetchRoomContext(room.RoomId);
+ if (newRoomContext is not null) RoomContexts[room.RoomId] = newRoomContext;
+ return roomContext;
+ }
+
+ public async Task<RoomContext?> GetRoomContext(string roomId) {
+ if (RoomContexts.TryGetValue(roomId, out var roomContext) && !roomContext.NeedsUpdate) return roomContext;
+
+ var newRoomContext = await FetchRoomContext(roomId);
+ if (newRoomContext is not null) RoomContexts[roomId] = newRoomContext;
+ return roomContext;
+ }
+
+ public async Task<RoomContext?> FetchRoomContext(string roomId) {
+ var span = currentTransaction.StartSpan($"FetchRoomContext - {roomId}", ApiConstants.TypeApp);
+ if (!LocalUsersByRoom.ContainsKey(roomId))
+ await UpdateLocalUserRoomLists();
+ if (!LocalUsersByRoom.TryGetValue(roomId, out var localUsers)) return null;
+ if (localUsers.Count == 0) return null;
+
+ var roomContext = new RoomContext {
+ RoomId = roomId
+ };
+
+ var room = (await userProvider.GetImpersonatedHomeserver(localUsers.First())).GetRoom(roomId);
+ var roomMembers = await room.GetMembersListAsync(false);
+
+ roomContext.UserCountByMembership = roomMembers.GroupBy(x => (x.TypedContent as RoomMemberEventContent)?.Membership)
+ .ToDictionary(x => x.Key, x => x.Count());
+ roomContext.LocalUsers = roomMembers.Select(x => x.StateKey).Where(x => x.EndsWith(':' + config.ServerName)).ToList();
+ roomContext.CurrentLocalUsers = roomMembers.Where(x => x.StateKey.EndsWith(':' + config.ServerName) && (x.TypedContent as RoomMemberEventContent)?.Membership == "join")
+ .Select(x => x.StateKey).ToList();
+
+ var powerLevels = await room.GetPowerLevelsAsync();
+ roomContext.LocalUsersByStatePermission = powerLevels?.Events?
+ .Select(@event => (@event.Key, roomContext.CurrentLocalUsers.Where(clu => powerLevels.UserHasStatePermission(clu, @event.Key))))
+ .Where(x => x.Item2.Any())
+ .ToDictionary(x => x.Key, x => x.Item2.ToList());
+
+ roomContext.LastUpdate = DateTime.Now;
+
+ span.End();
+ return roomContext;
+ }
+
+ private async Task UpdateLocalUserRoomLists() {
+ var span = currentTransaction.StartSpan("UpdateLocalUserRoomLists", ApiConstants.TypeApp);
+ var newLocalUsersByRoom = new Dictionary<string, List<string>>();
+ var users = await userProvider.GetValidUsers();
+ var getRoomsSpan = currentTransaction.StartSpan("GetRooms", ApiConstants.TypeApp);
+ var userRoomLists = users.Values.Select(ahs => (ahs, ahs.GetJoinedRooms())).ToList();
+ await Task.WhenAll(userRoomLists.Select(x => x.Item2));
+ getRoomsSpan.End();
+ foreach (var (ahs, rooms) in userRoomLists) {
+ foreach (var room in rooms.Result) {
+ newLocalUsersByRoom.TryAdd(room.RoomId, new List<string>());
+ newLocalUsersByRoom[room.RoomId].Add(ahs.UserId);
+ }
+ }
+ span.End();
+ LocalUsersByRoom = newLocalUsersByRoom;
+ }
+
+ public class RoomContext {
+ public string RoomId { get; set; }
+ public List<string> LocalUsers { get; set; }
+ public List<string> CurrentLocalUsers { get; set; }
+ public Dictionary<string, int> UserCountByMembership { get; set; }
+ public Dictionary<string, List<string>> LocalUsersByStatePermission { get; set; }
+
+ public DateTime LastUpdate { get; set; }
+ public bool NeedsUpdate => DateTime.Now - LastUpdate > TimeSpan.FromMinutes(5);
+ }
+
+ public async Task<GenericRoom?> GetRoomReferenceById(string roomId) {
+ var roomContext = await GetRoomContext(roomId);
+ var localUsers = roomContext.LocalUsers.Select(userProvider.GetImpersonatedHomeserver).ToAsyncEnumerable();
+ await foreach (var localUser in localUsers) {
+ var room = localUser.GetRoom(roomId);
+ try {
+ if (await room.GetCreateEventAsync() is not null)
+ return room;
+ }
+ catch (MatrixException e) {
+ if (e is { ErrorCode: "M_UNAUTHORIZED" }) continue;
+ throw;
+ }
+ }
+
+ return null;
+ }
+
+ private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction;
+}
\ No newline at end of file
diff --git a/ModAS.Server/Services/RoomStateCacheService.cs b/ModAS.Server/Services/RoomStateCacheService.cs
new file mode 100644
index 0000000..3fdf29a
--- /dev/null
+++ b/ModAS.Server/Services/RoomStateCacheService.cs
@@ -0,0 +1,66 @@
+using System.Collections.Frozen;
+using ArcaneLibs.Extensions;
+using Elastic.Apm;
+using Elastic.Apm.Api;
+using LibMatrix;
+using LibMatrix.RoomTypes;
+
+namespace ModAS.Server.Services;
+
+public class RoomStateCacheService(RoomContextService roomContextService) {
+ public FrozenDictionary<string, FrozenSet<StateEventResponse>> RoomStateCache { get; private set; } = FrozenDictionary<string, FrozenSet<StateEventResponse>>.Empty;
+ private SemaphoreSlim updateLock = new(1, 1);
+ public async Task<FrozenSet<StateEventResponse>> GetRoomState(string roomId, GenericRoom? roomReference = null) {
+ if (RoomStateCache.TryGetValue(roomId, out var roomState)) return roomState;
+ return await InvalidateRoomState(roomId, roomReference);
+ }
+
+ public async Task<FrozenSet<StateEventResponse>> InvalidateRoomState(string roomId, GenericRoom? roomReference = null) {
+ var invalidateSpan = currentTransaction.StartSpan($"invalidateRoomState - {roomId}", ApiConstants.TypeApp);
+ var getRoomReferenceSpan = currentTransaction.StartSpan($"getRoomReference - {roomId}", ApiConstants.TypeApp);
+ if (roomReference is null) {
+ var rc = await roomContextService.GetRoomContext(roomId);
+ if (rc is null) return FrozenSet<StateEventResponse>.Empty;
+ roomReference = await roomContextService.GetRoomReferenceById(roomId);
+ }
+
+ if (roomReference is null) {
+ currentTransaction.CaptureException(new Exception("Could not get room reference for room state invalidation"), roomId, true);
+ return FrozenSet<StateEventResponse>.Empty;
+ }
+
+ getRoomReferenceSpan.End();
+
+ var updateSpan = currentTransaction.StartSpan($"updateRoomState - {roomId}", ApiConstants.TypeApp);
+ await updateLock.WaitAsync();
+ var unfrozen = RoomStateCache.ToDictionary();
+ unfrozen[roomId] = (await roomReference.GetFullStateAsListAsync()).ToFrozenSet();
+ RoomStateCache = unfrozen.ToFrozenDictionary();
+ updateSpan.End();
+ updateLock.Release();
+
+ invalidateSpan.End();
+ if (!RoomStateCache.ContainsKey(roomId)) {
+ currentTransaction.CaptureException(new Exception("Room state cache does not contain room after invalidation"), roomId, false);
+ if (!unfrozen.ContainsKey(roomId))
+ currentTransaction.CaptureException(new Exception("Unfrozen room state cache does not contain room after invalidation either..."), roomId, false);
+ }
+
+ return RoomStateCache[roomId];
+ }
+
+ public async Task EnsureCachedFromRoomList(IEnumerable<GenericRoom> rooms) {
+ await updateLock.WaitAsync();
+ var unfrozen = RoomStateCache.ToDictionary();
+
+ var tasks = rooms.Select(async room => {
+ if (RoomStateCache.ContainsKey(room.RoomId)) return;
+ unfrozen[room.RoomId] = (await room.GetFullStateAsListAsync()).ToFrozenSet();
+ }).ToList();
+ await Task.WhenAll(tasks);
+ RoomStateCache = unfrozen.ToFrozenDictionary();
+ updateLock.Release();
+ }
+
+ private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction;
+}
\ No newline at end of file
diff --git a/ModAS.Server/Services/UserProviderService.cs b/ModAS.Server/Services/UserProviderService.cs
new file mode 100644
index 0000000..bb281c4
--- /dev/null
+++ b/ModAS.Server/Services/UserProviderService.cs
@@ -0,0 +1,95 @@
+using System.Collections.Concurrent;
+using ArcaneLibs.Extensions;
+using Elastic.Apm;
+using Elastic.Apm.Api;
+using LibMatrix;
+using LibMatrix.Homeservers;
+using LibMatrix.RoomTypes;
+using LibMatrix.Services;
+using MxApiExtensions.Services;
+
+namespace ModAS.Server.Services;
+
+public class UserProviderService(
+ AuthenticationService authenticationService,
+ HomeserverProviderService homeserverProviderService,
+ IHttpContextAccessor request,
+ ModASConfiguration config,
+ AppServiceRegistration asRegistration
+) {
+ public HttpContext? _context = request.HttpContext;
+
+ private static ITransaction currentTransaction => Agent.Tracer.CurrentTransaction;
+
+ private SemaphoreSlim updateLock = new(1,1);
+
+ public ConcurrentDictionary<string, AuthenticatedHomeserverGeneric> KnownUsers { get; set; } = new();
+ public ConcurrentDictionary<string, DateTime> UserValidationExpiry { get; set; } = new();
+ public ConcurrentDictionary<string, List<string>> CachedUserRooms { get; set; } = new();
+ public ConcurrentDictionary<string, DateTime> CachedUserRoomsExpiry { get; set; } = new();
+
+
+ public async Task<AuthenticatedHomeserverGeneric> GetImpersonatedHomeserver(string mxid) {
+ var span = currentTransaction.StartSpan("GetImpersonatedHomeserver", ApiConstants.TypeApp);
+ if (!KnownUsers.TryGetValue(mxid, out var homeserver)) {
+ var getUserSpan = currentTransaction.StartSpan($"GetUser - {mxid}", ApiConstants.TypeApp);
+ homeserver = await homeserverProviderService.GetAuthenticatedWithToken(config.ServerName, asRegistration.AppServiceToken, config.HomeserverUrl, mxid);
+ KnownUsers.TryAdd(mxid, homeserver);
+ getUserSpan.End();
+ }
+
+ await homeserver.SetImpersonate(mxid);
+ span.End();
+ return homeserver;
+ }
+
+ public async Task<Dictionary<string, AuthenticatedHomeserverGeneric>> GetValidUsers() {
+ var span = currentTransaction.StartSpan("GetValidUsers", ApiConstants.TypeApp);
+ var tasks = KnownUsers.Select(kvp => ValidateUser(kvp.Key, kvp.Value)).ToList();
+ var results = await Task.WhenAll(tasks);
+ var validUsers = results.Where(r => r.Value is not null).ToDictionary(r => r.Key, r => r.Value!);
+ span.End();
+ return validUsers;
+ }
+
+ public async Task<List<GenericRoom>> GetUserRoomsCached(string mxid) {
+ var span = currentTransaction.StartSpan($"GetUserRoomsCached - {mxid}", ApiConstants.TypeApp);
+ var hs = await GetImpersonatedHomeserver(mxid);
+ if (CachedUserRoomsExpiry.TryGetValue(mxid, out var expiry) && expiry > DateTime.Now) {
+ if (CachedUserRooms.TryGetValue(mxid, out var rooms)) {
+ span.End();
+ return rooms.Select(hs.GetRoom).ToList();
+ }
+ }
+
+ var userRooms = await hs.GetJoinedRooms();
+ await updateLock.WaitAsync();
+ CachedUserRooms[mxid] = userRooms.Select(r => r.RoomId).ToList();
+ CachedUserRoomsExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5);
+ updateLock.Release();
+ span.End();
+ return userRooms;
+ }
+
+ private async Task<KeyValuePair<string, AuthenticatedHomeserverGeneric?>> ValidateUser(string mxid, AuthenticatedHomeserverGeneric hs) {
+ if(UserValidationExpiry.TryGetValue(mxid, out var expires))
+ if(DateTime.Now < expires) return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, hs);
+ var span = currentTransaction.StartSpan($"ValidateUser - {mxid}", ApiConstants.TypeApp);
+ try {
+ await hs.GetJoinedRooms();
+ await updateLock.WaitAsync();
+ UserValidationExpiry[mxid] = DateTime.Now + TimeSpan.FromMinutes(5);
+ updateLock.Release();
+ return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, hs);
+ }
+ catch (MatrixException e) {
+ if (e.ErrorCode == "M_FORBIDDEN") {
+ return new KeyValuePair<string, AuthenticatedHomeserverGeneric?>(mxid, null);
+ }
+ throw;
+ }
+ finally {
+ span.End();
+ }
+ }
+}
\ No newline at end of file
diff --git a/ModAS.sln b/ModAS.sln
index dbaf8ce..b4af365 100644
--- a/ModAS.sln
+++ b/ModAS.sln
@@ -6,6 +6,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LibMatrix", "LibMatrix", "{
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix", "LibMatrix\LibMatrix\LibMatrix.csproj", "{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.EventTypes", "LibMatrix\LibMatrix.EventTypes\LibMatrix.EventTypes.csproj", "{D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModAS.Classes", "ModAS.Classes\ModAS.Classes.csproj", "{70CA5BEC-8902-4CDC-A684-50D93B354643}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -20,8 +24,17 @@ Global
{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335}.Release|Any CPU.Build.0 = Release|Any CPU
+ {70CA5BEC-8902-4CDC-A684-50D93B354643}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {70CA5BEC-8902-4CDC-A684-50D93B354643}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {70CA5BEC-8902-4CDC-A684-50D93B354643}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {70CA5BEC-8902-4CDC-A684-50D93B354643}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{5C0EFE64-FCFF-474E-B55E-4DC30E8921D5} = {45E81953-A24E-40B8-AD1E-CD2C51020BE8}
+ {D9679A92-2DA4-47FB-BC4C-D6D65ECA1335} = {45E81953-A24E-40B8-AD1E-CD2C51020BE8}
EndGlobalSection
EndGlobal
|
