Minor cleanups, support for loading access tokens from disk or appservice github/master

15 files changed, 102 insertions, 25 deletions

diff --git a/ArcaneLibs b/ArcaneLibs
-Subproject b7685c786b29e7f8ae2db6ff0f79a52efc57020
+Subproject 952f5ef673862fdb7ff6f51d032eca4577dab9c
diff --git a/LibMatrix.EventTypes/EventContent.cs b/LibMatrix.EventTypes/EventContent.cs
index c582cf2..d65a47f 100644
--- a/LibMatrix.EventTypes/EventContent.cs
+++ b/LibMatrix.EventTypes/EventContent.cs
@@ -37,6 +37,10 @@ public abstract class TimelineEventContent : EventContent {
         [JsonPropertyName("rel_type")]
         public string? RelationType { get; set; }
 
+        // used for reactions
+        [JsonPropertyName("key")]
+        public string? Key { get; set; }
+
         public class EventInReplyTo {
             [JsonPropertyName("event_id")]
             public string? EventId { get; set; }
diff --git a/LibMatrix.EventTypes/Spec/RoomMessageEventContent.cs b/LibMatrix.EventTypes/Spec/RoomMessageEventContent.cs
index ae893f8..9602bf3 100644
--- a/LibMatrix.EventTypes/Spec/RoomMessageEventContent.cs
+++ b/LibMatrix.EventTypes/Spec/RoomMessageEventContent.cs
@@ -29,6 +29,7 @@ public class RoomMessageEventContent : TimelineEventContent {
     [JsonPropertyName("url")]
     public string? Url { get; set; }
 
+    [JsonPropertyName("filename")]
     public string? FileName { get; set; }
 
     [JsonPropertyName("info")]
diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs
index 39eb7e5..4145a16 100644
--- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs
+++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs
@@ -2,6 +2,7 @@
 // #define SYNC_HTTPCLIENT // Only allow one request as a time, for debugging
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Net;
 using System.Net.Http.Headers;
 using System.Reflection;
 using System.Security.Cryptography.X509Certificates;
@@ -73,12 +74,15 @@ public class MatrixHttpClient {
         await _rateLimitSemaphore.WaitAsync(cancellationToken);
 #endif
 
-        Console.WriteLine($"Sending {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)})");
+        Console.WriteLine($"Sending {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.GetContentLength())})");
 
         if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
         if (!request.RequestUri.IsAbsoluteUri) request.RequestUri = new Uri(BaseAddress, request.RequestUri);
         foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value);
-        foreach (var (key, value) in DefaultRequestHeaders) request.Headers.Add(key, value);
+        foreach (var (key, value) in DefaultRequestHeaders) {
+            if (request.Headers.Contains(key)) continue;
+            request.Headers.Add(key, value);
+        }
 
         request.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse"), true);
 
@@ -106,7 +110,13 @@ public class MatrixHttpClient {
     public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) {
         var responseMessage = await SendUnhandledAsync(request, cancellationToken);
         if (responseMessage.IsSuccessStatusCode) return responseMessage;
-
+        
+        //retry on gateway timeout
+        if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) {
+            request.ResetSendStatus();
+            return await SendAsync(request, cancellationToken);
+        }
+        
         //error handling
         var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken);
         if (content.Length == 0)
diff --git a/LibMatrix/Helpers/MessageBuilder.cs b/LibMatrix/Helpers/MessageBuilder.cs
index d897078..b639e1f 100644
--- a/LibMatrix/Helpers/MessageBuilder.cs
+++ b/LibMatrix/Helpers/MessageBuilder.cs
@@ -91,6 +91,18 @@ public class MessageBuilder(string msgType = "m.text", string format = "org.matr
         return this;
     }
 
+    public MessageBuilder WithMention(string id, string? displayName = null) {
+        Content.Body += $"@{displayName ?? id}";
+        Content.FormattedBody += $"<a href=\"https://matrix.to/#/{id}\">{displayName ?? id}</a>";
+        return this;
+    }
+
+    public MessageBuilder WithNewline() {
+        Content.Body += "\n";
+        Content.FormattedBody += "<br>";
+        return this;
+    }
+
     public MessageBuilder WithTable(Action<TableBuilder> tableBuilder) {
         var tb = new TableBuilder(this);
         this.WithHtmlTag("table", msb => tableBuilder(tb));
diff --git a/LibMatrix/Helpers/SyncHelper.cs b/LibMatrix/Helpers/SyncHelper.cs
index 1833bd0..c9ca85d 100644
--- a/LibMatrix/Helpers/SyncHelper.cs
+++ b/LibMatrix/Helpers/SyncHelper.cs
@@ -4,6 +4,7 @@ using ArcaneLibs.Extensions;
 using LibMatrix.Filters;
 using LibMatrix.Homeservers;
 using LibMatrix.Responses;
+using LibMatrix.Utilities;
 using Microsoft.Extensions.Logging;
 
 namespace LibMatrix.Helpers;
@@ -42,6 +43,7 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg
             _filter = value;
             _filterIsDirty = true;
             _filterId = null;
+            _namedFilterName = null;
         }
     }
 
@@ -81,16 +83,16 @@ public class SyncHelper(AuthenticatedHomeserverGeneric homeserver, ILogger? logg
         if (!string.IsNullOrWhiteSpace(Since)) url += $"&since={Since}";
         if (_filterId is not null) url += $"&filter={_filterId}";
 
-        logger?.LogInformation("SyncHelper: Calling: {}", url);
+        // logger?.LogInformation("SyncHelper: Calling: {}", url);
 
         try {
             var httpResp = await homeserver.ClientHttpClient.GetAsync(url, cancellationToken ?? CancellationToken.None);
             if (httpResp is null) throw new NullReferenceException("Failed to send HTTP request");
-            logger?.LogInformation("Got sync response: {} bytes, {} elapsed", httpResp.Content.Headers.ContentLength ?? -1, sw.Elapsed);
+            logger?.LogTrace("Got sync response: {} bytes, {} elapsed", httpResp.GetContentLength(), sw.Elapsed);
             var deserializeSw = Stopwatch.StartNew();
             var resp = await httpResp.Content.ReadFromJsonAsync<SyncResponse>(cancellationToken: cancellationToken ?? CancellationToken.None,
                 jsonTypeInfo: SyncResponseSerializerContext.Default.SyncResponse);
-            logger?.LogInformation("Deserialized sync response: {} bytes, {} elapsed, {} total", httpResp.Content.Headers.ContentLength ?? -1, deserializeSw.Elapsed, sw.Elapsed);
+            logger?.LogInformation("Deserialized sync response: {} bytes, {} elapsed, {} total", httpResp.GetContentLength(), deserializeSw.Elapsed, sw.Elapsed);
             var timeToWait = MinimumDelay.Subtract(sw.Elapsed);
             if (timeToWait.TotalMilliseconds > 0)
                 await Task.Delay(timeToWait);
diff --git a/LibMatrix/LibMatrix.csproj b/LibMatrix/LibMatrix.csproj
index 1e3a8aa..6158ff8 100644
--- a/LibMatrix/LibMatrix.csproj
+++ b/LibMatrix/LibMatrix.csproj
@@ -27,8 +27,10 @@
         <ProjectReference Include="..\LibMatrix.EventTypes\LibMatrix.EventTypes.csproj"/>
     </ItemGroup>
 
+    <!-- 
     <Target Name="ArcaneLibsNugetWarning" AfterTargets="AfterBuild">
         <Warning Text="ArcaneLibs is being referenced from NuGet, which is dangerous. Please read the warning in LibMatrix.csproj!" Condition="!Exists('..\ArcaneLibs\ArcaneLibs\ArcaneLibs.csproj')"/>
     </Target>
+    -->
 
 </Project>
diff --git a/LibMatrix/Responses/SyncResponse.cs b/LibMatrix/Responses/SyncResponse.cs
index e4addb6..b2308c5 100644
--- a/LibMatrix/Responses/SyncResponse.cs
+++ b/LibMatrix/Responses/SyncResponse.cs
@@ -39,7 +39,7 @@ public class SyncResponse {
     // supporting classes
     public class PresenceDataStructure {
         [JsonPropertyName("events")]
-        public List<StateEventResponse> Events { get; set; } = new();
+        public List<StateEventResponse>? Events { get; set; }
     }
 
     public class RoomsDataStructure {
diff --git a/LibMatrix/RoomTypes/GenericRoom.cs b/LibMatrix/RoomTypes/GenericRoom.cs
index 2ec8571..8398ab9 100644
--- a/LibMatrix/RoomTypes/GenericRoom.cs
+++ b/LibMatrix/RoomTypes/GenericRoom.cs
@@ -431,6 +431,16 @@ public class GenericRoom {
 
         return await res.Content.ReadFromJsonAsync<T>();
     }
+    
+    public async Task<T?> GetRoomAccountDataOrNullAsync<T>(string key) {
+        try {
+            return await GetRoomAccountDataAsync<T>(key);
+        }
+        catch (MatrixException e) {
+            if (e.ErrorCode == "M_NOT_FOUND") return default;
+            throw;
+        }
+    }
 
     public async Task SetRoomAccountDataAsync(string key, object data) {
         var res = await Homeserver.ClientHttpClient.PutAsJsonAsync($"/_matrix/client/v3/user/{Homeserver.UserId}/rooms/{RoomId}/account_data/{key}", data);
@@ -443,10 +453,17 @@ public class GenericRoom {
     public Task<StateEventResponse> GetEventAsync(string eventId) =>
         Homeserver.ClientHttpClient.GetFromJsonAsync<StateEventResponse>($"/_matrix/client/v3/rooms/{RoomId}/event/{eventId}");
 
-    public async Task<EventIdResponse> RedactEventAsync(string eventToRedact, string reason) {
+    public async Task<EventIdResponse> RedactEventAsync(string eventToRedact, string? reason = null) {
         var data = new { reason };
-        return (await (await Homeserver.ClientHttpClient.PutAsJsonAsync(
-            $"/_matrix/client/v3/rooms/{RoomId}/redact/{eventToRedact}/{Guid.NewGuid()}", data)).Content.ReadFromJsonAsync<EventIdResponse>())!;
+        var url = $"/_matrix/client/v3/rooms/{RoomId}/redact/{eventToRedact}/{Guid.NewGuid().ToString()}";
+        while (true) {
+            try {
+                return (await (await Homeserver.ClientHttpClient.PutAsJsonAsync(url, data)).Content.ReadFromJsonAsync<EventIdResponse>())!;
+            } catch (MatrixException e) {
+                if (e is { ErrorCode: MatrixException.ErrorCodes.M_FORBIDDEN }) throw;
+                throw;
+            }
+        }
     }
 
 #endregion
diff --git a/LibMatrix/Services/ServiceInstaller.cs b/LibMatrix/Services/ServiceInstaller.cs
index 06ea9de..8b7e54b 100644
--- a/LibMatrix/Services/ServiceInstaller.cs
+++ b/LibMatrix/Services/ServiceInstaller.cs
@@ -5,23 +5,13 @@ namespace LibMatrix.Services;
 
 public static class ServiceInstaller {
     public static IServiceCollection AddRoryLibMatrixServices(this IServiceCollection services, RoryLibMatrixConfiguration? config = null) {
-        //Check required services
-        // if (!services.Any(x => x.ServiceType == typeof(TieredStorageService)))
-        // throw new Exception("[RMUCore/DI] No TieredStorageService has been registered!");
         //Add config
         services.AddSingleton(config ?? new RoryLibMatrixConfiguration());
 
         //Add services
         services.AddSingleton<HomeserverResolverService>(sp => new HomeserverResolverService(sp.GetRequiredService<ILogger<HomeserverResolverService>>()));
-
-        // if (services.First(x => x.ServiceType == typeof(TieredStorageService)).Lifetime == ServiceLifetime.Singleton) {
         services.AddSingleton<HomeserverProviderService>();
-        // }
-        // else {
-        // services.AddScoped<HomeserverProviderService>();
-        // }
 
-        // services.AddScoped<MatrixHttpClient>();
         return services;
     }
 }
diff --git a/LibMatrix/StateEvent.cs b/LibMatrix/StateEvent.cs
index 8b44d2c..cc870e4 100644
--- a/LibMatrix/StateEvent.cs
+++ b/LibMatrix/StateEvent.cs
@@ -166,7 +166,7 @@ public class StateEventResponse : StateEvent {
     public string? Sender { get; set; }
 
     [JsonPropertyName("unsigned")]
-    public UnsignedData? Unsigned { get; set; }
+    public JsonObject? Unsigned { get; set; }
 
     [JsonPropertyName("event_id")]
     public string? EventId { get; set; }
diff --git a/Utilities/LibMatrix.TestDataGenerator/Program.cs b/Utilities/LibMatrix.TestDataGenerator/Program.cs
index 2583817..f3750a8 100644
--- a/Utilities/LibMatrix.TestDataGenerator/Program.cs
+++ b/Utilities/LibMatrix.TestDataGenerator/Program.cs
@@ -2,6 +2,7 @@
 
 using LibMatrix.Services;
 using LibMatrix.Utilities.Bot;
+using LibMatrix.Utilities.Bot.AppServices;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using TestDataGenerator.Bot;
diff --git a/Utilities/LibMatrix.Utilities.Bot/AppServiceConfiguration.cs b/Utilities/LibMatrix.Utilities.Bot/AppServices/AppServiceConfiguration.cs
index afda89e..2cfcf32 100644
--- a/Utilities/LibMatrix.Utilities.Bot/AppServiceConfiguration.cs
+++ b/Utilities/LibMatrix.Utilities.Bot/AppServices/AppServiceConfiguration.cs
@@ -1,23 +1,47 @@
-namespace LibMatrix.Utilities.Bot;
+using System.Text.Json.Serialization;
+
+namespace LibMatrix.Utilities.Bot.AppServices;
 
 public class AppServiceConfiguration {
+    [JsonPropertyName("id")]
     public string Id { get; set; } = null!;
+
+    [JsonPropertyName("url")]
     public string? Url { get; set; } = null!;
+
+    [JsonPropertyName("sender_localpart")]
     public string SenderLocalpart { get; set; } = null!;
+
+    [JsonPropertyName("as_token")]
     public string AppserviceToken { get; set; } = null!;
+
+    [JsonPropertyName("hs_token")]
     public string HomeserverToken { get; set; } = null!;
+
+    [JsonPropertyName("protocols")]
     public List<string>? Protocols { get; set; } = null!;
+
+    [JsonPropertyName("rate_limited")]
     public bool? RateLimited { get; set; } = null!;
 
+    [JsonPropertyName("namespaces")]
     public AppserviceNamespaces Namespaces { get; set; } = null!;
 
     public class AppserviceNamespaces {
+        [JsonPropertyName("users")]
         public List<AppserviceNamespace>? Users { get; set; } = null;
+
+        [JsonPropertyName("aliases")]
         public List<AppserviceNamespace>? Aliases { get; set; } = null;
+
+        [JsonPropertyName("rooms")]
         public List<AppserviceNamespace>? Rooms { get; set; } = null;
 
         public class AppserviceNamespace {
+            [JsonPropertyName("exclusive")]
             public bool Exclusive { get; set; }
+
+            [JsonPropertyName("regex")]
             public string Regex { get; set; } = null!;
         }
     }
diff --git a/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs b/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs
index 621c1ee..ca6a4d8 100644
--- a/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs
+++ b/Utilities/LibMatrix.Utilities.Bot/BotCommandInstaller.cs
@@ -1,8 +1,7 @@
 using ArcaneLibs;
-using LibMatrix.EventTypes.Spec.State;
 using LibMatrix.Homeservers;
-using LibMatrix.Responses;
 using LibMatrix.Services;
+using LibMatrix.Utilities.Bot.AppServices;
 using LibMatrix.Utilities.Bot.Interfaces;
 using LibMatrix.Utilities.Bot.Services;
 using Microsoft.Extensions.DependencyInjection;
@@ -22,6 +21,20 @@ public class BotInstaller(IServiceCollection services) {
         services.AddScoped<AuthenticatedHomeserverGeneric>(x => {
             var config = x.GetService<LibMatrixBotConfiguration>() ?? throw new Exception("No configuration found!");
             var hsProvider = x.GetService<HomeserverProviderService>() ?? throw new Exception("No homeserver provider found!");
+            
+            if (x.GetService<AppServiceConfiguration>() is AppServiceConfiguration appsvcConfig)
+                config.AccessToken = appsvcConfig.AppserviceToken;
+            else if (Environment.GetEnvironmentVariable("LIBMATRIX_ACCESS_TOKEN_PATH") is string path)
+                config.AccessTokenPath = path;
+            
+            if(string.IsNullOrWhiteSpace(config.AccessToken) && string.IsNullOrWhiteSpace(config.AccessTokenPath))
+                throw new Exception("Unable to add bot service without an access token or access token path!");
+            
+            if(!string.IsNullOrWhiteSpace(config.AccessTokenPath)) {
+                var token = File.ReadAllText(config.AccessTokenPath);
+                config.AccessToken = token;
+            }
+            
             var hs = hsProvider.GetAuthenticatedWithToken(config.Homeserver, config.AccessToken).Result;
 
             return hs;
diff --git a/Utilities/LibMatrix.Utilities.Bot/LibMatrixBotConfiguration.cs b/Utilities/LibMatrix.Utilities.Bot/LibMatrixBotConfiguration.cs
index 245442f..728b169 100644
--- a/Utilities/LibMatrix.Utilities.Bot/LibMatrixBotConfiguration.cs
+++ b/Utilities/LibMatrix.Utilities.Bot/LibMatrixBotConfiguration.cs
@@ -6,7 +6,8 @@ namespace LibMatrix.Utilities.Bot;
 public class LibMatrixBotConfiguration {
     public LibMatrixBotConfiguration(IConfiguration config) => config.GetRequiredSection("LibMatrixBot").Bind(this);
     public string Homeserver { get; set; }
-    public string AccessToken { get; set; }
+    public string? AccessToken { get; set; }
+    public string? AccessTokenPath { get; set; }
     public List<string> Prefixes { get; set; }
     public bool MentionPrefix { get; set; }
     public string? LogRoom { get; set; }