diff --git a/LibMatrix/Extensions/CanonicalJsonSerializer.cs b/LibMatrix/Extensions/CanonicalJsonSerializer.cs
index 55a4b1a..ae535aa 100644
--- a/LibMatrix/Extensions/CanonicalJsonSerializer.cs
+++ b/LibMatrix/Extensions/CanonicalJsonSerializer.cs
@@ -1,6 +1,7 @@
using System.Collections.Frozen;
using System.Reflection;
using System.Text.Json;
+using System.Text.Json.Nodes;
using System.Text.Json.Serialization.Metadata;
using ArcaneLibs.Extensions;
@@ -57,6 +58,14 @@ public static class CanonicalJsonSerializer {
// public static String Serialize<TValue>(TValue value, JsonTypeInfo<TValue> jsonTypeInfo) => JsonSerializer.Serialize(value, jsonTypeInfo, _options);
// public static String Serialize(Object value, JsonTypeInfo jsonTypeInfo)
+ public static byte[] SerializeToUtf8Bytes<T>(T value, JsonSerializerOptions? options = null) {
+ var newOptions = MergeOptions(null);
+ return JsonSerializer.SerializeToNode(value, options) // We want to allow passing custom converters for eg. double/float -> string here...
+ .SortProperties()!
+ .CanonicalizeNumbers()!
+ .ToJsonString(newOptions).AsBytes().ToArray();
+ }
+
#endregion
// ReSharper disable once UnusedType.Local
diff --git a/LibMatrix/Extensions/EnumerableExtensions.cs b/LibMatrix/Extensions/EnumerableExtensions.cs
index 4dcf26e..88e79f0 100644
--- a/LibMatrix/Extensions/EnumerableExtensions.cs
+++ b/LibMatrix/Extensions/EnumerableExtensions.cs
@@ -4,7 +4,7 @@ using System.Collections.Immutable;
namespace LibMatrix.Extensions;
public static class EnumerableExtensions {
- public static void MergeStateEventLists(this IList<StateEvent> oldState, IList<StateEvent> newState) {
+ public static void MergeStateEventLists(this IList<MatrixEvent> oldState, IList<MatrixEvent> newState) {
// foreach (var stateEvent in newState) {
// var old = oldState.FirstOrDefault(x => x.Type == stateEvent.Type && x.StateKey == stateEvent.StateKey);
// if (old is null) {
@@ -27,7 +27,7 @@ public static class EnumerableExtensions {
}
}
- int FindIndex(StateEvent needle) {
+ int FindIndex(MatrixEvent needle) {
for (int i = 0; i < oldState.Count; i++) {
var old = oldState[i];
if (old.Type == needle.Type && old.StateKey == needle.StateKey)
@@ -38,7 +38,7 @@ public static class EnumerableExtensions {
}
}
- public static void MergeStateEventLists(this IList<StateEventResponse> oldState, IList<StateEventResponse> newState) {
+ public static void MergeStateEventLists(this IList<MatrixEventResponse> oldState, IList<MatrixEventResponse> newState) {
foreach (var e in newState) {
switch (FindIndex(e)) {
case -1:
@@ -50,7 +50,7 @@ public static class EnumerableExtensions {
}
}
- int FindIndex(StateEventResponse needle) {
+ int FindIndex(MatrixEventResponse needle) {
for (int i = 0; i < oldState.Count; i++) {
var old = oldState[i];
if (old.Type == needle.Type && old.StateKey == needle.StateKey)
@@ -61,7 +61,7 @@ public static class EnumerableExtensions {
}
}
- public static void MergeStateEventLists(this List<StateEventResponse> oldState, List<StateEventResponse> newState) {
+ public static void MergeStateEventLists(this List<MatrixEventResponse> oldState, List<MatrixEventResponse> newState) {
foreach (var e in newState) {
switch (FindIndex(e)) {
case -1:
@@ -73,7 +73,7 @@ public static class EnumerableExtensions {
}
}
- int FindIndex(StateEventResponse needle) {
+ int FindIndex(MatrixEventResponse needle) {
for (int i = 0; i < oldState.Count; i++) {
var old = oldState[i];
if (old.Type == needle.Type && old.StateKey == needle.StateKey)
diff --git a/LibMatrix/Extensions/JsonElementExtensions.cs b/LibMatrix/Extensions/JsonElementExtensions.cs
index dfec95b..9225f58 100644
--- a/LibMatrix/Extensions/JsonElementExtensions.cs
+++ b/LibMatrix/Extensions/JsonElementExtensions.cs
@@ -8,7 +8,7 @@ namespace LibMatrix.Extensions;
public static class JsonElementExtensions {
public static bool FindExtraJsonElementFields(this JsonElement obj, Type objectType, string objectPropertyName) {
if (objectPropertyName == "content" && objectType == typeof(JsonObject))
- objectType = typeof(StateEventResponse);
+ objectType = typeof(MatrixEventResponse);
// if (t == typeof(JsonNode))
// return false;
@@ -35,9 +35,9 @@ public static class JsonElementExtensions {
continue;
}
- if (field.Name == "content" && (objectType == typeof(StateEventResponse) || objectType == typeof(StateEvent))) {
+ if (field.Name == "content" && (objectType == typeof(MatrixEventResponse) || objectType == typeof(MatrixEvent))) {
unknownPropertyFound |= field.FindExtraJsonPropertyFieldsByValueKind(
- StateEvent.GetStateEventType(obj.GetProperty("type").GetString()!), // We expect type to always be present
+ MatrixEvent.GetEventType(obj.GetProperty("type").GetString()!), // We expect type to always be present
mappedProperty.PropertyType);
continue;
}
diff --git a/LibMatrix/Extensions/MatrixHttpClient.Single.cs b/LibMatrix/Extensions/MatrixHttpClient.Single.cs
index baa4a2c..cd82071 100644
--- a/LibMatrix/Extensions/MatrixHttpClient.Single.cs
+++ b/LibMatrix/Extensions/MatrixHttpClient.Single.cs
@@ -4,13 +4,14 @@ using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Net.Sockets;
using System.Reflection;
+using System.Security.Authentication;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
-using ArcaneLibs;
using ArcaneLibs.Extensions;
-using LibMatrix.Homeservers.ImplementationDetails.Synapse.Models.Requests;
namespace LibMatrix.Extensions;
@@ -32,12 +33,13 @@ public class MatrixHttpClient {
};
}
catch (PlatformNotSupportedException e) {
- Console.WriteLine("Failed to create HttpClient with connection pooling, continuing without connection pool!");
- Console.WriteLine("Original exception (safe to ignore!):");
- Console.WriteLine(e);
+ Console.WriteLine("HTTP connection pooling is not supported on this platform, continuing without connection pooling!");
+ // Console.WriteLine("Original exception (safe to ignore!):");
+ // Console.WriteLine(e);
Client = new HttpClient {
- DefaultRequestVersion = new Version(3, 0)
+ DefaultRequestVersion = new Version(3, 0),
+ Timeout = TimeSpan.FromDays(1)
};
}
catch (Exception e) {
@@ -55,6 +57,20 @@ public class MatrixHttpClient {
public Dictionary<string, string> AdditionalQueryParameters { get; set; } = new();
public Uri? BaseAddress { get; set; }
+ public static bool DefaultRetryOnNetworkError { get; set; } = true;
+ public static bool DefaultRetryOnMatrixError { get; set; } = true;
+ public bool RetryOnNetworkError { get; set; } = DefaultRetryOnNetworkError;
+ public bool RetryOnMatrixError { get; set; } = DefaultRetryOnMatrixError;
+
+ public static int DefaultMinRetryIntervalMs { get; set; } = 1000;
+ public static int DefaultMaxRetryIntervalMs { get; set; } = 2000;
+ public static int DefaultMaxRetries { get; set; } = 20;
+
+ public int MinRetryIntervalMs { get; set; } = DefaultMinRetryIntervalMs;
+ public int MaxRetryIntervalMs { get; set; } = DefaultMaxRetryIntervalMs;
+ public int MaxRetries { get; set; } = DefaultMaxRetries;
+
+ private Dictionary<HttpRequestMessage, int> _retries = [];
// default headers, not bound to client
public HttpRequestHeaders DefaultRequestHeaders { get; set; } =
@@ -101,22 +117,22 @@ public class MatrixHttpClient {
responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
catch (Exception e) {
- if (attempt >= 5) {
- Console.WriteLine(
- $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
- throw;
- }
-
- if (e is TaskCanceledException or TimeoutException or HttpRequestException) {
- if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) {
- await Task.Delay(Random.Shared.Next(500, 2500), cancellationToken);
- request.ResetSendStatus();
- return await SendUnhandledAsync(request, cancellationToken, attempt + 1);
- }
- }
- else if (!e.ToString().StartsWith("TypeError: NetworkError"))
- Console.WriteLine(
- $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
+ // if (attempt >= 5) {
+ // Console.WriteLine(
+ // $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
+ // throw;
+ // }
+ //
+ // if (e is TaskCanceledException or TimeoutException or HttpRequestException) {
+ // if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) {
+ // await Task.Delay(Random.Shared.Next(100, 1000), cancellationToken);
+ // request.ResetSendStatus();
+ // return await SendUnhandledAsync(request, cancellationToken, attempt + 1);
+ // }
+ // }
+ // else if (!e.ToString().StartsWith("TypeError: NetworkError"))
+ // Console.WriteLine(
+ // $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
throw;
}
@@ -144,15 +160,73 @@ public class MatrixHttpClient {
"X-Content-Security-Policy",
"Referrer-Policy",
"X-Robots-Tag",
- "Content-Security-Policy"
+ "Content-Security-Policy",
+ "Alt-Svc",
+ // evil
+ "CF-Cache-Status",
+ "CF-Ray",
+ "x-amz-request-id",
+ "x-do-app-origin",
+ "x-do-orig-status",
+ "x-rgw-object-type",
+ "Report-To"
]));
return responseMessage;
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) {
- var responseMessage = await SendUnhandledAsync(request, cancellationToken);
- if (responseMessage.IsSuccessStatusCode) return responseMessage;
+ _retries.TryAdd(request, MaxRetries);
+ HttpResponseMessage responseMessage;
+ try {
+ responseMessage = await SendUnhandledAsync(request, cancellationToken);
+ }
+ catch (HttpRequestException ex) {
+ if (RetryOnNetworkError) {
+ if (_retries[request]-- <= 0) throw;
+ // browser exceptions
+ if (ex.InnerException?.GetType().FullName == "System.Runtime.InteropServices.JavaScript.JSException")
+ Console.WriteLine($"Got JSException, likely a CORS error due to a reverse proxy misconfiguration and error, retrying ({_retries[request]} left)...");
+ // native exceptions
+ else if (ex.InnerException is SocketException sockEx)
+ if (sockEx.SocketErrorCode == SocketError.HostNotFound) {
+ throw new LibMatrixNetworkException(ex) {
+ Error = $"Host {request.RequestUri?.Host ?? "(null)"} not found",
+ ErrorCode = LibMatrixNetworkException.ErrorCodes.RLM_NET_UNKNOWN_HOST
+ };
+ }
+ else { } // empty
+ else if (ex.InnerException is AuthenticationException authEx)
+ if (authEx.Message.Contains("The remote certificate is invalid")) {
+ throw new LibMatrixNetworkException(ex) {
+ Error = ex.Message,
+ ErrorCode = LibMatrixNetworkException.ErrorCodes.RLM_NET_INVALID_REMOTE_CERTIFICATE
+ };
+ }
+ else { } // empty
+
+ else
+ Console.WriteLine(new {
+ ex.HttpRequestError,
+ ex.StatusCode,
+ ex.Data,
+ ex.Message,
+ InnerException = ex.InnerException?.ToString(),
+ InnerExceptionType = ex.InnerException?.GetType().FullName
+ }.ToJson());
+
+ await Task.Delay(Random.Shared.Next(MinRetryIntervalMs, MaxRetryIntervalMs), cancellationToken);
+ request.ResetSendStatus();
+ return await SendAsync(request, cancellationToken);
+ }
+
+ throw;
+ }
+
+ if (responseMessage.IsSuccessStatusCode) {
+ _retries.Remove(request);
+ return responseMessage;
+ }
//retry on gateway timeout
// if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) {
@@ -169,31 +243,46 @@ public class MatrixHttpClient {
};
// if (!content.StartsWith('{')) throw new InvalidDataException("Encountered invalid data:\n" + content);
- if (!content.TrimStart().StartsWith('{')) {
- responseMessage.EnsureSuccessStatusCode();
- throw new InvalidDataException("Encountered invalid data:\n" + content);
- }
- //we have a matrix error
+ if (content.TrimStart().StartsWith('{')) {
+ //we have a matrix error, most likely
+ MatrixException? ex;
+ try {
+ ex = JsonSerializer.Deserialize<MatrixException>(content);
+ }
+ catch (JsonException e) {
+ throw new LibMatrixException() {
+ ErrorCode = "M_INVALID_JSON",
+ Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken)
+ };
+ }
- MatrixException? ex;
- try {
- ex = JsonSerializer.Deserialize<MatrixException>(content);
+ Debug.Assert(ex != null, nameof(ex) + " != null");
+ ex.RawContent = content;
+ // Console.WriteLine($"Failed to send request: {ex}");
+ if (ex.ErrorCode == MatrixException.ErrorCodes.M_LIMIT_EXCEEDED) {
+ // if (ex.RetryAfterMs is null) throw ex!;
+ //we have a ratelimit error
+ await Task.Delay(ex.RetryAfterMs ?? responseMessage.Headers.RetryAfter?.Delta?.Milliseconds ?? MinRetryIntervalMs, cancellationToken);
+ request.ResetSendStatus();
+ return await SendAsync(request, cancellationToken);
+ }
+
+ throw ex;
}
- catch (JsonException e) {
- throw new LibMatrixException() {
- ErrorCode = "M_INVALID_JSON",
- Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken)
- };
+
+ if (responseMessage.StatusCode == HttpStatusCode.BadGateway) {
+ // spread out retries
+ if (RetryOnNetworkError) {
+ if (_retries[request]-- <= 0) throw new InvalidDataException("Encountered invalid data:\n" + content);
+ Console.WriteLine($"Got 502 Bad Gateway, retrying ({_retries[request]} left)...");
+ await Task.Delay(Random.Shared.Next(MinRetryIntervalMs, MaxRetryIntervalMs), cancellationToken);
+ request.ResetSendStatus();
+ return await SendAsync(request, cancellationToken);
+ }
}
- Debug.Assert(ex != null, nameof(ex) + " != null");
- ex.RawContent = content;
- // Console.WriteLine($"Failed to send request: {ex}");
- if (ex.RetryAfterMs is null) throw ex!;
- //we have a ratelimit error
- await Task.Delay(ex.RetryAfterMs.Value, cancellationToken);
- request.ResetSendStatus();
- return await SendAsync(request, cancellationToken);
+ responseMessage.EnsureSuccessStatusCode();
+ throw new InvalidDataException("Encountered invalid data:\n" + content);
}
// GetAsync
@@ -241,22 +330,16 @@ public class MatrixHttpClient {
options = GetJsonSerializerOptions(options);
var request = new HttpRequestMessage(HttpMethod.Put, requestUri);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
- Encoding.UTF8, "application/json");
+ request.Content = JsonContent.Create(value, value.GetType(), new MediaTypeHeaderValue("application/json", "utf-8"), options);
return await SendAsync(request, cancellationToken);
}
public async Task<HttpResponseMessage> PostAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
CancellationToken cancellationToken = default) where T : notnull {
- options ??= new JsonSerializerOptions();
- options.Converters.Add(new JsonFloatStringConverter());
- options.Converters.Add(new JsonDoubleStringConverter());
- options.Converters.Add(new JsonDecimalStringConverter());
- options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+ options = GetJsonSerializerOptions(options);
var request = new HttpRequestMessage(HttpMethod.Post, requestUri);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
- Encoding.UTF8, "application/json");
+ request.Content = JsonContent.Create(value, value.GetType(), new MediaTypeHeaderValue("application/json", "utf-8"), options);
return await SendAsync(request, cancellationToken);
}
@@ -286,9 +369,9 @@ public class MatrixHttpClient {
return await SendAsync(request, cancellationToken);
}
- public async Task DeleteAsync(string url) {
+ public async Task<HttpResponseMessage> DeleteAsync(string url) {
var request = new HttpRequestMessage(HttpMethod.Delete, url);
- await SendAsync(request);
+ return await SendAsync(request);
}
public async Task<HttpResponseMessage> DeleteAsJsonAsync<T>(string url, T payload) {
@@ -297,5 +380,66 @@ public class MatrixHttpClient {
};
return await SendAsync(request);
}
+
+ public async Task<HttpResponseMessage> PostAsyncEnumerableAsJsonAsync<T>(string url, IAsyncEnumerable<T> payload, JsonSerializerOptions? options = null,
+ CancellationToken cancellationToken = default) {
+ options = GetJsonSerializerOptions(options);
+ var request = new HttpRequestMessage(HttpMethod.Post, url);
+ request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+ request.Content = new AsyncEnumerableJsonContent<T>(payload, options);
+ return await SendAsync(request, cancellationToken);
+ }
+
+ public async Task<HttpResponseMessage> PutAsyncEnumerableAsJsonAsync<T>(string url, IAsyncEnumerable<T> payload, JsonSerializerOptions? options = null,
+ CancellationToken cancellationToken = default) {
+ options = GetJsonSerializerOptions(options);
+ var request = new HttpRequestMessage(HttpMethod.Put, url);
+ request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+ request.Content = new AsyncEnumerableJsonContent<T>(payload, options);
+ return await SendAsync(request, cancellationToken);
+ }
+}
+
+public class AsyncEnumerableJsonContent<T>(IAsyncEnumerable<T> payload, JsonSerializerOptions? options) : HttpContent {
+ protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) {
+ await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions {
+ Indented = options?.WriteIndented ?? true,
+ Encoder = options?.Encoder,
+ IndentCharacter = options?.IndentCharacter ?? ' ',
+ IndentSize = options?.IndentSize ?? 4,
+ MaxDepth = options?.MaxDepth ?? 0,
+ NewLine = options?.NewLine ?? Environment.NewLine
+ });
+
+ writer.WriteStartArray();
+ await writer.FlushAsync();
+ await stream.FlushAsync();
+
+ await foreach (var item in payload) {
+ if (item is null) {
+ writer.WriteNullValue();
+ }
+ else {
+ using var memoryStream = new MemoryStream();
+ await JsonSerializer.SerializeAsync(memoryStream, item, item.GetType(), options);
+
+ memoryStream.Position = 0;
+ var jsonBytes = memoryStream.ToArray();
+ writer.WriteRawValue(jsonBytes, skipInputValidation: true);
+ }
+
+ await writer.FlushAsync();
+ await stream.FlushAsync();
+ }
+
+ writer.WriteEndArray();
+ await writer.FlushAsync();
+ await stream.FlushAsync();
+ }
+
+ protected override bool TryComputeLength(out long length) {
+ length = 0;
+ return false;
+ }
}
#endif
\ No newline at end of file
|
