diff --git a/extra/admin-api/Utilities/DiscordEmojiConverter/DiscordEmojiConverter.csproj b/extra/admin-api/Utilities/DiscordEmojiConverter/DiscordEmojiConverter.csproj
index 72d306b3..7ed79afd 100644
--- a/extra/admin-api/Utilities/DiscordEmojiConverter/DiscordEmojiConverter.csproj
+++ b/extra/admin-api/Utilities/DiscordEmojiConverter/DiscordEmojiConverter.csproj
@@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="ArcaneLibs" Version="1.0.0-preview.20251207-164820" />
+ <PackageReference Include="ArcaneLibs" Version="1.0.1-preview.20260126-091403" />
</ItemGroup>
</Project>
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Spacebar.AdminApi.PrepareTestData.csproj b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Spacebar.AdminApi.PrepareTestData.csproj
index 5daabdd0..546875d2 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Spacebar.AdminApi.PrepareTestData.csproj
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Spacebar.AdminApi.PrepareTestData.csproj
@@ -10,7 +10,7 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="ArcaneLibs" Version="1.0.0-preview.20251207-164820" />
+ <PackageReference Include="ArcaneLibs" Version="1.0.1-preview.20260126-091403" />
</ItemGroup>
</Project>
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Guilds.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Guilds.razor
index f8cc15dd..b105968e 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Guilds.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Guilds.razor
@@ -1,6 +1,7 @@
@page "/Guilds"
@using System.Net.Http.Headers
@using System.Reflection
+@using ArcaneLibs
@using Spacebar.Models.AdminApi
@using Spacebar.AdminApi.TestClient.Services
@using ArcaneLibs.Blazor.Components
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/HttpTestClient.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/HttpTestClient.razor
index b70f4efb..91aec659 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/HttpTestClient.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/HttpTestClient.razor
@@ -1,5 +1,6 @@
@page "/HttpTestClient"
@using System.Text.Json
+@using ArcaneLibs
@using ArcaneLibs.Blazor.Components
@using ArcaneLibs.Extensions
@using Spacebar.AdminApi.TestClient.Classes.OpenAPI
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/NonAdmin/StickerManager.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/NonAdmin/StickerManager.razor
index 73ce4ec2..3e613104 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/NonAdmin/StickerManager.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/NonAdmin/StickerManager.razor
@@ -3,6 +3,7 @@
@using System.Diagnostics.CodeAnalysis
@using System.Net.Http.Headers
@using System.Text.Json.Serialization
+@using ArcaneLibs
@using ArcaneLibs.Blazor.Components
@using ArcaneLibs.Blazor.Components.Services
@using ArcaneLibs.Extensions
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/ServerConfig.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/ServerConfig.razor
index 7a83bd52..4ecbf9d3 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/ServerConfig.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/ServerConfig.razor
@@ -2,6 +2,7 @@
@inject Config Config
@using System.Net.Http.Headers
@using System.Text.Json
+@using ArcaneLibs
@using Spacebar.AdminApi.TestClient.Services
@using Spacebar.ConfigModel.Extensions
<h3>Server Config</h3>
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Users.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Users.razor
index 939c5dca..fa8ad23a 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Users.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/Users.razor
@@ -1,6 +1,7 @@
@page "/Users"
@using System.Net.Http.Headers
@using System.Reflection
+@using ArcaneLibs
@using Spacebar.Models.AdminApi
@using Spacebar.AdminApi.TestClient.Services
@using ArcaneLibs.Blazor.Components
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/UsersDelete.razor b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/UsersDelete.razor
index b2116ebd..e9a59f3b 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/UsersDelete.razor
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Pages/UsersDelete.razor
@@ -2,6 +2,7 @@
@using System.Net.Http.Headers
@using System.Text.Json
@using System.Text.Json.Nodes
+@using ArcaneLibs
@using ArcaneLibs.Extensions
@using Spacebar.Models.AdminApi
@using Spacebar.AdminApi.TestClient.Services
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Services/StreamingHttpClient.cs b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Services/StreamingHttpClient.cs
deleted file mode 100644
index 311eb457..00000000
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Services/StreamingHttpClient.cs
+++ /dev/null
@@ -1,304 +0,0 @@
-#define SINGLE_HTTPCLIENT // Use a single HttpClient instance for all MatrixHttpClient instances
-// #define SYNC_HTTPCLIENT // Only allow one request as a time, for debugging
-using System.Data;
-using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
-using System.Net.Http.Headers;
-using System.Reflection;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using ArcaneLibs;
-using ArcaneLibs.Extensions;
-
-namespace Spacebar.AdminApi.TestClient.Services;
-
-#if SINGLE_HTTPCLIENT
-// TODO: Add URI wrapper for
-public class StreamingHttpClient {
- private static readonly HttpClient Client;
-
- static StreamingHttpClient() {
- try {
- var handler = new SocketsHttpHandler {
- PooledConnectionLifetime = TimeSpan.FromMinutes(15),
- MaxConnectionsPerServer = 4096,
- EnableMultipleHttp2Connections = true
- };
- Client = new HttpClient(handler) {
- DefaultRequestVersion = new Version(3, 0),
- Timeout = TimeSpan.FromDays(1)
- };
- }
- catch (PlatformNotSupportedException e) {
- Console.WriteLine("Failed to create HttpClient with connection pooling, continuing without connection pool!");
- Console.WriteLine("Original exception (safe to ignore!):");
- Console.WriteLine(e);
-
- Client = new HttpClient {
- DefaultRequestVersion = new Version(3, 0)
- };
- }
- catch (Exception e) {
- Console.WriteLine("Failed to create HttpClient:");
- Console.WriteLine(e);
- throw;
- }
- }
-
-#if SYNC_HTTPCLIENT
- internal SemaphoreSlim _rateLimitSemaphore { get; } = new(1, 1);
-#endif
-
- public static bool LogRequests = true;
- public Dictionary<string, string> AdditionalQueryParameters { get; set; } = new();
-
- public Uri? BaseAddress { get; set; }
-
- // default headers, not bound to client
- public HttpRequestHeaders DefaultRequestHeaders { get; set; } =
- typeof(HttpRequestHeaders).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, [], null)?.Invoke([]) as HttpRequestHeaders ??
- throw new InvalidOperationException("Failed to create HttpRequestHeaders");
-
- private static JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) {
- options ??= new JsonSerializerOptions();
- // options.Converters.Add(new JsonFloatStringConverter());
- // options.Converters.Add(new JsonDoubleStringConverter());
- // options.Converters.Add(new JsonDecimalStringConverter());
- options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
- return options;
- }
-
- public async Task<HttpResponseMessage> SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) {
- if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
- // if (!request.RequestUri.IsAbsoluteUri)
- request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!);
- var swWait = Stopwatch.StartNew();
-#if SYNC_HTTPCLIENT
- await _rateLimitSemaphore.WaitAsync(cancellationToken);
-#endif
-
- if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
- if (!request.RequestUri.IsAbsoluteUri)
- request.RequestUri = new Uri(BaseAddress ?? throw new InvalidOperationException("Relative URI passed, but no BaseAddress is specified!"), request.RequestUri);
- swWait.Stop();
- var swExec = Stopwatch.StartNew();
-
- foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value);
- foreach (var (key, value) in DefaultRequestHeaders) {
- if (request.Headers.Contains(key)) continue;
- request.Headers.Add(key, value);
- }
-
- request.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse"), true);
-
- if (LogRequests)
- Console.WriteLine("Sending " + request.Summarise(includeHeaders: true, includeQuery: true, includeContentIfText: false, hideHeaders: ["Accept"]));
-
- HttpResponseMessage? responseMessage;
- try {
- responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
- }
- catch (Exception e) {
- if (e is TaskCanceledException or TimeoutException) {
- if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) {
- await Task.Delay(Random.Shared.Next(500, 2500), cancellationToken);
- request.ResetSendStatus();
- return await SendAsync(request, cancellationToken);
- }
- }
- else if (!e.ToString().StartsWith("TypeError: NetworkError"))
- Console.WriteLine(
- $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
-
- throw;
- }
-#if SYNC_HTTPCLIENT
- finally {
- _rateLimitSemaphore.Release();
- }
-#endif
-
- // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)");
- if (LogRequests)
- Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [
- "Server",
- "Date",
- "Transfer-Encoding",
- "Connection",
- "Vary",
- "Content-Length",
- "Access-Control-Allow-Origin",
- "Access-Control-Allow-Methods",
- "Access-Control-Allow-Headers",
- "Access-Control-Expose-Headers",
- "Cache-Control",
- "Cross-Origin-Resource-Policy",
- "X-Content-Security-Policy",
- "Referrer-Policy",
- "X-Robots-Tag",
- "Content-Security-Policy"
- ]));
-
- return responseMessage;
- }
-
- public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) {
- var responseMessage = await SendUnhandledAsync(request, cancellationToken);
- if (responseMessage.IsSuccessStatusCode) return responseMessage;
-
- //retry on gateway timeout
- // if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) {
- // request.ResetSendStatus();
- // return await SendAsync(request, cancellationToken);
- // }
-
- //error handling
- var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken);
- if (content.Length == 0)
- throw new DataException("Content was empty");
- // throw new MatrixException() {
- // ErrorCode = "M_UNKNOWN",
- // Error = "Unknown error, server returned no content"
- // };
-
- // if (!content.StartsWith('{')) throw new InvalidDataException("Encountered invalid data:\n" + content);
- if (!content.TrimStart().StartsWith('{')) {
- responseMessage.EnsureSuccessStatusCode();
- throw new InvalidDataException("Encountered invalid data:\n" + content);
- }
- //we have a matrix error
-
- throw new Exception("Unknown http exception");
- // MatrixException? ex;
- // try {
- // ex = JsonSerializer.Deserialize<MatrixException>(content);
- // }
- // catch (JsonException e) {
- // throw new LibMatrixException() {
- // ErrorCode = "M_INVALID_JSON",
- // Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken)
- // };
- // }
- //
- // Debug.Assert(ex != null, nameof(ex) + " != null");
- // ex.RawContent = content;
- // // Console.WriteLine($"Failed to send request: {ex}");
- // if (ex.RetryAfterMs is null) throw ex!;
- // //we have a ratelimit error
- // await Task.Delay(ex.RetryAfterMs.Value, cancellationToken);
- request.ResetSendStatus();
- return await SendAsync(request, cancellationToken);
- }
-
- // GetAsync
- public Task<HttpResponseMessage> GetAsync([StringSyntax("Uri")] string? requestUri, CancellationToken? cancellationToken = null) =>
- SendAsync(new HttpRequestMessage(HttpMethod.Get, requestUri), cancellationToken ?? CancellationToken.None);
-
- // GetFromJsonAsync
- public async Task<T?> TryGetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
- try {
- return await GetFromJsonAsync<T>(requestUri, options, cancellationToken);
- }
- catch (JsonException e) {
- Console.WriteLine($"Failed to deserialize response from {requestUri}: {e.Message}");
- return default;
- }
- catch (HttpRequestException e) {
- Console.WriteLine($"Failed to get {requestUri}: {e.Message}");
- return default;
- }
- }
-
- public async Task<T> GetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
- options = GetJsonSerializerOptions(options);
- var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
- request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- var response = await SendAsync(request, cancellationToken);
- response.EnsureSuccessStatusCode();
- await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken);
-
- return await JsonSerializer.DeserializeAsync<T>(responseStream, options, cancellationToken) ??
- throw new InvalidOperationException("Failed to deserialize response");
- }
-
- // GetStreamAsync
- public async Task<Stream> GetStreamAsync(string requestUri, CancellationToken cancellationToken = default) {
- var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
- request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- var response = await SendAsync(request, cancellationToken);
- response.EnsureSuccessStatusCode();
- return await response.Content.ReadAsStreamAsync(cancellationToken);
- }
-
- public async Task<HttpResponseMessage> PutAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
- CancellationToken cancellationToken = default) where T : notnull {
- options = GetJsonSerializerOptions(options);
- var request = new HttpRequestMessage(HttpMethod.Put, requestUri);
- request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
- Encoding.UTF8, "application/json");
- return await SendAsync(request, cancellationToken);
- }
-
- public async Task<HttpResponseMessage> PostAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
- CancellationToken cancellationToken = default) where T : notnull {
- options ??= new JsonSerializerOptions();
- // options.Converters.Add(new JsonFloatStringConverter());
- // options.Converters.Add(new JsonDoubleStringConverter());
- // options.Converters.Add(new JsonDecimalStringConverter());
- options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
- var request = new HttpRequestMessage(HttpMethod.Post, requestUri);
- request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
- request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
- Encoding.UTF8, "application/json");
- return await SendAsync(request, cancellationToken);
- }
-
- public async IAsyncEnumerable<T?> GetAsyncEnumerableFromJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, JsonSerializerOptions? options = null) {
- options = GetJsonSerializerOptions(options);
- var res = await GetAsync(requestUri);
- options.PropertyNameCaseInsensitive = true;
- var result = JsonSerializer.DeserializeAsyncEnumerable<T>(await res.Content.ReadAsStreamAsync(), options);
- await foreach (var resp in result) yield return resp;
- }
-
- public static async Task<bool> CheckSuccessStatus(string url) {
- //cors causes failure, try to catch
- try {
- var resp = await Client.GetAsync(url);
- return resp.IsSuccessStatusCode;
- }
- catch (Exception e) {
- Console.WriteLine($"Failed to check success status: {e.Message}");
- return false;
- }
- }
-
- public async Task<HttpResponseMessage> PostAsync(string uri, HttpContent? content, CancellationToken cancellationToken = default) {
- var request = new HttpRequestMessage(HttpMethod.Post, uri) {
- Content = content
- };
- return await SendAsync(request, cancellationToken);
- }
-
- public async Task<HttpResponseMessage> DeleteAsync(string url) {
- var request = new HttpRequestMessage(HttpMethod.Delete, url);
- return await SendAsync(request);
- }
-
- public async Task<HttpResponseMessage> DeleteAsJsonAsync<T>(string url, T payload) {
- var request = new HttpRequestMessage(HttpMethod.Delete, url) {
- Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
- };
- return await SendAsync(request);
- }
-
- public async Task<HttpResponseMessage> PatchAsJsonAsync<T>(string url, T payload) {
- var request = new HttpRequestMessage(new HttpMethod("PATCH"), url) {
- Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
- };
- return await SendAsync(request);
- }
-}
-#endif
\ No newline at end of file
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Spacebar.AdminApi.TestClient.csproj b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Spacebar.AdminApi.TestClient.csproj
index 552dbd3e..d1656c63 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Spacebar.AdminApi.TestClient.csproj
+++ b/extra/admin-api/Utilities/Spacebar.AdminApi.TestClient/Spacebar.AdminApi.TestClient.csproj
@@ -18,8 +18,8 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="ArcaneLibs" Version="1.0.0-preview.20251207-164820" />
- <PackageReference Include="ArcaneLibs.Blazor.Components" Version="1.0.0-preview.20251207-164820" />
+ <PackageReference Include="ArcaneLibs" Version="1.0.1-preview.20260126-091403" />
+ <PackageReference Include="ArcaneLibs.Blazor.Components" Version="1.0.1-preview.20260126-091403" />
<PackageReference Include="Blazored.LocalStorage" Version="4.5.0" />
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.2" />
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="10.0.2" PrivateAssets="all" />
diff --git a/extra/admin-api/Utilities/Spacebar.AdminApiTest/Spacebar.AdminApiTest.csproj b/extra/admin-api/Utilities/Spacebar.AdminApiTest/Spacebar.AdminApiTest.csproj
index 5daabdd0..546875d2 100644
--- a/extra/admin-api/Utilities/Spacebar.AdminApiTest/Spacebar.AdminApiTest.csproj
+++ b/extra/admin-api/Utilities/Spacebar.AdminApiTest/Spacebar.AdminApiTest.csproj
@@ -10,7 +10,7 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="ArcaneLibs" Version="1.0.0-preview.20251207-164820" />
+ <PackageReference Include="ArcaneLibs" Version="1.0.1-preview.20260126-091403" />
</ItemGroup>
</Project>
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/FsckService.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/FsckService.cs
new file mode 100644
index 00000000..c0b0bbc3
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/FsckService.cs
@@ -0,0 +1,117 @@
+using System.Diagnostics;
+using Microsoft.EntityFrameworkCore;
+using Spacebar.Interop.Cdn.Abstractions;
+using Spacebar.Models.Db.Contexts;
+
+namespace Spacebar.Cdn.Fsck;
+
+public class FsckService(ILogger<FsckService> logger, IServiceScopeFactory serviceScopeFactory, IFileSource fs) : IHostedService {
+ private SpacebarDbContext _db = null!;
+
+ public async Task StartAsync(CancellationToken cancellationToken) {
+ var sw = Stopwatch.StartNew();
+ await using var scope = serviceScopeFactory.CreateAsyncScope();
+ _db = scope.ServiceProvider.GetRequiredService<SpacebarDbContext>();
+ logger.LogInformation("Starting fsck on {source}...", $"{fs.GetType().FullName}({fs.BaseUrl})");
+ await RunFsckAsync("User Avatars", "/avatars", EnumerateUserAvatarFilesAsync());
+ await RunFsckAsync("User Banners", "/banners", EnumerateUserBannerPathsAsync());
+ await RunFsckAsync("Guild Icons", "/icons", EnumerateGuildIconPathsAsync());
+ await RunFsckAsync("Stickers", "/stickers", EnumerateStickerPathsAsync());
+ await RunFsckAsync("Emojis", "/emojis", EnumerateEmojiPathsAsync());
+ logger.LogInformation("Fsck complete in {time}.", sw.Elapsed);
+ }
+
+ public async Task StopAsync(CancellationToken cancellationToken) { }
+
+ private readonly Stopwatch _lastUpdateSw = Stopwatch.StartNew();
+ private readonly SemaphoreSlim _fsckSemaphore = new SemaphoreSlim(32, 32);
+
+ public struct FsckItem {
+ public string Path;
+ public string ItemId;
+ }
+
+ private async Task RunFsckAsync(string name, string path, IQueryable<FsckItem> items) {
+ int i = 0, count = await items.CountAsync();
+ List<Task> tasks = [];
+
+ await foreach (var item in items.AsAsyncEnumerable()) {
+ tasks.Add(Task.Run(async () => {
+ await _fsckSemaphore.WaitAsync();
+ if (_lastUpdateSw.ElapsedMilliseconds >= (1000 / 30) || i == 0) {
+ _lastUpdateSw.Restart();
+ Console.Write($"{name} fsck: {i}/{count}: {item.Path,-64}\r");
+ }
+
+ i++;
+ if (!await fs.FileExists(item.Path))
+ logger.LogWarning("{itemType} {itemId} is missing at {path}", name, item.ItemId, item.Path);
+
+ _fsckSemaphore.Release();
+ }));
+ }
+
+ await Task.WhenAll(tasks);
+ logger.LogInformation("Validated {count} items for {path}.", i, path);
+ }
+
+#region User Assets
+
+ public IQueryable<FsckItem> EnumerateUserAvatarFilesAsync() =>
+ _db.Users
+ .Where(x => !string.IsNullOrWhiteSpace(x.Avatar))
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/avatars/{x.Id}/{x.Avatar}",
+ ItemId = x.Id
+ });
+
+ public IQueryable<FsckItem> EnumerateUserBannerPathsAsync() =>
+ _db.Users
+ .Where(x => !string.IsNullOrWhiteSpace(x.Banner))
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/banners/{x.Id}/{x.Banner}",
+ ItemId = x.Id
+ });
+
+#endregion
+
+#region Guild Assets
+
+ public IQueryable<FsckItem> EnumerateGuildIconPathsAsync() =>
+ _db.Guilds
+ .Where(x => !string.IsNullOrWhiteSpace(x.Icon))
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/icons/{x.Id}/{x.Icon}",
+ ItemId = x.Id
+ });
+
+ public IQueryable<FsckItem> EnumerateRoleIconPathsAsync() =>
+ _db.Roles
+ .Where(x => !string.IsNullOrWhiteSpace(x.Icon))
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/role-icons/{x.Id}/{x.Icon}",
+ ItemId = x.Id
+ });
+
+ public IQueryable<FsckItem> EnumerateStickerPathsAsync() =>
+ _db.Stickers
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/stickers/{x.Id}.png",
+ ItemId = x.Id
+ });
+
+ public IQueryable<FsckItem> EnumerateEmojiPathsAsync() =>
+ _db.Emojis
+ .OrderBy(x => x.Id)
+ .Select(x => new FsckItem {
+ Path = $"/emojis/{x.Id}",
+ ItemId = x.Id
+ });
+
+#endregion
+}
\ No newline at end of file
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs
new file mode 100644
index 00000000..15822cc9
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs
@@ -0,0 +1,21 @@
+using ArcaneLibs;
+using Microsoft.EntityFrameworkCore;
+using Spacebar.Cdn.Fsck;
+using Spacebar.Interop.Cdn.Abstractions;
+using Spacebar.Models.Db.Contexts;
+
+var builder = Host.CreateApplicationBuilder(args);
+builder.Services.AddSingleton<IFileSource>(new ProxyFileSource("http://cdn.old.server.spacebar.chat"));
+builder.Services.AddSingleton<LruFileCache>(new LruFileCache(1*1024*1024*1024));
+builder.Services.AddHostedService<FsckService>();
+
+builder.Services.AddDbContextPool<SpacebarDbContext>(options => {
+ options
+ .UseNpgsql(builder.Configuration.GetConnectionString("Spacebar"))
+ .EnableDetailedErrors();
+});
+
+StreamingHttpClient.LogRequests = false;
+
+var host = builder.Build();
+host.Start();
\ No newline at end of file
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Properties/launchSettings.json b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Properties/launchSettings.json
new file mode 100644
index 00000000..d2903516
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Properties/launchSettings.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "https://json.schemastore.org/launchsettings.json",
+ "profiles": {
+ "Development": {
+ "commandName": "Project",
+ "dotnetRunMessages": true,
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Development"
+ }
+ },
+ "Local": {
+ "commandName": "Project",
+ "dotnetRunMessages": true,
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Local"
+ }
+ }
+ }
+}
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Spacebar.Cdn.Fsck.csproj b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Spacebar.Cdn.Fsck.csproj
new file mode 100644
index 00000000..e08d3ae8
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Spacebar.Cdn.Fsck.csproj
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk.Worker">
+
+ <PropertyGroup>
+ <TargetFramework>net10.0</TargetFramework>
+ <Nullable>enable</Nullable>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <UserSecretsId>dotnet-Spacebar.Cdn.Fsck-a4aee86f-7e64-4c71-88d7-d2a76cc8b77e</UserSecretsId>
+ <NoDefaultLaunchSettingsFile>True</NoDefaultLaunchSettingsFile>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.2"/>
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" />
+ <ProjectReference Include="..\..\Models\Spacebar.Models.Db\Spacebar.Models.Db.csproj" />
+ </ItemGroup>
+</Project>
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.Development.json b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.Development.json
new file mode 100644
index 00000000..b2dcdb67
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.Development.json
@@ -0,0 +1,8 @@
+{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.Hosting.Lifetime": "Information"
+ }
+ }
+}
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.json b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.json
new file mode 100644
index 00000000..b2dcdb67
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/appsettings.json
@@ -0,0 +1,8 @@
+{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.Hosting.Lifetime": "Information"
+ }
+ }
+}
diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/deps.json b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/deps.json
new file mode 100644
index 00000000..fe51488c
--- /dev/null
+++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/deps.json
@@ -0,0 +1 @@
+[]
diff --git a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/deps.json b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/deps.json
index 3bf6a98c..269c9e39 100644
--- a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/deps.json
+++ b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/deps.json
@@ -5,39 +5,14 @@
"hash": "sha256-EXvojddPu+9JKgOG9NSQgUTfWq1RpOYw7adxDPKDJ6o="
},
{
- "pname": "Microsoft.Build",
- "version": "17.7.2",
- "hash": "sha256-k35nFdPxC8t0zAltVSmAJtsepp/ubNIjPOsJ6k8jSqM="
- },
- {
"pname": "Microsoft.Build.Framework",
- "version": "17.14.28",
- "hash": "sha256-7RzEyIipumafwLW1xN1q23114NafG6PT0+RADElNsiM="
+ "version": "17.11.31",
+ "hash": "sha256-YS4oASrmC5dmZrx5JPS7SfKmUpIJErlUpVDsU3VrfFE="
},
{
"pname": "Microsoft.Build.Framework",
- "version": "17.7.2",
- "hash": "sha256-fNWmVQYFTJDveAGmxEdNqJRAczV6+Ep8RA8clKBJFqw="
- },
- {
- "pname": "Microsoft.Build.Tasks.Core",
- "version": "17.14.28",
- "hash": "sha256-M9zRXYijH2HtLlRXbrUK1a1LQ9zkT+DC9ZmMiiVZwv0="
- },
- {
- "pname": "Microsoft.Build.Tasks.Core",
- "version": "17.7.2",
- "hash": "sha256-OrV/qWgZHzGlNUmaSfX5wDBcmg1aQeF3/OUHpSH+uZU="
- },
- {
- "pname": "Microsoft.Build.Utilities.Core",
- "version": "17.14.28",
- "hash": "sha256-VFfO+UpyTpw2X/qiCCOCYzvMLuu7B+XVSSpJZQLkPzU="
- },
- {
- "pname": "Microsoft.Build.Utilities.Core",
- "version": "17.7.2",
- "hash": "sha256-oatF0KfuP1nb4+OLNKg2/R/ZLO4EiACaO5leaxMEY4A="
+ "version": "18.0.2",
+ "hash": "sha256-fO31KAdDs2J0RUYD1ov9UB3ucsbALan7K0YdWW+yg7A="
},
{
"pname": "Microsoft.CodeAnalysis.Analyzers",
@@ -46,28 +21,28 @@
},
{
"pname": "Microsoft.CodeAnalysis.Common",
- "version": "4.14.0",
- "hash": "sha256-ne/zxH3GqoGB4OemnE8oJElG5mai+/67ASaKqwmL2BE="
+ "version": "5.0.0",
+ "hash": "sha256-g4ALvBSNyHEmSb1l5TFtWW7zEkiRmhqLx4XWZu9sr2U="
},
{
"pname": "Microsoft.CodeAnalysis.CSharp",
- "version": "4.14.0",
- "hash": "sha256-5Mzj3XkYYLkwDWh17r1NEXSbXwwWYQPiOmkSMlgo1JY="
+ "version": "5.0.0",
+ "hash": "sha256-ctBCkQGFpH/xT5rRE3xibu9YxPD108RuC4a4Z25koG8="
},
{
"pname": "Microsoft.CodeAnalysis.CSharp.Workspaces",
- "version": "4.14.0",
- "hash": "sha256-aNbV1a0yYBs0fpQawG6LXcbyoE8en+YFSpV5vcYE4J4="
+ "version": "5.0.0",
+ "hash": "sha256-yWVcLt/f2CouOfFy966glGdtSFy+RcgrU1dd9UtlL/Q="
},
{
"pname": "Microsoft.CodeAnalysis.Workspaces.Common",
- "version": "4.14.0",
- "hash": "sha256-0YfeaJe01WBUm9avy4a8FacQJXA1NkpnDpiXu4yz88I="
+ "version": "5.0.0",
+ "hash": "sha256-Bir5e1gEhgQQ6upQmVKQHAKLRfenAu60DAzNupNnZsQ="
},
{
"pname": "Microsoft.CodeAnalysis.Workspaces.MSBuild",
- "version": "4.14.0",
- "hash": "sha256-5SJfpRqzqCK0UbkmAaJpA/r1XJb0YAriMMeQHYC4d+o="
+ "version": "5.0.0",
+ "hash": "sha256-+58+iqTayTiE0pDaog1U8mjaDA8bNNDLA8gjCQZZudo="
},
{
"pname": "Microsoft.EntityFrameworkCore",
@@ -91,8 +66,8 @@
},
{
"pname": "Microsoft.EntityFrameworkCore.Design",
- "version": "10.0.1",
- "hash": "sha256-GGNZIGNEMhSGaMRFkRN4bOuCUBs5YVnX8klXarm319U="
+ "version": "10.0.2",
+ "hash": "sha256-bTShsGux0y/49PIIMb/4ZX3x5+rPacvT5/NcooNCI1Y="
},
{
"pname": "Microsoft.EntityFrameworkCore.Relational",
@@ -101,8 +76,8 @@
},
{
"pname": "Microsoft.EntityFrameworkCore.Relational",
- "version": "10.0.1",
- "hash": "sha256-zLgxr/iW9HP8Fip1IDgr7X0Ar8OWKDvVmoEt65gG6VY="
+ "version": "10.0.2",
+ "hash": "sha256-Y4jPpoYhKizg5wF6QfkBX4sYlE2FU1bYhfoDN3xkhKM="
},
{
"pname": "Microsoft.Extensions.Caching.Abstractions",
@@ -111,16 +86,6 @@
},
{
"pname": "Microsoft.Extensions.Caching.Memory",
- "version": "10.0.0",
- "hash": "sha256-AMgDSm1k6q0s17spGtyR5q8nAqUFDOxl/Fe38f9M+d4="
- },
- {
- "pname": "Microsoft.Extensions.Caching.Memory",
- "version": "10.0.1",
- "hash": "sha256-Qb7xK6VEZDas0lJFaW1suKdFjtkSYwLHHxkQEfWIU2A="
- },
- {
- "pname": "Microsoft.Extensions.Caching.Memory",
"version": "10.0.2",
"hash": "sha256-sRUF7DM0s1yzZnfjM/hF9A/IysE6Er23gZ6jST+RWh0="
},
@@ -131,16 +96,6 @@
},
{
"pname": "Microsoft.Extensions.Configuration.Abstractions",
- "version": "10.0.0",
- "hash": "sha256-GcgrnTAieCV7AVT13zyOjfwwL86e99iiO/MiMOxPGG0="
- },
- {
- "pname": "Microsoft.Extensions.Configuration.Abstractions",
- "version": "10.0.1",
- "hash": "sha256-s4PDp+vtzdxKIxnOT3+dDRoTDopyl8kqmmw4KDnkOtQ="
- },
- {
- "pname": "Microsoft.Extensions.Configuration.Abstractions",
"version": "10.0.2",
"hash": "sha256-P+0kaDGO+xB9KxF9eWHDJ4hzi05sUGM/uMNEX5NdBTE="
},
@@ -191,8 +146,8 @@
},
{
"pname": "Microsoft.Extensions.DependencyModel",
- "version": "10.0.1",
- "hash": "sha256-XIj2jEURe25YA4RhBSuCqQpic0YP+TZaO/dbBPCjad8="
+ "version": "10.0.2",
+ "hash": "sha256-w/dGIjtZiGH+KW3969BPOdQpQEV+WB7RPTa2MK2DavE="
},
{
"pname": "Microsoft.Extensions.Diagnostics",
@@ -231,16 +186,6 @@
},
{
"pname": "Microsoft.Extensions.Logging",
- "version": "10.0.0",
- "hash": "sha256-P+zPAadLL63k/GqK34/qChqQjY9aIRxZfxlB9lqsSrs="
- },
- {
- "pname": "Microsoft.Extensions.Logging",
- "version": "10.0.1",
- "hash": "sha256-zuLP3SIpCToMOlIPOEv3Kq8y/minecd8k8GSkxFo13E="
- },
- {
- "pname": "Microsoft.Extensions.Logging",
"version": "10.0.2",
"hash": "sha256-9+gfQwK32JMYscW1YvyCWEzc9mRZOjCACoD9U1vVaJI="
},
@@ -306,11 +251,6 @@
},
{
"pname": "Microsoft.Extensions.Primitives",
- "version": "10.0.1",
- "hash": "sha256-EXmukq09erT4s+miQpBSYy3IY4HxxKlwEPL43/KoyEc="
- },
- {
- "pname": "Microsoft.Extensions.Primitives",
"version": "10.0.2",
"hash": "sha256-8Ccrjjv9cFVf9RyCc7GS/Byt8+DXdSNea0UX3A5BEdA="
},
@@ -320,14 +260,9 @@
"hash": "sha256-ZNLusK1CRuq5BZYZMDqaz04PIKScE2Z7sS2tehU7EJs="
},
{
- "pname": "Microsoft.NET.StringTools",
- "version": "17.14.28",
- "hash": "sha256-UzREyvDxkiOQ4cEOQ5UCjkwXGrldIDCcbefECTPGjXI="
- },
- {
- "pname": "Microsoft.NET.StringTools",
- "version": "17.7.2",
- "hash": "sha256-hQE07TCgcQuyu9ZHVq2gPDb0+xe8ECJUdrgh17bJP4o="
+ "pname": "Microsoft.VisualStudio.SolutionPersistence",
+ "version": "1.0.52",
+ "hash": "sha256-KZGPtOXe6Hv8RrkcsgoLKTRyaCScIpQEa2NhNB3iOXw="
},
{
"pname": "Mono.TextTemplating",
@@ -355,16 +290,6 @@
"hash": "sha256-uPetUFZyHfxjScu5x4agjk9pIhbCkt5rG4Axj25npcQ="
},
{
- "pname": "System.CodeDom",
- "version": "7.0.0",
- "hash": "sha256-7IPt39cY+0j0ZcRr/J45xPtEjnSXdUJ/5ai3ebaYQiE="
- },
- {
- "pname": "System.CodeDom",
- "version": "9.0.0",
- "hash": "sha256-578lcBgswW0eM16r0EnJzfGodPx86RxxFoZHc2PSzsw="
- },
- {
"pname": "System.Composition",
"version": "9.0.0",
"hash": "sha256-FehOkQ2u1p8mQ0/wn3cZ+24HjhTLdck8VZYWA1CcgbM="
@@ -395,83 +320,8 @@
"hash": "sha256-F5fpTUs3Rr7yP/NyIzr+Xn5NdTXXp8rrjBnF9UBBUog="
},
{
- "pname": "System.Configuration.ConfigurationManager",
- "version": "7.0.0",
- "hash": "sha256-SgBexTTjRn23uuXvkzO0mz0qOfA23MiS4Wv+qepMLZE="
- },
- {
- "pname": "System.Configuration.ConfigurationManager",
- "version": "9.0.0",
- "hash": "sha256-+pLnTC0YDP6Kjw5DVBiFrV/Q3x5is/+6N6vAtjvhVWk="
- },
- {
"pname": "System.Diagnostics.EventLog",
"version": "10.0.2",
"hash": "sha256-rezZk0M4+MWRxwGSFzXfvekrhL8qLTp7Pc8YsSy/4nE="
- },
- {
- "pname": "System.Diagnostics.EventLog",
- "version": "9.0.0",
- "hash": "sha256-tPvt6yoAp56sK/fe+/ei8M65eavY2UUhRnbrREj/Ems="
- },
- {
- "pname": "System.Formats.Nrbf",
- "version": "9.0.0",
- "hash": "sha256-c4qf6CocQUZB0ySGQd8s15PXY7xfrjQqMGXxkwytKyw="
- },
- {
- "pname": "System.Reflection.MetadataLoadContext",
- "version": "7.0.0",
- "hash": "sha256-VYl6SFD130K9Aw4eJH16ApJ9Sau4Xu0dcxEip2veuTI="
- },
- {
- "pname": "System.Resources.Extensions",
- "version": "9.0.0",
- "hash": "sha256-y2gLEMuAy6QfEyNJxABC/ayMWGnwlpX735jsUQLktho="
- },
- {
- "pname": "System.Security.Cryptography.Pkcs",
- "version": "7.0.0",
- "hash": "sha256-3J3vL9hcKSuZjT2GKappa2A9p2xJm1nH2asTNAl8ZCA="
- },
- {
- "pname": "System.Security.Cryptography.Pkcs",
- "version": "7.0.2",
- "hash": "sha256-qS5Z/Yo8J+f3ExVX5Qkcpj1Z57oUZqz5rWa1h5bVpl8="
- },
- {
- "pname": "System.Security.Cryptography.Pkcs",
- "version": "9.0.0",
- "hash": "sha256-AjG14mGeSc2Ka4QSelGBM1LrGBW3VJX60lnihKyJjGY="
- },
- {
- "pname": "System.Security.Cryptography.ProtectedData",
- "version": "9.0.0",
- "hash": "sha256-gPgPU7k/InTqmXoRzQfUMEKL3QuTnOKowFqmXTnWaBQ="
- },
- {
- "pname": "System.Security.Cryptography.Xml",
- "version": "7.0.1",
- "hash": "sha256-CH8+JVC8LyCSW75/6ZQ7ecMbSOAE1c16z4dG8JTp01w="
- },
- {
- "pname": "System.Security.Cryptography.Xml",
- "version": "9.0.0",
- "hash": "sha256-SQJWwAFrJUddEU6JiZB52FM9tGjRlJAYH8oYVzG5IJU="
- },
- {
- "pname": "System.Security.Permissions",
- "version": "7.0.0",
- "hash": "sha256-DOFoX+AKRmrkllykHheR8FfUXYx/Ph+I/HYuReQydXI="
- },
- {
- "pname": "System.Security.Permissions",
- "version": "9.0.0",
- "hash": "sha256-BFrA9ottmQtLIAiKiGRbfSUpzNJwuaOCeFRDN4Z0ku0="
- },
- {
- "pname": "System.Windows.Extensions",
- "version": "9.0.0",
- "hash": "sha256-RErD+Ju15qtnwdwB7E0SjjJGAnhXwJyC7UPcl24Z3Vs="
}
]
|
