1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
using System.Diagnostics;
using Microsoft.EntityFrameworkCore;
using Spacebar.Interop.Cdn.Abstractions;
using Spacebar.Models.Db.Contexts;
namespace Spacebar.Cdn.Fsck;
public class FsckService(ILogger<FsckService> logger, IServiceScopeFactory serviceScopeFactory, IFileSource fs) : IHostedService {
private SpacebarDbContext _db = null!;
public async Task StartAsync(CancellationToken cancellationToken) {
var sw = Stopwatch.StartNew();
await using var scope = serviceScopeFactory.CreateAsyncScope();
_db = scope.ServiceProvider.GetRequiredService<SpacebarDbContext>();
logger.LogInformation("Starting fsck on {source}...", $"{fs.GetType().FullName}({fs.BaseUrl})");
await RunFsckAsync("User Avatars", "/avatars", EnumerateUserAvatarFilesAsync());
await RunFsckAsync("User Banners", "/banners", EnumerateUserBannerPathsAsync());
await RunFsckAsync("Guild Icons", "/icons", EnumerateGuildIconPathsAsync());
await RunFsckAsync("Stickers", "/stickers", EnumerateStickerPathsAsync());
await RunFsckAsync("Emojis", "/emojis", EnumerateEmojiPathsAsync());
logger.LogInformation("Fsck complete in {time}.", sw.Elapsed);
}
public async Task StopAsync(CancellationToken cancellationToken) { }
private readonly Stopwatch _lastUpdateSw = Stopwatch.StartNew();
private readonly SemaphoreSlim _fsckSemaphore = new SemaphoreSlim(32, 32);
public struct FsckItem {
public string Path;
public string ItemId;
}
private async Task RunFsckAsync(string name, string path, IQueryable<FsckItem> items) {
int i = 0, count = await items.CountAsync();
List<Task> tasks = [];
await foreach (var item in items.AsAsyncEnumerable()) {
tasks.Add(Task.Run(async () => {
await _fsckSemaphore.WaitAsync();
if (_lastUpdateSw.ElapsedMilliseconds >= (1000 / 30) || i == 0) {
_lastUpdateSw.Restart();
Console.Write($"{name} fsck: {i}/{count}: {item.Path,-64}\r");
}
i++;
if (!await fs.FileExists(item.Path))
logger.LogWarning("{itemType} {itemId} is missing at {path}", name, item.ItemId, item.Path);
_fsckSemaphore.Release();
}));
}
await Task.WhenAll(tasks);
logger.LogInformation("Validated {count} items for {path}.", i, path);
}
#region User Assets
public IQueryable<FsckItem> EnumerateUserAvatarFilesAsync() =>
_db.Users
.Where(x => !string.IsNullOrWhiteSpace(x.Avatar))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/avatars/{x.Id}/{x.Avatar}",
ItemId = x.Id
});
public IQueryable<FsckItem> EnumerateUserBannerPathsAsync() =>
_db.Users
.Where(x => !string.IsNullOrWhiteSpace(x.Banner))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/banners/{x.Id}/{x.Banner}",
ItemId = x.Id
});
#endregion
#region Guild Assets
public IQueryable<FsckItem> EnumerateGuildIconPathsAsync() =>
_db.Guilds
.Where(x => !string.IsNullOrWhiteSpace(x.Icon))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/icons/{x.Id}/{x.Icon}",
ItemId = x.Id
});
public IQueryable<FsckItem> EnumerateRoleIconPathsAsync() =>
_db.Roles
.Where(x => !string.IsNullOrWhiteSpace(x.Icon))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/role-icons/{x.Id}/{x.Icon}",
ItemId = x.Id
});
public IQueryable<FsckItem> EnumerateStickerPathsAsync() =>
_db.Stickers
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/stickers/{x.Id}.png",
ItemId = x.Id
});
public IQueryable<FsckItem> EnumerateEmojiPathsAsync() =>
_db.Emojis
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/emojis/{x.Id}",
ItemId = x.Id
});
#endregion
}
|
