1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
using System.Diagnostics;
using Microsoft.EntityFrameworkCore;
using Spacebar.Interop.Cdn.Abstractions;
using Spacebar.Models.Db.Contexts;
namespace Spacebar.Cdn.Fsck;
public class FsckService(ILogger<FsckService> logger, IServiceScopeFactory serviceScopeFactory, IFileSource fs) : IHostedService {
private SpacebarDbContext _db = null!;
public async Task StartAsync(CancellationToken cancellationToken) {
var sw = Stopwatch.StartNew();
await using var scope = serviceScopeFactory.CreateAsyncScope();
_db = scope.ServiceProvider.GetRequiredService<SpacebarDbContext>();
logger.LogInformation("Starting fsck on {source}...", $"{fs.GetType().FullName}({fs.BaseUrl})");
await RunFsckAsync("User Avatars", "/avatars", EnumerateUserAvatarFilesAsync());
await RunFsckAsync("User Banners", "/banners", EnumerateUserBannerPathsAsync());
await RunFsckAsync("Guild Icons", "/icons", EnumerateGuildIconPathsAsync());
await RunFsckAsync("Stickers", "/stickers", EnumerateStickerPathsAsync());
await RunFsckAsync("Emojis", "/emojis", EnumerateEmojiPathsAsync());
logger.LogInformation("Fsck complete in {time}.", sw.Elapsed);
}
public async Task StopAsync(CancellationToken cancellationToken) { }
private readonly Stopwatch _lastUpdateSw = Stopwatch.StartNew();
private readonly SemaphoreSlim _fsckSemaphore = new SemaphoreSlim(32, 32);
public struct FsckItem {
public string Path;
public string ItemId;
public bool IsSingleSubDirFile;
}
private async Task RunFsckAsync(string name, string path, IQueryable<FsckItem> items) {
int i = 0, count = await items.CountAsync(), missing = 0;
List<Task> tasks = [];
await foreach (var item in items.AsAsyncEnumerable()) {
tasks.Add(Task.Run(async () => {
await _fsckSemaphore.WaitAsync();
if (_lastUpdateSw.ElapsedMilliseconds >= (1000 / 30) || i == 0) {
_lastUpdateSw.Restart();
Console.Write($"{name} fsck: {i}/{count}: {item.Path,-64}\r");
}
i++;
if (!item.IsSingleSubDirFile) {
if (!await fs.FileExists(item.Path)) {
logger.LogWarning("{itemType} {itemId} is missing at {path}", name, item.ItemId, item.Path);
missing++;
}
}
else if (item.IsSingleSubDirFile && fs is FilesystemFileSource ffs) {
if (!await ffs.DirectoryExists(Path.GetDirectoryName(item.Path))) {
logger.LogWarning("{itemType} {itemId} is missing at {path} (directory missing)", name, item.ItemId, item.Path);
missing++;
}
}
else {
logger.LogWarning("Unhandled case: {itemType} {itemId} -> {path} (IsSingleSubDirFile: {isSingleSubDirFile}, fstype: {fsType})", name, item.ItemId, item.Path, item.IsSingleSubDirFile, fs.GetType().Name);
}
_fsckSemaphore.Release();
}));
}
await Task.WhenAll(tasks);
logger.LogInformation("Validated {count} items for {path}, {missing} missing.", i, path, missing);
}
#region User Assets
public IQueryable<FsckItem> EnumerateUserAvatarFilesAsync() =>
_db.Users
.Where(x => !string.IsNullOrWhiteSpace(x.Avatar))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/avatars/{x.Id}/{x.Avatar}",
ItemId = x.Id.ToString()
});
public IQueryable<FsckItem> EnumerateUserBannerPathsAsync() =>
_db.Users
.Where(x => !string.IsNullOrWhiteSpace(x.Banner))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/banners/{x.Id}/{x.Banner}",
ItemId = x.Id.ToString()
});
#endregion
#region Guild Assets
public IQueryable<FsckItem> EnumerateGuildIconPathsAsync() =>
_db.Guilds
.Where(x => !string.IsNullOrWhiteSpace(x.Icon))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/icons/{x.Id}/{x.Icon}",
ItemId = x.Id.ToString()
});
public IQueryable<FsckItem> EnumerateRoleIconPathsAsync() =>
_db.Roles
.Where(x => !string.IsNullOrWhiteSpace(x.Icon))
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/role-icons/{x.Id}/{x.Icon}",
ItemId = x.Id.ToString()
});
public IQueryable<FsckItem> EnumerateStickerPathsAsync() =>
_db.Stickers
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/stickers/{x.Id}",
ItemId = x.Id.ToString()
});
public IQueryable<FsckItem> EnumerateEmojiPathsAsync() =>
_db.Emojis
.OrderBy(x => x.Id)
.Select(x => new FsckItem {
Path = $"/emojis/{x.Id}",
ItemId = x.Id.ToString()
});
#endregion
}
|
