1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
|
using System.ComponentModel;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.CompilerServices;
using ArcaneLibs;
namespace LibBeatmapDownload;
public class DownloadTask : INotifyPropertyChanged {
private static readonly HttpClient Client = new();
private static readonly List<string> Mirrors = new() {
"https://api.osu.direct/d/", "https://osu.direct/api/d/",
"https://api.chimu.moe/v1/download/",
"https://api.nerinyan.moe/d/", "https://proxy.nerinyan.moe/d/",
"https://storage.kurikku.pw/d/",
"https://storage.ripple.moe/d/",
"https://catboy.best/d/",
"https://txy1.sayobot.cn/beatmaps/download/full/"
};
public static readonly Dictionary<string, DomainStats> MirrorStats = Mirrors.ToDictionary(x => x, x => new DomainStats(x));
private static int _mirrorIndex = 0;
// private void AllPropertiesChanged() {
// typeof(DownloadTask).GetProperties().ToList().ForEach(x => OnPropertyChanged(x.Name));
// }
private static readonly Dictionary<string, DateTime> NextRatelimitFree = Mirrors.ToDictionary(x => x, x => DateTime.MinValue);
public int Id { get; }
private long _expectedSize = 0;
public long ExpectedSize {
get => _expectedSize;
set {
if (value == _expectedSize) return;
_expectedSize = value;
// OnPropertyChanged();
OnPropertyChanged(nameof(ProgressString));
}
}
private long _downloadedSize = 0;
private Stopwatch sw = Stopwatch.StartNew();
public long DownloadedSize {
get => _downloadedSize;
set {
if (value == _downloadedSize) return;
_downloadedSize = value;
//if (sw.ElapsedMilliseconds < 1000) return;
if(!value.ToString().EndsWith('0')) return;
OnPropertyChanged(nameof(Progress));
OnPropertyChanged(nameof(ProgressString));
sw.Restart();
}
}
public string OutPath { get; set; } = Path.GetTempFileName();
public int Attempt { get; set; } = 0;
private Status _status = Status.NotQueued;
public Status Status {
get => _status;
set {
if (value == _status) return;
_status = value;
OnPropertyChanged();
OnPropertyChanged(nameof(ProgressString));
}
}
public string? OriginalFilename { get; set; }
public double Progress => DownloadedSize / (double)(ExpectedSize == 0 ? 1 : ExpectedSize) * 100;
public string ProgressString => $"{Id}" +
$"{(_title == null ? "" : $" - {_title}")}" +
// $"{(OriginalFilename == null ? "" : $" ({OriginalFilename})")}" +
$": " +
Status switch {
Status.Downloading => $"{Progress / 100d:P2} ({Util.BytesToString(DownloadedSize)}/{Util.BytesToString(ExpectedSize)})",
Status.Finished => $"Finished downloading {Util.BytesToString(DownloadedSize)}",
Status.Error => $"Errored, retrying later, so far {Attempt} attempts...",
_ => Status
};
private static readonly SemaphoreSlim Semaphore = new(Mirrors.Count, Mirrors.Count);
private readonly int _id;
private readonly string? _title;
public static List<DownloadTask> DownloadTasks { get; set; } = new();
public DownloadTask(int id, string? title = null) {
_id = id;
_title = title;
Id = id;
DownloadTasks.Add(this);
}
public async Task<string?> Download() {
if(_status == Status.Error)
while (Semaphore.CurrentCount < Mirrors.Count/2) {
await Task.Delay(100);
}
Status = Status.Waiting;
var currentMirrorIndex = Interlocked.Increment(ref _mirrorIndex);
string mirror = Mirrors[currentMirrorIndex % Mirrors.Count];
await Semaphore.WaitAsync();
Status = Status.Connecting;
while (Attempt++ % Mirrors.Count == 0) {
mirror = Mirrors[currentMirrorIndex++ % Mirrors.Count];
if (DateTime.Now < NextRatelimitFree[mirror]) continue;
var url = $"{mirror}{Id}";
Debug.WriteLine($"Attempting to download {Id} from {mirror}: {url}");
var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("Accept", "*/*");
// request.Headers.Add("User-Agent", "osu_beatmap_download_test");
// set user agent to entry binary name
request.Headers.Add("User-Agent", Assembly.GetEntryAssembly()?.GetName().Name ?? "LibBeatmapDownload");
var sw = Stopwatch.StartNew();
try {
var response = await Client.SendAsync(request);
Debug.WriteLine($"Got status code {(int)response.StatusCode} {response.StatusCode} for {url} after {sw.Elapsed}");
if (response.IsSuccessStatusCode) {
Status = Status.Downloading;
OriginalFilename = response.Content.Headers.ContentDisposition?.FileName ?? $"{_id}.osz";
ExpectedSize = response.Content.Headers.ContentLength ?? 1;
Debug.WriteLine(
$"Got success status code {response.StatusCode} from {mirror} for {_id} after {sw.Elapsed}, filename is {OriginalFilename}, expected size is {Util.BytesToString(ExpectedSize)}");
var streamWithFileBody = await response.Content.ReadAsStreamAsync();
// write to temp file first
await using var fileStream = File.Create(OutPath);
var buffer = new byte[8192];
int read;
while ((read = await streamWithFileBody.ReadAsync(buffer, 0, buffer.Length)) > 0) {
await fileStream.WriteAsync(buffer, 0, read);
DownloadedSize += read;
// Debug.WriteLine($"Read {read} bytes from {url}, total downloaded is {Util.BytesToString(DownloadedSize)}");
}
fileStream.Flush();
fileStream.Close();
Semaphore.Release();
Status = Status.Finished;
MirrorStats[mirror].Success++;
return OutPath;
}
Status = Status.Connecting;
//ratelimits
if (response.Headers.Contains("X-Burst-RateLimit-Reset")) {
NextRatelimitFree[mirror] = DateTime.Now.AddSeconds(int.Parse(response.Headers.GetValues("X-Burst-RateLimit-Reset").First()));
Debug.WriteLine($"We got burst ratelimited by {mirror}: {response.Headers.GetValues("X-Burst-RateLimit-Reset").First()} seconds");
}
else if (response.Headers.Contains("X-RateLimit-Reset")) {
NextRatelimitFree[mirror] = DateTime.Now.AddSeconds(int.Parse(response.Headers.GetValues("X-RateLimit-Reset").First()));
Debug.WriteLine($"We got ratelimited by {mirror}: {response.Headers.GetValues("X-RateLimit-Reset").First()} seconds");
}
else {
Debug.WriteLine($"Got non-success status code {response.StatusCode} from {mirror} for {_id} and no ratelimit headers were found...");
Debug.WriteLine(await response.Content.ReadAsStringAsync());
}
}
catch (Exception e) {
Debug.WriteLine(e.ToString());
}
MirrorStats[mirror].Failed++;
}
Status = Status.Error;
Debug.WriteLine($"Failed to download {_id} after {Attempt + 1} attempts, retrying...");
Semaphore.Release();
await Task.Delay(500);
return await Download();
}
public event PropertyChangedEventHandler? PropertyChanged;
protected virtual void OnPropertyChanged([CallerMemberName] string? propertyName = null) {
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
}
protected bool SetField<T>(ref T field, T value, [CallerMemberName] string? propertyName = null) {
if (EqualityComparer<T>.Default.Equals(field, value)) return false;
field = value;
OnPropertyChanged(propertyName);
return true;
}
}
|
