summary refs log tree commit diff
path: root/LibBeatmapDownload
diff options
context:
space:
mode:
Diffstat (limited to 'LibBeatmapDownload')
-rw-r--r--LibBeatmapDownload/DomainStats.cs46
-rw-r--r--LibBeatmapDownload/DownloadTask.cs201
-rw-r--r--LibBeatmapDownload/DownloadTaskList.cs64
-rw-r--r--LibBeatmapDownload/LibBeatmapDownload.csproj14
-rw-r--r--LibBeatmapDownload/Status.cs10
5 files changed, 335 insertions, 0 deletions
diff --git a/LibBeatmapDownload/DomainStats.cs b/LibBeatmapDownload/DomainStats.cs
new file mode 100644
index 0000000..a026b7d
--- /dev/null
+++ b/LibBeatmapDownload/DomainStats.cs
@@ -0,0 +1,46 @@
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
+namespace LibBeatmapDownload;
+
+public class DomainStats(string url) : INotifyPropertyChanged {
+	public string Domain { get; } = new Uri(url).Host;
+
+	private int _total = 0;
+
+	public int Total => Success + Failed;
+
+	private int _success = 0;
+
+	public int Success {
+		get => _success;
+		set => SetField(ref _success, value);
+	}
+
+	private int _failed = 0;
+
+	public int Failed {
+		get => _failed;
+		set => SetField(ref _failed, value);
+	}
+
+	public double Progress => Total == 0 ? 100 : (double)Success / Total * 100;
+	public string ProgressString => $"{Domain} {Progress/100:P0} ({Failed}/{Total} failed)";
+
+	public event PropertyChangedEventHandler? PropertyChanged;
+
+	private readonly Stopwatch sw = Stopwatch.StartNew();
+	protected virtual void OnPropertyChanged([CallerMemberName] string? propertyName = null) {
+		PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
+		PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Progress)));
+		PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(ProgressString)));
+	}
+
+	protected bool SetField<T>(ref T field, T value, [CallerMemberName] string? propertyName = null) {
+		if (EqualityComparer<T>.Default.Equals(field, value)) return false;
+		field = value;
+		OnPropertyChanged(propertyName);
+		return true;
+	}
+}
diff --git a/LibBeatmapDownload/DownloadTask.cs b/LibBeatmapDownload/DownloadTask.cs
new file mode 100644
index 0000000..2e9222f
--- /dev/null
+++ b/LibBeatmapDownload/DownloadTask.cs
@@ -0,0 +1,201 @@
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using ArcaneLibs;
+
+namespace LibBeatmapDownload;
+
+public class DownloadTask : INotifyPropertyChanged {
+	private static readonly HttpClient Client = new();
+
+	private static readonly List<string> Mirrors = new() {
+		"https://api.osu.direct/d/", "https://osu.direct/api/d/",
+		"https://api.chimu.moe/v1/download/",
+		"https://api.nerinyan.moe/d/", "https://proxy.nerinyan.moe/d/",
+		"https://storage.kurikku.pw/d/",
+		"https://storage.ripple.moe/d/",
+		"https://catboy.best/d/",
+		"https://txy1.sayobot.cn/beatmaps/download/full/"
+	};
+
+	public static readonly Dictionary<string, DomainStats> MirrorStats = Mirrors.ToDictionary(x => x, x => new DomainStats(x));
+
+	private static int _mirrorIndex = 0;
+
+	// private void AllPropertiesChanged() {
+	// typeof(DownloadTask).GetProperties().ToList().ForEach(x => OnPropertyChanged(x.Name));
+	// }
+
+	private static readonly Dictionary<string, DateTime> NextRatelimitFree = Mirrors.ToDictionary(x => x, x => DateTime.MinValue);
+
+	public int Id { get; }
+
+	private long _expectedSize = 0;
+
+	public long ExpectedSize {
+		get => _expectedSize;
+		set {
+			if (value == _expectedSize) return;
+			_expectedSize = value;
+			// OnPropertyChanged();
+			OnPropertyChanged(nameof(ProgressString));
+		}
+	}
+
+	private long _downloadedSize = 0;
+
+	private Stopwatch sw = Stopwatch.StartNew();
+
+	public long DownloadedSize {
+		get => _downloadedSize;
+		set {
+			if (value == _downloadedSize) return;
+			_downloadedSize = value;
+
+			//if (sw.ElapsedMilliseconds < 1000) return;
+			if(!value.ToString().EndsWith('0')) return;
+			OnPropertyChanged(nameof(Progress));
+			OnPropertyChanged(nameof(ProgressString));
+			sw.Restart();
+		}
+	}
+
+	public string OutPath { get; set; } = Path.GetTempFileName();
+
+	public int Attempt { get; set; } = 0;
+
+	private Status _status = Status.NotQueued;
+
+	public Status Status {
+		get => _status;
+		set {
+			if (value == _status) return;
+			_status = value;
+			OnPropertyChanged();
+			OnPropertyChanged(nameof(ProgressString));
+		}
+	}
+
+	public string? OriginalFilename { get; set; }
+	public double Progress => DownloadedSize / (double)(ExpectedSize == 0 ? 1 : ExpectedSize) * 100;
+
+	public string ProgressString => $"{Id}" +
+	                                $"{(_title == null ? "" : $" - {_title}")}" +
+	                                // $"{(OriginalFilename == null ? "" : $" ({OriginalFilename})")}" +
+	                                $": " +
+	                                Status switch {
+		                                Status.Downloading => $"{Progress / 100d:P2} ({Util.BytesToString(DownloadedSize)}/{Util.BytesToString(ExpectedSize)})",
+		                                Status.Finished => $"Finished downloading {Util.BytesToString(DownloadedSize)}",
+		                                Status.Error => $"Errored, retrying later, so far {Attempt} attempts...",
+		                                _ => Status
+	                                };
+
+	private static readonly SemaphoreSlim Semaphore = new(Mirrors.Count, Mirrors.Count);
+	private readonly int _id;
+	private readonly string? _title;
+
+	public static List<DownloadTask> DownloadTasks { get; set; } = new();
+	public DownloadTask(int id, string? title = null) {
+		_id = id;
+		_title = title;
+		Id = id;
+		DownloadTasks.Add(this);
+	}
+
+	public async Task<string?> Download() {
+		if(_status == Status.Error)
+			while (Semaphore.CurrentCount < Mirrors.Count/2) {
+				await Task.Delay(100);
+			}
+		Status = Status.Waiting;
+		var currentMirrorIndex = Interlocked.Increment(ref _mirrorIndex);
+		string mirror = Mirrors[currentMirrorIndex % Mirrors.Count];
+		await Semaphore.WaitAsync();
+		Status = Status.Connecting;
+
+		while (Attempt++ % Mirrors.Count == 0) {
+			mirror = Mirrors[currentMirrorIndex++ % Mirrors.Count];
+
+			if (DateTime.Now < NextRatelimitFree[mirror]) continue;
+			var url = $"{mirror}{Id}";
+			Debug.WriteLine($"Attempting to download {Id} from {mirror}: {url}");
+			var request = new HttpRequestMessage(HttpMethod.Get, url);
+			request.Headers.Add("Accept", "*/*");
+			// request.Headers.Add("User-Agent", "osu_beatmap_download_test");
+			// set user agent to entry binary name
+			request.Headers.Add("User-Agent", Assembly.GetEntryAssembly()?.GetName().Name ?? "LibBeatmapDownload");
+
+			var sw = Stopwatch.StartNew();
+			try {
+				var response = await Client.SendAsync(request);
+				Debug.WriteLine($"Got status code {(int)response.StatusCode} {response.StatusCode} for {url} after {sw.Elapsed}");
+				if (response.IsSuccessStatusCode) {
+					Status = Status.Downloading;
+					OriginalFilename = response.Content.Headers.ContentDisposition?.FileName ?? $"{_id}.osz";
+					ExpectedSize = response.Content.Headers.ContentLength ?? 1;
+					Debug.WriteLine(
+						$"Got success status code {response.StatusCode} from {mirror} for {_id} after {sw.Elapsed}, filename is {OriginalFilename}, expected size is {Util.BytesToString(ExpectedSize)}");
+					var streamWithFileBody = await response.Content.ReadAsStreamAsync();
+					// write to temp file first
+					await using var fileStream = File.Create(OutPath);
+					var buffer = new byte[8192];
+					int read;
+					while ((read = await streamWithFileBody.ReadAsync(buffer, 0, buffer.Length)) > 0) {
+						await fileStream.WriteAsync(buffer, 0, read);
+						DownloadedSize += read;
+						// Debug.WriteLine($"Read {read} bytes from {url}, total downloaded is {Util.BytesToString(DownloadedSize)}");
+					}
+
+					fileStream.Flush();
+					fileStream.Close();
+					Semaphore.Release();
+					Status = Status.Finished;
+					MirrorStats[mirror].Success++;
+					return OutPath;
+				}
+
+				Status = Status.Connecting;
+
+				//ratelimits
+				if (response.Headers.Contains("X-Burst-RateLimit-Reset")) {
+					NextRatelimitFree[mirror] = DateTime.Now.AddSeconds(int.Parse(response.Headers.GetValues("X-Burst-RateLimit-Reset").First()));
+					Debug.WriteLine($"We got burst ratelimited by {mirror}: {response.Headers.GetValues("X-Burst-RateLimit-Reset").First()} seconds");
+				}
+				else if (response.Headers.Contains("X-RateLimit-Reset")) {
+					NextRatelimitFree[mirror] = DateTime.Now.AddSeconds(int.Parse(response.Headers.GetValues("X-RateLimit-Reset").First()));
+					Debug.WriteLine($"We got ratelimited by {mirror}: {response.Headers.GetValues("X-RateLimit-Reset").First()} seconds");
+				}
+				else {
+					Debug.WriteLine($"Got non-success status code {response.StatusCode} from {mirror} for {_id} and no ratelimit headers were found...");
+					Debug.WriteLine(await response.Content.ReadAsStringAsync());
+				}
+			}
+			catch (Exception e) {
+				Debug.WriteLine(e.ToString());
+			}
+
+			MirrorStats[mirror].Failed++;
+		}
+
+		Status = Status.Error;
+		Debug.WriteLine($"Failed to download {_id} after {Attempt + 1} attempts, retrying...");
+		Semaphore.Release();
+		await Task.Delay(500);
+
+		return await Download();
+	}
+
+	public event PropertyChangedEventHandler? PropertyChanged;
+
+	protected virtual void OnPropertyChanged([CallerMemberName] string? propertyName = null) {
+		PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
+	}
+
+	protected bool SetField<T>(ref T field, T value, [CallerMemberName] string? propertyName = null) {
+		if (EqualityComparer<T>.Default.Equals(field, value)) return false;
+		field = value;
+		OnPropertyChanged(propertyName);
+		return true;
+	}
+}
diff --git a/LibBeatmapDownload/DownloadTaskList.cs b/LibBeatmapDownload/DownloadTaskList.cs
new file mode 100644
index 0000000..a449e96
--- /dev/null
+++ b/LibBeatmapDownload/DownloadTaskList.cs
@@ -0,0 +1,64 @@
+using System.Collections.ObjectModel;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
+namespace LibBeatmapDownload;
+
+public class DownloadTaskList : INotifyPropertyChanged {
+	public DownloadTaskList() {
+		Tasks.CollectionChanged += (sender, args) => {
+			if (args.NewItems is { Count: > 0 })
+				foreach (var downloadTask in args.NewItems) {
+					if (downloadTask is not DownloadTask task) {
+						Debug.WriteLine($"[DownloadTaskList] New task is {downloadTask?.GetType().FullName ?? "null"}!");
+						continue;
+					}
+
+					task.PropertyChanged += (taskSender, taskArgs) => {
+						if (taskArgs.PropertyName == nameof(DownloadTask.Status)) {
+							if (task.Status == Status.Finished) {
+								Task.Run(async () => {
+									await Task.Delay(1000);
+									Tasks.Remove(task);
+								});
+							}
+
+							SetField(ref _tasks,
+								new ObservableCollection<DownloadTask>(
+									_tasks
+										.OrderByDescending(x => x.Progress)
+										.ThenByDescending(x => x.Status)
+								),
+								nameof(Tasks));
+							OnPropertyChanged(nameof(Tasks));
+						}
+					};
+				}
+
+			// OnPropertyChanged(nameof(Tasks));
+		};
+	}
+
+	public event PropertyChangedEventHandler? PropertyChanged;
+
+	private ObservableCollection<DownloadTask> _tasks = new();
+
+	public ObservableCollection<DownloadTask> Tasks {
+		get => _tasks;
+		set => SetField(ref _tasks, value);
+	}
+
+	protected virtual void OnPropertyChanged([CallerMemberName] string? propertyName = null) {
+		PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
+	}
+
+	protected bool SetField<T>(ref T field, T value, [CallerMemberName] string? propertyName = null) {
+		//check ObservableCollection by order
+
+		if (EqualityComparer<T>.Default.Equals(field, value)) return false;
+		field = value;
+		OnPropertyChanged(propertyName);
+		return true;
+	}
+}
diff --git a/LibBeatmapDownload/LibBeatmapDownload.csproj b/LibBeatmapDownload/LibBeatmapDownload.csproj
new file mode 100644
index 0000000..829366e
--- /dev/null
+++ b/LibBeatmapDownload/LibBeatmapDownload.csproj
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <LangVersion>preview</LangVersion>
+    </PropertyGroup>
+    
+    <ItemGroup>
+        <PackageReference Include="ArcaneLibs" Version="1.0.0-preview6437853305.78f6d30" />
+    </ItemGroup>
+
+</Project>
diff --git a/LibBeatmapDownload/Status.cs b/LibBeatmapDownload/Status.cs
new file mode 100644
index 0000000..f5602a4
--- /dev/null
+++ b/LibBeatmapDownload/Status.cs
@@ -0,0 +1,10 @@
+namespace LibBeatmapDownload;
+
+public enum Status {
+	NotQueued,
+	Error,
+	Waiting,
+	Connecting,
+	Downloading,
+	Finished
+}
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-11-08 15:16:24 +0000