NetworkTaskFactory

2024-09-29 08:21:48 +05:00
parent 45c3f90da0
commit 4704f1217a
24 changed files with 817 additions and 287 deletions
--- a/Млаумчерб.Клиент/сеть/DataSize.cs
+++ b/Млаумчерб.Клиент/сеть/DataSize.cs
@@ -0,0 +1,23 @@
+namespace Млаумчерб.Клиент.сеть;
+
+public record struct DataSize(long Bytes)
+{
+    static string BytesToHumanReadable(long bytes)
+    {
+        long K = bytes / 1024;
+        if (K == 0)
+            return $"{bytes}b";
+        float M = K / 1024f;
+        if (M < 1)
+            return $"{K:N1}Kb";
+        float G = M / 1024f;
+        if (G < 1)
+            return $"{M:N1}Mb";
+        return $"{G:N1}Gb";
+    }
+
+    public override string ToString() => BytesToHumanReadable(Bytes);
+    
+    public static implicit operator DataSize(long bytes) => new DataSize(bytes);
+}
+
--- a/Млаумчерб.Клиент/сеть/NetworkProgressReporter.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkProgressReporter.cs
@@ -0,0 +1,62 @@
+using Млаумчерб.Клиент.видимое;
+using Timer = DTLib.Timer;
+
+namespace Млаумчерб.Клиент.сеть;
+
+public record struct DownloadProgress(DataSize Downloaded, DataSize Total, DataSize PerSecond)
+{
+    public override string ToString()
+    {
+        return $"{Downloaded}/{Total} ({PerSecond}/s)";
+    }
+}
+
+public class NetworkProgressReporter : IDisposable
+{
+    private readonly Action<DownloadProgress> _reportProgressDelegate;
+    private long _totalSize;
+    private long _curSize;
+    private long _prevSize;
+    private int _timerDelay = 1000;
+    private Timer _timer;
+    
+    public NetworkProgressReporter(long totalSize, Action<DownloadProgress> reportProgressDelegate)
+    {
+        _totalSize = totalSize;
+        _reportProgressDelegate = reportProgressDelegate;
+        _timer = new Timer(true, _timerDelay, ReportProgress);
+    }
+
+    // atomic add
+    public void AddBytesCount(ArraySegment<byte> chunk)
+    {
+        long chunkSize = chunk.Count;
+        Interlocked.Add(ref _curSize, chunkSize);
+    }
+    
+    public void Start()
+    {
+        _timer.Start();
+    }
+
+    public void Stop()
+    {
+        _timer.Stop();
+        _timer.InvokeAction();
+    }
+
+    public void Dispose()
+    {
+        Stop();
+    }
+
+    void ReportProgress()
+    {
+        long bytesPerSec = (_curSize - _prevSize) / (_timerDelay / 1000);
+        _prevSize = _curSize;
+        var p = new DownloadProgress(_curSize, _totalSize, bytesPerSec);
+        Приложение.Логгер.LogDebug(nameof(ReportProgress),
+            $"download progress {p}");
+        _reportProgressDelegate(p);
+    }
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTask.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTask.cs
@@ -0,0 +1,71 @@
+namespace Млаумчерб.Клиент.сеть;
+
+public class NetworkTask : IDisposable
+{
+    public readonly string Name;
+
+    public enum Status
+    {
+        Initialized,
+        Running,
+        Completed,
+        Cancelled,
+        Failed
+    };
+
+    public Status DownloadStatus { get; private set; } = Status.Initialized;
+
+    public event Action<DownloadProgress>? OnProgress;
+    public event Action<Status>? OnStop; 
+
+    public delegate Task DownloadAction(NetworkProgressReporter progressReporter, CancellationToken ct);
+
+    private readonly DownloadAction _downloadAction;
+    private CancellationTokenSource _cts = new();
+    private NetworkProgressReporter _progressReporter;
+
+    public NetworkTask(string name, long dataSize, DownloadAction downloadAction)
+    {
+        Name = name;
+        _downloadAction = downloadAction;
+        _progressReporter = new NetworkProgressReporter(dataSize, ReportProgress);
+    }
+    
+    private void ReportProgress(DownloadProgress p) => OnProgress?.Invoke(p);
+
+    public async Task StartAsync()
+    {
+        if(DownloadStatus == Status.Running || DownloadStatus == Status.Completed)
+            return;
+        DownloadStatus = Status.Running;
+        try
+        {
+            _progressReporter.Start();
+            await _downloadAction(_progressReporter, _cts.Token);
+            DownloadStatus = Status.Completed;
+        }
+        catch
+        {
+            DownloadStatus = Status.Failed;
+            throw;
+        }
+        finally
+        {
+            _progressReporter.Stop();
+            OnStop?.Invoke(DownloadStatus);
+        }
+    } 
+
+    public void Cancel()
+    {
+        DownloadStatus = Status.Cancelled;
+        _cts.Cancel();
+        _cts = new CancellationTokenSource();
+    }
+
+    public void Dispose()
+    {
+        _cts.Dispose();
+        _progressReporter.Dispose();
+    }
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/AssetsDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/AssetsDownloadTaskFactory.cs
@@ -0,0 +1,112 @@
+using System.Security.Cryptography;
+using DTLib.Extensions;
+using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.классы;
+using static Млаумчерб.Клиент.сеть.Сеть;
+
+namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;
+
+public class AssetsDownloadTaskFactory : INetworkTaskFactory
+{
+    private const string ASSET_SERVER_URL = "https://resources.download.minecraft.net/";
+    private VersionDescriptor _descriptor;
+    private SHA1 _hasher;
+    private IOPath _indexFilePath;
+    List<AssetDownloadProperties> _assetsToDownload = new();
+
+    public AssetsDownloadTaskFactory(VersionDescriptor descriptor)
+    {
+        _descriptor = descriptor;
+        _hasher = SHA1.Create();
+        _indexFilePath = Пути.GetAssetIndexFilePath(_descriptor.assetIndex.id);
+    }
+    
+    public async Task<NetworkTask?> CreateAsync(bool checkHashes)
+    {
+        if (!await CheckFilesAsync(checkHashes))
+            return new NetworkTask(
+                $"assets '{_descriptor.assetIndex.id}'",
+                GetTotalSize(),
+                Download
+            );
+        
+        return null;
+    }
+    private async Task<bool> CheckFilesAsync(bool checkHashes)
+    {
+        if(!File.Exists(_indexFilePath))
+        {
+            Приложение.Логгер.LogInfo(nameof(Сеть), $"started downloading asset index to '{_indexFilePath}'");
+            await DownloadFileHTTP(_descriptor.assetIndex.url, _indexFilePath);
+            Приложение.Логгер.LogInfo(nameof(Сеть), "finished downloading asset index");
+        }
+
+        string indexFileText = File.ReadAllText(_indexFilePath);
+        var assetIndex = JsonConvert.DeserializeObject<AssetIndex>(indexFileText) 
+                     ?? throw new Exception($"can't deserialize asset index file '{_indexFilePath}'");
+        
+        _assetsToDownload.Clear();
+        // removing duplicates for Dictionary (idk how can it be possible, but Newtonsoft.Json creates them)
+        HashSet<string> assetHashes = new HashSet<string>();
+        foreach (var pair in assetIndex.objects)
+        {
+            if (assetHashes.Add(pair.Value.hash))
+            {
+                var a = new AssetDownloadProperties(pair.Key, pair.Value);
+                if (!File.Exists(a.filePath))
+                {
+                    _assetsToDownload.Add(a);
+                }
+                else if(checkHashes)
+                {
+                    await using var fs = File.OpenRead(a.filePath);
+                    string hash = _hasher.ComputeHash(fs).HashToString();
+                    if (hash != a.hash)
+                        _assetsToDownload.Add(a);
+                }
+            }
+        }
+        
+        return _assetsToDownload.Count == 0;
+    }
+
+    private long GetTotalSize()
+    {
+        long totalSize = 0;
+        foreach (var a in _assetsToDownload) 
+            totalSize += a.size;
+        return totalSize;
+    }
+
+    private class AssetDownloadProperties
+    {
+        public string name;
+        public string hash;
+        public long size;
+        public string url;
+        public IOPath filePath;
+
+        public AssetDownloadProperties(string key, AssetProperties p)
+        {
+            name = key;
+            hash = p.hash;
+            size = p.size;
+            string hashStart = hash.Substring(0, 2);
+            url = $"{ASSET_SERVER_URL}/{hashStart}/{hash}";
+            filePath = Path.Concat(IOPath.ArrayCast(["assets", "objects", hashStart, hash], true));
+        }
+    }
+    
+    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
+    {
+        Приложение.Логгер.LogInfo(nameof(Сеть), "started downloading assets");
+        ParallelOptions opt = new() { MaxDegreeOfParallelism = ParallelDownloadsN, CancellationToken = ct };
+        await Parallel.ForEachAsync(_assetsToDownload, opt, 
+            async (a, _ct) =>
+            {
+                Приложение.Логгер.LogDebug(nameof(Сеть), $"downloading asset '{a.name}' {a.hash}");
+                await DownloadFileHTTP(a.url, a.filePath, _ct, pr.AddBytesCount);
+            });
+        Приложение.Логгер.LogInfo(nameof(Сеть), "finished downloading assets");
+    }
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/INetworkTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/INetworkTaskFactory.cs
@@ -0,0 +1,7 @@
+namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;
+
+public interface INetworkTaskFactory
+{
+    /// <returns>unstarted network task or null if there is nothing to download</returns>
+    Task<NetworkTask?> CreateAsync(bool checkHashes);
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/JavaDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/JavaDownloadTaskFactory.cs
@@ -0,0 +1,46 @@
+using System.Security.Cryptography;
+using Млаумчерб.Клиент.классы;
+using static Млаумчерб.Клиент.сеть.Сеть;
+
+namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;
+
+public class JavaDownloadTaskFactory : INetworkTaskFactory
+{
+    private VersionDescriptor _descriptor;
+    private SHA1 _hasher;
+    IOPath _javaVersionDir;
+    
+    public JavaDownloadTaskFactory(VersionDescriptor descriptor)
+    {
+        _descriptor = descriptor;
+        _hasher = SHA1.Create();
+        _javaVersionDir = Пути.GetJavaRuntimeDir(_descriptor.javaVersion.component);
+    }
+
+    public Task<NetworkTask?> CreateAsync(bool checkHashes)
+    {
+        NetworkTask? networkTask = null;
+        if (!CheckFiles(checkHashes))
+            networkTask = new(
+                    $"java runtime '{_descriptor.javaVersion.component}'",
+                    GetTotalSize(),
+                    Download
+                );
+        return Task.FromResult(networkTask);
+    }
+
+    private bool CheckFiles(bool checkHashes)
+    {
+        throw new NotImplementedException();
+    }
+    
+    private long GetTotalSize()
+    {
+        throw new NotImplementedException();
+    }
+
+    private Task Download(NetworkProgressReporter pr, CancellationToken ct)
+    {
+        throw new NotImplementedException();
+    }
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/LibrariesDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/LibrariesDownloadTaskFactory.cs
@@ -0,0 +1,78 @@
+using System.Security.Cryptography;
+using DTLib.Extensions;
+using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.классы;
+using static Млаумчерб.Клиент.сеть.Сеть;
+
+namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;
+
+public class LibrariesDownloadTaskFactory : INetworkTaskFactory
+{
+    private VersionDescriptor _descriptor;
+    private Libraries _libraries;
+    private SHA1 _hasher;
+    private List<Libraries.JarLib> _libsToDownload = new();
+
+    public LibrariesDownloadTaskFactory(VersionDescriptor descriptor, Libraries libraries)
+    {
+        _descriptor = descriptor;
+        _libraries = libraries;
+        _hasher = SHA1.Create();
+    }
+    
+    public Task<NetworkTask?> CreateAsync(bool checkHashes)
+    {
+        NetworkTask? networkTask = null;
+        if (!CheckFiles(checkHashes))
+            networkTask = new NetworkTask(
+                $"libraries '{_descriptor.id}'",
+                GetTotalSize(),
+                Download
+            );
+        return Task.FromResult(networkTask);
+    }
+    
+    private bool CheckFiles(bool checkHashes)
+    {
+        _libsToDownload.Clear();
+        
+        foreach (var l in _libraries.Libs)
+        {
+            if (!File.Exists(l.jarFilePath))
+            {
+                _libsToDownload.Add(l);
+            }
+            else if (checkHashes)
+            {
+                using var fs = File.OpenRead(l.jarFilePath);
+                string hash = _hasher.ComputeHash(fs).HashToString();
+                if(hash != l.artifact.sha1)
+                    _libsToDownload.Add(l);
+            }
+        }
+
+        return _libsToDownload.Count == 0;
+    }
+    
+    private long GetTotalSize()
+    {
+        long total = 0;
+        foreach (var l in _libsToDownload)
+            total += l.artifact.size;
+        return total;
+    }
+
+    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
+    {
+        Приложение.Логгер.LogInfo(nameof(Сеть), "started downloading libraries");
+        ParallelOptions opt = new() { MaxDegreeOfParallelism = ParallelDownloadsN, CancellationToken = ct };
+        await Parallel.ForEachAsync(_libsToDownload, opt, async (l, _ct) =>
+        {
+            Приложение.Логгер.LogDebug(nameof(Сеть), 
+                $"downloading library '{l.name}' to '{l.jarFilePath}'");
+            await DownloadFileHTTP(l.artifact.url, l.jarFilePath, _ct, pr.AddBytesCount);
+            //TODO: extract natives from jar
+        });
+        Приложение.Логгер.LogInfo(nameof(Сеть), "finished downloading libraries");
+    }
+}
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/VersionFileDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/VersionFileDownloadTaskFactory.cs
@@ -0,0 +1,53 @@
+using System.Security.Cryptography;
+using DTLib.Extensions;
+using Млаумчерб.Клиент.классы;
+using static Млаумчерб.Клиент.сеть.Сеть;
+
+namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;
+
+public class VersionFileDownloadTaskFactory : INetworkTaskFactory
+{
+    private VersionDescriptor _descriptor;
+    private IOPath _filePath;
+    private SHA1 _hasher;
+    
+    public VersionFileDownloadTaskFactory(VersionDescriptor descriptor)
+    {
+        _descriptor = descriptor;
+        _filePath = Пути.GetVersionJarFilePath(_descriptor.id);
+        _hasher = SHA1.Create();
+    }
+
+    public Task<NetworkTask?> CreateAsync(bool checkHashes)
+    {
+        NetworkTask? networkTask = null;
+        if (!CheckFiles(checkHashes))
+            networkTask = new NetworkTask(
+                    $"version file '{_descriptor.id}'",
+                    GetTotalSize(),
+                    Download
+                );
+        return Task.FromResult(networkTask);
+    }
+
+    private bool CheckFiles(bool checkHashes)
+    {
+        if (!File.Exists(_filePath))
+            return false;
+        if (!checkHashes)
+            return true;
+        using var fs = File.OpenRead(_filePath);
+        string hash = _hasher.ComputeHash(fs).HashToString();
+        return hash == _descriptor.downloads.client.sha1;
+    }
+
+    private long GetTotalSize()
+    {
+        return _descriptor.downloads.client.size;
+    }
+    
+    private Task Download(NetworkProgressReporter pr, CancellationToken ct)
+    {
+        return DownloadFileHTTP(_descriptor.downloads.client.url, _filePath, ct, pr.AddBytesCount);
+    }
+}
--- a/Млаумчерб.Клиент/сеть/Сеть.cs
+++ b/Млаумчерб.Клиент/сеть/Сеть.cs
@@ -0,0 +1,97 @@
+using System.Buffers;
+using System.Net.Http;
+using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.классы;
+
+namespace Млаумчерб.Клиент.сеть;
+
+public static class Сеть
+{
+    public static int ParallelDownloadsN = 32;
+    public static HttpClient http = new();
+
+    public static async Task DownloadFileHTTP(string url, IOPath outPath, CancellationToken ct = default,
+        Action<ArraySegment<byte>>? transformFunc = null)
+    {
+        await using var src = await http.GetStreamAsync(url, ct);
+        await using var dst = File.OpenWrite(outPath);
+
+        await src.CopyTransformAsync(dst, transformFunc, ct).ConfigureAwait(false);
+    }
+
+    public static async Task CopyTransformAsync(this Stream src, Stream dst,
+        Action<ArraySegment<byte>>? transformFunc = null, CancellationToken ct = default)
+    {
+        // default dotnet runtime buffer size
+        int bufferSize = 81920;
+        byte[] readBuffer = ArrayPool<byte>.Shared.Rent(bufferSize);
+        byte[] writeBuffer = ArrayPool<byte>.Shared.Rent(bufferSize);
+        try
+        {
+            var readTask = src.ReadAsync(readBuffer, 0, bufferSize, ct).ConfigureAwait(false);
+            while (true)
+            {
+                int readCount = await readTask;
+                if (readCount == 0)
+                    break;
+                (readBuffer, writeBuffer) = (writeBuffer, readBuffer);
+                readTask = src.ReadAsync(readBuffer, 0, bufferSize, ct).ConfigureAwait(false);
+                transformFunc?.Invoke(new ArraySegment<byte>(writeBuffer, 0, readCount));
+                dst.Write(writeBuffer, 0, readCount);
+            }
+        }
+        catch (OperationCanceledException)
+        {
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(readBuffer);
+            ArrayPool<byte>.Shared.Return(writeBuffer);
+        }
+    }
+    
+    private static readonly string[] VERSION_MANIFEST_URLS =
+    {
+        "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"
+    };
+
+    private static async Task<List<RemoteVersionDescriptorProps>> GetRemoteVersionDescriptorsAsync()
+    {
+        List<RemoteVersionDescriptorProps> descriptors = new();
+        foreach (var url in VERSION_MANIFEST_URLS)
+        {
+            try
+            {
+                var manifestText = await http.GetStringAsync(url);
+                var catalog = JsonConvert.DeserializeObject<VersionCatalog>(manifestText);
+                if (catalog != null)
+                    descriptors.AddRange(catalog.versions);
+            }
+            catch (Exception ex)
+            {
+                Приложение.Логгер.LogWarn(nameof(Сеть), ex);
+            }
+        }
+
+        return descriptors;
+    }
+
+    private static List<GameVersionProps>? _versionPropsList;
+
+    /// <returns>empty list if couldn't find any remote versions</returns>
+    public static async Task<IReadOnlyList<GameVersionProps>> GetDownloadableVersions()
+    {
+        if (_versionPropsList == null)
+        {
+            _versionPropsList = new();
+            var rvdlist = await GetRemoteVersionDescriptorsAsync();
+            foreach (var r in rvdlist)
+            {
+                if (r.type == "release")
+                    _versionPropsList.Add(new GameVersionProps(r.id, r.url));
+            }
+        }
+
+        return _versionPropsList;
+    }
+}