its downloading!!!

2024-11-04 00:41:25 +05:00
parent ae7e5096fc
commit 612976dfe6
35 changed files with 635 additions and 297 deletions
--- a/Млаумчерб.Клиент/сеть/NetworkProgressReporter.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkProgressReporter.cs
@@ -1,4 +1,4 @@
-using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.зримое;
 using Timer = DTLib.Timer;

 namespace Млаумчерб.Клиент.сеть;
@@ -28,10 +28,9 @@ public class NetworkProgressReporter : IDisposable
    }

    // atomic add
-    public void AddBytesCount(ArraySegment<byte> chunk)
+    public void AddBytesCount(byte[] buffer, int offset, int count)
    {
-        long chunkSize = chunk.Count;
-        Interlocked.Add(ref _curSize, chunkSize);
+        Interlocked.Add(ref _curSize, count);
    }
    
    public void Start()
--- a/Млаумчерб.Клиент/сеть/NetworkTask.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTask.cs
@@ -7,7 +7,7 @@ public class NetworkTask : IDisposable
    public enum Status
    {
        Initialized,
-        Running,
+        Started,
        Completed,
        Cancelled,
        Failed
@@ -15,6 +15,7 @@ public class NetworkTask : IDisposable

    public Status DownloadStatus { get; private set; } = Status.Initialized;

+    public event Action? OnStart; 
    public event Action<DownloadProgress>? OnProgress;
    public event Action<Status>? OnStop; 

@@ -35,15 +36,20 @@ public class NetworkTask : IDisposable

    public async Task StartAsync()
    {
-        if(DownloadStatus == Status.Running || DownloadStatus == Status.Completed)
+        if(DownloadStatus == Status.Started || DownloadStatus == Status.Completed)
            return;
-        DownloadStatus = Status.Running;
+        DownloadStatus = Status.Started;
        try
        {
            _progressReporter.Start();
+            OnStart?.Invoke();
            await _downloadAction(_progressReporter, _cts.Token);
            DownloadStatus = Status.Completed;
        }
+        catch (OperationCanceledException)
+        {
+            DownloadStatus = Status.Cancelled;
+        }
        catch
        {
            DownloadStatus = Status.Failed;
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/AssetsDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/AssetsDownloadTaskFactory.cs
@@ -1,6 +1,8 @@
-using System.Security.Cryptography;
+using System.Net;
+using System.Net.Http;
+using System.Security.Cryptography;
 using DTLib.Extensions;
-using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.зримое;
 using Млаумчерб.Клиент.классы;
 using static Млаумчерб.Клиент.сеть.Сеть;

@@ -37,7 +39,7 @@ public class AssetsDownloadTaskFactory : INetworkTaskFactory
        if(!File.Exists(_indexFilePath))
        {
            Приложение.Логгер.LogInfo(nameof(Сеть), $"started downloading asset index to '{_indexFilePath}'");
-            await DownloadFileHTTP(_descriptor.assetIndex.url, _indexFilePath);
+            await DownloadFile(_descriptor.assetIndex.url, _indexFilePath);
            Приложение.Логгер.LogInfo(nameof(Сеть), "finished downloading asset index");
        }

@@ -86,7 +88,7 @@ public class AssetsDownloadTaskFactory : INetworkTaskFactory
        public string url;
        public IOPath filePath;

-        public AssetDownloadProperties(string key, GameAssetProperties p)
+        public AssetDownloadProperties(string key, AssetProperties p)
        {
            name = key;
            hash = p.hash;
@@ -100,13 +102,35 @@ public class AssetsDownloadTaskFactory : INetworkTaskFactory
    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
    {
        Приложение.Логгер.LogInfo(nameof(Сеть), $"started downloading assets '{_descriptor.assetIndex.id}'");
-        ParallelOptions opt = new() { MaxDegreeOfParallelism = ParallelDownloadsN, CancellationToken = ct };
+        ParallelOptions opt = new()
+        {
+            MaxDegreeOfParallelism = Приложение.Настройки.максимум_параллельных_загрузок,
+            CancellationToken = ct
+        };
        await Parallel.ForEachAsync(_assetsToDownload, opt, 
-            async (a, _ct) =>
+        async (a, _ct) =>
+        {
+            bool completed = false;
+            while(!completed)
            {
                Приложение.Логгер.LogDebug(nameof(Сеть), $"downloading asset '{a.name}' {a.hash}");
-                await DownloadFileHTTP(a.url, a.filePath, _ct, pr.AddBytesCount);
-            });
+                try
+                {
+                    await DownloadFile(a.url, a.filePath, _ct, pr.AddBytesCount);
+                    completed = true;
+                }
+                catch (HttpRequestException httpException)
+                {
+                    // wait on rate limit
+                    if(httpException.StatusCode == HttpStatusCode.TooManyRequests)
+                    {
+                        Приложение.Логгер.LogDebug(nameof(Сеть), "rate limit hit");
+                        await Task.Delay(1000, _ct);
+                    }
+                    else throw;
+                }
+            }
+        });
        Приложение.Логгер.LogInfo(nameof(Сеть), $"finished downloading assets '{_descriptor.assetIndex.id}'");
    }
 }
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/JavaDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/JavaDownloadTaskFactory.cs
@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
-using EasyCompressor;
+using DTLib.Extensions;
+using Млаумчерб.Клиент.зримое;
 using Млаумчерб.Клиент.классы;
 using static Млаумчерб.Клиент.сеть.Сеть;

@@ -7,23 +8,27 @@ namespace Млаумчерб.Клиент.сеть.NetworkTaskFactories;

 public class JavaDownloadTaskFactory : INetworkTaskFactory
 {
-    private const string INDEX_URL =
+    private const string CATALOG_URL =
        "https://launchermeta.mojang.com/v1/products/java-runtime/2ec0cc96c44e5a76b9c8b7c39df7210883d12871/all.json";
    private GameVersionDescriptor _descriptor;
    private IOPath _javaVersionDir;
    private SHA1 _hasher;
-    private LZMACompressor _lzma;
+    private JavaDistributiveManifest? _distributiveManifest;
+    private List<(IOPath path, JavaDistributiveElementProps props)> _filesToDownload = new();
    
    public JavaDownloadTaskFactory(GameVersionDescriptor descriptor)
    {
        _descriptor = descriptor;
        _javaVersionDir = Пути.GetJavaRuntimeDir(_descriptor.javaVersion.component);
        _hasher = SHA1.Create();
-        _lzma = new LZMACompressor();
    }

-    public Task<NetworkTask?> CreateAsync(bool checkHashes)
+    public async Task<NetworkTask?> CreateAsync(bool checkHashes)
    {
+        var catalog = await DownloadStringAndDeserialize<JavaVersionCatalog>(CATALOG_URL);
+        var versionProps = catalog.GetVersionProps(_descriptor.javaVersion);
+        _distributiveManifest = await DownloadStringAndDeserialize<JavaDistributiveManifest>(versionProps.manifest.url);
+        
        NetworkTask? networkTask = null;
        if (!CheckFiles(checkHashes))
            networkTask = new(
@@ -31,26 +36,87 @@ public class JavaDownloadTaskFactory : INetworkTaskFactory
                    GetTotalSize(),
                    Download
                );
-        return Task.FromResult(networkTask);
+        return networkTask;
    }

    private bool CheckFiles(bool checkHashes)
    {
-        //TODO: download catalog
-        //TODO: download manifest for required runtime
-        //TODO: check whether files from manifest exist and match hashes
-        throw new NotImplementedException();
+        _filesToDownload.Clear();
+        foreach (var pair in _distributiveManifest!.files)
+        {
+            if (pair.Value.type != "file")
+                continue;
+
+            if (pair.Value.downloads != null)
+            {
+                var artifact = pair.Value.downloads;
+                IOPath file_path = Path.Concat(_javaVersionDir, pair.Key);
+                if (!File.Exists(file_path))
+                {
+                    _filesToDownload.Add((file_path, pair.Value));
+                }
+                else if(checkHashes)
+                {
+                    using var fs = File.OpenRead(file_path);
+                    if (_hasher.ComputeHash(fs).HashToString() != artifact.raw.sha1)
+                    {
+                        _filesToDownload.Add((file_path, pair.Value));
+                    }
+                }
+            }
+        }
+
+        return _filesToDownload.Count == 0;
    }
    
    private long GetTotalSize()
    {
-        //TODO: sum up size of all files invalidated by CheckFiles
-        throw new NotImplementedException();
+        long totalSize = 0;
+        foreach (var file in _filesToDownload)
+        {
+            if(file.props.downloads == null)
+                continue;
+            totalSize += file.props.downloads.lzma?.size ?? file.props.downloads.raw.size;
+        }
+        return totalSize;
    }

-    private Task Download(NetworkProgressReporter pr, CancellationToken ct)
+    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
    {
-        //TODO: download files using lzma decompression
-        throw new NotImplementedException();
+        Приложение.Логгер.LogInfo(nameof(Сеть), "started downloading java runtime " +
+                    $"{_descriptor.javaVersion.majorVersion} '{_descriptor.javaVersion.component}'");
+        
+        ParallelOptions opt = new()
+        {
+            MaxDegreeOfParallelism = Приложение.Настройки.максимум_параллельных_загрузок,
+            CancellationToken = ct
+        };
+        await Parallel.ForEachAsync(_filesToDownload, opt, async (f, _ct) =>
+        {
+            if (f.props.downloads!.lzma != null)
+            {
+                Приложение.Логгер.LogDebug(nameof(Сеть), $"downloading lzma-compressed file '{f.path}'");
+                await using var pipe = new TransformStream(await GetStream(f.props.downloads.lzma.url, _ct));
+                pipe.AddTransform(pr.AddBytesCount);
+                await using var fs = File.OpenWrite(f.path);
+                LZMACompressor.Decompress(pipe, fs);
+            }
+            else
+            {
+                Приложение.Логгер.LogDebug(nameof(Сеть), $"downloading raw file '{f.path}'");
+                await DownloadFile(f.props.downloads.raw.url, f.path, _ct, pr.AddBytesCount);
+            }
+            
+            if(!OperatingSystem.IsWindows() && f.props.executable is true)
+            {
+                Приложение.Логгер.LogDebug(nameof(Сеть), $"adding execute rights to file '{f.path}'");
+                System.IO.File.SetUnixFileMode(f.path.ToString(), UnixFileMode.UserExecute);
+            }
+        });
+        
+        Приложение.Логгер.LogInfo(nameof(Сеть), "finished downloading java runtime " + 
+                    $"{_descriptor.javaVersion.majorVersion} '{_descriptor.javaVersion.component}'");
    }
+    
+    
 }
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/LibrariesDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/LibrariesDownloadTaskFactory.cs
@@ -1,7 +1,7 @@
 using System.IO.Compression;
 using System.Security.Cryptography;
 using DTLib.Extensions;
-using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.зримое;
 using Млаумчерб.Клиент.классы;
 using static Млаумчерб.Клиент.сеть.Сеть;

@@ -73,12 +73,16 @@ public class LibrariesDownloadTaskFactory : INetworkTaskFactory
    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
    {
        Приложение.Логгер.LogInfo(nameof(Сеть), $"started downloading libraries '{_descriptor.id}'");
-        ParallelOptions opt = new() { MaxDegreeOfParallelism = ParallelDownloadsN, CancellationToken = ct };
+        
+        ParallelOptions opt = new()
+        {
+            MaxDegreeOfParallelism = Приложение.Настройки.максимум_параллельных_загрузок,
+            CancellationToken = ct
+        };
        await Parallel.ForEachAsync(_libsToDownload, opt, async (l, _ct) =>
        {
-            Приложение.Логгер.LogDebug(nameof(Сеть), 
-                $"downloading library '{l.name}' to '{l.jarFilePath}'");
-            await DownloadFileHTTP(l.artifact.url, l.jarFilePath, _ct, pr.AddBytesCount);
+            Приложение.Логгер.LogDebug(nameof(Сеть), $"downloading library '{l.name}' to '{l.jarFilePath}'");
+            await DownloadFile(l.artifact.url, l.jarFilePath, _ct, pr.AddBytesCount);
            if (l is Libraries.NativeLib n)
            {
                var zipf = File.OpenRead(n.jarFilePath);
@@ -96,6 +100,7 @@ public class LibrariesDownloadTaskFactory : INetworkTaskFactory
                }
            }
        });
+        
        Приложение.Логгер.LogInfo(nameof(Сеть), $"finished downloading libraries '{_descriptor.id}'");
    }
 }
--- a/Млаумчерб.Клиент/сеть/NetworkTaskFactories/VersionFileDownloadTaskFactory.cs
+++ b/Млаумчерб.Клиент/сеть/NetworkTaskFactories/VersionFileDownloadTaskFactory.cs
@@ -1,6 +1,6 @@
 using System.Security.Cryptography;
 using DTLib.Extensions;
-using Млаумчерб.Клиент.видимое;
+using Млаумчерб.Клиент.зримое;
 using Млаумчерб.Клиент.классы;
 using static Млаумчерб.Клиент.сеть.Сеть;

@@ -50,7 +50,7 @@ public class VersionFileDownloadTaskFactory : INetworkTaskFactory
    private async Task Download(NetworkProgressReporter pr, CancellationToken ct)
    {
        Приложение.Логгер.LogInfo(nameof(Сеть), $"started downloading version file '{_descriptor.id}'");
-        await DownloadFileHTTP(_descriptor.downloads.client.url, _filePath, ct, pr.AddBytesCount);
+        await DownloadFile(_descriptor.downloads.client.url, _filePath, ct, pr.AddBytesCount);
        Приложение.Логгер.LogInfo(nameof(Сеть), $"finished downloading version file '{_descriptor.id}'");
    }
 }
--- a/Млаумчерб.Клиент/сеть/Сеть.cs
+++ b/Млаумчерб.Клиент/сеть/Сеть.cs
@@ -1,53 +1,46 @@
-using System.Buffers;
-using System.Net.Http;
-using Млаумчерб.Клиент.видимое;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using Млаумчерб.Клиент.зримое;
 using Млаумчерб.Клиент.классы;

 namespace Млаумчерб.Клиент.сеть;

 public static class Сеть
 {
-    public static int ParallelDownloadsN = 32;
-    public static HttpClient http = new();
+    private static HttpClient _http = new();

-    public static async Task DownloadFileHTTP(string url, IOPath outPath, CancellationToken ct = default,
-        Action<ArraySegment<byte>>? transformFunc = null)
+    static Сеть()
    {
-        await using var src = await http.GetStreamAsync(url, ct);
-        await using var dst = File.OpenWrite(outPath);
-
-        await src.CopyTransformAsync(dst, transformFunc, ct).ConfigureAwait(false);
+        // thanks for Sashok :3
+        // https://github.com/new-sashok724/Launcher/blob/23485c3f7de6620d2c6b7b2dd9339c3beb6a0366/Launcher/source/helper/IOHelper.java#L259
+        _http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)");
    }

-    public static async Task CopyTransformAsync(this Stream src, Stream dst,
-        Action<ArraySegment<byte>>? transformFunc = null, CancellationToken ct = default)
+    public static Task<string> GetString(string url, CancellationToken ct = default) => _http.GetStringAsync(url, ct);
+    public static Task<Stream> GetStream(string url, CancellationToken ct = default) => _http.GetStreamAsync(url, ct);
+    
+    public static async Task DownloadFile(string url, Stream outStream, CancellationToken ct = default, 
+        params TransformStream.TransformFuncDelegate[] transforms)
    {
-        // default dotnet runtime buffer size
-        int bufferSize = 81920;
-        byte[] readBuffer = ArrayPool<byte>.Shared.Rent(bufferSize);
-        byte[] writeBuffer = ArrayPool<byte>.Shared.Rent(bufferSize);
-        try
-        {
-            var readTask = src.ReadAsync(readBuffer, 0, bufferSize, ct).ConfigureAwait(false);
-            while (true)
-            {
-                int readCount = await readTask;
-                if (readCount == 0)
-                    break;
-                (readBuffer, writeBuffer) = (writeBuffer, readBuffer);
-                readTask = src.ReadAsync(readBuffer, 0, bufferSize, ct).ConfigureAwait(false);
-                transformFunc?.Invoke(new ArraySegment<byte>(writeBuffer, 0, readCount));
-                dst.Write(writeBuffer, 0, readCount);
-            }
-        }
-        catch (OperationCanceledException)
-        {
-        }
-        finally
-        {
-            ArrayPool<byte>.Shared.Return(readBuffer);
-            ArrayPool<byte>.Shared.Return(writeBuffer);
-        }
+        await using var pipe = new TransformStream(await GetStream(url, ct));
+        if (transforms.Length > 0) 
+            pipe.AddTransforms(transforms);
+        await pipe.CopyToAsync(outStream, ct);
+    }
+    
+    public static async Task DownloadFile(string url, IOPath outPath, CancellationToken ct = default, 
+        params TransformStream.TransformFuncDelegate[] transforms)
+    {
+        await using var file = File.OpenWrite(outPath);
+        await DownloadFile(url, file, ct, transforms);
+    }
+    
+    public static async Task<T> DownloadStringAndDeserialize<T>(string url)
+    {
+        var text = await _http.GetStringAsync(url);
+        var result = JsonConvert.DeserializeObject<T>(text)
+            ?? throw new Exception($"can't deserialize {typeof(T).Name}");
+        return result;
    }
    
    private static readonly string[] VERSION_MANIFEST_URLS =
@@ -62,10 +55,8 @@ public static class Сеть
        {
            try
            {
-                var manifestText = await http.GetStringAsync(url);
-                var catalog = JsonConvert.DeserializeObject<GameVersionCatalog>(manifestText);
-                if (catalog != null)
-                    descriptors.AddRange(catalog.versions);
+                var catalog = await DownloadStringAndDeserialize<GameVersionCatalog>(url);
+                descriptors.AddRange(catalog.versions);
            }
            catch (Exception ex)
            {