ffmpeg wrapper

DTLib

@@ -1 +1 @@
-Subproject commit eec2ec60bee84dc9b307c2e0a2803e399fca02ca
+Subproject commit 3ebb5be5819fa46d36705a5752aecd885c406a8b

VkAudioDownloader.CLI/Program.cs

@@ -6,45 +6,47 @@ using VkAudioDownloader;
 using DTLib.Logging.New;
 using VkAudioDownloader.VkM3U8;
 
-
 if(!File.Exists("config.dtsod"))
+{
     File.Copy("config.dtsod.default", "config.dtsod");
+    throw new Exception("No config detected, default created. Edit it!");
+}
+
+var config = VkClientConfig.FromDtsod(new DtsodV23(File.ReadAllText("config.dtsod")));
 
 var logger = new CompositeLogger(new DefaultLogFormat(true), 
     new ConsoleLogger(), 
             new FileLogger("logs", "VkAudioDownloaer"));
+var _logger = new LoggerContext(logger, "main");
+
+try
+{
 #if DEBUG
-logger.DebugLogEnabled = true;
-#endif
     AudioAesDecryptor.TestAes();
+#endif
+    
+    var client = new VkClient(config, logger);
+    _logger.LogDebug("initializing api...");
+    await client.ConnectAsync();
     
-var client = new VkClient(
-    VkClientConfig.FromDtsod(new DtsodV23(File.ReadAllText("config.dtsod"))), 
-        logger);
-logger.Log("main", LogSeverity.Debug, "initializing api...");
-client.Connect();
 // getting audio from vk
-var http = new HttpHelper();
-var audio = client.FindAudio("гражданская оборона", 1).First();
-Console.WriteLine($"{audio.Title} -- {audio.Artist} [{TimeSpan.FromSeconds(audio.Duration)}]");
-var m3u8 = await http.GetStringAsync(audio.Url);
-Console.WriteLine("downloaded m3u8 playlist\n");
-// parsing index.m3u8
-var parser = new M3U8Parser();
-var playlist = parser.Parse(audio.Url, m3u8);
-Console.WriteLine(playlist);
-// downloading parts
-var frag = playlist.Fragments[3];
-var kurl =frag.EncryptionKeyUrl ?? throw new NullReferenceException();
-await http.DownloadAsync(kurl, "key.pub");
-if(Directory.Exists("playlist"))
-    Directory.Delete("playlist",true);
-Directory.CreateDirectory("playlist");
-await http.DownloadAsync(playlist, "playlist");
+    var audios = client.FindAudio("сталинский костюм").ToArray();
 
-// var decryptor = new AudioAesDecryptor();
-// string key = "cca42800074d7aeb";
-// using var encryptedFile = File.Open("encrypted.ts", FileMode.Open, FileAccess.ReadWrite);
-// using var cryptoStream = decryptor.DecryptStream(encryptedFile, key);
-// using var decryptedFile = File.Open("out.ts", FileMode.Create);
-// await cryptoStream.CopyToAsync(decryptedFile);
+    for (var i = 0; i < audios.Length; i++)
+    {
+        var a = audios[i];
+        Console.WriteLine($"[{i}] {a.AudioToString()}");
+    }
+
+    Console.Write("choose audio: ");
+    int ain = Convert.ToInt32(Console.ReadLine());
+    var audio = audios[ain];
+    Console.WriteLine($"selected \"{audio.Title}\" -- {audio.Artist} [{TimeSpan.FromSeconds(audio.Duration)}]");
+    // downloading parts
+    string downloadedFile = await client.DownloadAudioAsync(audio, "downloads");
+    _logger.LogInfo($"audio {audio.AudioToString()} downloaded to {downloadedFile}");
+}
+catch (Exception ex)
+{
+    _logger.LogException(ex);
+}

VkAudioDownloader.CLI/config.dtsod.default

@@ -1,3 +1,4 @@
 app_id: 0ul;
 login: "";
 password: "";
+ffmpeg_dir: "";

VkAudioDownloader.sln.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeEditing/SuppressUninitializedWarningFix/Enabled/@EntryValue">False</s:Boolean></wpf:ResourceDictionary>

VkAudioDownloader/AudioHelper.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Runtime.CompilerServices;
+using VkNet.Model.Attachments;
+
+namespace VkAudioDownloader;
+
+public static class AudioHelper
+{
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static string AudioToString(this Audio a)
+        => $"\"{a.Title}\" -- {a.Artist} ({TimeSpan.FromSeconds(a.Duration)})";
+}

VkAudioDownloader/FFMPegHelper.cs

@@ -0,0 +1,104 @@
+using System;
+using CliWrap;
+using DTLib.Filesystem;
+using DTLib.Extensions;
+using DTLib.Logging.New;
+
+namespace VkAudioDownloader;
+
+public class FFMPegHelper
+{
+    private LoggerContext _logger;
+    private readonly string ffmpeg;    
+    public FFMPegHelper(ILogger logger, string ffmpegDir)
+    {
+        _logger = new LoggerContext(logger, nameof(FFMPegHelper));
+        ffmpeg=Path.Concat(ffmpegDir,"ffmpeg");
+    }
+    
+    /// <summary>creates fragments list for ffmppeg concat</summary>
+    /// <param name="fragmentsDir">there file list.txt will be created</param>
+    /// <param name="fragments">audio files in fragmentsDir (with or without dir in path)</param>
+    /// <returns></returns>
+    public string CreateFragmentList(string fragmentsDir, string[] fragments)
+    {
+        string listFile = Path.Concat(fragmentsDir, "list.txt");
+        using var playlistFile = File.OpenWrite(listFile);
+        for (var i = 0; i < fragments.Length; i++)
+        {
+            var clearFileName = fragments[i].AsSpan().AfterLast(Path.Sep);
+            playlistFile.Write($"file '{clearFileName}'\n".ToBytes(StringConverter.UTF8));
+        }
+
+        return listFile;
+    }
+    
+    
+    /// <summary>converts ts files in directory to opus</summary>
+    /// <param name="localDir">directory with ts fragment files</param>
+    /// <returns>paths to created opus files</returns>
+    public Task<string[]> ToOpus(string localDir) => 
+        ToOpus(Directory.GetFiles(localDir, "*.ts"));
+
+    /// <summary>
+    /// converts ts files in to opus
+    /// </summary>
+    /// <param name="fragments">ts fragment files</param>
+    /// <returns>paths to created opus files</returns>
+    public async Task<string[]> ToOpus(string[] fragments)
+    {
+        string[] output = new string[fragments.Length];
+        var tasks = new Task<CommandResult>[fragments.Length];
+        
+        for (var i = 0; i < fragments.Length; i++)
+        {
+            string tsFile = fragments[i];
+            string opusFile = tsFile.Replace(".ts",".opus");
+            _logger.LogDebug($"{tsFile} -> {opusFile}");
+            var command = Cli.Wrap(ffmpeg).WithArguments(new[]
+                {
+                    "-i", tsFile, // input
+                    "-loglevel", "warning", "-hide_banner", "-nostats", // print only warnings and errors
+                    "-map", "0:0", // select first audio track (sometimes there are blank buggy second thack) 
+                    "-filter:a", "asetpts=PTS-STARTPTS", // fixes pts
+                    "-c", "libopus", "-b:a", "96k", // encoding params
+                    opusFile // output
+                })
+                // ffmpeg prints all log to stderr, because in stdout it ptints encoded file
+                .WithStandardErrorPipe(PipeTarget.ToDelegate(
+                    msg => _logger.LogWarn(msg)));
+
+            tasks[i] = command.ExecuteAsync();
+            output[i] = opusFile;
+        }
+
+        await Task.WhenAll(tasks);
+        return output;
+    }
+
+    public async Task Concat(string outfile, string fragmentListFile, string codec="libopus")
+    {
+        _logger.LogDebug($"{fragmentListFile} -> {outfile}");
+        var command = Cli.Wrap(ffmpeg).WithArguments(new[]
+            {
+                "-f", "concat", // mode
+                "-i", fragmentListFile, // input list
+                "-loglevel", "warning", "-hide_banner", "-nostats", // print only warnings and errors
+                "-filter:a", "asetpts=PTS-STARTPTS", // fixes pts
+                "-c", codec, "-b:a", "96k", // encoding params
+                outfile, "-y" // output override
+            })
+            // ffmpeg prints all log to stderr, because in stdout it ptints encoded file
+            .WithStandardErrorPipe(PipeTarget.ToDelegate(
+                msg => _logger.LogWarn(msg)))
+            .WithValidation(CommandResultValidation.None);
+        
+        var rezult =await command.ExecuteAsync();
+        // log time
+        if (rezult.ExitCode != 0)
+        {
+            _logger.LogError($"command failed with code {rezult.ExitCode}");
+            throw new Exception($"command: {command} failed");
+        }
+    }
+}

VkAudioDownloader/VkAudioDownloader.csproj

@@ -17,4 +17,8 @@
         <ProjectReference Include="..\vknet\VkNet\VkNet.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="CliWrap" Version="3.5.0" />
+    </ItemGroup>
+
 </Project>

VkAudioDownloader/VkClient.cs

@@ -1,6 +1,4 @@
 using System;
-using System.IO;
-using System.Net.Http;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using VkNet;
@@ -12,7 +10,8 @@ using VkNet.Model.Attachments;
 using VkNet.AudioBypassService.Extensions;
 using DTLib.Logging.DependencyInjection;
 using DTLib.Logging.New;
-using ILogger = DTLib.Logging.New.ILogger;
+using DTLib.Filesystem;
+using VkAudioDownloader.VkM3U8;
 
 namespace VkAudioDownloader;
 
@@ -22,19 +21,23 @@ public class VkClient : IDisposable
 {
     public VkApi Api;
     public VkClientConfig Config;
-    private ILogger _logger;
+    private  DTLib.Logging.New.ILogger _logger;
+    private HttpHelper _http;
+    private FFMPegHelper _ffmpeg;
     
-    public VkClient(VkClientConfig conf, ILogger logger)
+    public VkClient(VkClientConfig conf,  DTLib.Logging.New.ILogger logger)
     {
         Config = conf;
         _logger = logger;
+        _http = new HttpHelper();
+        _ffmpeg = new FFMPegHelper(logger,conf.FFMPegDir);
         var services = new ServiceCollection()
             .Add(new LoggerService<VkApi>(logger))
             .AddAudioBypass();
         Api = new VkApi(services);
     }
 
-    public void Connect()
+    public async Task ConnectAsync(int attempts=5)
     {
         var authParams = new ApiAuthParams
         {
@@ -52,7 +55,19 @@ public class VkClient : IDisposable
             authParams.Login = Config.Login;
             authParams.Password = Config.Password;
         }
-        Api.Authorize(authParams);
+
+        for (int authAttempt = 0; authAttempt < attempts; authAttempt++)
+        {
+            try
+            {
+                await Api.AuthorizeAsync(authParams);
+                break;
+            }
+            catch (Exception aex)
+            {
+                _logger.LogException(nameof(VkClient),aex);
+            }
+        }
     }
 
     public VkCollection<Audio> FindAudio(string query, int maxRezults=10) =>
@@ -62,22 +77,41 @@ public class VkClient : IDisposable
             Count = maxRezults,
         });
     
-    public Stream DownloadAudio(Audio audio)
-    {
-        HttpClient http = new HttpClient();
-        var stream = http.GetStreamAsync(audio.Url).GetAwaiter().GetResult();
-        return stream;
-    }
-    public void DownloadAudio(Audio audio, TimeSpan lengthLimit)
-    {
     
+    public Task<string> DownloadAudioAsync(Audio audio, string localDir) => 
+        DownloadAudioAsync(audio, localDir,TimeSpan.FromHours(1));
+
+    public async Task<string> DownloadAudioAsync(Audio audio, string localDir, TimeSpan durationLimit)
+    {
+        if (!audio.Url.ToString().StartsWith("http"))
+            throw new Exception($"incorrect audio url: {audio.Url}");
+
+        string outFile = Path.Concat(localDir, DTLib.Filesystem.Path.CorrectString($"{audio.Artist}-{audio.Title}.opus"));
+        string fragmentDir = $"{outFile}_{DateTime.Now.Ticks}";
+        if(File.Exists(outFile))
+            _logger.LogWarn(nameof(VkClient), $"file {outFile} already exists");
+        
+        string m3u8 = await _http.GetStringAsync(audio.Url);
+        var parser = new M3U8Parser();
+        var hls = parser.Parse(audio.Url, m3u8);
+        if (hls.Duration > durationLimit.TotalSeconds)
+            throw new Exception($"duration limit <{durationLimit}> exceeded by track <{audio}> - <{hls.Duration}>");
+        
+        await _http.DownloadAsync(hls, fragmentDir);
+        string[] opusFragments = await _ffmpeg.ToOpus(fragmentDir);
+        string listFile = _ffmpeg.CreateFragmentList(fragmentDir, opusFragments);
+        await _ffmpeg.Concat(outFile, listFile);
+        // Directory.Delete(fragmentDir);
+        
+        return outFile;
     }
 
-    private bool Disposed = false;
+    private bool _disposed = false;
     public void Dispose()
     {
-        if (Disposed) return;
+        if (_disposed) return;
         Api.Dispose();
-        Disposed = true;
+        _http.Dispose();
+        _disposed = true;
     }
 }

VkAudioDownloader/VkClientConfig.cs

@@ -4,6 +4,8 @@ namespace VkAudioDownloader;
 
 public class VkClientConfig
 {
+    /// directory where ffmpeg and ffprobe binaries are stored
+    public string FFMPegDir;
     /// vk app id from https://vk.com/apps?act=manage
     public ulong AppId;
     /// account password
@@ -12,12 +14,32 @@ public class VkClientConfig
     public string? Login;
     /// can be used instead of login and password
     public string? Token;
+
+
+    public VkClientConfig(string ffmPegDir, ulong appId, string? token)
+    {
+        FFMPegDir = ffmPegDir;
+        AppId = appId;
+        Token = token;
+    }
+
+    public VkClientConfig(string ffmPegDir, ulong appId, string? password, string? login)
+    {
+        FFMPegDir = ffmPegDir;
+        AppId = appId;
+        Password = password;
+        Login = login;
+    }
+    
+    private VkClientConfig(string ffmPegDir, ulong appId)
+    {
+        FFMPegDir = ffmPegDir;
+        AppId = appId;
+    }
+
     public static VkClientConfig FromDtsod(DtsodV23 dtsod)
     {
-        var config = new VkClientConfig
-        {
-            AppId = dtsod["app_id"]
-        };
+        var config = new VkClientConfig(dtsod["ffmpeg_dir"], dtsod["app_id"]);
         if (dtsod.TryGetValue("login", out var login))
             config.Login = login;
         if (dtsod.TryGetValue("password", out var password))
@@ -33,7 +55,8 @@ public class VkClientConfig
             { "app_id", AppId },
             { "password", Password ?? null },
             { "login", Login ?? null },
-            { "token", Token ?? null }
+            { "token", Token ?? null },
+            { "ffmpeg_dir", FFMPegDir}
         };
 
 }

VkAudioDownloader/VkM3U8/AudioAesDecryptor.cs

@@ -1,6 +1,7 @@
 using System;
-using System.IO;
 using System.Security.Cryptography;
+using DTLib.Filesystem;
+using Stream = System.IO.Stream;
 
 namespace VkAudioDownloader.VkM3U8;
 

VkAudioDownloader/VkM3U8/HLSPlaylist.cs

@@ -1,18 +1,14 @@
-using System.IO;
-using System.Text;
-
 namespace VkAudioDownloader.VkM3U8;
 
 public class HLSPlaylist
 {
-
-    public HLSFragment[] Fragments { get; internal set; } 
+    public HLSFragment[] Fragments { get; } 
 
     /// content duration in seconds
-    public float Duration { get; internal set; }
+    public float Duration { get; }
     
     /// url before index.m3u8
-    public string BaseUrl { get; internal set; }
+    public string BaseUrl { get; }
     
     internal HLSPlaylist(HLSFragment[] fragments, float duration, string baseUrl)
     {
@@ -25,16 +21,4 @@ public class HLSPlaylist
         $"BaseUrl: {BaseUrl}\n" +
         $"Duration: {Duration}\n" +
         $"Fragments: HLSFragment[{Fragments.Length}]";
-
-    public void CreateFragmentListFile(string path)
-    {
-        using var playlistFile = File.Open(path, FileMode.Create);
-        foreach (var fragment in Fragments)
-        {
-            playlistFile.Write(Encoding.ASCII.GetBytes("file '"));
-            playlistFile.Write(Encoding.ASCII.GetBytes(fragment.Name));
-            playlistFile.WriteByte((byte)'\'');
-            playlistFile.WriteByte((byte)'\n');
-        }
-    }
 }

VkAudioDownloader/VkM3U8/HttpHelper.cs

@@ -1,7 +1,7 @@
 global using System.Threading.Tasks;
-using System.IO;
 using System.Net.Http;
-using System.Text;
+using DTLib.Filesystem;
+using Stream = System.IO.Stream;
 
 
 namespace VkAudioDownloader.VkM3U8;
@@ -29,8 +29,8 @@ public class HttpHelper : HttpClient
         return Decryptor.DecryptStream(fragmentStream, key);
     }
 
-    public async Task DownloadAsync(HLSFragment fragment, string localDir) 
-        => await WriteStreamAsync(await GetStreamAsync(fragment), Path.Combine(localDir, fragment.Name));
+    public async Task DownloadAsync(HLSFragment fragment, string localDir) => 
+        await WriteStreamAsync(await GetStreamAsync(fragment), Path.Concat(localDir, fragment.Name));
 
     public async Task DownloadAsync(HLSPlaylist playlist, string localDir)
     {
@@ -38,7 +38,7 @@ public class HttpHelper : HttpClient
         {
             //TODO log file download progress
             await DownloadAsync(fragment, localDir);
-            playlist.CreateFragmentListFile(Path.Join(localDir, "playlist.txt"));
+            // playlist.CreateFragmentList();
         }
     }
 }

VkAudioDownloader/VkM3U8/M3U8Parser.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using DTLib.Extensions;
 
 namespace VkAudioDownloader.VkM3U8;
 

VkAudioDownloader/VkM3U8/SpanHelper.cs

@@ -1,39 +0,0 @@
-using System;
-
-namespace VkAudioDownloader.VkM3U8;
-
-internal static class SpanHelper
-{
-    public static ReadOnlySpan<char> After(this ReadOnlySpan<char> span, char c)
-    {
-        var index = span.IndexOf(c);
-        if (index == -1)
-            throw new Exception($"char <{c}> not found in span <{span}>");
-        return span.Slice(index+1);
-    }
-    
-    public static ReadOnlySpan<char> After(this ReadOnlySpan<char> span, ReadOnlySpan<char> s)
-    {
-        var index = span.IndexOf(s);
-        if (index == -1)
-            throw new Exception($"span <{s}> not found in span <{span}>");
-        return span.Slice(index+s.Length);
-    }
-    
-    
-    public static ReadOnlySpan<char> Before(this ReadOnlySpan<char> span, char c)
-    {
-        var index = span.IndexOf(c);
-        if (index == -1)
-            throw new Exception($"char <{c}> not found in span <{span}>");
-        return span.Slice(0,index);
-    }
-    
-    public static ReadOnlySpan<char> Before(this ReadOnlySpan<char> span, ReadOnlySpan<char> s)
-    {
-        var index = span.IndexOf(s);
-        if (index == -1)
-            throw new Exception($"span <{s}> not found in span <{span}>");
-        return span.Slice(0,index);
-    }
-}