m3u8 audio downloading, and decrypting

DTLib

@@ -1 +1 @@
-Subproject commit bef458bcdda81ea0fd955597be12b0966804f7ee
+Subproject commit eec2ec60bee84dc9b307c2e0a2803e399fca02ca

VkAudioDownloader.CLI/Program.cs

@@ -1,8 +1,7 @@
 using System;
 using System.Linq;
-using System.Net.Http;
 using DTLib.Dtsod;
-using DTLib.Filesystem;
+using System.IO;
 using VkAudioDownloader;
 using DTLib.Logging.New;
 using VkAudioDownloader.VkM3U8;
@@ -14,17 +13,38 @@ if(!File.Exists("config.dtsod"))
 var logger = new CompositeLogger(new DefaultLogFormat(true), 
     new ConsoleLogger(), 
             new FileLogger("logs", "VkAudioDownloaer"));
+#if DEBUG
+logger.DebugLogEnabled = true;
+#endif
+AudioAesDecryptor.TestAes();
+
 var client = new VkClient(
     VkClientConfig.FromDtsod(new DtsodV23(File.ReadAllText("config.dtsod"))), 
         logger);
-logger.Log("main", LogSeverity.Info, "initializing api...");
+logger.Log("main", LogSeverity.Debug, "initializing api...");
-logger.DebugLogEnabled = true;
 client.Connect();
+// getting audio from vk
+var http = new HttpHelper();
 var audio = client.FindAudio("гражданская оборона", 1).First();
 Console.WriteLine($"{audio.Title} -- {audio.Artist} [{TimeSpan.FromSeconds(audio.Duration)}]");
-var Http = new HttpClient();
+var m3u8 = await http.GetStringAsync(audio.Url);
-var m3u8 = await Http.GetStringAsync(audio.Url);
+Console.WriteLine("downloaded m3u8 playlist\n");
-Console.WriteLine("downloaded m3u8 playlist:\n" + m3u8);
+// parsing index.m3u8
 var parser = new M3U8Parser();
-var HLSPlaylist = parser.Parse(audio.Url, m3u8);
+var playlist = parser.Parse(audio.Url, m3u8);
-Console.WriteLine(HLSPlaylist);
+Console.WriteLine(playlist);
+// downloading parts
+var frag = playlist.Fragments[3];
+var kurl =frag.EncryptionKeyUrl ?? throw new NullReferenceException();
+await http.DownloadAsync(kurl, "key.pub");
+if(Directory.Exists("playlist"))
+    Directory.Delete("playlist",true);
+Directory.CreateDirectory("playlist");
+await http.DownloadAsync(playlist, "playlist");
+
+// var decryptor = new AudioAesDecryptor();
+// string key = "cca42800074d7aeb";
+// using var encryptedFile = File.Open("encrypted.ts", FileMode.Open, FileAccess.ReadWrite);
+// using var cryptoStream = decryptor.DecryptStream(encryptedFile, key);
+// using var decryptedFile = File.Open("out.ts", FileMode.Create);
+// await cryptoStream.CopyToAsync(decryptedFile);

VkAudioDownloader/VkM3U8/AudioAesDecryptor.cs

@@ -0,0 +1,191 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace VkAudioDownloader.VkM3U8;
+
+public class AudioAesDecryptor : IDisposable
+{
+    private Aes aes;
+    private byte[] _key;
+    private byte[] _iv;
+
+    public byte[] Key
+    {
+        get => _key;
+        set
+        {
+            if (value.Length != 16) 
+                throw new Exception($"key.Length!=16, key: [{value.Length}]{{{string.Join(",",value)}}}");
+            _key = value;
+            aes.Key = value;
+        }
+    }
+    public byte[] IV
+    {
+        get => _iv;
+        set
+        {
+            if (value.Length != 16) 
+                throw new Exception($"iv.Length!=16, iv: [{value.Length}]{{{string.Join(",",value)}}}");
+            _iv = value;
+            aes.IV = value;
+        }
+    }
+    
+    public AudioAesDecryptor(byte[] key, byte[] iv)
+    {
+        aes = CreateAes();
+        _iv = iv;
+        _key = key;
+        Key = key;
+        IV = iv;
+    }
+
+    public AudioAesDecryptor() : this(GetZeroIV(), GetZeroIV())
+    {}
+
+    static byte[] GetZeroIV()
+    {
+        var iv= new byte[16];
+        for (int i = 0; i < 16; i++) 
+            iv[i] = 0;
+        return iv;
+    }
+
+    // analog of xxd -p key.pub
+    public static byte[] KeyToBytes(string key)
+    {
+        if (key.Length != 16)
+            throw new Exception($"invalid key string: {key}");
+        var bytes = new byte[16];
+        for (int i = 0; i < 16; i++) 
+            bytes[i] = (byte)key[i];
+        return bytes;
+    }
+
+    public void ResetIV()
+    {
+        aes.IV = _iv;
+    }
+    
+    public Stream DecryptStream(Stream fragment, string key)
+    {
+        if (string.IsNullOrEmpty(key))
+            throw new NullReferenceException("key is null");
+        aes.Key = KeyToBytes(key);
+        aes.IV = GetZeroIV();
+        var decryptor = aes.CreateDecryptor();
+		// aes.Dispose(); decryptor.Dispose();
+        return new CryptoStream(fragment, decryptor, CryptoStreamMode.Read);
+    }
+
+    public static Aes CreateAes()
+    {
+        var aes=Aes.Create();
+        aes.KeySize = 128;
+        // aes.BlockSize = 128;
+        aes.Mode = CipherMode.CBC;
+        aes.Padding = PaddingMode.PKCS7;
+        return aes;
+    }
+
+    static byte HexToByte(char c)
+        =>c switch
+        {
+            '0' => 0,
+            '1' => 1,
+            '2' => 2,
+            '3' => 3,
+            '4' => 4,
+            '5' => 5,
+            '6' => 6,
+            '7' => 7,
+            '8' => 8,
+            '9' => 9,
+            'A' or 'a' => 10,
+            'B' or 'b' => 11,
+            'C' or 'c' => 12,
+            'D' or 'd' => 13,
+            'E' or 'e' => 14,
+            'F' or 'f' => 15,
+            _ => throw new ArgumentOutOfRangeException(nameof(c), c, null)
+        };
+    
+    static char HalfByteToHex(byte b)
+        =>b switch
+        {
+            0 => '0',
+            1 => '1',
+            2 => '2',
+            3 => '3',
+            4 => '4',
+            5 => '5',
+            6 => '6',
+            7 => '7',
+            8 => '8',
+            9 => '9',
+            0xA => 'a',
+            0xB => 'b',
+            0xC => 'c',
+            0xD => 'd',
+            0xE => 'e',
+            0xF => 'f',
+            _ => throw new ArgumentOutOfRangeException(nameof(b), b, null)
+        };
+
+    static byte[] HexToBytes(string hex)
+    {
+        if (hex.Length % 2 != 0)
+            throw new Exception("argument length is not even");
+        byte[] bytes = new byte[hex.Length / 2];
+        for (int i = 0; i < hex.Length; i++)
+        {
+            bytes[i / 2] = (byte)(HexToByte(hex[i]) * 16 + HexToByte(hex[++i]));
+        }
+        return bytes;
+    }
+    static string BytesToHex(byte[] bytes)
+    {
+        if (bytes.Length % 2 != 0)
+            throw new Exception("argument length is not even");
+        char[] hex = new char[bytes.Length * 2];
+        for (int i = 0; i < bytes.Length; i++)
+        {
+            byte b = bytes[i];
+            hex[i * 2] = HalfByteToHex((byte)(b / 16));
+            hex[i * 2 +1] = HalfByteToHex((byte)(b % 16));
+        }
+        return new string(hex);
+    }
+    
+    
+    public static void TestAes()
+    {
+        const string PLAINTEXT =  "6a84867cd77e12ad07ea1be895c53fa3";
+        byte[] PLAINTEXT_B = HexToBytes(PLAINTEXT);
+        const string CIPHERTEXT = "732281c0a0aab8f7a54a0c67a0c45ecfcf52019292387d1b2c9d44c45d418a48";
+        byte[] CIPHERTEXT_B = HexToBytes(CIPHERTEXT);
+        var aes = new AudioAesDecryptor();
+        var padding = PaddingMode.PKCS7;
+        var enc = aes.aes.EncryptCbc(PLAINTEXT_B, aes._iv, padding);
+        var encs = BytesToHex(enc);
+        if (encs != CIPHERTEXT)
+            throw new Exception("encryption went wrong");
+        aes.ResetIV();
+        var dec = aes.aes.DecryptCbc(enc, aes._iv, padding);
+        var decs = BytesToHex(dec);
+        if (decs != PLAINTEXT)
+            throw new Exception("decryption went wrong");
+    }
+    
+
+    private bool _disposed = false;
+    public void Dispose()
+    {
+        if (_disposed) return;
+        aes.Dispose();
+    }
+
+    ~AudioAesDecryptor() => Dispose();
+}

VkAudioDownloader/VkM3U8/AudioDecryptor.cs

@@ -1,35 +0,0 @@
-using System;
-using System.IO;
-using System.Security.Cryptography;
-
-namespace VkAudioDownloader.VkM3U8;
-
-public class AudioDecryptor : IDisposable
-{
-    private Aes Aes;
-    private ICryptoTransform Decryptor;
-
-    AudioDecryptor()
-    {
-        Aes=Aes.Create();
-        Aes.KeySize = 128;
-        Aes.Mode = CipherMode.CBC;
-        Aes.IV = new byte[4] { 0, 0, 0, 0 };
-        Aes.Padding = PaddingMode.Zeros;
-        Decryptor = Aes.CreateDecryptor();
-    }
-
-    public Stream Decrypt(Stream fragment) 
-        => new CryptoStream(fragment, Decryptor, CryptoStreamMode.Read);
-    
-    private bool _disposed;
-    public void Dispose()
-    {
-        if(_disposed) return;
-        Aes.Dispose();
-        Decryptor.Dispose();
-        _disposed = true;
-    }
-
-    ~AudioDecryptor() => Dispose();
-}

VkAudioDownloader/VkM3U8/HLSFragment.cs

@@ -1,9 +1,10 @@
 namespace VkAudioDownloader.VkM3U8;
 
-public record struct HLSFragment
+public readonly record struct HLSFragment
-{
+(
-    public string Name;
+    string Name,
-    // public int Duration;
+    string Url,
-    public bool Encrypted;
+    float Duration,
-    public string? EncryptionKeyUrl;
+    bool Encrypted,
-}
+    string? EncryptionKeyUrl
+);

VkAudioDownloader/VkM3U8/HLSPlaylist.cs

@@ -1,3 +1,6 @@
+using System.IO;
+using System.Text;
+
 namespace VkAudioDownloader.VkM3U8;
 
 public class HLSPlaylist
@@ -6,12 +9,12 @@ public class HLSPlaylist
     public HLSFragment[] Fragments { get; internal set; } 
 
     /// content duration in seconds
-    public int Duration { get; internal set; }
+    public float Duration { get; internal set; }
     
     /// url before index.m3u8
     public string BaseUrl { get; internal set; }
     
-    internal HLSPlaylist(HLSFragment[] fragments, int duration, string baseUrl)
+    internal HLSPlaylist(HLSFragment[] fragments, float duration, string baseUrl)
     {
         Fragments = fragments;
         Duration = duration;
@@ -22,4 +25,16 @@ public class HLSPlaylist
         $"BaseUrl: {BaseUrl}\n" +
         $"Duration: {Duration}\n" +
         $"Fragments: HLSFragment[{Fragments.Length}]";
+
+    public void CreateFragmentListFile(string path)
+    {
+        using var playlistFile = File.Open(path, FileMode.Create);
+        foreach (var fragment in Fragments)
+        {
+            playlistFile.Write(Encoding.ASCII.GetBytes("file '"));
+            playlistFile.Write(Encoding.ASCII.GetBytes(fragment.Name));
+            playlistFile.WriteByte((byte)'\'');
+            playlistFile.WriteByte((byte)'\n');
+        }
+    }
 }

VkAudioDownloader/VkM3U8/HttpHelper.cs

@@ -0,0 +1,44 @@
+global using System.Threading.Tasks;
+using System.IO;
+using System.Net.Http;
+using System.Text;
+
+
+namespace VkAudioDownloader.VkM3U8;
+
+public class HttpHelper : HttpClient
+{
+    private AudioAesDecryptor Decryptor = new();
+
+    public static async Task WriteStreamAsync(Stream stream, string localFilePath, bool disposeStream=true)
+    {
+        await using var file = File.OpenWrite(localFilePath);
+        await stream.CopyToAsync(file);
+        if(disposeStream)
+            await stream.DisposeAsync();
+    }
+    public async Task DownloadAsync(string url, string localFilePath) => 
+        await WriteStreamAsync(await GetStreamAsync(url), localFilePath);
+
+    public async Task<Stream> GetStreamAsync(HLSFragment fragment)
+    {
+        var fragmentStream = await GetStreamAsync(fragment.Url);
+        if (!fragment.Encrypted)
+            return fragmentStream;
+        string key = await GetStringAsync(fragment.EncryptionKeyUrl);
+        return Decryptor.DecryptStream(fragmentStream, key);
+    }
+
+    public async Task DownloadAsync(HLSFragment fragment, string localDir) 
+        => await WriteStreamAsync(await GetStreamAsync(fragment), Path.Combine(localDir, fragment.Name));
+
+    public async Task DownloadAsync(HLSPlaylist playlist, string localDir)
+    {
+        foreach (var fragment in playlist.Fragments)
+        {
+            //TODO log file download progress
+            await DownloadAsync(fragment, localDir);
+            playlist.CreateFragmentListFile(Path.Join(localDir, "playlist.txt"));
+        }
+    }
+}

VkAudioDownloader/VkM3U8/M3U8Parser.cs

@@ -1,20 +1,29 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 
 namespace VkAudioDownloader.VkM3U8;
 
 public class M3U8Parser
 {
-    private string _m3u8="";
+    #nullable disable
+    private string _m3u8;
     private int _pos;
     private List<HLSFragment> _fragments = new();
-    private int _playlistDuration = 0;
+    private string _baseUrl;
-    private HLSFragment _currentFragment = default;
+    private float _playlistDuration;
+    private string _fragmentName;
+    private float _fragmentDuration;
+    private string _fragmentEncryptionKeyUrl;
+    private bool _fragmentEncrypted;
     
     // parses m3u8 playlist and resets state
     public HLSPlaylist Parse(Uri m3u8Url, string m3u8Content)
     {
         _m3u8 = m3u8Content;
+        var urlStr = m3u8Url.ToString();
+        _baseUrl = urlStr.Remove(urlStr.LastIndexOf('/') + 1);
+        
         var line = NextLine();
         while (!line.IsEmpty)
         {
@@ -22,19 +31,28 @@ public class M3U8Parser
                 ParseHashTag(line);
             else
             {
-                _currentFragment.Name = line.ToString();
+                _fragmentName = line.ToString();
-                _fragments.Add(_currentFragment);
+                _fragments.Add(new HLSFragment(
-                _currentFragment = default;
+                    _fragmentName, 
+                    _baseUrl+_fragmentName,
+                    _fragmentDuration,
+                    _fragmentEncrypted, 
+                    _fragmentEncryptionKeyUrl));
+                _playlistDuration += _fragmentDuration;
+                // m3u8 format uses hashtags to replace some properties, so there is no need to reset them after every fragment name
+                // _fragmentName = null;
+                // _fragmentDuration = 0;
+                // _fragmentEncrypted = false;
+                // _fragmentEncryptionKeyUrl = null;
             }
                 
             line = NextLine();
         }
 
-        string urlStr = m3u8Url.ToString();
         var rezult = new HLSPlaylist(
             _fragments.ToArray(),
             _playlistDuration,
-            urlStr.Remove(urlStr.LastIndexOf('/')+1));
+            _baseUrl);
         Clear();
         return rezult;
     }
@@ -56,15 +74,18 @@ public class M3U8Parser
 
     private void ParseHashTag(ReadOnlySpan<char> line)
     {
-        if(line.StartsWith("EXT-X-TARGETDURATION:"))
+        if(line.StartsWith("#EXTINF:"))
-            _playlistDuration=Int32.Parse(line.After(':'));
+        {
+            var duration = line.After(':').Before(',');
+            _fragmentDuration = float.Parse(duration, NumberStyles.Any, CultureInfo.InvariantCulture.NumberFormat);
+        }
         else if (line.StartsWith("#EXT-X-KEY:METHOD="))
         {
             var method = line.After("#EXT-X-KEY:METHOD=");
             
             if (method.ToString() == "NONE")
             {
-                _currentFragment.Encrypted = false;
+                _fragmentEncrypted = false;
                 return;
             }
 
@@ -76,18 +97,21 @@ public class M3U8Parser
             if (!keyUrl.StartsWith("http"))
                 throw new Exception($"key uri is not url: {keyUrl}");
             
-            // AES-128 which AudioDecryptor can decrypt
+            // AES-128 which AudioAesDecryptor can decrypt
-            _currentFragment.Encrypted = true;
+            _fragmentEncrypted = true;
-            _currentFragment.EncryptionKeyUrl = keyUrl.ToString();
+            _fragmentEncryptionKeyUrl = keyUrl.ToString();
         }
     }
 
     private void Clear()
     {
-        _m3u8 = "";
+        _baseUrl = null;
+        _m3u8 = null;
         _pos=0;
-        _fragments.Clear();
-        _currentFragment = default;
         _playlistDuration = 0;
+        _fragments.Clear();
+        _fragmentName = null;
+        _fragmentEncrypted = false;
+        _fragmentEncryptionKeyUrl = null;
     }
 }