From 1299e83d4eb9728fa43951893b09010adc14ad23 Mon Sep 17 00:00:00 2001 From: timerix Date: Sat, 18 Mar 2023 05:08:12 +0600 Subject: [PATCH] DiffConflicts --- diff-text/Program.cs | 110 ++++---- diff-text/diff-text.csproj | 2 +- paradox-mod-merger/Diff.cs | 273 +++++++++++++++---- paradox-mod-merger/Merge.cs | 41 ++- paradox-mod-merger/Program.cs | 39 ++- paradox-mod-merger/Workshop.cs | 2 +- paradox-mod-merger/paradox-mod-merger.csproj | 6 +- publish_debug.sh | 4 + 8 files changed, 354 insertions(+), 123 deletions(-) create mode 100644 publish_debug.sh diff --git a/diff-text/Program.cs b/diff-text/Program.cs index 4584952..c1b1eee 100644 --- a/diff-text/Program.cs +++ b/diff-text/Program.cs @@ -4,60 +4,70 @@ using System.Text; using DiffMatchPatch; using DTLib.Filesystem; -Console.InputEncoding=Encoding.UTF8; -Console.OutputEncoding=Encoding.UTF8; +namespace diff_text; -if (args.Length != 2) +public static class DiffText { - Console.WriteLine("usage: [file0] [file1]"); - return; -} - -var _diff=FileDiff(args[0], args[1]); -PrintDiff(_diff); - - -List FileDiff(string file0, string file1) -{ - string fileText0 = File.ReadAllText(file0); - string fileText1 = File.ReadAllText(file1); - return TextDiff(fileText0, fileText1); -} - -List TextDiff(string text0, string text1) -{ - var diff = Diff.Compute(text0, text1, checklines:true); - diff.CleanupSemantic(); - return diff; -} - -void PrintDiff(List diff, bool ignoreWhitespaces=false) -{ - foreach (var d in diff) + internal static void Main(string[] args) { - bool whitespaceOnly = d.WhitespaceOnlyDiff; - if(ignoreWhitespaces && whitespaceOnly) - continue; - - switch(d.Operation) + Console.InputEncoding = Encoding.UTF8; + Console.OutputEncoding = Encoding.UTF8; + + if (args.Length != 2) { - case Operation.Delete: - Console.BackgroundColor = ConsoleColor.DarkRed; - Console.ForegroundColor = ConsoleColor.Black; - Console.Write(whitespaceOnly ? d.FormattedText : d.Text); - Console.ResetColor(); - break; - case Operation.Insert: - Console.BackgroundColor = ConsoleColor.DarkGreen; - Console.ForegroundColor = ConsoleColor.Black; - Console.Write(whitespaceOnly ? d.FormattedText : d.Text); - Console.ResetColor(); - break; - case Operation.Equal: - Console.Write(d.Text); - break; - default: - throw new ArgumentOutOfRangeException(d.Operation.ToString()); + Console.WriteLine("usage: [file0] [file1]"); + return; + } + + var _diff = FileDiff(args[0], args[1]); + PrintDiff(_diff); + } + + public static List FileDiff(string file0, string file1) + { + string fileText0 = File.ReadAllText(file0); + string fileText1 = File.ReadAllText(file1); + return TextDiff(fileText0, fileText1); + } + + public static List TextDiff(string text0, string text1) + { + var diff = Diff.Compute(text0, text1, checklines: true); + diff.CleanupSemantic(); + return diff; + } + + public static void PrintDiff(List diff, bool ignoreWhitespaces = false) + { + Console.ResetColor(); + foreach (var d in diff) + { + bool whitespaceOnly = d.WhitespaceOnlyDiff; + if (ignoreWhitespaces && whitespaceOnly) + continue; + + string text; + switch (d.Operation) + { + case Operation.Delete: + Console.BackgroundColor = ConsoleColor.DarkRed; + Console.ForegroundColor = ConsoleColor.Black; + text = whitespaceOnly ? d.FormattedText : d.Text; + break; + case Operation.Insert: + Console.BackgroundColor = ConsoleColor.DarkGreen; + Console.ForegroundColor = ConsoleColor.Black; + text = whitespaceOnly ? d.FormattedText : d.Text; + break; + case Operation.Equal: + text = d.Text; + break; + default: + throw new ArgumentOutOfRangeException(d.Operation.ToString()); + } + + Console.Write(text); + Console.ResetColor(); } } } \ No newline at end of file diff --git a/diff-text/diff-text.csproj b/diff-text/diff-text.csproj index ab07cfc..0d61b7e 100644 --- a/diff-text/diff-text.csproj +++ b/diff-text/diff-text.csproj @@ -9,7 +9,7 @@ - + diff --git a/paradox-mod-merger/Diff.cs b/paradox-mod-merger/Diff.cs index 54df3ac..14305a1 100644 --- a/paradox-mod-merger/Diff.cs +++ b/paradox-mod-merger/Diff.cs @@ -1,76 +1,247 @@ +using System.Linq; +using DTLib.Console; +using DTLib.Dtsod; +using diff_text; +using DiffMatchPatch; +using DTLib.Ben.Demystifier; + namespace ParadoxModMerger; + +public record struct ConflictingModFile(string FilePath, string[] Mods); + +public enum DiffState +{ + Added, Equal, Removed, Changed +} + +public record struct DiffPart(T Value, DiffState State); + static class Diff { static ConsoleLogger logger = new($"logs", "diff"); static void Log(params string[] msg) => logger.Log(msg); - public static void DiffMods(string connectedPathes) + public static void DiffCommandHandler(string connected_pathes) { - string[] split = connectedPathes.Split(';'); - DiffMods(split[0], split[1]); + IOPath[] moddirs = Program.SplitStringToPaths(connected_pathes); + var conflicts = FindModConflicts(moddirs); + LogModConflicts(conflicts); + } + + public static void DiffConflictsCommandHandler(IOPath conflicts_dtsod_path) + { + var dtsod = new DtsodV23(File.ReadAllText(conflicts_dtsod_path)); + var conflicts = new ConflictingModFile[dtsod.Count]; + int i = 0; + foreach (var p in dtsod) + { + conflicts[i]=new ConflictingModFile(p.Key, ((List)p.Value).Select(m=>(string)m).ToArray()); + i++; + } + ShowConflictsTextDiff(conflicts); } - public static void DiffMods(IOPath moddir0, IOPath moddir1) + public static void ShowConflictsTextDiff(ConflictingModFile[] conflicts) { - var hasher = new Hasher(); - var diff = new Dictionary(); - // добавление файлов из первой папки - List files = Directory.GetAllFiles(moddir0); - var mods = new List(); - for (short i = 0; i < files.Count; i++) - { - byte[] hash = hasher.HashFile(files[i]); - files[i] = files[i].ReplaceBase(moddir0, ""); - diff.Add(files[i], hash); - AddMod(files[i]); - } + int selected_confl_i = 0; + int selected_mod0_i = 0; + int selected_mod1_i = 1; + int line_buffer_offset = 0; + + int line_i=0; + for (int i = 0; i < conflicts.Length; i++) + line_i += 1 + conflicts[i].Mods.Length; - // убирание совпадающих файлов - files = Directory.GetAllFiles(moddir1); - for (short i = 0; i < files.Count; i++) + var lines = new (ConsoleColor color, string text)[line_i]; + // set lines text + line_i = 0; + for (int confl_i = 0; confl_i < conflicts.Length; confl_i++) { - byte[] hash = hasher.HashFile(files[i]); - files[i] = files[i].RemoveBase(moddir1); - if (diff.ContainsKey(files[i]) && diff[files[i]].HashToString() == hash.HashToString()) - diff.Remove(files[i]); - else + lines[line_i].color = ConsoleColor.White; + lines[line_i++].text = $"[{confl_i}]{conflicts[confl_i].FilePath}"; + for (int mod_i = 0; mod_i < conflicts[confl_i].Mods.Length; mod_i++) { - diff.Add(Path.Concat(moddir1,files[i]), hash); - AddMod(files[i]); + lines[line_i].color = ConsoleColor.Gray; + lines[line_i++].text = $" [{mod_i}] {conflicts[confl_i].Mods[mod_i]}"; } } - - void AddMod(IOPath mod) + + while (true) { - mod = mod.Remove(0, 1); - mod = mod.Remove(mod.IndexOf(Path.Sep)); - if (!mods.Contains(mod)) mods.Add(mod); - } - - // вывод результата - StringBuilder output = new StringBuilder(); - output.Append($"[{DateTime.Now}]\n\n"); - foreach (var mod in mods) - { - output.Append('\n').Append(mod).Append("\n{\n"); - foreach (var file in diff.Keys) + try { - if (file.Contains(mod)) + // set line colors + line_i = 0; + for (int confl_i = 0; confl_i < conflicts.Length; confl_i++) { - output.Append('\t'); - if (!file.Contains(moddir1)) output.Append(moddir0).Append(file).Append('\n'); - output.Append(file).Append('\n'); + if (confl_i == selected_confl_i) + lines[line_i].color = ConsoleColor.Blue; + else lines[line_i].color = ConsoleColor.White; + line_i++; + + for (int mod_i = 0; mod_i < conflicts[confl_i].Mods.Length; mod_i++) + { + if (confl_i == selected_confl_i) + { + if (mod_i == selected_mod0_i) + lines[line_i].color = ConsoleColor.Yellow; + else if (mod_i == selected_mod1_i) + lines[line_i].color = ConsoleColor.Green; + else lines[line_i].color = ConsoleColor.Gray; + } + else lines[line_i].color = ConsoleColor.Gray; + line_i++; + } + } + + // print lines + Console.Clear(); + ColoredConsole.WriteLine("c", + "[Q]exit [Up/Down]select file [Left/Right][number]select mod [Enter]show diff"); + for (int i = line_buffer_offset; + i < lines.Length && i < Console.WindowHeight + line_buffer_offset - 2; + i++) + ColoredConsole.WriteLine(lines[i].color, lines[i].text); + + // read input + ConsoleKeyInfo key = Console.ReadKey(); + switch (key.Key) + { + case ConsoleKey.Q: + return; + case ConsoleKey.UpArrow: + if (selected_confl_i > 0) + { + line_buffer_offset -= conflicts[selected_confl_i].Mods.Length; + line_buffer_offset--; + selected_confl_i--; + } + + break; + case ConsoleKey.DownArrow: + if (selected_confl_i < conflicts.Length - 1) + { + selected_confl_i++; + line_buffer_offset++; + line_buffer_offset += conflicts[selected_confl_i].Mods.Length; + } + + break; + case ConsoleKey.Enter: + { + var conflict = conflicts[selected_confl_i]; + var path0 = Path.Concat(conflict.Mods[selected_mod0_i], conflict.FilePath); + var path1 = Path.Concat(conflict.Mods[selected_mod1_i], conflict.FilePath); + if (path0.Extension() == "dds") + { + ColoredConsole.Write("r", $"file {path0} is not text file\n", + "c", "press enter to continue: "); + Console.ReadLine(); + } + var textDiff = DiffText.FileDiff(path0.Str, path1.Str); + Console.Clear(); + DiffText.PrintDiff(textDiff, true); + ColoredConsole.Write("c", "\npress enter to continue: "); + Console.ReadLine(); + break; + } + case ConsoleKey.LeftArrow: + { + ColoredConsole.Write("w", "enter left mod number: "); + string answ = Console.ReadLine(); + selected_mod0_i = answ.ToInt(); + break; + } + case ConsoleKey.RightArrow: + { + ColoredConsole.Write("w", "enter right mod number: "); + string answ = Console.ReadLine(); + selected_mod1_i = answ.ToInt(); + break; + } } } - - output.Append("}\n"); - // не убирать, это полезное - if (output[output.Length - 4] == '{') - output.Remove(output.Length - mod.Length - 5, mod.Length + 5); + catch (Exception ex) + { + Log("r",ex.ToStringDemystified()); + ColoredConsole.Write("c", "\npress enter to continue: "); + Console.ReadLine(); + } + } + } + + + public static ICollection FindModConflicts(IOPath[] modpaths) + { + var all_files = new Dictionary>(); + foreach (var modp in modpaths) + { + foreach (var _file in Directory.GetAllFiles(modp)) + { + var file = _file.RemoveBase(modp); + if (all_files.TryGetValue(file.Str, out var associated_mods)) + associated_mods.Add(modp.Str); + else all_files.Add(file.Str, new List(1) { modp.Str }); + } } - var _outStr = output.ToString(); - Log("w", _outStr); + var output = new List(); + foreach (var p in all_files) + if (p.Value.Count > 1) + output.Add(new ConflictingModFile(p.Key, p.Value.ToArray())); + return output; + } + + public static IEnumerable> DiffCollections(ICollection col0, ICollection col1) + { + foreach (T el in col0) + yield return new DiffPart(el, col1.Contains(el) ? DiffState.Equal : DiffState.Removed); + foreach (var el in col1) + if (!col0.Contains(el)) + yield return new DiffPart(el, DiffState.Added); + } + + public static IEnumerable> DiffDirs(IOPath dir0, IOPath dir1) + { + var files0 = Directory.GetAllFiles(dir0).Select(p=>p.RemoveBase(dir0)).ToList(); + var files1 = Directory.GetAllFiles(dir1).Select(p=>p.RemoveBase(dir1)).ToList(); + var filesMerged = DiffCollections(files0, files1); + + foreach (var filePathDiff in filesMerged) + { + if (filePathDiff.State == DiffState.Equal) + { + Hasher hasher = new Hasher(); + string hash0=hasher.HashFile(Path.Concat(dir0, filePathDiff.Value)).HashToString(); + string hash1=hasher.HashFile(Path.Concat(dir1, filePathDiff.Value)).HashToString(); + if (hash0 != hash1) + yield return filePathDiff with { State = DiffState.Changed }; + else yield return filePathDiff; + } + else yield return filePathDiff; + } + } + + + // вывод конфликтующих файлов при -merge и -clear если такие есть + public static void LogModConflicts(ICollection conflicts) + { + if(conflicts.Count==0) return; + + Log("m",$"found {conflicts.Count} conflicting files:"); + var dtsod = new DtsodV23(); + foreach (var cfl in conflicts) + { + Log("m", "file ","c", cfl.FilePath, "m", "in mods", "b", cfl.Mods.MergeToString(", ")); + dtsod.Add(cfl.FilePath, cfl.Mods); + } + + string timeStr = DateTime.Now.ToString(MyTimeFormat.ForFileNames); + IOPath conflicts_dtsod_path = Path.Concat("conflicts", $"conflicts_{timeStr}.dtsod"); + File.WriteAllText(conflicts_dtsod_path, dtsod.ToString()); + conflicts_dtsod_path = Path.Concat("conflicts", "conflicts_latest.dtsod"); + File.WriteAllText(conflicts_dtsod_path, dtsod.ToString()); + Log("m",$"conflicts have written to {conflicts_dtsod_path}"); } } \ No newline at end of file diff --git a/paradox-mod-merger/Merge.cs b/paradox-mod-merger/Merge.cs index e5ee8e1..c816b0b 100644 --- a/paradox-mod-merger/Merge.cs +++ b/paradox-mod-merger/Merge.cs @@ -1,4 +1,7 @@ -namespace ParadoxModMerger; +using System.Linq; +using DTLib.Console; + +namespace ParadoxModMerger; static class Merge { @@ -8,17 +11,43 @@ static class Merge public static void MergeAll(IOPath[] moddirs, IOPath outDir) { Log("b", $"found {moddirs.Length} mod dirs"); + HandleConflicts(moddirs); + for (short i = 0; i < moddirs.Length; i++) { Log("b", $"[{i + 1}/{moddirs.Length}] merging mod ", "c", $"{moddirs[i]}"); - Directory.Copy(moddirs[i], outDir, true, out var _conflicts); - Program.LogConflicts(_conflicts); + Directory.Copy(moddirs[i], outDir, true); } } - public static void MergeSingle(IOPath moddir, IOPath outDir) + public static void MergeInto(IOPath moddir, IOPath outDir) { - Directory.Copy(moddir, outDir, true, out var _conflicts); - Program.LogConflicts(_conflicts); + HandleConflicts(new[] { moddir, outDir }); + Directory.Copy(moddir, outDir, true); + } + + public static void ConsoleAskYN(string question, Action yes, Action no) + { + Log("y", question + " [y/n]"); + string answ = ColoredConsole.Read("w"); + if (answ == "y") yes(); + else no(); + } + + static void HandleConflicts(IOPath[] moddirs) + { + var conflicts = Diff.FindModConflicts(moddirs); + if (conflicts.Count <= 0) return; + + Diff.LogModConflicts(conflicts); + ConsoleAskYN("continue merge?", + () =>Log("y", "merge continued"), + () => + { + Log("y", "merge interrupted"); + ConsoleAskYN("show text diff?", + () => Diff.ShowConflictsTextDiff(conflicts.ToArray()), + () => {}); + }); } } \ No newline at end of file diff --git a/paradox-mod-merger/Program.cs b/paradox-mod-merger/Program.cs index 28dbc1c..fb5685c 100644 --- a/paradox-mod-merger/Program.cs +++ b/paradox-mod-merger/Program.cs @@ -36,26 +36,36 @@ public static class Program "workshop_dir", 1), new LaunchArgument(new []{"diff"}, - "Compare mod files by hash", - p=>Diff.DiffMods(p), - "first_mod_directory;second_mod_directory", 1), + "Compares mod files by hash", + p=>Diff.DiffCommandHandler(p), + "first_mod_directory:second_mod_directory:...", + 1), + new LaunchArgument(new []{"diff-conflicts"}, + "reads conflicts_XXX.dtsod file and shows text diff for each file", + p=>Diff.DiffConflictsCommandHandler(p), + "conflicts_dtsod_path", + 1 + ), new LaunchArgument(new []{"merge-subdirs"}, - "Merge mods and show conflicts. Requires -o", + "Merges mods and shows conflicts. Requires -o", d => Merge.MergeAll(Directory.GetDirectories(d), outPath), "dir_with_mods", 1), - new LaunchArgument(new []{"merge-single"}, + new LaunchArgument(new []{"merge-into", "merge-single"}, "Merges one mod into output dir and shows conflicts. Requires -o", - mod=>Merge.MergeSingle(mod, outPath), + mod=>Merge.MergeInto(mod, outPath), "mod_dir", 1), new LaunchArgument(new []{"gen-rus-locale"}, "Creates l_russian copy of english locale in output directory. Requires -o", eng=>Localisation.GenerateRussian(eng, outPath), - "english_locale_path", 1), + "english_locale_path", + 1), new LaunchArgument(new []{"desc"}, "Downloads mod description from steam to new file in outDir. Requires -o", - id=>Workshop.CreateDescFile(id, outPath), "mod_id") + id=>Workshop.CreateDescFile(id, outPath), + "mod_id", + 1) ).ParseAndHandle(args); } catch (LaunchArgumentParser.ExitAfterHelpException) @@ -67,11 +77,14 @@ public static class Program Console.ResetColor(); } - - // вывод конфликтующих файлов при -merge и -clear если такие есть - public static void LogConflicts(List conflicts) + public static IOPath[] SplitStringToPaths(string connected_paths) { - if(conflicts.Count>0) - Log("y", $"conflicts found: {conflicts.Count}\n{conflicts.MergeToString("\n")}"); + if (!connected_paths.Contains(':')) + throw new Exception($"<{connected_paths}> doesn't contain any separators (:)"); + string[] split = connected_paths.Split(':'); + IOPath[] split_iop = new IOPath[split.Length]; + for (int i = 0; i < split.Length; i++) + split_iop[i] = new IOPath(split[i]); + return split_iop; } } \ No newline at end of file diff --git a/paradox-mod-merger/Workshop.cs b/paradox-mod-merger/Workshop.cs index a47b67b..b3c89c7 100644 --- a/paradox-mod-merger/Workshop.cs +++ b/paradox-mod-merger/Workshop.cs @@ -78,7 +78,7 @@ static class Workshop Directory.Copy(Path.Concat(moddirs[i], subdirs[n]), Path.Concat(outModDir, subdirs[n]), true, out var _conflicts); - Program.LogConflicts(_conflicts); + Log("y", $"found {_conflicts.Count} conflicts:\n{_conflicts.MergeToString('\n')}"); break; } } diff --git a/paradox-mod-merger/paradox-mod-merger.csproj b/paradox-mod-merger/paradox-mod-merger.csproj index 5b27a72..6af4f1e 100644 --- a/paradox-mod-merger/paradox-mod-merger.csproj +++ b/paradox-mod-merger/paradox-mod-merger.csproj @@ -11,7 +11,8 @@ - + + @@ -20,4 +21,7 @@ + + + \ No newline at end of file diff --git a/publish_debug.sh b/publish_debug.sh new file mode 100644 index 0000000..8ddd9c7 --- /dev/null +++ b/publish_debug.sh @@ -0,0 +1,4 @@ +#!/bin/sh +rm -rf publish +mkdir publish +dotnet publish -c debug -o publish -f net7.0