diff --git a/src/jit-analyze/DisassemblyReader.cs b/src/jit-analyze/DisassemblyReader.cs new file mode 100644 index 00000000..953c9ad1 --- /dev/null +++ b/src/jit-analyze/DisassemblyReader.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace ManagedCodeGen +{ + // Return borrowed line spans so the overwhelmingly common instruction lines need no strings. + internal sealed class DisassemblyReader : IDisposable + { + private readonly StreamReader _reader; + private char[] _buffer = new char[32 * 1024]; + private int _start; + private int _end; + private bool _skipLF; + private bool _eof; + + public DisassemblyReader(string path) + { + _reader = new StreamReader(path); + } + + public bool ReadLine(out ReadOnlySpan line) + { + int scanned = 0; + while (true) + { + ReadOnlySpan remaining = _buffer.AsSpan(_start, _end - _start); + if (_skipLF && !remaining.IsEmpty) + { + _skipLF = false; + if (remaining[0] == '\n') + { + _start++; + remaining = remaining.Slice(1); + } + } + + int newline = remaining.Slice(scanned).IndexOfAny('\r', '\n'); + if (newline >= 0) + { + newline += scanned; + line = remaining.Slice(0, newline); + _skipLF = remaining[newline] == '\r'; + _start += newline + 1; + return true; + } + + if (_eof) + { + line = remaining; + _start = _end; + return !line.IsEmpty; + } + + scanned = remaining.Length; + if (remaining.Length == _buffer.Length) + { + Array.Resize(ref _buffer, checked(_buffer.Length * 2)); + } + else + { + remaining.CopyTo(_buffer); + } + + _start = 0; + _end = scanned; + int read = _reader.Read(_buffer.AsSpan(_end)); + _end += read; + _eof = read == 0; + } + } + + public void Dispose() => _reader.Dispose(); + } +} diff --git a/src/jit-analyze/MetricCollection.cs b/src/jit-analyze/MetricCollection.cs index 35307b2c..b0c26358 100644 --- a/src/jit-analyze/MetricCollection.cs +++ b/src/jit-analyze/MetricCollection.cs @@ -32,16 +32,14 @@ static MetricCollection() } } + private readonly double[] _values; + [JsonInclude] - private Metric[] metrics; + private Metric[] metrics => s_metrics.Select(m => GetMetric(m.Name)).ToArray(); public MetricCollection() { - metrics = new Metric[s_metrics.Length]; - for (int i = 0; i < s_metrics.Length; i++) - { - metrics[i] = s_metrics[i].Clone(); - } + _values = new double[s_metrics.Length]; } public MetricCollection(MetricCollection other) : this() @@ -51,16 +49,27 @@ public MetricCollection(MetricCollection other) : this() public static IEnumerable AllMetrics => s_metrics; + // Materialize display metadata only for reports; analysis uses the compact values directly. public Metric GetMetric(string metricName) { int index; if (s_metricNameToIndex.TryGetValue(metricName, out index)) { - return metrics[index]; + Metric metric = s_metrics[index].Clone(); + metric.Value = _values[index]; + return metric; } return null; } + public double GetValue(string metricName) => _values[s_metricNameToIndex[metricName]]; + + public void AddInt(string metricName, int value) + { + int index = s_metricNameToIndex[metricName]; + _values[index] = checked((int)_values[index] + value); + } + public static bool ValidateMetric(string name) { return s_metricNameToIndex.TryGetValue(name, out _); @@ -104,47 +113,51 @@ public override string ToString() public void Add(MetricCollection other) { - for (int i = 0; i < metrics.Length; i++) + for (int i = 0; i < _values.Length; i++) { - metrics[i].Add(other.metrics[i]); + _values[i] += other._values[i]; } } public void Add(string metricName, double value) { - Metric m = GetMetric(metricName); - m.Value += value; + _values[s_metricNameToIndex[metricName]] += value; } public void Sub(MetricCollection other) { - for (int i = 0; i < metrics.Length; i++) + for (int i = 0; i < _values.Length; i++) { - metrics[i].Sub(other.metrics[i]); + _values[i] -= other._values[i]; } } public void Rel(MetricCollection other) { - for (int i = 0; i < metrics.Length; i++) + for (int i = 0; i < _values.Length; i++) { - metrics[i].Rel(other.metrics[i]); + _values[i] = (_values[i] - other._values[i]) / other._values[i]; } } - public void SetValueFrom(MetricCollection other) + public void AddRelativeDifference(MetricCollection diff, MetricCollection baseline) { - for (int i = 0; i < metrics.Length; i++) + for (int i = 0; i < _values.Length; i++) { - metrics[i].SetValueFrom(other.metrics[i]); + _values[i] += (diff._values[i] - baseline._values[i]) / baseline._values[i]; } } + public void SetValueFrom(MetricCollection other) + { + other._values.CopyTo(_values, 0); + } + public bool IsZero() { - for (int i = 0; i < metrics.Length; i++) + for (int i = 0; i < _values.Length; i++) { - if (metrics[i].Value != 0) return false; + if (_values[i] != 0) return false; } return true; } diff --git a/src/jit-analyze/Program.cs b/src/jit-analyze/Program.cs index 7d030175..dc667319 100644 --- a/src/jit-analyze/Program.cs +++ b/src/jit-analyze/Program.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Buffers; using System.Collections.Generic; using System.CommandLine; using System.CommandLine.Parsing; @@ -14,10 +15,11 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; +using IOFileInfo = System.IO.FileInfo; namespace ManagedCodeGen { - internal sealed class Program + internal sealed partial class Program { private readonly JitAnalyzeRootCommand _command; private readonly bool _reconcile; @@ -27,6 +29,9 @@ internal sealed class Program private readonly int _count; private readonly string _basePath; private readonly string _diffPath; + private readonly string _baseDirectory; + private Dictionary _textDiffCounts; + private HashSet _filesNeedingTextDiffCounts; private static string METRIC_SEP = new string('-', 80); @@ -51,12 +56,13 @@ public Program(JitAnalyzeRootCommand command) _count = Get(command.Count); _basePath = Get(command.BasePath); _diffPath = Get(command.DiffPath); + _baseDirectory = Directory.Exists(_basePath) ? Path.GetFullPath(_basePath) : Path.GetDirectoryName(Path.GetFullPath(_basePath)); } public class FileInfo { public string name; - public IEnumerable methodList; + public string[] paths; public bool isExplicitOnlyFile; public override string ToString() @@ -92,7 +98,7 @@ public class MethodInfo public MetricCollection Metrics => metrics; public string name; public int functionCount; - public IEnumerable functionOffsets; + public List functionOffsets; public MethodInfo() { @@ -256,17 +262,17 @@ public List ExtractFileInfo(string path, string filter, string fileExt new FileInfo { name = Path.GetFileName(path), - methodList = ExtractMethodInfo(Directory.EnumerateFiles(fullRootPath, searchPattern, searchOption).ToArray()), + paths = Directory.EnumerateFiles(fullRootPath, searchPattern, searchOption).ToArray(), isExplicitOnlyFile = true, }, }; } return Directory.EnumerateFiles(fullRootPath, searchPattern, searchOption) - .AsParallel().Select(p => new FileInfo + .Select(p => new FileInfo { name = p.Substring(fullRootPath.Length).TrimStart(Path.DirectorySeparatorChar), - methodList = ExtractMethodInfo(new[] { p }) + paths = new[] { p } }).ToList(); } else @@ -277,7 +283,7 @@ public List ExtractFileInfo(string path, string filter, string fileExt { new FileInfo { name = Path.GetFileName(path), - methodList = ExtractMethodInfo(new[] {path }), + paths = new[] { path }, isExplicitOnlyFile = true, } }; @@ -289,145 +295,190 @@ public List ExtractFileInfo(string path, string filter, string fileExt // and offset in the file. // // This is the method that knows how to parse jit output and recover the metrics. + private static IEnumerable<(string line, int index)> ReadMetricLines(string[] filePaths) + { + int index = 0; + foreach (string path in filePaths) + { + using var reader = new DisassemblyReader(path); + while (reader.ReadLine(out ReadOnlySpan line)) + { + if (line.StartsWith("; Total bytes of code", StringComparison.Ordinal) || + line.StartsWith("; Assembly listing for method", StringComparison.Ordinal) || + line.StartsWith("; Variable debug info:", StringComparison.Ordinal)) + { + yield return (line.ToString(), index); + } + index = checked(index + 1); + } + } + } + public static IEnumerable ExtractMethodInfo(string[] filePaths) { - Regex namePattern = new Regex(@"for method (.*)$"); - Regex codeSizePattern = new Regex(@"^; Total bytes of code ([0-9]{1,}).* for method "); - Regex prologSizePattern = new Regex(@"prolog size ([0-9]{1,})"); - // use new regex for perf score so we can still parse older files that did not have it. - Regex perfScorePattern = new Regex(@"(PerfScore|perf score) (\d+(\.\d+)?)"); - Regex instrCountPattern = new Regex(@"instruction count ([0-9]{1,})"); - Regex allocSizePattern = new Regex(@"allocated bytes for code ([0-9]{1,})"); - Regex debugInfoPattern = new Regex(@"Variable debug info: ([0-9]{1,}) live range\(s\), ([0-9]{1,}) var\(s\)"); - Regex spillInfoPattern = new Regex(@"SpillCount (\d+) SpillCountWt (\d+\.\d+)"); - Regex resolutionInfoPattern = new Regex(@"ResolutionMovs (\d+) ResolutionMovsWt (\d+\.\d+)"); - - var result = - filePaths.SelectMany(filePath => File.ReadLines(filePath)) - .Select((x, i) => new { line = x, index = i }) - .Where(l => l.line.StartsWith(@"; Total bytes of code", StringComparison.Ordinal) - || l.line.StartsWith(@"; Assembly listing for method", StringComparison.Ordinal) - || l.line.StartsWith(@"; Variable debug info:", StringComparison.Ordinal)) - .Select((x) => - { - var nameMatch = namePattern.Match(x.line); - var codeSizeMatch = codeSizePattern.Match(x.line); - var prologSizeMatch = prologSizePattern.Match(x.line); - var perfScoreMatch = perfScorePattern.Match(x.line); - var instrCountMatch = instrCountPattern.Match(x.line); - var allocSizeMatch = allocSizePattern.Match(x.line); - var debugInfoMatch = debugInfoPattern.Match(x.line); - var spillInfoMatch = spillInfoPattern.Match(x.line); - var resolutionInfoMatch = resolutionInfoPattern.Match(x.line); - return new - { - name = nameMatch.Groups[1].Value, - // Use matched data or default to 0 - totalBytes = codeSizeMatch.Success ? - int.Parse(codeSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - prologBytes = prologSizeMatch.Success ? - int.Parse(prologSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - perfScore = perfScoreMatch.Success ? - double.Parse(perfScoreMatch.Groups[2].Value, CultureInfo.InvariantCulture) : 0, - instrCount = instrCountMatch.Success ? - int.Parse(instrCountMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - allocSize = allocSizeMatch.Success ? - int.Parse(allocSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - debugClauseCount = debugInfoMatch.Success ? - int.Parse(debugInfoMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - debugVarCount = debugInfoMatch.Success ? - int.Parse(debugInfoMatch.Groups[2].Value, CultureInfo.InvariantCulture) : 0, - spillCount = spillInfoMatch.Success ? - int.Parse(spillInfoMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - spillWeight = spillInfoMatch.Success ? - double.Parse(spillInfoMatch.Groups[2].Value, CultureInfo.InvariantCulture) : 0, - resolutionCount = resolutionInfoMatch.Success ? - int.Parse(resolutionInfoMatch.Groups[1].Value, CultureInfo.InvariantCulture) : 0, - resolutionWeight = resolutionInfoMatch.Success ? - double.Parse(resolutionInfoMatch.Groups[2].Value, CultureInfo.InvariantCulture) : 0, - // Use function index only from non-data lines (the name line) - functionOffset = codeSizeMatch.Success ? - 0 : x.index - }; - }) - .GroupBy(x => x.name) - .Select(x => - { - MethodInfo mi = new MethodInfo - { - name = x.Key, - functionCount = x.Select(z => z).Where(z => z.totalBytes == 0).Count(), - // for all non-zero function offsets create list. - functionOffsets = x.Select(z => z) - .Where(z => z.functionOffset != 0) - .Select(z => z.functionOffset).ToList() - }; - - int totalCodeSize = x.Sum(z => z.totalBytes); - int totalAllocSize = x.Sum(z => z.allocSize); - Debug.Assert((totalAllocSize == 0) || (totalCodeSize <= totalAllocSize)); - - mi.Metrics.Add("CodeSize", totalCodeSize); - mi.Metrics.Add("PrologSize", x.Sum(z => z.prologBytes)); - mi.Metrics.Add("PerfScore", x.Sum(z => z.perfScore)); - mi.Metrics.Add("InstrCount", x.Sum(z => z.instrCount)); - mi.Metrics.Add("AllocSize", totalAllocSize); - mi.Metrics.Add("ExtraAllocBytes", totalAllocSize == 0 ? 0 : totalAllocSize - totalCodeSize); - mi.Metrics.Add("DebugClauseCount", x.Sum(z => z.debugClauseCount)); - mi.Metrics.Add("DebugVarCount", x.Sum(z => z.debugVarCount)); - mi.Metrics.Add("SpillCount", x.Sum(z => z.spillCount)); - mi.Metrics.Add("SpillWeight", x.Sum(z => z.spillWeight)); - mi.Metrics.Add("ResolutionCount", x.Sum(z => z.resolutionCount)); - mi.Metrics.Add("ResolutionWeight", x.Sum(z => z.resolutionWeight)); - return mi; - }).ToList(); - - return result; + var methods = new Dictionary(StringComparer.Ordinal); + var lookup = methods.GetAlternateLookup>(); + foreach (var record in ReadMetricLines(filePaths)) + { + string line = record.line; + int nameStart = line.IndexOf("for method ", StringComparison.Ordinal); + ReadOnlySpan name = nameStart < 0 ? ReadOnlySpan.Empty : line.AsSpan(nameStart + 11); + if (!lookup.TryGetValue(name, out MethodInfo method)) + { + method = new MethodInfo { name = name.ToString(), functionOffsets = new List() }; + methods.Add(method.name, method); + } + + Match codeSize = CodeSizePattern().Match(line); + int totalBytes = ReadInt(codeSize); + if (totalBytes == 0) + { + method.functionCount = checked(method.functionCount + 1); + } + if (!codeSize.Success && record.index != 0) + { + method.functionOffsets.Add(record.index); + } + + method.Metrics.AddInt("CodeSize", totalBytes); + method.Metrics.AddInt("PrologSize", ReadInt(PrologSizePattern().Match(line))); + method.Metrics.Add("PerfScore", ReadDouble(PerfScorePattern().Match(line), 2)); + method.Metrics.AddInt("InstrCount", ReadInt(InstrCountPattern().Match(line))); + method.Metrics.AddInt("AllocSize", ReadInt(AllocSizePattern().Match(line))); + Match debugInfo = DebugInfoPattern().Match(line); + method.Metrics.AddInt("DebugClauseCount", ReadInt(debugInfo)); + method.Metrics.AddInt("DebugVarCount", ReadInt(debugInfo, 2)); + Match spillInfo = SpillInfoPattern().Match(line); + method.Metrics.AddInt("SpillCount", ReadInt(spillInfo)); + method.Metrics.Add("SpillWeight", ReadDouble(spillInfo, 2)); + Match resolutionInfo = ResolutionInfoPattern().Match(line); + method.Metrics.AddInt("ResolutionCount", ReadInt(resolutionInfo)); + method.Metrics.Add("ResolutionWeight", ReadDouble(resolutionInfo, 2)); + } + + foreach (MethodInfo method in methods.Values) + { + double totalCodeSize = method.Metrics.GetValue("CodeSize"); + double totalAllocSize = method.Metrics.GetValue("AllocSize"); + Debug.Assert(totalAllocSize == 0 || totalCodeSize <= totalAllocSize); + method.Metrics.Add("ExtraAllocBytes", totalAllocSize == 0 ? 0 : totalAllocSize - totalCodeSize); + } + return methods.Values.ToList(); + + static int ReadInt(Match match, int group = 1) => + match.Success ? int.Parse(match.Groups[group].ValueSpan, CultureInfo.InvariantCulture) : 0; + + static double ReadDouble(Match match, int group) => + match.Success ? double.Parse(match.Groups[group].ValueSpan, CultureInfo.InvariantCulture) : 0; + } + [GeneratedRegex(@"^; Total bytes of code ([0-9]{1,}).* for method ")] + private static partial Regex CodeSizePattern(); + [GeneratedRegex(@"prolog size ([0-9]{1,})")] + private static partial Regex PrologSizePattern(); + [GeneratedRegex(@"(PerfScore|perf score) (\d+(\.\d+)?)")] + private static partial Regex PerfScorePattern(); + [GeneratedRegex(@"instruction count ([0-9]{1,})")] + private static partial Regex InstrCountPattern(); + [GeneratedRegex(@"allocated bytes for code ([0-9]{1,})")] + private static partial Regex AllocSizePattern(); + [GeneratedRegex(@"Variable debug info: ([0-9]{1,}) live range\(s\), ([0-9]{1,}) var\(s\)")] + private static partial Regex DebugInfoPattern(); + [GeneratedRegex(@"SpillCount (\d+) SpillCountWt (\d+\.\d+)")] + private static partial Regex SpillInfoPattern(); + [GeneratedRegex(@"ResolutionMovs (\d+) ResolutionMovsWt (\d+\.\d+)")] + private static partial Regex ResolutionInfoPattern(); + // Compare base and diff file lists and produce a sorted list of method // deltas by file. Delta is computed diffBytes - baseBytes so positive // numbers are regressions. (lower is better) // // Todo: handle metrics where "higher is better" - public IEnumerable Comparator(IEnumerable baseInfo, - IEnumerable diffInfo, string metricName) + public FileDelta[][] Comparator(IEnumerable baseInfo, + IEnumerable diffInfo, string[] metricNames) { - MethodInfoComparer methodInfoComparer = new MethodInfoComparer(); - return baseInfo.Join(diffInfo, b => b.isExplicitOnlyFile ? "" : b.name, d => d.isExplicitOnlyFile ? "" : d.name, (b, d) => - { - var jointList = b.methodList.Join(d.methodList, - x => x.name, y => y.name, (x, y) => new MethodDelta - { - name = x.name, - baseMetrics = new MetricCollection(x.Metrics), - diffMetrics = new MetricCollection(y.Metrics), - baseOffsets = x.functionOffsets, - diffOffsets = y.functionOffsets - }) - .OrderByDescending(r => r.deltaMetrics.GetMetric(metricName).Value); - - FileDelta f = new FileDelta - { - baseName = b.name, - diffName = d.name, - baseMetrics = jointList.Sum(x => x.baseMetrics), - diffMetrics = jointList.Sum(x => x.diffMetrics), - deltaMetrics = jointList.Sum(x => x.deltaMetrics), - relDeltaMetrics = jointList.Sum(x => x.relDeltaMetrics), - methodsInBoth = jointList.Count(), - methodsOnlyInBase = b.methodList.Except(d.methodList, methodInfoComparer), - methodsOnlyInDiff = d.methodList.Except(b.methodList, methodInfoComparer), - methodDeltaList = jointList.Where(x => x.deltaMetrics.GetMetric(metricName).Value != 0) - }; + // Keep only one pair's parsed methods per worker, and reuse them for every metric. + return baseInfo.Join(diffInfo, b => b.isExplicitOnlyFile ? "" : b.name, + d => d.isExplicitOnlyFile ? "" : d.name, (b, d) => (Base: b, Diff: d)) + .AsParallel().AsOrdered() + .Select(pair => + { + var baseMethods = ExtractMethodInfo(pair.Base.paths); + bool identical = pair.Base.paths.Length == pair.Diff.paths.Length && + pair.Base.paths.Zip(pair.Diff.paths).All(paths => FilesEqual(paths.First, paths.Second)); + var diffMethods = identical ? baseMethods : ExtractMethodInfo(pair.Diff.paths); + return metricNames.Select(metricName => + CompareFile(pair.Base.name, pair.Diff.name, baseMethods, diffMethods, metricName)).ToArray(); + }).ToArray(); + } - if (_reconcile) + private FileDelta CompareFile(string baseName, string diffName, + IEnumerable baseMethods, IEnumerable diffMethods, string metricName) + { + if (ReferenceEquals(baseMethods, diffMethods)) + { + var total = new MetricCollection(); + var relative = new MetricCollection(); + int count = 0; + foreach (MethodInfo method in baseMethods) { - f.Reconcile(); + total.Add(method.Metrics); + // Preserve 0/0 (NaN) for metrics absent from an otherwise identical method. + relative.AddRelativeDifference(method.Metrics, method.Metrics); + count++; } + var unchanged = new FileDelta + { + baseName = baseName, + diffName = diffName, + baseMetrics = total, + diffMetrics = new MetricCollection(total), + deltaMetrics = new MetricCollection(), + relDeltaMetrics = relative, + methodsInBoth = count, + methodsOnlyInBase = Array.Empty(), + methodsOnlyInDiff = Array.Empty(), + methodDeltaList = Array.Empty(), + }; + if (_reconcile) + unchanged.Reconcile(); + return unchanged; + } - return f; - }).ToList(); + MethodInfoComparer methodInfoComparer = new MethodInfoComparer(); + var jointList = baseMethods.Join(diffMethods, + x => x.name, y => y.name, (x, y) => new MethodDelta + { + name = x.name, + baseMetrics = x.Metrics, + diffMetrics = y.Metrics, + baseOffsets = x.functionOffsets, + diffOffsets = y.functionOffsets + }) + .OrderByDescending(r => r.deltaMetrics.GetValue(metricName)) + .ToList(); + + FileDelta f = new FileDelta + { + baseName = baseName, + diffName = diffName, + baseMetrics = jointList.Sum(x => x.baseMetrics), + diffMetrics = jointList.Sum(x => x.diffMetrics), + deltaMetrics = jointList.Sum(x => x.deltaMetrics), + relDeltaMetrics = jointList.Sum(x => x.relDeltaMetrics), + methodsInBoth = jointList.Count(), + methodsOnlyInBase = baseMethods.Except(diffMethods, methodInfoComparer).ToList(), + methodsOnlyInDiff = diffMethods.Except(baseMethods, methodInfoComparer).ToList(), + methodDeltaList = jointList.Where(x => x.deltaMetrics.GetValue(metricName) != 0).ToList() + }; + + if (_reconcile) + { + f.Reconcile(); + } + + return f; } // Summarize differences across all the files. @@ -646,11 +697,13 @@ void DisplayMethodMetric(string headerText, string subtext, int methodCount, dyn { // Show files with text diffs but no metric diffs. - Dictionary diffCounts = DiffInText(_diffPath, _basePath); + Dictionary diffCounts = _textDiffCounts ??= DiffInText(_diffPath, _basePath, _filesNeedingTextDiffCounts); // TODO: resolve diffs to particular methods in the files. - var zeroDiffFilesWithDiffs = fileDeltaList.Where(x => diffCounts.ContainsKey(x.diffName) && (x.deltaMetrics.IsZero())) - .OrderByDescending(x => diffCounts[x.baseName]); + var zeroDiffFilesWithDiffs = fileDeltaList + .Select(file => (File: file, Path: Path.GetFullPath(Path.Combine(_baseDirectory, file.baseName)))) + .Where(x => !Get(_command.ConcatFiles) && diffCounts.ContainsKey(x.Path) && x.File.deltaMetrics.IsZero()) + .OrderByDescending(x => diffCounts[x.Path]); int zeroDiffFilesWithDiffCount = zeroDiffFilesWithDiffs.Count(); if (zeroDiffFilesWithDiffCount > 0) @@ -658,7 +711,7 @@ void DisplayMethodMetric(string headerText, string subtext, int methodCount, dyn summaryContents.AppendLine($"\n{zeroDiffFilesWithDiffCount} files had text diffs but no metric diffs."); foreach (var zerofile in zeroDiffFilesWithDiffs.Take(_count)) { - summaryContents.AppendLine($"{zerofile.baseName} had {diffCounts[zerofile.baseName]} diffs"); + summaryContents.AppendLine($"{zerofile.File.baseName} had {diffCounts[zerofile.Path]} diffs"); } } } @@ -842,70 +895,125 @@ public static StringBuilder GenerateTSV(IEnumerable compareList) // For example: // 6\t6\t\0d:\root\dasmset_8\base\Vector3Interop_ro.dasm\0d:\root\dasmset_8\diff\Vector3Interop_ro.dasm\0 // - public static Dictionary DiffInText(string diffPath, string basePath) + public static Dictionary DiffInText(string diffPath, string basePath) => + DiffInText(diffPath, basePath, filesNeedingCounts: null); + + private static Dictionary DiffInText(string diffPath, string basePath, HashSet filesNeedingCounts) { - // run get diff command to see if we have textual diffs. - // (use git diff since it's already a dependency and cross platform) - List commandArgs = new List(); - commandArgs.Add("diff"); - commandArgs.Add("--no-index"); - commandArgs.Add("--diff-filter=M"); - commandArgs.Add("--exit-code"); - commandArgs.Add("--numstat"); - commandArgs.Add("-z"); - commandArgs.Add(basePath); - commandArgs.Add(diffPath); - - ProcessResult result = Utility.ExecuteProcess("git", commandArgs, true); - Dictionary fileToTextDiffCount = new Dictionary(); ; - - if (result.ExitCode != 0) - { - // There are files with diffs. Build up a dictionary mapping base file name to net text diff count. - - var rawLines = result.StdOut.Split(new[] { "\0", Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); - if (rawLines.Length % 3 != 0) - { - Console.WriteLine($"Error parsing output: {result.StdOut}"); - return fileToTextDiffCount; - } + basePath = Path.GetFullPath(basePath); + diffPath = Path.GetFullPath(diffPath); + IEnumerable<(string Base, string Diff)> pairs; + bool baseDirectory = IsRealDirectory(basePath); + bool diffDirectory = IsRealDirectory(diffPath); + if (baseDirectory && diffDirectory) + { + pairs = EnumerateTextPairs(new DirectoryInfo(basePath), new DirectoryInfo(diffPath)); + } + else + { + if (baseDirectory) + basePath = Path.Combine(basePath, Path.GetFileName(diffPath)); + if (diffDirectory) + diffPath = Path.Combine(diffPath, Path.GetFileName(basePath)); + pairs = new[] { (basePath, diffPath) }; + } - for (int i = 0; i < rawLines.Length; i += 3) - { - string rawStats = rawLines[i]; - string rawBasePath = rawLines[i + 1]; - string rawDiffPath = rawLines[i + 2]; + // Initialize the process manager on the caller thread before starting parallel workers. + _ = ProcessManager.Instance; + var changes = pairs.AsParallel() + .Select(pair => (pair.Base, Result: CompareText(pair.Base, pair.Diff, + countLines: filesNeedingCounts == null || filesNeedingCounts.Contains(pair.Base)))) + .Where(pair => pair.Result.HasChanges) + .ToArray(); + + if (changes.Length != 0) + Console.WriteLine($"Found {changes.Length} files with textual diffs."); + return changes.Where(pair => pair.Result.LineCount.HasValue) + .ToDictionary(pair => pair.Base, pair => pair.Result.LineCount.Value, StringComparer.Ordinal); + } - string[] fields = rawStats.Split(new char[] { ' ', '\t', '"' }, StringSplitOptions.RemoveEmptyEntries); + // Directory.Exists follows links; Git compares directory links themselves instead. + private static bool IsRealDirectory(string path) => + (File.GetAttributes(path) & (FileAttributes.Directory | FileAttributes.ReparsePoint)) == FileAttributes.Directory; - string parsedFullDiffFilePath = Path.GetFullPath(rawDiffPath); - string parsedFullBaseFilePath = Path.GetFullPath(rawBasePath); + private static IEnumerable<(string Base, string Diff)> EnumerateTextPairs(DirectoryInfo baseline, DirectoryInfo diff) + { + var diffEntries = diff.EnumerateFileSystemInfos().ToDictionary(entry => entry.Name, StringComparer.Ordinal); + foreach (FileSystemInfo entry in baseline.EnumerateFileSystemInfos()) + { + if (!diffEntries.TryGetValue(entry.Name, out FileSystemInfo other)) + continue; - if (!File.Exists(parsedFullBaseFilePath)) - { - Console.WriteLine($"Error parsing path '{rawBasePath}'. `{parsedFullBaseFilePath}` doesn't exist."); - continue; - } + bool baseDirectory = IsRealDirectory(entry.FullName); + bool diffDirectory = IsRealDirectory(other.FullName); + if (baseDirectory && diffDirectory) + { + foreach (var pair in EnumerateTextPairs((DirectoryInfo)entry, (DirectoryInfo)other)) + yield return pair; + } + else if (!baseDirectory && !diffDirectory) + { + yield return (entry.FullName, other.FullName); + } + } + } + private static bool FilesEqual(string basePath, string diffPath) + { + // Git compares symbolic links themselves, not the contents of their targets. + var baseInfo = new IOFileInfo(basePath); + var diffInfo = new IOFileInfo(diffPath); + if (baseInfo.LinkTarget != null || diffInfo.LinkTarget != null) + return false; - if (!File.Exists(parsedFullDiffFilePath)) - { - Console.WriteLine($"Error parsing path '{rawDiffPath}'. `{parsedFullDiffFilePath}` doesn't exist."); - continue; - } + if (baseInfo.Length != diffInfo.Length) + return false; - // Sometimes .dasm is parsed as binary and we don't get numbers, just dashes - int addCount = 0; - int delCount = 0; - Int32.TryParse(fields[0], out addCount); - Int32.TryParse(fields[1], out delCount); - fileToTextDiffCount[parsedFullBaseFilePath] = addCount + delCount; - } + using var baseStream = File.OpenRead(basePath); + using var diffStream = File.OpenRead(diffPath); - Console.WriteLine($"Found {fileToTextDiffCount.Count()} files with textual diffs."); + byte[] baseBuffer = ArrayPool.Shared.Rent(64 * 1024); + byte[] diffBuffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + int read; + while ((read = baseStream.Read(baseBuffer)) != 0) + { + if (diffStream.ReadAtLeast(diffBuffer.AsSpan(0, read), read, throwOnEndOfStream: false) != read || + !baseBuffer.AsSpan(0, read).SequenceEqual(diffBuffer.AsSpan(0, read))) + return false; + } + return diffStream.ReadByte() == -1; } + finally + { + ArrayPool.Shared.Return(baseBuffer); + ArrayPool.Shared.Return(diffBuffer); + } + } - return fileToTextDiffCount; + private static (bool HasChanges, int? LineCount) CompareText(string basePath, string diffPath, bool countLines) + { + var startInfo = new ProcessStartInfo("git"); + foreach (string argument in new[] { "diff", "--no-index", "--diff-filter=M", "--exit-code", countLines ? "--numstat" : "--quiet", "-z", "--", basePath, diffPath }) + startInfo.ArgumentList.Add(argument); + + ProcessResult result = Utility.ExecuteProcess(startInfo, capture: true); + if (result.ExitCode == 0) + return (false, null); + if (result.ExitCode != 1) + throw new InvalidOperationException($"git diff failed for '{basePath}' and '{diffPath}' (exit {result.ExitCode}): {result.StdErr}"); + + if (!countLines) + return (true, null); + + string[] fields = result.StdOut.Split('\t', 3); + if (fields.Length != 3) + throw new InvalidOperationException($"Invalid git numstat output for '{basePath}': {result.StdOut}"); + // Binary files have '-' in both numeric fields. + return (true, ParseCount(fields[0]) + ParseCount(fields[1])); + + static int ParseCount(string value) => value == "-" ? 0 : int.Parse(value, CultureInfo.InvariantCulture); } private T Get(Option option) => _command.Result.GetValue(option); @@ -940,9 +1048,18 @@ public int Run() string json = Get(_command.Json); string tsv = Get(_command.Tsv); string md = Get(_command.MD); - foreach (var metricName in Get(_command.Metrics)) + string[] metricNames = Get(_command.Metrics).ToArray(); + FileDelta[][] comparisons = Comparator(baseList, diffList, metricNames); + // Detailed text counts are only displayed for files with no metric differences. + // Still ask Git whether every other file changed, without computing unused numstat data. + _filesNeedingTextDiffCounts = comparisons.SelectMany(files => files) + .Where(file => file.deltaMetrics.IsZero() && !Get(_command.ConcatFiles)) + .Select(file => Path.GetFullPath(Path.Combine(_baseDirectory, file.baseName))) + .ToHashSet(StringComparer.Ordinal); + for (int metricIndex = 0; metricIndex < metricNames.Length; metricIndex++) { - compareList = Comparator(baseList, diffList, metricName); + string metricName = metricNames[metricIndex]; + compareList = comparisons.Select(files => files[metricIndex]).ToArray(); if (tsv != null) { diff --git a/src/util/util.cs b/src/util/util.cs index b0eee54e..1aa3307e 100644 --- a/src/util/util.cs +++ b/src/util/util.cs @@ -175,10 +175,6 @@ public static ProcessResult ExecuteProcess(string name, IEnumerable comm { var startInfo = new ProcessStartInfo { - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardError = true, - RedirectStandardOutput = true, WorkingDirectory = workingDirectory, FileName = name, Arguments = string.Join(" ", commandArgs) @@ -192,11 +188,22 @@ public static ProcessResult ExecuteProcess(string name, IEnumerable comm } } + return ExecuteProcess(startInfo, capture); + } + + // Supports ArgumentList without changing the legacy overload's pre-quoted argument handling. + public static ProcessResult ExecuteProcess(ProcessStartInfo startInfo, bool capture = false) + { + startInfo.UseShellExecute = false; + startInfo.CreateNoWindow = true; + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + // set up the pipe for the stdout and builder for stderr StringBuilder _errorDataStringBuilder = new StringBuilder(); StringBuilder _outputDataStringBuilder = new StringBuilder(); - Process process = ProcessManager.Instance.Start(startInfo); + using Process process = ProcessManager.Instance.Start(startInfo); if (capture) { @@ -233,7 +240,7 @@ public static ProcessResult ExecuteProcess(string name, IEnumerable comm catch (System.Exception e) { // Maybe the program we're spawning wasn't found (ERROR_FILE_NOT_FOUND == 2). - Console.Error.WriteLine($"Error: failed to start '{name} {startInfo.Arguments}': {e.Message}"); + Console.Error.WriteLine($"Error: failed to start '{startInfo.FileName} {startInfo.Arguments}': {e.Message}"); return new ProcessResult() {