Add benchmarking tool

src/Harp.Toolkit/Benchmark/BenchmarkCommand.cs

@@ -0,0 +1,229 @@
+using System.CommandLine;
+using Spectre.Console;
+using Harp.Generators;
+using Harp.Toolkit.Benchmark;
+using Harp.Toolkit.Benchmark.Suites;
+using Harp.Toolkit.Generate;
+
+namespace Harp.Toolkit;
+public class BenchmarkCommand : Command
+{
+    public BenchmarkCommand()
+        : base("benchmark", "Run benchmark tests on the device.")
+    {
+        PortNameOption portNameOption = new();
+        Option<FileInfo?> fileOption = new("--report")
+        {
+            Description = "Path to the HTML report generated after running tests.",
+            Required = false,
+        };
+
+        Option<bool> verboseOption = new("--verbose")
+        {
+            Description = "Show detailed results for each test.",
+            Required = false,
+        };
+
+        Option<string?> clockPortOption = new("--clock-port")
+        {
+            Description = "Serial port of the reference clock device. Enables clock alignment tests.",
+            Required = false,
+        };
+
+        Option<int?> regClockOption = new("--pps-address")
+        {
+            Description = "Register address on the tested device (--port) that emits an event whenever the incoming PPS signal goes high. Enables PPS alignment test.",
+            Required = false,
+        };
+
+        Option<int> clockSamplesOption = new("--clock-samples")
+        {
+            Description = "Number of PPS event pairs to collect for the PPS alignment test. Default: 5.",
+            Required = false,
+        };
+        clockSamplesOption.DefaultValueFactory = _ => 5;
+
+        Option<FileInfo> deviceYmlOption = new("--device-yml")
+        {
+            Description = "Path to the device's device.yml. Enables validation of the generated C# interface against a live read of every declared register, and cross-checks WhoAmI/firmware/hardware versions.",
+            Required = false,
+        };
+        OptionValidation.AcceptExistingOnly(deviceYmlOption);
+
+        Options.Add(portNameOption);
+        Options.Add(fileOption);
+        Options.Add(verboseOption);
+        Options.Add(clockPortOption);
+        Options.Add(regClockOption);
+        Options.Add(clockSamplesOption);
+        Options.Add(deviceYmlOption);
+        SetAction(parsedResult =>
+        {
+            string portName = parsedResult.GetRequiredValue(portNameOption);
+            FileInfo? reportFile = parsedResult.GetValue(fileOption);
+            bool verbose = parsedResult.GetValue(verboseOption);
+            string? clockPort = parsedResult.GetValue(clockPortOption);
+            ClockTestOptions? clockOptions = clockPort is null ? null : new ClockTestOptions(
+                ClockPort: clockPort,
+                PpsAddress: parsedResult.GetValue(regClockOption),
+                ClockSamples: parsedResult.GetValue(clockSamplesOption));
+            FileInfo? deviceYml = parsedResult.GetValue(deviceYmlOption);
+            return RunBenchmarks(portName, reportFile, verbose, clockOptions, deviceYml, CancellationToken.None);
+        });
+    }
+
+    static async Task RunBenchmarks(string portName, FileInfo? reportFile, bool verbose, ClockTestOptions? clockOptions, FileInfo? deviceYml, CancellationToken cancellationToken)
+    {
+        AnsiConsole.MarkupLine($"Running tests on [bold]{portName}[/]...");
+        if (clockOptions is not null)
+            AnsiConsole.MarkupLine($"Clock reference device: [bold]{clockOptions.ClockPort}[/]");
+
+        DeviceInfo? deviceMetadata = null;
+        string? deviceRawYaml = null;
+        if (deviceYml is not null)
+        {
+            AnsiConsole.Markup($"Loading device metadata from [bold]{deviceYml.FullName}[/]...");
+            deviceMetadata = GeneratorHelper.ReadDeviceMetadata(deviceYml.FullName);
+            deviceRawYaml = await File.ReadAllTextAsync(deviceYml.FullName, cancellationToken);
+            AnsiConsole.MarkupLine($" [green]Done![/] ({deviceMetadata.Registers.Count} registers)");
+        }
+
+        var runner = new CoreRunner(clockOptions, deviceMetadata, deviceRawYaml);
+        var report = new Report
+        {
+            DeviceName = $"Harp Device ({portName})",
+            RunDate = DateTime.Now
+        };
+
+        int currentTest = 0;
+        await foreach (var (suite, result) in runner.RunAllAsync(portName, cancellationToken, (suite, testName, testDesc) =>
+        {
+            // Print "Running" status before test execution (without newline)
+            currentTest++;
+            Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running...");
+        }))
+        {
+            // Clear the line by moving cursor to start and overwriting with spaces, then print result
+            Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
+            AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}");
+
+            var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name);
+            if (suiteResult == null)
+            {
+                suiteResult = new SuiteResult
+                {
+                    Name = suite.GetType().Name,
+                    Description = suite.Description
+                };
+                report.Suites.Add(suiteResult);
+            }
+            suiteResult.Results.Add(result);
+        }
+
+        if (verbose)
+        {
+            AnsiConsole.WriteLine();
+            AnsiConsole.Write(new Rule("[yellow]Detailed Results[/]"));
+            foreach (var suite in report.Suites)
+            {
+                AnsiConsole.MarkupLine($"[bold underline]{suite.Name}[/]");
+                AnsiConsole.MarkupLine($"[dim]{suite.Description}[/]");
+
+                var table = new Table();
+                table.AddColumn("Test Case");
+                table.AddColumn("Status");
+                table.AddColumn("Details");
+                table.AddColumn("Message");
+
+                foreach (var test in suite.Results)
+                {
+                    string details = "";
+                    string message = test.Result.Message ?? "";
+
+                    if (test.Result is NumericBenchmarkResult bsr)
+                    {
+                        details = $"Mean: {bsr.Summary.Mean:F4}\nMedian: {bsr.Summary.Median:F4}\nStdDev: {bsr.Summary.StdDev:F4}\nMin: {bsr.Summary.Min:F4}\nMax: {bsr.Summary.Max:F4}\nPercentiles: 99th={bsr.Summary.Percentile99:F4}, 01th={bsr.Summary.Percentile01:F4}";
+                    }
+                    else if (test.Result is ErrorResult er)
+                    {
+                        details = $"{er.Exception.GetType().Name}";
+                    }
+                    else
+                    {
+                        var valProp = test.Result?.GetType().GetProperty("Value");
+                        if (valProp != null)
+                        {
+                            var val = valProp.GetValue(test.Result);
+                            details = val?.ToString() ?? "";
+                        }
+                    }
+
+                    table.AddRow(
+                        new Markup($"[bold]{Markup.Escape(test.Name)}[/]\n[dim]{Markup.Escape(test.Description)}[/]"),
+                        new Markup(GetResultMarkup(test.Result)),
+                        new Markup(Markup.Escape(details)),
+                        new Markup(Markup.Escape(message))
+                    );
+                }
+                AnsiConsole.Write(table);
+                AnsiConsole.WriteLine();
+            }
+        }
+
+        if (reportFile != null)
+        {
+            AnsiConsole.Markup("Generating HTML report...");
+            string html = await HtmlReportGenerator.GenerateAsync(report);
+            string fileName = reportFile?.FullName ?? $"TestReport_{DateTime.Now:yyyyMMdd_HHmmss}.html";
+            await File.WriteAllTextAsync(fileName, html, cancellationToken);
+            AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]");
+        }
+    }
+
+    static string GetResultMarkup(IResult result)
+    {
+        return result.Status switch
+        {
+            Status.Passed => "[green]Passed[/]",
+            Status.Failed => "[red]Failed[/]",
+            Status.Error => "[red]Error[/]",
+            Status.Skipped => "[yellow]Skipped[/]",
+            _ => $"[white]{result.Status}[/]"
+        };
+    }
+
+    class CoreRunner : Runner
+    {
+        public CoreRunner(
+            ClockTestOptions? clockOptions = null,
+            DeviceInfo? deviceMetadata = null,
+            string? deviceRawYaml = null) : base()
+        {
+            AddSuite(new R_WHO_AM_I());
+            AddSuite(new R_HW_VERSION_H());
+            AddSuite(new R_HW_VERSION_L());
+            AddSuite(new R_ASSEMBLY_VERSION());
+            AddSuite(new R_CORE_VERSION_H());
+            AddSuite(new R_CORE_VERSION_L());
+            AddSuite(new R_FW_VERSION_H());
+            AddSuite(new R_FW_VERSION_L());
+            AddSuite(new R_TIMESTAMP_SECOND());
+            AddSuite(new R_TIMESTAMP_MICRO());
+            AddSuite(new R_OPERATION_CTRL());
+            AddSuite(new R_RESET_DEV());
+            AddSuite(new R_DEVICE_NAME());
+            AddSuite(new R_SERIAL_NUMBER());
+            AddSuite(new R_CLOCK_CONFIG());
+            AddSuite(new R_TIMESTAMP_OFFSET());
+            AddSuite(new R_UID());
+            AddSuite(new R_TAG());
+            AddSuite(new R_HEARTBEAT());
+            AddSuite(new R_VERSION());
+            AddSuite(new RoundTripTestSuite());
+            AddSuite(new ClockTestSuite(clockOptions));
+            AddSuite(new DeviceInterfaceSuite(deviceMetadata, deviceRawYaml));
+        }
+    }
+}
+
+

src/Harp.Toolkit/Benchmark/ClockTestOptions.cs

@@ -0,0 +1,18 @@
+namespace Harp.Toolkit.Benchmark;
+
+/// <summary>
+/// Options for clock alignment and PPS synchronization tests run against a reference clock device.
+/// </summary>
+/// <param name="ClockPort">
+/// Serial port of the reference clock device (WhiteRabbit). Enabling this option runs the
+/// simultaneous WhoAmI timestamp comparison test.
+/// </param>
+/// <param name="PpsAddress">
+/// Register address on the tested device that emits an event whenever the incoming PPS signal
+/// goes high. When provided, also runs the PPS alignment test.
+/// </param>
+/// <param name="ClockSamples">Number of PPS event pairs to collect for the PPS alignment test.</param>
+internal record ClockTestOptions(
+    string ClockPort,
+    int? PpsAddress = null,
+    int ClockSamples = 5);

src/Harp.Toolkit/Benchmark/GeneratedInterfaceCompiler.cs

@@ -0,0 +1,78 @@
+using System.Reflection;
+using System.Text;
+using Harp.Generators;
+using Harp.Toolkit.Generate;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Emit;
+using Microsoft.Extensions.DependencyModel;
+
+namespace Harp.Toolkit.Benchmark;
+
+/// <summary>
+/// Generates the C# interface for a device.yml (via <see cref="InterfaceGenerator"/>),
+/// compiles it in-memory, and returns its register address-to-type map so callers can
+/// invoke each register's own generated parser reflectively.
+/// </summary>
+internal static class GeneratedInterfaceCompiler
+{
+    public static IReadOnlyDictionary<int, Type> Compile(DeviceInfo deviceOnlyMetadata, string rawDeviceYaml, string @namespace)
+    {
+        var generator = new InterfaceGenerator(deviceOnlyMetadata, @namespace);
+        var implementation = generator.GenerateImplementation();
+        if (!GeneratorHelper.AssertNoGeneratorErrors(generator.Errors))
+            throw new InvalidOperationException("Interface generation from device.yml completed with errors.");
+
+        var syntaxTree = CSharpSyntaxTree.ParseText(implementation.Device);
+        var compilation = CSharpCompilation.Create(
+            $"HarpGeneratedInterface_{@namespace}",
+            new[] { syntaxTree },
+            GetMetadataReferences(),
+            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: true));
+
+        // The generated Device class's static constructor reads device.yml back from
+        // an embedded "{Namespace}.device.yml" manifest resource (e.g. to expose it via
+        // the Metadata property) - without it, merely accessing RegisterMap throws.
+        var rawYamlBytes = Encoding.UTF8.GetBytes(rawDeviceYaml);
+        var deviceYamlResource = new ResourceDescription(
+            $"{@namespace}.device.yml",
+            () => new MemoryStream(rawYamlBytes),
+            isPublic: true);
+
+        using var peStream = new MemoryStream();
+        var result = compilation.Emit(peStream, manifestResources: new[] { deviceYamlResource });
+        if (!result.Success)
+        {
+            var errors = string.Join(Environment.NewLine, result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error));
+            throw new InvalidOperationException($"Failed to compile the interface generated from device.yml:{Environment.NewLine}{errors}");
+        }
+
+        var assembly = Assembly.Load(peStream.ToArray());
+        var deviceType = assembly.GetType($"{@namespace}.Device")
+            ?? throw new InvalidOperationException($"Compiled assembly does not contain type '{@namespace}.Device'.");
+        var registerMapProperty = deviceType.GetProperty("RegisterMap", BindingFlags.Public | BindingFlags.Static)
+            ?? throw new InvalidOperationException($"'{@namespace}.Device' does not expose a static RegisterMap property.");
+
+        return (IReadOnlyDictionary<int, Type>)registerMapProperty.GetValue(null)!;
+    }
+
+    // Reuses this project's existing PreserveCompilationContext setup (already required
+    // for RazorLight's own runtime compilation) to resolve the full reference-assembly
+    // closure, including Bonsai.Harp/Bonsai.Core, which the generated code depends on.
+    private static IReadOnlyList<MetadataReference> GetMetadataReferences()
+    {
+        var context = DependencyContext.Default
+            ?? throw new InvalidOperationException("No DependencyContext available for compiling the generated interface.");
+
+        var paths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        foreach (var library in context.CompileLibraries)
+        {
+            foreach (var path in library.ResolveReferencePaths())
+            {
+                paths.Add(path);
+            }
+        }
+
+        return paths.Select(path => (MetadataReference)MetadataReference.CreateFromFile(path)).ToList();
+    }
+}

src/Harp.Toolkit/Benchmark/HarpTestAttribute.cs

@@ -0,0 +1,7 @@
+namespace Harp.Toolkit;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class HarpTestAttribute : Attribute
+{
+    public string? Description { get; set; }
+}

src/Harp.Toolkit/Benchmark/HtmlReportGenerator.cs

@@ -0,0 +1,21 @@
+using System.Reflection;
+using RazorLight;
+
+namespace Harp.Toolkit;
+
+public static class HtmlReportGenerator
+{
+    public static async Task<string> GenerateAsync(Report report)
+    {
+        var engine = new RazorLightEngineBuilder()
+            .UseFileSystemProject(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))
+            .UseMemoryCachingProvider()
+            .Build();
+
+        // The template is copied to the output directory under Reporting/ReportTemplate.cshtml
+        // RazorLight expects the path relative to the project root (which we set to the assembly location)
+        string templatePath = Path.Combine("Benchmark", "ReportTemplate.cshtml");
+
+        return await engine.CompileRenderAsync(templatePath, report);
+    }
+}

src/Harp.Toolkit/Benchmark/Report.cs

@@ -0,0 +1,8 @@
+namespace Harp.Toolkit;
+
+public class Report
+{
+    public string DeviceName { get; set; } = "Unknown Device";
+    public DateTime RunDate { get; set; } = DateTime.Now;
+    public List<SuiteResult> Suites { get; set; } = new();
+}