Adam
/
AyCode.Core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor Output to BenchmarkReportWriter 

- Removed Output.cs and migrated all reporting, formatting, and statistics logic to a new BenchmarkReportWriter static class.
- Extended ReportingContext with run-header fields for richer output metadata.
- Updated BenchmarkLoop to use BenchmarkReportWriter and the new ReportingContext.
- Centralized all output file generation (.log, .LLM, .output) and formatting helpers in BenchmarkReportWriter.
- Improved separation of concerns and unified output artifact naming and metadata.
This commit is contained in:
Loretta 2026-05-15 20:18:13 +02:00
parent 9dcb62ce23
commit ed03d754ec
3 changed files with 95 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
AyCode.Core.Serializers.Console/Output.cs → AyCode.Benchmark/Reporting/BenchmarkReportWriter.cs
View File

@ -1,4 +1,3 @@
using AyCode.Core.Benchmarks.Reporting;
using AyCode.Core.Benchmarks.Workloads.Scenarios; using AyCode.Core.Benchmarks.Workloads.Scenarios;
using AyCode.Core.Serializers.Binaries; using AyCode.Core.Serializers.Binaries;
using AyCode.Core.Tests.TestModels; using AyCode.Core.Tests.TestModels;
@ -6,15 +5,20 @@ using System.Globalization;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Text; using System.Text;
namespace AyCode.Core.Serializers.Console; namespace AyCode.Core.Benchmarks.Reporting;
/// <summary> /// <summary>
/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file. /// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file, .output
/// Consumes <see cref="BenchmarkResult"/> rows produced by the benchmark loop and emits human-readable /// binary hex dump. Consumes <see cref="BenchmarkResult"/> rows produced by the benchmark execution loop
/// + LLM-friendly outputs. Display-only helpers (per-op µs conversion, allocation KB formatting, /// (Console-side <c>BenchmarkLoop</c> or BDN-side <c>BdnSummaryAdapter</c>) and emits human-readable +
/// range/CV formatting, paired-aggregation stats) live here too — they are display-side concerns. /// LLM-friendly outputs.
///
/// <para>The <see cref="ReportingContext"/> parameter encapsulates per-run state — <see cref="ReportingContext.SourceTag"/>
/// drives the filename prefix ("Console" / "Bdn"), <see cref="ReportingContext.ResultsDirectory"/> is the
/// resolved (walk-up-to-.sln) output folder, and the remaining fields (charset, iter counts, target sample
/// window, CV threshold) carry the run-header info embedded in every emitted artifact.</para>
/// </summary> /// </summary>
internal static class Output public static class BenchmarkReportWriter
{ {
/// <summary> /// <summary>
/// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation /// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation
@ -27,18 +31,18 @@ internal static class Output
/// <param name="AcAvg">Arithmetic mean AcBinary value (µs/op or bytes).</param> /// <param name="AcAvg">Arithmetic mean AcBinary value (µs/op or bytes).</param>
/// <param name="MpAvg">Arithmetic mean MemPack value.</param> /// <param name="MpAvg">Arithmetic mean MemPack value.</param>
/// <param name="CellCount">Number of paired cells contributing to the geo/median.</param> /// <param name="CellCount">Number of paired cells contributing to the geo/median.</param>
internal record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount); public record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount);
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0; public static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0;
// Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever // Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever
// averaging or comparison happens across rows with potentially different iter counts (Winners summary, // averaging or comparison happens across rows with potentially different iter counts (Winners summary,
// Overall comparison, per-cell summary row). Keeping these as methods rather than properties on // Overall comparison, per-cell summary row). Keeping these as methods rather than properties on
// BenchmarkResult preserves the result-as-data-bag distinction. // BenchmarkResult preserves the result-as-data-bag distinction.
internal static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations); public static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations);
internal static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations); public static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations);
internal static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations); public static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations);
/// <summary> /// <summary>
/// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can /// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can
@ -47,7 +51,7 @@ internal static class Output
/// integers untouched. /// integers untouched.
/// </summary> /// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static double ToKilobytes(long bytes) => bytes / 1024.0; public static double ToKilobytes(long bytes) => bytes / 1024.0;
/// <summary> /// <summary>
/// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison /// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison
@ -55,7 +59,7 @@ internal static class Output
/// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats. /// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats.
/// Returns null when no paired cell has a valid value. /// Returns null when no paired cell has a valid value.
/// </summary> /// </summary>
internal static OverallStats? ComputeOverallStats(List<BenchmarkResult> acResults, List<BenchmarkResult> mpResults, Func<BenchmarkResult, double> getValue) public static OverallStats? ComputeOverallStats(List<BenchmarkResult> acResults, List<BenchmarkResult> mpResults, Func<BenchmarkResult, double> getValue)
{ {
if (acResults.Count == 0 || mpResults.Count == 0) return null; if (acResults.Count == 0 || mpResults.Count == 0) return null;
@ -93,12 +97,12 @@ internal static class Output
/// <summary> /// <summary>
/// Formats a per-op micros value with its inter-sample range and CV-threshold marker as /// Formats a per-op micros value with its inter-sample range and CV-threshold marker as
/// <c>"26.86 (24.5..29.1)"</c> or <c>"26.86 (24.5..29.1) ⚠️5.2%"</c>. Median first, range in parentheses, /// <c>"26.86 (24.5..29.1)"</c> or <c>"26.86 (24.5..29.1) ⚠️5.2%"</c>. Median first, range in parentheses,
/// CV warning suffix only when CV > <see cref="Configuration.UnstableCVThreshold"/>. When min == max == median /// CV warning suffix only when CV > <paramref name="unstableCvThreshold"/>. When min == max == median
/// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter. /// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter.
/// All time inputs are total-batch milliseconds; <paramref name="iterations"/> is the per-row iter /// All time inputs are total-batch milliseconds; <paramref name="iterations"/> is the per-row iter
/// count (post-adaptive-calibration). /// count (post-adaptive-calibration).
/// </summary> /// </summary>
internal static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv) public static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv, double unstableCvThreshold)
{ {
var med = ToPerOpMicros(medianMs, iterations); var med = ToPerOpMicros(medianMs, iterations);
// No range data (single-sample fast path) — surface as bare median, identical to the prior format. // No range data (single-sample fast path) — surface as bare median, identical to the prior format.
@ -114,7 +118,7 @@ internal static class Output
if (medianMs > 0 && stdDevMs > 0) if (medianMs > 0 && stdDevMs > 0)
{ {
var cv = stdDevMs / medianMs; var cv = stdDevMs / medianMs;
if (cv > Configuration.UnstableCVThreshold) if (cv > unstableCvThreshold)
{ {
var cvPct = (cv * 100).ToString("F1", inv); var cvPct = (cv * 100).ToString("F1", inv);
return $"{range} ⚠️{cvPct}%"; return $"{range} ⚠️{cvPct}%";
@ -128,14 +132,14 @@ internal static class Output
/// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`). /// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`).
/// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block. /// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block.
/// </summary> /// </summary>
internal static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%"; public static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%";
/// <summary> /// <summary>
/// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means. /// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means.
/// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when /// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when
/// stats is null (no paired data). /// stats is null (no paired data).
/// </summary> /// </summary>
internal static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2") public static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2")
{ {
if (stats == null) return; if (stats == null) return;
@ -150,13 +154,13 @@ internal static class Output
/// Same as <see cref="WriteOverallLine"/> but appends to a <see cref="StringBuilder"/> (no color). /// Same as <see cref="WriteOverallLine"/> but appends to a <see cref="StringBuilder"/> (no color).
/// Used by the .log and .LLM file writers. /// Used by the .log and .LLM file writers.
/// </summary> /// </summary>
internal static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2") public static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2")
{ {
if (stats == null) return; if (stats == null) return;
sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
} }
internal static void PrintResult(BenchmarkResult result) public static void PrintResult(BenchmarkResult result)
{ {
// Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op). // Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op).
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A"; var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A";
@ -166,7 +170,7 @@ internal static class Output
System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)"); System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)");
} }
internal static void PrintGroupedResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets) public static void PrintGroupedResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets)
{ {
System.Console.WriteLine("\n"); System.Console.WriteLine("\n");
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
@ -407,14 +411,21 @@ internal static class Output
WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0"); WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0");
} }
internal static void SaveResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets) /// <summary>
/// Writes the unified file triplet — <c>{SourceTag}.FullBenchmark_{Build}_{timestamp}.{log, LLM, output}</c>
/// — to <see cref="ReportingContext.ResultsDirectory"/>. The <c>.log</c> is the human-readable formatted
/// view, the <c>.LLM</c> is the markdown LLM-paste-friendly view, and the <c>.output</c> is a binary hex
/// dump of the Large test data's AcBinary-Default serialization (for raw inspection / wire-debugging).
/// </summary>
public static void SaveAll(ReportingContext ctx, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
{ {
Directory.CreateDirectory(Configuration.ResultsDirectory); Directory.CreateDirectory(ctx.ResultsDirectory);
var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}"; var baseFileName = $"{ctx.SourceTag}.FullBenchmark_{ctx.BuildConfiguration}_{timestamp}";
var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log"); var logFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.log");
var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output"); var outputFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.output");
var llmFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.LLM");
// Save binary output to separate .output file. // Save binary output to separate .output file.
// Cast to TestDataSet<TestOrder_All_False> because Phase 1 hardcodes the benchmark variant. // Cast to TestDataSet<TestOrder_All_False> because Phase 1 hardcodes the benchmark variant.
@ -436,7 +447,7 @@ internal static class Output
outputSb.AppendLine("Hex dump:"); outputSb.AppendLine("Hex dump:");
outputSb.AppendLine(FormatHexDump(serializedBytes)); outputSb.AppendLine(FormatHexDump(serializedBytes));
File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom); File.WriteAllText(outputFilePath, outputSb.ToString(), ctx.Utf8NoBom);
System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}"); System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}");
} }
@ -445,10 +456,11 @@ internal static class Output
sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║"); sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║");
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║"); sb.AppendLine($"║ Source: {ctx.SourceTag}".PadRight(100) + "║");
sb.AppendLine($"║ Charset: {Configuration.GetCurrentCharsetName()}".PadRight(100) + "║"); sb.AppendLine($"║ Build: {ctx.BuildConfiguration}".PadRight(100) + "║");
sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║"); sb.AppendLine($"║ Charset: {ctx.CharsetName}".PadRight(100) + "║");
sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║"); sb.AppendLine($"║ Iterations: per-cell adaptive (~{ctx.TargetSampleMs} ms target)".PadRight(100) + "║");
sb.AppendLine($"║ Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║");
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
sb.AppendLine(); sb.AppendLine();
@ -471,6 +483,10 @@ internal static class Output
} }
// CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely. // CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely.
// InvariantCulture is mandatory here: the decimal-separator dimension MUST be `.` so the comma-separated
// field delimiters don't collide with locale-specific decimal commas (e.g. Hungarian "7,38" would split
// a single F2 value across two CSV fields).
var inv = CultureInfo.InvariantCulture;
sb.AppendLine("=== RAW DATA (CSV) ==="); sb.AppendLine("=== RAW DATA (CSV) ===");
sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes"); sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes");
@ -479,7 +495,7 @@ internal static class Output
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList(); var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList();
foreach (var result in testResults) foreach (var result in testResults)
{ {
sb.AppendLine($"{result.TestDataName},{result.Engine.ToDisplay()},{result.IoMode.ToDisplay()},{result.DispatchMode.ToDisplay()},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}"); sb.AppendLine($"{result.TestDataName},{result.Engine.ToDisplay()},{result.IoMode.ToDisplay()},{result.DispatchMode.ToDisplay()},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result).ToString("F2", inv)},{DesPerOp(result).ToString("F2", inv)},{RtPerOp(result).ToString("F2", inv)},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}");
} }
} }
sb.AppendLine(); sb.AppendLine();
@ -546,11 +562,10 @@ internal static class Output
if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0) if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0)
{ {
sb.AppendLine(" (Comparison requires both serialize and deserialize data)"); sb.AppendLine(" (Comparison requires both serialize and deserialize data)");
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); SaveLlmResults(ctx, llmFilePath, results, testDataSets);
SaveLlmResults(llmFilePathEarly, results, testDataSets);
return; return;
} }
@ -564,23 +579,22 @@ internal static class Output
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp)); AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp));
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0"); AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0");
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
// Save LLM-optimized results // Save LLM-optimized results
var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); SaveLlmResults(ctx, llmFilePath, results, testDataSets);
SaveLlmResults(llmFilePath, results, testDataSets);
} }
internal static void SaveLlmResults(string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets) private static void SaveLlmResults(ReportingContext ctx, string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
{ {
var sb = new StringBuilder(); var sb = new StringBuilder();
sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); sb.AppendLine($"# AcBinary Benchmark [{ctx.SourceTag}] {ctx.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
sb.AppendLine($"Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%"); sb.AppendLine($"Charset: {ctx.CharsetName} | Iterations: per-cell adaptive (target ~{ctx.TargetSampleMs} ms/sample) | Warmup: {ctx.WarmupIterations} per phase (Ser/Des isolated) | Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | UnstableCV threshold: {ctx.UnstableCVThreshold * 100:F0}%");
sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup"); sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup");
// Options summary. Bracketed [OrderType] surfaces the TestOrder variant each preset serialised — // Options summary. Bracketed [OrderType] surfaces the TestOrder variant each preset serialised —
// see SaveResults for the variant-dispatch rationale. // see SaveAll for the variant-dispatch rationale.
var optionsMap = results var optionsMap = results
.Where(r => r.OptionsDescription != null) .Where(r => r.OptionsDescription != null)
.Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!)) .Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!))
@ -601,6 +615,8 @@ internal static class Output
sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des"); sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des");
sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---"); sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---");
var inv = CultureInfo.InvariantCulture;
foreach (var testData in testDataSets) foreach (var testData in testDataSets)
{ {
var testResults = results var testResults = results
@ -610,12 +626,11 @@ internal static class Output
foreach (var r in testResults) foreach (var r in testResults)
{ {
var inv = CultureInfo.InvariantCulture; var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv, ctx.UnstableCVThreshold) : "-";
var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-"; var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv, ctx.UnstableCVThreshold) : "-";
var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-";
var rt = r.RoundTripTimeMs > 0 var rt = r.RoundTripTimeMs > 0
? (r.IsRoundTripOnly ? (r.IsRoundTripOnly
? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv) ? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv, ctx.UnstableCVThreshold)
: RtPerOp(r).ToString("F2", inv)) : RtPerOp(r).ToString("F2", inv))
: "-"; : "-";
@ -658,14 +673,14 @@ internal static class Output
sb.AppendLine("```"); sb.AppendLine("```");
} }
File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom); File.WriteAllText(filePath, sb.ToString(), ctx.Utf8NoBom);
System.Console.WriteLine($"✓ LLM results saved to: {filePath}"); System.Console.WriteLine($"✓ LLM results saved to: {filePath}");
} }
/// <summary> /// <summary>
/// Formats byte array as hex dump with offset, hex values, and ASCII representation. /// Formats byte array as hex dump with offset, hex values, and ASCII representation.
/// </summary> /// </summary>
internal static string FormatHexDump(byte[] bytes, int bytesPerLine = 16) public static string FormatHexDump(byte[] bytes, int bytesPerLine = 16)
{ {
var sb = new StringBuilder(); var sb = new StringBuilder();
for (var i = 0; i < bytes.Length; i += bytesPerLine) for (var i = 0; i < bytes.Length; i += bytesPerLine)

27
AyCode.Benchmark/Reporting/ReportingContext.cs
View File

@ -4,37 +4,34 @@ namespace AyCode.Core.Benchmarks.Reporting;
/// <summary> /// <summary>
/// Context bundle for the unified benchmark report writer. Same record on both sides (Console / BDN), /// Context bundle for the unified benchmark report writer. Same record on both sides (Console / BDN),
/// only the <see cref="SourceTag"/> differs ("Console" / "Bdn") — that drives the filename prefix /// the <see cref="SourceTag"/> differs ("Console" / "Bdn") and drives the filename prefix
/// (e.g. <c>Console.FullBenchmark_Release_{timestamp}.LLM</c> vs <c>Bdn.FullBenchmark_Release_{timestamp}.LLM</c>). /// (e.g. <c>Console.FullBenchmark_Release_{timestamp}.LLM</c> vs <c>Bdn.FullBenchmark_Release_{timestamp}.LLM</c>).
/// The <see cref="ResultsDirectory"/> resolution walks up from <see cref="AppContext.BaseDirectory"/> to the /// The <see cref="ResultsDirectory"/> resolution walks up from <see cref="AppContext.BaseDirectory"/> to the
/// nearest <c>AyCode.Core.sln</c> and combines with <c>Test_Benchmark_Results\Benchmark</c> — works across /// nearest <c>AyCode.Core.sln</c> and combines with <c>Test_Benchmark_Results\Benchmark</c> — works across
/// build modes (Debug / Release / AOT publish) and worktrees (each worktree has its own .sln, so its bench /// build modes (Debug / Release / AOT publish) and worktrees (each worktree has its own .sln, so its bench
/// results land alongside its code). /// results land alongside its code). The remaining fields capture run-header information (charset, iter
/// counts, target sample window, CV threshold) so the writer can render a self-documenting header in both
/// the <c>.log</c> and <c>.LLM</c> outputs.
/// </summary> /// </summary>
public sealed record ReportingContext( public sealed record ReportingContext(
string SourceTag, string SourceTag,
string ResultsDirectory, string ResultsDirectory,
string BuildConfiguration, string BuildConfiguration,
UTF8Encoding Utf8NoBom) UTF8Encoding Utf8NoBom,
string CharsetName,
int WarmupIterations,
int BenchmarkSamples,
int TargetSampleMs,
double UnstableCVThreshold)
{ {
/// <summary> /// <summary>
/// Resolves the canonical <see cref="ResultsDirectory"/> by walking up from /// Walks up from the assembly's BaseDirectory to find the repo root (marker: <c>AyCode.Core.sln</c>).
/// <see cref="AppContext.BaseDirectory"/> to the nearest <c>AyCode.Core.sln</c>, then combining
/// with <c>Test_Benchmark_Results\Benchmark</c>. The build configuration is supplied by the caller
/// (Console resolves via <c>#if AYCODE_NATIVEAOT|DEBUG|else Release</c>; BDN-side currently mirrors
/// the JIT vs AOT discriminator from <see cref="System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled"/>).
/// </summary>
public static ReportingContext Create(string sourceTag, string buildConfiguration) =>
new(sourceTag, ResolveResultsDirectory(), buildConfiguration, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
/// <summary>
/// Walk-up from the assembly's BaseDirectory to find the repo root (marker: <c>AyCode.Core.sln</c>).
/// Returns <c>{repoRoot}\Test_Benchmark_Results\Benchmark</c>. Worktree-aware: if running from a /// Returns <c>{repoRoot}\Test_Benchmark_Results\Benchmark</c>. Worktree-aware: if running from a
/// worktree, the walk finds the worktree's own .sln (each worktree has its own checkout), so /// worktree, the walk finds the worktree's own .sln (each worktree has its own checkout), so
/// results land in the worktree's results folder — the natural place when the worktree's code /// results land in the worktree's results folder — the natural place when the worktree's code
/// changes are what produced the numbers. /// changes are what produced the numbers.
/// </summary> /// </summary>
private static string ResolveResultsDirectory() public static string ResolveResultsDirectory()
{ {
var dir = new DirectoryInfo(AppContext.BaseDirectory); var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AyCode.Core.sln"))) while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AyCode.Core.sln")))

26
AyCode.Core.Serializers.Console/BenchmarkLoop.cs
View File

@ -127,11 +127,23 @@ internal static class BenchmarkLoop
allResults.AddRange(results); allResults.AddRange(results);
} }
// Print grouped results // Build the reporting context (resolves path via walk-up to .sln, snapshots run-config).
Output.PrintGroupedResults(allResults, testDataSets); var ctx = new ReportingContext(
SourceTag: "Console",
ResultsDirectory: ReportingContext.ResolveResultsDirectory(),
BuildConfiguration: Configuration.BuildConfiguration,
Utf8NoBom: Configuration.Utf8NoBom,
CharsetName: Configuration.GetCurrentCharsetName(),
WarmupIterations: Configuration.WarmupIterations,
BenchmarkSamples: Configuration.BenchmarkSamples,
TargetSampleMs: Configuration.TargetSampleMs,
UnstableCVThreshold: Configuration.UnstableCVThreshold);
// Save results to file // Print grouped results
Output.SaveResults(allResults, testDataSets); BenchmarkReportWriter.PrintGroupedResults(allResults, testDataSets);
// Save results to file (.log + .LLM + .output)
BenchmarkReportWriter.SaveAll(ctx, allResults, testDataSets);
System.Console.WriteLine("\n✓ Benchmark complete!"); System.Console.WriteLine("\n✓ Benchmark complete!");
} }
@ -275,8 +287,8 @@ internal static class BenchmarkLoop
// batch-time addition would be misleading. Instead: compute per-op µs (iter-independent), // batch-time addition would be misleading. Instead: compute per-op µs (iter-independent),
// then synthesize RoundTripTimeMs against RoundTripIterations = max(serIter, desIter) so that // then synthesize RoundTripTimeMs against RoundTripIterations = max(serIter, desIter) so that
// RoundTripTimeMs / RoundTripIterations * 1000 == Output.SerPerOp + Output.DesPerOp. // RoundTripTimeMs / RoundTripIterations * 1000 == Output.SerPerOp + Output.DesPerOp.
var serPerOp = Output.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations); var serPerOp = BenchmarkReportWriter.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations);
var desPerOp = Output.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations); var desPerOp = BenchmarkReportWriter.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations);
var rtPerOp = serPerOp + desPerOp; var rtPerOp = serPerOp + desPerOp;
result.RoundTripIterations = Math.Max(result.SerializeIterations, result.DeserializeIterations); result.RoundTripIterations = Math.Max(result.SerializeIterations, result.DeserializeIterations);
result.RoundTripTimeMs = rtPerOp / 1000.0 * result.RoundTripIterations; result.RoundTripTimeMs = rtPerOp / 1000.0 * result.RoundTripIterations;
@ -284,7 +296,7 @@ internal static class BenchmarkLoop
} }
results.Add(result); results.Add(result);
Output.PrintResult(result); BenchmarkReportWriter.PrintResult(result);
} }
// Dispose any IDisposable serializers (NamedPipe / FileStream variants own OS resources that must be released // Dispose any IDisposable serializers (NamedPipe / FileStream variants own OS resources that must be released