Refactor Output to BenchmarkReportWriter
- Removed Output.cs and migrated all reporting, formatting, and statistics logic to a new BenchmarkReportWriter static class. - Extended ReportingContext with run-header fields for richer output metadata. - Updated BenchmarkLoop to use BenchmarkReportWriter and the new ReportingContext. - Centralized all output file generation (.log, .LLM, .output) and formatting helpers in BenchmarkReportWriter. - Improved separation of concerns and unified output artifact naming and metadata.
This commit is contained in:
parent
9dcb62ce23
commit
ed03d754ec
|
|
@ -1,4 +1,3 @@
|
|||
using AyCode.Core.Benchmarks.Reporting;
|
||||
using AyCode.Core.Benchmarks.Workloads.Scenarios;
|
||||
using AyCode.Core.Serializers.Binaries;
|
||||
using AyCode.Core.Tests.TestModels;
|
||||
|
|
@ -6,15 +5,20 @@ using System.Globalization;
|
|||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
namespace AyCode.Core.Serializers.Console;
|
||||
namespace AyCode.Core.Benchmarks.Reporting;
|
||||
|
||||
/// <summary>
|
||||
/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file.
|
||||
/// Consumes <see cref="BenchmarkResult"/> rows produced by the benchmark loop and emits human-readable
|
||||
/// + LLM-friendly outputs. Display-only helpers (per-op µs conversion, allocation KB formatting,
|
||||
/// range/CV formatting, paired-aggregation stats) live here too — they are display-side concerns.
|
||||
/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file, .output
|
||||
/// binary hex dump. Consumes <see cref="BenchmarkResult"/> rows produced by the benchmark execution loop
|
||||
/// (Console-side <c>BenchmarkLoop</c> or BDN-side <c>BdnSummaryAdapter</c>) and emits human-readable +
|
||||
/// LLM-friendly outputs.
|
||||
///
|
||||
/// <para>The <see cref="ReportingContext"/> parameter encapsulates per-run state — <see cref="ReportingContext.SourceTag"/>
|
||||
/// drives the filename prefix ("Console" / "Bdn"), <see cref="ReportingContext.ResultsDirectory"/> is the
|
||||
/// resolved (walk-up-to-.sln) output folder, and the remaining fields (charset, iter counts, target sample
|
||||
/// window, CV threshold) carry the run-header info embedded in every emitted artifact.</para>
|
||||
/// </summary>
|
||||
internal static class Output
|
||||
public static class BenchmarkReportWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation
|
||||
|
|
@ -27,18 +31,18 @@ internal static class Output
|
|||
/// <param name="AcAvg">Arithmetic mean AcBinary value (µs/op or bytes).</param>
|
||||
/// <param name="MpAvg">Arithmetic mean MemPack value.</param>
|
||||
/// <param name="CellCount">Number of paired cells contributing to the geo/median.</param>
|
||||
internal record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount);
|
||||
public record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0;
|
||||
public static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0;
|
||||
|
||||
// Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever
|
||||
// averaging or comparison happens across rows with potentially different iter counts (Winners summary,
|
||||
// Overall comparison, per-cell summary row). Keeping these as methods rather than properties on
|
||||
// BenchmarkResult preserves the result-as-data-bag distinction.
|
||||
internal static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations);
|
||||
internal static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations);
|
||||
internal static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations);
|
||||
public static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations);
|
||||
public static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations);
|
||||
public static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations);
|
||||
|
||||
/// <summary>
|
||||
/// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can
|
||||
|
|
@ -47,7 +51,7 @@ internal static class Output
|
|||
/// integers untouched.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal static double ToKilobytes(long bytes) => bytes / 1024.0;
|
||||
public static double ToKilobytes(long bytes) => bytes / 1024.0;
|
||||
|
||||
/// <summary>
|
||||
/// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison
|
||||
|
|
@ -55,7 +59,7 @@ internal static class Output
|
|||
/// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats.
|
||||
/// Returns null when no paired cell has a valid value.
|
||||
/// </summary>
|
||||
internal static OverallStats? ComputeOverallStats(List<BenchmarkResult> acResults, List<BenchmarkResult> mpResults, Func<BenchmarkResult, double> getValue)
|
||||
public static OverallStats? ComputeOverallStats(List<BenchmarkResult> acResults, List<BenchmarkResult> mpResults, Func<BenchmarkResult, double> getValue)
|
||||
{
|
||||
if (acResults.Count == 0 || mpResults.Count == 0) return null;
|
||||
|
||||
|
|
@ -93,12 +97,12 @@ internal static class Output
|
|||
/// <summary>
|
||||
/// Formats a per-op micros value with its inter-sample range and CV-threshold marker as
|
||||
/// <c>"26.86 (24.5..29.1)"</c> or <c>"26.86 (24.5..29.1) ⚠️5.2%"</c>. Median first, range in parentheses,
|
||||
/// CV warning suffix only when CV > <see cref="Configuration.UnstableCVThreshold"/>. When min == max == median
|
||||
/// CV warning suffix only when CV > <paramref name="unstableCvThreshold"/>. When min == max == median
|
||||
/// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter.
|
||||
/// All time inputs are total-batch milliseconds; <paramref name="iterations"/> is the per-row iter
|
||||
/// count (post-adaptive-calibration).
|
||||
/// </summary>
|
||||
internal static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv)
|
||||
public static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv, double unstableCvThreshold)
|
||||
{
|
||||
var med = ToPerOpMicros(medianMs, iterations);
|
||||
// No range data (single-sample fast path) — surface as bare median, identical to the prior format.
|
||||
|
|
@ -114,7 +118,7 @@ internal static class Output
|
|||
if (medianMs > 0 && stdDevMs > 0)
|
||||
{
|
||||
var cv = stdDevMs / medianMs;
|
||||
if (cv > Configuration.UnstableCVThreshold)
|
||||
if (cv > unstableCvThreshold)
|
||||
{
|
||||
var cvPct = (cv * 100).ToString("F1", inv);
|
||||
return $"{range} ⚠️{cvPct}%";
|
||||
|
|
@ -128,14 +132,14 @@ internal static class Output
|
|||
/// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`).
|
||||
/// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block.
|
||||
/// </summary>
|
||||
internal static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%";
|
||||
public static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%";
|
||||
|
||||
/// <summary>
|
||||
/// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means.
|
||||
/// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when
|
||||
/// stats is null (no paired data).
|
||||
/// </summary>
|
||||
internal static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2")
|
||||
public static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2")
|
||||
{
|
||||
if (stats == null) return;
|
||||
|
||||
|
|
@ -150,13 +154,13 @@ internal static class Output
|
|||
/// Same as <see cref="WriteOverallLine"/> but appends to a <see cref="StringBuilder"/> (no color).
|
||||
/// Used by the .log and .LLM file writers.
|
||||
/// </summary>
|
||||
internal static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2")
|
||||
public static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2")
|
||||
{
|
||||
if (stats == null) return;
|
||||
sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
|
||||
}
|
||||
|
||||
internal static void PrintResult(BenchmarkResult result)
|
||||
public static void PrintResult(BenchmarkResult result)
|
||||
{
|
||||
// Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op).
|
||||
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A";
|
||||
|
|
@ -166,7 +170,7 @@ internal static class Output
|
|||
System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)");
|
||||
}
|
||||
|
||||
internal static void PrintGroupedResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
public static void PrintGroupedResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
{
|
||||
System.Console.WriteLine("\n");
|
||||
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||||
|
|
@ -407,14 +411,21 @@ internal static class Output
|
|||
WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0");
|
||||
}
|
||||
|
||||
internal static void SaveResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
/// <summary>
|
||||
/// Writes the unified file triplet — <c>{SourceTag}.FullBenchmark_{Build}_{timestamp}.{log, LLM, output}</c>
|
||||
/// — to <see cref="ReportingContext.ResultsDirectory"/>. The <c>.log</c> is the human-readable formatted
|
||||
/// view, the <c>.LLM</c> is the markdown LLM-paste-friendly view, and the <c>.output</c> is a binary hex
|
||||
/// dump of the Large test data's AcBinary-Default serialization (for raw inspection / wire-debugging).
|
||||
/// </summary>
|
||||
public static void SaveAll(ReportingContext ctx, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
{
|
||||
Directory.CreateDirectory(Configuration.ResultsDirectory);
|
||||
Directory.CreateDirectory(ctx.ResultsDirectory);
|
||||
|
||||
var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
|
||||
var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}";
|
||||
var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log");
|
||||
var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output");
|
||||
var baseFileName = $"{ctx.SourceTag}.FullBenchmark_{ctx.BuildConfiguration}_{timestamp}";
|
||||
var logFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.log");
|
||||
var outputFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.output");
|
||||
var llmFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.LLM");
|
||||
|
||||
// Save binary output to separate .output file.
|
||||
// Cast to TestDataSet<TestOrder_All_False> because Phase 1 hardcodes the benchmark variant.
|
||||
|
|
@ -436,7 +447,7 @@ internal static class Output
|
|||
outputSb.AppendLine("Hex dump:");
|
||||
outputSb.AppendLine(FormatHexDump(serializedBytes));
|
||||
|
||||
File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom);
|
||||
File.WriteAllText(outputFilePath, outputSb.ToString(), ctx.Utf8NoBom);
|
||||
System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}");
|
||||
}
|
||||
|
||||
|
|
@ -445,10 +456,11 @@ internal static class Output
|
|||
sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||||
sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║");
|
||||
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Charset: {Configuration.GetCurrentCharsetName()}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Source: {ctx.SourceTag}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Build: {ctx.BuildConfiguration}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Charset: {ctx.CharsetName}".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Iterations: per-cell adaptive (~{ctx.TargetSampleMs} ms target)".PadRight(100) + "║");
|
||||
sb.AppendLine($"║ Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║");
|
||||
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||||
sb.AppendLine();
|
||||
|
||||
|
|
@ -471,6 +483,10 @@ internal static class Output
|
|||
}
|
||||
|
||||
// CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely.
|
||||
// InvariantCulture is mandatory here: the decimal-separator dimension MUST be `.` so the comma-separated
|
||||
// field delimiters don't collide with locale-specific decimal commas (e.g. Hungarian "7,38" would split
|
||||
// a single F2 value across two CSV fields).
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
sb.AppendLine("=== RAW DATA (CSV) ===");
|
||||
sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes");
|
||||
|
||||
|
|
@ -479,7 +495,7 @@ internal static class Output
|
|||
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList();
|
||||
foreach (var result in testResults)
|
||||
{
|
||||
sb.AppendLine($"{result.TestDataName},{result.Engine.ToDisplay()},{result.IoMode.ToDisplay()},{result.DispatchMode.ToDisplay()},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}");
|
||||
sb.AppendLine($"{result.TestDataName},{result.Engine.ToDisplay()},{result.IoMode.ToDisplay()},{result.DispatchMode.ToDisplay()},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result).ToString("F2", inv)},{DesPerOp(result).ToString("F2", inv)},{RtPerOp(result).ToString("F2", inv)},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}");
|
||||
}
|
||||
}
|
||||
sb.AppendLine();
|
||||
|
|
@ -546,11 +562,10 @@ internal static class Output
|
|||
if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0)
|
||||
{
|
||||
sb.AppendLine(" (Comparison requires both serialize and deserialize data)");
|
||||
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom);
|
||||
File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
|
||||
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
||||
|
||||
var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM");
|
||||
SaveLlmResults(llmFilePathEarly, results, testDataSets);
|
||||
SaveLlmResults(ctx, llmFilePath, results, testDataSets);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -564,23 +579,22 @@ internal static class Output
|
|||
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp));
|
||||
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0");
|
||||
|
||||
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom);
|
||||
File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
|
||||
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
||||
|
||||
// Save LLM-optimized results
|
||||
var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM");
|
||||
SaveLlmResults(llmFilePath, results, testDataSets);
|
||||
SaveLlmResults(ctx, llmFilePath, results, testDataSets);
|
||||
}
|
||||
|
||||
internal static void SaveLlmResults(string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
private static void SaveLlmResults(ReportingContext ctx, string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
|
||||
sb.AppendLine($"Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%");
|
||||
sb.AppendLine($"# AcBinary Benchmark [{ctx.SourceTag}] {ctx.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
|
||||
sb.AppendLine($"Charset: {ctx.CharsetName} | Iterations: per-cell adaptive (target ~{ctx.TargetSampleMs} ms/sample) | Warmup: {ctx.WarmupIterations} per phase (Ser/Des isolated) | Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | UnstableCV threshold: {ctx.UnstableCVThreshold * 100:F0}%");
|
||||
sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup");
|
||||
|
||||
// Options summary. Bracketed [OrderType] surfaces the TestOrder variant each preset serialised —
|
||||
// see SaveResults for the variant-dispatch rationale.
|
||||
// see SaveAll for the variant-dispatch rationale.
|
||||
var optionsMap = results
|
||||
.Where(r => r.OptionsDescription != null)
|
||||
.Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!))
|
||||
|
|
@ -601,6 +615,8 @@ internal static class Output
|
|||
sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des");
|
||||
sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---");
|
||||
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
|
||||
foreach (var testData in testDataSets)
|
||||
{
|
||||
var testResults = results
|
||||
|
|
@ -610,12 +626,11 @@ internal static class Output
|
|||
|
||||
foreach (var r in testResults)
|
||||
{
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-";
|
||||
var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-";
|
||||
var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv, ctx.UnstableCVThreshold) : "-";
|
||||
var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv, ctx.UnstableCVThreshold) : "-";
|
||||
var rt = r.RoundTripTimeMs > 0
|
||||
? (r.IsRoundTripOnly
|
||||
? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv)
|
||||
? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv, ctx.UnstableCVThreshold)
|
||||
: RtPerOp(r).ToString("F2", inv))
|
||||
: "-";
|
||||
|
||||
|
|
@ -658,14 +673,14 @@ internal static class Output
|
|||
sb.AppendLine("```");
|
||||
}
|
||||
|
||||
File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom);
|
||||
File.WriteAllText(filePath, sb.ToString(), ctx.Utf8NoBom);
|
||||
System.Console.WriteLine($"✓ LLM results saved to: {filePath}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats byte array as hex dump with offset, hex values, and ASCII representation.
|
||||
/// </summary>
|
||||
internal static string FormatHexDump(byte[] bytes, int bytesPerLine = 16)
|
||||
public static string FormatHexDump(byte[] bytes, int bytesPerLine = 16)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
for (var i = 0; i < bytes.Length; i += bytesPerLine)
|
||||
|
|
@ -4,37 +4,34 @@ namespace AyCode.Core.Benchmarks.Reporting;
|
|||
|
||||
/// <summary>
|
||||
/// Context bundle for the unified benchmark report writer. Same record on both sides (Console / BDN),
|
||||
/// only the <see cref="SourceTag"/> differs ("Console" / "Bdn") — that drives the filename prefix
|
||||
/// the <see cref="SourceTag"/> differs ("Console" / "Bdn") and drives the filename prefix
|
||||
/// (e.g. <c>Console.FullBenchmark_Release_{timestamp}.LLM</c> vs <c>Bdn.FullBenchmark_Release_{timestamp}.LLM</c>).
|
||||
/// The <see cref="ResultsDirectory"/> resolution walks up from <see cref="AppContext.BaseDirectory"/> to the
|
||||
/// nearest <c>AyCode.Core.sln</c> and combines with <c>Test_Benchmark_Results\Benchmark</c> — works across
|
||||
/// build modes (Debug / Release / AOT publish) and worktrees (each worktree has its own .sln, so its bench
|
||||
/// results land alongside its code).
|
||||
/// results land alongside its code). The remaining fields capture run-header information (charset, iter
|
||||
/// counts, target sample window, CV threshold) so the writer can render a self-documenting header in both
|
||||
/// the <c>.log</c> and <c>.LLM</c> outputs.
|
||||
/// </summary>
|
||||
public sealed record ReportingContext(
|
||||
string SourceTag,
|
||||
string ResultsDirectory,
|
||||
string BuildConfiguration,
|
||||
UTF8Encoding Utf8NoBom)
|
||||
UTF8Encoding Utf8NoBom,
|
||||
string CharsetName,
|
||||
int WarmupIterations,
|
||||
int BenchmarkSamples,
|
||||
int TargetSampleMs,
|
||||
double UnstableCVThreshold)
|
||||
{
|
||||
/// <summary>
|
||||
/// Resolves the canonical <see cref="ResultsDirectory"/> by walking up from
|
||||
/// <see cref="AppContext.BaseDirectory"/> to the nearest <c>AyCode.Core.sln</c>, then combining
|
||||
/// with <c>Test_Benchmark_Results\Benchmark</c>. The build configuration is supplied by the caller
|
||||
/// (Console resolves via <c>#if AYCODE_NATIVEAOT|DEBUG|else Release</c>; BDN-side currently mirrors
|
||||
/// the JIT vs AOT discriminator from <see cref="System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled"/>).
|
||||
/// </summary>
|
||||
public static ReportingContext Create(string sourceTag, string buildConfiguration) =>
|
||||
new(sourceTag, ResolveResultsDirectory(), buildConfiguration, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
|
||||
/// <summary>
|
||||
/// Walk-up from the assembly's BaseDirectory to find the repo root (marker: <c>AyCode.Core.sln</c>).
|
||||
/// Walks up from the assembly's BaseDirectory to find the repo root (marker: <c>AyCode.Core.sln</c>).
|
||||
/// Returns <c>{repoRoot}\Test_Benchmark_Results\Benchmark</c>. Worktree-aware: if running from a
|
||||
/// worktree, the walk finds the worktree's own .sln (each worktree has its own checkout), so
|
||||
/// results land in the worktree's results folder — the natural place when the worktree's code
|
||||
/// changes are what produced the numbers.
|
||||
/// </summary>
|
||||
private static string ResolveResultsDirectory()
|
||||
public static string ResolveResultsDirectory()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AyCode.Core.sln")))
|
||||
|
|
|
|||
|
|
@ -127,11 +127,23 @@ internal static class BenchmarkLoop
|
|||
allResults.AddRange(results);
|
||||
}
|
||||
|
||||
// Print grouped results
|
||||
Output.PrintGroupedResults(allResults, testDataSets);
|
||||
// Build the reporting context (resolves path via walk-up to .sln, snapshots run-config).
|
||||
var ctx = new ReportingContext(
|
||||
SourceTag: "Console",
|
||||
ResultsDirectory: ReportingContext.ResolveResultsDirectory(),
|
||||
BuildConfiguration: Configuration.BuildConfiguration,
|
||||
Utf8NoBom: Configuration.Utf8NoBom,
|
||||
CharsetName: Configuration.GetCurrentCharsetName(),
|
||||
WarmupIterations: Configuration.WarmupIterations,
|
||||
BenchmarkSamples: Configuration.BenchmarkSamples,
|
||||
TargetSampleMs: Configuration.TargetSampleMs,
|
||||
UnstableCVThreshold: Configuration.UnstableCVThreshold);
|
||||
|
||||
// Save results to file
|
||||
Output.SaveResults(allResults, testDataSets);
|
||||
// Print grouped results
|
||||
BenchmarkReportWriter.PrintGroupedResults(allResults, testDataSets);
|
||||
|
||||
// Save results to file (.log + .LLM + .output)
|
||||
BenchmarkReportWriter.SaveAll(ctx, allResults, testDataSets);
|
||||
|
||||
System.Console.WriteLine("\n✓ Benchmark complete!");
|
||||
}
|
||||
|
|
@ -275,8 +287,8 @@ internal static class BenchmarkLoop
|
|||
// batch-time addition would be misleading. Instead: compute per-op µs (iter-independent),
|
||||
// then synthesize RoundTripTimeMs against RoundTripIterations = max(serIter, desIter) so that
|
||||
// RoundTripTimeMs / RoundTripIterations * 1000 == Output.SerPerOp + Output.DesPerOp.
|
||||
var serPerOp = Output.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations);
|
||||
var desPerOp = Output.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations);
|
||||
var serPerOp = BenchmarkReportWriter.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations);
|
||||
var desPerOp = BenchmarkReportWriter.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations);
|
||||
var rtPerOp = serPerOp + desPerOp;
|
||||
result.RoundTripIterations = Math.Max(result.SerializeIterations, result.DeserializeIterations);
|
||||
result.RoundTripTimeMs = rtPerOp / 1000.0 * result.RoundTripIterations;
|
||||
|
|
@ -284,7 +296,7 @@ internal static class BenchmarkLoop
|
|||
}
|
||||
|
||||
results.Add(result);
|
||||
Output.PrintResult(result);
|
||||
BenchmarkReportWriter.PrintResult(result);
|
||||
}
|
||||
|
||||
// Dispose any IDisposable serializers (NamedPipe / FileStream variants own OS resources that must be released
|
||||
|
|
|
|||
Loading…
Reference in New Issue