732 lines
50 KiB
C#
732 lines
50 KiB
C#
using AyCode.Core.Benchmarks.Workloads.Scenarios;
|
||
using AyCode.Core.Serializers.Binaries;
|
||
using AyCode.Core.Tests.TestModels;
|
||
using System.Globalization;
|
||
using System.Runtime.CompilerServices;
|
||
using System.Text;
|
||
|
||
namespace AyCode.Core.Benchmarks.Reporting;
|
||
|
||
/// <summary>
|
||
/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file, .output
|
||
/// binary hex dump. Consumes <see cref="BenchmarkResult"/> rows produced by the benchmark execution loop
|
||
/// (Console-side <c>BenchmarkLoop</c> or BDN-side <c>BdnSummaryAdapter</c>) and emits human-readable +
|
||
/// LLM-friendly outputs.
|
||
///
|
||
/// <para>The <see cref="ReportingContext"/> parameter encapsulates per-run state — <see cref="ReportingContext.SourceTag"/>
|
||
/// drives the filename prefix ("Console" / "Bdn"), <see cref="ReportingContext.ResultsDirectory"/> is the
|
||
/// resolved (walk-up-to-.sln) output folder, and the remaining fields (charset, iter counts, target sample
|
||
/// window, CV threshold) carry the run-header info embedded in every emitted artifact.</para>
|
||
/// </summary>
|
||
public static class BenchmarkReportWriter
|
||
{
|
||
/// <summary>
|
||
/// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation
|
||
/// strategies so the reader can judge whether the headline delta is dominated by one large cell
|
||
/// (arithmetic mean) or representative of typical workload (geometric mean / median).
|
||
/// </summary>
|
||
/// <param name="ArithMeanPct">Arithmetic mean of µs/op — magnitude-weighted; biased toward Large cell.</param>
|
||
/// <param name="GeoMeanPct">Geometric mean of per-cell ratios — magnitude-neutral; each cell weighted equally.</param>
|
||
/// <param name="MedianPct">Median of per-cell ratios — outlier-resistant.</param>
|
||
/// <param name="AcAvg">Arithmetic mean AcBinary value (µs/op or bytes).</param>
|
||
/// <param name="MpAvg">Arithmetic mean MemPack value.</param>
|
||
/// <param name="CellCount">Number of paired cells contributing to the geo/median.</param>
|
||
public record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount);
|
||
|
||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||
public static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0;
|
||
|
||
// Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever
|
||
// averaging or comparison happens across rows with potentially different iter counts (Winners summary,
|
||
// Overall comparison, per-cell summary row). Keeping these as methods rather than properties on
|
||
// BenchmarkResult preserves the result-as-data-bag distinction.
|
||
public static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations);
|
||
public static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations);
|
||
public static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations);
|
||
|
||
/// <summary>
|
||
/// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can
|
||
/// render compact F2 KB values (e.g. <c>4.05 KB</c> instead of <c>4,144 B</c>) — header carries
|
||
/// the unit so per-row entries stay numbers-only. CSV / raw-data outputs keep the precise byte
|
||
/// integers untouched.
|
||
/// </summary>
|
||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||
public static double ToKilobytes(long bytes) => bytes / 1024.0;
|
||
|
||
/// <summary>
|
||
/// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison
|
||
/// across paired cells (joined by <c>TestDataName</c>). Per-cell pairing is required for the
|
||
/// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats.
|
||
/// Returns null when no paired cell has a valid value.
|
||
/// </summary>
|
||
public static OverallStats? ComputeOverallStats(List<BenchmarkResult> acResults, List<BenchmarkResult> mpResults, Func<BenchmarkResult, double> getValue)
|
||
{
|
||
if (acResults.Count == 0 || mpResults.Count == 0) return null;
|
||
|
||
var pairs = (from ac in acResults
|
||
join mp in mpResults on ac.TestDataName equals mp.TestDataName
|
||
let acV = getValue(ac)
|
||
let mpV = getValue(mp)
|
||
where acV > 0 && mpV > 0
|
||
select (ac: acV, mp: mpV)).ToList();
|
||
|
||
if (pairs.Count == 0) return null;
|
||
|
||
var acAvg = pairs.Average(p => p.ac);
|
||
var mpAvg = pairs.Average(p => p.mp);
|
||
var ratios = pairs.Select(p => p.ac / p.mp).ToList();
|
||
|
||
// Geometric mean: exp(avg(ln(ratios))) — numerically stable vs Π ratios then ^(1/N).
|
||
var geoMean = Math.Exp(ratios.Sum(Math.Log) / ratios.Count);
|
||
|
||
// Median (paired-ratio): for even N use the midpoint of the two middle values.
|
||
var sorted = ratios.OrderBy(r => r).ToList();
|
||
var median = sorted.Count % 2 == 1
|
||
? sorted[sorted.Count / 2]
|
||
: (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2.0;
|
||
|
||
return new OverallStats(
|
||
ArithMeanPct: (acAvg / mpAvg - 1) * 100,
|
||
GeoMeanPct: (geoMean - 1) * 100,
|
||
MedianPct: (median - 1) * 100,
|
||
AcAvg: acAvg,
|
||
MpAvg: mpAvg,
|
||
CellCount: ratios.Count);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Formats a per-op micros value with its inter-sample range and CV-threshold marker as
|
||
/// <c>"26.86 (24.5..29.1)"</c> or <c>"26.86 (24.5..29.1) ⚠️5.2%"</c>. Median first, range in parentheses,
|
||
/// CV warning suffix only when CV > <paramref name="unstableCvThreshold"/>. When min == max == median
|
||
/// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter.
|
||
/// All time inputs are total-batch milliseconds; <paramref name="iterations"/> is the per-row iter
|
||
/// count (post-adaptive-calibration).
|
||
/// </summary>
|
||
public static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv, double unstableCvThreshold)
|
||
{
|
||
var med = ToPerOpMicros(medianMs, iterations);
|
||
// No range data (single-sample fast path) — surface as bare median, identical to the prior format.
|
||
if (minMs <= 0 && maxMs <= 0) return med.ToString("F2", inv);
|
||
if (minMs >= medianMs && maxMs <= medianMs) return med.ToString("F2", inv);
|
||
|
||
var min = ToPerOpMicros(minMs, iterations);
|
||
var max = ToPerOpMicros(maxMs, iterations);
|
||
var range = $"{med.ToString("F2", inv)} ({min.ToString("F2", inv)}..{max.ToString("F2", inv)})";
|
||
|
||
// CV (coefficient of variation = stddev / mean) — flag rows above the unstable threshold so a
|
||
// small inter-engine delta on a high-CV row is easy to discount as noise.
|
||
if (medianMs > 0 && stdDevMs > 0)
|
||
{
|
||
var cv = stdDevMs / medianMs;
|
||
if (cv > unstableCvThreshold)
|
||
{
|
||
var cvPct = (cv * 100).ToString("F1", inv);
|
||
return $"{range} ⚠️{cvPct}%";
|
||
}
|
||
}
|
||
|
||
return range;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`).
|
||
/// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block.
|
||
/// </summary>
|
||
public static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%";
|
||
|
||
/// <summary>
|
||
/// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means.
|
||
/// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when
|
||
/// stats is null (no paired data).
|
||
/// </summary>
|
||
public static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2")
|
||
{
|
||
if (stats == null) return;
|
||
|
||
// Color follows geo-mean (the magnitude-neutral signal). The arith-mean column may show a
|
||
// different sign when a single big cell dominates — that's exactly the signal we want to surface.
|
||
System.Console.ForegroundColor = stats.GeoMeanPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.WriteLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} │ geo {FormatPctSigned(stats.GeoMeanPct)} │ median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
|
||
System.Console.ResetColor();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Same as <see cref="WriteOverallLine"/> but appends to a <see cref="StringBuilder"/> (no color).
|
||
/// Used by the .log and .LLM file writers.
|
||
/// </summary>
|
||
public static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2")
|
||
{
|
||
if (stats == null) return;
|
||
sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
|
||
}
|
||
|
||
public static void PrintResult(BenchmarkResult result)
|
||
{
|
||
// Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op).
|
||
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A";
|
||
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result),7:F2}" : " N/A";
|
||
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp),7:F2}" : " N/A";
|
||
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp),7:F2}" : " N/A";
|
||
System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)");
|
||
}
|
||
|
||
public static void PrintGroupedResults(List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||
{
|
||
System.Console.WriteLine("\n");
|
||
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||
System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║");
|
||
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||
|
||
// Print serializer options. [OrderType] suffix shows which TestOrder variant each preset serialised.
|
||
var optionsMap = results
|
||
.Where(r => r.OptionsDescription != null)
|
||
.Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!))
|
||
.Distinct()
|
||
.ToList();
|
||
|
||
if (optionsMap.Count > 0)
|
||
{
|
||
System.Console.WriteLine();
|
||
System.Console.WriteLine(" Serializer Options:");
|
||
foreach (var (name, orderType, opts) in optionsMap)
|
||
System.Console.WriteLine($" {name} [{orderType}]: {opts}");
|
||
}
|
||
|
||
foreach (var testData in testDataSets)
|
||
{
|
||
// Order by Engine (so the same engine column-position stays stable across cells, especially
|
||
// when two engines are within noise floor on a given cell — flip-flopping speed-rank produces
|
||
// diff-hostile output across runs). RtPerOp is the secondary tiebreaker for cells where
|
||
// multiple variants of the same engine exist (e.g. AcBinary SGen vs Runtime).
|
||
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => r.Engine).ThenBy(r => RtPerOp(r)).ToList();
|
||
// Baseline switched MessagePack → MemoryPack: MemoryPack is the SOTA performance leader.
|
||
var memPackResult = testResults.FirstOrDefault(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray));
|
||
// Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated).
|
||
// The Runtime variant is shown alongside in the table for context, not used as the headline number.
|
||
var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen));
|
||
|
||
System.Console.WriteLine($"\n┌─ {testData.DisplayName} ─".PadRight(172, '─') + "┐");
|
||
// Header-only units; per-row entries are numbers (µs/op for time, KB/op for alloc, KB pair "ser / des" for Setup, B for Size).
|
||
System.Console.WriteLine($"│ {"#",-4} │ {"Engine",-11} │ {"Options",-22} │ {"IO",-12} │ {"Mode",-8} │ {"Setup S/D KB",-14} │ {"Size B",-8} │ {"Ser µs/op",-10} │ {"SerAlc KB",-10} │ {"Des µs/op",-10} │ {"DesAlc KB",-10} │ {"RT µs/op",-10} │ {"RTAlc KB",-10} │");
|
||
System.Console.WriteLine($"├{"─".PadRight(6, '─')}┼{"─".PadRight(13, '─')}┼{"─".PadRight(24, '─')}┼{"─".PadRight(14, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤");
|
||
|
||
var rank = 1;
|
||
foreach (var result in testResults)
|
||
{
|
||
var size = $"{result.SerializedSize:N0}";
|
||
var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}";
|
||
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A";
|
||
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A";
|
||
var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A";
|
||
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A";
|
||
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A";
|
||
var rtAlloc = result.RoundTripAllocBytesPerOp > 0 ? $"{ToKilobytes(result.RoundTripAllocBytesPerOp):F2}" : "N/A";
|
||
|
||
// Highlight MemoryPack baseline (any Byte[]) and AcBinary headline contender (Byte[] + SGen) with win/lose colors.
|
||
// The AcBinary Byte[]+Runtime variant is shown unhighlighted — it's contextual (SGen speed-up reference), not the headline.
|
||
var isHighlighted = (result.Engine == BenchmarkEngine.MemoryPack && result.IoMode == BenchmarkIoMode.ByteArray)
|
||
|| (result.Engine == BenchmarkEngine.AcBinary && result.IoMode == BenchmarkIoMode.ByteArray && result.DispatchMode == BenchmarkDispatchMode.SGen);
|
||
|
||
var prefix = isHighlighted ? "│►" : "│ ";
|
||
var suffix = isHighlighted ? "◄│" : " │";
|
||
|
||
// Color logic: Green = winner (faster), Red = loser (slower)
|
||
if (isHighlighted && memPackResult != null && acBinaryResult != null)
|
||
{
|
||
var isMemPack = (result.Engine == BenchmarkEngine.MemoryPack && result.IoMode == BenchmarkIoMode.ByteArray);
|
||
var memPackFaster = RtPerOp(memPackResult) < RtPerOp(acBinaryResult);
|
||
|
||
if (isMemPack)
|
||
{
|
||
System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Green : ConsoleColor.Red;
|
||
}
|
||
else
|
||
{
|
||
System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Red : ConsoleColor.Green;
|
||
}
|
||
}
|
||
|
||
System.Console.WriteLine($"{prefix}{rank++,4} │ {result.Engine.ToDisplay(),-11} │ {result.OptionsPreset,-22} │ {result.IoMode.ToDisplay(),-12} │ {result.DispatchMode.ToDisplay(),-8} │ {setup,14} │ {size,8} │ {ser,10} │ {serAlloc,10} │ {des,10} │ {desAlloc,10} │ {rt,10} │ {rtAlloc,10}{suffix}");
|
||
|
||
if (isHighlighted)
|
||
{
|
||
System.Console.ResetColor();
|
||
}
|
||
}
|
||
|
||
// Footer row: AcBinary (Byte[]) vs MemoryPack (Byte[]) comparison per column
|
||
if (memPackResult != null && acBinaryResult != null)
|
||
{
|
||
var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100;
|
||
// Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows.
|
||
var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0;
|
||
var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0;
|
||
var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0;
|
||
|
||
var serAllocPct = memPackResult.SerializeAllocBytesPerOp > 0 ? (acBinaryResult.SerializeAllocBytesPerOp / (double)memPackResult.SerializeAllocBytesPerOp - 1) * 100 : 0;
|
||
var desAllocPct = memPackResult.DeserializeAllocBytesPerOp > 0 ? (acBinaryResult.DeserializeAllocBytesPerOp / (double)memPackResult.DeserializeAllocBytesPerOp - 1) * 100 : 0;
|
||
var rtAllocPct = memPackResult.RoundTripAllocBytesPerOp > 0 ? (acBinaryResult.RoundTripAllocBytesPerOp / (double)memPackResult.RoundTripAllocBytesPerOp - 1) * 100 : 0;
|
||
|
||
// Footer separator: merge first 5 cols (#, Engine, Options, IO, Mode) → comparison label;
|
||
// remaining 8 cols stay aligned (Setup S/D KB, Size, Ser µs/op, SerAlc KB, Des µs/op, DesAlc KB, RT µs/op, RTAlc KB).
|
||
System.Console.WriteLine($"├{"─".PadRight(6, '─')}┴{"─".PadRight(13, '─')}┴{"─".PadRight(24, '─')}┴{"─".PadRight(14, '─')}┴{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤");
|
||
// Merged label cell width = 4 + 11 + 22 + 12 + 8 + 4*3 (dropped separators) = 69
|
||
System.Console.Write($"│ {"► AcBinary (Byte[]) vs MemoryPack (Byte[])",-69} │ ");
|
||
|
||
// Setup S/D KB (n/a for Byte[] vs Byte[] — neither pre-allocates)
|
||
System.Console.Write($"{"—",14}");
|
||
System.Console.Write(" │ ");
|
||
|
||
// Size
|
||
System.Console.ForegroundColor = sizePct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{sizePct,+7:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Serialize
|
||
System.Console.ForegroundColor = serPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{serPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Serialize Alloc
|
||
System.Console.ForegroundColor = serAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{serAllocPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Deserialize
|
||
System.Console.ForegroundColor = desPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{desPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Deserialize Alloc
|
||
System.Console.ForegroundColor = desAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{desAllocPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Round-trip
|
||
System.Console.ForegroundColor = rtPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{rtPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.Write(" │ ");
|
||
|
||
// Round-trip Alloc
|
||
System.Console.ForegroundColor = rtAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
|
||
System.Console.Write($"{rtAllocPct,+9:+0;-0}%");
|
||
System.Console.ResetColor();
|
||
System.Console.WriteLine(" │");
|
||
}
|
||
|
||
// Closing line: merged on left (─ between cols 1-5), ┴ on the right (cols 6-13 boundary, 8 unmerged cells).
|
||
System.Console.WriteLine($"└{"─".PadRight(6, '─')}─{"─".PadRight(13, '─')}─{"─".PadRight(24, '─')}─{"─".PadRight(14, '─')}─{"─".PadRight(10, '─')}┴{"─".PadRight(16, '─')}┴{"─".PadRight(10, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┘");
|
||
}
|
||
|
||
// Summary: Best serializer for each category
|
||
System.Console.WriteLine("\n");
|
||
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||
System.Console.WriteLine("║ SUMMARY: WINNERS ║");
|
||
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||
|
||
System.Console.WriteLine($"\n{"Category",-20} │ {"Winner",-40} │ {"Avg Value",-18}");
|
||
System.Console.WriteLine($"{"─".PadRight(20, '─')}─┼─{"─".PadRight(40, '─')}─┼─{"─".PadRight(18, '─')}");
|
||
|
||
// Fastest Serialize — round-trip-only serializers (NamedPipe etc.) excluded:
|
||
// their Serialize() captures the full round-trip and isn't comparable to a pure Ser metric.
|
||
var fastestSer = results.Where(r => r.SerializeTimeMs > 0 && !r.IsRoundTripOnly)
|
||
.GroupBy(r => r.SerializerName)
|
||
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => SerPerOp(r)) })
|
||
.OrderBy(x => x.AvgPerOp)
|
||
.FirstOrDefault();
|
||
|
||
if (fastestSer != null)
|
||
System.Console.WriteLine($"{"Fastest Serialize",-20} │ {fastestSer.Name,-40} │ {fastestSer.AvgPerOp,12:F2} µs/op");
|
||
|
||
// Fastest Deserialize — round-trip-only serializers excluded (their Deserialize() is a no-op).
|
||
var fastestDes = results.Where(r => r.DeserializeTimeMs > 0 && !r.IsRoundTripOnly)
|
||
.GroupBy(r => r.SerializerName)
|
||
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => DesPerOp(r)) })
|
||
.OrderBy(x => x.AvgPerOp)
|
||
.FirstOrDefault();
|
||
|
||
if (fastestDes != null)
|
||
System.Console.WriteLine($"{"Fastest Deserialize",-20} │ {fastestDes.Name,-40} │ {fastestDes.AvgPerOp,12:F2} µs/op");
|
||
|
||
// Smallest Size
|
||
var smallestSize = results
|
||
.GroupBy(r => r.SerializerName)
|
||
.Select(g => new { Name = g.Key, AvgSize = g.Average(r => r.SerializedSize) })
|
||
.OrderBy(x => x.AvgSize)
|
||
.FirstOrDefault();
|
||
|
||
if (smallestSize != null)
|
||
System.Console.WriteLine($"{"Smallest Size",-20} │ {smallestSize.Name,-40} │ {smallestSize.AvgSize,15:F0} B");
|
||
|
||
// Fastest Round-trip — iter-independent per-op average.
|
||
var fastestRt = results.Where(r => r.RoundTripTimeMs > 0)
|
||
.GroupBy(r => r.SerializerName)
|
||
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => RtPerOp(r)) })
|
||
.OrderBy(x => x.AvgPerOp)
|
||
.FirstOrDefault();
|
||
|
||
if (fastestRt != null)
|
||
System.Console.WriteLine($"{"Fastest Round-trip",-20} │ {fastestRt.Name,-40} │ {fastestRt.AvgPerOp,12:F2} µs/op");
|
||
|
||
// Overall AcBinary (SGen) vs MemoryPack comparison.
|
||
var memPackSerResults = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.SerializeTimeMs > 0).ToList();
|
||
var memPackDesResults = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.DeserializeTimeMs > 0).ToList();
|
||
var memPackRtResults = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.RoundTripTimeMs > 0).ToList();
|
||
|
||
var acBinarySerResults = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.SerializeTimeMs > 0).ToList();
|
||
var acBinaryDesResults = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.DeserializeTimeMs > 0).ToList();
|
||
var acBinaryRtResults = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.RoundTripTimeMs > 0).ToList();
|
||
|
||
// Skip comparison if no data available
|
||
if (memPackRtResults.Count == 0 || acBinaryRtResults.Count == 0)
|
||
{
|
||
System.Console.WriteLine();
|
||
System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──");
|
||
System.Console.WriteLine(" (Comparison requires both serialize and deserialize data)");
|
||
return;
|
||
}
|
||
|
||
// All averages are over per-op µs (iter-independent). Three aggregations per metric.
|
||
var sizeAcResults = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen)).ToList();
|
||
var sizeMpResults = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray)).ToList();
|
||
|
||
var serStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, SerPerOp);
|
||
var desStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, DesPerOp);
|
||
var rtStats = ComputeOverallStats(acBinaryRtResults, memPackRtResults, RtPerOp);
|
||
var sizeStats = ComputeOverallStats(sizeAcResults, sizeMpResults, r => r.SerializedSize);
|
||
var serAllocStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, r => r.SerializeAllocBytesPerOp);
|
||
var desAllocStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, r => r.DeserializeAllocBytesPerOp);
|
||
|
||
System.Console.WriteLine();
|
||
System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──");
|
||
|
||
WriteOverallLine("Serialize", "µs/op", serStats);
|
||
WriteOverallLine("Deserialize", "µs/op", desStats);
|
||
WriteOverallLine("Round-trip", "µs/op", rtStats);
|
||
WriteOverallLine("Size", "B", sizeStats, "F0");
|
||
WriteOverallLine("Ser Alloc", "B/op", serAllocStats, "F0");
|
||
WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0");
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes the unified file triplet — <c>{SourceTag}.FullBenchmark_{Build}_{timestamp}.{log, LLM, output}</c>
|
||
/// — to <see cref="ReportingContext.ResultsDirectory"/>. The <c>.log</c> is the human-readable formatted
|
||
/// view, the <c>.LLM</c> is the markdown LLM-paste-friendly view, and the <c>.output</c> is a binary hex
|
||
/// dump of the Large test data's AcBinary-Default serialization (for raw inspection / wire-debugging).
|
||
/// </summary>
|
||
public static void SaveAll(ReportingContext ctx, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||
{
|
||
Directory.CreateDirectory(ctx.ResultsDirectory);
|
||
|
||
var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
|
||
var baseFileName = $"{ctx.SourceTag}.FullBenchmark_{ctx.BuildConfiguration}_{timestamp}";
|
||
var logFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.log");
|
||
var outputFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.output");
|
||
var llmFilePath = Path.Combine(ctx.ResultsDirectory, $"{baseFileName}.LLM");
|
||
|
||
// Save binary output to separate .output file.
|
||
// Cast to TestDataSet<TestOrder_All_False> because Phase 1 hardcodes the benchmark variant.
|
||
// Phase 2 will replace the cast with an options-driven dispatch (matching CreateSerializers).
|
||
var largeTestData = testDataSets.FirstOrDefault(t => t.Name.StartsWith("Large")) as TestDataSet<TestOrder_All_False>;
|
||
if (largeTestData != null)
|
||
{
|
||
var outputSb = new StringBuilder();
|
||
outputSb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||
outputSb.AppendLine("║ SERIALIZED BINARY OUTPUT ║");
|
||
outputSb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
|
||
outputSb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||
outputSb.AppendLine();
|
||
|
||
outputSb.AppendLine("=== SERIALIZED BYTES: Large (5x5x5x10) - AcBinary (Default) ===");
|
||
var serializedBytes = AcBinarySerializer.Serialize(largeTestData.Order, AcBinarySerializerOptions.Default);
|
||
outputSb.AppendLine($"Size: {serializedBytes.Length:N0} bytes");
|
||
outputSb.AppendLine();
|
||
outputSb.AppendLine("Hex dump:");
|
||
outputSb.AppendLine(FormatHexDump(serializedBytes));
|
||
|
||
File.WriteAllText(outputFilePath, outputSb.ToString(), ctx.Utf8NoBom);
|
||
System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}");
|
||
}
|
||
|
||
// Save benchmark results to .log file
|
||
var sb = new StringBuilder();
|
||
sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
|
||
sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║");
|
||
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
|
||
sb.AppendLine($"║ Source: {ctx.SourceTag}".PadRight(100) + "║");
|
||
sb.AppendLine($"║ Build: {ctx.BuildConfiguration}".PadRight(100) + "║");
|
||
sb.AppendLine($"║ Charset: {ctx.CharsetName}".PadRight(100) + "║");
|
||
// For BDN-sourced contexts, warmup / samples / target are managed inside BDN's job config (not by
|
||
// our adaptive engine) — surfacing the placeholder zeros as concrete numbers would be misleading.
|
||
// Print "BDN-managed" instead; raw BDN config is recoverable from the BDN-native artifacts under .../BDN/.
|
||
var isBdn = ctx.SourceTag == "Bdn";
|
||
var iterationsHeader = isBdn ? "Iterations: BDN-managed" : $"Iterations: per-cell adaptive (~{ctx.TargetSampleMs} ms target)";
|
||
var samplesHeader = isBdn ? "Samples: BDN-managed" : $"Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded";
|
||
sb.AppendLine($"║ {iterationsHeader}".PadRight(100) + "║");
|
||
sb.AppendLine($"║ {samplesHeader}".PadRight(100) + "║");
|
||
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||
sb.AppendLine();
|
||
|
||
// Serializer options summary. The bracketed [OrderType] suffix shows which TestOrder variant
|
||
// graph each benchmark serialised — AcBinary picks variant per options preset
|
||
// (FastMode → _All_False, Default → _All_True; see BenchmarkLoop.UsesAllFalseVariant),
|
||
// MemPack / MsgPack always use _All_False. Distinct() de-dupes across cells (each preset
|
||
// appears once even though it runs on every test data set).
|
||
var optionsMap = results
|
||
.Where(r => r.OptionsDescription != null)
|
||
.Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!))
|
||
.Distinct()
|
||
.ToList();
|
||
if (optionsMap.Count > 0)
|
||
{
|
||
sb.AppendLine("=== SERIALIZER OPTIONS ===");
|
||
foreach (var (name, orderType, opts) in optionsMap)
|
||
sb.AppendLine($" {name} [{orderType}]: {opts}");
|
||
sb.AppendLine();
|
||
}
|
||
|
||
// CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely.
|
||
// InvariantCulture is mandatory here: the decimal-separator dimension MUST be `.` so the comma-separated
|
||
// field delimiters don't collide with locale-specific decimal commas (e.g. Hungarian "7,38" would split
|
||
// a single F2 value across two CSV fields).
|
||
var inv = CultureInfo.InvariantCulture;
|
||
sb.AppendLine("=== RAW DATA (CSV) ===");
|
||
sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes");
|
||
|
||
foreach (var testData in testDataSets)
|
||
{
|
||
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList();
|
||
foreach (var result in testResults)
|
||
{
|
||
sb.AppendLine($"{result.TestDataName},{result.Engine.ToDisplay()},{result.IoMode.ToDisplay()},{result.DispatchMode.ToDisplay()},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result).ToString("F2", inv)},{DesPerOp(result).ToString("F2", inv)},{RtPerOp(result).ToString("F2", inv)},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}");
|
||
}
|
||
}
|
||
sb.AppendLine();
|
||
|
||
// Formatted results
|
||
sb.AppendLine("=== FORMATTED RESULTS BY TEST DATA ===");
|
||
sb.AppendLine("(►) = Highlighted: MemoryPack (Byte[]) (baseline) and AcBinary (Byte[])");
|
||
sb.AppendLine();
|
||
|
||
foreach (var testData in testDataSets)
|
||
{
|
||
// Order by Engine (stable column-position across cells, see PrintGroupedResults for rationale);
|
||
// RtPerOp is the secondary tiebreaker between same-engine variants (SGen vs Runtime).
|
||
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => r.Engine).ThenBy(r => RtPerOp(r)).ToList();
|
||
var memPackResult = testResults.FirstOrDefault(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray));
|
||
var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen));
|
||
|
||
sb.AppendLine();
|
||
sb.AppendLine($"--- {testData.DisplayName} ---");
|
||
sb.AppendLine($"{"#",-4} {"Serializer",-42} {"Size B",-12} {"Setup S/D KB",-14} {"Ser µs/op",-12} {"Des µs/op",-12} {"RT µs/op",-12} {"SerAlc KB",-11} {"DesAlc KB",-11}");
|
||
sb.AppendLine(new string('-', 140));
|
||
|
||
var rank = 1;
|
||
foreach (var result in testResults)
|
||
{
|
||
var isHighlighted = ((result.Engine == BenchmarkEngine.MemoryPack || result.Engine == BenchmarkEngine.AcBinary) && result.IoMode == BenchmarkIoMode.ByteArray);
|
||
var prefix = isHighlighted ? "► " : " ";
|
||
|
||
var size = $"{result.SerializedSize:N0}";
|
||
var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}";
|
||
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A";
|
||
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A";
|
||
var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A";
|
||
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A";
|
||
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A";
|
||
|
||
sb.AppendLine($"{rank++,2} {prefix}{result.SerializerName,-40} {size,-12} {setup,-14} {ser,-12} {des,-12} {rt,-12} {serAlloc,-11} {desAlloc,-11}");
|
||
}
|
||
|
||
// Summary row for this test data (vs MemoryPack)
|
||
if (memPackResult != null && acBinaryResult != null)
|
||
{
|
||
var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100;
|
||
var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0;
|
||
var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0;
|
||
var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0;
|
||
|
||
sb.AppendLine($" AcBinary (Byte[]) vs MemoryPack (Byte[]): Size {sizePct:+0;-0}% │ Ser {serPct:+0;-0}% │ Des {desPct:+0;-0}% │ RT {rtPct:+0;-0}%");
|
||
}
|
||
}
|
||
|
||
// Summary comparison (vs MemoryPack)
|
||
sb.AppendLine();
|
||
sb.AppendLine("=== AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ===");
|
||
|
||
var memPackSerResults2 = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.SerializeTimeMs > 0).ToList();
|
||
var memPackDesResults2 = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.DeserializeTimeMs > 0).ToList();
|
||
var memPackRtResults2 = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray) && r.RoundTripTimeMs > 0).ToList();
|
||
|
||
var acBinarySerResults2 = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.SerializeTimeMs > 0).ToList();
|
||
var acBinaryDesResults2 = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.DeserializeTimeMs > 0).ToList();
|
||
var acBinaryRtResults2 = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen) && r.RoundTripTimeMs > 0).ToList();
|
||
|
||
// Skip comparison block if either side has no Byte[] data
|
||
if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0)
|
||
{
|
||
sb.AppendLine(" (Comparison requires both serialize and deserialize data)");
|
||
File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
|
||
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
||
|
||
SaveLlmResults(ctx, llmFilePath, results, testDataSets);
|
||
return;
|
||
}
|
||
|
||
var sizeAcResults2 = results.Where(r => (r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen)).ToList();
|
||
var sizeMpResults2 = results.Where(r => (r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray)).ToList();
|
||
|
||
AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, SerPerOp));
|
||
AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, r => r.SerializeAllocBytesPerOp), "F0");
|
||
AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, DesPerOp));
|
||
AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, r => r.DeserializeAllocBytesPerOp), "F0");
|
||
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp));
|
||
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0");
|
||
|
||
File.WriteAllText(logFilePath, sb.ToString(), ctx.Utf8NoBom);
|
||
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
||
|
||
// Save LLM-optimized results
|
||
SaveLlmResults(ctx, llmFilePath, results, testDataSets);
|
||
}
|
||
|
||
private static void SaveLlmResults(ReportingContext ctx, string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||
{
|
||
var sb = new StringBuilder();
|
||
sb.AppendLine($"# AcBinary Benchmark [{ctx.SourceTag}] {ctx.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
|
||
// BDN-sourced: warmup / iter / samples are BDN-job-config-managed (see .../BDN/ artifacts for raw N).
|
||
// Console-sourced: our adaptive engine emits real numbers.
|
||
var runStatsHeader = ctx.SourceTag == "Bdn"
|
||
? "Iterations: BDN-managed | Warmup: BDN-managed | Samples: BDN-managed"
|
||
: $"Iterations: per-cell adaptive (target ~{ctx.TargetSampleMs} ms/sample) | Warmup: {ctx.WarmupIterations} per phase (Ser/Des isolated) | Samples: {ctx.BenchmarkSamples} (median) + 1 pilot discarded";
|
||
sb.AppendLine($"Charset: {ctx.CharsetName} | {runStatsHeader} | .NET: {Environment.Version} | UnstableCV threshold: {ctx.UnstableCVThreshold * 100:F0}%");
|
||
sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup");
|
||
|
||
// Options summary. Bracketed [OrderType] surfaces the TestOrder variant each preset serialised —
|
||
// see SaveAll for the variant-dispatch rationale.
|
||
var optionsMap = results
|
||
.Where(r => r.OptionsDescription != null)
|
||
.Select(r => (r.SerializerName, r.OrderTypeName, r.OptionsDescription!))
|
||
.Distinct()
|
||
.ToList();
|
||
if (optionsMap.Count > 0)
|
||
{
|
||
sb.AppendLine();
|
||
sb.AppendLine("## Options");
|
||
sb.AppendLine();
|
||
foreach (var (name, orderType, opts) in optionsMap)
|
||
sb.AppendLine($"- **{name} [{orderType}]**: {opts}");
|
||
}
|
||
|
||
sb.AppendLine();
|
||
sb.AppendLine("## Results");
|
||
sb.AppendLine();
|
||
sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des");
|
||
sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---");
|
||
|
||
var inv = CultureInfo.InvariantCulture;
|
||
|
||
foreach (var testData in testDataSets)
|
||
{
|
||
// Order by Engine for stable column-position across cells (see PrintGroupedResults for rationale).
|
||
var testResults = results
|
||
.Where(r => r.TestDataName == testData.DisplayName)
|
||
.OrderBy(r => r.Engine).ThenBy(RtPerOp)
|
||
.ToList();
|
||
|
||
foreach (var r in testResults)
|
||
{
|
||
var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv, ctx.UnstableCVThreshold) : "-";
|
||
var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv, ctx.UnstableCVThreshold) : "-";
|
||
var rt = r.RoundTripTimeMs > 0
|
||
? (r.IsRoundTripOnly
|
||
? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv, ctx.UnstableCVThreshold)
|
||
: RtPerOp(r).ToString("F2", inv))
|
||
: "-";
|
||
|
||
var serAlloc = r.SerializeTimeMs > 0 ? ToKilobytes(r.SerializeAllocBytesPerOp).ToString("F2", inv) : "-";
|
||
var desAlloc = r.DeserializeTimeMs > 0 ? ToKilobytes(r.DeserializeAllocBytesPerOp).ToString("F2", inv) : "-";
|
||
var rtAlloc = r.RoundTripAllocBytesPerOp > 0 ? ToKilobytes(r.RoundTripAllocBytesPerOp).ToString("F2", inv) : "-";
|
||
var setupAlloc = $"{ToKilobytes(r.SetupSerializeAllocBytes).ToString("F2", inv)} / {ToKilobytes(r.SetupDeserializeAllocBytes).ToString("F2", inv)}";
|
||
|
||
var iterCol = r.IsRoundTripOnly
|
||
? r.RoundTripIterations.ToString(inv)
|
||
: $"{(r.SerializeIterations > 0 ? r.SerializeIterations.ToString(inv) : "-")} / {(r.DeserializeIterations > 0 ? r.DeserializeIterations.ToString(inv) : "-")}";
|
||
sb.AppendLine($"{r.TestDataName} | {r.Engine.ToDisplay()} | {r.IoMode.ToDisplay()} | {r.DispatchMode.ToDisplay()} | {r.OptionsPreset} | {r.SerializedSize} | {ser} | {des} | {rt} | {serAlloc} | {desAlloc} | {rtAlloc} | {setupAlloc} | {iterCol}");
|
||
}
|
||
}
|
||
|
||
// Overall AcBinary (SGen, Byte[]) vs MemoryPack (Byte[]) comparison
|
||
var memPackByteArrayResults = results.Where(r => r.Engine == BenchmarkEngine.MemoryPack && r.IoMode == BenchmarkIoMode.ByteArray).ToList();
|
||
var acBinarySGenByteArrayResults = results.Where(r => r.Engine == BenchmarkEngine.AcBinary && r.IoMode == BenchmarkIoMode.ByteArray && r.DispatchMode == BenchmarkDispatchMode.SGen).ToList();
|
||
var memPackSerResultsLlm = memPackByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList();
|
||
var memPackDesResultsLlm = memPackByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList();
|
||
var memPackRtResultsLlm = memPackByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList();
|
||
var acBinarySerResultsLlm = acBinarySGenByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList();
|
||
var acBinaryDesResultsLlm = acBinarySGenByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList();
|
||
var acBinaryRtResultsLlm = acBinarySGenByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList();
|
||
|
||
if (memPackRtResultsLlm.Count > 0 && acBinaryRtResultsLlm.Count > 0)
|
||
{
|
||
sb.AppendLine();
|
||
sb.AppendLine("## Overall: AcBinary (Byte[], SGen) vs MemoryPack (Byte[])");
|
||
sb.AppendLine();
|
||
sb.AppendLine("Three aggregations of per-cell results: **arith** = arithmetic mean of µs/op (magnitude-weighted, Large cell dominates); **geo** = geometric mean of per-cell ratios (each cell weighted equally); **median** = median of per-cell ratios (outlier-resistant). Negative % = AcBinary faster/smaller; positive % = MemPack faster/smaller. The geo/median variants surface when a single big cell skews the arithmetic mean.");
|
||
sb.AppendLine();
|
||
sb.AppendLine("```");
|
||
AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, SerPerOp));
|
||
AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, r => r.SerializeAllocBytesPerOp), "F0");
|
||
AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, DesPerOp));
|
||
AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, r => r.DeserializeAllocBytesPerOp), "F0");
|
||
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResultsLlm, memPackRtResultsLlm, RtPerOp));
|
||
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(acBinarySGenByteArrayResults, memPackByteArrayResults, r => r.SerializedSize), "F0");
|
||
sb.AppendLine("```");
|
||
}
|
||
|
||
File.WriteAllText(filePath, sb.ToString(), ctx.Utf8NoBom);
|
||
System.Console.WriteLine($"✓ LLM results saved to: {filePath}");
|
||
}
|
||
|
||
/// <summary>
|
||
/// Formats byte array as hex dump with offset, hex values, and ASCII representation.
|
||
/// </summary>
|
||
public static string FormatHexDump(byte[] bytes, int bytesPerLine = 16)
|
||
{
|
||
var sb = new StringBuilder();
|
||
for (var i = 0; i < bytes.Length; i += bytesPerLine)
|
||
{
|
||
// Offset
|
||
sb.Append($"{i:X8} ");
|
||
|
||
// Hex bytes
|
||
for (var j = 0; j < bytesPerLine; j++)
|
||
{
|
||
if (i + j < bytes.Length)
|
||
sb.Append($"{bytes[i + j]:X2} ");
|
||
else
|
||
sb.Append(" ");
|
||
|
||
if (j == 7) sb.Append(' '); // Extra space in middle
|
||
}
|
||
|
||
sb.Append(" |");
|
||
|
||
// ASCII representation
|
||
for (var j = 0; j < bytesPerLine && i + j < bytes.Length; j++)
|
||
{
|
||
var b = bytes[i + j];
|
||
sb.Append(b is >= 32 and < 127 ? (char)b : '.');
|
||
}
|
||
|
||
sb.AppendLine("|");
|
||
}
|
||
return sb.ToString();
|
||
}
|
||
}
|