using AyCode.Core.Serializers.Binaries;
using AyCode.Core.Tests.TestModels;
using System.Globalization;
using System.Runtime.CompilerServices;
using System.Text;
namespace AyCode.Core.Serializers.Console;
///
/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file.
/// Consumes rows produced by the benchmark loop and emits human-readable
/// + LLM-friendly outputs. Display-only helpers (per-op µs conversion, allocation KB formatting,
/// range/CV formatting, paired-aggregation stats) live here too — they are display-side concerns.
///
internal static class Output
{
///
/// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation
/// strategies so the reader can judge whether the headline delta is dominated by one large cell
/// (arithmetic mean) or representative of typical workload (geometric mean / median).
///
/// Arithmetic mean of µs/op — magnitude-weighted; biased toward Large cell.
/// Geometric mean of per-cell ratios — magnitude-neutral; each cell weighted equally.
/// Median of per-cell ratios — outlier-resistant.
/// Arithmetic mean AcBinary value (µs/op or bytes).
/// Arithmetic mean MemPack value.
/// Number of paired cells contributing to the geo/median.
internal record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0;
// Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever
// averaging or comparison happens across rows with potentially different iter counts (Winners summary,
// Overall comparison, per-cell summary row). Keeping these as methods rather than properties on
// BenchmarkResult preserves the result-as-data-bag distinction.
internal static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations);
internal static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations);
internal static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations);
///
/// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can
/// render compact F2 KB values (e.g. 4.05 KB instead of 4,144 B) — header carries
/// the unit so per-row entries stay numbers-only. CSV / raw-data outputs keep the precise byte
/// integers untouched.
///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static double ToKilobytes(long bytes) => bytes / 1024.0;
///
/// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison
/// across paired cells (joined by TestDataName). Per-cell pairing is required for the
/// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats.
/// Returns null when no paired cell has a valid value.
///
internal static OverallStats? ComputeOverallStats(List acResults, List mpResults, Func getValue)
{
if (acResults.Count == 0 || mpResults.Count == 0) return null;
var pairs = (from ac in acResults
join mp in mpResults on ac.TestDataName equals mp.TestDataName
let acV = getValue(ac)
let mpV = getValue(mp)
where acV > 0 && mpV > 0
select (ac: acV, mp: mpV)).ToList();
if (pairs.Count == 0) return null;
var acAvg = pairs.Average(p => p.ac);
var mpAvg = pairs.Average(p => p.mp);
var ratios = pairs.Select(p => p.ac / p.mp).ToList();
// Geometric mean: exp(avg(ln(ratios))) — numerically stable vs Π ratios then ^(1/N).
var geoMean = Math.Exp(ratios.Sum(Math.Log) / ratios.Count);
// Median (paired-ratio): for even N use the midpoint of the two middle values.
var sorted = ratios.OrderBy(r => r).ToList();
var median = sorted.Count % 2 == 1
? sorted[sorted.Count / 2]
: (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2.0;
return new OverallStats(
ArithMeanPct: (acAvg / mpAvg - 1) * 100,
GeoMeanPct: (geoMean - 1) * 100,
MedianPct: (median - 1) * 100,
AcAvg: acAvg,
MpAvg: mpAvg,
CellCount: ratios.Count);
}
///
/// Formats a per-op micros value with its inter-sample range and CV-threshold marker as
/// "26.86 (24.5..29.1)" or "26.86 (24.5..29.1) ⚠️5.2%". Median first, range in parentheses,
/// CV warning suffix only when CV > . When min == max == median
/// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter.
/// All time inputs are total-batch milliseconds; is the per-row iter
/// count (post-adaptive-calibration).
///
internal static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv)
{
var med = ToPerOpMicros(medianMs, iterations);
// No range data (single-sample fast path) — surface as bare median, identical to the prior format.
if (minMs <= 0 && maxMs <= 0) return med.ToString("F2", inv);
if (minMs >= medianMs && maxMs <= medianMs) return med.ToString("F2", inv);
var min = ToPerOpMicros(minMs, iterations);
var max = ToPerOpMicros(maxMs, iterations);
var range = $"{med.ToString("F2", inv)} ({min.ToString("F2", inv)}..{max.ToString("F2", inv)})";
// CV (coefficient of variation = stddev / mean) — flag rows above the unstable threshold so a
// small inter-engine delta on a high-CV row is easy to discount as noise.
if (medianMs > 0 && stdDevMs > 0)
{
var cv = stdDevMs / medianMs;
if (cv > Configuration.UnstableCVThreshold)
{
var cvPct = (cv * 100).ToString("F1", inv);
return $"{range} ⚠️{cvPct}%";
}
}
return range;
}
///
/// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`).
/// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block.
///
internal static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%";
///
/// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means.
/// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when
/// stats is null (no paired data).
///
internal static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2")
{
if (stats == null) return;
// Color follows geo-mean (the magnitude-neutral signal). The arith-mean column may show a
// different sign when a single big cell dominates — that's exactly the signal we want to surface.
System.Console.ForegroundColor = stats.GeoMeanPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.WriteLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} │ geo {FormatPctSigned(stats.GeoMeanPct)} │ median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
System.Console.ResetColor();
}
///
/// Same as but appends to a (no color).
/// Used by the .log and .LLM file writers.
///
internal static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2")
{
if (stats == null) return;
sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)");
}
internal static void PrintResult(BenchmarkResult result)
{
// Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op).
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A";
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result),7:F2}" : " N/A";
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp),7:F2}" : " N/A";
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp),7:F2}" : " N/A";
System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)");
}
internal static void PrintGroupedResults(List results, List testDataSets)
{
System.Console.WriteLine("\n");
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║");
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
// Print serializer options
var optionsMap = results
.Where(r => r.OptionsDescription != null)
.Select(r => (r.SerializerName, r.OptionsDescription!))
.Distinct()
.ToList();
if (optionsMap.Count > 0)
{
System.Console.WriteLine();
System.Console.WriteLine(" Serializer Options:");
foreach (var (name, opts) in optionsMap)
System.Console.WriteLine($" {name}: {opts}");
}
foreach (var testData in testDataSets)
{
// Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration.
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList();
// Baseline switched MessagePack → MemoryPack: MemoryPack is the SOTA performance leader.
var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray));
// Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated).
// The Runtime variant is shown alongside in the table for context, not used as the headline number.
var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen));
System.Console.WriteLine($"\n┌─ {testData.DisplayName} ─".PadRight(172, '─') + "┐");
// Header-only units; per-row entries are numbers (µs/op for time, KB/op for alloc, KB pair "ser / des" for Setup, B for Size).
System.Console.WriteLine($"│ {"#",-4} │ {"Engine",-11} │ {"Options",-22} │ {"IO",-12} │ {"Mode",-8} │ {"Setup S/D KB",-14} │ {"Size B",-8} │ {"Ser µs/op",-10} │ {"SerAlc KB",-10} │ {"Des µs/op",-10} │ {"DesAlc KB",-10} │ {"RT µs/op",-10} │ {"RTAlc KB",-10} │");
System.Console.WriteLine($"├{"─".PadRight(6, '─')}┼{"─".PadRight(13, '─')}┼{"─".PadRight(24, '─')}┼{"─".PadRight(14, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤");
var rank = 1;
foreach (var result in testResults)
{
var size = $"{result.SerializedSize:N0}";
var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}";
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A";
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A";
var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A";
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A";
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A";
var rtAlloc = result.RoundTripAllocBytesPerOp > 0 ? $"{ToKilobytes(result.RoundTripAllocBytesPerOp):F2}" : "N/A";
// Highlight MemoryPack baseline (any Byte[]) and AcBinary headline contender (Byte[] + SGen) with win/lose colors.
// The AcBinary Byte[]+Runtime variant is shown unhighlighted — it's contextual (SGen speed-up reference), not the headline.
var isHighlighted = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray)
|| (result.Engine == Configuration.EngineAcBinary && result.IoMode == Configuration.IoByteArray && result.DispatchMode == Configuration.ModeSGen);
var prefix = isHighlighted ? "│►" : "│ ";
var suffix = isHighlighted ? "◄│" : " │";
// Color logic: Green = winner (faster), Red = loser (slower)
if (isHighlighted && memPackResult != null && acBinaryResult != null)
{
var isMemPack = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray);
var memPackFaster = RtPerOp(memPackResult) < RtPerOp(acBinaryResult);
if (isMemPack)
{
System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Green : ConsoleColor.Red;
}
else
{
System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Red : ConsoleColor.Green;
}
}
System.Console.WriteLine($"{prefix}{rank++,4} │ {result.Engine,-11} │ {result.OptionsPreset,-22} │ {result.IoMode,-12} │ {result.DispatchMode,-8} │ {setup,14} │ {size,8} │ {ser,10} │ {serAlloc,10} │ {des,10} │ {desAlloc,10} │ {rt,10} │ {rtAlloc,10}{suffix}");
if (isHighlighted)
{
System.Console.ResetColor();
}
}
// Footer row: AcBinary (Byte[]) vs MemoryPack (Byte[]) comparison per column
if (memPackResult != null && acBinaryResult != null)
{
var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100;
// Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows.
var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0;
var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0;
var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0;
var serAllocPct = memPackResult.SerializeAllocBytesPerOp > 0 ? (acBinaryResult.SerializeAllocBytesPerOp / (double)memPackResult.SerializeAllocBytesPerOp - 1) * 100 : 0;
var desAllocPct = memPackResult.DeserializeAllocBytesPerOp > 0 ? (acBinaryResult.DeserializeAllocBytesPerOp / (double)memPackResult.DeserializeAllocBytesPerOp - 1) * 100 : 0;
var rtAllocPct = memPackResult.RoundTripAllocBytesPerOp > 0 ? (acBinaryResult.RoundTripAllocBytesPerOp / (double)memPackResult.RoundTripAllocBytesPerOp - 1) * 100 : 0;
// Footer separator: merge first 5 cols (#, Engine, Options, IO, Mode) → comparison label;
// remaining 8 cols stay aligned (Setup S/D KB, Size, Ser µs/op, SerAlc KB, Des µs/op, DesAlc KB, RT µs/op, RTAlc KB).
System.Console.WriteLine($"├{"─".PadRight(6, '─')}┴{"─".PadRight(13, '─')}┴{"─".PadRight(24, '─')}┴{"─".PadRight(14, '─')}┴{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤");
// Merged label cell width = 4 + 11 + 22 + 12 + 8 + 4*3 (dropped separators) = 69
System.Console.Write($"│ {"► AcBinary (Byte[]) vs MemoryPack (Byte[])",-69} │ ");
// Setup S/D KB (n/a for Byte[] vs Byte[] — neither pre-allocates)
System.Console.Write($"{"—",14}");
System.Console.Write(" │ ");
// Size
System.Console.ForegroundColor = sizePct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{sizePct,+7:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Serialize
System.Console.ForegroundColor = serPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{serPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Serialize Alloc
System.Console.ForegroundColor = serAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{serAllocPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Deserialize
System.Console.ForegroundColor = desPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{desPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Deserialize Alloc
System.Console.ForegroundColor = desAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{desAllocPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Round-trip
System.Console.ForegroundColor = rtPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{rtPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.Write(" │ ");
// Round-trip Alloc
System.Console.ForegroundColor = rtAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red;
System.Console.Write($"{rtAllocPct,+9:+0;-0}%");
System.Console.ResetColor();
System.Console.WriteLine(" │");
}
// Closing line: merged on left (─ between cols 1-5), ┴ on the right (cols 6-13 boundary, 8 unmerged cells).
System.Console.WriteLine($"└{"─".PadRight(6, '─')}─{"─".PadRight(13, '─')}─{"─".PadRight(24, '─')}─{"─".PadRight(14, '─')}─{"─".PadRight(10, '─')}┴{"─".PadRight(16, '─')}┴{"─".PadRight(10, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┘");
}
// Summary: Best serializer for each category
System.Console.WriteLine("\n");
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
System.Console.WriteLine("║ SUMMARY: WINNERS ║");
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
System.Console.WriteLine($"\n{"Category",-20} │ {"Winner",-40} │ {"Avg Value",-18}");
System.Console.WriteLine($"{"─".PadRight(20, '─')}─┼─{"─".PadRight(40, '─')}─┼─{"─".PadRight(18, '─')}");
// Fastest Serialize — round-trip-only serializers (NamedPipe etc.) excluded:
// their Serialize() captures the full round-trip and isn't comparable to a pure Ser metric.
var fastestSer = results.Where(r => r.SerializeTimeMs > 0 && !r.IsRoundTripOnly)
.GroupBy(r => r.SerializerName)
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => SerPerOp(r)) })
.OrderBy(x => x.AvgPerOp)
.FirstOrDefault();
if (fastestSer != null)
System.Console.WriteLine($"{"Fastest Serialize",-20} │ {fastestSer.Name,-40} │ {fastestSer.AvgPerOp,12:F2} µs/op");
// Fastest Deserialize — round-trip-only serializers excluded (their Deserialize() is a no-op).
var fastestDes = results.Where(r => r.DeserializeTimeMs > 0 && !r.IsRoundTripOnly)
.GroupBy(r => r.SerializerName)
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => DesPerOp(r)) })
.OrderBy(x => x.AvgPerOp)
.FirstOrDefault();
if (fastestDes != null)
System.Console.WriteLine($"{"Fastest Deserialize",-20} │ {fastestDes.Name,-40} │ {fastestDes.AvgPerOp,12:F2} µs/op");
// Smallest Size
var smallestSize = results
.GroupBy(r => r.SerializerName)
.Select(g => new { Name = g.Key, AvgSize = g.Average(r => r.SerializedSize) })
.OrderBy(x => x.AvgSize)
.FirstOrDefault();
if (smallestSize != null)
System.Console.WriteLine($"{"Smallest Size",-20} │ {smallestSize.Name,-40} │ {smallestSize.AvgSize,15:F0} B");
// Fastest Round-trip — iter-independent per-op average.
var fastestRt = results.Where(r => r.RoundTripTimeMs > 0)
.GroupBy(r => r.SerializerName)
.Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => RtPerOp(r)) })
.OrderBy(x => x.AvgPerOp)
.FirstOrDefault();
if (fastestRt != null)
System.Console.WriteLine($"{"Fastest Round-trip",-20} │ {fastestRt.Name,-40} │ {fastestRt.AvgPerOp,12:F2} µs/op");
// Overall AcBinary (SGen) vs MemoryPack comparison.
var memPackSerResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList();
var memPackDesResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList();
var memPackRtResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList();
var acBinarySerResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList();
var acBinaryDesResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList();
var acBinaryRtResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList();
// Skip comparison if no data available
if (memPackRtResults.Count == 0 || acBinaryRtResults.Count == 0)
{
System.Console.WriteLine();
System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──");
System.Console.WriteLine(" (Comparison requires both serialize and deserialize data)");
return;
}
// All averages are over per-op µs (iter-independent). Three aggregations per metric.
var sizeAcResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList();
var sizeMpResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList();
var serStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, SerPerOp);
var desStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, DesPerOp);
var rtStats = ComputeOverallStats(acBinaryRtResults, memPackRtResults, RtPerOp);
var sizeStats = ComputeOverallStats(sizeAcResults, sizeMpResults, r => r.SerializedSize);
var serAllocStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, r => r.SerializeAllocBytesPerOp);
var desAllocStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, r => r.DeserializeAllocBytesPerOp);
System.Console.WriteLine();
System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──");
WriteOverallLine("Serialize", "µs/op", serStats);
WriteOverallLine("Deserialize", "µs/op", desStats);
WriteOverallLine("Round-trip", "µs/op", rtStats);
WriteOverallLine("Size", "B", sizeStats, "F0");
WriteOverallLine("Ser Alloc", "B/op", serAllocStats, "F0");
WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0");
}
internal static void SaveResults(List results, List testDataSets)
{
Directory.CreateDirectory(Configuration.ResultsDirectory);
var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}";
var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log");
var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output");
// Save binary output to separate .output file
var largeTestData = testDataSets.FirstOrDefault(t => t.Name.StartsWith("Large"));
if (largeTestData != null)
{
var outputSb = new StringBuilder();
outputSb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
outputSb.AppendLine("║ SERIALIZED BINARY OUTPUT ║");
outputSb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
outputSb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
outputSb.AppendLine();
outputSb.AppendLine("=== SERIALIZED BYTES: Large (5x5x5x10) - AcBinary (Default) ===");
var serializedBytes = AcBinarySerializer.Serialize(largeTestData.Order, AcBinarySerializerOptions.Default);
outputSb.AppendLine($"Size: {serializedBytes.Length:N0} bytes");
outputSb.AppendLine();
outputSb.AppendLine("Hex dump:");
outputSb.AppendLine(FormatHexDump(serializedBytes));
File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom);
System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}");
}
// Save benchmark results to .log file
var sb = new StringBuilder();
sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗");
sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║");
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║");
sb.AppendLine($"║ Charset: {Configuration.GetCurrentCharsetName()}".PadRight(100) + "║");
sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║");
sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║");
sb.AppendLine($"║ Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}".PadRight(100) + "║");
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
sb.AppendLine();
// Serializer options summary
var optionsMap = results
.Where(r => r.OptionsDescription != null)
.Select(r => (r.SerializerName, r.OptionsDescription!))
.Distinct()
.ToList();
if (optionsMap.Count > 0)
{
sb.AppendLine("=== SERIALIZER OPTIONS ===");
foreach (var (name, opts) in optionsMap)
sb.AppendLine($" {name}: {opts}");
sb.AppendLine();
}
// CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely.
sb.AppendLine("=== RAW DATA (CSV) ===");
sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes");
foreach (var testData in testDataSets)
{
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList();
foreach (var result in testResults)
{
sb.AppendLine($"{result.TestDataName},{result.Engine},{result.IoMode},{result.DispatchMode},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}");
}
}
sb.AppendLine();
// Formatted results
sb.AppendLine("=== FORMATTED RESULTS BY TEST DATA ===");
sb.AppendLine("(►) = Highlighted: MemoryPack (Byte[]) (baseline) and AcBinary (Byte[])");
sb.AppendLine();
foreach (var testData in testDataSets)
{
// Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration.
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList();
var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray));
var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen));
sb.AppendLine();
sb.AppendLine($"--- {testData.DisplayName} ---");
sb.AppendLine($"{"#",-4} {"Serializer",-42} {"Size B",-12} {"Setup S/D KB",-14} {"Ser µs/op",-12} {"Des µs/op",-12} {"RT µs/op",-12} {"SerAlc KB",-11} {"DesAlc KB",-11}");
sb.AppendLine(new string('-', 140));
var rank = 1;
foreach (var result in testResults)
{
var isHighlighted = ((result.Engine == Configuration.EngineMemoryPack || result.Engine == Configuration.EngineAcBinary) && result.IoMode == Configuration.IoByteArray);
var prefix = isHighlighted ? "► " : " ";
var size = $"{result.SerializedSize:N0}";
var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}";
var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A";
var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A";
var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A";
var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A";
var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A";
sb.AppendLine($"{rank++,2} {prefix}{result.SerializerName,-40} {size,-12} {setup,-14} {ser,-12} {des,-12} {rt,-12} {serAlloc,-11} {desAlloc,-11}");
}
// Summary row for this test data (vs MemoryPack)
if (memPackResult != null && acBinaryResult != null)
{
var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100;
var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0;
var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0;
var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0;
sb.AppendLine($" AcBinary (Byte[]) vs MemoryPack (Byte[]): Size {sizePct:+0;-0}% │ Ser {serPct:+0;-0}% │ Des {desPct:+0;-0}% │ RT {rtPct:+0;-0}%");
}
}
// Summary comparison (vs MemoryPack)
sb.AppendLine();
sb.AppendLine("=== AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ===");
var memPackSerResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList();
var memPackDesResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList();
var memPackRtResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList();
var acBinarySerResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList();
var acBinaryDesResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList();
var acBinaryRtResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList();
// Skip comparison block if either side has no Byte[] data
if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0)
{
sb.AppendLine(" (Comparison requires both serialize and deserialize data)");
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom);
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM");
SaveLlmResults(llmFilePathEarly, results, testDataSets);
return;
}
var sizeAcResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList();
var sizeMpResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList();
AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, SerPerOp));
AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, r => r.SerializeAllocBytesPerOp), "F0");
AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, DesPerOp));
AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, r => r.DeserializeAllocBytesPerOp), "F0");
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp));
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0");
File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom);
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
// Save LLM-optimized results
var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM");
SaveLlmResults(llmFilePath, results, testDataSets);
}
internal static void SaveLlmResults(string filePath, List results, List testDataSets)
{
var sb = new StringBuilder();
var testTypeName = testDataSets.FirstOrDefault()?.TypeName ?? "unknown";
sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
sb.AppendLine($"Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | TestType: {testTypeName} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%");
sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup");
// Options summary
var optionsMap = results
.Where(r => r.OptionsDescription != null)
.Select(r => (r.SerializerName, r.OptionsDescription!))
.Distinct()
.ToList();
if (optionsMap.Count > 0)
{
sb.AppendLine();
sb.AppendLine("## Options");
sb.AppendLine();
foreach (var (name, opts) in optionsMap)
sb.AppendLine($"- **{name}**: {opts}");
}
sb.AppendLine();
sb.AppendLine("## Results");
sb.AppendLine();
sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des");
sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---");
foreach (var testData in testDataSets)
{
var testResults = results
.Where(r => r.TestDataName == testData.DisplayName)
.OrderBy(RtPerOp)
.ToList();
foreach (var r in testResults)
{
var inv = CultureInfo.InvariantCulture;
var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-";
var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-";
var rt = r.RoundTripTimeMs > 0
? (r.IsRoundTripOnly
? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv)
: RtPerOp(r).ToString("F2", inv))
: "-";
var serAlloc = r.SerializeTimeMs > 0 ? ToKilobytes(r.SerializeAllocBytesPerOp).ToString("F2", inv) : "-";
var desAlloc = r.DeserializeTimeMs > 0 ? ToKilobytes(r.DeserializeAllocBytesPerOp).ToString("F2", inv) : "-";
var rtAlloc = r.RoundTripAllocBytesPerOp > 0 ? ToKilobytes(r.RoundTripAllocBytesPerOp).ToString("F2", inv) : "-";
var setupAlloc = $"{ToKilobytes(r.SetupSerializeAllocBytes).ToString("F2", inv)} / {ToKilobytes(r.SetupDeserializeAllocBytes).ToString("F2", inv)}";
var iterCol = r.IsRoundTripOnly
? r.RoundTripIterations.ToString(inv)
: $"{(r.SerializeIterations > 0 ? r.SerializeIterations.ToString(inv) : "-")} / {(r.DeserializeIterations > 0 ? r.DeserializeIterations.ToString(inv) : "-")}";
sb.AppendLine($"{r.TestDataName} | {r.Engine} | {r.IoMode} | {r.DispatchMode} | {r.OptionsPreset} | {r.SerializedSize} | {ser} | {des} | {rt} | {serAlloc} | {desAlloc} | {rtAlloc} | {setupAlloc} | {iterCol}");
}
}
// Overall AcBinary (SGen, Byte[]) vs MemoryPack (Byte[]) comparison
var memPackByteArrayResults = results.Where(r => r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray).ToList();
var acBinarySGenByteArrayResults = results.Where(r => r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen).ToList();
var memPackSerResultsLlm = memPackByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList();
var memPackDesResultsLlm = memPackByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList();
var memPackRtResultsLlm = memPackByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList();
var acBinarySerResultsLlm = acBinarySGenByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList();
var acBinaryDesResultsLlm = acBinarySGenByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList();
var acBinaryRtResultsLlm = acBinarySGenByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList();
if (memPackRtResultsLlm.Count > 0 && acBinaryRtResultsLlm.Count > 0)
{
sb.AppendLine();
sb.AppendLine("## Overall: AcBinary (Byte[], SGen) vs MemoryPack (Byte[])");
sb.AppendLine();
sb.AppendLine("Three aggregations of per-cell results: **arith** = arithmetic mean of µs/op (magnitude-weighted, Large cell dominates); **geo** = geometric mean of per-cell ratios (each cell weighted equally); **median** = median of per-cell ratios (outlier-resistant). Negative % = AcBinary faster/smaller; positive % = MemPack faster/smaller. The geo/median variants surface when a single big cell skews the arithmetic mean.");
sb.AppendLine();
sb.AppendLine("```");
AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, SerPerOp));
AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, r => r.SerializeAllocBytesPerOp), "F0");
AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, DesPerOp));
AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, r => r.DeserializeAllocBytesPerOp), "F0");
AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResultsLlm, memPackRtResultsLlm, RtPerOp));
AppendOverallLine(sb, "Size", "B", ComputeOverallStats(acBinarySGenByteArrayResults, memPackByteArrayResults, r => r.SerializedSize), "F0");
sb.AppendLine("```");
}
File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom);
System.Console.WriteLine($"✓ LLM results saved to: {filePath}");
}
///
/// Formats byte array as hex dump with offset, hex values, and ASCII representation.
///
internal static string FormatHexDump(byte[] bytes, int bytesPerLine = 16)
{
var sb = new StringBuilder();
for (var i = 0; i < bytes.Length; i += bytesPerLine)
{
// Offset
sb.Append($"{i:X8} ");
// Hex bytes
for (var j = 0; j < bytesPerLine; j++)
{
if (i + j < bytes.Length)
sb.Append($"{bytes[i + j]:X2} ");
else
sb.Append(" ");
if (j == 7) sb.Append(' '); // Extra space in middle
}
sb.Append(" |");
// ASCII representation
for (var j = 0; j < bytesPerLine && i + j < bytes.Length; j++)
{
var b = bytes[i + j];
sb.Append(b is >= 32 and < 127 ? (char)b : '.');
}
sb.AppendLine("|");
}
return sb.ToString();
}
}