using AyCode.Core.Serializers.Binaries; using AyCode.Core.Tests.TestModels; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; namespace AyCode.Core.Serializers.Console; /// /// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file. /// Consumes rows produced by the benchmark loop and emits human-readable /// + LLM-friendly outputs. Display-only helpers (per-op µs conversion, allocation KB formatting, /// range/CV formatting, paired-aggregation stats) live here too — they are display-side concerns. /// internal static class Output { /// /// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation /// strategies so the reader can judge whether the headline delta is dominated by one large cell /// (arithmetic mean) or representative of typical workload (geometric mean / median). /// /// Arithmetic mean of µs/op — magnitude-weighted; biased toward Large cell. /// Geometric mean of per-cell ratios — magnitude-neutral; each cell weighted equally. /// Median of per-cell ratios — outlier-resistant. /// Arithmetic mean AcBinary value (µs/op or bytes). /// Arithmetic mean MemPack value. /// Number of paired cells contributing to the geo/median. internal record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0; // Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever // averaging or comparison happens across rows with potentially different iter counts (Winners summary, // Overall comparison, per-cell summary row). Keeping these as methods rather than properties on // BenchmarkResult preserves the result-as-data-bag distinction. internal static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations); internal static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations); internal static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations); /// /// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can /// render compact F2 KB values (e.g. 4.05 KB instead of 4,144 B) — header carries /// the unit so per-row entries stay numbers-only. CSV / raw-data outputs keep the precise byte /// integers untouched. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static double ToKilobytes(long bytes) => bytes / 1024.0; /// /// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison /// across paired cells (joined by TestDataName). Per-cell pairing is required for the /// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats. /// Returns null when no paired cell has a valid value. /// internal static OverallStats? ComputeOverallStats(List acResults, List mpResults, Func getValue) { if (acResults.Count == 0 || mpResults.Count == 0) return null; var pairs = (from ac in acResults join mp in mpResults on ac.TestDataName equals mp.TestDataName let acV = getValue(ac) let mpV = getValue(mp) where acV > 0 && mpV > 0 select (ac: acV, mp: mpV)).ToList(); if (pairs.Count == 0) return null; var acAvg = pairs.Average(p => p.ac); var mpAvg = pairs.Average(p => p.mp); var ratios = pairs.Select(p => p.ac / p.mp).ToList(); // Geometric mean: exp(avg(ln(ratios))) — numerically stable vs Π ratios then ^(1/N). var geoMean = Math.Exp(ratios.Sum(Math.Log) / ratios.Count); // Median (paired-ratio): for even N use the midpoint of the two middle values. var sorted = ratios.OrderBy(r => r).ToList(); var median = sorted.Count % 2 == 1 ? sorted[sorted.Count / 2] : (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2.0; return new OverallStats( ArithMeanPct: (acAvg / mpAvg - 1) * 100, GeoMeanPct: (geoMean - 1) * 100, MedianPct: (median - 1) * 100, AcAvg: acAvg, MpAvg: mpAvg, CellCount: ratios.Count); } /// /// Formats a per-op micros value with its inter-sample range and CV-threshold marker as /// "26.86 (24.5..29.1)" or "26.86 (24.5..29.1) ⚠️5.2%". Median first, range in parentheses, /// CV warning suffix only when CV > . When min == max == median /// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter. /// All time inputs are total-batch milliseconds; is the per-row iter /// count (post-adaptive-calibration). /// internal static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv) { var med = ToPerOpMicros(medianMs, iterations); // No range data (single-sample fast path) — surface as bare median, identical to the prior format. if (minMs <= 0 && maxMs <= 0) return med.ToString("F2", inv); if (minMs >= medianMs && maxMs <= medianMs) return med.ToString("F2", inv); var min = ToPerOpMicros(minMs, iterations); var max = ToPerOpMicros(maxMs, iterations); var range = $"{med.ToString("F2", inv)} ({min.ToString("F2", inv)}..{max.ToString("F2", inv)})"; // CV (coefficient of variation = stddev / mean) — flag rows above the unstable threshold so a // small inter-engine delta on a high-CV row is easy to discount as noise. if (medianMs > 0 && stdDevMs > 0) { var cv = stdDevMs / medianMs; if (cv > Configuration.UnstableCVThreshold) { var cvPct = (cv * 100).ToString("F1", inv); return $"{range} ⚠️{cvPct}%"; } } return range; } /// /// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`). /// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block. /// internal static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%"; /// /// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means. /// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when /// stats is null (no paired data). /// internal static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2") { if (stats == null) return; // Color follows geo-mean (the magnitude-neutral signal). The arith-mean column may show a // different sign when a single big cell dominates — that's exactly the signal we want to surface. System.Console.ForegroundColor = stats.GeoMeanPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.WriteLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} │ geo {FormatPctSigned(stats.GeoMeanPct)} │ median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); System.Console.ResetColor(); } /// /// Same as but appends to a (no color). /// Used by the .log and .LLM file writers. /// internal static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2") { if (stats == null) return; sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); } internal static void PrintResult(BenchmarkResult result) { // Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op). var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A"; var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result),7:F2}" : " N/A"; var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp),7:F2}" : " N/A"; var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp),7:F2}" : " N/A"; System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)"); } internal static void PrintGroupedResults(List results, List testDataSets) { System.Console.WriteLine("\n"); System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║"); System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); // Print serializer options var optionsMap = results .Where(r => r.OptionsDescription != null) .Select(r => (r.SerializerName, r.OptionsDescription!)) .Distinct() .ToList(); if (optionsMap.Count > 0) { System.Console.WriteLine(); System.Console.WriteLine(" Serializer Options:"); foreach (var (name, opts) in optionsMap) System.Console.WriteLine($" {name}: {opts}"); } foreach (var testData in testDataSets) { // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); // Baseline switched MessagePack → MemoryPack: MemoryPack is the SOTA performance leader. var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); // Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated). // The Runtime variant is shown alongside in the table for context, not used as the headline number. var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); System.Console.WriteLine($"\n┌─ {testData.DisplayName} ─".PadRight(172, '─') + "┐"); // Header-only units; per-row entries are numbers (µs/op for time, KB/op for alloc, KB pair "ser / des" for Setup, B for Size). System.Console.WriteLine($"│ {"#",-4} │ {"Engine",-11} │ {"Options",-22} │ {"IO",-12} │ {"Mode",-8} │ {"Setup S/D KB",-14} │ {"Size B",-8} │ {"Ser µs/op",-10} │ {"SerAlc KB",-10} │ {"Des µs/op",-10} │ {"DesAlc KB",-10} │ {"RT µs/op",-10} │ {"RTAlc KB",-10} │"); System.Console.WriteLine($"├{"─".PadRight(6, '─')}┼{"─".PadRight(13, '─')}┼{"─".PadRight(24, '─')}┼{"─".PadRight(14, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); var rank = 1; foreach (var result in testResults) { var size = $"{result.SerializedSize:N0}"; var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; var rtAlloc = result.RoundTripAllocBytesPerOp > 0 ? $"{ToKilobytes(result.RoundTripAllocBytesPerOp):F2}" : "N/A"; // Highlight MemoryPack baseline (any Byte[]) and AcBinary headline contender (Byte[] + SGen) with win/lose colors. // The AcBinary Byte[]+Runtime variant is shown unhighlighted — it's contextual (SGen speed-up reference), not the headline. var isHighlighted = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray) || (result.Engine == Configuration.EngineAcBinary && result.IoMode == Configuration.IoByteArray && result.DispatchMode == Configuration.ModeSGen); var prefix = isHighlighted ? "│►" : "│ "; var suffix = isHighlighted ? "◄│" : " │"; // Color logic: Green = winner (faster), Red = loser (slower) if (isHighlighted && memPackResult != null && acBinaryResult != null) { var isMemPack = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray); var memPackFaster = RtPerOp(memPackResult) < RtPerOp(acBinaryResult); if (isMemPack) { System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Green : ConsoleColor.Red; } else { System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Red : ConsoleColor.Green; } } System.Console.WriteLine($"{prefix}{rank++,4} │ {result.Engine,-11} │ {result.OptionsPreset,-22} │ {result.IoMode,-12} │ {result.DispatchMode,-8} │ {setup,14} │ {size,8} │ {ser,10} │ {serAlloc,10} │ {des,10} │ {desAlloc,10} │ {rt,10} │ {rtAlloc,10}{suffix}"); if (isHighlighted) { System.Console.ResetColor(); } } // Footer row: AcBinary (Byte[]) vs MemoryPack (Byte[]) comparison per column if (memPackResult != null && acBinaryResult != null) { var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; // Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows. var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; var serAllocPct = memPackResult.SerializeAllocBytesPerOp > 0 ? (acBinaryResult.SerializeAllocBytesPerOp / (double)memPackResult.SerializeAllocBytesPerOp - 1) * 100 : 0; var desAllocPct = memPackResult.DeserializeAllocBytesPerOp > 0 ? (acBinaryResult.DeserializeAllocBytesPerOp / (double)memPackResult.DeserializeAllocBytesPerOp - 1) * 100 : 0; var rtAllocPct = memPackResult.RoundTripAllocBytesPerOp > 0 ? (acBinaryResult.RoundTripAllocBytesPerOp / (double)memPackResult.RoundTripAllocBytesPerOp - 1) * 100 : 0; // Footer separator: merge first 5 cols (#, Engine, Options, IO, Mode) → comparison label; // remaining 8 cols stay aligned (Setup S/D KB, Size, Ser µs/op, SerAlc KB, Des µs/op, DesAlc KB, RT µs/op, RTAlc KB). System.Console.WriteLine($"├{"─".PadRight(6, '─')}┴{"─".PadRight(13, '─')}┴{"─".PadRight(24, '─')}┴{"─".PadRight(14, '─')}┴{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); // Merged label cell width = 4 + 11 + 22 + 12 + 8 + 4*3 (dropped separators) = 69 System.Console.Write($"│ {"► AcBinary (Byte[]) vs MemoryPack (Byte[])",-69} │ "); // Setup S/D KB (n/a for Byte[] vs Byte[] — neither pre-allocates) System.Console.Write($"{"—",14}"); System.Console.Write(" │ "); // Size System.Console.ForegroundColor = sizePct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{sizePct,+7:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Serialize System.Console.ForegroundColor = serPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{serPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Serialize Alloc System.Console.ForegroundColor = serAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{serAllocPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Deserialize System.Console.ForegroundColor = desPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{desPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Deserialize Alloc System.Console.ForegroundColor = desAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{desAllocPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Round-trip System.Console.ForegroundColor = rtPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{rtPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.Write(" │ "); // Round-trip Alloc System.Console.ForegroundColor = rtAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; System.Console.Write($"{rtAllocPct,+9:+0;-0}%"); System.Console.ResetColor(); System.Console.WriteLine(" │"); } // Closing line: merged on left (─ between cols 1-5), ┴ on the right (cols 6-13 boundary, 8 unmerged cells). System.Console.WriteLine($"└{"─".PadRight(6, '─')}─{"─".PadRight(13, '─')}─{"─".PadRight(24, '─')}─{"─".PadRight(14, '─')}─{"─".PadRight(10, '─')}┴{"─".PadRight(16, '─')}┴{"─".PadRight(10, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┘"); } // Summary: Best serializer for each category System.Console.WriteLine("\n"); System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); System.Console.WriteLine("║ SUMMARY: WINNERS ║"); System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); System.Console.WriteLine($"\n{"Category",-20} │ {"Winner",-40} │ {"Avg Value",-18}"); System.Console.WriteLine($"{"─".PadRight(20, '─')}─┼─{"─".PadRight(40, '─')}─┼─{"─".PadRight(18, '─')}"); // Fastest Serialize — round-trip-only serializers (NamedPipe etc.) excluded: // their Serialize() captures the full round-trip and isn't comparable to a pure Ser metric. var fastestSer = results.Where(r => r.SerializeTimeMs > 0 && !r.IsRoundTripOnly) .GroupBy(r => r.SerializerName) .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => SerPerOp(r)) }) .OrderBy(x => x.AvgPerOp) .FirstOrDefault(); if (fastestSer != null) System.Console.WriteLine($"{"Fastest Serialize",-20} │ {fastestSer.Name,-40} │ {fastestSer.AvgPerOp,12:F2} µs/op"); // Fastest Deserialize — round-trip-only serializers excluded (their Deserialize() is a no-op). var fastestDes = results.Where(r => r.DeserializeTimeMs > 0 && !r.IsRoundTripOnly) .GroupBy(r => r.SerializerName) .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => DesPerOp(r)) }) .OrderBy(x => x.AvgPerOp) .FirstOrDefault(); if (fastestDes != null) System.Console.WriteLine($"{"Fastest Deserialize",-20} │ {fastestDes.Name,-40} │ {fastestDes.AvgPerOp,12:F2} µs/op"); // Smallest Size var smallestSize = results .GroupBy(r => r.SerializerName) .Select(g => new { Name = g.Key, AvgSize = g.Average(r => r.SerializedSize) }) .OrderBy(x => x.AvgSize) .FirstOrDefault(); if (smallestSize != null) System.Console.WriteLine($"{"Smallest Size",-20} │ {smallestSize.Name,-40} │ {smallestSize.AvgSize,15:F0} B"); // Fastest Round-trip — iter-independent per-op average. var fastestRt = results.Where(r => r.RoundTripTimeMs > 0) .GroupBy(r => r.SerializerName) .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => RtPerOp(r)) }) .OrderBy(x => x.AvgPerOp) .FirstOrDefault(); if (fastestRt != null) System.Console.WriteLine($"{"Fastest Round-trip",-20} │ {fastestRt.Name,-40} │ {fastestRt.AvgPerOp,12:F2} µs/op"); // Overall AcBinary (SGen) vs MemoryPack comparison. var memPackSerResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); var memPackDesResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); var memPackRtResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); var acBinarySerResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); var acBinaryDesResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); var acBinaryRtResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); // Skip comparison if no data available if (memPackRtResults.Count == 0 || acBinaryRtResults.Count == 0) { System.Console.WriteLine(); System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); System.Console.WriteLine(" (Comparison requires both serialize and deserialize data)"); return; } // All averages are over per-op µs (iter-independent). Three aggregations per metric. var sizeAcResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); var sizeMpResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); var serStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, SerPerOp); var desStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, DesPerOp); var rtStats = ComputeOverallStats(acBinaryRtResults, memPackRtResults, RtPerOp); var sizeStats = ComputeOverallStats(sizeAcResults, sizeMpResults, r => r.SerializedSize); var serAllocStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, r => r.SerializeAllocBytesPerOp); var desAllocStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, r => r.DeserializeAllocBytesPerOp); System.Console.WriteLine(); System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); WriteOverallLine("Serialize", "µs/op", serStats); WriteOverallLine("Deserialize", "µs/op", desStats); WriteOverallLine("Round-trip", "µs/op", rtStats); WriteOverallLine("Size", "B", sizeStats, "F0"); WriteOverallLine("Ser Alloc", "B/op", serAllocStats, "F0"); WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0"); } internal static void SaveResults(List results, List testDataSets) { Directory.CreateDirectory(Configuration.ResultsDirectory); var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}"; var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log"); var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output"); // Save binary output to separate .output file var largeTestData = testDataSets.FirstOrDefault(t => t.Name.StartsWith("Large")); if (largeTestData != null) { var outputSb = new StringBuilder(); outputSb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); outputSb.AppendLine("║ SERIALIZED BINARY OUTPUT ║"); outputSb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); outputSb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); outputSb.AppendLine(); outputSb.AppendLine("=== SERIALIZED BYTES: Large (5x5x5x10) - AcBinary (Default) ==="); var serializedBytes = AcBinarySerializer.Serialize(largeTestData.Order, AcBinarySerializerOptions.Default); outputSb.AppendLine($"Size: {serializedBytes.Length:N0} bytes"); outputSb.AppendLine(); outputSb.AppendLine("Hex dump:"); outputSb.AppendLine(FormatHexDump(serializedBytes)); File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom); System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}"); } // Save benchmark results to .log file var sb = new StringBuilder(); sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║"); sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║"); sb.AppendLine($"║ Charset: {Configuration.GetCurrentCharsetName()}".PadRight(100) + "║"); sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║"); sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║"); sb.AppendLine($"║ Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}".PadRight(100) + "║"); sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); sb.AppendLine(); // Serializer options summary var optionsMap = results .Where(r => r.OptionsDescription != null) .Select(r => (r.SerializerName, r.OptionsDescription!)) .Distinct() .ToList(); if (optionsMap.Count > 0) { sb.AppendLine("=== SERIALIZER OPTIONS ==="); foreach (var (name, opts) in optionsMap) sb.AppendLine($" {name}: {opts}"); sb.AppendLine(); } // CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely. sb.AppendLine("=== RAW DATA (CSV) ==="); sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes"); foreach (var testData in testDataSets) { var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList(); foreach (var result in testResults) { sb.AppendLine($"{result.TestDataName},{result.Engine},{result.IoMode},{result.DispatchMode},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}"); } } sb.AppendLine(); // Formatted results sb.AppendLine("=== FORMATTED RESULTS BY TEST DATA ==="); sb.AppendLine("(►) = Highlighted: MemoryPack (Byte[]) (baseline) and AcBinary (Byte[])"); sb.AppendLine(); foreach (var testData in testDataSets) { // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); sb.AppendLine(); sb.AppendLine($"--- {testData.DisplayName} ---"); sb.AppendLine($"{"#",-4} {"Serializer",-42} {"Size B",-12} {"Setup S/D KB",-14} {"Ser µs/op",-12} {"Des µs/op",-12} {"RT µs/op",-12} {"SerAlc KB",-11} {"DesAlc KB",-11}"); sb.AppendLine(new string('-', 140)); var rank = 1; foreach (var result in testResults) { var isHighlighted = ((result.Engine == Configuration.EngineMemoryPack || result.Engine == Configuration.EngineAcBinary) && result.IoMode == Configuration.IoByteArray); var prefix = isHighlighted ? "► " : " "; var size = $"{result.SerializedSize:N0}"; var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; sb.AppendLine($"{rank++,2} {prefix}{result.SerializerName,-40} {size,-12} {setup,-14} {ser,-12} {des,-12} {rt,-12} {serAlloc,-11} {desAlloc,-11}"); } // Summary row for this test data (vs MemoryPack) if (memPackResult != null && acBinaryResult != null) { var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; sb.AppendLine($" AcBinary (Byte[]) vs MemoryPack (Byte[]): Size {sizePct:+0;-0}% │ Ser {serPct:+0;-0}% │ Des {desPct:+0;-0}% │ RT {rtPct:+0;-0}%"); } } // Summary comparison (vs MemoryPack) sb.AppendLine(); sb.AppendLine("=== AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ==="); var memPackSerResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); var memPackDesResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); var memPackRtResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); var acBinarySerResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); var acBinaryDesResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); var acBinaryRtResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); // Skip comparison block if either side has no Byte[] data if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0) { sb.AppendLine(" (Comparison requires both serialize and deserialize data)"); File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); SaveLlmResults(llmFilePathEarly, results, testDataSets); return; } var sizeAcResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); var sizeMpResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, SerPerOp)); AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, r => r.SerializeAllocBytesPerOp), "F0"); AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, DesPerOp)); AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, r => r.DeserializeAllocBytesPerOp), "F0"); AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp)); AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0"); File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); // Save LLM-optimized results var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); SaveLlmResults(llmFilePath, results, testDataSets); } internal static void SaveLlmResults(string filePath, List results, List testDataSets) { var sb = new StringBuilder(); var testTypeName = testDataSets.FirstOrDefault()?.TypeName ?? "unknown"; sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); sb.AppendLine($"Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | TestType: {testTypeName} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%"); sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup"); // Options summary var optionsMap = results .Where(r => r.OptionsDescription != null) .Select(r => (r.SerializerName, r.OptionsDescription!)) .Distinct() .ToList(); if (optionsMap.Count > 0) { sb.AppendLine(); sb.AppendLine("## Options"); sb.AppendLine(); foreach (var (name, opts) in optionsMap) sb.AppendLine($"- **{name}**: {opts}"); } sb.AppendLine(); sb.AppendLine("## Results"); sb.AppendLine(); sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des"); sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---"); foreach (var testData in testDataSets) { var testResults = results .Where(r => r.TestDataName == testData.DisplayName) .OrderBy(RtPerOp) .ToList(); foreach (var r in testResults) { var inv = CultureInfo.InvariantCulture; var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-"; var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-"; var rt = r.RoundTripTimeMs > 0 ? (r.IsRoundTripOnly ? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv) : RtPerOp(r).ToString("F2", inv)) : "-"; var serAlloc = r.SerializeTimeMs > 0 ? ToKilobytes(r.SerializeAllocBytesPerOp).ToString("F2", inv) : "-"; var desAlloc = r.DeserializeTimeMs > 0 ? ToKilobytes(r.DeserializeAllocBytesPerOp).ToString("F2", inv) : "-"; var rtAlloc = r.RoundTripAllocBytesPerOp > 0 ? ToKilobytes(r.RoundTripAllocBytesPerOp).ToString("F2", inv) : "-"; var setupAlloc = $"{ToKilobytes(r.SetupSerializeAllocBytes).ToString("F2", inv)} / {ToKilobytes(r.SetupDeserializeAllocBytes).ToString("F2", inv)}"; var iterCol = r.IsRoundTripOnly ? r.RoundTripIterations.ToString(inv) : $"{(r.SerializeIterations > 0 ? r.SerializeIterations.ToString(inv) : "-")} / {(r.DeserializeIterations > 0 ? r.DeserializeIterations.ToString(inv) : "-")}"; sb.AppendLine($"{r.TestDataName} | {r.Engine} | {r.IoMode} | {r.DispatchMode} | {r.OptionsPreset} | {r.SerializedSize} | {ser} | {des} | {rt} | {serAlloc} | {desAlloc} | {rtAlloc} | {setupAlloc} | {iterCol}"); } } // Overall AcBinary (SGen, Byte[]) vs MemoryPack (Byte[]) comparison var memPackByteArrayResults = results.Where(r => r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray).ToList(); var acBinarySGenByteArrayResults = results.Where(r => r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen).ToList(); var memPackSerResultsLlm = memPackByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); var memPackDesResultsLlm = memPackByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); var memPackRtResultsLlm = memPackByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); var acBinarySerResultsLlm = acBinarySGenByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); var acBinaryDesResultsLlm = acBinarySGenByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); var acBinaryRtResultsLlm = acBinarySGenByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); if (memPackRtResultsLlm.Count > 0 && acBinaryRtResultsLlm.Count > 0) { sb.AppendLine(); sb.AppendLine("## Overall: AcBinary (Byte[], SGen) vs MemoryPack (Byte[])"); sb.AppendLine(); sb.AppendLine("Three aggregations of per-cell results: **arith** = arithmetic mean of µs/op (magnitude-weighted, Large cell dominates); **geo** = geometric mean of per-cell ratios (each cell weighted equally); **median** = median of per-cell ratios (outlier-resistant). Negative % = AcBinary faster/smaller; positive % = MemPack faster/smaller. The geo/median variants surface when a single big cell skews the arithmetic mean."); sb.AppendLine(); sb.AppendLine("```"); AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, SerPerOp)); AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, r => r.SerializeAllocBytesPerOp), "F0"); AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, DesPerOp)); AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, r => r.DeserializeAllocBytesPerOp), "F0"); AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResultsLlm, memPackRtResultsLlm, RtPerOp)); AppendOverallLine(sb, "Size", "B", ComputeOverallStats(acBinarySGenByteArrayResults, memPackByteArrayResults, r => r.SerializedSize), "F0"); sb.AppendLine("```"); } File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom); System.Console.WriteLine($"✓ LLM results saved to: {filePath}"); } /// /// Formats byte array as hex dump with offset, hex values, and ASCII representation. /// internal static string FormatHexDump(byte[] bytes, int bytesPerLine = 16) { var sb = new StringBuilder(); for (var i = 0; i < bytes.Length; i += bytesPerLine) { // Offset sb.Append($"{i:X8} "); // Hex bytes for (var j = 0; j < bytesPerLine; j++) { if (i + j < bytes.Length) sb.Append($"{bytes[i + j]:X2} "); else sb.Append(" "); if (j == 7) sb.Append(' '); // Extra space in middle } sb.Append(" |"); // ASCII representation for (var j = 0; j < bytesPerLine && i + j < bytes.Length; j++) { var b = bytes[i + j]; sb.Append(b is >= 32 and < 127 ? (char)b : '.'); } sb.AppendLine("|"); } return sb.ToString(); } }