diff --git a/AyCode.Core.Serializers.Console/BenchmarkResult.cs b/AyCode.Core.Serializers.Console/BenchmarkResult.cs new file mode 100644 index 0000000..7f80c20 --- /dev/null +++ b/AyCode.Core.Serializers.Console/BenchmarkResult.cs @@ -0,0 +1,74 @@ +namespace AyCode.Core.Serializers.Console; + +/// +/// Per-cell benchmark result row. Populated by the benchmark execution loop in +/// BenchmarkLoop.RunBenchmarksForTestData; consumed by the output formatters in +/// Output (console table + .log + .LLM file writers). Pure DTO — no behaviour. +/// +internal sealed class BenchmarkResult +{ + public string TestDataName { get; set; } = ""; + public string Engine { get; set; } = ""; + public string IoMode { get; set; } = ""; + public string DispatchMode { get; set; } = ""; + public string OptionsPreset { get; set; } = ""; + + /// True if Serialize() captures a full round-trip and Deserialize() is a no-op + /// (single-use streaming transports like NamedPipe). Excluded from "Fastest Serialize" / "Fastest Deserialize" + /// winners rankings; still ranked in "Fastest Round-trip". Display-side: Ser µs/op / SerAlloc / Des µs/op / DesAlloc + /// all show "N/A" since they were never measured separately; RT µs/op / RT Alloc carry the full round-trip values. + public bool IsRoundTripOnly { get; set; } + + /// Synthesized display name for backwards compatibility / single-string-row scenarios. Includes DispatchMode so SGen and Runtime variants of the same preset don't collide in grouping (e.g. SUMMARY: WINNERS). + public string SerializerName => $"{Engine} ({IoMode}, {OptionsPreset}, {DispatchMode})"; + + public string? OptionsDescription { get; set; } + public int SerializedSize { get; set; } + public double SerializeTimeMs { get; set; } + public double DeserializeTimeMs { get; set; } + + // Per-sample min/max alongside the median (median is the *Time*Ms field above). Surfaces + // inter-sample range — the visible noise floor for the row. 0 when the operation was skipped + // (mode != "all"/"ser"/"des") or when a single-sample fast path was used (min == max == median). + public double SerializeTimeMinMs { get; set; } + public double SerializeTimeMaxMs { get; set; } + public double DeserializeTimeMinMs { get; set; } + public double DeserializeTimeMaxMs { get; set; } + + // Sample-population stddev (ms). Used by FormatMicrosWithRange to compute CV (stddev/mean) + // and emit the ⚠️ marker on rows above Configuration.UnstableCVThreshold. 0 in single-sample mode. + public double SerializeTimeStdDevMs { get; set; } + public double DeserializeTimeStdDevMs { get; set; } + + // Per-row adaptive iteration count (post-CalibrateIterations). Each Ser and Des function calibrates + // independently to land its sample window at ~Configuration.TargetSampleMs; per-op µs is then iter-independent + // (`SerializeTimeMs / SerializeIterations * 1000`). For round-trip-only rows (NamedPipe etc.), + // RoundTripIterations carries the calibrated iter count; SerializeIterations and DeserializeIterations + // stay 0 (Ser and Des are not separately measurable on those rows). + public int SerializeIterations { get; set; } + public int DeserializeIterations { get; set; } + public int RoundTripIterations { get; set; } + + public long SerializeAllocBytesPerOp { get; set; } + public long DeserializeAllocBytesPerOp { get; set; } + public long SetupSerializeAllocBytes { get; set; } + public long SetupDeserializeAllocBytes { get; set; } + + /// Total round-trip time. For in-memory benchmarks: synthesized so that + /// RoundTripTimeMs / RoundTripIterations yields the correct SerPerOp + DesPerOp µs/op + /// (necessary because Ser and Des may have different iter counts post-calibration). + /// For round-trip-only benchmarks (NamedPipe etc.): the directly-measured pipe round-trip time. + public double RoundTripTimeMs { get; set; } + + // Round-trip min/max + stddev — only populated for round-trip-only benchmarks (NamedPipe etc.) where + // RT is directly measured. For in-memory rows RT = Ser + Des, which has no single-sample + // distribution; surface Ser/Des range separately instead. + public double RoundTripTimeMinMs { get; set; } + public double RoundTripTimeMaxMs { get; set; } + public double RoundTripTimeStdDevMs { get; set; } + + /// Total round-trip allocation per op. For in-memory benchmarks: SerializeAlloc + DeserializeAlloc. + /// For round-trip-only benchmarks: process-wide allocation measured via + /// (covers ALL threads — client, server-drain, channel internals — not just the caller). + public long RoundTripAllocBytesPerOp { get; set; } +} diff --git a/AyCode.Core.Serializers.Console/Configuration.cs b/AyCode.Core.Serializers.Console/Configuration.cs index e1b72d0..d7290de 100644 --- a/AyCode.Core.Serializers.Console/Configuration.cs +++ b/AyCode.Core.Serializers.Console/Configuration.cs @@ -1,5 +1,6 @@ using AyCode.Core.Serializers.Attributes; using AyCode.Core.Serializers.Binaries; +using AyCode.Core.Tests.TestModels; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; @@ -130,4 +131,26 @@ internal static class Configuration idTracking: attrs.All(a => a!.EnableIdTrackingFeature), propertyFilter: attrs.All(a => a!.EnablePropertyFilterFeature)); } + + /// + /// Returns a human-readable name for the currently-active BenchmarkTestDataProvider.LongStringSuffix + /// charset. Returns "Custom" when the suffix doesn't match any of the predefined + /// constants. Used in menu state display, console run header, and + /// the .LLM / .log output headers so per-charset bench files are self-documenting. + /// + internal static string GetCurrentCharsetName() + { + var s = BenchmarkTestDataProvider.LongStringSuffix; + + return s switch + { + CharsetSuffixes.Latin1FixAscii => "Latin1FixAscii", + CharsetSuffixes.Latin1Short => "Latin1Short", + CharsetSuffixes.Latin1Long => "Latin1Long", + CharsetSuffixes.CjkBmp => "CjkBmp", + CharsetSuffixes.Cyrillic => "Cyrillic", + CharsetSuffixes.Mixed => "Mixed", + _ => "Custom" + }; + } } diff --git a/AyCode.Core.Serializers.Console/Menu.cs b/AyCode.Core.Serializers.Console/Menu.cs new file mode 100644 index 0000000..48b6ae9 --- /dev/null +++ b/AyCode.Core.Serializers.Console/Menu.cs @@ -0,0 +1,222 @@ +using AyCode.Core.Serializers.Binaries; +using AyCode.Core.Tests.TestModels; + +namespace AyCode.Core.Serializers.Console; + +/// +/// Interactive console menu for the benchmark application. Shown when the user runs without CLI args: +/// top-level layer/mode selection + nested Settings sub-menus (iteration counts, wire mode, charset). +/// All settings mutate in place; the menu loop returns control to the +/// caller (Program.Main) once the user picks a benchmark layer or quits. +/// +internal static class Menu +{ + /// + /// Interactive menu shown when no CLI args. Returns the layer keyword (core/comprehensive/edge/all) or null on Quit. + /// Loops on settings-changes ([S]) — user is returned to this menu after modifying iteration counts. + /// + internal static (string layer, string serializerMode)? ShowInteractiveMenu() + { + while (true) + { + System.Console.WriteLine(); + System.Console.WriteLine("╔══════════════════════════════════════════════════════════╗"); + System.Console.WriteLine("║ AcBinary Benchmark Suite ║"); + System.Console.WriteLine("╚══════════════════════════════════════════════════════════╝"); + System.Console.WriteLine(); + System.Console.WriteLine("Select benchmark layer:"); + System.Console.WriteLine(); + System.Console.WriteLine(" [1] Core — daily iteration"); + System.Console.WriteLine(" [2] Comprehensive — release validation"); + System.Console.WriteLine(" [3] Edge cases — refactor verification"); + System.Console.WriteLine(" [A] All layers"); + System.Console.WriteLine(" [F] FastestByte — AcBinary FastMode Byte[] vs MemoryPack Byte[] only (tight optimization loop)"); + System.Console.WriteLine(" [P] AsyncPipe — streaming I/O isolation (only AsyncPipe, all test data)"); + System.Console.WriteLine($" [S] Settings — Iteration / WireMode (current: {Configuration.SelectedWireMode})"); + System.Console.WriteLine(" [Q] Quit"); + System.Console.Write("\nSelection: "); + + var key = System.Console.ReadKey(intercept: false).KeyChar; + System.Console.WriteLine(); + + switch (char.ToLower(key)) + { + case '1': return ("core", "standard"); + case '2': return ("comprehensive", "standard"); + case '3': return ("edge", "standard"); + case 'a': return ("all", "standard"); + case 'f': return ("all", "fastestbyte"); + case 'p': return ("all", "asyncpipe"); + case 's': + ShowSettingsMenu(); + continue; // re-display the main menu after settings update + case 'q': return null; + default: return ("all", "standard"); + } + } + } + + /// + /// Settings sub-menu — dispatches to per-area sub-menus (iteration counts, wire mode, charset). + /// Returns to the caller (which re-displays the main menu) when [B]ack is pressed. + /// + private static void ShowSettingsMenu() + { + while (true) + { + System.Console.WriteLine(); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine("Settings"); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine(" [1] Iteration — Warmup / Iterations / Samples"); + System.Console.WriteLine($" [2] WireMode — current: {Configuration.SelectedWireMode}"); + System.Console.WriteLine($" [3] Charset — current: {Configuration.GetCurrentCharsetName()}"); + System.Console.WriteLine(" [B] Back"); + System.Console.Write("\nSelection: "); + + var key = System.Console.ReadKey(intercept: false).KeyChar; + System.Console.WriteLine(); + + switch (char.ToLower(key)) + { + case '1': + ShowIterationSettingsMenu(); + break; + case '2': + ShowWireModeSettingsMenu(); + break; + case '3': + ShowCharsetSettingsMenu(); + break; + case 'b': + return; + default: + continue; + } + } + } + + private static void ShowCharsetSettingsMenu() + { + while (true) + { + System.Console.WriteLine(); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine("Charset settings — long-string suffix profile"); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine($"Current: {Configuration.GetCurrentCharsetName()}"); + System.Console.WriteLine(); + System.Console.WriteLine(" [1] Latin1FixAscii — empty suffix; short FixStr-fast-path stress (Latin1 baseline values stay short)"); + System.Console.WriteLine(" [2] Latin1Short — \" árvíztűrő tükörfúrógép\" (~24 char Hungarian mixed)"); + System.Console.WriteLine(" [3] Latin1Long — ~47-char Latin1 mixed (default; exceeds FixStr boundary)"); + System.Console.WriteLine(" [4] CjkBmp — CJK BMP (long 3-byte runs)"); + System.Console.WriteLine(" [5] Cyrillic — Russian Cyrillic (long 2-byte runs)"); + System.Console.WriteLine(" [6] Mixed — Hungarian + CJK + Cyrillic + emoji (full-spectrum + surrogate pairs)"); + System.Console.WriteLine(" [B] Back"); + System.Console.Write("\nSelection: "); + + var key = System.Console.ReadKey(intercept: false).KeyChar; + System.Console.WriteLine(); + + switch (char.ToLower(key)) + { + case '1': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1FixAscii; + System.Console.WriteLine("✓ Charset set to Latin1FixAscii"); + return; + case '2': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1Short; + System.Console.WriteLine("✓ Charset set to Latin1Short"); + return; + case '3': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1Long; + System.Console.WriteLine("✓ Charset set to Latin1Long"); + return; + case '4': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.CjkBmp; + System.Console.WriteLine("✓ Charset set to CjkBmp"); + return; + case '5': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Cyrillic; + System.Console.WriteLine("✓ Charset set to Cyrillic"); + return; + case '6': + BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Mixed; + System.Console.WriteLine("✓ Charset set to Mixed"); + return; + case 'b': + return; + default: + continue; + } + } + } + + private static void ShowIterationSettingsMenu() + { + System.Console.WriteLine(); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine("Iteration settings — press Enter to keep current value"); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine(); + + Configuration.WarmupIterations = PromptInt("WarmupIterations", Configuration.WarmupIterations, min: 0); + Configuration.TestIterations = PromptInt("TestIterations ", Configuration.TestIterations, min: 1); + Configuration.BenchmarkSamples = PromptInt("BenchmarkSamples", Configuration.BenchmarkSamples, min: 1); + + System.Console.WriteLine(); + System.Console.WriteLine($"✓ Iteration settings updated: Warmup={Configuration.WarmupIterations} | Iterations={Configuration.TestIterations} | Samples={Configuration.BenchmarkSamples}"); + } + + private static void ShowWireModeSettingsMenu() + { + while (true) + { + System.Console.WriteLine(); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine("WireMode settings"); + System.Console.WriteLine("─────────────────────────────────────────────"); + System.Console.WriteLine($"Current: {Configuration.SelectedWireMode}"); + System.Console.WriteLine(" [1] Compact"); + System.Console.WriteLine(" [2] Fast"); + System.Console.WriteLine(" [B] Back"); + System.Console.Write("\nSelection: "); + + var key = System.Console.ReadKey(intercept: false).KeyChar; + System.Console.WriteLine(); + + switch (char.ToLower(key)) + { + case '1': + Configuration.SelectedWireMode = WireMode.Compact; + System.Console.WriteLine("✓ WireMode set to Compact"); + return; + case '2': + Configuration.SelectedWireMode = WireMode.Fast; + System.Console.WriteLine("✓ WireMode set to Fast"); + return; + case 'b': + return; + default: + continue; + } + } + } + + /// + /// Prompts the user for an integer with a default (current value). Returns the current value if + /// the user presses Enter on empty input or if parsing fails / value is below the minimum. + /// + private static int PromptInt(string name, int currentValue, int min) + { + System.Console.Write($" {name} [{currentValue}]: "); + + var input = System.Console.ReadLine()?.Trim() ?? ""; + if (input.Length == 0) return currentValue; + + if (int.TryParse(input, out var newValue) && newValue >= min) return newValue; + + System.Console.WriteLine($" ! Invalid value (need int ≥ {min}) — keeping {currentValue}"); + return currentValue; + } +} diff --git a/AyCode.Core.Serializers.Console/Output.cs b/AyCode.Core.Serializers.Console/Output.cs new file mode 100644 index 0000000..bfa9280 --- /dev/null +++ b/AyCode.Core.Serializers.Console/Output.cs @@ -0,0 +1,692 @@ +using AyCode.Core.Serializers.Binaries; +using AyCode.Core.Tests.TestModels; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; + +namespace AyCode.Core.Serializers.Console; + +/// +/// Output formatters for the benchmark suite: per-row console table, .log file, .LLM markdown file. +/// Consumes rows produced by the benchmark loop and emits human-readable +/// + LLM-friendly outputs. Display-only helpers (per-op µs conversion, allocation KB formatting, +/// range/CV formatting, paired-aggregation stats) live here too — they are display-side concerns. +/// +internal static class Output +{ + /// + /// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation + /// strategies so the reader can judge whether the headline delta is dominated by one large cell + /// (arithmetic mean) or representative of typical workload (geometric mean / median). + /// + /// Arithmetic mean of µs/op — magnitude-weighted; biased toward Large cell. + /// Geometric mean of per-cell ratios — magnitude-neutral; each cell weighted equally. + /// Median of per-cell ratios — outlier-resistant. + /// Arithmetic mean AcBinary value (µs/op or bytes). + /// Arithmetic mean MemPack value. + /// Number of paired cells contributing to the geo/median. + internal record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0; + + // Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever + // averaging or comparison happens across rows with potentially different iter counts (Winners summary, + // Overall comparison, per-cell summary row). Keeping these as methods rather than properties on + // BenchmarkResult preserves the result-as-data-bag distinction. + internal static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations); + internal static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations); + internal static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations); + + /// + /// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can + /// render compact F2 KB values (e.g. 4.05 KB instead of 4,144 B) — header carries + /// the unit so per-row entries stay numbers-only. CSV / raw-data outputs keep the precise byte + /// integers untouched. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static double ToKilobytes(long bytes) => bytes / 1024.0; + + /// + /// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison + /// across paired cells (joined by TestDataName). Per-cell pairing is required for the + /// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats. + /// Returns null when no paired cell has a valid value. + /// + internal static OverallStats? ComputeOverallStats(List acResults, List mpResults, Func getValue) + { + if (acResults.Count == 0 || mpResults.Count == 0) return null; + + var pairs = (from ac in acResults + join mp in mpResults on ac.TestDataName equals mp.TestDataName + let acV = getValue(ac) + let mpV = getValue(mp) + where acV > 0 && mpV > 0 + select (ac: acV, mp: mpV)).ToList(); + + if (pairs.Count == 0) return null; + + var acAvg = pairs.Average(p => p.ac); + var mpAvg = pairs.Average(p => p.mp); + var ratios = pairs.Select(p => p.ac / p.mp).ToList(); + + // Geometric mean: exp(avg(ln(ratios))) — numerically stable vs Π ratios then ^(1/N). + var geoMean = Math.Exp(ratios.Sum(Math.Log) / ratios.Count); + + // Median (paired-ratio): for even N use the midpoint of the two middle values. + var sorted = ratios.OrderBy(r => r).ToList(); + var median = sorted.Count % 2 == 1 + ? sorted[sorted.Count / 2] + : (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2.0; + + return new OverallStats( + ArithMeanPct: (acAvg / mpAvg - 1) * 100, + GeoMeanPct: (geoMean - 1) * 100, + MedianPct: (median - 1) * 100, + AcAvg: acAvg, + MpAvg: mpAvg, + CellCount: ratios.Count); + } + + /// + /// Formats a per-op micros value with its inter-sample range and CV-threshold marker as + /// "26.86 (24.5..29.1)" or "26.86 (24.5..29.1) ⚠️5.2%". Median first, range in parentheses, + /// CV warning suffix only when CV > . When min == max == median + /// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter. + /// All time inputs are total-batch milliseconds; is the per-row iter + /// count (post-adaptive-calibration). + /// + internal static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, CultureInfo inv) + { + var med = ToPerOpMicros(medianMs, iterations); + // No range data (single-sample fast path) — surface as bare median, identical to the prior format. + if (minMs <= 0 && maxMs <= 0) return med.ToString("F2", inv); + if (minMs >= medianMs && maxMs <= medianMs) return med.ToString("F2", inv); + + var min = ToPerOpMicros(minMs, iterations); + var max = ToPerOpMicros(maxMs, iterations); + var range = $"{med.ToString("F2", inv)} ({min.ToString("F2", inv)}..{max.ToString("F2", inv)})"; + + // CV (coefficient of variation = stddev / mean) — flag rows above the unstable threshold so a + // small inter-engine delta on a high-CV row is easy to discount as noise. + if (medianMs > 0 && stdDevMs > 0) + { + var cv = stdDevMs / medianMs; + if (cv > Configuration.UnstableCVThreshold) + { + var cvPct = (cv * 100).ToString("F1", inv); + return $"{range} ⚠️{cvPct}%"; + } + } + + return range; + } + + /// + /// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`). + /// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block. + /// + internal static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", CultureInfo.InvariantCulture).PadLeft(6) + "%"; + + /// + /// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means. + /// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when + /// stats is null (no paired data). + /// + internal static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2") + { + if (stats == null) return; + + // Color follows geo-mean (the magnitude-neutral signal). The arith-mean column may show a + // different sign when a single big cell dominates — that's exactly the signal we want to surface. + System.Console.ForegroundColor = stats.GeoMeanPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.WriteLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} │ geo {FormatPctSigned(stats.GeoMeanPct)} │ median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); + System.Console.ResetColor(); + } + + /// + /// Same as but appends to a (no color). + /// Used by the .log and .LLM file writers. + /// + internal static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2") + { + if (stats == null) return; + sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); + } + + internal static void PrintResult(BenchmarkResult result) + { + // Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op). + var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A"; + var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result),7:F2}" : " N/A"; + var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp),7:F2}" : " N/A"; + var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp),7:F2}" : " N/A"; + System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)"); + } + + internal static void PrintGroupedResults(List results, List testDataSets) + { + System.Console.WriteLine("\n"); + System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); + System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║"); + System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); + + // Print serializer options + var optionsMap = results + .Where(r => r.OptionsDescription != null) + .Select(r => (r.SerializerName, r.OptionsDescription!)) + .Distinct() + .ToList(); + + if (optionsMap.Count > 0) + { + System.Console.WriteLine(); + System.Console.WriteLine(" Serializer Options:"); + foreach (var (name, opts) in optionsMap) + System.Console.WriteLine($" {name}: {opts}"); + } + + foreach (var testData in testDataSets) + { + // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. + var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); + // Baseline switched MessagePack → MemoryPack: MemoryPack is the SOTA performance leader. + var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); + // Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated). + // The Runtime variant is shown alongside in the table for context, not used as the headline number. + var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); + + System.Console.WriteLine($"\n┌─ {testData.DisplayName} ─".PadRight(172, '─') + "┐"); + // Header-only units; per-row entries are numbers (µs/op for time, KB/op for alloc, KB pair "ser / des" for Setup, B for Size). + System.Console.WriteLine($"│ {"#",-4} │ {"Engine",-11} │ {"Options",-22} │ {"IO",-12} │ {"Mode",-8} │ {"Setup S/D KB",-14} │ {"Size B",-8} │ {"Ser µs/op",-10} │ {"SerAlc KB",-10} │ {"Des µs/op",-10} │ {"DesAlc KB",-10} │ {"RT µs/op",-10} │ {"RTAlc KB",-10} │"); + System.Console.WriteLine($"├{"─".PadRight(6, '─')}┼{"─".PadRight(13, '─')}┼{"─".PadRight(24, '─')}┼{"─".PadRight(14, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); + + var rank = 1; + foreach (var result in testResults) + { + var size = $"{result.SerializedSize:N0}"; + var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; + var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; + var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; + var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; + var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; + var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; + var rtAlloc = result.RoundTripAllocBytesPerOp > 0 ? $"{ToKilobytes(result.RoundTripAllocBytesPerOp):F2}" : "N/A"; + + // Highlight MemoryPack baseline (any Byte[]) and AcBinary headline contender (Byte[] + SGen) with win/lose colors. + // The AcBinary Byte[]+Runtime variant is shown unhighlighted — it's contextual (SGen speed-up reference), not the headline. + var isHighlighted = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray) + || (result.Engine == Configuration.EngineAcBinary && result.IoMode == Configuration.IoByteArray && result.DispatchMode == Configuration.ModeSGen); + + var prefix = isHighlighted ? "│►" : "│ "; + var suffix = isHighlighted ? "◄│" : " │"; + + // Color logic: Green = winner (faster), Red = loser (slower) + if (isHighlighted && memPackResult != null && acBinaryResult != null) + { + var isMemPack = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray); + var memPackFaster = RtPerOp(memPackResult) < RtPerOp(acBinaryResult); + + if (isMemPack) + { + System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Green : ConsoleColor.Red; + } + else + { + System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Red : ConsoleColor.Green; + } + } + + System.Console.WriteLine($"{prefix}{rank++,4} │ {result.Engine,-11} │ {result.OptionsPreset,-22} │ {result.IoMode,-12} │ {result.DispatchMode,-8} │ {setup,14} │ {size,8} │ {ser,10} │ {serAlloc,10} │ {des,10} │ {desAlloc,10} │ {rt,10} │ {rtAlloc,10}{suffix}"); + + if (isHighlighted) + { + System.Console.ResetColor(); + } + } + + // Footer row: AcBinary (Byte[]) vs MemoryPack (Byte[]) comparison per column + if (memPackResult != null && acBinaryResult != null) + { + var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; + // Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows. + var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; + var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; + var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; + var serAllocPct = memPackResult.SerializeAllocBytesPerOp > 0 ? (acBinaryResult.SerializeAllocBytesPerOp / (double)memPackResult.SerializeAllocBytesPerOp - 1) * 100 : 0; + var desAllocPct = memPackResult.DeserializeAllocBytesPerOp > 0 ? (acBinaryResult.DeserializeAllocBytesPerOp / (double)memPackResult.DeserializeAllocBytesPerOp - 1) * 100 : 0; + var rtAllocPct = memPackResult.RoundTripAllocBytesPerOp > 0 ? (acBinaryResult.RoundTripAllocBytesPerOp / (double)memPackResult.RoundTripAllocBytesPerOp - 1) * 100 : 0; + + // Footer separator: merge first 5 cols (#, Engine, Options, IO, Mode) → comparison label; + // remaining 8 cols stay aligned (Setup S/D KB, Size, Ser µs/op, SerAlc KB, Des µs/op, DesAlc KB, RT µs/op, RTAlc KB). + System.Console.WriteLine($"├{"─".PadRight(6, '─')}┴{"─".PadRight(13, '─')}┴{"─".PadRight(24, '─')}┴{"─".PadRight(14, '─')}┴{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); + // Merged label cell width = 4 + 11 + 22 + 12 + 8 + 4*3 (dropped separators) = 69 + System.Console.Write($"│ {"► AcBinary (Byte[]) vs MemoryPack (Byte[])",-69} │ "); + + // Setup S/D KB (n/a for Byte[] vs Byte[] — neither pre-allocates) + System.Console.Write($"{"—",14}"); + System.Console.Write(" │ "); + + // Size + System.Console.ForegroundColor = sizePct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{sizePct,+7:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Serialize + System.Console.ForegroundColor = serPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{serPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Serialize Alloc + System.Console.ForegroundColor = serAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{serAllocPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Deserialize + System.Console.ForegroundColor = desPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{desPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Deserialize Alloc + System.Console.ForegroundColor = desAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{desAllocPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Round-trip + System.Console.ForegroundColor = rtPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{rtPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.Write(" │ "); + + // Round-trip Alloc + System.Console.ForegroundColor = rtAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; + System.Console.Write($"{rtAllocPct,+9:+0;-0}%"); + System.Console.ResetColor(); + System.Console.WriteLine(" │"); + } + + // Closing line: merged on left (─ between cols 1-5), ┴ on the right (cols 6-13 boundary, 8 unmerged cells). + System.Console.WriteLine($"└{"─".PadRight(6, '─')}─{"─".PadRight(13, '─')}─{"─".PadRight(24, '─')}─{"─".PadRight(14, '─')}─{"─".PadRight(10, '─')}┴{"─".PadRight(16, '─')}┴{"─".PadRight(10, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┘"); + } + + // Summary: Best serializer for each category + System.Console.WriteLine("\n"); + System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); + System.Console.WriteLine("║ SUMMARY: WINNERS ║"); + System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); + + System.Console.WriteLine($"\n{"Category",-20} │ {"Winner",-40} │ {"Avg Value",-18}"); + System.Console.WriteLine($"{"─".PadRight(20, '─')}─┼─{"─".PadRight(40, '─')}─┼─{"─".PadRight(18, '─')}"); + + // Fastest Serialize — round-trip-only serializers (NamedPipe etc.) excluded: + // their Serialize() captures the full round-trip and isn't comparable to a pure Ser metric. + var fastestSer = results.Where(r => r.SerializeTimeMs > 0 && !r.IsRoundTripOnly) + .GroupBy(r => r.SerializerName) + .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => SerPerOp(r)) }) + .OrderBy(x => x.AvgPerOp) + .FirstOrDefault(); + + if (fastestSer != null) + System.Console.WriteLine($"{"Fastest Serialize",-20} │ {fastestSer.Name,-40} │ {fastestSer.AvgPerOp,12:F2} µs/op"); + + // Fastest Deserialize — round-trip-only serializers excluded (their Deserialize() is a no-op). + var fastestDes = results.Where(r => r.DeserializeTimeMs > 0 && !r.IsRoundTripOnly) + .GroupBy(r => r.SerializerName) + .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => DesPerOp(r)) }) + .OrderBy(x => x.AvgPerOp) + .FirstOrDefault(); + + if (fastestDes != null) + System.Console.WriteLine($"{"Fastest Deserialize",-20} │ {fastestDes.Name,-40} │ {fastestDes.AvgPerOp,12:F2} µs/op"); + + // Smallest Size + var smallestSize = results + .GroupBy(r => r.SerializerName) + .Select(g => new { Name = g.Key, AvgSize = g.Average(r => r.SerializedSize) }) + .OrderBy(x => x.AvgSize) + .FirstOrDefault(); + + if (smallestSize != null) + System.Console.WriteLine($"{"Smallest Size",-20} │ {smallestSize.Name,-40} │ {smallestSize.AvgSize,15:F0} B"); + + // Fastest Round-trip — iter-independent per-op average. + var fastestRt = results.Where(r => r.RoundTripTimeMs > 0) + .GroupBy(r => r.SerializerName) + .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => RtPerOp(r)) }) + .OrderBy(x => x.AvgPerOp) + .FirstOrDefault(); + + if (fastestRt != null) + System.Console.WriteLine($"{"Fastest Round-trip",-20} │ {fastestRt.Name,-40} │ {fastestRt.AvgPerOp,12:F2} µs/op"); + + // Overall AcBinary (SGen) vs MemoryPack comparison. + var memPackSerResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); + var memPackDesResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); + var memPackRtResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); + + var acBinarySerResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); + var acBinaryDesResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); + var acBinaryRtResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); + + // Skip comparison if no data available + if (memPackRtResults.Count == 0 || acBinaryRtResults.Count == 0) + { + System.Console.WriteLine(); + System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); + System.Console.WriteLine(" (Comparison requires both serialize and deserialize data)"); + return; + } + + // All averages are over per-op µs (iter-independent). Three aggregations per metric. + var sizeAcResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); + var sizeMpResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); + + var serStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, SerPerOp); + var desStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, DesPerOp); + var rtStats = ComputeOverallStats(acBinaryRtResults, memPackRtResults, RtPerOp); + var sizeStats = ComputeOverallStats(sizeAcResults, sizeMpResults, r => r.SerializedSize); + var serAllocStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, r => r.SerializeAllocBytesPerOp); + var desAllocStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, r => r.DeserializeAllocBytesPerOp); + + System.Console.WriteLine(); + System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); + + WriteOverallLine("Serialize", "µs/op", serStats); + WriteOverallLine("Deserialize", "µs/op", desStats); + WriteOverallLine("Round-trip", "µs/op", rtStats); + WriteOverallLine("Size", "B", sizeStats, "F0"); + WriteOverallLine("Ser Alloc", "B/op", serAllocStats, "F0"); + WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0"); + } + + internal static void SaveResults(List results, List testDataSets) + { + Directory.CreateDirectory(Configuration.ResultsDirectory); + + var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); + var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}"; + var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log"); + var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output"); + + // Save binary output to separate .output file + var largeTestData = testDataSets.FirstOrDefault(t => t.Name.StartsWith("Large")); + if (largeTestData != null) + { + var outputSb = new StringBuilder(); + outputSb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); + outputSb.AppendLine("║ SERIALIZED BINARY OUTPUT ║"); + outputSb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); + outputSb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); + outputSb.AppendLine(); + + outputSb.AppendLine("=== SERIALIZED BYTES: Large (5x5x5x10) - AcBinary (Default) ==="); + var serializedBytes = AcBinarySerializer.Serialize(largeTestData.Order, AcBinarySerializerOptions.Default); + outputSb.AppendLine($"Size: {serializedBytes.Length:N0} bytes"); + outputSb.AppendLine(); + outputSb.AppendLine("Hex dump:"); + outputSb.AppendLine(FormatHexDump(serializedBytes)); + + File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom); + System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}"); + } + + // Save benchmark results to .log file + var sb = new StringBuilder(); + sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); + sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║"); + sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); + sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║"); + sb.AppendLine($"║ Charset: {Configuration.GetCurrentCharsetName()}".PadRight(100) + "║"); + sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║"); + sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║"); + sb.AppendLine($"║ Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}".PadRight(100) + "║"); + sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); + sb.AppendLine(); + + // Serializer options summary + var optionsMap = results + .Where(r => r.OptionsDescription != null) + .Select(r => (r.SerializerName, r.OptionsDescription!)) + .Distinct() + .ToList(); + if (optionsMap.Count > 0) + { + sb.AppendLine("=== SERIALIZER OPTIONS ==="); + foreach (var (name, opts) in optionsMap) + sb.AppendLine($" {name}: {opts}"); + sb.AppendLine(); + } + + // CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely. + sb.AppendLine("=== RAW DATA (CSV) ==="); + sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes"); + + foreach (var testData in testDataSets) + { + var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList(); + foreach (var result in testResults) + { + sb.AppendLine($"{result.TestDataName},{result.Engine},{result.IoMode},{result.DispatchMode},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}"); + } + } + sb.AppendLine(); + + // Formatted results + sb.AppendLine("=== FORMATTED RESULTS BY TEST DATA ==="); + sb.AppendLine("(►) = Highlighted: MemoryPack (Byte[]) (baseline) and AcBinary (Byte[])"); + sb.AppendLine(); + + foreach (var testData in testDataSets) + { + // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. + var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); + var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); + var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); + + sb.AppendLine(); + sb.AppendLine($"--- {testData.DisplayName} ---"); + sb.AppendLine($"{"#",-4} {"Serializer",-42} {"Size B",-12} {"Setup S/D KB",-14} {"Ser µs/op",-12} {"Des µs/op",-12} {"RT µs/op",-12} {"SerAlc KB",-11} {"DesAlc KB",-11}"); + sb.AppendLine(new string('-', 140)); + + var rank = 1; + foreach (var result in testResults) + { + var isHighlighted = ((result.Engine == Configuration.EngineMemoryPack || result.Engine == Configuration.EngineAcBinary) && result.IoMode == Configuration.IoByteArray); + var prefix = isHighlighted ? "► " : " "; + + var size = $"{result.SerializedSize:N0}"; + var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; + var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; + var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; + var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; + var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; + var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; + + sb.AppendLine($"{rank++,2} {prefix}{result.SerializerName,-40} {size,-12} {setup,-14} {ser,-12} {des,-12} {rt,-12} {serAlloc,-11} {desAlloc,-11}"); + } + + // Summary row for this test data (vs MemoryPack) + if (memPackResult != null && acBinaryResult != null) + { + var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; + var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; + var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; + var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; + + sb.AppendLine($" AcBinary (Byte[]) vs MemoryPack (Byte[]): Size {sizePct:+0;-0}% │ Ser {serPct:+0;-0}% │ Des {desPct:+0;-0}% │ RT {rtPct:+0;-0}%"); + } + } + + // Summary comparison (vs MemoryPack) + sb.AppendLine(); + sb.AppendLine("=== AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ==="); + + var memPackSerResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); + var memPackDesResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); + var memPackRtResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); + + var acBinarySerResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); + var acBinaryDesResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); + var acBinaryRtResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); + + // Skip comparison block if either side has no Byte[] data + if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0) + { + sb.AppendLine(" (Comparison requires both serialize and deserialize data)"); + File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); + System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); + + var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); + SaveLlmResults(llmFilePathEarly, results, testDataSets); + return; + } + + var sizeAcResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); + var sizeMpResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); + + AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, SerPerOp)); + AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, r => r.SerializeAllocBytesPerOp), "F0"); + AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, DesPerOp)); + AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, r => r.DeserializeAllocBytesPerOp), "F0"); + AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp)); + AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0"); + + File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); + System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); + + // Save LLM-optimized results + var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); + SaveLlmResults(llmFilePath, results, testDataSets); + } + + internal static void SaveLlmResults(string filePath, List results, List testDataSets) + { + var sb = new StringBuilder(); + var testTypeName = testDataSets.FirstOrDefault()?.TypeName ?? "unknown"; + sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + sb.AppendLine($"Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | TestType: {testTypeName} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%"); + sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup"); + + // Options summary + var optionsMap = results + .Where(r => r.OptionsDescription != null) + .Select(r => (r.SerializerName, r.OptionsDescription!)) + .Distinct() + .ToList(); + if (optionsMap.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("## Options"); + sb.AppendLine(); + foreach (var (name, opts) in optionsMap) + sb.AppendLine($"- **{name}**: {opts}"); + } + + sb.AppendLine(); + sb.AppendLine("## Results"); + sb.AppendLine(); + sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des"); + sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---"); + + foreach (var testData in testDataSets) + { + var testResults = results + .Where(r => r.TestDataName == testData.DisplayName) + .OrderBy(RtPerOp) + .ToList(); + + foreach (var r in testResults) + { + var inv = CultureInfo.InvariantCulture; + var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-"; + var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-"; + var rt = r.RoundTripTimeMs > 0 + ? (r.IsRoundTripOnly + ? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv) + : RtPerOp(r).ToString("F2", inv)) + : "-"; + + var serAlloc = r.SerializeTimeMs > 0 ? ToKilobytes(r.SerializeAllocBytesPerOp).ToString("F2", inv) : "-"; + var desAlloc = r.DeserializeTimeMs > 0 ? ToKilobytes(r.DeserializeAllocBytesPerOp).ToString("F2", inv) : "-"; + var rtAlloc = r.RoundTripAllocBytesPerOp > 0 ? ToKilobytes(r.RoundTripAllocBytesPerOp).ToString("F2", inv) : "-"; + var setupAlloc = $"{ToKilobytes(r.SetupSerializeAllocBytes).ToString("F2", inv)} / {ToKilobytes(r.SetupDeserializeAllocBytes).ToString("F2", inv)}"; + + var iterCol = r.IsRoundTripOnly + ? r.RoundTripIterations.ToString(inv) + : $"{(r.SerializeIterations > 0 ? r.SerializeIterations.ToString(inv) : "-")} / {(r.DeserializeIterations > 0 ? r.DeserializeIterations.ToString(inv) : "-")}"; + sb.AppendLine($"{r.TestDataName} | {r.Engine} | {r.IoMode} | {r.DispatchMode} | {r.OptionsPreset} | {r.SerializedSize} | {ser} | {des} | {rt} | {serAlloc} | {desAlloc} | {rtAlloc} | {setupAlloc} | {iterCol}"); + } + } + + // Overall AcBinary (SGen, Byte[]) vs MemoryPack (Byte[]) comparison + var memPackByteArrayResults = results.Where(r => r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray).ToList(); + var acBinarySGenByteArrayResults = results.Where(r => r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen).ToList(); + var memPackSerResultsLlm = memPackByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); + var memPackDesResultsLlm = memPackByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); + var memPackRtResultsLlm = memPackByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); + var acBinarySerResultsLlm = acBinarySGenByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); + var acBinaryDesResultsLlm = acBinarySGenByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); + var acBinaryRtResultsLlm = acBinarySGenByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); + + if (memPackRtResultsLlm.Count > 0 && acBinaryRtResultsLlm.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("## Overall: AcBinary (Byte[], SGen) vs MemoryPack (Byte[])"); + sb.AppendLine(); + sb.AppendLine("Three aggregations of per-cell results: **arith** = arithmetic mean of µs/op (magnitude-weighted, Large cell dominates); **geo** = geometric mean of per-cell ratios (each cell weighted equally); **median** = median of per-cell ratios (outlier-resistant). Negative % = AcBinary faster/smaller; positive % = MemPack faster/smaller. The geo/median variants surface when a single big cell skews the arithmetic mean."); + sb.AppendLine(); + sb.AppendLine("```"); + AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, SerPerOp)); + AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, r => r.SerializeAllocBytesPerOp), "F0"); + AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, DesPerOp)); + AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, r => r.DeserializeAllocBytesPerOp), "F0"); + AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResultsLlm, memPackRtResultsLlm, RtPerOp)); + AppendOverallLine(sb, "Size", "B", ComputeOverallStats(acBinarySGenByteArrayResults, memPackByteArrayResults, r => r.SerializedSize), "F0"); + sb.AppendLine("```"); + } + + File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom); + System.Console.WriteLine($"✓ LLM results saved to: {filePath}"); + } + + /// + /// Formats byte array as hex dump with offset, hex values, and ASCII representation. + /// + internal static string FormatHexDump(byte[] bytes, int bytesPerLine = 16) + { + var sb = new StringBuilder(); + for (var i = 0; i < bytes.Length; i += bytesPerLine) + { + // Offset + sb.Append($"{i:X8} "); + + // Hex bytes + for (var j = 0; j < bytesPerLine; j++) + { + if (i + j < bytes.Length) + sb.Append($"{bytes[i + j]:X2} "); + else + sb.Append(" "); + + if (j == 7) sb.Append(' '); // Extra space in middle + } + + sb.Append(" |"); + + // ASCII representation + for (var j = 0; j < bytesPerLine && i + j < bytes.Length; j++) + { + var b = bytes[i + j]; + sb.Append(b is >= 32 and < 127 ? (char)b : '.'); + } + + sb.AppendLine("|"); + } + return sb.ToString(); + } +} diff --git a/AyCode.Core.Serializers.Console/Program.cs b/AyCode.Core.Serializers.Console/Program.cs index 61f31a1..80b31c0 100644 --- a/AyCode.Core.Serializers.Console/Program.cs +++ b/AyCode.Core.Serializers.Console/Program.cs @@ -1,4 +1,4 @@ -using AyCode.Core.Compression; +using AyCode.Core.Compression; using AyCode.Core.Serializers.Attributes; using AyCode.Core.Serializers.Binaries; using AyCode.Core.Tests.Serialization; // DrainFromAsync extension (test-only, used by benchmark) @@ -91,112 +91,8 @@ public static class Program /// produce the same per-op µs (within noise). Necessary because per-cell adaptive iteration makes /// iterations a per-row property — there is no longer a single global Configuration.TestIterations to divide by. /// - private static double ToPerOpMicros(double totalMs, int iterations) => iterations > 0 ? totalMs / iterations * 1000.0 : 0; - - // Per-row per-op µs accessors — pull batch-time + iter from BenchmarkResult and convert. Used wherever - // averaging or comparison happens across rows with potentially different iter counts (Winners summary, - // Overall comparison, per-cell summary row). Keeping these as methods rather than properties on - // BenchmarkResult preserves the result-as-data-bag distinction. - private static double SerPerOp(BenchmarkResult r) => ToPerOpMicros(r.SerializeTimeMs, r.SerializeIterations); - private static double DesPerOp(BenchmarkResult r) => ToPerOpMicros(r.DeserializeTimeMs, r.DeserializeIterations); - private static double RtPerOp(BenchmarkResult r) => ToPerOpMicros(r.RoundTripTimeMs, r.RoundTripIterations); - - /// - /// Per-cell-paired aggregation of an overall comparison. Captures three different aggregation - /// strategies so the reader can judge whether the headline delta is dominated by one large cell - /// (arithmetic mean) or representative of typical workload (geometric mean / median). - /// - /// Arithmetic mean of µs/op — magnitude-weighted; biased toward Large cell. - /// Geometric mean of per-cell ratios — magnitude-neutral; each cell weighted equally. - /// Median of per-cell ratios — outlier-resistant. - /// Arithmetic mean AcBinary value (µs/op or bytes). - /// Arithmetic mean MemPack value. - /// Number of paired cells contributing to the geo/median. - private record OverallStats(double ArithMeanPct, double GeoMeanPct, double MedianPct, double AcAvg, double MpAvg, int CellCount); - - /// - /// Computes arithmetic + geometric + median aggregation of an AcBinary-vs-MemPack comparison - /// across paired cells (joined by TestDataName). Per-cell pairing is required for the - /// geo/median variants — a cell where AcBinary or MemPack is missing is dropped from all stats. - /// Returns null when no paired cell has a valid value. - /// - private static OverallStats? ComputeOverallStats(List acResults, List mpResults, Func getValue) - { - if (acResults.Count == 0 || mpResults.Count == 0) return null; - - var pairs = (from ac in acResults - join mp in mpResults on ac.TestDataName equals mp.TestDataName - let acV = getValue(ac) - let mpV = getValue(mp) - where acV > 0 && mpV > 0 - select (ac: acV, mp: mpV)).ToList(); - - if (pairs.Count == 0) return null; - - var acAvg = pairs.Average(p => p.ac); - var mpAvg = pairs.Average(p => p.mp); - var ratios = pairs.Select(p => p.ac / p.mp).ToList(); - - // Geometric mean: exp(avg(ln(ratios))) — numerically stable vs Π ratios then ^(1/N). - var geoMean = Math.Exp(ratios.Sum(Math.Log) / ratios.Count); - - // Median (paired-ratio): for even N use the midpoint of the two middle values. - var sorted = ratios.OrderBy(r => r).ToList(); - var median = sorted.Count % 2 == 1 - ? sorted[sorted.Count / 2] - : (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2.0; - - return new OverallStats( - ArithMeanPct: (acAvg / mpAvg - 1) * 100, - GeoMeanPct: (geoMean - 1) * 100, - MedianPct: (median - 1) * 100, - AcAvg: acAvg, - MpAvg: mpAvg, - CellCount: ratios.Count); - } - - /// - /// Formats a per-op micros value with its inter-sample range and CV-threshold marker as - /// "26.86 (24.5..29.1)" or "26.86 (24.5..29.1) ⚠️5.2%". Median first, range in parentheses, - /// CV warning suffix only when CV > . When min == max == median - /// (single-sample / Debug / quick mode), collapses to bare median to avoid visual clutter. - /// All time inputs are total-batch milliseconds; is the per-row iter - /// count (post-adaptive-calibration). - /// - private static string FormatMicrosWithRange(double medianMs, double minMs, double maxMs, double stdDevMs, int iterations, System.Globalization.CultureInfo inv) - { - var med = ToPerOpMicros(medianMs, iterations); - // No range data (single-sample fast path) — surface as bare median, identical to the prior format. - if (minMs <= 0 && maxMs <= 0) return med.ToString("F2", inv); - if (minMs >= medianMs && maxMs <= medianMs) return med.ToString("F2", inv); - - var min = ToPerOpMicros(minMs, iterations); - var max = ToPerOpMicros(maxMs, iterations); - var range = $"{med.ToString("F2", inv)} ({min.ToString("F2", inv)}..{max.ToString("F2", inv)})"; - - // CV (coefficient of variation = stddev / mean) — flag rows above the unstable threshold so a - // small inter-engine delta on a high-CV row is easy to discount as noise. - if (medianMs > 0 && stdDevMs > 0) - { - var cv = stdDevMs / medianMs; - if (cv > Configuration.UnstableCVThreshold) - { - var cvPct = (cv * 100).ToString("F1", inv); - return $"{range} ⚠️{cvPct}%"; - } - } - - return range; - } - - /// - /// Converts a byte count to KB (1 KB = 1024 B). Display-only helper so allocation columns can - /// render compact F2 KB values (e.g. 4.05 KB instead of 4,144 B) — header carries - /// the unit so per-row entries stay numbers-only. CSV / raw-data outputs keep the precise byte - /// integers untouched. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static double ToKilobytes(long bytes) => bytes / 1024.0; + // Output helpers (PrintResult, SaveResults, OverallStats, FormatMicrosWithRange, etc.) → Output.cs + // BenchmarkResult DTO → BenchmarkResult.cs public static void Main(string[] args) { @@ -221,7 +117,7 @@ public static class Program // Q exits the menu (and the application). while (true) { - var selection = ShowInteractiveMenu(); + var selection = Menu.ShowInteractiveMenu(); if (selection == null) return; // user pressed Q RunBenchmark(selection.Value.layer, "all", selection.Value.serializerMode); @@ -344,7 +240,7 @@ public static class Program var allTestDataSets = BenchmarkTestDataProvider.CreateTestDataSets(); var testDataSets = FilterByLayer(allTestDataSets, layer); - System.Console.WriteLine($"Layer: {layer} | OpMode: {opMode} | SerializerMode: {serializerMode} | Charset: {GetCurrentCharsetName()} | Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + pilot discard"); + System.Console.WriteLine($"Layer: {layer} | OpMode: {opMode} | SerializerMode: {serializerMode} | Charset: {Configuration.GetCurrentCharsetName()} | Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + pilot discard"); System.Console.WriteLine($"Build: {Configuration.BuildConfiguration} | .NET: {Environment.Version} | Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"} | Test Cells: {testDataSets.Count}/{allTestDataSets.Count}"); System.Console.WriteLine(); @@ -366,9 +262,11 @@ public static class Program { foreach (var s in preSerializers) { - // Light warmup just to trigger Tier 0 → Tier 1 promotion. The per-cell 5000-iter warmup - // inside RunBenchmarksForTestData still runs afterwards for cache/BTB warming. - s.Warmup(2000); + // Light warmup just to trigger Tier 0 → Tier 1 promotion. Phase-isolated: + // Ser path first, then Des path — same pattern as the per-cell warmup in + // RunBenchmarksForTestData (which still runs afterwards for cache/BTB warming). + s.WarmupSerialize(2000); + s.WarmupDeserialize(2000); } } finally @@ -394,10 +292,10 @@ public static class Program } // Print grouped results - PrintGroupedResults(allResults, testDataSets); + Output.PrintGroupedResults(allResults, testDataSets); // Save results to file - SaveResults(allResults, testDataSets); + Output.SaveResults(allResults, testDataSets); System.Console.WriteLine("\n✓ Benchmark complete!"); } @@ -557,9 +455,9 @@ public static class Program // Compose RT from Ser+Des. Because Ser and Des may have DIFFERENT iter counts post-calibration, // batch-time addition would be misleading. Instead: compute per-op µs (iter-independent), // then synthesize RoundTripTimeMs against RoundTripIterations = max(serIter, desIter) so that - // RoundTripTimeMs / RoundTripIterations * 1000 == SerPerOp + DesPerOp. - var serPerOp = ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations); - var desPerOp = ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations); + // RoundTripTimeMs / RoundTripIterations * 1000 == Output.SerPerOp + Output.DesPerOp. + var serPerOp = Output.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations); + var desPerOp = Output.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations); var rtPerOp = serPerOp + desPerOp; result.RoundTripIterations = Math.Max(result.SerializeIterations, result.DeserializeIterations); result.RoundTripTimeMs = rtPerOp / 1000.0 * result.RoundTripIterations; @@ -567,7 +465,7 @@ public static class Program } results.Add(result); - PrintResult(result); + Output.PrintResult(result); } // Dispose any IDisposable serializers (NamedPipe / FileStream variants own OS resources that must be released @@ -1046,238 +944,6 @@ public static class Program } } - /// - /// Interactive menu shown when no CLI args. Returns the layer keyword (core/comprehensive/edge/all) or null on Quit. - /// Loops on settings-changes ([S]) — user is returned to this menu after modifying iteration counts. - /// - private static (string layer, string serializerMode)? ShowInteractiveMenu() - { - while (true) - { - System.Console.WriteLine(); - System.Console.WriteLine("╔══════════════════════════════════════════════════════════╗"); - System.Console.WriteLine("║ AcBinary Benchmark Suite ║"); - System.Console.WriteLine("╚══════════════════════════════════════════════════════════╝"); - System.Console.WriteLine(); - System.Console.WriteLine("Select benchmark layer:"); - System.Console.WriteLine(); - System.Console.WriteLine(" [1] Core — daily iteration"); - System.Console.WriteLine(" [2] Comprehensive — release validation"); - System.Console.WriteLine(" [3] Edge cases — refactor verification"); - System.Console.WriteLine(" [A] All layers"); - System.Console.WriteLine(" [F] FastestByte — AcBinary FastMode Byte[] vs MemoryPack Byte[] only (tight optimization loop)"); - System.Console.WriteLine(" [P] AsyncPipe — streaming I/O isolation (only AsyncPipe, all test data)"); - System.Console.WriteLine($" [S] Settings — Iteration / WireMode (current: {Configuration.SelectedWireMode})"); - System.Console.WriteLine(" [Q] Quit"); - System.Console.Write("\nSelection: "); - - var key = System.Console.ReadKey(intercept: false).KeyChar; - System.Console.WriteLine(); - - switch (char.ToLower(key)) - { - case '1': return ("core", "standard"); - case '2': return ("comprehensive", "standard"); - case '3': return ("edge", "standard"); - case 'a': return ("all", "standard"); - case 'f': return ("all", "fastestbyte"); - case 'p': return ("all", "asyncpipe"); - case 's': - ShowSettingsMenu(); - continue; // re-display the main menu after settings update - case 'q': return null; - default: return ("all", "standard"); - } - } - } - - /// - /// Settings sub-menu — prompts for Warmup / Iterations / Samples values. Empty input keeps the - /// current value. Validation: Configuration.WarmupIterations ≥ 0; Configuration.TestIterations ≥ 1; Configuration.BenchmarkSamples ≥ 1. - /// Returns to the caller (which re-displays the main menu). - /// - private static void ShowSettingsMenu() - { - while (true) - { - System.Console.WriteLine(); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine("Settings"); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine(" [1] Iteration — Warmup / Iterations / Samples"); - System.Console.WriteLine($" [2] WireMode — current: {Configuration.SelectedWireMode}"); - System.Console.WriteLine($" [3] Charset — current: {GetCurrentCharsetName()}"); - System.Console.WriteLine(" [B] Back"); - System.Console.Write("\nSelection: "); - - var key = System.Console.ReadKey(intercept: false).KeyChar; - System.Console.WriteLine(); - - switch (char.ToLower(key)) - { - case '1': - ShowIterationSettingsMenu(); - break; - case '2': - ShowWireModeSettingsMenu(); - break; - case '3': - ShowCharsetSettingsMenu(); - break; - case 'b': - return; - default: - continue; - } - } - } - - /// - /// Returns a human-readable name for the currently-active BenchmarkTestDataProvider.LongStringSuffix - /// charset. Returns "Custom" when the suffix doesn't match any of the predefined - /// constants. Used in menu state display, console run header, and - /// the .LLM markdown output header so per-charset bench files are self-documenting. - /// - private static string GetCurrentCharsetName() - { - var s = BenchmarkTestDataProvider.LongStringSuffix; - - return s switch - { - CharsetSuffixes.Latin1FixAscii => "Latin1FixAscii", - CharsetSuffixes.Latin1Short => "Latin1Short", - CharsetSuffixes.Latin1Long => "Latin1Long", - CharsetSuffixes.CjkBmp => "CjkBmp", - CharsetSuffixes.Cyrillic => "Cyrillic", - CharsetSuffixes.Mixed => "Mixed", - _ => "Custom" - }; - } - - private static void ShowCharsetSettingsMenu() - { - while (true) - { - System.Console.WriteLine(); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine("Charset settings — long-string suffix profile"); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine($"Current: {GetCurrentCharsetName()}"); - System.Console.WriteLine(); - System.Console.WriteLine(" [1] Latin1FixAscii — empty suffix; short FixStr-fast-path stress (Latin1 baseline values stay short)"); - System.Console.WriteLine(" [2] Latin1Short — \" árvíztűrő tükörfúrógép\" (~24 char Hungarian mixed)"); - System.Console.WriteLine(" [3] Latin1Long — ~47-char Latin1 mixed (default; exceeds FixStr boundary)"); - System.Console.WriteLine(" [4] CjkBmp — CJK BMP (long 3-byte runs)"); - System.Console.WriteLine(" [5] Cyrillic — Russian Cyrillic (long 2-byte runs)"); - System.Console.WriteLine(" [6] Mixed — Hungarian + CJK + Cyrillic + emoji (full-spectrum + surrogate pairs)"); - System.Console.WriteLine(" [B] Back"); - System.Console.Write("\nSelection: "); - - var key = System.Console.ReadKey(intercept: false).KeyChar; - System.Console.WriteLine(); - - switch (char.ToLower(key)) - { - case '1': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1FixAscii; - System.Console.WriteLine("✓ Charset set to Latin1FixAscii"); - return; - case '2': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1Short; - System.Console.WriteLine("✓ Charset set to Latin1Short"); - return; - case '3': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1Long; - System.Console.WriteLine("✓ Charset set to Latin1Long"); - return; - case '4': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.CjkBmp; - System.Console.WriteLine("✓ Charset set to CjkBmp"); - return; - case '5': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Cyrillic; - System.Console.WriteLine("✓ Charset set to Cyrillic"); - return; - case '6': - BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Mixed; - System.Console.WriteLine("✓ Charset set to Mixed"); - return; - case 'b': - return; - default: - continue; - } - } - } - - private static void ShowIterationSettingsMenu() - { - System.Console.WriteLine(); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine("Iteration settings — press Enter to keep current value"); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine(); - - Configuration.WarmupIterations = PromptInt("Configuration.WarmupIterations", Configuration.WarmupIterations, min: 0); - Configuration.TestIterations = PromptInt("Configuration.TestIterations ", Configuration.TestIterations, min: 1); - Configuration.BenchmarkSamples = PromptInt("Configuration.BenchmarkSamples", Configuration.BenchmarkSamples, min: 1); - - System.Console.WriteLine(); - System.Console.WriteLine($"✓ Iteration settings updated: Warmup={Configuration.WarmupIterations} | Iterations={Configuration.TestIterations} | Samples={Configuration.BenchmarkSamples}"); - } - - private static void ShowWireModeSettingsMenu() - { - while (true) - { - System.Console.WriteLine(); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine("WireMode settings"); - System.Console.WriteLine("─────────────────────────────────────────────"); - System.Console.WriteLine($"Current: {Configuration.SelectedWireMode}"); - System.Console.WriteLine(" [1] Compact"); - System.Console.WriteLine(" [2] Fast"); - System.Console.WriteLine(" [B] Back"); - System.Console.Write("\nSelection: "); - - var key = System.Console.ReadKey(intercept: false).KeyChar; - System.Console.WriteLine(); - - switch (char.ToLower(key)) - { - case '1': - Configuration.SelectedWireMode = WireMode.Compact; - System.Console.WriteLine("✓ WireMode set to Compact"); - return; - case '2': - Configuration.SelectedWireMode = WireMode.Fast; - System.Console.WriteLine("✓ WireMode set to Fast"); - return; - case 'b': - return; - default: - continue; - } - } - } - - /// - /// Prompts the user for an integer with a default (current value). Returns the current value if - /// the user presses Enter on empty input or if parsing fails / value is below the minimum. - /// - private static int PromptInt(string name, int currentValue, int min) - { - System.Console.Write($" {name} [{currentValue}]: "); - - var input = System.Console.ReadLine()?.Trim() ?? ""; - if (input.Length == 0) return currentValue; - - if (int.TryParse(input, out var newValue) && newValue >= min) return newValue; - - System.Console.WriteLine($" ! Invalid value (need int ≥ {min}) — keeping {currentValue}"); - return currentValue; - } - /// /// Filters test data sets by layer keyword. Layered approach lets you run only what's needed for the iteration cadence. /// P1: only "Core" data exists (Small/Medium/Large/Repeated/Deep). Comprehensive and Edge layers will be expanded in P2. @@ -1338,11 +1004,6 @@ public static class Program /// rankings (because both metrics are misleading there) — they still participate in "Fastest Round-trip". /// Default false for in-memory IO modes which measure Ser and Des separately. bool IsRoundTripOnly => false; - /// Combined warmup (Ser + Deser interleaved). Currently unused — kept as a legacy entry point - /// for any external caller that still wants single-call warmup. The benchmark loop uses the split - /// + pair for cache-isolated measurements. - void Warmup(int iterations); - /// Warm only the Serialize path. Default body iterates N times. /// Overrides are only needed when the implementor wants Ser-specific warmup-state (e.g. pre-allocate buffers). /// On benchmarks (NamedPipe-style) performs the full RT, @@ -1391,15 +1052,6 @@ public static class Program //_options.UseCompression = Lz4CompressionMode.Block; } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -1446,15 +1098,6 @@ public static class Program _serialized = MemoryPackSerializer.Serialize(order, _options); } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() => MemoryPackSerializer.Serialize(_order, _options); @@ -1504,15 +1147,6 @@ public static class Program _serialized = MessagePackSerializer.Serialize(order, _options); } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() => MessagePackSerializer.Serialize(_order, _options); @@ -1565,15 +1199,6 @@ public static class Program _serialized = AcBinarySerializer.Serialize(order, _options); } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -1763,14 +1388,6 @@ public static class Program } } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -1954,11 +1571,6 @@ public static class Program } } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) Serialize(); - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2163,14 +1775,6 @@ public static class Program } } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2343,11 +1947,6 @@ public static class Program } } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) Serialize(); - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2433,15 +2032,6 @@ public static class Program _serialized = MemoryPackSerializer.Serialize(order, _options); } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2496,15 +2086,6 @@ public static class Program SetupSerializeAllocBytes = afterSetup - beforeSetup; } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2565,15 +2146,6 @@ public static class Program SetupSerializeAllocBytes = afterSetup - beforeSetup; } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() { @@ -2624,15 +2196,6 @@ public static class Program _serializedUtf8 = Configuration.Utf8NoBom.GetBytes(_serialized); } - public void Warmup(int iterations) - { - for (var i = 0; i < iterations; i++) - { - Serialize(); - Deserialize(); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] public void Serialize() => JsonSerializer.Serialize(_order, _options); @@ -2649,674 +2212,7 @@ public static class Program #endregion - #region Results + // Results / output formatters → Output.cs + // BenchmarkResult DTO → BenchmarkResult.cs - private sealed class BenchmarkResult - { - public string TestDataName { get; set; } = ""; - public string Engine { get; set; } = ""; - public string IoMode { get; set; } = ""; - public string DispatchMode { get; set; } = ""; - public string OptionsPreset { get; set; } = ""; - /// True if Serialize() captures a full round-trip and Deserialize() is a no-op - /// (single-use streaming transports like NamedPipe). Excluded from "Fastest Serialize" / "Fastest Deserialize" - /// winners rankings; still ranked in "Fastest Round-trip". Display-side: Ser µs/op / SerAlloc / Des µs/op / DesAlloc - /// all show "N/A" since they were never measured separately; RT µs/op / RT Alloc carry the full round-trip values. - public bool IsRoundTripOnly { get; set; } - /// Synthesized display name for backwards compatibility / single-string-row scenarios. Includes DispatchMode so SGen and Runtime variants of the same preset don't collide in grouping (e.g. SUMMARY: WINNERS). - public string SerializerName => $"{Engine} ({IoMode}, {OptionsPreset}, {DispatchMode})"; - public string? OptionsDescription { get; set; } - public int SerializedSize { get; set; } - public double SerializeTimeMs { get; set; } - public double DeserializeTimeMs { get; set; } - // Per-sample min/max alongside the median (median is the *Time*Ms field above). Surfaces - // inter-sample range — the visible noise floor for the row. 0 when the operation was skipped - // (mode != "all"/"ser"/"des") or when a single-sample fast path was used (min == max == median). - public double SerializeTimeMinMs { get; set; } - public double SerializeTimeMaxMs { get; set; } - public double DeserializeTimeMinMs { get; set; } - public double DeserializeTimeMaxMs { get; set; } - // Sample-population stddev (ms). Used by FormatMicrosWithRange to compute CV (stddev/mean) - // and emit the ⚠️ marker on rows above Configuration.UnstableCVThreshold. 0 in single-sample mode. - public double SerializeTimeStdDevMs { get; set; } - public double DeserializeTimeStdDevMs { get; set; } - // Per-row adaptive iteration count (post-CalibrateIterations). Each Ser and Des function calibrates - // independently to land its sample window at ~Configuration.TargetSampleMs; per-op µs is then iter-independent - // (`SerializeTimeMs / SerializeIterations * 1000`). For round-trip-only rows (NamedPipe etc.), - // RoundTripIterations carries the calibrated iter count; SerializeIterations and DeserializeIterations - // stay 0 (Ser and Des are not separately measurable on those rows). - public int SerializeIterations { get; set; } - public int DeserializeIterations { get; set; } - public int RoundTripIterations { get; set; } - public long SerializeAllocBytesPerOp { get; set; } - public long DeserializeAllocBytesPerOp { get; set; } - public long SetupSerializeAllocBytes { get; set; } - public long SetupDeserializeAllocBytes { get; set; } - /// Total round-trip time. For in-memory benchmarks: synthesized so that - /// RoundTripTimeMs / RoundTripIterations yields the correct SerPerOp + DesPerOp µs/op - /// (necessary because Ser and Des may have different iter counts post-calibration). - /// For round-trip-only benchmarks (NamedPipe etc.): the directly-measured pipe round-trip time. - public double RoundTripTimeMs { get; set; } - // Round-trip min/max + stddev — only populated for round-trip-only benchmarks (NamedPipe etc.) where - // RT is directly measured. For in-memory rows RT = Ser + Des, which has no single-sample - // distribution; surface Ser/Des range separately instead. - public double RoundTripTimeMinMs { get; set; } - public double RoundTripTimeMaxMs { get; set; } - public double RoundTripTimeStdDevMs { get; set; } - /// Total round-trip allocation per op. For in-memory benchmarks: SerializeAlloc + DeserializeAlloc. - /// For round-trip-only benchmarks: process-wide allocation measured via - /// (covers ALL threads — client, server-drain, channel internals — not just the caller). - public long RoundTripAllocBytesPerOp { get; set; } - } - - private static void PrintResult(BenchmarkResult result) - { - // Numbers-only per-row entries; the column-headers carry units (µs/op, KB/op). - var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result),7:F2}" : " N/A"; - var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result),7:F2}" : " N/A"; - var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp),7:F2}" : " N/A"; - var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp),7:F2}" : " N/A"; - System.Console.WriteLine($" {result.SerializerName,-40} | Size: {result.SerializedSize,8:N0} B | Ser: {ser} µs/op ({serAlloc} KB/op) | Des: {des} µs/op ({desAlloc} KB/op)"); - } - - private static void PrintGroupedResults(List results, List testDataSets) - { - System.Console.WriteLine("\n"); - System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); - System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║"); - System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); - - // Print serializer options - var optionsMap = results - .Where(r => r.OptionsDescription != null) - .Select(r => (r.SerializerName, r.OptionsDescription!)) - .Distinct() - .ToList(); - - if (optionsMap.Count > 0) - { - System.Console.WriteLine(); - System.Console.WriteLine(" Serializer Options:"); - foreach (var (name, opts) in optionsMap) - System.Console.WriteLine($" {name}: {opts}"); - } - - foreach (var testData in testDataSets) - { - // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. - var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); - // Baseline switched MessagePack → MemoryPack: MemoryPack is the SOTA performance leader. - var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); - // Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated). - // The Runtime variant is shown alongside in the table for context, not used as the headline number. - var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); - - System.Console.WriteLine($"\n┌─ {testData.DisplayName} ─".PadRight(172, '─') + "┐"); - // Header-only units; per-row entries are numbers (µs/op for time, KB/op for alloc, KB pair "ser / des" for Setup, B for Size). - System.Console.WriteLine($"│ {"#",-4} │ {"Engine",-11} │ {"Options",-22} │ {"IO",-12} │ {"Mode",-8} │ {"Setup S/D KB",-14} │ {"Size B",-8} │ {"Ser µs/op",-10} │ {"SerAlc KB",-10} │ {"Des µs/op",-10} │ {"DesAlc KB",-10} │ {"RT µs/op",-10} │ {"RTAlc KB",-10} │"); - System.Console.WriteLine($"├{"─".PadRight(6, '─')}┼{"─".PadRight(13, '─')}┼{"─".PadRight(24, '─')}┼{"─".PadRight(14, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); - - var rank = 1; - foreach (var result in testResults) - { - var size = $"{result.SerializedSize:N0}"; - var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; - var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; - var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; - var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; - var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; - var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; - var rtAlloc = result.RoundTripAllocBytesPerOp > 0 ? $"{ToKilobytes(result.RoundTripAllocBytesPerOp):F2}" : "N/A"; - - // Highlight MemoryPack baseline (any Byte[]) and AcBinary headline contender (Byte[] + SGen) with win/lose colors. - // The AcBinary Byte[]+Runtime variant is shown unhighlighted — it's contextual (SGen speed-up reference), not the headline. - var isHighlighted = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray) - || (result.Engine == Configuration.EngineAcBinary && result.IoMode == Configuration.IoByteArray && result.DispatchMode == Configuration.ModeSGen); - - var prefix = isHighlighted ? "│►" : "│ "; - var suffix = isHighlighted ? "◄│" : " │"; - - // Color logic: Green = winner (faster), Red = loser (slower) - if (isHighlighted && memPackResult != null && acBinaryResult != null) - { - var isMemPack = (result.Engine == Configuration.EngineMemoryPack && result.IoMode == Configuration.IoByteArray); - var memPackFaster = RtPerOp(memPackResult) < RtPerOp(acBinaryResult); - - if (isMemPack) - { - System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Green : ConsoleColor.Red; - } - else - { - System.Console.ForegroundColor = memPackFaster ? ConsoleColor.Red : ConsoleColor.Green; - } - } - - System.Console.WriteLine($"{prefix}{rank++,4} │ {result.Engine,-11} │ {result.OptionsPreset,-22} │ {result.IoMode,-12} │ {result.DispatchMode,-8} │ {setup,14} │ {size,8} │ {ser,10} │ {serAlloc,10} │ {des,10} │ {desAlloc,10} │ {rt,10} │ {rtAlloc,10}{suffix}"); - - if (isHighlighted) - { - System.Console.ResetColor(); - } - } - - // Footer row: AcBinary (Byte[]) vs MemoryPack (Byte[]) comparison per column - if (memPackResult != null && acBinaryResult != null) - { - var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; - // Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows. - var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; - var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; - var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; - var serAllocPct = memPackResult.SerializeAllocBytesPerOp > 0 ? (acBinaryResult.SerializeAllocBytesPerOp / (double)memPackResult.SerializeAllocBytesPerOp - 1) * 100 : 0; - var desAllocPct = memPackResult.DeserializeAllocBytesPerOp > 0 ? (acBinaryResult.DeserializeAllocBytesPerOp / (double)memPackResult.DeserializeAllocBytesPerOp - 1) * 100 : 0; - var rtAllocPct = memPackResult.RoundTripAllocBytesPerOp > 0 ? (acBinaryResult.RoundTripAllocBytesPerOp / (double)memPackResult.RoundTripAllocBytesPerOp - 1) * 100 : 0; - - // Footer separator: merge first 5 cols (#, Engine, Options, IO, Mode) → comparison label; - // remaining 8 cols stay aligned (Setup S/D KB, Size, Ser µs/op, SerAlc KB, Des µs/op, DesAlc KB, RT µs/op, RTAlc KB). - System.Console.WriteLine($"├{"─".PadRight(6, '─')}┴{"─".PadRight(13, '─')}┴{"─".PadRight(24, '─')}┴{"─".PadRight(14, '─')}┴{"─".PadRight(10, '─')}┼{"─".PadRight(16, '─')}┼{"─".PadRight(10, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┼{"─".PadRight(12, '─')}┤"); - // Merged label cell width = 4 + 11 + 22 + 12 + 8 + 4*3 (dropped separators) = 69 - System.Console.Write($"│ {"► AcBinary (Byte[]) vs MemoryPack (Byte[])",-69} │ "); - - // Setup S/D KB (n/a for Byte[] vs Byte[] — neither pre-allocates) - System.Console.Write($"{"—",14}"); - System.Console.Write(" │ "); - - // Size - System.Console.ForegroundColor = sizePct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{sizePct,+7:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Serialize - System.Console.ForegroundColor = serPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{serPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Serialize Alloc - System.Console.ForegroundColor = serAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{serAllocPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Deserialize - System.Console.ForegroundColor = desPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{desPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Deserialize Alloc - System.Console.ForegroundColor = desAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{desAllocPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Round-trip - System.Console.ForegroundColor = rtPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{rtPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.Write(" │ "); - - // Round-trip Alloc - System.Console.ForegroundColor = rtAllocPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.Write($"{rtAllocPct,+9:+0;-0}%"); - System.Console.ResetColor(); - System.Console.WriteLine(" │"); - } - - // Closing line: merged on left (─ between cols 1-5), ┴ on the right (cols 6-13 boundary, 8 unmerged cells). - System.Console.WriteLine($"└{"─".PadRight(6, '─')}─{"─".PadRight(13, '─')}─{"─".PadRight(24, '─')}─{"─".PadRight(14, '─')}─{"─".PadRight(10, '─')}┴{"─".PadRight(16, '─')}┴{"─".PadRight(10, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┴{"─".PadRight(12, '─')}┘"); - //System.Console.WriteLine($"GrowBufferCount: {AcBinarySerializer.GrowBufferCount}"); - //System.Console.WriteLine($"GrowBufferTotalBytes: {AcBinarySerializer.GrowBufferTotalBytes:N0} bytes"); - } - - // Summary: Best serializer for each category - System.Console.WriteLine("\n"); - System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); - System.Console.WriteLine("║ SUMMARY: WINNERS ║"); - System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); - - System.Console.WriteLine($"\n{"Category",-20} │ {"Winner",-40} │ {"Avg Value",-18}"); - System.Console.WriteLine($"{"─".PadRight(20, '─')}─┼─{"─".PadRight(40, '─')}─┼─{"─".PadRight(18, '─')}"); - - // Fastest Serialize — round-trip-only serializers (NamedPipe etc.) excluded: - // their Serialize() captures the full round-trip and isn't comparable to a pure Ser metric. - // Average is over per-op µs (iter-independent) instead of batch-time, since rows may now - // have different iter counts post-calibration. - var fastestSer = results.Where(r => r.SerializeTimeMs > 0 && !r.IsRoundTripOnly) - .GroupBy(r => r.SerializerName) - .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => SerPerOp(r)) }) - .OrderBy(x => x.AvgPerOp) - .FirstOrDefault(); - - if (fastestSer != null) - System.Console.WriteLine($"{"Fastest Serialize",-20} │ {fastestSer.Name,-40} │ {fastestSer.AvgPerOp,12:F2} µs/op"); - - // Fastest Deserialize — round-trip-only serializers excluded (their Deserialize() is a no-op). - var fastestDes = results.Where(r => r.DeserializeTimeMs > 0 && !r.IsRoundTripOnly) - .GroupBy(r => r.SerializerName) - .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => DesPerOp(r)) }) - .OrderBy(x => x.AvgPerOp) - .FirstOrDefault(); - - if (fastestDes != null) - System.Console.WriteLine($"{"Fastest Deserialize",-20} │ {fastestDes.Name,-40} │ {fastestDes.AvgPerOp,12:F2} µs/op"); - - // Smallest Size - var smallestSize = results - .GroupBy(r => r.SerializerName) - .Select(g => new { Name = g.Key, AvgSize = g.Average(r => r.SerializedSize) }) - .OrderBy(x => x.AvgSize) - .FirstOrDefault(); - - if (smallestSize != null) - System.Console.WriteLine($"{"Smallest Size",-20} │ {smallestSize.Name,-40} │ {smallestSize.AvgSize,15:F0} B"); - - // Fastest Round-trip — iter-independent per-op average. - var fastestRt = results.Where(r => r.RoundTripTimeMs > 0) - .GroupBy(r => r.SerializerName) - .Select(g => new { Name = g.Key, AvgPerOp = g.Average(r => RtPerOp(r)) }) - .OrderBy(x => x.AvgPerOp) - .FirstOrDefault(); - - if (fastestRt != null) - System.Console.WriteLine($"{"Fastest Round-trip",-20} │ {fastestRt.Name,-40} │ {fastestRt.AvgPerOp,12:F2} µs/op"); - - // Overall AcBinary (SGen) vs MemoryPack comparison (baseline switched MessagePack → MemoryPack as SOTA reference). - // AcBinary side is restricted to DispatchMode == SGen — apples-to-apples vs MemoryPack which is also source-generated. - // The Runtime variant is shown side-by-side in each per-test fancy table for SGen-speedup context, but excluded from this headline. - var memPackSerResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); - var memPackDesResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); - var memPackRtResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); - - var acBinarySerResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); - var acBinaryDesResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); - var acBinaryRtResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); - - // Skip comparison if no data available - if (memPackRtResults.Count == 0 || acBinaryRtResults.Count == 0) - { - System.Console.WriteLine(); - System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); - System.Console.WriteLine(" (Comparison requires both serialize and deserialize data)"); - return; - } - - // All averages are over per-op µs (iter-independent). Batch-time averaging would mix rows - // measured with different iter counts (post-calibration), producing meaningless numbers. - // Three aggregations per metric: - // - Arithmetic mean (current behavior) — magnitude-weighted, biased toward Large cell. - // - Geometric mean of per-cell ratios — magnitude-neutral, each cell weighted equally. - // - Median of per-cell ratios — outlier-resistant. - // The geo/median variants surface when a single cell dominates the arithmetic average - // (typical when one cell's µs-per-op is an order of magnitude larger than the others). - var sizeAcResults = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); - var sizeMpResults = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); - - var serStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, SerPerOp); - var desStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, DesPerOp); - var rtStats = ComputeOverallStats(acBinaryRtResults, memPackRtResults, RtPerOp); - var sizeStats = ComputeOverallStats(sizeAcResults, sizeMpResults, r => r.SerializedSize); - var serAllocStats = ComputeOverallStats(acBinarySerResults, memPackSerResults, r => r.SerializeAllocBytesPerOp); - var desAllocStats = ComputeOverallStats(acBinaryDesResults, memPackDesResults, r => r.DeserializeAllocBytesPerOp); - - System.Console.WriteLine(); - System.Console.WriteLine("── AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ──"); - - WriteOverallLine("Serialize", "µs/op", serStats); - WriteOverallLine("Deserialize", "µs/op", desStats); - WriteOverallLine("Round-trip", "µs/op", rtStats); - WriteOverallLine("Size", "B", sizeStats, "F0"); - WriteOverallLine("Ser Alloc", "B/op", serAllocStats, "F0"); - WriteOverallLine("Des Alloc", "B/op", desAllocStats, "F0"); - } - - /// - /// Formats a signed percent delta with explicit sign for positive values (`+1.5%`, `-3.0%`, `0.0%`). - /// Padded to 7 chars (e.g. ` +12.3%`, `-100.0%`) for column alignment in the Overall block. - /// - private static string FormatPctSigned(double pct) => pct.ToString("+0.0;-0.0;0.0", System.Globalization.CultureInfo.InvariantCulture).PadLeft(6) + "%"; - - /// - /// Renders one Overall row with arith / geo / median deltas + AcBinary/MemPack absolute means. - /// Color is driven by the geometric-mean delta (magnitude-neutral signal). Skips silently when - /// stats is null (no paired data). - /// - private static void WriteOverallLine(string label, string unit, OverallStats? stats, string fmt = "F2") - { - if (stats == null) return; - - // Color follows geo-mean (the magnitude-neutral signal). The arith-mean column may show a - // different sign when a single big cell dominates — that's exactly the signal we want to surface. - System.Console.ForegroundColor = stats.GeoMeanPct <= 0 ? ConsoleColor.Green : ConsoleColor.Red; - System.Console.WriteLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} │ geo {FormatPctSigned(stats.GeoMeanPct)} │ median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, System.Globalization.CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, System.Globalization.CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); - System.Console.ResetColor(); - } - - /// - /// Same as but appends to a (no color). - /// Used by the .log and .LLM file writers. - /// - private static void AppendOverallLine(StringBuilder sb, string label, string unit, OverallStats? stats, string fmt = "F2") - { - if (stats == null) return; - sb.AppendLine($" {label,-12} arith {FormatPctSigned(stats.ArithMeanPct)} | geo {FormatPctSigned(stats.GeoMeanPct)} | median {FormatPctSigned(stats.MedianPct)} ({stats.AcAvg.ToString(fmt, System.Globalization.CultureInfo.InvariantCulture)} {unit} vs {stats.MpAvg.ToString(fmt, System.Globalization.CultureInfo.InvariantCulture)} {unit}, {stats.CellCount} cells)"); - } - - private static void SaveResults(List results, List testDataSets) - { - Directory.CreateDirectory(Configuration.ResultsDirectory); - - var timestamp = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); - var baseFileName = $"Console.FullBenchmark_{Configuration.BuildConfiguration}_{timestamp}"; - var logFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.log"); - var outputFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.output"); - - // Save binary output to separate .output file - var largeTestData = testDataSets.FirstOrDefault(t => t.Name.StartsWith("Large")); - if (largeTestData != null) - { - var outputSb = new StringBuilder(); - outputSb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); - outputSb.AppendLine("║ SERIALIZED BINARY OUTPUT ║"); - outputSb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); - outputSb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); - outputSb.AppendLine(); - - outputSb.AppendLine("=== SERIALIZED BYTES: Large (5x5x5x10) - AcBinary (Default) ==="); - var serializedBytes = AcBinarySerializer.Serialize(largeTestData.Order, AcBinarySerializerOptions.Default); - outputSb.AppendLine($"Size: {serializedBytes.Length:N0} bytes"); - outputSb.AppendLine(); - outputSb.AppendLine("Hex dump:"); - outputSb.AppendLine(FormatHexDump(serializedBytes)); - - File.WriteAllText(outputFilePath, outputSb.ToString(), Configuration.Utf8NoBom); - System.Console.WriteLine($"✓ Binary output saved to: {outputFilePath}"); - } - - // Save benchmark results to .log file - var sb = new StringBuilder(); - sb.AppendLine("╔══════════════════════════════════════════════════════════════════════════════════════════════════════╗"); - sb.AppendLine("║ SERIALIZER BENCHMARK RESULTS ║"); - sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║"); - sb.AppendLine($"║ Build: {Configuration.BuildConfiguration}".PadRight(100) + "║"); - sb.AppendLine($"║ Charset: {GetCurrentCharsetName()}".PadRight(100) + "║"); - sb.AppendLine($"║ Iterations: per-cell adaptive (~{Configuration.TargetSampleMs} ms target)".PadRight(100) + "║"); - sb.AppendLine($"║ Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded".PadRight(100) + "║"); - sb.AppendLine($"║ Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}".PadRight(100) + "║"); - sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝"); - sb.AppendLine(); - - // Serializer options summary - var optionsMap = results - .Where(r => r.OptionsDescription != null) - .Select(r => (r.SerializerName, r.OptionsDescription!)) - .Distinct() - .ToList(); - if (optionsMap.Count > 0) - { - sb.AppendLine("=== SERIALIZER OPTIONS ==="); - foreach (var (name, opts) in optionsMap) - sb.AppendLine($" {name}: {opts}"); - sb.AppendLine(); - } - - // CSV-like data for easy import — keeps raw byte integers (no KB rounding) so external tools can compute precisely. - sb.AppendLine("=== RAW DATA (CSV) ==="); - sb.AppendLine("TestData,Engine,IO,Mode,Options,Size,SerializeMicrosPerOp,DeserializeMicrosPerOp,RoundTripMicrosPerOp,SerializeAllocBytesPerOp,DeserializeAllocBytesPerOp,RoundTripAllocBytesPerOp,SetupSerializeAllocBytes,SetupDeserializeAllocBytes"); - - foreach (var testData in testDataSets) - { - var testResults = results.Where(r => r.TestDataName == testData.DisplayName).ToList(); - foreach (var result in testResults) - { - sb.AppendLine($"{result.TestDataName},{result.Engine},{result.IoMode},{result.DispatchMode},{result.OptionsPreset},{result.SerializedSize},{SerPerOp(result):F2},{DesPerOp(result):F2},{RtPerOp(result):F2},{result.SerializeAllocBytesPerOp},{result.DeserializeAllocBytesPerOp},{result.RoundTripAllocBytesPerOp},{result.SetupSerializeAllocBytes},{result.SetupDeserializeAllocBytes}"); - } - } - sb.AppendLine(); - - // Formatted results - sb.AppendLine("=== FORMATTED RESULTS BY TEST DATA ==="); - sb.AppendLine("(►) = Highlighted: MemoryPack (Byte[]) (baseline) and AcBinary (Byte[])"); - sb.AppendLine(); - - foreach (var testData in testDataSets) - { - // Order by per-op µs (iter-independent) — rows may have different iter counts post-calibration. - var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => RtPerOp(r)).ToList(); - var memPackResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)); - // Pin the comparison to AcBinary's SGen variant — apples-to-apples vs MemoryPack (also source-generated). - // The Runtime variant is shown alongside in the table for context, not used as the headline number. - var acBinaryResult = testResults.FirstOrDefault(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)); - - sb.AppendLine(); - sb.AppendLine($"--- {testData.DisplayName} ---"); - sb.AppendLine($"{"#",-4} {"Serializer",-42} {"Size B",-12} {"Setup S/D KB",-14} {"Ser µs/op",-12} {"Des µs/op",-12} {"RT µs/op",-12} {"SerAlc KB",-11} {"DesAlc KB",-11}"); - sb.AppendLine(new string('-', 140)); - - var rank = 1; - foreach (var result in testResults) - { - var isHighlighted = ((result.Engine == Configuration.EngineMemoryPack || result.Engine == Configuration.EngineAcBinary) && result.IoMode == Configuration.IoByteArray); - var prefix = isHighlighted ? "► " : " "; - - var size = $"{result.SerializedSize:N0}"; - var setup = $"{ToKilobytes(result.SetupSerializeAllocBytes):F2} / {ToKilobytes(result.SetupDeserializeAllocBytes):F2}"; - var ser = result.SerializeTimeMs > 0 ? $"{SerPerOp(result):F2}" : "N/A"; - var des = result.DeserializeTimeMs > 0 ? $"{DesPerOp(result):F2}" : "N/A"; - var rt = result.RoundTripTimeMs > 0 ? $"{RtPerOp(result):F2}" : "N/A"; - var serAlloc = result.SerializeTimeMs > 0 ? $"{ToKilobytes(result.SerializeAllocBytesPerOp):F2}" : "N/A"; - var desAlloc = result.DeserializeTimeMs > 0 ? $"{ToKilobytes(result.DeserializeAllocBytesPerOp):F2}" : "N/A"; - - sb.AppendLine($"{rank++,2} {prefix}{result.SerializerName,-40} {size,-12} {setup,-14} {ser,-12} {des,-12} {rt,-12} {serAlloc,-11} {desAlloc,-11}"); - } - - // Summary row for this test data (vs MemoryPack — baseline switched MessagePack → MemoryPack) - if (memPackResult != null && acBinaryResult != null) - { - var sizePct = (acBinaryResult.SerializedSize / (double)memPackResult.SerializedSize - 1) * 100; - // Per-op µs ratio (iter-independent) — Ser/Des may have different iter counts on the two rows. - var serPct = SerPerOp(memPackResult) > 0 ? (SerPerOp(acBinaryResult) / SerPerOp(memPackResult) - 1) * 100 : 0; - var desPct = DesPerOp(memPackResult) > 0 ? (DesPerOp(acBinaryResult) / DesPerOp(memPackResult) - 1) * 100 : 0; - var rtPct = RtPerOp(memPackResult) > 0 ? (RtPerOp(acBinaryResult) / RtPerOp(memPackResult) - 1) * 100 : 0; - - sb.AppendLine($" AcBinary (Byte[]) vs MemoryPack (Byte[]): Size {sizePct:+0;-0}% │ Ser {serPct:+0;-0}% │ Des {desPct:+0;-0}% │ RT {rtPct:+0;-0}%"); - } - - //sb.AppendLine($"GrowBufferCount: {AcBinarySerializer.GrowBufferCount}"); - //sb.AppendLine($"GrowBufferTotalBytes: {AcBinarySerializer.GrowBufferTotalBytes:N0} bytes"); - } - - - // Summary comparison (vs MemoryPack) - // Restrict AcBinary side to SGen — the SGen vs Runtime variants are shown side-by-side - // in the per-test fancy table; the headline should compare apples-to-apples (both source-generated). - sb.AppendLine(); - sb.AppendLine("=== AcBinary (Byte[], SGen) vs MemoryPack (Byte[]) (Overall) ==="); - - var memPackSerResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.SerializeTimeMs > 0).ToList(); - var memPackDesResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.DeserializeTimeMs > 0).ToList(); - var memPackRtResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray) && r.RoundTripTimeMs > 0).ToList(); - - var acBinarySerResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.SerializeTimeMs > 0).ToList(); - var acBinaryDesResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.DeserializeTimeMs > 0).ToList(); - var acBinaryRtResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen) && r.RoundTripTimeMs > 0).ToList(); - - // Skip comparison block if either side has no Byte[] data — happens in AsyncPipe-only mode - // where only NamedPipe rows exist (no MemoryPack baseline, no AcBinary Byte[] reference). - // Mirrors the same early-return guard in PrintGroupedResults. - if (memPackRtResults2.Count == 0 || acBinaryRtResults2.Count == 0) - { - sb.AppendLine(" (Comparison requires both serialize and deserialize data)"); - File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); - System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); - - var llmFilePathEarly = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); - SaveLlmResults(llmFilePathEarly, results, testDataSets); - return; - } - - // Per-cell-paired aggregation: arithmetic / geometric / median. See PrintSummary's parallel - // block + the OverallStats record for the rationale (per-cell ratio vs magnitude-weighted mean). - var sizeAcResults2 = results.Where(r => (r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen)).ToList(); - var sizeMpResults2 = results.Where(r => (r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray)).ToList(); - - AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, SerPerOp)); - AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResults2, memPackSerResults2, r => r.SerializeAllocBytesPerOp), "F0"); - AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, DesPerOp)); - AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResults2, memPackDesResults2, r => r.DeserializeAllocBytesPerOp), "F0"); - AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResults2, memPackRtResults2, RtPerOp)); - AppendOverallLine(sb, "Size", "B", ComputeOverallStats(sizeAcResults2, sizeMpResults2, r => r.SerializedSize), "F0"); - - File.WriteAllText(logFilePath, sb.ToString(), Configuration.Utf8NoBom); - System.Console.WriteLine($"✓ Results saved to: {logFilePath}"); - - // Save LLM-optimized results - var llmFilePath = Path.Combine(Configuration.ResultsDirectory, $"{baseFileName}.LLM"); - SaveLlmResults(llmFilePath, results, testDataSets); - } - - private static void SaveLlmResults(string filePath, List results, List testDataSets) - { - var sb = new StringBuilder(); - var testTypeName = testDataSets.FirstOrDefault()?.TypeName ?? "unknown"; - sb.AppendLine($"# AcBinary Benchmark {Configuration.BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); - sb.AppendLine($"Charset: {GetCurrentCharsetName()} | Iterations: per-cell adaptive (target ~{Configuration.TargetSampleMs} ms/sample) | Warmup: {Configuration.WarmupIterations} per phase (Ser/Des isolated) | Samples: {Configuration.BenchmarkSamples} (median) + 1 pilot discarded | .NET: {Environment.Version} | TestType: {testTypeName} | UnstableCV threshold: {Configuration.UnstableCVThreshold * 100:F0}%"); - sb.AppendLine("Baseline: MemoryPack (Byte[]) (SOTA reference) | Verified: round-trip correctness checked once per cell before warmup"); - - // Options summary - var optionsMap = results - .Where(r => r.OptionsDescription != null) - .Select(r => (r.SerializerName, r.OptionsDescription!)) - .Distinct() - .ToList(); - if (optionsMap.Count > 0) - { - sb.AppendLine(); - sb.AppendLine("## Options"); - sb.AppendLine(); - foreach (var (name, opts) in optionsMap) - sb.AppendLine($"- **{name}**: {opts}"); - } - - // Flat results table sorted by test data then round-trip (now includes Alloc + Iter columns). - // Iter column shows per-row Ser/Des iteration counts (post-adaptive-calibration), so the reader - // can verify that each cell's batch sample landed near the Configuration.TargetSampleMs window. - sb.AppendLine(); - sb.AppendLine("## Results"); - sb.AppendLine(); - sb.AppendLine("TestData | Engine | IO | Mode | Options | Size(B) | Ser(µs/op) | Deser(µs/op) | RT(µs/op) | SerAlloc(KB/op) | DesAlloc(KB/op) | RTAlloc(KB/op) | Setup S/D(KB) | Iter Ser/Des"); - sb.AppendLine("---|---|---|---|---|---|---|---|---|---|---|---|---|---"); - - foreach (var testData in testDataSets) - { - var testResults = results - .Where(r => r.TestDataName == testData.DisplayName) - // Per-op µs (iter-independent) ordering — mixing iter counts within a cell is now expected. - .OrderBy(RtPerOp) - .ToList(); - - foreach (var r in testResults) - { - var inv = System.Globalization.CultureInfo.InvariantCulture; - // Per-cell median + inter-sample range (min..max) + CV-threshold marker (⚠️X.X% when CV > 3%). - // Range surfaces the noise floor for each row so a small inter-engine delta is easy to - // judge against the row's noise. Format: "26.86 (24.50..29.10)" or - // "26.86 (24.50..29.10) ⚠️5.2%" when stddev/mean exceeds the unstable threshold. - // When only one sample was taken (Debug / quick mode) min == max == median; collapse - // to bare median to avoid visual clutter. - var ser = r.SerializeTimeMs > 0 ? FormatMicrosWithRange(r.SerializeTimeMs, r.SerializeTimeMinMs, r.SerializeTimeMaxMs, r.SerializeTimeStdDevMs, r.SerializeIterations, inv) : "-"; - var des = r.DeserializeTimeMs > 0 ? FormatMicrosWithRange(r.DeserializeTimeMs, r.DeserializeTimeMinMs, r.DeserializeTimeMaxMs, r.DeserializeTimeStdDevMs, r.DeserializeIterations, inv) : "-"; - var rt = r.RoundTripTimeMs > 0 - ? (r.IsRoundTripOnly - ? FormatMicrosWithRange(r.RoundTripTimeMs, r.RoundTripTimeMinMs, r.RoundTripTimeMaxMs, r.RoundTripTimeStdDevMs, r.RoundTripIterations, inv) - : RtPerOp(r).ToString("F2", inv)) - : "-"; - - var serAlloc = r.SerializeTimeMs > 0 ? ToKilobytes(r.SerializeAllocBytesPerOp).ToString("F2", inv) : "-"; - var desAlloc = r.DeserializeTimeMs > 0 ? ToKilobytes(r.DeserializeAllocBytesPerOp).ToString("F2", inv) : "-"; - var rtAlloc = r.RoundTripAllocBytesPerOp > 0 ? ToKilobytes(r.RoundTripAllocBytesPerOp).ToString("F2", inv) : "-"; - var setupAlloc = $"{ToKilobytes(r.SetupSerializeAllocBytes).ToString("F2", inv)} / {ToKilobytes(r.SetupDeserializeAllocBytes).ToString("F2", inv)}"; - - // Iter Ser/Des column — per-row adaptive iter counts. RT-only rows show Iter for RT. - var iterCol = r.IsRoundTripOnly - ? r.RoundTripIterations.ToString(inv) - : $"{(r.SerializeIterations > 0 ? r.SerializeIterations.ToString(inv) : "-")} / {(r.DeserializeIterations > 0 ? r.DeserializeIterations.ToString(inv) : "-")}"; - sb.AppendLine($"{r.TestDataName} | {r.Engine} | {r.IoMode} | {r.DispatchMode} | {r.OptionsPreset} | {r.SerializedSize} | {ser} | {des} | {rt} | {serAlloc} | {desAlloc} | {rtAlloc} | {setupAlloc} | {iterCol}"); - } - } - - // Overall AcBinary (SGen, Byte[]) vs MemoryPack (Byte[]) comparison — same three aggregations - // as the .log / console output (arithmetic / geometric / median of per-cell ratios). The - // arith mean is magnitude-weighted (Large cell dominates); geo/median are per-cell-equal - // signals. Adding this lets an LLM diagnose whether a headline delta is a real overall - // win/loss or a single-cell artifact. - var memPackByteArrayResults = results.Where(r => r.Engine == Configuration.EngineMemoryPack && r.IoMode == Configuration.IoByteArray).ToList(); - var acBinarySGenByteArrayResults = results.Where(r => r.Engine == Configuration.EngineAcBinary && r.IoMode == Configuration.IoByteArray && r.DispatchMode == Configuration.ModeSGen).ToList(); - var memPackSerResultsLlm = memPackByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); - var memPackDesResultsLlm = memPackByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); - var memPackRtResultsLlm = memPackByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); - var acBinarySerResultsLlm = acBinarySGenByteArrayResults.Where(r => r.SerializeTimeMs > 0).ToList(); - var acBinaryDesResultsLlm = acBinarySGenByteArrayResults.Where(r => r.DeserializeTimeMs > 0).ToList(); - var acBinaryRtResultsLlm = acBinarySGenByteArrayResults.Where(r => r.RoundTripTimeMs > 0).ToList(); - - if (memPackRtResultsLlm.Count > 0 && acBinaryRtResultsLlm.Count > 0) - { - sb.AppendLine(); - sb.AppendLine("## Overall: AcBinary (Byte[], SGen) vs MemoryPack (Byte[])"); - sb.AppendLine(); - sb.AppendLine("Three aggregations of per-cell results: **arith** = arithmetic mean of µs/op (magnitude-weighted, Large cell dominates); **geo** = geometric mean of per-cell ratios (each cell weighted equally); **median** = median of per-cell ratios (outlier-resistant). Negative % = AcBinary faster/smaller; positive % = MemPack faster/smaller. The geo/median variants surface when a single big cell skews the arithmetic mean."); - sb.AppendLine(); - sb.AppendLine("```"); - AppendOverallLine(sb, "Serialize", "µs/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, SerPerOp)); - AppendOverallLine(sb, "Ser Alloc", "B/op", ComputeOverallStats(acBinarySerResultsLlm, memPackSerResultsLlm, r => r.SerializeAllocBytesPerOp), "F0"); - AppendOverallLine(sb, "Deserialize", "µs/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, DesPerOp)); - AppendOverallLine(sb, "Des Alloc", "B/op", ComputeOverallStats(acBinaryDesResultsLlm, memPackDesResultsLlm, r => r.DeserializeAllocBytesPerOp), "F0"); - AppendOverallLine(sb, "Round-trip", "µs/op", ComputeOverallStats(acBinaryRtResultsLlm, memPackRtResultsLlm, RtPerOp)); - AppendOverallLine(sb, "Size", "B", ComputeOverallStats(acBinarySGenByteArrayResults, memPackByteArrayResults, r => r.SerializedSize), "F0"); - sb.AppendLine("```"); - } - - File.WriteAllText(filePath, sb.ToString(), Configuration.Utf8NoBom); - System.Console.WriteLine($"✓ LLM results saved to: {filePath}"); - } - - /// - /// Formats byte array as hex dump with offset, hex values, and ASCII representation. - /// - private static string FormatHexDump(byte[] bytes, int bytesPerLine = 16) - { - var sb = new StringBuilder(); - for (var i = 0; i < bytes.Length; i += bytesPerLine) - { - // Offset - sb.Append($"{i:X8} "); - - // Hex bytes - for (var j = 0; j < bytesPerLine; j++) - { - if (i + j < bytes.Length) - sb.Append($"{bytes[i + j]:X2} "); - else - sb.Append(" "); - - if (j == 7) sb.Append(' '); // Extra space in middle - } - - sb.Append(" |"); - - // ASCII representation - for (var j = 0; j < bytesPerLine && i + j < bytes.Length; j++) - { - var b = bytes[i + j]; - sb.Append(b is >= 32 and < 127 ? (char)b : '.'); - } - - sb.AppendLine("|"); - } - return sb.ToString(); - } - - #endregion }