using AyCode.Core.Tests.TestModels; using MemoryPack; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text.Json; namespace AyCode.Core.Serializers.Console; /// /// Benchmark execution helpers: timing (), per-cell adaptive iteration /// calibration (), allocation measurement /// ( + ), in-place /// \r-progress reporting, full-GC phase-boundary helper (), /// startup validation (), and per-cell round-trip equality /// (). Pure benchmark-execution infrastructure — no display /// formatting (that lives in Output) and no per-engine glue (which lives with the /// individual ISerializerBenchmark implementations). /// internal static class BenchmarkLoop { /// /// Forces a full GC cycle at a phase boundary in the benchmark loop. Two-pass collect with finalizer drain /// in between: the first pass moves managed garbage to the finalization queue, WaitForPendingFinalizers /// runs the finalizers, the second pass reclaims any objects the finalizers released. After this returns the /// heap is in a known-quiescent state — the next warmup/measurement phase starts on a clean slate, isolated /// from the previous phase's residual allocations (write-buffer pools, intern cache, write-plan arrays, etc.). /// Called between every Ser-phase / Des-phase boundary in RunBenchmarksForTestData. /// [MethodImpl(MethodImplOptions.NoInlining)] internal static void ForceGcCollect() { GC.Collect(2, GCCollectionMode.Forced, blocking: true); GC.WaitForPendingFinalizers(); GC.Collect(2, GCCollectionMode.Forced, blocking: true); } /// /// Runs the action times for independent samples, /// returning the median, min, and max elapsed time. Multi-sample design reduces single-run variance /// from ~±15% to ~±5% by smoothing transient effects (background activity, thermal/turbo state). /// When <= 1, falls back to single-sample timing (Debug / quick mode). /// When is non-null, emits in-place \r progress updates so a /// stuck benchmark (e.g. deadlocked NamedPipe row) is visibly stuck at a specific %% rather than /// silently hanging. /// /// Stabilization (added 2026-05-07): /// 1) Pilot sample is run BEFORE the recorded loop and discarded. The first measurement after /// warmup tends to absorb residual JIT bookkeeping and GC bookkeeping; dropping it tightens /// the min/max range without throwing away signal (the median is the SAME data as before). /// 2) GC.Collect / WaitForPendingFinalizers / GC.Collect runs BEFORE every recorded sample. /// Without this, GC pressure from sample N occasionally triggered a Gen-2 pause inside /// sample N+1, painting it as an outlier; collecting up-front gives every sample the /// same starting heap shape. /// 3) Returns (median, min, max) so the caller can surface the inter-sample range — visible /// noise floor for the row, replacing the previous "median only" view. /// internal static (double medianMs, double minMs, double maxMs, double stdDevMs) RunTimed(Action action, int iterations, string? progressLabel = null) { var samples = Configuration.BenchmarkSamples; if (samples <= 1) { // Single-sample fast path (Debug or trivial run) — no allocation, no sort, no stddev. var sw = Stopwatch.StartNew(); RunWithProgress(action, iterations, progressLabel, samples: 1, sampleIndex: 0); sw.Stop(); var ms = sw.Elapsed.TotalMilliseconds; EndProgress(progressLabel, ms); return (ms, ms, ms, 0); } // Pilot sample (discarded). Counts as sample index 0 of (samples + 1) for progress display // so the user sees an extra "warmup-ish" tick before the recorded samples start. GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var pilotSw = Stopwatch.StartNew(); RunWithProgress(action, iterations, progressLabel, samples + 1, sampleIndex: 0); pilotSw.Stop(); // intentionally not stored var times = new double[samples]; for (var s = 0; s < samples; s++) { // Per-sample GC settle. Forces every sample to start from the same heap state, so // a Gen-2 pause caused by the previous sample doesn't bleed into the next sample's // timing. Cost is paid OUTSIDE the Stopwatch window — no impact on the measurement. GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var sw = Stopwatch.StartNew(); RunWithProgress(action, iterations, progressLabel, samples + 1, sampleIndex: s + 1); sw.Stop(); times[s] = sw.Elapsed.TotalMilliseconds; } // Capture min/max/sum/sumSq BEFORE sort to avoid order ambiguity (Array.Sort is in-place). var minMs = double.MaxValue; var maxMs = double.MinValue; var sum = 0.0; var sumSq = 0.0; for (var i = 0; i < times.Length; i++) { var t = times[i]; sum += t; sumSq += t * t; if (t < minMs) minMs = t; if (t > maxMs) maxMs = t; } // Population stddev (not sample-stddev — we treat the captured samples as the population for // CV computation). variance = E[X²] - E[X]² with Math.Max(0, ...) guard against tiny negative // values from FP rounding when samples are nearly identical. var mean = sum / times.Length; var variance = (sumSq / times.Length) - (mean * mean); var stdDevMs = Math.Sqrt(Math.Max(0.0, variance)); Array.Sort(times); // Median: middle value for odd sample counts, average of two middles for even counts. var medianMs = samples % 2 == 1 ? times[samples / 2] : (times[samples / 2 - 1] + times[samples / 2]) / 2.0; EndProgress(progressLabel, medianMs); return (medianMs, minMs, maxMs, stdDevMs); } /// /// Per-cell adaptive iteration calibration. Runs a 100-iter measurement after warmup and computes /// how many iterations are needed to reach wall-clock per sample. /// Returns iter rounded UP to the nearest 1000, floored at 1000 (the prior fixed minimum) and /// ceiling-capped at 200_000 (sanity bound for pathologically fast ops). In Debug single-sample mode /// (Configuration.BenchmarkSamples <= 1) returns the global unchanged — /// calibration overhead is unjustified there. Calibration runs OUTSIDE the timed sample loop and /// does NOT count toward warmup; its sole purpose is to measure per-op cost. /// internal static int CalibrateIterations(Action action, int targetMs) { if (Configuration.BenchmarkSamples <= 1) return Configuration.TestIterations; // Debug fast path GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); const int calibIter = 100; var sw = Stopwatch.StartNew(); for (var i = 0; i < calibIter; i++) action(); sw.Stop(); var ms = sw.Elapsed.TotalMilliseconds; // Pathologically-fast op below Stopwatch resolution — cap at ceiling (further calibration won't help). if (ms <= 0.0001) return 200_000; var iterPerMs = calibIter / ms; var raw = (int)Math.Ceiling(targetMs * iterPerMs); // Round UP to nearest 1000 — keeps numbers human-readable in the markdown output. var rounded = ((raw + 999) / 1000) * 1000; return rounded switch { < 1000 => 1000, > 200_000 => 200_000, _ => rounded }; } /// /// Measures per-call allocation in bytes after a clean GC. Single dedicated sample (no median) — keeps timing samples pure. /// internal static long MeasureAllocation(Action action, int iterations, string? progressLabel = null) { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var sw = Stopwatch.StartNew(); var before = GC.GetAllocatedBytesForCurrentThread(); RunWithProgress(action, iterations, progressLabel, samples: 1, sampleIndex: 0); var after = GC.GetAllocatedBytesForCurrentThread(); sw.Stop(); EndProgress(progressLabel, sw.Elapsed.TotalMilliseconds); return (after - before) / iterations; } /// /// Process-wide allocation measurement — needed for round-trip-only benchmarks (NamedPipe etc.) where /// the work happens across multiple threads. would /// only count the caller-thread allocations, missing the server-side new byte[len] buffers and /// any drain-pump-thread allocations. covers the entire process. /// Slightly noisier than the per-thread variant (background threads / GC bookkeeping leak in), but /// over 1000 iterations the signal dominates. /// internal static long MeasureAllocationTotal(Action action, int iterations, string? progressLabel = null) { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var sw = Stopwatch.StartNew(); var before = GC.GetTotalAllocatedBytes(precise: true); RunWithProgress(action, iterations, progressLabel, samples: 1, sampleIndex: 0); var after = GC.GetTotalAllocatedBytes(precise: true); sw.Stop(); EndProgress(progressLabel, sw.Elapsed.TotalMilliseconds); return (after - before) / iterations; } // ============================================================================================ // Progress reporting — \r-driven in-place updates so a stuck benchmark surfaces the exact phase // and % where it stopped, instead of appearing as a silent hang. Used by RunTimed and the // MeasureAllocation* helpers when the caller passes a non-null progressLabel. // ============================================================================================ // Tracks the longest line written by the current progress session, so EndProgress can clear // any leftover characters from a prior longer line (avoids "ghost" trailing chars after \r). private static int _progressLastLineLen; /// /// Runs times, emitting \r-overwriting /// progress every ~10% (approx. 10 progress prints per sample). When /// is null, runs without any progress output (zero overhead beyond a null check per iter). /// private static void RunWithProgress(Action action, int iterations, string? label, int samples, int sampleIndex) { if (label is null) { for (var i = 0; i < iterations; i++) action(); return; } // ~10 progress emits per sample run. Avoid emitting on every iter (Console.Write is // expensive enough to skew sub-µs benchmarks if overdone). var step = Math.Max(1, iterations / 10); for (var i = 0; i < iterations; i++) { action(); if ((i + 1) % step == 0 || i == iterations - 1) { var pct = (int)((i + 1) * 100L / iterations); var line = samples > 1 ? $" > {label} sample {sampleIndex + 1}/{samples} {pct,3}% ({i + 1}/{iterations})" : $" > {label} {pct,3}% ({i + 1}/{iterations})"; System.Console.Write('\r'); System.Console.Write(line); if (line.Length < _progressLastLineLen) System.Console.Write(new string(' ', _progressLastLineLen - line.Length)); _progressLastLineLen = line.Length; } } } /// /// Closes a progress line cleanly: clears any leftover chars and writes a final "done" line on /// the same row, terminated by \n so subsequent WriteLine calls render below. /// private static void EndProgress(string? label, double elapsedMs) { if (label is null) return; var done = $" > {label} done in {elapsedMs,7:F1} ms"; System.Console.Write('\r'); System.Console.Write(done); if (done.Length < _progressLastLineLen) System.Console.Write(new string(' ', _progressLastLineLen - done.Length)); System.Console.WriteLine(); _progressLastLineLen = 0; } #if !AYCODE_NATIVEAOT private static readonly JsonSerializerOptions VerifyJsonOpts = new() { WriteIndented = false, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, ReferenceHandler = System.Text.Json.Serialization.ReferenceHandler.IgnoreCycles }; #endif /// /// Round-trip equality check: serialize both via System.Text.Json (canonical form) and compare strings. /// Slower than property-by-property compare, but universal — works for any object graph without custom comparer. /// /// /// AOT publish skip: System.Text.Json's reflection path uses runtime closed-generic instantiation /// (JsonPropertyInfo<TestStatus> et al.) that the trimmer drops, causing /// NotSupportedException: missing native code or metadata. The validation is JIT-only — the actual /// benchmark Serialize/Deserialize loops don't touch this path. Under AOT we return true so all /// VerifyRoundTrip() calls pass without running the cross-format validation. /// internal static bool DeepEqualsViaJson(object? a, object? b) { #if AYCODE_NATIVEAOT // Skip cross-format validation under AOT — STJ reflection path is incompatible. The roundtrip // itself still runs (caller-side Serialize+Deserialize), just the JSON-canonical compare is bypassed. return true; #else if (a == null && b == null) return true; if (a == null || b == null) return false; var jsonA = JsonSerializer.Serialize(a, VerifyJsonOpts); var jsonB = JsonSerializer.Serialize(b, VerifyJsonOpts); return jsonA == jsonB; #endif } /// /// Validates MemoryPack setup at startup. Aborts the benchmark if TestOrder is not [MemoryPackable]. /// Without this attribute, MemoryPack falls back to runtime resolver (slower) — comparison would be INVALID. /// internal static void ValidateMemoryPackSetup() { var typesToCheck = new[] { typeof(TestOrder) }; foreach (var type in typesToCheck) { var hasAttr = type.GetCustomAttributes(typeof(MemoryPackableAttribute), inherit: true).Any(); if (!hasAttr) { System.Console.Error.WriteLine($"❌ FATAL: {type.FullName} is not [MemoryPackable] — MemoryPack would fall back to runtime resolver, comparison is INVALID for SGen-vs-SGen claim."); System.Console.Error.WriteLine("Add [MemoryPackable] to the type and any nested types referenced from it."); Environment.Exit(1); } } } /// /// Filters test data sets by layer keyword. Layered approach lets you run only what's needed for the iteration cadence. /// P1: only "Core" data exists (Small/Medium/Large/Repeated/Deep). Comprehensive and Edge layers will be expanded in P2. /// internal static List FilterByLayer(List all, string layer) { if (layer == "all") return all.ToList(); var coreNames = new[] { "Small", "Medium", "Large", "Repeated", "Deep" }; // P2 will add: "Flat", "Polymorphic", "Collection", "Numeric", "NonAscii", etc. var comprehensiveExtras = new string[] { /* P2 */ }; // P3 will add: "ColdStart", "VeryLarge", "PathologicalString", etc. var edgeExtras = new string[] { /* P3 */ }; return layer switch { "core" => all.Where(t => StartsWithAny(t.Name, coreNames)).ToList(), "comprehensive" => all.Where(t => StartsWithAny(t.Name, coreNames) || StartsWithAny(t.Name, comprehensiveExtras)).ToList(), "edge" => all.Where(t => StartsWithAny(t.Name, coreNames) || StartsWithAny(t.Name, comprehensiveExtras) || StartsWithAny(t.Name, edgeExtras)).ToList(), // Single-cell A/B mini-suite filters — match by case-insensitive prefix on Name. // Use case: tight optimization-iteration loop on one specific cell (e.g. `dotnet run -- repeated` // or interactive menu shortcut), avoiding the full ~110 sec suite when only one cell is in scope. "small" => all.Where(t => t.Name.StartsWith("Small", StringComparison.OrdinalIgnoreCase)).ToList(), "medium" => all.Where(t => t.Name.StartsWith("Medium", StringComparison.OrdinalIgnoreCase)).ToList(), "large" => all.Where(t => t.Name.StartsWith("Large", StringComparison.OrdinalIgnoreCase)).ToList(), "repeated" => all.Where(t => t.Name.StartsWith("Repeated", StringComparison.OrdinalIgnoreCase)).ToList(), "deep" => all.Where(t => t.Name.StartsWith("Deep", StringComparison.OrdinalIgnoreCase)).ToList(), _ => all.ToList() }; static bool StartsWithAny(string name, string[] prefixes) => prefixes.Any(name.StartsWith); } }