217 lines
12 KiB
C#
217 lines
12 KiB
C#
using AyCode.Core.Benchmarks.Reporting;
|
||
using AyCode.Core.Benchmarks.Workloads.Scenarios;
|
||
using AyCode.Core.Serializers;
|
||
using AyCode.Core.Serializers.Binaries;
|
||
using AyCode.Core.Tests.TestModels;
|
||
using BenchmarkDotNet.Reports;
|
||
using System.Text;
|
||
|
||
namespace AyCode.Core.Benchmarks;
|
||
|
||
/// <summary>
|
||
/// Translates <see cref="BenchmarkDotNet.Reports.Summary"/> (BDN's post-run aggregate) into the unified
|
||
/// <see cref="BenchmarkResult"/> rows consumed by <see cref="BenchmarkReportWriter"/>, then emits the
|
||
/// <c>Bdn.FullBenchmark_*.{log,LLM,output}</c> triplet alongside Console's counterparts in
|
||
/// <see cref="ReportingContext.ResolveResultsDirectory"/>'s <c>Test_Benchmark_Results/Benchmark/</c>.
|
||
///
|
||
/// <para><b>Why a separate adapter</b>: BDN's Summary is per-method (Serialize / Deserialize as separate
|
||
/// <see cref="BenchmarkReport"/>s, parameterised by TestData + Engine). The unified format collapses these
|
||
/// into per-cell rows (one row per <c>TestData × Engine</c> with both Ser and Des stats inline). The adapter
|
||
/// groups, transposes, and converts ns → ms before handing off to the shared writer.</para>
|
||
///
|
||
/// <para><b>Mean vs Median</b>: maps BDN's <see cref="BenchmarkDotNet.Mathematics.Statistics.Median"/> into
|
||
/// the BenchmarkResult's time columns — same convention as Console (which captures sample-median).
|
||
/// Min/Max/StdDev populate the inter-sample range surfaced in <see cref="BenchmarkReportWriter.FormatMicrosWithRange"/>
|
||
/// (incl. CV-warning ⚠️ marker when stddev/median exceeds <see cref="ReportingContext.UnstableCVThreshold"/>).</para>
|
||
///
|
||
/// <para><b>Iteration count = 1</b>: BDN reports <i>per-operation</i> time (ns) — already amortized across N
|
||
/// invocations. The unified BenchmarkResult expects total-batch time + iteration count (so <c>µs/op =
|
||
/// timeMs / iterations * 1000</c>). Storing Mean-in-ms with iterations = 1 makes the same formula yield
|
||
/// Mean-in-µs directly. The actual BDN N count is recorded in the BDN-native artifacts (<c>.../BDN/...</c>)
|
||
/// for anyone who wants the raw invocation count.</para>
|
||
/// </summary>
|
||
public static class BdnSummaryAdapter
|
||
{
|
||
/// <summary>
|
||
/// Post-run entry point — call once after <c>BenchmarkRunner.Run<AcBinaryVsMemPackBenchmark>(...)</c>
|
||
/// returns. Produces the BDN-side <c>Bdn.*</c> file triplet AND prints the grouped-results console table
|
||
/// (same view Console produces post-run) so the user sees the cell-level deltas immediately, without
|
||
/// having to open the .LLM file.
|
||
/// </summary>
|
||
public static void WriteResults(Summary summary)
|
||
{
|
||
// Parent-process counterpart of AcBinaryVsMemPackBenchmark.Setup's charset pin: the BDN child
|
||
// processes ran Latin1Short, but this adapter runs in the parent process where LongStringSuffix
|
||
// would still be the compile-time default (Latin1Long). Set it so GetCharsetName() labels the
|
||
// .LLM correctly AND the CreateTestDataSets()/CreateWorkload calls below compute matching Size(B).
|
||
BenchmarkTestDataProvider.LongStringSuffix = CharsetSuffixes.Latin1Short;
|
||
|
||
var allTestData = BenchmarkTestDataProvider_All_False.CreateTestDataSets();
|
||
var results = Translate(summary, allTestData);
|
||
var ctx = CreateContext();
|
||
|
||
BenchmarkReportWriter.PrintGroupedResults(results, allTestData);
|
||
BenchmarkReportWriter.SaveAll(ctx, results, allTestData);
|
||
}
|
||
|
||
private static ReportingContext CreateContext()
|
||
{
|
||
#if DEBUG
|
||
const string buildConfig = "Debug";
|
||
#elif SGEN_ONLY
|
||
const string buildConfig = "SGenOnly";
|
||
#else
|
||
const string buildConfig = "Release";
|
||
#endif
|
||
return new ReportingContext(
|
||
SourceTag: "Bdn",
|
||
ResultsDirectory: ReportingContext.ResolveResultsDirectory(),
|
||
BuildConfiguration: buildConfig,
|
||
Utf8NoBom: new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
|
||
CharsetName: GetCharsetName(),
|
||
// Warmup / Samples / TargetSampleMs are BDN-managed (not Console's adaptive engine). Zeros here
|
||
// signal "BDN handled internally" in the header; the BDN-native artifacts under .../BDN/ have
|
||
// the exact BDN config (warmup count, iteration count, run strategy) for anyone who needs it.
|
||
WarmupIterations: 0,
|
||
BenchmarkSamples: 0,
|
||
TargetSampleMs: 0,
|
||
UnstableCVThreshold: 0.03);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Looks up the human-readable name for the currently-active <see cref="BenchmarkTestDataProvider.LongStringSuffix"/>
|
||
/// charset. Mirrors Console's <c>Configuration.GetCurrentCharsetName</c>. The BDN serializer benchmark
|
||
/// pins the charset to <c>Latin1Short</c> — set in <see cref="WriteResults"/> (parent process) and in
|
||
/// <c>AcBinaryVsMemPackBenchmark.Setup</c> (child process); see those sites for the process-isolation
|
||
/// rationale (BDN's per-benchmark child processes don't inherit a parent static-field mutation).
|
||
/// </summary>
|
||
private static string GetCharsetName()
|
||
{
|
||
var s = BenchmarkTestDataProvider.LongStringSuffix;
|
||
return s switch
|
||
{
|
||
CharsetSuffixes.Latin1FixAscii => "Latin1FixAscii",
|
||
CharsetSuffixes.AsciiShort => "AsciiShort",
|
||
CharsetSuffixes.AsciiLong => "AsciiLong",
|
||
CharsetSuffixes.Latin1Short => "Latin1Short",
|
||
CharsetSuffixes.Latin1Long => "Latin1Long",
|
||
CharsetSuffixes.CjkBmpShort => "CjkBmpShort",
|
||
CharsetSuffixes.CjkBmpLong => "CjkBmpLong",
|
||
CharsetSuffixes.CyrillicShort => "CyrillicShort",
|
||
CharsetSuffixes.CyrillicLong => "CyrillicLong",
|
||
CharsetSuffixes.MixedShort => "MixedShort",
|
||
CharsetSuffixes.MixedLong => "MixedLong",
|
||
_ => "Custom"
|
||
};
|
||
}
|
||
|
||
private static List<BenchmarkResult> Translate(Summary summary, List<TestDataSet> allTestData)
|
||
{
|
||
var grouped = summary.Reports
|
||
.Where(r => r.Success && r.ResultStatistics != null)
|
||
.GroupBy(r => (
|
||
TestData: GetParam(r, "TestData"),
|
||
Engine: GetParam(r, "Engine")
|
||
))
|
||
.Where(g => !string.IsNullOrEmpty(g.Key.TestData) && !string.IsNullOrEmpty(g.Key.Engine))
|
||
.ToList();
|
||
|
||
var results = new List<BenchmarkResult>(grouped.Count);
|
||
|
||
foreach (var group in grouped)
|
||
{
|
||
var testDataSet = (TestDataSet<TestOrder_All_False>)allTestData.First(t => t.Name.StartsWith(group.Key.TestData));
|
||
var engineEnum = group.Key.Engine switch
|
||
{
|
||
"AcBinary" => BenchmarkEngine.AcBinary,
|
||
"MemoryPack" => BenchmarkEngine.MemoryPack,
|
||
_ => throw new InvalidOperationException($"Unknown engine in BDN params: {group.Key.Engine}")
|
||
};
|
||
|
||
// Construct the same workload instance AcBinaryVsMemPackBenchmark.Setup would build — same options,
|
||
// same wire mode. Reading SerializedSize + OptionsDescription from it keeps the BDN-side metadata
|
||
// in lockstep with what the workload actually serialised (no drift between hardcoded BDN strings
|
||
// and the workload's own OptionsDescription / SerializedSize).
|
||
var workload = CreateWorkload(testDataSet, group.Key.Engine);
|
||
|
||
var result = new BenchmarkResult
|
||
{
|
||
TestDataName = testDataSet.DisplayName,
|
||
Engine = engineEnum,
|
||
IoMode = BenchmarkIoMode.ByteArray,
|
||
DispatchMode = BenchmarkDispatchMode.SGen,
|
||
OptionsPreset = group.Key.Engine == "AcBinary" ? "FastMode" : "Default",
|
||
OrderTypeName = nameof(TestOrder_All_False),
|
||
SerializedSize = workload.SerializedSize,
|
||
OptionsDescription = workload.OptionsDescription,
|
||
};
|
||
|
||
// ns → ms (BenchmarkResult expects ms per op with iter=1, so µs/op = ms * 1000 / 1 = ms*1000).
|
||
const double nsToMs = 1.0 / 1_000_000.0;
|
||
|
||
foreach (var report in group)
|
||
{
|
||
var methodName = report.BenchmarkCase.Descriptor.WorkloadMethod.Name;
|
||
var stats = report.ResultStatistics!;
|
||
var allocBytes = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0;
|
||
|
||
if (methodName == "Serialize")
|
||
{
|
||
result.SerializeTimeMs = stats.Median * nsToMs;
|
||
result.SerializeTimeMinMs = stats.Min * nsToMs;
|
||
result.SerializeTimeMaxMs = stats.Max * nsToMs;
|
||
result.SerializeTimeStdDevMs = stats.StandardDeviation * nsToMs;
|
||
result.SerializeIterations = 1; // see class-doc "Iteration count = 1" note
|
||
result.SerializeAllocBytesPerOp = allocBytes;
|
||
}
|
||
else if (methodName == "Deserialize")
|
||
{
|
||
result.DeserializeTimeMs = stats.Median * nsToMs;
|
||
result.DeserializeTimeMinMs = stats.Min * nsToMs;
|
||
result.DeserializeTimeMaxMs = stats.Max * nsToMs;
|
||
result.DeserializeTimeStdDevMs = stats.StandardDeviation * nsToMs;
|
||
result.DeserializeIterations = 1;
|
||
result.DeserializeAllocBytesPerOp = allocBytes;
|
||
}
|
||
}
|
||
|
||
// Compose RT from Ser + Des per-op µs (same logic as Console BenchmarkLoop's in-memory
|
||
// composition — since BDN measures Ser and Des independently, RT here is the analytic sum).
|
||
var serPerOp = BenchmarkReportWriter.ToPerOpMicros(result.SerializeTimeMs, result.SerializeIterations);
|
||
var desPerOp = BenchmarkReportWriter.ToPerOpMicros(result.DeserializeTimeMs, result.DeserializeIterations);
|
||
var rtPerOp = serPerOp + desPerOp;
|
||
result.RoundTripIterations = Math.Max(result.SerializeIterations, result.DeserializeIterations);
|
||
result.RoundTripTimeMs = rtPerOp / 1000.0 * result.RoundTripIterations;
|
||
result.RoundTripAllocBytesPerOp = result.SerializeAllocBytesPerOp + result.DeserializeAllocBytesPerOp;
|
||
|
||
results.Add(result);
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
private static string GetParam(BenchmarkReport report, string name) =>
|
||
report.BenchmarkCase.Parameters.Items.FirstOrDefault(p => p.Name == name)?.Value?.ToString() ?? "";
|
||
|
||
/// <summary>
|
||
/// Constructs the same workload instance <see cref="AcBinaryVsMemPackBenchmark.Setup"/> would build —
|
||
/// same options, same wire mode. The adapter reads <see cref="ISerializerBenchmark.SerializedSize"/> and
|
||
/// <see cref="ISerializerBenchmark.OptionsDescription"/> from this instance so the BDN-side BenchmarkResult
|
||
/// rows carry the same workload-side metadata the Console rows have (no risk of drift between hardcoded
|
||
/// adapter strings and what the workload actually used).
|
||
///
|
||
/// Cost: one Serialize call inside the ctor per (TestData × Engine) cell — runs once during summary
|
||
/// translation, NOT in BDN's measured hot path. Negligible vs BDN's per-run cost.
|
||
/// </summary>
|
||
private static ISerializerBenchmark CreateWorkload(TestDataSet<TestOrder_All_False> testDataSet, string engine)
|
||
{
|
||
if (engine == "AcBinary")
|
||
{
|
||
var options = AcBinarySerializerOptions.FastMode;
|
||
options.WireMode = WireMode.Compact;
|
||
return new AcBinaryBenchmark<TestOrder_All_False>(testDataSet.Order, options, "FastMode");
|
||
}
|
||
return new MemoryPackBenchmark<TestOrder_All_False>(testDataSet.Order, WireMode.Compact, "Default");
|
||
}
|
||
}
|