Improve benchmark reporting and add LLM-friendly results
- Add TypeName to TestDataSet for clearer test scenario reporting - Display serializer options in console and log outputs - Extend BenchmarkResult with OptionsDescription - Serializer benchmarks now provide detailed config summaries - Log files now include test type and serializer options summary - Generate .LLM Markdown results for LLM consumption and docs - Reference .LLM results in BINARY_IMPLEMENTATION.md for visibility
This commit is contained in:
parent
9150df6982
commit
5ba2684ac4
|
|
@ -225,6 +225,8 @@ internal sealed class TestDataSet
|
||||||
IIdRefPercent = iidRefPercent;
|
IIdRefPercent = iidRefPercent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public string TypeName => Order.GetType().Name;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets display name including IId ref percentage if set.
|
/// Gets display name including IId ref percentage if set.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
|
||||||
|
|
@ -82,13 +82,13 @@ public static class Program
|
||||||
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════╗");
|
System.Console.WriteLine("╔══════════════════════════════════════════════════════════════════════╗");
|
||||||
System.Console.WriteLine("║ COMPREHENSIVE SERIALIZER BENCHMARK SUITE ║");
|
System.Console.WriteLine("║ COMPREHENSIVE SERIALIZER BENCHMARK SUITE ║");
|
||||||
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════╝");
|
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════╝");
|
||||||
System.Console.WriteLine($"Mode: {mode} | Iterations: {TestIterations} | Warmup: {WarmupIterations}");
|
|
||||||
System.Console.WriteLine($"Build: {BuildConfiguration} | .NET: {Environment.Version}");
|
|
||||||
System.Console.WriteLine();
|
|
||||||
|
|
||||||
var allResults = new List<BenchmarkResult>();
|
var allResults = new List<BenchmarkResult>();
|
||||||
var testDataSets = BenchmarkTestDataProvider.CreateTestDataSets();
|
var testDataSets = BenchmarkTestDataProvider.CreateTestDataSets();
|
||||||
|
|
||||||
|
System.Console.WriteLine($"Mode: {mode} | Iterations: {TestIterations} | Warmup: {WarmupIterations}");
|
||||||
|
System.Console.WriteLine($"Build: {BuildConfiguration} | .NET: {Environment.Version} | Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}");
|
||||||
|
System.Console.WriteLine();
|
||||||
|
|
||||||
foreach (var testData in testDataSets)
|
foreach (var testData in testDataSets)
|
||||||
{
|
{
|
||||||
System.Console.WriteLine($"\n{'═'.ToString().PadRight(70, '═')}");
|
System.Console.WriteLine($"\n{'═'.ToString().PadRight(70, '═')}");
|
||||||
|
|
@ -188,6 +188,7 @@ public static class Program
|
||||||
{
|
{
|
||||||
TestDataName = testData.DisplayName, // Use DisplayName for IId% info
|
TestDataName = testData.DisplayName, // Use DisplayName for IId% info
|
||||||
SerializerName = serializer.Name,
|
SerializerName = serializer.Name,
|
||||||
|
OptionsDescription = serializer.OptionsDescription,
|
||||||
SerializedSize = serializer.SerializedSize
|
SerializedSize = serializer.SerializedSize
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -271,6 +272,7 @@ public static class Program
|
||||||
{
|
{
|
||||||
string Name { get; }
|
string Name { get; }
|
||||||
int SerializedSize { get; }
|
int SerializedSize { get; }
|
||||||
|
string? OptionsDescription => null;
|
||||||
void Warmup(int iterations);
|
void Warmup(int iterations);
|
||||||
void Serialize();
|
void Serialize();
|
||||||
void Deserialize();
|
void Deserialize();
|
||||||
|
|
@ -284,6 +286,7 @@ public static class Program
|
||||||
|
|
||||||
public string Name { get; }
|
public string Name { get; }
|
||||||
public int SerializedSize => _serialized.Length;
|
public int SerializedSize => _serialized.Length;
|
||||||
|
public string OptionsDescription => $"WireMode={_options.WireMode}, RefHandling={_options.ReferenceHandling}, Interning={_options.UseStringInterning}, Metadata={_options.UseMetadata}, SGen={_options.UseGeneratedCode}, Compression={_options.UseCompression}";
|
||||||
|
|
||||||
public AcBinaryBenchmark(TestOrder order, AcBinarySerializerOptions options, string name)
|
public AcBinaryBenchmark(TestOrder order, AcBinarySerializerOptions options, string name)
|
||||||
{
|
{
|
||||||
|
|
@ -359,6 +362,7 @@ public static class Program
|
||||||
|
|
||||||
public string Name { get; }
|
public string Name { get; }
|
||||||
public int SerializedSize => _serialized.Length;
|
public int SerializedSize => _serialized.Length;
|
||||||
|
public string OptionsDescription { get; }
|
||||||
|
|
||||||
public MessagePackBenchmark(TestOrder order, string name)
|
public MessagePackBenchmark(TestOrder order, string name)
|
||||||
{
|
{
|
||||||
|
|
@ -369,6 +373,9 @@ public static class Program
|
||||||
//_options = ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4Block);
|
//_options = ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4Block);
|
||||||
_options = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.None);
|
_options = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.None);
|
||||||
|
|
||||||
|
var isContractless = _options.Resolver is ContractlessStandardResolver;
|
||||||
|
OptionsDescription = $"Mode={( isContractless ? "Contractless" : "ContractBased")}, Compression={_options.Compression}";
|
||||||
|
|
||||||
_serialized = MessagePackSerializer.Serialize(order, _options);
|
_serialized = MessagePackSerializer.Serialize(order, _options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -397,6 +404,7 @@ public static class Program
|
||||||
|
|
||||||
public string Name { get; }
|
public string Name { get; }
|
||||||
public int SerializedSize => _serialized.Length;
|
public int SerializedSize => _serialized.Length;
|
||||||
|
public string OptionsDescription => $"WireMode={_options.WireMode}, RefHandling={_options.ReferenceHandling}, Interning={_options.UseStringInterning}, Metadata={_options.UseMetadata}, SGen={_options.UseGeneratedCode}, Compression={_options.UseCompression}";
|
||||||
|
|
||||||
public AcBinaryBufferWriterBenchmark(TestOrder order, AcBinarySerializerOptions options, string name)
|
public AcBinaryBufferWriterBenchmark(TestOrder order, AcBinarySerializerOptions options, string name)
|
||||||
{
|
{
|
||||||
|
|
@ -476,6 +484,7 @@ public static class Program
|
||||||
{
|
{
|
||||||
public string TestDataName { get; set; } = "";
|
public string TestDataName { get; set; } = "";
|
||||||
public string SerializerName { get; set; } = "";
|
public string SerializerName { get; set; } = "";
|
||||||
|
public string? OptionsDescription { get; set; }
|
||||||
public int SerializedSize { get; set; }
|
public int SerializedSize { get; set; }
|
||||||
public double SerializeTimeMs { get; set; }
|
public double SerializeTimeMs { get; set; }
|
||||||
public double DeserializeTimeMs { get; set; }
|
public double DeserializeTimeMs { get; set; }
|
||||||
|
|
@ -496,6 +505,20 @@ public static class Program
|
||||||
System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║");
|
System.Console.WriteLine("║ GROUPED RESULTS BY TEST DATA ║");
|
||||||
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
System.Console.WriteLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||||||
|
|
||||||
|
// Print serializer options
|
||||||
|
var optionsMap = results
|
||||||
|
.Where(r => r.OptionsDescription != null)
|
||||||
|
.Select(r => (r.SerializerName, r.OptionsDescription!))
|
||||||
|
.Distinct()
|
||||||
|
.ToList();
|
||||||
|
if (optionsMap.Count > 0)
|
||||||
|
{
|
||||||
|
System.Console.WriteLine();
|
||||||
|
System.Console.WriteLine(" Serializer Options:");
|
||||||
|
foreach (var (name, opts) in optionsMap)
|
||||||
|
System.Console.WriteLine($" {name}: {opts}");
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var testData in testDataSets)
|
foreach (var testData in testDataSets)
|
||||||
{
|
{
|
||||||
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => r.RoundTripTimeMs).ToList();
|
var testResults = results.Where(r => r.TestDataName == testData.DisplayName).OrderBy(r => r.RoundTripTimeMs).ToList();
|
||||||
|
|
@ -724,9 +747,24 @@ public static class Program
|
||||||
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
|
sb.AppendLine($"║ Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}".PadRight(100) + "║");
|
||||||
sb.AppendLine($"║ Build: {BuildConfiguration}".PadRight(100) + "║");
|
sb.AppendLine($"║ Build: {BuildConfiguration}".PadRight(100) + "║");
|
||||||
sb.AppendLine($"║ Iterations: {TestIterations}".PadRight(100) + "║");
|
sb.AppendLine($"║ Iterations: {TestIterations}".PadRight(100) + "║");
|
||||||
|
sb.AppendLine($"║ Test Type: {testDataSets.FirstOrDefault()?.TypeName ?? "unknown"}".PadRight(100) + "║");
|
||||||
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
sb.AppendLine("╚══════════════════════════════════════════════════════════════════════════════════════════════════════╝");
|
||||||
sb.AppendLine();
|
sb.AppendLine();
|
||||||
|
|
||||||
|
// Serializer options summary
|
||||||
|
var optionsMap = results
|
||||||
|
.Where(r => r.OptionsDescription != null)
|
||||||
|
.Select(r => (r.SerializerName, r.OptionsDescription!))
|
||||||
|
.Distinct()
|
||||||
|
.ToList();
|
||||||
|
if (optionsMap.Count > 0)
|
||||||
|
{
|
||||||
|
sb.AppendLine("=== SERIALIZER OPTIONS ===");
|
||||||
|
foreach (var (name, opts) in optionsMap)
|
||||||
|
sb.AppendLine($" {name}: {opts}");
|
||||||
|
sb.AppendLine();
|
||||||
|
}
|
||||||
|
|
||||||
// CSV-like data for easy import
|
// CSV-like data for easy import
|
||||||
sb.AppendLine("=== RAW DATA (CSV) ===");
|
sb.AppendLine("=== RAW DATA (CSV) ===");
|
||||||
sb.AppendLine("TestData,Serializer,Size,SerializeMs,DeserializeMs,RoundTripMs");
|
sb.AppendLine("TestData,Serializer,Size,SerializeMs,DeserializeMs,RoundTripMs");
|
||||||
|
|
@ -825,6 +863,61 @@ public static class Program
|
||||||
|
|
||||||
File.WriteAllText(logFilePath, sb.ToString(), Utf8NoBom);
|
File.WriteAllText(logFilePath, sb.ToString(), Utf8NoBom);
|
||||||
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
System.Console.WriteLine($"✓ Results saved to: {logFilePath}");
|
||||||
|
|
||||||
|
// Save LLM-optimized results
|
||||||
|
var llmFilePath = Path.Combine(ResultsDirectory, $"{baseFileName}.LLM");
|
||||||
|
SaveLlmResults(llmFilePath, results, testDataSets);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void SaveLlmResults(string filePath, List<BenchmarkResult> results, List<TestDataSet> testDataSets)
|
||||||
|
{
|
||||||
|
var sb = new StringBuilder();
|
||||||
|
var testTypeName = testDataSets.FirstOrDefault()?.TypeName ?? "unknown";
|
||||||
|
sb.AppendLine($"# AcBinary Benchmark {BuildConfiguration} {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
|
||||||
|
sb.AppendLine($"Iterations: {TestIterations} | Warmup: {WarmupIterations} | .NET: {Environment.Version} | TestType: {testTypeName}");
|
||||||
|
|
||||||
|
// Options summary
|
||||||
|
var optionsMap = results
|
||||||
|
.Where(r => r.OptionsDescription != null)
|
||||||
|
.Select(r => (r.SerializerName, r.OptionsDescription!))
|
||||||
|
.Distinct()
|
||||||
|
.ToList();
|
||||||
|
if (optionsMap.Count > 0)
|
||||||
|
{
|
||||||
|
sb.AppendLine();
|
||||||
|
sb.AppendLine("## Options");
|
||||||
|
sb.AppendLine();
|
||||||
|
foreach (var (name, opts) in optionsMap)
|
||||||
|
sb.AppendLine($"- **{name}**: {opts}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flat results table sorted by test data then round-trip
|
||||||
|
sb.AppendLine();
|
||||||
|
sb.AppendLine("## Results");
|
||||||
|
sb.AppendLine();
|
||||||
|
sb.AppendLine("TestData | Serializer | Size(B) | Ser(ms) | Deser(ms) | RT(ms)");
|
||||||
|
sb.AppendLine("---|---|---|---|---|---");
|
||||||
|
|
||||||
|
foreach (var testData in testDataSets)
|
||||||
|
{
|
||||||
|
var testResults = results
|
||||||
|
.Where(r => r.TestDataName == testData.DisplayName)
|
||||||
|
.OrderBy(r => r.RoundTripTimeMs)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
foreach (var r in testResults)
|
||||||
|
{
|
||||||
|
var inv = System.Globalization.CultureInfo.InvariantCulture;
|
||||||
|
var ser = r.SerializeTimeMs > 0 ? r.SerializeTimeMs.ToString("F2", inv) : "-";
|
||||||
|
var des = r.DeserializeTimeMs > 0 ? r.DeserializeTimeMs.ToString("F2", inv) : "-";
|
||||||
|
var rt = r.RoundTripTimeMs > 0 ? r.RoundTripTimeMs.ToString("F2", inv) : "-";
|
||||||
|
sb.AppendLine($"{r.TestDataName} | {r.SerializerName} | {r.SerializedSize} | {ser} | {des} | {rt}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
File.WriteAllText(filePath, sb.ToString(), Utf8NoBom);
|
||||||
|
System.Console.WriteLine($"✓ LLM results saved to: {filePath}");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
Low-level technical decisions, memory management, internal structure of `AcBinarySerializer`. For framework developers modifying the serialization pipeline.
|
Low-level technical decisions, memory management, internal structure of `AcBinarySerializer`. For framework developers modifying the serialization pipeline.
|
||||||
|
|
||||||
> Format spec: `BINARY_FORMAT.md` | Options/presets: `BINARY_OPTIONS.md` | Features: `BINARY_FEATURES.md` | Output writers: `BINARY_WRITERS.md`
|
> Format spec: `BINARY_FORMAT.md` | Options/presets: `BINARY_OPTIONS.md` | Features: `BINARY_FEATURES.md` | Output writers: `BINARY_WRITERS.md`
|
||||||
|
> Benchmark results: `../../Test_Benchmark_Results/Benchmark/*.LLM`
|
||||||
|
|
||||||
## Zero-Allocation Buffer Management
|
## Zero-Allocation Buffer Management
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue