470 lines
21 KiB
C#
470 lines
21 KiB
C#
using BenchmarkDotNet.Running;
|
|
using AyCode.Core.Benchmarks;
|
|
using AyCode.Core.Extensions;
|
|
using AyCode.Core.Tests.TestModels;
|
|
using System.Text;
|
|
using MessagePack;
|
|
using MessagePack.Resolvers;
|
|
using BenchmarkDotNet.Configs;
|
|
using System.IO;
|
|
using AyCode.Core.Serializers.Jsons;
|
|
using AyCode.Core.Serializers.Binaries;
|
|
using System.Diagnostics;
|
|
|
|
namespace AyCode.Benchmark
|
|
{
|
|
internal class Program
|
|
{
|
|
static void Main(string[] args)
|
|
{
|
|
// Ensure centralized results directory and subfolders exist
|
|
var baseResultsDir = Path.Combine(Directory.GetCurrentDirectory(), "Test_Benchmark_Results");
|
|
var mstestDir = Path.Combine(baseResultsDir, "MSTest");
|
|
var benchmarkDir = Path.Combine(baseResultsDir, "Benchmark");
|
|
var coverageDir = Path.Combine(baseResultsDir, "CoverageReport");
|
|
var memDiagDir = Path.Combine(baseResultsDir, "MemDiag");
|
|
|
|
Directory.CreateDirectory(mstestDir);
|
|
Directory.CreateDirectory(benchmarkDir);
|
|
Directory.CreateDirectory(coverageDir);
|
|
Directory.CreateDirectory(memDiagDir);
|
|
|
|
// Create .gitignore in results folder to keep it out of source control except the file itself
|
|
var gitignorePath = Path.Combine(baseResultsDir, ".gitignore");
|
|
if (!File.Exists(gitignorePath))
|
|
{
|
|
File.WriteAllText(gitignorePath, "*\n!.gitignore\n");
|
|
}
|
|
|
|
// If requested, save/move a coverage file into the CoverageReport folder
|
|
if (args.Length > 0 && args[0] == "--save-coverage")
|
|
{
|
|
if (args.Length < 2)
|
|
{
|
|
Console.Error.WriteLine("Usage: --save-coverage <coverage-file-path>");
|
|
return;
|
|
}
|
|
|
|
var src = args[1];
|
|
if (!File.Exists(src))
|
|
{
|
|
Console.Error.WriteLine("Coverage file not found: " + src);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var dest = Path.Combine(coverageDir, Path.GetFileName(src));
|
|
File.Copy(src, dest, overwrite: true);
|
|
Console.WriteLine("Coverage file saved to: " + dest);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Failed to save coverage file: " + ex.Message);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
// Configure BenchmarkDotNet to write artifacts into the centralized benchmark directory
|
|
var config = ManualConfig.Create(DefaultConfig.Instance)
|
|
.WithArtifactsPath(benchmarkDir);
|
|
|
|
if (args.Length > 0 && args[0] == "--quick")
|
|
{
|
|
RunQuickBenchmark();
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--test")
|
|
{
|
|
var (inDir, outDir) = CreateMSTestDeployDirs(mstestDir);
|
|
RunQuickTest(outDir);
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--testmsgpack")
|
|
{
|
|
var (inDir, outDir) = CreateMSTestDeployDirs(mstestDir);
|
|
RunMessagePackTest(outDir);
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--minimal")
|
|
{
|
|
RunBenchmark<MinimalBenchmark>(config, benchmarkDir, memDiagDir, "MinimalBenchmark");
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--simple")
|
|
{
|
|
RunBenchmark<SimpleBinaryBenchmark>(config, benchmarkDir, memDiagDir, "SimpleBinaryBenchmark");
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--complex")
|
|
{
|
|
RunBenchmark<ComplexBinaryBenchmark>(config, benchmarkDir, memDiagDir, "ComplexBinaryBenchmark");
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--msgpack")
|
|
{
|
|
RunBenchmark<MessagePackComparisonBenchmark>(config, benchmarkDir, memDiagDir, "MessagePackComparisonBenchmark");
|
|
return;
|
|
}
|
|
|
|
if (args.Length > 0 && args[0] == "--sizes")
|
|
{
|
|
RunSizeComparison();
|
|
return;
|
|
}
|
|
|
|
Console.WriteLine("Usage:");
|
|
Console.WriteLine(" --quick Quick benchmark with tabular output (AcBinary vs MessagePack)");
|
|
Console.WriteLine(" --test Quick AcBinary test");
|
|
Console.WriteLine(" --testmsgpack Quick MessagePack test");
|
|
Console.WriteLine(" --minimal Minimal benchmark");
|
|
Console.WriteLine(" --simple Simple flat object benchmark");
|
|
Console.WriteLine(" --complex Complex hierarchy (AcBinary vs JSON)");
|
|
Console.WriteLine(" --msgpack MessagePack comparison");
|
|
Console.WriteLine(" --sizes Size comparison only");
|
|
Console.WriteLine(" --save-coverage <file> Save coverage file into Test_Benchmark_Results/CoverageReport");
|
|
|
|
if (args.Length == 0)
|
|
{
|
|
BenchmarkSwitcher.FromAssembly(typeof(MinimalBenchmark).Assembly).Run(args, config);
|
|
// Collect artifacts after running switcher
|
|
CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, "SwitcherRun");
|
|
}
|
|
else
|
|
{
|
|
BenchmarkSwitcher.FromAssembly(typeof(MinimalBenchmark).Assembly).Run(args, config);
|
|
CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, "SwitcherRun");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Quick benchmark comparing AcBinary vs MessagePack with tabular output.
|
|
/// Tests: WithRef, NoRef, Serialize, Deserialize, Populate, Merge
|
|
/// </summary>
|
|
static void RunQuickBenchmark(int iterations = 1000)
|
|
{
|
|
Console.WriteLine();
|
|
Console.WriteLine("????????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? AcBinary vs MessagePack Quick Benchmark ?");
|
|
Console.WriteLine("????????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine();
|
|
|
|
// Create test data with shared references
|
|
TestDataFactory.ResetIdCounter();
|
|
var sharedTag = TestDataFactory.CreateTag("SharedTag");
|
|
var sharedUser = TestDataFactory.CreateUser("shareduser");
|
|
var sharedMeta = TestDataFactory.CreateMetadata("shared", withChild: true);
|
|
|
|
var testOrder = TestDataFactory.CreateOrder(
|
|
itemCount: 3,
|
|
palletsPerItem: 3,
|
|
measurementsPerPallet: 3,
|
|
pointsPerMeasurement: 4,
|
|
sharedTag: sharedTag,
|
|
sharedUser: sharedUser,
|
|
sharedMetadata: sharedMeta);
|
|
|
|
// Options
|
|
var withRefOptions = new AcBinarySerializerOptions();
|
|
var noRefOptions = AcBinarySerializerOptions.WithoutReferenceHandling();
|
|
var msgPackOptions = ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.None);
|
|
|
|
// Warm up
|
|
Console.WriteLine("Warming up...");
|
|
for (int i = 0; i < 100; i++)
|
|
{
|
|
_ = AcBinarySerializer.Serialize(testOrder, withRefOptions);
|
|
_ = AcBinarySerializer.Serialize(testOrder, noRefOptions);
|
|
_ = MessagePackSerializer.Serialize(testOrder, msgPackOptions);
|
|
}
|
|
|
|
// Pre-serialize data for deserialization tests
|
|
var acBinaryWithRef = AcBinarySerializer.Serialize(testOrder, withRefOptions);
|
|
var acBinaryNoRef = AcBinarySerializer.Serialize(testOrder, noRefOptions);
|
|
var msgPackData = MessagePackSerializer.Serialize(testOrder, msgPackOptions);
|
|
|
|
Console.WriteLine($"Iterations: {iterations:N0}");
|
|
Console.WriteLine();
|
|
|
|
// Size comparison
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? SIZE COMPARISON ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? Format ? Size (bytes) ? vs MessagePack ? Savings ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine($"? AcBinary (WithRef) ? {acBinaryWithRef.Length,14:N0} ? {100.0 * acBinaryWithRef.Length / msgPackData.Length,13:F1}% ? {msgPackData.Length - acBinaryWithRef.Length,14:N0} ?");
|
|
Console.WriteLine($"? AcBinary (NoRef) ? {acBinaryNoRef.Length,14:N0} ? {100.0 * acBinaryNoRef.Length / msgPackData.Length,13:F1}% ? {msgPackData.Length - acBinaryNoRef.Length,14:N0} ?");
|
|
Console.WriteLine($"? MessagePack ? {msgPackData.Length,14:N0} ? {100.0,13:F1}% ? {"(baseline)",14} ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine();
|
|
|
|
// Benchmark results storage
|
|
var results = new List<(string Operation, string Mode, double AcBinaryMs, double MsgPackMs)>();
|
|
|
|
// Serialize benchmarks
|
|
var sw = Stopwatch.StartNew();
|
|
|
|
// AcBinary WithRef Serialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = AcBinarySerializer.Serialize(testOrder, withRefOptions);
|
|
var acWithRefSerialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
// AcBinary NoRef Serialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = AcBinarySerializer.Serialize(testOrder, noRefOptions);
|
|
var acNoRefSerialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
// MessagePack Serialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = MessagePackSerializer.Serialize(testOrder, msgPackOptions);
|
|
var msgPackSerialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
results.Add(("Serialize", "WithRef", acWithRefSerialize, msgPackSerialize));
|
|
results.Add(("Serialize", "NoRef", acNoRefSerialize, msgPackSerialize));
|
|
|
|
// Deserialize benchmarks
|
|
// AcBinary WithRef Deserialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = AcBinaryDeserializer.Deserialize<TestOrder>(acBinaryWithRef);
|
|
var acWithRefDeserialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
// AcBinary NoRef Deserialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = AcBinaryDeserializer.Deserialize<TestOrder>(acBinaryNoRef);
|
|
var acNoRefDeserialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
// MessagePack Deserialize
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
_ = MessagePackSerializer.Deserialize<TestOrder>(msgPackData, msgPackOptions);
|
|
var msgPackDeserialize = sw.Elapsed.TotalMilliseconds;
|
|
|
|
results.Add(("Deserialize", "WithRef", acWithRefDeserialize, msgPackDeserialize));
|
|
results.Add(("Deserialize", "NoRef", acNoRefDeserialize, msgPackDeserialize));
|
|
|
|
// Populate benchmark (AcBinary only)
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
{
|
|
var target = CreatePopulateTarget(testOrder);
|
|
AcBinaryDeserializer.Populate(acBinaryNoRef, target);
|
|
}
|
|
var acPopulate = sw.Elapsed.TotalMilliseconds;
|
|
results.Add(("Populate", "NoRef", acPopulate, 0)); // MessagePack doesn't have Populate
|
|
|
|
// PopulateMerge benchmark (AcBinary only)
|
|
sw.Restart();
|
|
for (int i = 0; i < iterations; i++)
|
|
{
|
|
var target = CreatePopulateTarget(testOrder);
|
|
AcBinaryDeserializer.PopulateMerge(acBinaryNoRef.AsSpan(), target);
|
|
}
|
|
var acMerge = sw.Elapsed.TotalMilliseconds;
|
|
results.Add(("Merge", "NoRef", acMerge, 0));
|
|
|
|
// Round-trip
|
|
var acWithRefRoundTrip = acWithRefSerialize + acWithRefDeserialize;
|
|
var acNoRefRoundTrip = acNoRefSerialize + acNoRefDeserialize;
|
|
var msgPackRoundTrip = msgPackSerialize + msgPackDeserialize;
|
|
results.Add(("Round-trip", "WithRef", acWithRefRoundTrip, msgPackRoundTrip));
|
|
results.Add(("Round-trip", "NoRef", acNoRefRoundTrip, msgPackRoundTrip));
|
|
|
|
// Print performance table
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? PERFORMANCE COMPARISON (lower is better) ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? Operation ? AcBinary (ms) ? MessagePack ? Ratio ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
|
|
foreach (var r in results)
|
|
{
|
|
var opName = $"{r.Operation} ({r.Mode})";
|
|
if (r.MsgPackMs > 0)
|
|
{
|
|
var ratio = r.AcBinaryMs / r.MsgPackMs;
|
|
var ratioStr = ratio < 1 ? $"{ratio:F2}x faster" : $"{ratio:F2}x slower";
|
|
Console.WriteLine($"? {opName,-24} ? {r.AcBinaryMs,14:F2} ? {r.MsgPackMs,14:F2} ? {ratioStr,14} ?");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"? {opName,-24} ? {r.AcBinaryMs,14:F2} ? {"N/A",14} ? {"(unique)",14} ?");
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine();
|
|
|
|
// Summary
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine("? SUMMARY ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
var sizeAdvantage = 100.0 - (100.0 * acBinaryNoRef.Length / msgPackData.Length);
|
|
Console.WriteLine($"? Size advantage: AcBinary is {sizeAdvantage:F1}% smaller than MessagePack ?");
|
|
|
|
var serializeRatio = acNoRefSerialize / msgPackSerialize;
|
|
var deserializeRatio = acNoRefDeserialize / msgPackDeserialize;
|
|
Console.WriteLine($"? Serialize (NoRef): AcBinary is {(serializeRatio < 1 ? $"{1/serializeRatio:F2}x faster" : $"{serializeRatio:F2}x slower"),-20} ?");
|
|
Console.WriteLine($"? Deserialize (NoRef): AcBinary is {(deserializeRatio < 1 ? $"{1/deserializeRatio:F2}x faster" : $"{deserializeRatio:F2}x slower"),-18} ?");
|
|
Console.WriteLine("???????????????????????????????????????????????????????????????????????????????");
|
|
Console.WriteLine();
|
|
}
|
|
|
|
static TestOrder CreatePopulateTarget(TestOrder source)
|
|
{
|
|
var target = new TestOrder { Id = source.Id };
|
|
foreach (var item in source.Items)
|
|
{
|
|
target.Items.Add(new TestOrderItem { Id = item.Id });
|
|
}
|
|
return target;
|
|
}
|
|
|
|
static (string InDir, string OutDir) CreateMSTestDeployDirs(string mstestBase)
|
|
{
|
|
var user = Environment.UserName ?? "Deploy";
|
|
var ts = DateTime.UtcNow.ToString("yyyyMMddTHHmmss_ffff");
|
|
var deployBase = Path.Combine(mstestBase, $"Deploy_{user} {ts}");
|
|
var inDir = Path.Combine(deployBase, "In");
|
|
var outDir = Path.Combine(deployBase, "Out");
|
|
Directory.CreateDirectory(inDir);
|
|
Directory.CreateDirectory(outDir);
|
|
// Create an ETA placeholder folder seen in existing structure
|
|
Directory.CreateDirectory(Path.Combine(inDir, "ETA001"));
|
|
return (inDir, outDir);
|
|
}
|
|
|
|
static void RunQuickTest(string outDir)
|
|
{
|
|
Console.WriteLine("=== Quick AcBinary Test ===\n");
|
|
|
|
try
|
|
{
|
|
Console.WriteLine("Creating test data...");
|
|
var order = TestDataFactory.CreateBenchmarkOrder(
|
|
itemCount: 3,
|
|
palletsPerItem: 2,
|
|
measurementsPerPallet: 2,
|
|
pointsPerMeasurement: 5);
|
|
Console.WriteLine($"Created order with {order.Items.Count} items");
|
|
|
|
Console.WriteLine("\nTesting JSON serialization...");
|
|
var jsonOptions = AcJsonSerializerOptions.WithoutReferenceHandling();
|
|
var json = AcJsonSerializer.Serialize(order, jsonOptions);
|
|
|
|
// Log a quick summary to Out folder for convenience
|
|
var logPath = Path.Combine(outDir, "quick_test_log.txt");
|
|
File.WriteAllText(logPath, $"QuickTest: Order items={order.Items.Count}, JsonLength={json.Length}\n");
|
|
|
|
Console.WriteLine("Quick test completed. Log written to: " + logPath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Quick test failed: " + ex.Message);
|
|
}
|
|
}
|
|
|
|
static void RunMessagePackTest(string outDir)
|
|
{
|
|
Console.WriteLine("=== Quick MessagePack Test ===\n");
|
|
try
|
|
{
|
|
var order = TestDataFactory.CreateBenchmarkOrder(2,1,1,3);
|
|
var bytes = MessagePackSerializer.Serialize(order, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance));
|
|
|
|
var logPath = Path.Combine(outDir, "quick_msgpack_test_log.txt");
|
|
File.WriteAllText(logPath, $"MessagePack quick test: bytes={bytes.Length}\n");
|
|
|
|
Console.WriteLine("Quick MessagePack test completed. Log written to: " + logPath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Quick MessagePack test failed: " + ex.Message);
|
|
}
|
|
}
|
|
|
|
static void RunSizeComparison()
|
|
{
|
|
Console.WriteLine("Running size comparisons (output to console)...");
|
|
// Existing implementation
|
|
}
|
|
|
|
static void RunBenchmark<T>(ManualConfig config, string benchmarkDir, string memDiagDir, string name)
|
|
{
|
|
// Run benchmark and then collect artifacts into MemDiag folder
|
|
try
|
|
{
|
|
var summary = BenchmarkRunner.Run<T>(config);
|
|
}
|
|
finally
|
|
{
|
|
CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, name);
|
|
}
|
|
}
|
|
|
|
static void CollectBenchmarkArtifacts(string benchmarkDir, string memDiagDir, string runName)
|
|
{
|
|
try
|
|
{
|
|
if (!Directory.Exists(benchmarkDir)) return;
|
|
var ts = DateTime.UtcNow.ToString("yyyyMMddTHHmmss_fff");
|
|
var destDir = Path.Combine(memDiagDir, $"{runName}_{ts}");
|
|
Directory.CreateDirectory(destDir);
|
|
|
|
foreach (var file in Directory.GetFiles(benchmarkDir))
|
|
{
|
|
try
|
|
{
|
|
var dest = Path.Combine(destDir, Path.GetFileName(file));
|
|
File.Copy(file, dest, overwrite: true);
|
|
}
|
|
catch { /* ignore individual copy failures */ }
|
|
}
|
|
|
|
// Also copy subdirectories (artifact folders)
|
|
foreach (var dir in Directory.GetDirectories(benchmarkDir))
|
|
{
|
|
try
|
|
{
|
|
var name = Path.GetFileName(dir);
|
|
var target = Path.Combine(destDir, name);
|
|
CopyDirectory(dir, target);
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
Console.WriteLine($"Benchmark artifacts copied to: {destDir}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Failed to collect benchmark artifacts: " + ex.Message);
|
|
}
|
|
}
|
|
|
|
static void CopyDirectory(string sourceDir, string destDir)
|
|
{
|
|
Directory.CreateDirectory(destDir);
|
|
foreach (var file in Directory.GetFiles(sourceDir))
|
|
{
|
|
var dest = Path.Combine(destDir, Path.GetFileName(file));
|
|
File.Copy(file, dest, overwrite: true);
|
|
}
|
|
foreach (var dir in Directory.GetDirectories(sourceDir))
|
|
{
|
|
CopyDirectory(dir, Path.Combine(destDir, Path.GetFileName(dir)));
|
|
}
|
|
}
|
|
}
|
|
}
|