using BenchmarkDotNet.Running; using AyCode.Core.Benchmarks; using AyCode.Core.Extensions; using AyCode.Core.Tests.TestModels; using System.Text; using MessagePack; using MessagePack.Resolvers; using BenchmarkDotNet.Configs; using System.IO; using AyCode.Core.Serializers.Jsons; using AyCode.Core.Serializers.Binaries; using System.Diagnostics; namespace AyCode.Benchmark { internal class Program { static void Main(string[] args) { // Ensure centralized results directory and subfolders exist var baseResultsDir = Path.Combine(Directory.GetCurrentDirectory(), "Test_Benchmark_Results"); var mstestDir = Path.Combine(baseResultsDir, "MSTest"); var benchmarkDir = Path.Combine(baseResultsDir, "Benchmark"); var coverageDir = Path.Combine(baseResultsDir, "CoverageReport"); var memDiagDir = Path.Combine(baseResultsDir, "MemDiag"); Directory.CreateDirectory(mstestDir); Directory.CreateDirectory(benchmarkDir); Directory.CreateDirectory(coverageDir); Directory.CreateDirectory(memDiagDir); // Create .gitignore in results folder to keep it out of source control except the file itself var gitignorePath = Path.Combine(baseResultsDir, ".gitignore"); if (!File.Exists(gitignorePath)) { File.WriteAllText(gitignorePath, "*\n!.gitignore\n"); } // If requested, save/move a coverage file into the CoverageReport folder if (args.Length > 0 && args[0] == "--save-coverage") { if (args.Length < 2) { Console.Error.WriteLine("Usage: --save-coverage "); return; } var src = args[1]; if (!File.Exists(src)) { Console.Error.WriteLine("Coverage file not found: " + src); return; } try { var dest = Path.Combine(coverageDir, Path.GetFileName(src)); File.Copy(src, dest, overwrite: true); Console.WriteLine("Coverage file saved to: " + dest); } catch (Exception ex) { Console.Error.WriteLine("Failed to save coverage file: " + ex.Message); } return; } // Configure BenchmarkDotNet to write artifacts into the centralized benchmark directory var config = ManualConfig.Create(DefaultConfig.Instance) .WithArtifactsPath(benchmarkDir); if (args.Length > 0 && args[0] == "--quick") { RunQuickBenchmark(); return; } if (args.Length > 0 && args[0] == "--test") { var (inDir, outDir) = CreateMSTestDeployDirs(mstestDir); RunQuickTest(outDir); return; } if (args.Length > 0 && args[0] == "--testmsgpack") { var (inDir, outDir) = CreateMSTestDeployDirs(mstestDir); RunMessagePackTest(outDir); return; } if (args.Length > 0 && args[0] == "--minimal") { RunBenchmark(config, benchmarkDir, memDiagDir, "MinimalBenchmark"); return; } if (args.Length > 0 && args[0] == "--simple") { RunBenchmark(config, benchmarkDir, memDiagDir, "SimpleBinaryBenchmark"); return; } if (args.Length > 0 && args[0] == "--complex") { RunBenchmark(config, benchmarkDir, memDiagDir, "ComplexBinaryBenchmark"); return; } if (args.Length > 0 && args[0] == "--msgpack") { RunBenchmark(config, benchmarkDir, memDiagDir, "MessagePackComparisonBenchmark"); return; } if (args.Length > 0 && args[0] == "--sizes") { RunSizeComparison(); return; } Console.WriteLine("Usage:"); Console.WriteLine(" --quick Quick benchmark with tabular output (AcBinary vs MessagePack)"); Console.WriteLine(" --test Quick AcBinary test"); Console.WriteLine(" --testmsgpack Quick MessagePack test"); Console.WriteLine(" --minimal Minimal benchmark"); Console.WriteLine(" --simple Simple flat object benchmark"); Console.WriteLine(" --complex Complex hierarchy (AcBinary vs JSON)"); Console.WriteLine(" --msgpack MessagePack comparison"); Console.WriteLine(" --sizes Size comparison only"); Console.WriteLine(" --save-coverage Save coverage file into Test_Benchmark_Results/CoverageReport"); if (args.Length == 0) { BenchmarkSwitcher.FromAssembly(typeof(MinimalBenchmark).Assembly).Run(args, config); // Collect artifacts after running switcher CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, "SwitcherRun"); } else { BenchmarkSwitcher.FromAssembly(typeof(MinimalBenchmark).Assembly).Run(args, config); CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, "SwitcherRun"); } } /// /// Quick benchmark comparing AcBinary vs MessagePack with tabular output. /// Tests: WithRef, NoRef, Serialize, Deserialize, Populate, Merge /// static void RunQuickBenchmark(int iterations = 1000) { Console.WriteLine(); Console.WriteLine("????????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? AcBinary vs MessagePack Quick Benchmark ?"); Console.WriteLine("????????????????????????????????????????????????????????????????????????????????"); Console.WriteLine(); // Create test data with shared references TestDataFactory.ResetIdCounter(); var sharedTag = TestDataFactory.CreateTag("SharedTag"); var sharedUser = TestDataFactory.CreateUser("shareduser"); var sharedMeta = TestDataFactory.CreateMetadata("shared", withChild: true); var testOrder = TestDataFactory.CreateOrder( itemCount: 3, palletsPerItem: 3, measurementsPerPallet: 3, pointsPerMeasurement: 4, sharedTag: sharedTag, sharedUser: sharedUser, sharedMetadata: sharedMeta); // Options var withRefOptions = new AcBinarySerializerOptions(); var noRefOptions = AcBinarySerializerOptions.WithoutReferenceHandling(); var msgPackOptions = ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.None); // Warm up Console.WriteLine("Warming up..."); for (int i = 0; i < 100; i++) { _ = AcBinarySerializer.Serialize(testOrder, withRefOptions); _ = AcBinarySerializer.Serialize(testOrder, noRefOptions); _ = MessagePackSerializer.Serialize(testOrder, msgPackOptions); } // Pre-serialize data for deserialization tests var acBinaryWithRef = AcBinarySerializer.Serialize(testOrder, withRefOptions); var acBinaryNoRef = AcBinarySerializer.Serialize(testOrder, noRefOptions); var msgPackData = MessagePackSerializer.Serialize(testOrder, msgPackOptions); Console.WriteLine($"Iterations: {iterations:N0}"); Console.WriteLine(); // Size comparison Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? SIZE COMPARISON ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? Format ? Size (bytes) ? vs MessagePack ? Savings ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine($"? AcBinary (WithRef) ? {acBinaryWithRef.Length,14:N0} ? {100.0 * acBinaryWithRef.Length / msgPackData.Length,13:F1}% ? {msgPackData.Length - acBinaryWithRef.Length,14:N0} ?"); Console.WriteLine($"? AcBinary (NoRef) ? {acBinaryNoRef.Length,14:N0} ? {100.0 * acBinaryNoRef.Length / msgPackData.Length,13:F1}% ? {msgPackData.Length - acBinaryNoRef.Length,14:N0} ?"); Console.WriteLine($"? MessagePack ? {msgPackData.Length,14:N0} ? {100.0,13:F1}% ? {"(baseline)",14} ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine(); // Benchmark results storage var results = new List<(string Operation, string Mode, double AcBinaryMs, double MsgPackMs)>(); // Serialize benchmarks var sw = Stopwatch.StartNew(); // AcBinary WithRef Serialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = AcBinarySerializer.Serialize(testOrder, withRefOptions); var acWithRefSerialize = sw.Elapsed.TotalMilliseconds; // AcBinary NoRef Serialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = AcBinarySerializer.Serialize(testOrder, noRefOptions); var acNoRefSerialize = sw.Elapsed.TotalMilliseconds; // MessagePack Serialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = MessagePackSerializer.Serialize(testOrder, msgPackOptions); var msgPackSerialize = sw.Elapsed.TotalMilliseconds; results.Add(("Serialize", "WithRef", acWithRefSerialize, msgPackSerialize)); results.Add(("Serialize", "NoRef", acNoRefSerialize, msgPackSerialize)); // Deserialize benchmarks // AcBinary WithRef Deserialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = AcBinaryDeserializer.Deserialize(acBinaryWithRef); var acWithRefDeserialize = sw.Elapsed.TotalMilliseconds; // AcBinary NoRef Deserialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = AcBinaryDeserializer.Deserialize(acBinaryNoRef); var acNoRefDeserialize = sw.Elapsed.TotalMilliseconds; // MessagePack Deserialize sw.Restart(); for (int i = 0; i < iterations; i++) _ = MessagePackSerializer.Deserialize(msgPackData, msgPackOptions); var msgPackDeserialize = sw.Elapsed.TotalMilliseconds; results.Add(("Deserialize", "WithRef", acWithRefDeserialize, msgPackDeserialize)); results.Add(("Deserialize", "NoRef", acNoRefDeserialize, msgPackDeserialize)); // Populate benchmark (AcBinary only) sw.Restart(); for (int i = 0; i < iterations; i++) { var target = CreatePopulateTarget(testOrder); AcBinaryDeserializer.Populate(acBinaryNoRef, target); } var acPopulate = sw.Elapsed.TotalMilliseconds; results.Add(("Populate", "NoRef", acPopulate, 0)); // MessagePack doesn't have Populate // PopulateMerge benchmark (AcBinary only) sw.Restart(); for (int i = 0; i < iterations; i++) { var target = CreatePopulateTarget(testOrder); AcBinaryDeserializer.PopulateMerge(acBinaryNoRef.AsSpan(), target); } var acMerge = sw.Elapsed.TotalMilliseconds; results.Add(("Merge", "NoRef", acMerge, 0)); // Round-trip var acWithRefRoundTrip = acWithRefSerialize + acWithRefDeserialize; var acNoRefRoundTrip = acNoRefSerialize + acNoRefDeserialize; var msgPackRoundTrip = msgPackSerialize + msgPackDeserialize; results.Add(("Round-trip", "WithRef", acWithRefRoundTrip, msgPackRoundTrip)); results.Add(("Round-trip", "NoRef", acNoRefRoundTrip, msgPackRoundTrip)); // Print performance table Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? PERFORMANCE COMPARISON (lower is better) ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? Operation ? AcBinary (ms) ? MessagePack ? Ratio ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); foreach (var r in results) { var opName = $"{r.Operation} ({r.Mode})"; if (r.MsgPackMs > 0) { var ratio = r.AcBinaryMs / r.MsgPackMs; var ratioStr = ratio < 1 ? $"{ratio:F2}x faster" : $"{ratio:F2}x slower"; Console.WriteLine($"? {opName,-24} ? {r.AcBinaryMs,14:F2} ? {r.MsgPackMs,14:F2} ? {ratioStr,14} ?"); } else { Console.WriteLine($"? {opName,-24} ? {r.AcBinaryMs,14:F2} ? {"N/A",14} ? {"(unique)",14} ?"); } } Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine(); // Summary Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine("? SUMMARY ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); var sizeAdvantage = 100.0 - (100.0 * acBinaryNoRef.Length / msgPackData.Length); Console.WriteLine($"? Size advantage: AcBinary is {sizeAdvantage:F1}% smaller than MessagePack ?"); var serializeRatio = acNoRefSerialize / msgPackSerialize; var deserializeRatio = acNoRefDeserialize / msgPackDeserialize; Console.WriteLine($"? Serialize (NoRef): AcBinary is {(serializeRatio < 1 ? $"{1/serializeRatio:F2}x faster" : $"{serializeRatio:F2}x slower"),-20} ?"); Console.WriteLine($"? Deserialize (NoRef): AcBinary is {(deserializeRatio < 1 ? $"{1/deserializeRatio:F2}x faster" : $"{deserializeRatio:F2}x slower"),-18} ?"); Console.WriteLine("???????????????????????????????????????????????????????????????????????????????"); Console.WriteLine(); } static TestOrder CreatePopulateTarget(TestOrder source) { var target = new TestOrder { Id = source.Id }; foreach (var item in source.Items) { target.Items.Add(new TestOrderItem { Id = item.Id }); } return target; } static (string InDir, string OutDir) CreateMSTestDeployDirs(string mstestBase) { var user = Environment.UserName ?? "Deploy"; var ts = DateTime.UtcNow.ToString("yyyyMMddTHHmmss_ffff"); var deployBase = Path.Combine(mstestBase, $"Deploy_{user} {ts}"); var inDir = Path.Combine(deployBase, "In"); var outDir = Path.Combine(deployBase, "Out"); Directory.CreateDirectory(inDir); Directory.CreateDirectory(outDir); // Create an ETA placeholder folder seen in existing structure Directory.CreateDirectory(Path.Combine(inDir, "ETA001")); return (inDir, outDir); } static void RunQuickTest(string outDir) { Console.WriteLine("=== Quick AcBinary Test ===\n"); try { Console.WriteLine("Creating test data..."); var order = TestDataFactory.CreateBenchmarkOrder( itemCount: 3, palletsPerItem: 2, measurementsPerPallet: 2, pointsPerMeasurement: 5); Console.WriteLine($"Created order with {order.Items.Count} items"); Console.WriteLine("\nTesting JSON serialization..."); var jsonOptions = AcJsonSerializerOptions.WithoutReferenceHandling(); var json = AcJsonSerializer.Serialize(order, jsonOptions); // Log a quick summary to Out folder for convenience var logPath = Path.Combine(outDir, "quick_test_log.txt"); File.WriteAllText(logPath, $"QuickTest: Order items={order.Items.Count}, JsonLength={json.Length}\n"); Console.WriteLine("Quick test completed. Log written to: " + logPath); } catch (Exception ex) { Console.Error.WriteLine("Quick test failed: " + ex.Message); } } static void RunMessagePackTest(string outDir) { Console.WriteLine("=== Quick MessagePack Test ===\n"); try { var order = TestDataFactory.CreateBenchmarkOrder(2,1,1,3); var bytes = MessagePackSerializer.Serialize(order, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance)); var logPath = Path.Combine(outDir, "quick_msgpack_test_log.txt"); File.WriteAllText(logPath, $"MessagePack quick test: bytes={bytes.Length}\n"); Console.WriteLine("Quick MessagePack test completed. Log written to: " + logPath); } catch (Exception ex) { Console.Error.WriteLine("Quick MessagePack test failed: " + ex.Message); } } static void RunSizeComparison() { Console.WriteLine("Running size comparisons (output to console)..."); // Existing implementation } static void RunBenchmark(ManualConfig config, string benchmarkDir, string memDiagDir, string name) { // Run benchmark and then collect artifacts into MemDiag folder try { var summary = BenchmarkRunner.Run(config); } finally { CollectBenchmarkArtifacts(benchmarkDir, memDiagDir, name); } } static void CollectBenchmarkArtifacts(string benchmarkDir, string memDiagDir, string runName) { try { if (!Directory.Exists(benchmarkDir)) return; var ts = DateTime.UtcNow.ToString("yyyyMMddTHHmmss_fff"); var destDir = Path.Combine(memDiagDir, $"{runName}_{ts}"); Directory.CreateDirectory(destDir); foreach (var file in Directory.GetFiles(benchmarkDir)) { try { var dest = Path.Combine(destDir, Path.GetFileName(file)); File.Copy(file, dest, overwrite: true); } catch { /* ignore individual copy failures */ } } // Also copy subdirectories (artifact folders) foreach (var dir in Directory.GetDirectories(benchmarkDir)) { try { var name = Path.GetFileName(dir); var target = Path.Combine(destDir, name); CopyDirectory(dir, target); } catch { } } Console.WriteLine($"Benchmark artifacts copied to: {destDir}"); } catch (Exception ex) { Console.Error.WriteLine("Failed to collect benchmark artifacts: " + ex.Message); } } static void CopyDirectory(string sourceDir, string destDir) { Directory.CreateDirectory(destDir); foreach (var file in Directory.GetFiles(sourceDir)) { var dest = Path.Combine(destDir, Path.GetFileName(file)); File.Copy(file, dest, overwrite: true); } foreach (var dir in Directory.GetDirectories(sourceDir)) { CopyDirectory(dir, Path.Combine(destDir, Path.GetFileName(dir))); } } } }