From b5680bc0e4dd52caf09e61a3dd047f82a089db78 Mon Sep 17 00:00:00 2001 From: Loretta Date: Mon, 23 Feb 2026 16:01:37 +0100 Subject: [PATCH] Improve circular reference handling in binary serializer Introduce new test models for circular refs, update tests to stress reference handling, and enhance deserializer to support ObjectRefFirst/WithMetadataRefFirst type codes. Fix intern cache index assignment, track generated readers in TypeMetadataWrapper, and disable UseGeneratedCode by default. Update benchmarks for reliability and diagnostics. These changes strengthen reference resolution, circular ref support, and performance. --- AyCode.Core.Serializers.Console/Program.cs | 2 +- .../AcBinarySerializerIIdReferenceTests.cs | 17 ++++-- .../Serialization/QuickBenchmark.cs | 26 ++++++++- .../TestModels/SharedTestModels.cs | 57 +++++++++++++++++++ .../Binaries/AcBinaryDeserializer.Populate.cs | 16 +++++- .../Binaries/AcBinaryDeserializer.cs | 45 +++++++++++++-- ...rySerializer.BinarySerializationContext.cs | 2 +- .../Binaries/AcBinarySerializerOptions.cs | 2 +- .../Serializers/TypeMetadataWrapper.cs | 9 +-- 9 files changed, 153 insertions(+), 23 deletions(-) diff --git a/AyCode.Core.Serializers.Console/Program.cs b/AyCode.Core.Serializers.Console/Program.cs index eeb06d6..a645910 100644 --- a/AyCode.Core.Serializers.Console/Program.cs +++ b/AyCode.Core.Serializers.Console/Program.cs @@ -50,7 +50,7 @@ public static class Program private static int TestIterations = 1; #else private static int WarmupIterations = 5000; - private static int TestIterations = 5000; + private static int TestIterations = 1000; //private static int WarmupIterations = 5000; //private static int TestIterations = 2000; diff --git a/AyCode.Core.Tests/Serialization/AcBinarySerializerIIdReferenceTests.cs b/AyCode.Core.Tests/Serialization/AcBinarySerializerIIdReferenceTests.cs index 6ea3363..0253c96 100644 --- a/AyCode.Core.Tests/Serialization/AcBinarySerializerIIdReferenceTests.cs +++ b/AyCode.Core.Tests/Serialization/AcBinarySerializerIIdReferenceTests.cs @@ -100,7 +100,7 @@ public class AcBinarySerializerIIdReferenceTests var sharedTag = new SharedTag { Id = 1, Name = "ImportantTag", Color = "#FF0000" }; var sharedUser = new SharedUser { Id = 1, Preferences = userPreferences }; - var order = new TestOrder + var order = new TestOrder_Circ_Ref { Id = 1, OrderNumber = "ORD-001", @@ -108,18 +108,22 @@ public class AcBinarySerializerIIdReferenceTests Owner = sharedUser, Items = [ - new TestOrderItem { Id = 1, ProductName = "Product-A", Tag = sharedTag, Assignee = sharedUser }, - new TestOrderItem { Id = 2, ProductName = "Product-B", Tag = sharedTag, Assignee = new SharedUser { Id = 2, Preferences = userPreferences }}, - new TestOrderItem { Id = 3, ProductName = "Product-C", Tag = sharedTag, Assignee = new SharedUser { Id = 3, Preferences = userPreferences } } + new TestOrderItem_Circ_Ref { Id = 1, ProductName = "Product-A", Tag = sharedTag, Assignee = sharedUser }, + new TestOrderItem_Circ_Ref { Id = 2, ProductName = "Product-B", Tag = sharedTag, Assignee = new SharedUser { Id = 2, Preferences = userPreferences }}, + new TestOrderItem_Circ_Ref { Id = 3, ProductName = "Product-C", Tag = sharedTag, Assignee = new SharedUser { Id = 3, Preferences = userPreferences } } ] }; + //order.Parent = order.Items[0]; + order.Parent = userPreferences; + order.Items[0].ParentOrder = order; + var options = new AcBinarySerializerOptions { ReferenceHandling = mode }; // Act var binary = AcBinarySerializer.Serialize(order, options); - WriteBinaryToConsole(binary); - var result = binary.BinaryTo(); // Options from header + //WriteBinaryToConsole(binary); + var result = binary.BinaryTo(); // Options from header var objectRefCount = CountObjectRefs(binary, false); Console.WriteLine($"Binary size: {binary.Length} bytes"); @@ -129,6 +133,7 @@ public class AcBinarySerializerIIdReferenceTests switch (mode) { case ReferenceHandlingMode.None: + //none esetén miért nincs infinite loop??? - J. Assert.AreEqual(0, objectRefCount, $"[{mode}] Should have 0 ObjectRefs"); break; diff --git a/AyCode.Core.Tests/Serialization/QuickBenchmark.cs b/AyCode.Core.Tests/Serialization/QuickBenchmark.cs index 3396f1a..e1002ae 100644 --- a/AyCode.Core.Tests/Serialization/QuickBenchmark.cs +++ b/AyCode.Core.Tests/Serialization/QuickBenchmark.cs @@ -130,7 +130,7 @@ public class QuickBenchmark var deserializeMs = sw.Elapsed.TotalMilliseconds; // JSON comparison - var jsonOptions = AcJsonSerializerOptions.WithoutReferenceHandling; + var jsonOptions = AcJsonSerializerOptions.Default; sw.Restart(); string json = null!; for (int i = 0; i < iterations; i++) @@ -512,6 +512,7 @@ public class QuickBenchmark var noRefOptions = AcBinarySerializerOptions.WithoutReferenceHandling; noRefOptions.UseMetadata = false; + //noRefOptions.ReferenceHandling = ReferenceHandlingMode.All; //noRefOptions.UseStringInterning = StringInterningMode.None; // Pre-serialize @@ -519,15 +520,20 @@ public class QuickBenchmark var acBinaryNoRef = AcBinarySerializer.Serialize(testOrder, noRefOptions); var msgPackData = MessagePackSerializer.Serialize(testOrder, MsgPackOptions); + Console.WriteLine("withRefOptions:"); + WriteBinaryToConsole(acBinaryWithRef); + // Warmup - MUST be 1000+ iterations for tiered JIT to complete Console.WriteLine($"\nSerialize warming up ({DefaultIterations} iterations each)..."); - for (int i = 0; i < 500; i++) + for (int i = 0; i < DefaultIterations; i++) { _ = AcBinarySerializer.Serialize(testOrder, withRefOptions); _ = AcBinarySerializer.Serialize(testOrder, noRefOptions); _ = MessagePackSerializer.Serialize(testOrder, MsgPackOptions); + Console.WriteLine("acBinaryWithRef"); _ = AcBinaryDeserializer.Deserialize(acBinaryWithRef); + Console.WriteLine("acBinaryNoRef"); _ = AcBinaryDeserializer.Deserialize(acBinaryNoRef); _ = MessagePackSerializer.Deserialize(msgPackData, MsgPackOptions); } @@ -724,13 +730,20 @@ public class QuickBenchmark pointsPerMeasurement: 4); var options = AcBinarySerializerOptions.WithoutReferenceHandling; + //options.ReferenceHandling = ReferenceHandlingMode.All; var binaryData = AcBinarySerializer.Serialize(testOrder, options); + WriteBinaryToConsole(binaryData); + Console.WriteLine("Warming up..."); - for (int i = 0; i < 50; i++) + for (int i = 0; i < DefaultIterations; i++) { var target = CreatePopulateTarget(testOrder); + + //Console.WriteLine("Populate"); AcBinaryDeserializer.Populate(binaryData, target); + + //Console.WriteLine("PopulateMerge"); AcBinaryDeserializer.PopulateMerge(binaryData.AsSpan(), target); } @@ -786,6 +799,13 @@ public class QuickBenchmark #endregion + private static void WriteBinaryToConsole(byte[] binary) + { + Console.WriteLine(); + Console.WriteLine(BitConverter.ToString(binary)); + Console.WriteLine(); + } + #region Test Models [AcBinarySerializable] diff --git a/AyCode.Core.Tests/TestModels/SharedTestModels.cs b/AyCode.Core.Tests/TestModels/SharedTestModels.cs index 11129a1..df12470 100644 --- a/AyCode.Core.Tests/TestModels/SharedTestModels.cs +++ b/AyCode.Core.Tests/TestModels/SharedTestModels.cs @@ -246,6 +246,40 @@ public partial class TestOrder : IId public object? Parent { get; set; } } +/// +/// Level 1: Main order - root of the hierarchy +/// +[AcBinarySerializable(true)] +public partial class TestOrder_Circ_Ref : IId +{ + public int Id { get; set; } + public string OrderNumber { get; set; } = ""; + public TestStatus Status { get; set; } = TestStatus.Pending; + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public DateTime? PaidDateUtc { get; set; } + public decimal TotalAmount { get; set; } + + // Level 2 collection + public List Items { get; set; } = []; + + // Shared reference properties (for $id/$ref testing) + public SharedTag? PrimaryTag { get; set; } + public SharedTag? SecondaryTag { get; set; } + public SharedUser? Owner { get; set; } + public SharedCategory? Category { get; set; } + + // Collection of shared references + public List Tags { get; set; } = []; + public MetadataInfo? OrderMetadata { get; set; } + public MetadataInfo? AuditMetadata { get; set; } + public List MetadataList { get; set; } = []; + + // NoMerge collection for testing replace behavior + [JsonNoMergeCollection] + public List NoMergeItems { get; set; } = []; + + public object? Parent { get; set; } +} /// /// Level 2: Order item with pallets /// @@ -286,6 +320,29 @@ public partial class TestOrderItem : IId public TestOrder? ParentOrder { get; set; } } +/// +/// Level 2: Order item with pallets +/// +[AcBinarySerializable(true)] +public partial class TestOrderItem_Circ_Ref : IId +{ + public int Id { get; set; } + [AcStringIntern(true)] + public string ProductName { get; set; } = ""; + public int Quantity { get; set; } + public decimal UnitPrice { get; set; } + public TestStatus Status { get; set; } = TestStatus.Pending; + + // Level 3 collection + public List Pallets { get; set; } = []; + + // Shared references + public SharedTag? Tag { get; set; } + public SharedUser? Assignee { get; set; } + public MetadataInfo? ItemMetadata { get; set; } + + public TestOrder_Circ_Ref? ParentOrder { get; set; } +} /// /// Level 3: Pallet containing measurements /// diff --git a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.Populate.cs b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.Populate.cs index 545e3c8..28a22b6 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.Populate.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.Populate.cs @@ -225,7 +225,8 @@ public static partial class AcBinaryDeserializer } // Handle nested complex objects - reuse existing if available - if ((typeCode == BinaryTypeCode.Object || typeCode == BinaryTypeCode.ObjectWithMetadata) && propInfo.IsComplexType) + if ((typeCode == BinaryTypeCode.Object || typeCode == BinaryTypeCode.ObjectWithMetadata + || typeCode == BinaryTypeCode.ObjectRefFirst || typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst) && propInfo.IsComplexType) { var existingObj = propInfo.GetValue(target); if (existingObj != null) @@ -421,12 +422,23 @@ public static partial class AcBinaryDeserializer // Read marker once — eliminates redundant PeekByte + ReadByte boundary checks var typeCode = context.ReadByte(); + // ObjectRefFirst: read cacheIndex before processing the object body + var cacheIndex = -1; + if (typeCode == BinaryTypeCode.ObjectRefFirst && elementMetadata != null) + { + cacheIndex = (int)context.ReadVarUInt(); + // Treat as Object from here on — same wire body, just with cacheIndex prefix + typeCode = BinaryTypeCode.Object; + } + // If we have an existing item at this index and the incoming is an object, reuse it if (i < existingCount && typeCode == BinaryTypeCode.Object && elementMetadata != null) { var existingItem = existingList[i]; if (existingItem != null) { + if (cacheIndex >= 0) + context.RegisterInternedValueAt(cacheIndex, existingItem); PopulateObjectPropertiesIndexed(context, existingItem, wrapper, nextDepth, skipDefaultWrite: false); continue; } @@ -436,7 +448,7 @@ public static partial class AcBinaryDeserializer object? value; if (typeCode == BinaryTypeCode.Object && elementMetadata != null) { - value = ReadObjectCoreWithWrapper(context, wrapper, nextDepth, cacheIndex: -1); + value = ReadObjectCoreWithWrapper(context, wrapper, nextDepth, cacheIndex: cacheIndex); } else { diff --git a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.cs b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.cs index 58f0139..53dc7b2 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.cs @@ -344,12 +344,30 @@ public static partial class AcBinaryDeserializer context.ReadByte(); PopulateObject(context, target, targetType, 0); } + else if (typeCode == BinaryTypeCode.ObjectRefFirst) + { + // ObjectRefFirst: [marker][VarUInt cacheIndex][props...] + // Read marker + cacheIndex, register target in intern cache, then populate + context.ReadByte(); + var cacheIndex = (int)context.ReadVarUInt(); + context.RegisterInternedValueAt(cacheIndex, target); + PopulateObject(context, target, targetType, 0); + } else if (typeCode == BinaryTypeCode.ObjectWithMetadata) { context.ReadByte(); ReadInlineMetadataForPopulate(context, targetType); PopulateObject(context, target, targetType, 0); } + else if (typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst) + { + // ObjectWithMetadataRefFirst: [marker][VarUInt cacheIndex][metadata...][props...] + context.ReadByte(); + var cacheIndex = (int)context.ReadVarUInt(); + context.RegisterInternedValueAt(cacheIndex, target); + ReadInlineMetadataForPopulate(context, targetType); + PopulateObject(context, target, targetType, 0); + } else if (typeCode == BinaryTypeCode.Array && target is IList targetList) { context.ReadByte(); @@ -487,12 +505,27 @@ public static partial class AcBinaryDeserializer context.ReadByte(); PopulateObject(context, target, targetType, 0); } + else if (typeCode == BinaryTypeCode.ObjectRefFirst) + { + context.ReadByte(); + var cacheIndex = (int)context.ReadVarUInt(); + context.RegisterInternedValueAt(cacheIndex, target); + PopulateObject(context, target, targetType, 0); + } else if (typeCode == BinaryTypeCode.ObjectWithMetadata) { context.ReadByte(); ReadInlineMetadataForPopulate(context, targetType); PopulateObject(context, target, targetType, 0); } + else if (typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst) + { + context.ReadByte(); + var cacheIndex = (int)context.ReadVarUInt(); + context.RegisterInternedValueAt(cacheIndex, target); + ReadInlineMetadataForPopulate(context, targetType); + PopulateObject(context, target, targetType, 0); + } else if (typeCode == BinaryTypeCode.Array && target is IList targetList) { context.ReadByte(); @@ -1125,17 +1158,19 @@ public static partial class AcBinaryDeserializer return ReadDictionaryAsObject(context, keyType!, valueType!, depth); } + var wrapper = context.GetWrapper(targetType); + // SGen fast path: generated reader bypasses GetWrapper + CreateInstance + PopulateObject entirely. - // Only when not in UseMetadata mode (cross-type CacheMap not known at compile time) - // and not in ChainMode (needs post-read identity tracking). - if (!context.HasMetadata && !context.IsChainMode) + // Only when not in UseMetadata mode (cross-type CacheMap not known at compile time), + // not in ChainMode (needs post-read identity tracking), + // and UseGeneratedCode is enabled (matching serializer-side check). + if (!context.HasMetadata && !context.IsChainMode && context.Options.UseGeneratedCode) { - var generatedReader = GeneratedReaderRegistry.TryGet(targetType); + var generatedReader = wrapper.GeneratedReader; if (generatedReader != null) return generatedReader.ReadObject(context, depth, cacheIndex); } - var wrapper = context.GetWrapper(targetType); return ReadObjectCoreWithWrapper(context, wrapper, depth, cacheIndex); } diff --git a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs index d703e2d..051193c 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs @@ -862,7 +862,7 @@ public static partial class AcBinarySerializer if (entry.CacheIndex == -1) { // 2nd occurrence: assign CacheIndex + add StringFirst entry at first visit position - entry.CacheIndex = ++_nextCacheIndex; + entry.CacheIndex = _nextCacheIndex++; entry.IsFirstWrite = true; AddWriteDuplicateEntry(entry.FirstIndex, entry.CacheIndex, isFirst: true, value); } diff --git a/AyCode.Core/Serializers/Binaries/AcBinarySerializerOptions.cs b/AyCode.Core/Serializers/Binaries/AcBinarySerializerOptions.cs index 6b01f75..4f4d363 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinarySerializerOptions.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinarySerializerOptions.cs @@ -84,7 +84,7 @@ public sealed class AcBinarySerializerOptions : AcSerializerOptions /// public bool UseMetadata { get; set; } = false; - public bool UseGeneratedCode { get; set; } = true; + public bool UseGeneratedCode { get; set; } = false; /// /// Controls how string interning is applied during serialization. diff --git a/AyCode.Core/Serializers/TypeMetadataWrapper.cs b/AyCode.Core/Serializers/TypeMetadataWrapper.cs index 597f81d..833e48d 100644 --- a/AyCode.Core/Serializers/TypeMetadataWrapper.cs +++ b/AyCode.Core/Serializers/TypeMetadataWrapper.cs @@ -64,6 +64,7 @@ public sealed class TypeMetadataWrapper where TMetadata : TypeMetadat /// Checked once per object in WriteObject (not per property). /// internal IGeneratedBinaryWriter? GeneratedWriter; + internal IGeneratedBinaryReader? GeneratedReader; /// /// Options-filtered subset of metadata.ReferenceProperties for the scan pass. @@ -145,6 +146,7 @@ public sealed class TypeMetadataWrapper where TMetadata : TypeMetadat // Lookup generated writer from registry (once per wrapper creation, not per serialization) GeneratedWriter = AcBinarySerializer.GeneratedWriterRegistry.TryGet(metadata.SourceType); + GeneratedReader = AcBinaryDeserializer.GeneratedReaderRegistry.TryGet(metadata.SourceType); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -223,8 +225,7 @@ public sealed class TypeMetadataWrapper where TMetadata : TypeMetadat ref var entry = ref map.GetValueRef(slotIndex); if (entry.CacheIndex == -1) { - // Exact 2nd occurrence: assign CacheIndex + return first visit index - entry.CacheIndex = ++nextCacheIndex; + entry.CacheIndex = nextCacheIndex++; entry.IsFirstWrite = true; firstVisitIndex = entry.FirstIndex; } @@ -260,7 +261,7 @@ public sealed class TypeMetadataWrapper where TMetadata : TypeMetadat ref var entry = ref map.GetValueRef(slotIndex); if (entry.CacheIndex == -1) { - entry.CacheIndex = ++nextCacheIndex; + entry.CacheIndex = nextCacheIndex++; entry.IsFirstWrite = true; firstVisitIndex = entry.FirstIndex; } @@ -295,7 +296,7 @@ public sealed class TypeMetadataWrapper where TMetadata : TypeMetadat ref var entry = ref map.GetValueRef(slotIndex); if (entry.CacheIndex == -1) { - entry.CacheIndex = ++nextCacheIndex; + entry.CacheIndex = nextCacheIndex++; entry.IsFirstWrite = true; firstVisitIndex = entry.FirstIndex; }