Improve circular reference handling in binary serializer

Introduce new test models for circular refs, update tests to stress reference handling, and enhance deserializer to support ObjectRefFirst/WithMetadataRefFirst type codes. Fix intern cache index assignment, track generated readers in TypeMetadataWrapper, and disable UseGeneratedCode by default. Update benchmarks for reliability and diagnostics. These changes strengthen reference resolution, circular ref support, and performance.

AyCode.Core.Serializers.Console/Program.cs

@@ -50,7 +50,7 @@ public static class Program
     private static int TestIterations = 1;
     #else
     private static int WarmupIterations = 5000;
-    private static int TestIterations = 5000;
+    private static int TestIterations = 1000;
 
     //private static int WarmupIterations = 5000;
     //private static int TestIterations = 2000;

AyCode.Core.Tests/Serialization/AcBinarySerializerIIdReferenceTests.cs

@@ -100,7 +100,7 @@ public class AcBinarySerializerIIdReferenceTests
             var sharedTag = new SharedTag { Id = 1, Name = "ImportantTag", Color = "#FF0000" };
             var sharedUser = new SharedUser { Id = 1, Preferences = userPreferences };
 
-            var order = new TestOrder
+            var order = new TestOrder_Circ_Ref
             {
                 Id = 1,
                 OrderNumber = "ORD-001",
@@ -108,18 +108,22 @@ public class AcBinarySerializerIIdReferenceTests
                 Owner = sharedUser,
                 Items =
                 [
-                    new TestOrderItem { Id = 1, ProductName = "Product-A", Tag = sharedTag, Assignee = sharedUser },
+                    new TestOrderItem_Circ_Ref { Id = 1, ProductName = "Product-A", Tag = sharedTag, Assignee = sharedUser },
-                    new TestOrderItem { Id = 2, ProductName = "Product-B", Tag = sharedTag, Assignee = new SharedUser { Id = 2, Preferences = userPreferences }},
+                    new TestOrderItem_Circ_Ref { Id = 2, ProductName = "Product-B", Tag = sharedTag, Assignee = new SharedUser { Id = 2, Preferences = userPreferences }},
-                    new TestOrderItem { Id = 3, ProductName = "Product-C", Tag = sharedTag, Assignee = new SharedUser { Id = 3, Preferences = userPreferences } }
+                    new TestOrderItem_Circ_Ref { Id = 3, ProductName = "Product-C", Tag = sharedTag, Assignee = new SharedUser { Id = 3, Preferences = userPreferences } }
                 ]
             };
 
+            //order.Parent = order.Items[0];
+            order.Parent = userPreferences;
+            order.Items[0].ParentOrder = order;
+
             var options = new AcBinarySerializerOptions { ReferenceHandling = mode };
 
             // Act
             var binary = AcBinarySerializer.Serialize(order, options);
-            WriteBinaryToConsole(binary);
+            //WriteBinaryToConsole(binary);
-            var result = binary.BinaryTo<TestOrder>(); // Options from header
+            var result = binary.BinaryTo<TestOrder_Circ_Ref>(); // Options from header
 
             var objectRefCount = CountObjectRefs(binary, false);
             Console.WriteLine($"Binary size: {binary.Length} bytes");
@@ -129,6 +133,7 @@ public class AcBinarySerializerIIdReferenceTests
             switch (mode)
             {
                 case ReferenceHandlingMode.None:
+                    //none esetén miért nincs infinite loop??? - J.
                     Assert.AreEqual(0, objectRefCount, $"[{mode}] Should have 0 ObjectRefs");
                     break;
                     

AyCode.Core.Tests/Serialization/QuickBenchmark.cs

@@ -130,7 +130,7 @@ public class QuickBenchmark
         var deserializeMs = sw.Elapsed.TotalMilliseconds;
 
         // JSON comparison
-        var jsonOptions = AcJsonSerializerOptions.WithoutReferenceHandling;
+        var jsonOptions = AcJsonSerializerOptions.Default;
         sw.Restart();
         string json = null!;
         for (int i = 0; i < iterations; i++)
@@ -512,6 +512,7 @@ public class QuickBenchmark
 
         var noRefOptions = AcBinarySerializerOptions.WithoutReferenceHandling;
         noRefOptions.UseMetadata = false;
+        //noRefOptions.ReferenceHandling = ReferenceHandlingMode.All;
         //noRefOptions.UseStringInterning = StringInterningMode.None;
 
         // Pre-serialize
@@ -519,15 +520,20 @@ public class QuickBenchmark
         var acBinaryNoRef = AcBinarySerializer.Serialize(testOrder, noRefOptions);
         var msgPackData = MessagePackSerializer.Serialize(testOrder, MsgPackOptions);
 
+        Console.WriteLine("withRefOptions:");
+        WriteBinaryToConsole(acBinaryWithRef);
+
         // Warmup - MUST be 1000+ iterations for tiered JIT to complete
         Console.WriteLine($"\nSerialize warming up ({DefaultIterations} iterations each)...");
-        for (int i = 0; i < 500; i++)
+        for (int i = 0; i < DefaultIterations; i++)
         {
             _ = AcBinarySerializer.Serialize(testOrder, withRefOptions);
             _ = AcBinarySerializer.Serialize(testOrder, noRefOptions);
             _ = MessagePackSerializer.Serialize(testOrder, MsgPackOptions);
 
+            Console.WriteLine("acBinaryWithRef");
             _ = AcBinaryDeserializer.Deserialize<TestOrder>(acBinaryWithRef);
+            Console.WriteLine("acBinaryNoRef");
             _ = AcBinaryDeserializer.Deserialize<TestOrder>(acBinaryNoRef);
             _ = MessagePackSerializer.Deserialize<TestOrder>(msgPackData, MsgPackOptions);
         }
@@ -724,13 +730,20 @@ public class QuickBenchmark
             pointsPerMeasurement: 4);
 
         var options = AcBinarySerializerOptions.WithoutReferenceHandling;
+        //options.ReferenceHandling = ReferenceHandlingMode.All;
         var binaryData = AcBinarySerializer.Serialize(testOrder, options);
 
+        WriteBinaryToConsole(binaryData);
+
         Console.WriteLine("Warming up...");
-        for (int i = 0; i < 50; i++)
+        for (int i = 0; i < DefaultIterations; i++)
         {
             var target = CreatePopulateTarget(testOrder);
+
+            //Console.WriteLine("Populate");
             AcBinaryDeserializer.Populate(binaryData, target);
+
+            //Console.WriteLine("PopulateMerge");
             AcBinaryDeserializer.PopulateMerge(binaryData.AsSpan(), target);
         }
 
@@ -786,6 +799,13 @@ public class QuickBenchmark
 
     #endregion
 
+    private static void WriteBinaryToConsole(byte[] binary)
+    {
+        Console.WriteLine();
+        Console.WriteLine(BitConverter.ToString(binary));
+        Console.WriteLine();
+    }
+
     #region Test Models
 
     [AcBinarySerializable]

AyCode.Core.Tests/TestModels/SharedTestModels.cs

@@ -246,6 +246,40 @@ public partial class TestOrder : IId<int>
     public object? Parent { get; set; }
 }
 
+/// <summary>
+/// Level 1: Main order - root of the hierarchy
+/// </summary>
+[AcBinarySerializable(true)]
+public partial class TestOrder_Circ_Ref : IId<int>
+{
+    public int Id { get; set; }
+    public string OrderNumber { get; set; } = "";
+    public TestStatus Status { get; set; } = TestStatus.Pending;
+    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+    public DateTime? PaidDateUtc { get; set; }
+    public decimal TotalAmount { get; set; }
+
+    // Level 2 collection
+    public List<TestOrderItem_Circ_Ref> Items { get; set; } = [];
+
+    // Shared reference properties (for $id/$ref testing)
+    public SharedTag? PrimaryTag { get; set; }
+    public SharedTag? SecondaryTag { get; set; }
+    public SharedUser? Owner { get; set; }
+    public SharedCategory? Category { get; set; }
+    
+    // Collection of shared references
+    public List<SharedTag> Tags { get; set; } = [];
+    public MetadataInfo? OrderMetadata { get; set; }
+    public MetadataInfo? AuditMetadata { get; set; }
+    public List<MetadataInfo> MetadataList { get; set; } = [];
+
+    // NoMerge collection for testing replace behavior
+    [JsonNoMergeCollection]
+    public List<TestOrderItem_Circ_Ref> NoMergeItems { get; set; } = [];
+
+    public object? Parent { get; set; }
+}
 /// <summary>
 /// Level 2: Order item with pallets
 /// </summary>
@@ -286,6 +320,29 @@ public partial class TestOrderItem : IId<int>
     public TestOrder? ParentOrder { get; set; }
 }
 
+/// <summary>
+/// Level 2: Order item with pallets
+/// </summary>
+[AcBinarySerializable(true)]
+public partial class TestOrderItem_Circ_Ref : IId<int>
+{
+    public int Id { get; set; }
+    [AcStringIntern(true)]
+    public string ProductName { get; set; } = "";
+    public int Quantity { get; set; }
+    public decimal UnitPrice { get; set; }
+    public TestStatus Status { get; set; } = TestStatus.Pending;
+
+    // Level 3 collection
+    public List<TestPallet> Pallets { get; set; } = [];
+
+    // Shared references
+    public SharedTag? Tag { get; set; }
+    public SharedUser? Assignee { get; set; }
+    public MetadataInfo? ItemMetadata { get; set; }
+
+    public TestOrder_Circ_Ref? ParentOrder { get; set; }
+}
 /// <summary>
 /// Level 3: Pallet containing measurements
 /// </summary>

AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.Populate.cs

@@ -225,7 +225,8 @@ public static partial class AcBinaryDeserializer
         }
 
         // Handle nested complex objects - reuse existing if available
-        if ((typeCode == BinaryTypeCode.Object || typeCode == BinaryTypeCode.ObjectWithMetadata) && propInfo.IsComplexType)
+        if ((typeCode == BinaryTypeCode.Object || typeCode == BinaryTypeCode.ObjectWithMetadata
+            || typeCode == BinaryTypeCode.ObjectRefFirst || typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst) && propInfo.IsComplexType)
         {
             var existingObj = propInfo.GetValue(target);
             if (existingObj != null)
@@ -421,12 +422,23 @@ public static partial class AcBinaryDeserializer
                 // Read marker once — eliminates redundant PeekByte + ReadByte boundary checks
                 var typeCode = context.ReadByte();
 
+                // ObjectRefFirst: read cacheIndex before processing the object body
+                var cacheIndex = -1;
+                if (typeCode == BinaryTypeCode.ObjectRefFirst && elementMetadata != null)
+                {
+                    cacheIndex = (int)context.ReadVarUInt();
+                    // Treat as Object from here on — same wire body, just with cacheIndex prefix
+                    typeCode = BinaryTypeCode.Object;
+                }
+
                 // If we have an existing item at this index and the incoming is an object, reuse it
                 if (i < existingCount && typeCode == BinaryTypeCode.Object && elementMetadata != null)
                 {
                     var existingItem = existingList[i];
                     if (existingItem != null)
                     {
+                        if (cacheIndex >= 0)
+                            context.RegisterInternedValueAt(cacheIndex, existingItem);
                         PopulateObjectPropertiesIndexed(context, existingItem, wrapper, nextDepth, skipDefaultWrite: false);
                         continue;
                     }
@@ -436,7 +448,7 @@ public static partial class AcBinaryDeserializer
                 object? value;
                 if (typeCode == BinaryTypeCode.Object && elementMetadata != null)
                 {
-                    value = ReadObjectCoreWithWrapper(context, wrapper, nextDepth, cacheIndex: -1);
+                    value = ReadObjectCoreWithWrapper(context, wrapper, nextDepth, cacheIndex: cacheIndex);
                 }
                 else
                 {

AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.cs

@@ -344,12 +344,30 @@ public static partial class AcBinaryDeserializer
                 context.ReadByte();
                 PopulateObject(context, target, targetType, 0);
             }
+            else if (typeCode == BinaryTypeCode.ObjectRefFirst)
+            {
+                // ObjectRefFirst: [marker][VarUInt cacheIndex][props...]
+                // Read marker + cacheIndex, register target in intern cache, then populate
+                context.ReadByte();
+                var cacheIndex = (int)context.ReadVarUInt();
+                context.RegisterInternedValueAt(cacheIndex, target);
+                PopulateObject(context, target, targetType, 0);
+            }
             else if (typeCode == BinaryTypeCode.ObjectWithMetadata)
             {
                 context.ReadByte();
                 ReadInlineMetadataForPopulate(context, targetType);
                 PopulateObject(context, target, targetType, 0);
             }
+            else if (typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst)
+            {
+                // ObjectWithMetadataRefFirst: [marker][VarUInt cacheIndex][metadata...][props...]
+                context.ReadByte();
+                var cacheIndex = (int)context.ReadVarUInt();
+                context.RegisterInternedValueAt(cacheIndex, target);
+                ReadInlineMetadataForPopulate(context, targetType);
+                PopulateObject(context, target, targetType, 0);
+            }
             else if (typeCode == BinaryTypeCode.Array && target is IList targetList)
             {
                 context.ReadByte();
@@ -487,12 +505,27 @@ public static partial class AcBinaryDeserializer
                 context.ReadByte();
                 PopulateObject(context, target, targetType, 0);
             }
+            else if (typeCode == BinaryTypeCode.ObjectRefFirst)
+            {
+                context.ReadByte();
+                var cacheIndex = (int)context.ReadVarUInt();
+                context.RegisterInternedValueAt(cacheIndex, target);
+                PopulateObject(context, target, targetType, 0);
+            }
             else if (typeCode == BinaryTypeCode.ObjectWithMetadata)
             {
                 context.ReadByte();
                 ReadInlineMetadataForPopulate(context, targetType);
                 PopulateObject(context, target, targetType, 0);
             }
+            else if (typeCode == BinaryTypeCode.ObjectWithMetadataRefFirst)
+            {
+                context.ReadByte();
+                var cacheIndex = (int)context.ReadVarUInt();
+                context.RegisterInternedValueAt(cacheIndex, target);
+                ReadInlineMetadataForPopulate(context, targetType);
+                PopulateObject(context, target, targetType, 0);
+            }
             else if (typeCode == BinaryTypeCode.Array && target is IList targetList)
             {
                 context.ReadByte();
@@ -1125,17 +1158,19 @@ public static partial class AcBinaryDeserializer
             return ReadDictionaryAsObject(context, keyType!, valueType!, depth);
         }
 
+        var wrapper = context.GetWrapper(targetType);
+
         // SGen fast path: generated reader bypasses GetWrapper + CreateInstance + PopulateObject entirely.
-        // Only when not in UseMetadata mode (cross-type CacheMap not known at compile time)
+        // Only when not in UseMetadata mode (cross-type CacheMap not known at compile time),
-        // and not in ChainMode (needs post-read identity tracking).
+        // not in ChainMode (needs post-read identity tracking),
-        if (!context.HasMetadata && !context.IsChainMode)
+        // and UseGeneratedCode is enabled (matching serializer-side check).
+        if (!context.HasMetadata && !context.IsChainMode && context.Options.UseGeneratedCode)
         {
-            var generatedReader = GeneratedReaderRegistry.TryGet(targetType);
+            var generatedReader = wrapper.GeneratedReader;
             if (generatedReader != null)
                 return generatedReader.ReadObject(context, depth, cacheIndex);
         }
 
-        var wrapper = context.GetWrapper(targetType);
         return ReadObjectCoreWithWrapper(context, wrapper, depth, cacheIndex);
     }
 

AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs

@@ -862,7 +862,7 @@ public static partial class AcBinarySerializer
                 if (entry.CacheIndex == -1)
                 {
                     // 2nd occurrence: assign CacheIndex + add StringFirst entry at first visit position
-                    entry.CacheIndex = ++_nextCacheIndex;
+                    entry.CacheIndex = _nextCacheIndex++;
                     entry.IsFirstWrite = true;
                     AddWriteDuplicateEntry(entry.FirstIndex, entry.CacheIndex, isFirst: true, value);
                 }

AyCode.Core/Serializers/Binaries/AcBinarySerializerOptions.cs

@@ -84,7 +84,7 @@ public sealed class AcBinarySerializerOptions : AcSerializerOptions
     /// </summary>
     public bool UseMetadata { get; set; } = false;
 
-    public bool UseGeneratedCode { get; set; } = true;
+    public bool UseGeneratedCode { get; set; } = false;
 
     /// <summary>
     /// Controls how string interning is applied during serialization.

AyCode.Core/Serializers/TypeMetadataWrapper.cs

@@ -64,6 +64,7 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
     /// Checked once per object in WriteObject (not per property).
     /// </summary>
     internal IGeneratedBinaryWriter? GeneratedWriter;
+    internal IGeneratedBinaryReader? GeneratedReader;
 
     /// <summary>
     /// Options-filtered subset of metadata.ReferenceProperties for the scan pass.
@@ -145,6 +146,7 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
 
         // Lookup generated writer from registry (once per wrapper creation, not per serialization)
         GeneratedWriter = AcBinarySerializer.GeneratedWriterRegistry.TryGet(metadata.SourceType);
+        GeneratedReader = AcBinaryDeserializer.GeneratedReaderRegistry.TryGet(metadata.SourceType);
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -223,8 +225,7 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
             ref var entry = ref map.GetValueRef(slotIndex);
             if (entry.CacheIndex == -1)
             {
-                // Exact 2nd occurrence: assign CacheIndex + return first visit index
+                entry.CacheIndex = nextCacheIndex++;
-                entry.CacheIndex = ++nextCacheIndex;
                 entry.IsFirstWrite = true;
                 firstVisitIndex = entry.FirstIndex;
             }
@@ -260,7 +261,7 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
             ref var entry = ref map.GetValueRef(slotIndex);
             if (entry.CacheIndex == -1)
             {
-                entry.CacheIndex = ++nextCacheIndex;
+                entry.CacheIndex = nextCacheIndex++;
                 entry.IsFirstWrite = true;
                 firstVisitIndex = entry.FirstIndex;
             }
@@ -295,7 +296,7 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
             ref var entry = ref map.GetValueRef(slotIndex);
             if (entry.CacheIndex == -1)
             {
-                entry.CacheIndex = ++nextCacheIndex;
+                entry.CacheIndex = nextCacheIndex++;
                 entry.IsFirstWrite = true;
                 firstVisitIndex = entry.FirstIndex;
             }