Refactor serialization core, add pooled buffer support

Centralize serialization steps into a new SerializeCore method for consistency and maintainability. Rework property metadata registration to operate on types instead of instances, improving efficiency. Replace property index tracking with stack-allocated or pooled buffers to reduce allocations. Add SerializeToPooledBuffer and BinarySerializationResult for zero-copy serialization with proper buffer pooling and disposal. Simplify string writing logic and use GC.AllocateUninitializedArray for result arrays. Refactor and add helper methods for buffer management and metadata handling.

AyCode.Core/Extensions/AcBinarySerializer.cs

@@ -1,6 +1,7 @@
 using System.Buffers;
 using System.Collections;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Globalization;
 using System.Linq.Expressions;
 using System.Reflection;
@@ -47,34 +48,10 @@ public static class AcBinarySerializer
             return [BinaryTypeCode.Null];
         }
 
-        var type = value.GetType();
+        var runtimeType = value.GetType();
-
+        var context = SerializeCore(value, runtimeType, options);
-        // Use context-based serialization for all types to ensure consistent format
-        var context = BinarySerializationContextPool.Get(options);
         try
         {
-            // Reserve space for header
-            context.WriteHeaderPlaceholder();
-
-            // Phase 1: If reference handling enabled, scan for multi-referenced objects
-            if (options.UseReferenceHandling && !IsPrimitiveOrStringFast(type))
-            {
-                ScanReferences(value, context, 0);
-            }
-
-            // Phase 2: If metadata enabled, collect property names
-            if (options.UseMetadata && !IsPrimitiveOrStringFast(type))
-            {
-                CollectPropertyNames(value, context, 0);
-            }
-            
-            // Write metadata section
-            context.WriteMetadata();
-
-            // Phase 3: Write the actual data
-            WriteValue(value, type, context, 0);
-
-            // Finalize and return
             return context.ToArray();
         }
         finally
@@ -97,26 +74,10 @@ public static class AcBinarySerializer
             return;
         }
 
-        var type = value.GetType();
+        var runtimeType = value.GetType();
-        var context = BinarySerializationContextPool.Get(options);
+        var context = SerializeCore(value, runtimeType, options);
         try
         {
-            context.WriteHeaderPlaceholder();
-
-            if (options.UseReferenceHandling && !IsPrimitiveOrStringFast(type))
-            {
-                ScanReferences(value, context, 0);
-            }
-
-            if (options.UseMetadata && !IsPrimitiveOrStringFast(type))
-            {
-                CollectPropertyNames(value, context, 0);
-            }
-            
-            context.WriteMetadata();
-            WriteValue(value, type, context, 0);
-
-            // Write directly to the IBufferWriter instead of creating a new array
             context.WriteTo(writer);
         }
         finally
@@ -133,25 +94,10 @@ public static class AcBinarySerializer
     {
         if (value == null) return 1;
 
-        var type = value.GetType();
+        var runtimeType = value.GetType();
-        var context = BinarySerializationContextPool.Get(options);
+        var context = SerializeCore(value, runtimeType, options);
         try
         {
-            context.WriteHeaderPlaceholder();
-
-            if (options.UseReferenceHandling && !IsPrimitiveOrStringFast(type))
-            {
-                ScanReferences(value, context, 0);
-            }
-
-            if (options.UseMetadata && !IsPrimitiveOrStringFast(type))
-            {
-                CollectPropertyNames(value, context, 0);
-            }
-            
-            context.WriteMetadata();
-            WriteValue(value, type, context, 0);
-
             return context.Position;
         }
         finally
@@ -160,6 +106,49 @@ public static class AcBinarySerializer
         }
     }
 
+    /// <summary>
+    /// Serialize object and keep the pooled buffer for zero-copy consumers.
+    /// Caller must dispose the returned result to release the buffer.
+    /// </summary>
+    public static BinarySerializationResult SerializeToPooledBuffer<T>(T value, AcBinarySerializerOptions options)
+    {
+        if (value == null)
+        {
+            return BinarySerializationResult.FromImmutable([BinaryTypeCode.Null]);
+        }
+
+        var runtimeType = value.GetType();
+        var context = SerializeCore(value, runtimeType, options);
+        try
+        {
+            return context.DetachResult();
+        }
+        finally
+        {
+            BinarySerializationContextPool.Return(context);
+        }
+    }
+
+    private static BinarySerializationContext SerializeCore(object value, Type runtimeType, AcBinarySerializerOptions options)
+    {
+        var context = BinarySerializationContextPool.Get(options);
+        context.WriteHeaderPlaceholder();
+
+        if (options.UseReferenceHandling && !IsPrimitiveOrStringFast(runtimeType))
+        {
+            ScanReferences(value, context, 0);
+        }
+
+        if (options.UseMetadata && !IsPrimitiveOrStringFast(runtimeType))
+        {
+            RegisterMetadataForType(runtimeType, context);
+        }
+        
+        context.WriteMetadata();
+        WriteValue(value, runtimeType, context, 0);
+        return context;
+    }
+
     #endregion
 
     #region Reference Scanning
@@ -205,33 +194,28 @@ public static class AcBinarySerializer
 
     #endregion
 
-    #region Property Name Collection
+    #region Property Metadata Registration
 
-    private static void CollectPropertyNames(object? value, BinarySerializationContext context, int depth)
+    private static void RegisterMetadataForType(Type type, BinarySerializationContext context, HashSet<Type>? visited = null)
     {
-        if (value == null || depth > context.MaxDepth) return;
-
-        var type = value.GetType();
         if (IsPrimitiveOrStringFast(type)) return;
 
-        if (value is byte[]) return;
+        visited ??= new HashSet<Type>();
+        if (!visited.Add(type)) return;
 
-        if (value is IDictionary dictionary)
+        if (IsDictionaryType(type, out var keyType, out var valueType))
         {
-            foreach (DictionaryEntry entry in dictionary)
+            if (keyType != null) RegisterMetadataForType(keyType, context, visited);
-            {
+            if (valueType != null) RegisterMetadataForType(valueType, context, visited);
-                if (entry.Value != null)
-                    CollectPropertyNames(entry.Value, context, depth + 1);
-            }
             return;
         }
 
-        if (value is IEnumerable enumerable && !ReferenceEquals(type, StringType))
+        if (typeof(IEnumerable).IsAssignableFrom(type) && !ReferenceEquals(type, StringType))
         {
-            foreach (var item in enumerable)
+            var elementType = GetCollectionElementType(type);
+            if (elementType != null)
             {
-                if (item != null)
+                RegisterMetadataForType(elementType, context, visited);
-                    CollectPropertyNames(item, context, depth + 1);
             }
             return;
         }
@@ -240,12 +224,46 @@ public static class AcBinarySerializer
         foreach (var prop in metadata.Properties)
         {
             context.RegisterPropertyName(prop.Name);
-            var propValue = prop.GetValue(value);
+
-            if (propValue != null)
+            if (TryResolveNestedMetadataType(prop.PropertyType, out var nestedType))
-                CollectPropertyNames(propValue, context, depth + 1);
+            {
+                RegisterMetadataForType(nestedType, context, visited);
+            }
         }
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static bool TryResolveNestedMetadataType(Type propertyType, out Type nestedType)
+    {
+        nestedType = Nullable.GetUnderlyingType(propertyType) ?? propertyType;
+
+        if (IsPrimitiveOrStringFast(nestedType))
+            return false;
+
+        if (IsDictionaryType(nestedType, out var _, out var valueType) && valueType != null)
+        {
+            if (!IsPrimitiveOrStringFast(valueType))
+            {
+                nestedType = valueType;
+                return true;
+            }
+            return false;
+        }
+
+        if (typeof(IEnumerable).IsAssignableFrom(nestedType) && !ReferenceEquals(nestedType, StringType))
+        {
+            var elementType = GetCollectionElementType(nestedType);
+            if (elementType != null && !IsPrimitiveOrStringFast(elementType))
+            {
+                nestedType = elementType;
+                return true;
+            }
+            return false;
+        }
+
+        return true;
+    }
+
     #endregion
 
     #region Value Writing
@@ -606,27 +624,35 @@ public static class AcBinarySerializer
         var properties = metadata.Properties;
         var propCount = properties.Length;
         
-        // Single-pass: count and collect non-null, non-default properties
+        const int StackThreshold = 64;
-        // Use stackalloc for small property counts to avoid allocation
+        byte[]? rentedStates = null;
-        Span<int> validIndices = propCount <= 32 ? stackalloc int[propCount] : new int[propCount];
+        Span<byte> propertyStates = propCount <= StackThreshold
+            ? stackalloc byte[propCount]
+            : (rentedStates = context.RentPropertyStateBuffer(propCount)).AsSpan(0, propCount);
+        propertyStates.Clear();
         var writtenCount = 0;
 
         for (var i = 0; i < propCount; i++)
         {
-            var prop = properties[i];
+            if (IsPropertyDefaultOrNull(value, properties[i]))
-            if (IsPropertyDefaultOrNull(value, prop))
+            {
+                propertyStates[i] = 0;
                 continue;
-            validIndices[writtenCount++] = i;
+            }
+
+            propertyStates[i] = 1;
+            writtenCount++;
         }
 
         context.WriteVarUInt((uint)writtenCount);
 
-        // Write only the valid properties
+        for (var i = 0; i < propCount; i++)
-        for (var j = 0; j < writtenCount; j++)
         {
-            var prop = properties[validIndices[j]];
+            if (propertyStates[i] == 0)
+                continue;
+
+            var prop = properties[i];
 
-            // Write property index or name
             if (context.UseMetadata)
             {
                 var propIndex = context.GetPropertyNameIndex(prop.Name);
@@ -637,9 +663,13 @@ public static class AcBinarySerializer
                 WriteString(prop.Name, context);
             }
 
-            // Use typed writers to avoid boxing
             WritePropertyValue(value, prop, context, nextDepth);
         }
+
+        if (rentedStates != null)
+        {
+            context.ReturnPropertyStateBuffer(rentedStates);
+        }
     }
 
     /// <summary>
@@ -1089,9 +1119,12 @@ public static class AcBinarySerializer
     {
         private byte[] _buffer;
         private int _position;
+        private int _initialBufferSize;
 
         // Minimum buffer size for ArrayPool (reduces fragmentation)
         private const int MinBufferSize = 256;
+        private const int PropertyIndexBufferMaxCache = 512;
+        private const int PropertyStateBufferMaxCache = 512;
 
         // Reference handling
         private Dictionary<object, int>? _scanOccurrences;
@@ -1106,6 +1139,8 @@ public static class AcBinarySerializer
         // Property name table
         private Dictionary<string, int>? _propertyNames;
         private List<string>? _propertyNameList;
+        private int[]? _propertyIndexBuffer;
+        private byte[]? _propertyStateBuffer;
 
         public bool UseReferenceHandling { get; private set; }
         public bool UseStringInterning { get; private set; }
@@ -1115,8 +1150,8 @@ public static class AcBinarySerializer
 
         public BinarySerializationContext(AcBinarySerializerOptions options)
         {
-            var size = Math.Max(options.InitialBufferCapacity, MinBufferSize);
+            _initialBufferSize = Math.Max(options.InitialBufferCapacity, MinBufferSize);
-            _buffer = ArrayPool<byte>.Shared.Rent(size);
+            _buffer = ArrayPool<byte>.Shared.Rent(_initialBufferSize);
             Reset(options);
         }
 
@@ -1129,6 +1164,13 @@ public static class AcBinarySerializer
             UseMetadata = options.UseMetadata;
             MaxDepth = options.MaxDepth;
             MinStringInternLength = options.MinStringInternLength;
+            _initialBufferSize = Math.Max(options.InitialBufferCapacity, MinBufferSize);
+
+            if (_buffer.Length < _initialBufferSize)
+            {
+                ArrayPool<byte>.Shared.Return(_buffer);
+                _buffer = ArrayPool<byte>.Shared.Rent(_initialBufferSize);
+            }
         }
 
         public void Clear()
@@ -1146,29 +1188,17 @@ public static class AcBinarySerializer
             
             _internedStringList?.Clear();
             _propertyNameList?.Clear();
-        }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length > PropertyIndexBufferMaxCache)
-        private static void ClearAndTrimIfNeeded<TKey, TValue>(Dictionary<TKey, TValue>? dict, int maxCapacity) where TKey : notnull
-        {
-            if (dict == null) return;
-            dict.Clear();
-            // TrimExcess only if the dictionary grew significantly beyond initial capacity
-            if (dict.EnsureCapacity(0) > maxCapacity)
             {
-                dict.TrimExcess();
+                ArrayPool<int>.Shared.Return(_propertyIndexBuffer);
+                _propertyIndexBuffer = null;
             }
-        }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            if (_propertyStateBuffer != null && _propertyStateBuffer.Length > PropertyStateBufferMaxCache)
-        private static void ClearAndTrimIfNeeded<T>(HashSet<T>? set, int maxCapacity)
-        {
-            if (set == null) return;
-            set.Clear();
-            // TrimExcess only if the set grew significantly beyond initial capacity
-            if (set.EnsureCapacity(0) > maxCapacity)
             {
-                set.TrimExcess();
+                ArrayPool<byte>.Shared.Return(_propertyStateBuffer);
+                _propertyStateBuffer = null;
             }
         }
 
@@ -1179,6 +1209,18 @@ public static class AcBinarySerializer
                 ArrayPool<byte>.Shared.Return(_buffer);
                 _buffer = null!;
             }
+
+            if (_propertyIndexBuffer != null)
+            {
+                ArrayPool<int>.Shared.Return(_propertyIndexBuffer);
+                _propertyIndexBuffer = null;
+            }
+
+            if (_propertyStateBuffer != null)
+            {
+                ArrayPool<byte>.Shared.Return(_propertyStateBuffer);
+                _propertyStateBuffer = null;
+            }
         }
 
         #region Optimized Buffer Writing
@@ -1341,22 +1383,11 @@ public static class AcBinarySerializer
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void WriteStringUtf8(string value)
         {
-            // For small strings, use stackalloc to avoid GetByteCount call
+            var byteCount = Utf8NoBom.GetByteCount(value);
-            if (value.Length <= 128)
+            WriteVarUInt((uint)byteCount);
-            {
+            EnsureCapacity(byteCount);
-                Span<byte> tempBuffer = stackalloc byte[value.Length * 3]; // Max UTF8 expansion
+            Utf8NoBom.GetBytes(value.AsSpan(), _buffer.AsSpan(_position, byteCount));
-                var bytesWritten = Encoding.UTF8.GetBytes(value.AsSpan(), tempBuffer);
+            _position += byteCount;
-                WriteVarUInt((uint)bytesWritten);
-                WriteBytes(tempBuffer.Slice(0, bytesWritten));
-            }
-            else
-            {
-                var utf8Length = Encoding.UTF8.GetByteCount(value);
-                WriteVarUInt((uint)utf8Length);
-                EnsureCapacity(utf8Length);
-                Encoding.UTF8.GetBytes(value, _buffer.AsSpan(_position, utf8Length));
-                _position += utf8Length;
-            }
         }
 
         #endregion
@@ -1620,15 +1651,78 @@ public static class AcBinarySerializer
             return _propertyNames!.TryGetValue(name, out var index) ? index : -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int[] RentPropertyIndexBuffer(int minimumLength)
+        {
+            if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length >= minimumLength)
+            {
+                var buffer = _propertyIndexBuffer;
+                _propertyIndexBuffer = null;
+                return buffer;
+            }
+
+            return ArrayPool<int>.Shared.Rent(minimumLength);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void ReturnPropertyIndexBuffer(int[] buffer)
+        {
+            if (_propertyIndexBuffer == null && buffer.Length <= PropertyIndexBufferMaxCache)
+            {
+                _propertyIndexBuffer = buffer;
+                return;
+            }
+
+            ArrayPool<int>.Shared.Return(buffer);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public byte[] RentPropertyStateBuffer(int minimumLength)
+        {
+            if (_propertyStateBuffer != null && _propertyStateBuffer.Length >= minimumLength)
+            {
+                var buffer = _propertyStateBuffer;
+                _propertyStateBuffer = null;
+                return buffer;
+            }
+
+            return ArrayPool<byte>.Shared.Rent(minimumLength);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void ReturnPropertyStateBuffer(byte[] buffer)
+        {
+            if (_propertyStateBuffer == null && buffer.Length <= PropertyStateBufferMaxCache)
+            {
+                _propertyStateBuffer = buffer;
+                return;
+            }
+
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+
         #endregion
 
         public byte[] ToArray()
         {
-            var result = new byte[_position];
+            var result = GC.AllocateUninitializedArray<byte>(_position);
             _buffer.AsSpan(0, _position).CopyTo(result);
             return result;
         }
 
+        public BinarySerializationResult DetachResult()
+        {
+            var buffer = _buffer;
+            var length = _position;
+            var result = new BinarySerializationResult(buffer, length, pooled: true);
+
+            var newSize = Math.Max(_initialBufferSize, MinBufferSize);
+            _buffer = ArrayPool<byte>.Shared.Rent(newSize);
+            _position = 0;
+
+            return result;
+        }
+
         public void WriteTo(IBufferWriter<byte> writer)
         {
             // Directly write the internal buffer to the IBufferWriter
@@ -1638,6 +1732,71 @@ public static class AcBinarySerializer
         }
 
         public int Position => _position;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static void ClearAndTrimIfNeeded<TKey, TValue>(Dictionary<TKey, TValue>? dict, int maxCapacity) where TKey : notnull
+        {
+            if (dict == null) return;
+            dict.Clear();
+            if (dict.EnsureCapacity(0) > maxCapacity)
+            {
+                dict.TrimExcess();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static void ClearAndTrimIfNeeded<T>(HashSet<T>? set, int maxCapacity)
+        {
+            if (set == null) return;
+            set.Clear();
+            if (set.EnsureCapacity(0) > maxCapacity)
+            {
+                set.TrimExcess();
+            }
+        }
+    }
+
+    #endregion
+
+    #region Serialization Result
+
+    public sealed class BinarySerializationResult : IDisposable
+    {
+        private readonly bool _pooled;
+        private bool _disposed;
+
+        internal BinarySerializationResult(byte[] buffer, int length, bool pooled)
+        {
+            Buffer = buffer;
+            Length = length;
+            _pooled = pooled;
+        }
+
+        public byte[] Buffer { get; }
+        public int Length { get; }
+        public ReadOnlySpan<byte> Span => Buffer.AsSpan(0, Length);
+        public ReadOnlyMemory<byte> Memory => new(Buffer, 0, Length);
+
+        public byte[] ToArray()
+        {
+            var result = GC.AllocateUninitializedArray<byte>(Length);
+            Buffer.AsSpan(0, Length).CopyTo(result);
+            return result;
+        }
+
+        public void Dispose()
+        {
+            if (_disposed) return;
+            _disposed = true;
+
+            if (_pooled)
+            {
+                ArrayPool<byte>.Shared.Return(Buffer);
+            }
+        }
+
+        internal static BinarySerializationResult FromImmutable(byte[] buffer)
+            => new(buffer, buffer.Length, pooled: false);
     }
 
     #endregion