AyCode.Core/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySe...

785 lines
27 KiB
C#

using System;
using System.Buffers;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using static AyCode.Core.Helpers.JsonUtilities;
namespace AyCode.Core.Serializers.Binaries;
public static partial class AcBinarySerializer
{
private static class BinarySerializationContextPool<TOutput> where TOutput : struct, IBinaryOutputBase
{
private static readonly ConcurrentQueue<BinarySerializationContext<TOutput>> Pool = new();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BinarySerializationContext<TOutput> Get(AcBinarySerializerOptions options)
{
if (Pool.TryDequeue(out var context))
{
context.Reset(options);
return context;
}
return new BinarySerializationContext<TOutput>(options);
}
public static void ReturnAsync(BinarySerializationContext<TOutput> context)
{
// 🔥 FIRE-AND-FORGET: cleanup háttérben
ThreadPool.UnsafeQueueUserWorkItem(Return, context, preferLocal: true);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Return(BinarySerializationContext<TOutput> context)
{
if (Pool.Count < context.Options.MaxContextPoolSize)
{
context.Clear();
Pool.Enqueue(context);
}
else
{
context.Dispose();
}
}
}
/// <summary>
/// Binary serialization context. Generic on TOutput for output strategy selection.
/// Owns _buffer/_position for zero virtual dispatch on the hot path.
/// All write operations (WriteByte, WriteVarUInt, etc.) are inline methods here.
/// TOutput Output handles only cold-path buffer management (Grow/Initialize) and finalization.
/// </summary>
internal sealed class BinarySerializationContext<TOutput>
: SerializationContextBase<BinarySerializeTypeMetadata, AcBinarySerializerOptions>, IDisposable
where TOutput : struct, IBinaryOutputBase
{
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
private const int PropertyIndexBufferMaxCache = 512;
private const int PropertyStateBufferMaxCache = 512;
/// <summary>
/// Output target — handles only Grow (cold path) and finalization (AsSpan/ToArray/Flush).
/// </summary>
public TOutput Output;
/// <summary>
/// True if Output has been assigned (struct can't be null-checked).
/// </summary>
public bool OutputInitialized;
#region Buffer State owned by context for zero virtual dispatch
/// <summary>Current writable buffer (from ArrayPool or IBufferWriter chunk).</summary>
internal byte[] _buffer = null!;
/// <summary>Current write position within _buffer.</summary>
internal int _position;
/// <summary>One past the last writable index in _buffer. Write must satisfy _position &lt; _bufferEnd.</summary>
internal int _bufferEnd;
#endregion
private IdentityMap<string, InternEntry>? _stringInternMap;
private int _nextCacheIndex; // Next dense cache index to assign (starts at 0, uses ++_nextCacheIndex)
public int NextFirstIndex; // Next first occurrence index for scan pass. Direct access for performance.
/// <summary>
/// Next cache index reference for scan pass. Direct ref access for TryTrack methods.
/// </summary>
public ref int NextCacheIndexRef
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref _nextCacheIndex;
}
private int[]? _propertyIndexBuffer;
private byte[]? _propertyStateBuffer;
/// <summary>
/// Current property being serialized. Set in WriteObject property loops.
/// Used by WriteString for per-property interning control (UseStringPropertyInterning).
/// null when writing non-property values (arrays, dictionaries, root value).
/// </summary>
//internal BinaryPropertyAccessorBase? CurrentProperty;
#if DEBUG
/// <summary>
/// DEBUG ONLY: Callback invoked when a string is registered for interning.
/// Parameters: (propertyPath, stringValue)
/// Use this to analyze which properties have repeated string values.
/// </summary>
internal Action<string?, string>? OnStringInterned;
#endif
// These properties delegate to Options for convenience
internal bool UseStringInterning => Options.UseStringInterning != StringInterningMode.None;
public bool IsValidForInterningString(int strLength)
{
return strLength >= MinStringInternLength && (MaxStringInternLength == 0 || strLength <= MaxStringInternLength);
}
/// <summary>
/// True if we have interning/ref tracking (cache count needed in header).
/// </summary>
public bool HasCaching => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None;
public bool UseMetadata => Options.UseMetadata;
public bool UseGeneratedCode => Options.UseGeneratedCode;
public byte MinStringInternLength => Options.MinStringInternLength;
public byte MaxStringInternLength => Options.MaxStringInternLength;
public BinaryPropertyFilter? PropertyFilter => Options.PropertyFilter;
/// <summary>
/// Cached check for PropertyFilter != null. Set in Reset() to avoid property getter in hot loop.
/// </summary>
public bool HasPropertyFilter { get; private set; }
/// <summary>
/// Current output position (total bytes written so far).
/// Cold path — uses virtual dispatch through Output.GetTotalPosition.
/// </summary>
public int Position
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => Output.GetTotalPosition(_position);
}
public BinarySerializationContext(AcBinarySerializerOptions options)
{
Reset(options);
}
/// <summary>
/// Factory for creating BinarySerializeTypeMetadata instances.
/// </summary>
protected override Func<Type, BinarySerializeTypeMetadata> MetadataFactory
=> static t => new BinarySerializeTypeMetadata(t, HasJsonIgnoreAttribute);
public override void Reset(AcBinarySerializerOptions options)
{
// IMPORTANT: base.Reset sets Options first, so derived code can use Options-derived properties
base.Reset(options);
HasPropertyFilter = Options.PropertyFilter != null;
}
public override void Clear()
{
// Reset output buffer (large buffers → return to pool, rent halved)
if (OutputInitialized)
{
Output.Reset();
}
_stringInternMap?.Reset();
_nextCacheIndex = 0;
NextFirstIndex = 0;
if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length > PropertyIndexBufferMaxCache)
{
ArrayPool<int>.Shared.Return(_propertyIndexBuffer);
_propertyIndexBuffer = null;
}
if (_propertyStateBuffer != null && _propertyStateBuffer.Length > PropertyStateBufferMaxCache)
{
ArrayPool<byte>.Shared.Return(_propertyStateBuffer);
_propertyStateBuffer = null;
}
// Clear wrapper tracking - returns IdentityMap arrays to pool
base.Clear();
}
public void Dispose()
{
if (_propertyIndexBuffer != null)
{
ArrayPool<int>.Shared.Return(_propertyIndexBuffer);
_propertyIndexBuffer = null;
}
if (_propertyStateBuffer != null)
{
ArrayPool<byte>.Shared.Return(_propertyStateBuffer);
_propertyStateBuffer = null;
}
// Dispose the output if it implements IDisposable (e.g. ArrayBinaryOutput returns buffer to pool)
if (Output is IDisposable disposableOutput)
disposableOutput.Dispose();
}
#region Write Methods inline, zero virtual dispatch
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void EnsureCapacity(int additionalBytes)
{
if (_position + additionalBytes > _bufferEnd)
Output.Grow(ref _buffer, ref _position, ref _bufferEnd, additionalBytes);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteByte(byte value)
{
if (_position >= _bufferEnd)
Output.Grow(ref _buffer, ref _position, ref _bufferEnd, 1);
_buffer[_position++] = value;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteTwoBytes(byte b1, byte b2)
{
EnsureCapacity(2);
_buffer[_position++] = b1;
_buffer[_position++] = b2;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteBytes(ReadOnlySpan<byte> data)
{
EnsureCapacity(data.Length);
data.CopyTo(_buffer.AsSpan(_position));
_position += data.Length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteRaw<T>(T value) where T : unmanaged
{
var size = Unsafe.SizeOf<T>();
EnsureCapacity(size);
Unsafe.WriteUnaligned(ref _buffer[_position], value);
_position += size;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteTypeCodeAndRaw<T>(byte typeCode, T value) where T : unmanaged
{
var size = 1 + Unsafe.SizeOf<T>();
EnsureCapacity(size);
_buffer[_position++] = typeCode;
Unsafe.WriteUnaligned(ref _buffer[_position], value);
_position += Unsafe.SizeOf<T>();
}
#endregion
#region VarInt Encoding inline
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteVarUInt(uint value)
{
if (value < 0x80)
{
if (_position >= _bufferEnd)
Output.Grow(ref _buffer, ref _position, ref _bufferEnd, 1);
_buffer[_position++] = (byte)value;
return;
}
EnsureCapacity(5);
while (value >= 0x80)
{
_buffer[_position++] = (byte)(value | 0x80);
value >>= 7;
}
_buffer[_position++] = (byte)value;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteVarInt(int value)
{
var encoded = (uint)((value << 1) ^ (value >> 31));
WriteVarUInt(encoded);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteVarULong(ulong value)
{
if (value < 0x80)
{
if (_position >= _bufferEnd)
Output.Grow(ref _buffer, ref _position, ref _bufferEnd, 1);
_buffer[_position++] = (byte)value;
return;
}
EnsureCapacity(10);
while (value >= 0x80)
{
_buffer[_position++] = (byte)(value | 0x80);
value >>= 7;
}
_buffer[_position++] = (byte)value;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteVarLong(long value)
{
var encoded = (ulong)((value << 1) ^ (value >> 63));
WriteVarULong(encoded);
}
#endregion
#region Specialized Types inline
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteDecimalBits(decimal value)
{
EnsureCapacity(16);
Span<int> bits = stackalloc int[4];
decimal.TryGetBits(value, bits, out _);
MemoryMarshal.AsBytes(bits).CopyTo(_buffer.AsSpan(_position, 16));
_position += 16;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteDateTimeBits(DateTime value)
{
EnsureCapacity(9);
Unsafe.WriteUnaligned(ref _buffer[_position], value.Ticks);
_buffer[_position + 8] = (byte)value.Kind;
_position += 9;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteGuidBits(Guid value)
{
EnsureCapacity(16);
value.TryWriteBytes(_buffer.AsSpan(_position, 16));
_position += 16;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteDateTimeOffsetBits(DateTimeOffset value)
{
EnsureCapacity(10);
Unsafe.WriteUnaligned(ref _buffer[_position], value.UtcTicks);
Unsafe.WriteUnaligned(ref _buffer[_position + 8], (short)value.Offset.TotalMinutes);
_position += 10;
}
#endregion
#region String Writes inline
public void WriteStringUtf8(string value)
{
var charLength = value.Length;
// Speculative ASCII fast path: assume byteCount == charLength
// Single-pass Ascii.FromUtf16 (scan+copy combined) instead of
// Ascii.IsValid (scan) + Ascii.FromUtf16 (scan+copy) = double traversal
var savedPosition = _position;
WriteVarUInt((uint)charLength);
EnsureCapacity(charLength);
if (Ascii.FromUtf16(value.AsSpan(), _buffer.AsSpan(_position, charLength), out _) == OperationStatus.Done)
{
_position += charLength;
return;
}
// Non-ASCII fallback: rewind VarUInt, encode with UTF-8
// FromUtf16 fails fast on first non-ASCII char, so speculative cost is minimal
_position = savedPosition;
var byteCount = Utf8NoBom.GetByteCount(value);
WriteVarUInt((uint)byteCount);
EnsureCapacity(byteCount);
Utf8NoBom.GetBytes(value.AsSpan(), _buffer.AsSpan(_position, byteCount));
_position += byteCount;
}
public void WriteFixStr(string value)
{
var length = value.Length;
EnsureCapacity(1 + length);
_buffer[_position++] = BinaryTypeCode.EncodeFixStr(length);
Ascii.FromUtf16(value.AsSpan(), _buffer.AsSpan(_position, length), out _);
_position += length;
}
public void WriteFixStrDirect(string value)
{
var length = value.Length;
EnsureCapacity(1 + length);
var destSpan = _buffer.AsSpan(_position + 1, length);
var status = Ascii.FromUtf16(value.AsSpan(), destSpan, out var bytesWritten);
if (status == OperationStatus.Done && bytesWritten == length)
{
_buffer[_position] = BinaryTypeCode.EncodeFixStr(length);
_position += 1 + length;
}
else
{
_buffer[_position++] = BinaryTypeCode.String;
WriteStringUtf8Internal(value);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteFixStrBytes(ReadOnlySpan<byte> utf8Bytes)
{
var length = utf8Bytes.Length;
EnsureCapacity(1 + length);
_buffer[_position++] = BinaryTypeCode.EncodeFixStr(length);
utf8Bytes.CopyTo(_buffer.AsSpan(_position, length));
_position += length;
}
public void WritePreencodedPropertyName(ReadOnlySpan<byte> utf8Name)
{
WriteByte(BinaryTypeCode.String);
WriteVarUInt((uint)utf8Name.Length);
WriteBytes(utf8Name);
}
private void WriteStringUtf8Internal(string value)
{
var byteCount = Utf8NoBom.GetByteCount(value);
WriteVarUInt((uint)byteCount);
EnsureCapacity(byteCount);
Utf8NoBom.GetBytes(value.AsSpan(), _buffer.AsSpan(_position, byteCount));
_position += byteCount;
}
#endregion
#region Bulk Array Writes inline
public void WriteDoubleArrayBulk(double[] array)
{
EnsureCapacity(array.Length * 9);
for (var i = 0; i < array.Length; i++)
{
_buffer[_position++] = BinaryTypeCode.Float64;
Unsafe.WriteUnaligned(ref _buffer[_position], array[i]);
_position += 8;
}
}
public void WriteFloatArrayBulk(float[] array)
{
EnsureCapacity(array.Length * 5);
for (var i = 0; i < array.Length; i++)
{
_buffer[_position++] = BinaryTypeCode.Float32;
Unsafe.WriteUnaligned(ref _buffer[_position], array[i]);
_position += 4;
}
}
public void WriteGuidArrayBulk(Guid[] array)
{
EnsureCapacity(array.Length * 17);
for (var i = 0; i < array.Length; i++)
{
_buffer[_position++] = BinaryTypeCode.Guid;
array[i].TryWriteBytes(_buffer.AsSpan(_position, 16));
_position += 16;
}
}
public void WriteInt32ArrayOptimized(int[] array)
{
for (var i = 0; i < array.Length; i++)
{
var value = array[i];
if (BinaryTypeCode.TryEncodeTinyInt(value, out var tiny))
{
WriteByte(tiny);
}
else
{
WriteByte(BinaryTypeCode.Int32);
WriteVarInt(value);
}
}
}
public void WriteLongArrayOptimized(long[] array)
{
for (var i = 0; i < array.Length; i++)
{
var value = array[i];
if (value >= int.MinValue && value <= int.MaxValue)
{
var intValue = (int)value;
if (BinaryTypeCode.TryEncodeTinyInt(intValue, out var tiny))
{
WriteByte(tiny);
}
else
{
WriteByte(BinaryTypeCode.Int32);
WriteVarInt(intValue);
}
}
else
{
WriteByte(BinaryTypeCode.Int64);
WriteVarLong(value);
}
}
}
public void WriteBytesSimd(ReadOnlySpan<byte> source)
{
EnsureCapacity(source.Length);
var destination = _buffer.AsSpan(_position, source.Length);
if (Vector.IsHardwareAccelerated && source.Length >= Vector<byte>.Count * 2)
{
var vectorSize = Vector<byte>.Count;
var i = 0;
var length = source.Length;
var vectorCount = length / vectorSize;
for (var v = 0; v < vectorCount; v++)
{
var vec = new Vector<byte>(source.Slice(i, vectorSize));
vec.CopyTo(destination.Slice(i, vectorSize));
i += vectorSize;
}
if (i < length)
source.Slice(i).CopyTo(destination.Slice(i));
}
else
{
source.CopyTo(destination);
}
_position += source.Length;
}
#endregion
#region String Interning
/// <summary>
/// Serialize pass: looks up interned string state.
/// Returns the entry ref for caller to check IsFirstWrite and update it.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref InternEntry GetInternedStringEntry(string value, out bool found)
{
if (_stringInternMap == null)
{
found = false;
return ref Unsafe.NullRef<InternEntry>();
}
if (_stringInternMap.TryAdd(value, out var slotIndex))
{
// Not in map (shouldn't happen after scan pass for cached strings)
found = false;
return ref _stringInternMap.GetValueRef(slotIndex);
}
found = true;
return ref _stringInternMap.GetValueRef(slotIndex);
}
/// <summary>
/// Scan pass: tracks a string for interning. Assigns CacheIndex immediately on 2nd occurrence.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ScanInternString(string value)
{
_stringInternMap ??= new IdentityMap<string, InternEntry>();
if (!_stringInternMap.TryAdd(value, out var slotIndex))
{
// 2+ occurrence: assign CacheIndex immediately
ref var entry = ref _stringInternMap.GetValueRef(slotIndex);
if (entry.CacheIndex == -1)
{
entry.CacheIndex = ++_nextCacheIndex;
entry.IsFirstWrite = true;
}
return;
}
// 1st occurrence: store FirstIndex for validation, CacheIndex = -1 (not cached yet)
ref var newEntry = ref _stringInternMap.GetValueRef(slotIndex);
newEntry.FirstIndex = NextFirstIndex++;
newEntry.CacheIndex = -1;
}
/// <summary>
/// Returns true if there are any interned strings that occurred more than once.
/// </summary>
public bool HasInternedStrings => _stringInternMap != null && _stringInternMap.Count > 0;
/// <summary>
/// Gets the count of cached values (string intern + object ref that occurred more than once).
/// </summary>
public int GetCacheCount() => _nextCacheIndex;
#endregion
#region UseMetadata Type Tracking
/// <summary>
/// Regisztrálja a típust UseMetadata módban.
/// Visszaadja true-t ha ez az első előfordulás (inline hash-eket kell írni),
/// false-t ha ismételt (csak propNameHash kell).
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool RegisterMetadataType(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
{
if (wrapper.MetadataFooterIndex >= 0)
return false; // ismételt
wrapper.MetadataFooterIndex = 0; // jelöljük hogy már regisztrálva
return true; // első előfordulás
}
/// <summary>
/// Inline metadata kiírása az ObjectWithMetadata marker után.
/// Első előfordulás: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// Ismételt: [propNameHash (4b)]
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteInlineMetadata(BinarySerializeTypeMetadata metadata, bool isFirstOccurrence)
{
WriteRaw(metadata.PropNameHash);
if (isFirstOccurrence)
{
var hashes = metadata.MetadataPropertyHashes;
WriteVarUInt((uint)hashes.Length);
for (var i = 0; i < hashes.Length; i++)
{
WriteRaw(hashes[i]);
}
}
}
#endregion
#region Property State Buffer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte[] RentPropertyStateBuffer(int size)
{
if (_propertyStateBuffer != null && _propertyStateBuffer.Length >= size)
{
return _propertyStateBuffer;
}
if (_propertyStateBuffer != null)
{
ArrayPool<byte>.Shared.Return(_propertyStateBuffer);
}
_propertyStateBuffer = ArrayPool<byte>.Shared.Rent(size);
return _propertyStateBuffer;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReturnPropertyStateBuffer(byte[] buffer)
{
// Buffer stays cached for reuse.
}
#endregion
#region Property Filtering
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ShouldSerializeProperty(object instance, BinaryPropertyAccessor property)
{
if (PropertyFilter == null)
{
return true;
}
var context = new BinaryPropertyFilterContext(
instance,
property.DeclaringType,
property.Name,
property.PropertyType,
property.DynamicGetter);
return PropertyFilter(context);
}
public bool CheckDuplicatePropName => Options.CheckDuplicatePropName;
#endregion
#region Header
/// <summary>
/// Writes the binary header directly. Call AFTER ScanForDuplicates (cacheCount is known).
/// No placeholder, no shift — single forward write.
/// Layout: [version (1b)][flags (1b)][cacheCount (VarUInt, if caching)]
/// </summary>
public void WriteHeader()
{
var flags = BinaryTypeCode.HeaderFlagsBase;
if (UseMetadata)
flags |= BinaryTypeCode.HeaderFlag_Metadata;
if (ReferenceHandling == ReferenceHandlingMode.OnlyId)
flags |= BinaryTypeCode.HeaderFlag_RefHandling_OnlyId;
else if (ReferenceHandling == ReferenceHandlingMode.All)
flags |= (byte)(BinaryTypeCode.HeaderFlag_RefHandling_OnlyId | BinaryTypeCode.HeaderFlag_RefHandling_All);
if (HasCaching)
flags |= BinaryTypeCode.HeaderFlag_HasCacheCount;
WriteByte(AcBinarySerializerOptions.FormatVersion);
WriteByte(flags);
if (HasCaching)
{
WriteVarUInt((uint)GetCacheCount());
}
}
#endregion
#region Helpers
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ClearAndTrimIfNeeded<TKey, TValue>(Dictionary<TKey, TValue>? dict, int maxCapacity)
where TKey : notnull
{
if (dict == null)
{
return;
}
dict.Clear();
if (dict.EnsureCapacity(0) > maxCapacity)
{
dict.TrimExcess();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ClearAndTrimIfNeeded<T>(HashSet<T>? set, int maxCapacity)
{
if (set == null)
{
return;
}
set.Clear();
if (set.EnsureCapacity(0) > maxCapacity)
{
set.TrimExcess();
}
}
#endregion
}
}