Enable cross-type deserialization via property hashes

Introduce UseMetadata mode with FNV-1a property name hashing.
Write per-type property hashes to metadata footer for robust
property matching during deserialization. Remove legacy property
name table logic. Add ObjectWithMetadata marker and cachemap
logic for nested objects. Enable duplicate hash detection and
make UseMetadata default. Improves schema evolution support.
This commit is contained in:
Loretta 2026-02-04 09:38:49 +01:00
parent b7cb6256a0
commit 3da902b575
14 changed files with 633 additions and 299 deletions

View File

@ -29,7 +29,8 @@
"Bash(dotnet exec vstest:*)",
"Bash(dotnet new:*)",
"Bash(Remove-Item \"H:\\\\Applications\\\\Aycode\\\\Source\\\\AyCode.Core\\\\AyCode.Core\\\\Serializers\\\\Toons\\\\AcToonSerializer.RelationshipDetection.cs\")",
"Bash(find:*)"
"Bash(find:*)",
"Bash(dir:*)"
]
}
}

View File

@ -17,7 +17,6 @@ public static partial class AcBinaryDeserializer
{
private readonly ReadOnlySpan<byte> _buffer;
private int _position;
private List<string>? _propertyNames;
private Dictionary<int, string>? _stringCache;
// Position-based interning: flat int[] for cache-friendly access
@ -74,7 +73,6 @@ public static partial class AcBinaryDeserializer
{
_buffer = data;
_position = 0;
_propertyNames = null;
_stringCache = null;
// Position-based interning fields (shared: string + IId)
@ -151,17 +149,7 @@ public static partial class AcBinaryDeserializer
HasMetadata = hasPropertyTable;
if (hasPropertyTable)
{
var propertyCount = (int)ReadVarUInt();
_propertyNames = new List<string>(propertyCount);
for (var i = 0; i < propertyCount; i++)
{
_propertyNames.Add(ReadHeaderString());
}
}
// Footer-based: read intern indices from footer (string + IId)
// Footer-based: read intern indices and metadata from footer
if (hasInternFooter && footerPosition > 0)
{
ReadFooterIndices(footerPosition);
@ -182,7 +170,7 @@ public static partial class AcBinaryDeserializer
// Seek to footer
_position = footerPosition;
// Read dup count
// Read dup count (intern entries)
var dupCount = (int)ReadVarUInt();
if (dupCount == 0)
{
@ -207,10 +195,44 @@ public static partial class AcBinaryDeserializer
_nextDupPosition = _dupData[0];
}
// Read UseMetadata footer section (per-type property hashes)
if (HasMetadata && _position < _buffer.Length)
{
ReadMetadataFooter();
}
// Seek back to data position
_position = dataPosition;
}
/// <summary>
/// UseMetadata footer olvasása: entry-k flat array-be.
/// Formátum: [entryCount (VarUInt)]
/// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// Entry 0 = root, 1+ = nested object-ek.
/// </summary>
private void ReadMetadataFooter()
{
var entryCount = (int)ReadVarUInt();
for (var i = 0; i < entryCount; i++)
{
EnsureAvailable(4);
var propNameHash = Unsafe.ReadUnaligned<int>(ref Unsafe.AsRef(in _buffer[_position]));
_position += 4;
var propCount = (int)ReadVarUInt();
var hashes = new int[propCount];
for (var p = 0; p < propCount; p++)
{
EnsureAvailable(4);
hashes[p] = Unsafe.ReadUnaligned<int>(ref Unsafe.AsRef(in _buffer[_position]));
_position += 4;
}
ContextClass.RegisterFooterEntry(i, propNameHash, hashes);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte ReadByte() => ReadByteInternal();
@ -551,6 +573,19 @@ public static partial class AcBinaryDeserializer
return h.ToHashCode();
}
/// <summary>
/// Reads a raw 4-byte int32 (little-endian) without type code.
/// Used for reading type name hashes in UseMetadata mode.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ReadInt32Raw()
{
EnsureAvailable(4);
var value = Unsafe.ReadUnaligned<int>(ref Unsafe.AsRef(in _buffer[_position]));
_position += 4;
return value;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Skip(int count)
{
@ -619,17 +654,6 @@ public static partial class AcBinaryDeserializer
return result;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public string GetPropertyName(int index)
{
if (_propertyNames == null || (uint)index >= (uint)_propertyNames.Count)
{
throw new AcBinaryDeserializationException($"Invalid property metadata index '{index}'.", _position);
}
return _propertyNames[index];
}
private void EnsureAvailable(int length)
{
if (_position > _buffer.Length - length)
@ -638,11 +662,5 @@ public static partial class AcBinaryDeserializer
}
}
private string ReadHeaderString()
{
var byteLength = (int)ReadVarUInt();
return ReadStringUtf8(byteLength);
}
}
}

View File

@ -1,5 +1,7 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace AyCode.Core.Serializers.Binaries;
@ -29,6 +31,23 @@ public static partial class AcBinaryDeserializer
// Small arrays: keep across calls. Large arrays: return to pool in Clear().
private const int SmallArrayThreshold = 256;
/// <summary>
/// Footer metadata bejegyzések flat array-ben.
/// Index = footer entry index (0 = root, 1+ = nested).
/// Minden entry tartalmazza a source propNameHash-t és a source property hash-eket.
/// </summary>
private MetadataFooterEntry[]? _footerEntries;
private int _footerEntryCount;
/// <summary>
/// Egy footer bejegyzés a deserializer számára.
/// </summary>
internal struct MetadataFooterEntry
{
public int PropNameHash; // source típus propNameHash (FNV-1a)
public int[] PropertyHashes; // source property hash-ek sorrendben
}
public BinaryDeserializationContextClass()
{
}
@ -84,10 +103,79 @@ public static partial class AcBinaryDeserializer
_lastInternCacheUsed = count;
}
/// <summary>
/// Footer bejegyzés regisztrálása a metadata footer olvasásakor.
/// Az entryIndex a footer-ben lévő sorrend (0 = root).
/// </summary>
public void RegisterFooterEntry(int entryIndex, int propNameHash, int[] propertyHashes)
{
// Szükség esetén növeljük a tömböt
if (_footerEntries == null || _footerEntries.Length <= entryIndex)
{
var newSize = Math.Max(entryIndex + 1, 8);
var newArray = new MetadataFooterEntry[newSize];
if (_footerEntries != null)
Array.Copy(_footerEntries, newArray, _footerEntryCount);
_footerEntries = newArray;
}
_footerEntries[entryIndex] = new MetadataFooterEntry
{
PropNameHash = propNameHash,
PropertyHashes = propertyHashes
};
if (entryIndex >= _footerEntryCount)
_footerEntryCount = entryIndex + 1;
}
/// <summary>
/// Footer entry lekérése index alapján.
/// A root mindig a 0. elem.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref MetadataFooterEntry GetFooterEntry(int entryIndex)
{
return ref _footerEntries![entryIndex];
}
/// <summary>
/// Footer entry-k száma.
/// </summary>
public int FooterEntryCount => _footerEntryCount;
/// <summary>
/// Cachemap felépítése: source property hash-ek → target PropertySetter?[] mapping.
/// Null entry = source property-nek nincs megfelelője a target-ben → skip.
/// </summary>
public BinaryPropertySetterInfo?[] BuildCacheMap(int footerEntryIndex, BinaryDeserializeTypeMetadata targetMetadata)
{
ref var entry = ref _footerEntries![footerEntryIndex];
var sourceHashes = entry.PropertyHashes;
var targetProperties = targetMetadata.PropertiesArray;
var mapping = new BinaryPropertySetterInfo?[sourceHashes.Length];
for (var i = 0; i < sourceHashes.Length; i++)
{
var sourceHash = sourceHashes[i];
for (var j = 0; j < targetProperties.Length; j++)
{
if (targetProperties[j].PropertyNameHash == sourceHash)
{
mapping[i] = targetProperties[j];
break;
}
}
// ha nincs match → mapping[i] marad null → skip
}
return mapping;
}
public override void Clear()
{
base.Clear();
_footerEntryCount = 0;
// Intern cache: clear GC roots, return large arrays to pool
if (_pooledInternCache != null)
{

View File

@ -16,6 +16,14 @@ public static partial class AcBinaryDeserializer
/// </summary>
public BinaryPropertySetterInfo[] PropertiesArray { get; }
/// <summary>
/// UseMetadata cachemap: source property index → target PropertySetter.
/// Null entry = source property-nek nincs megfelelője a target-ben → skip.
/// Lazy módon épül az első találkozáskor, utána újrahasználódik.
/// A wrapper mindig új vagy pool-ból jön (clear-elve), tehát nem kell invalidálni.
/// </summary>
internal BinaryPropertySetterInfo?[]? CacheMap;
/// <summary>
/// True if this type has a Source Generator generated deserializer available.
/// Note: Due to ref struct limitations, the generated code cannot be called via delegates.

View File

@ -35,6 +35,13 @@ public static partial class AcBinaryDeserializer
private static void PopulateObject(ref BinaryDeserializationContext context, object target, Type targetType, int depth)
{
var wrapper = context.ContextClass.GetWrapper(targetType);
// UseMetadata: Populate esetén a root = footer entry 0
if (context.HasMetadata && wrapper.Metadata.CacheMap == null && context.ContextClass.FooterEntryCount > 0)
{
wrapper.Metadata.CacheMap = context.ContextClass.BuildCacheMap(0, wrapper.Metadata);
}
PopulateObjectCore(ref context, target, wrapper, depth, skipDefaultWrite: false);
}
@ -54,8 +61,9 @@ public static partial class AcBinaryDeserializer
}
/// <summary>
/// Populates object properties only - does NOT read hashcode prefix.
/// Used by ReadObject where hashcode is already handled separately.
/// Property-k olvasása. UseMetadata módban cachemap-alapú (marker-vezérelt),
/// egyébként index-alapú (mint eddig).
/// A cachemap a metadata.CacheMap-ben van, a ReadObject/ReadObjectWithMetadata építi fel.
/// </summary>
private static void PopulateObjectProperties(
ref BinaryDeserializationContext context,
@ -63,6 +71,130 @@ public static partial class AcBinaryDeserializer
BinaryDeserializeTypeMetadata metadata,
int depth,
bool skipDefaultWrite)
{
if (context.HasMetadata && metadata.CacheMap != null)
{
// UseMetadata: marker-vezérelt olvasás, cachemap-ből vesszük a setter-t
PopulateWithCacheMap(ref context, target, metadata.CacheMap, metadata, depth, skipDefaultWrite);
}
else
{
PopulateObjectPropertiesIndexed(ref context, target, metadata, depth, skipDefaultWrite);
}
}
/// <summary>
/// UseMetadata olvasás cachemap-pel. Marker-vezérelt, pontosan úgy olvas mint az index-alapú,
/// de a property index-ből a cachemap adja vissza a setter-t. Ha null → skip.
/// </summary>
private static void PopulateWithCacheMap(
ref BinaryDeserializationContext context,
object target,
BinaryPropertySetterInfo?[] cacheMap,
BinaryDeserializeTypeMetadata metadata,
int depth,
bool skipDefaultWrite)
{
var nextDepth = depth + 1;
var isMergeMode = context.IsMergeMode;
for (int i = 0; i < cacheMap.Length; i++)
{
var propInfo = cacheMap[i];
var peekCode = context.PeekByte();
// Nincs megfelelő target property → skip
if (propInfo == null)
{
SkipValue(ref context, metadata);
continue;
}
// PropertySkip marker - default/null érték
if (peekCode == BinaryTypeCode.PropertySkip)
{
context.ReadByte();
if (!skipDefaultWrite)
SetPropertyToDefault(target, propInfo);
continue;
}
// Null érték
if (peekCode == BinaryTypeCode.Null)
{
context.ReadByte();
propInfo.SetValue(target, null);
continue;
}
// Kollekció kezelés
if (peekCode == BinaryTypeCode.Array && propInfo.IsCollection)
{
var existingCollection = propInfo.GetValue(target);
if (existingCollection is IList existingList)
{
context.ReadByte();
if (isMergeMode && propInfo.IsIIdCollection)
MergeIIdCollection(ref context, existingList, propInfo, nextDepth);
else
PopulateListOptimized(ref context, existingList, propInfo, nextDepth);
continue;
}
}
// Nested object kezelés - Object és ObjectWithMetadata marker
if ((peekCode == BinaryTypeCode.Object || peekCode == BinaryTypeCode.ObjectWithMetadata) && propInfo.IsComplexType)
{
var existingObj = propInfo.GetValue(target);
if (existingObj != null)
{
// A ReadValue kezeli mindkét markert (Object és ObjectWithMetadata)
var value = ReadValue(ref context, propInfo.PropertyType, nextDepth);
if (value != null)
{
CopyProperties(value, existingObj, context.ContextClass.GetWrapper(propInfo.PropertyType).Metadata);
}
continue;
}
}
// Default: érték olvasás és beállítás
var positionBeforeRead = context.Position;
try
{
if (propInfo.AccessorType != PropertyAccessorType.Object &&
TryReadAndSetTypedValue(ref context, target, propInfo, peekCode))
continue;
var value = ReadValue(ref context, propInfo.PropertyType, nextDepth);
propInfo.SetValue(target, value);
}
catch (InvalidCastException ex)
{
var targetType = target.GetType();
throw new AcBinaryDeserializationException(
$"Típushibás property '{propInfo.Name}' (cachemap index {i}), target: '{targetType.Name}'. " +
$"Várt típus: '{propInfo.PropertyType.FullName}'. " +
$"PeekCode: {peekCode} (0x{peekCode:X2}). " +
$"Pozíció: {positionBeforeRead} → {context.Position}. Depth: {depth}. " +
$"Hiba: {ex.Message}",
positionBeforeRead,
propInfo.PropertyType,
ex);
}
}
}
/// <summary>
/// Index-based property populate (original logic, used when UseMetadata is off).
/// Properties are read in order matching the target type's property array.
/// </summary>
private static void PopulateObjectPropertiesIndexed(
ref BinaryDeserializationContext context,
object target,
BinaryDeserializeTypeMetadata metadata,
int depth,
bool skipDefaultWrite)
{
var properties = metadata.PropertiesArray;
var nextDepth = depth + 1;
@ -119,15 +251,18 @@ public static partial class AcBinaryDeserializer
}
// Handle nested complex objects - reuse existing if available
if (peekCode == BinaryTypeCode.Object && propInfo.IsComplexType)
if ((peekCode == BinaryTypeCode.Object || peekCode == BinaryTypeCode.ObjectWithMetadata) && propInfo.IsComplexType)
{
var existingObj = propInfo.GetValue(target);
if (existingObj != null)
{
context.ReadByte(); // consume Object marker
var nestedWrapper = context.ContextClass.GetWrapper(propInfo.PropertyType);
PopulateObjectCore(ref context, existingObj, nestedWrapper, nextDepth, skipDefaultWrite: false);
// ReadValue kezeli mindkét markert
var nestedValue = ReadValue(ref context, propInfo.PropertyType, nextDepth);
if (nestedValue != null)
{
var nestedMeta = context.ContextClass.GetWrapper(propInfo.PropertyType).Metadata;
CopyProperties(nestedValue, existingObj, nestedMeta);
}
continue;
}
}

View File

@ -72,6 +72,7 @@ public static partial class AcBinaryDeserializer
RegisterReader(BinaryTypeCode.Guid, static (ref BinaryDeserializationContext ctx, Type _, int _) => ctx.ReadGuidUnsafe());
RegisterReader(BinaryTypeCode.Enum, static (ref BinaryDeserializationContext ctx, Type type, int _) => ReadEnumValue(ref ctx, type));
RegisterReader(BinaryTypeCode.Object, ReadObject);
RegisterReader(BinaryTypeCode.ObjectWithMetadata, ReadObjectWithMetadata);
RegisterReader(BinaryTypeCode.ObjectRef, ReadObjectRef);
RegisterReader(BinaryTypeCode.Array, ReadArray);
RegisterReader(BinaryTypeCode.Dictionary, ReadDictionary);
@ -924,6 +925,21 @@ public static partial class AcBinaryDeserializer
/// <summary>
/// Reads an ObjectRef - looks up previously registered object from shared intern cache.
/// Wire format: [ObjectRef][VarUInt cacheIndex]
///
/// IMPORTANT / BUG FIX TODO:
/// Cross-type deserialization esetén ha egy objektum SkipObject-tel lett átugorva
/// (mert a target típuson nincs megfelelő property), de később ObjectRef hivatkozik rá,
/// az intern cache-ben nincs objektum → exception.
///
/// Megoldás: SkipObject során a stream pozíciót kell beírni a cache-be (boxolt int).
/// Itt az "is int" check-kel meg kell különböztetni:
/// - Ha a cached value valódi objektum → visszaadjuk (jelenlegi működés)
/// - Ha a cached value boxolt int (stream pozíció) → reposition + ReadObject a target type-ra,
/// majd az eredményt visszaírjuk a cache-be (hogy a következő ref ne olvasson újra)
///
/// Az "is int" check biztonságos, mert az intern cache-be csak string és class instance
/// kerülhet — egyik sem matchel az "is int"-re. A check az ObjectRef path-ban van,
/// ami ritka eset (2+ referencia), tehát nem lassítja a hot path-ot.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static object? ReadObjectRef(ref BinaryDeserializationContext context, Type targetType, int depth)
@ -933,11 +949,9 @@ public static partial class AcBinaryDeserializer
}
/// <summary>
/// Reads an Object.
/// Wire format:
/// - IId types: [Object][props 0-tól...] - Id a props-ban, populate után register
/// - Non-IId + All: [Object][hashcode][props 0-tól...] - hashcode előre olvasva
/// - Ref=Off: [Object][props 0-tól...] - semmi extra
/// Root object olvasása.
/// Wire format: [Object][props 0-tól...] - Id a props-ban, nincs extra
/// UseMetadata esetén a root = footer entry 0 (nincs footer index a body-ban).
/// </summary>
private static object? ReadObject(ref BinaryDeserializationContext context, Type targetType, int depth)
{
@ -953,36 +967,87 @@ public static partial class AcBinaryDeserializer
var wrapper = context.ContextClass.GetWrapper(targetType);
var metadata = wrapper.Metadata;
// Wire format: [Object][props...] - no hashcode prefix, no Id prefix
// Position-based footer handles all reference tracking
var instance = CreateInstance(targetType, metadata);
if (instance == null) return null;
// Register in shared intern cache BEFORE populate (position-based sequential check)
// Instance is populated in-place, so registering early is safe.
// Must happen before reading inner content (strings/objects) that may also need registration.
if (context.ContextClass.UseTypeReferenceHandling(metadata))
{
context.RegisterInternedValue(instance, streamPosition);
}
// UseMetadata: root = footer entry 0, cachemap felépítése ha még nincs
if (context.HasMetadata && context.ContextClass.FooterEntryCount > 0)
{
if (metadata.CacheMap == null)
metadata.CacheMap = context.ContextClass.BuildCacheMap(0, metadata);
}
PopulateObject(ref context, instance, metadata, depth, skipDefaultWrite: true);
// ChainMode: Register/retrieve from chain tracker (separate from reference handling)
// ChainMode kezelés
if (context.IsChainMode && metadata.IsIId && metadata.IdType != null)
{
var id = GetIdBoxed(instance, metadata);
if (id != null && !IsDefaultValue(id, metadata.IdType))
{
// Check if we already have this object
if (context.ChainTracker!.TryGetObject(targetType, id, out var existingObj))
{
// Update existing object's properties and return it
CopyProperties(instance, existingObj!, metadata);
return existingObj;
}
context.ChainTracker.TryRegisterIIdObject(instance);
}
}
// Register this new object
return instance;
}
/// <summary>
/// Nested object olvasása UseMetadata módban.
/// Wire format: [ObjectWithMetadata][footerIndex (VarUInt)][props...]
/// A footer index-ből megkapjuk a source property hash-eket → cachemap felépítése.
/// </summary>
private static object? ReadObjectWithMetadata(ref BinaryDeserializationContext context, Type targetType, int depth)
{
var streamPosition = context.Position - 1;
// Footer index olvasása
var footerIndex = (int)context.ReadVarUInt();
// Handle dictionary types
if (IsDictionaryType(targetType, out var keyType, out var valueType))
{
return ReadDictionaryAsObject(ref context, keyType!, valueType!, depth);
}
var wrapper = context.ContextClass.GetWrapper(targetType);
var metadata = wrapper.Metadata;
var instance = CreateInstance(targetType, metadata);
if (instance == null) return null;
if (context.ContextClass.UseTypeReferenceHandling(metadata))
{
context.RegisterInternedValue(instance, streamPosition);
}
// Cachemap felépítése ha még nincs (lazy, 1x per wrapper)
if (metadata.CacheMap == null)
metadata.CacheMap = context.ContextClass.BuildCacheMap(footerIndex, metadata);
PopulateObject(ref context, instance, metadata, depth, skipDefaultWrite: true);
// ChainMode kezelés
if (context.IsChainMode && metadata.IsIId && metadata.IdType != null)
{
var id = GetIdBoxed(instance, metadata);
if (id != null && !IsDefaultValue(id, metadata.IdType))
{
if (context.ChainTracker!.TryGetObject(targetType, id, out var existingObj))
{
CopyProperties(instance, existingObj!, metadata);
return existingObj;
}
context.ChainTracker.TryRegisterIIdObject(instance);
}
}
@ -1292,7 +1357,11 @@ public static partial class AcBinaryDeserializer
if (enumByte == BinaryTypeCode.Int32) context.ReadVarInt();
return;
case BinaryTypeCode.Object:
SkipObject(ref context, metaData);
SkipObject(ref context, metaData, footerIndex: -1);
return;
case BinaryTypeCode.ObjectWithMetadata:
var skipFooterIndex = (int)context.ReadVarUInt();
SkipObject(ref context, metaData, skipFooterIndex);
return;
case BinaryTypeCode.ObjectRef:
context.ReadVarInt();
@ -1351,13 +1420,35 @@ public static partial class AcBinaryDeserializer
// }
//}
private static void SkipObject(ref BinaryDeserializationContext context, BinaryDeserializeTypeMetadata metaData)
/// <summary>
/// Object kihagyása. UseMetadata módban a footer entry-ből tudjuk a property számot.
/// footerIndex: -1 = root (footer entry 0), >=0 = nested (a body-ból olvasott index).
/// </summary>
private static void SkipObject(ref BinaryDeserializationContext context, BinaryDeserializeTypeMetadata metaData, int footerIndex)
{
// Wire format: [Object][props...] - no hashcode prefix in new format
// NEW FORMAT: Can't skip without knowing property count!
if (context.HasMetadata)
{
// Root object: footer entry 0, nested: a megadott footerIndex
var entryIndex = footerIndex < 0 ? 0 : footerIndex;
if (entryIndex >= context.ContextClass.FooterEntryCount)
{
throw new AcBinaryDeserializationException(
$"SkipObject: Érvénytelen footer index {entryIndex}.",
context.Position);
}
ref var entry = ref context.ContextClass.GetFooterEntry(entryIndex);
var propCount = entry.PropertyHashes.Length;
for (var i = 0; i < propCount; i++)
{
SkipValue(ref context, metaData);
}
return;
}
// Nincs metadata → nem tudjuk kihagyni az object-et
throw new NotSupportedException(
"SkipObject is not supported with SKIP marker format. " +
"Unable to determine property count without type metadata.");
"SkipObject nem támogatott metadata nélkül. " +
"A property szám nem határozható meg típus metadata nélkül.");
}
private static void SkipArray(ref BinaryDeserializationContext context, BinaryDeserializeTypeMetadata metaData)

View File

@ -74,7 +74,13 @@ public static partial class AcBinarySerializer
private const int PropertyIndexBufferMaxCache = 512;
private const int PropertyStateBufferMaxCache = 512;
private const int InitialInternCapacity = 32;
private const int InitialPropertyNameCapacity = 32;
/// <summary>
/// UseMetadata footer bejegyzések sorrendben.
/// Minden bejegyzés egy (Type, propertyHashes) pár.
/// A 0. elem mindig a root object.
/// A body-ban a nested object-eknél az index (VarUInt) kerül kiírásra.
/// </summary>
private List<MetadataFooterEntry>? _metadataEntries;
private byte[] _buffer;
private int _position;
@ -100,8 +106,6 @@ public static partial class AcBinarySerializer
private IdentityMap<string, InternEntry>? _stringInternMap;
private int _nextCacheIndex; // Next dense cache index to assign
private Dictionary<string, int>? _propertyNames;
private List<string>? _propertyNameList;
private int[]? _propertyIndexBuffer;
private byte[]? _propertyStateBuffer;
@ -123,9 +127,9 @@ public static partial class AcBinarySerializer
// These properties delegate to Options for convenience
public bool UseStringInterning => Options.UseStringInterning != StringInterningMode.None;
/// <summary>
/// True if we need footer position in header (string interning OR reference handling).
/// True if we need footer position in header (string interning OR reference handling OR metadata).
/// </summary>
public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None;
public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None || UseMetadata;
public bool UseMetadata => Options.UseMetadata;
public byte MinStringInternLength => Options.MinStringInternLength;
public byte MaxStringInternLength => Options.MaxStringInternLength;
@ -176,9 +180,7 @@ public static partial class AcBinarySerializer
//_refTracker.Reset();
_stringInternMap?.Reset();
ClearAndTrimIfNeeded(_propertyNames, InitialPropertyNameCapacity * 4);
_propertyNameList?.Clear();
_metadataEntries?.Clear();
_nextCacheIndex = 0;
if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length > PropertyIndexBufferMaxCache)
@ -362,25 +364,74 @@ public static partial class AcBinarySerializer
#endregion
#region Property Name Table
#region UseMetadata Type Tracking
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void RegisterPropertyName(string name)
/// <summary>
/// Egy footer bejegyzés: típus propNameHash + property hash-ek.
/// </summary>
internal readonly struct MetadataFooterEntry
{
_propertyNames ??= new Dictionary<string, int>(InitialPropertyNameCapacity, StringComparer.Ordinal);
_propertyNameList ??= new List<string>(InitialPropertyNameCapacity);
public readonly int PropNameHash; // FNV-1a hash a típus nevéből
public readonly int[] PropertyHashes; // property name hash-ek sorrendben
if (!_propertyNames.ContainsKey(name))
public MetadataFooterEntry(int propNameHash, int[] propertyHashes)
{
var index = _propertyNameList.Count;
_propertyNames[name] = index;
_propertyNameList.Add(name);
PropNameHash = propNameHash;
PropertyHashes = propertyHashes;
}
}
/// <summary>
/// Regisztrálja a típust a UseMetadata footer-be.
/// Visszaadja a footer index-et. Ha már regisztrálva van (wrapper.MetadataFooterIndex >= 0),
/// a meglévő index-et adja vissza. Nincs Dictionary lookup — a wrapper tárolja az indexet.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int GetPropertyNameIndex(string name)
=> _propertyNames != null && _propertyNames.TryGetValue(name, out var index) ? index : -1;
public int RegisterMetadataType(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
{
if (wrapper.MetadataFooterIndex >= 0)
return wrapper.MetadataFooterIndex;
return RegisterMetadataTypeSlow(wrapper);
}
[MethodImpl(MethodImplOptions.NoInlining)]
private int RegisterMetadataTypeSlow(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
{
_metadataEntries ??= new List<MetadataFooterEntry>();
// PropNameHash and MetadataPropertyHashes are lazy-computed once per type in metadata.
// Duplicate hash validation also happens once (in MetadataPropertyHashes getter).
var index = _metadataEntries.Count;
_metadataEntries.Add(new MetadataFooterEntry(
wrapper.Metadata.PropNameHash,
wrapper.Metadata.MetadataPropertyHashes));
wrapper.MetadataFooterIndex = index;
return index;
}
/// <summary>
/// UseMetadata footer kiírása.
/// Formátum: [entryCount (VarUInt)]
/// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// </summary>
public void WriteMetadataFooter()
{
if (_metadataEntries == null || _metadataEntries.Count == 0) return;
WriteVarUInt((uint)_metadataEntries.Count);
for (var i = 0; i < _metadataEntries.Count; i++)
{
var entry = _metadataEntries[i];
WriteRaw(entry.PropNameHash);
WriteVarUInt((uint)entry.PropertyHashes.Length);
for (var j = 0; j < entry.PropertyHashes.Length; j++)
{
WriteRaw(entry.PropertyHashes[j]);
}
}
}
#endregion
@ -459,22 +510,7 @@ public static partial class AcBinarySerializer
return PropertyFilter(context);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ShouldIncludePropertyInMetadata(BinaryPropertyAccessor property)
{
if (PropertyFilter == null)
{
return true;
}
var context = new BinaryPropertyFilterContext(
null,
property.DeclaringType,
property.Name,
property.PropertyType,
null);
return PropertyFilter(context);
}
public bool CheckDuplicatePropName => Options.CheckDuplicatePropName;
#endregion
@ -1005,102 +1041,49 @@ public static partial class AcBinarySerializer
// Body: data with StringInterned indices
// Footer: interned strings table
/// <summary>
/// Estimates header payload size based on registered property names.
/// String interning now uses footer, so no estimation needed for strings.
/// </summary>
public int EstimateHeaderPayloadSize()
{
var size = 0;
// Only property names are in header now
if (UseMetadata && _propertyNameList is { Count: > 0 })
{
size += GetVarUIntSize((uint)_propertyNameList.Count);
for (var i = 0; i < _propertyNameList.Count; i++)
{
var name = _propertyNameList[i];
var byteCount = name.Length; // Assume ASCII (common case)
size += GetVarUIntSize((uint)byteCount) + byteCount;
}
}
return size;
}
public void WriteHeaderPlaceholder()
{
// Header layout:
// [0] version (1 byte)
// [1] flags (1 byte)
// [2-5] footer position (4 bytes, if string interning OR reference handling)
// [2-5] footer position (4 bytes, if footer is needed)
EnsureCapacity(HasFooter ? 6 : 2);
_headerPosition = _position;
_position += HasFooter ? 6 : 2;
}
/// <summary>
/// Reserves space for property name table in header.
/// </summary>
public void ReserveHeaderSpace(int estimatedSize)
{
if (estimatedSize <= 0) return;
EnsureCapacity(estimatedSize);
_position += estimatedSize;
}
public void FinalizeHeaderSections()
{
var hasPropertyNames = UseMetadata && _propertyNameList is { Count: > 0 };
var dupCount = GetDupCount(); // Shared counter: string intern + ID tracking
var hasInternTable = dupCount > 0;
// Calculate property names header size (strings go to footer now)
var headerPayloadSize = 0;
if (hasPropertyNames)
{
headerPayloadSize += GetVarUIntSize((uint)_propertyNameList!.Count);
for (var i = 0; i < _propertyNameList.Count; i++)
{
var name = _propertyNameList[i];
var byteCount = Ascii.IsValid(name) ? name.Length : Utf8NoBom.GetByteCount(name);
headerPayloadSize += GetVarUIntSize((uint)byteCount) + byteCount;
}
}
// Write property names to header if needed
var headerPayloadStart = _headerPosition + (HasFooter ? 6 : 2);
if (hasPropertyNames)
{
var headerPos = headerPayloadStart;
headerPos = WriteVarUIntAt(headerPos, (uint)_propertyNameList!.Count);
for (var i = 0; i < _propertyNameList.Count; i++)
{
var name = _propertyNameList[i];
headerPos = WriteStringAtOptimized(headerPos, name);
}
}
var hasMetadataFooter = UseMetadata && _metadataEntries is { Count: > 0 };
// Footer: write merged intern entries (string + ID)
var footerPosition = 0;
if (hasInternTable)
if (hasInternTable || hasMetadataFooter)
{
footerPosition = _position;
// Intern footer
WriteVarUInt((uint)dupCount);
if (hasInternTable)
WriteInternedFooter();
// Metadata footer (per-type property hashes)
if (hasMetadataFooter)
WriteMetadataFooter();
}
// Write header
var flags = BinaryTypeCode.HeaderFlagsBase;
if (hasPropertyNames)
if (hasMetadataFooter)
flags |= BinaryTypeCode.HeaderFlag_Metadata;
// Encode ReferenceHandlingMode using separate bits
if (ReferenceHandling == ReferenceHandlingMode.OnlyId)
flags |= BinaryTypeCode.HeaderFlag_RefHandling_OnlyId;
else if (ReferenceHandling == ReferenceHandlingMode.All)
flags |= (byte)(BinaryTypeCode.HeaderFlag_RefHandling_OnlyId | BinaryTypeCode.HeaderFlag_RefHandling_All);
// Set footer position flag if footer is needed (string interning OR ref handling)
// Set footer position flag if footer is needed
if (HasFooter)
flags |= BinaryTypeCode.HeaderFlag_HasFooterPosition;
@ -1108,34 +1091,12 @@ public static partial class AcBinarySerializer
_buffer[_headerPosition + 1] = flags;
// Write footer position if footer is needed
// (even if there's no actual interned data - footer position will be 0)
if (HasFooter)
{
Unsafe.WriteUnaligned(ref _buffer[_headerPosition + 2], footerPosition);
}
}
/// <summary>
/// Writes UTF8 string at specific position, optimized for ASCII strings.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int WriteStringAtOptimized(int pos, string value)
{
// Fast path for ASCII strings
if (Ascii.IsValid(value))
{
pos = WriteVarUIntAt(pos, (uint)value.Length);
Ascii.FromUtf16(value.AsSpan(), _buffer.AsSpan(pos, value.Length), out _);
return pos + value.Length;
}
// Standard path for multi-byte UTF8
var byteCount = Utf8NoBom.GetByteCount(value);
pos = WriteVarUIntAt(pos, (uint)byteCount);
Utf8NoBom.GetBytes(value.AsSpan(), _buffer.AsSpan(pos, byteCount));
return pos + byteCount;
}
/// <summary>
/// Writes VarUInt at specific position and returns new position.
/// </summary>

View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.CompilerServices;
using AyCode.Core.Serializers;
@ -23,7 +24,65 @@ public static partial class AcBinarySerializer
/// </summary>
public Type? GeneratedSerializerType { get; }
/// <summary>
/// Lazy-computed FNV-1a hash of the type name (SourceType.Name).
/// Only computed once per type, on first access when UseMetadata=true.
/// </summary>
private int _propNameHash;
private bool _propNameHashComputed;
public int PropNameHash
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
if (!_propNameHashComputed)
ComputePropNameHash();
return _propNameHash;
}
}
[MethodImpl(MethodImplOptions.NoInlining)]
private void ComputePropNameHash()
{
_propNameHash = FnvHash.ComputeString(SourceType.Name);
_propNameHashComputed = true;
}
/// <summary>
/// Lazy-computed property name hashes array for UseMetadata footer.
/// Includes duplicate hash validation (computed once per type, not per session).
/// </summary>
private int[]? _metadataPropertyHashes;
public int[] MetadataPropertyHashes
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _metadataPropertyHashes ??= ComputeMetadataPropertyHashes();
}
[MethodImpl(MethodImplOptions.NoInlining)]
private int[] ComputeMetadataPropertyHashes()
{
var hashes = new int[Properties.Length];
HashSet<int>? seen = null;
for (var i = 0; i < Properties.Length; i++)
{
var hash = Properties[i].PropertyNameHash;
seen ??= new HashSet<int>(Properties.Length);
if (!seen.Add(hash))
{
throw new InvalidOperationException(
$"UseMetadata: Duplikált property name hash a '{SourceType.FullName}' típuson. " +
$"Property '{Properties[i].Name}' FNV-1a hash ütközés. " +
$"Használj [AcPropertyName] attribútumot a feloldáshoz.");
}
hashes[i] = hash;
}
return hashes;
}
public BinarySerializeTypeMetadata(Type type, Func<PropertyInfo, bool> ignorePropertyFilter) : base(type,ignorePropertyFilter)

View File

@ -90,8 +90,6 @@ public static partial class AcBinarySerializer
// Run serialization to trigger callbacks
context.WriteHeaderPlaceholder();
var estimatedHeaderSize = context.EstimateHeaderPayloadSize();
context.ReserveHeaderSpace(estimatedHeaderSize);
WriteValue(value, runtimeType, context, 0);
context.FinalizeHeaderSections();
@ -359,10 +357,7 @@ public static partial class AcBinarySerializer
// - No header size estimation needed (strings go to footer)
// - No body shifting (footer is appended at the end)
// - Reference tracking happens inline via TryTrack during WriteObject
// Reserve space only for property name table (if metadata is enabled)
var estimatedHeaderSize = context.EstimateHeaderPayloadSize();
context.ReserveHeaderSpace(estimatedHeaderSize);
// - UseMetadata: per-type property hashes written to footer
WriteValue(value, runtimeType, context, 0);
context.FinalizeHeaderSections();
@ -371,89 +366,6 @@ public static partial class AcBinarySerializer
#endregion
#region Property Metadata Registration
private static void RegisterMetadataForType(Type type, BinarySerializationContext context, HashSet<Type>? visited = null)
{
if (IsPrimitiveOrStringFast(type)) return;
visited ??= new HashSet<Type>();
if (!visited.Add(type)) return;
if (IsDictionaryType(type, out var keyType, out var valueType))
{
if (keyType != null) RegisterMetadataForType(keyType, context, visited);
if (valueType != null) RegisterMetadataForType(valueType, context, visited);
return;
}
if (typeof(IEnumerable).IsAssignableFrom(type) && !ReferenceEquals(type, StringType))
{
var elementType = GetCollectionElementType(type);
if (elementType != null)
{
RegisterMetadataForType(elementType, context, visited);
}
return;
}
var wrapper = context.GetWrapper(type);
var metadata = wrapper.Metadata;
var properties = metadata.Properties;
// Use index-based iteration for array access
for (var i = 0; i < properties.Length; i++)
{
var prop = properties[i];
if (!context.ShouldIncludePropertyInMetadata(prop))
{
continue;
}
context.RegisterPropertyName(prop.Name);
if (TryResolveNestedMetadataType(prop.PropertyType, out var nestedType))
{
RegisterMetadataForType(nestedType, context, visited);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryResolveNestedMetadataType(Type propertyType, out Type nestedType)
{
nestedType = Nullable.GetUnderlyingType(propertyType) ?? propertyType;
if (IsPrimitiveOrStringFast(nestedType))
return false;
if (IsDictionaryType(nestedType, out var _, out var valueType) && valueType != null)
{
if (!IsPrimitiveOrStringFast(valueType))
{
nestedType = valueType;
return true;
}
return false;
}
if (typeof(IEnumerable).IsAssignableFrom(nestedType) && !ReferenceEquals(nestedType, StringType))
{
var elementType = GetCollectionElementType(nestedType);
if (elementType != null && !IsPrimitiveOrStringFast(elementType))
{
nestedType = elementType;
return true;
}
return false;
}
return true;
}
#endregion
#region Value Writing
private static void WriteValue(object? value, Type type, BinarySerializationContext context, int depth)
@ -496,7 +408,7 @@ public static partial class AcBinarySerializer
}
// Handle complex objects with single-pass reference tracking
WriteObject(value, type, context, depth);
WriteObject(value, type, context, depth, isNested: depth > 0);
}
/// <summary>
@ -829,18 +741,29 @@ public static partial class AcBinarySerializer
#region Complex Type Writers
private static void WriteObject(object value, Type type, BinarySerializationContext context, int depth)
private static void WriteObject(object value, Type type, BinarySerializationContext context, int depth, bool isNested = false)
{
var wrapper = context.GetWrapper(type);
var metadata = wrapper.Metadata;
// Wire format:
// - IId types: [Object][props 0-tól...] - Id a props-ban, nincs extra
// - Non-IId + All: [Object][hashcode][props 0-tól...] - hashcode előre
// - Ref=Off: [Object][props 0-tól...] - semmi extra
// - IId types: [Object/ObjectWithMetadata][props 0-tól...] - Id a props-ban, nincs extra
// - Non-IId + All: [Object/ObjectWithMetadata][props 0-tól...] - no hashcode prefix
// - Ref=Off: [Object/ObjectWithMetadata][props 0-tól...] - no prefix
// ObjectRef format:
// - IId: [ObjectRef][Id érték]
// - Non-IId: [ObjectRef][hashcode]
// - IId: [ObjectRef][cacheIndex]
// - Non-IId: [ObjectRef][cacheIndex]
//
// UseMetadata:
// - Root (depth==0): [Object] marker, footer entry 0
// - Nested: [ObjectWithMetadata][footerIndex (VarUInt)]
// UseMetadata: regisztráljuk a típust a footer-be (index kell a marker kiíráshoz)
var metadataFooterIndex = -1;
if (context.UseMetadata)
{
metadataFooterIndex = context.RegisterMetadataType(wrapper);
}
if (context.UseTypeReferenceHandling(metadata))
{
@ -852,13 +775,10 @@ public static partial class AcBinarySerializer
case IdAccessorType.Int32:
if (!context.TryTrackObject(wrapper, value, out int cacheIndex32))
{
// Already seen → ObjectRef + cacheIndex
context.WriteByte(BinaryTypeCode.ObjectRef);
context.WriteVarUInt((uint)cacheIndex32);
return;
}
// First occurrence → Object (no extra data, Id in props)
context.WriteByte(BinaryTypeCode.Object);
break;
case IdAccessorType.Int64:
@ -868,7 +788,6 @@ public static partial class AcBinarySerializer
context.WriteVarUInt((uint)cacheIndex64);
return;
}
context.WriteByte(BinaryTypeCode.Object);
break;
case IdAccessorType.Guid:
@ -878,7 +797,6 @@ public static partial class AcBinarySerializer
context.WriteVarUInt((uint)cacheIndexGuid);
return;
}
context.WriteByte(BinaryTypeCode.Object);
break;
}
}
@ -887,18 +805,21 @@ public static partial class AcBinarySerializer
// Non-IId + RefHandling=All: track by hashcode
if (!context.TryTrackObject(wrapper, value, out int cacheIndexHash))
{
// Already seen → ObjectRef + cacheIndex
context.WriteByte(BinaryTypeCode.ObjectRef);
context.WriteVarUInt((uint)cacheIndexHash);
return;
}
// First occurrence → Object (no extra prefix for Non-IId in new format)
context.WriteByte(BinaryTypeCode.Object);
}
}
// Marker kiírása: UseMetadata nested → ObjectWithMetadata + footer index, egyébként Object
if (context.UseMetadata && isNested)
{
context.WriteByte(BinaryTypeCode.ObjectWithMetadata);
context.WriteVarUInt((uint)metadataFooterIndex);
}
else
{
// No reference handling - just write object marker
context.WriteByte(BinaryTypeCode.Object);
}

View File

@ -76,12 +76,20 @@ public sealed class AcBinarySerializerOptions : AcSerializerOptions
public int MaxCachedStringLength { get; init; } = 64;
/// <summary>
/// Whether to include metadata header with property names.
/// NOTE: Currently unused - deserializer uses ordered property indices, not names.
/// Kept for potential future schema evolution support.
/// Whether to include property hash metadata in footer for cross-type deserialization.
/// When enabled, property name hashes (FNV-1a) are written per type in the footer,
/// allowing the deserializer to match properties by name between different types.
/// Default: false (no overhead)
/// </summary>
public bool UseMetadata { get; init; } = false;
public bool UseMetadata { get; init; } = true;
/// <summary>
/// When true, checks for duplicate property name hashes during serialization (UseMetadata mode).
/// Throws exception if FNV-1a hash collision is detected between property names of the same type.
/// Should be enabled during development/testing, can be disabled in production for performance.
/// Default: true (safety first)
/// </summary>
public bool CheckDuplicatePropName { get; init; } = true;
/// <summary>
/// Controls how string interning is applied during serialization.

View File

@ -49,6 +49,7 @@ internal static class BinaryTypeCode
public const byte Array = 28; // Start of array/list
public const byte Dictionary = 29; // Start of dictionary
public const byte ByteArray = 30; // Optimized byte[] storage
public const byte ObjectWithMetadata = 31; // Object with metadata footer index (UseMetadata nested objects)
// Special markers (32+, for header/meta)
// Header flags byte structure (for values >= 64):

View File

@ -0,0 +1,26 @@
using System.Runtime.CompilerServices;
namespace AyCode.Core.Serializers;
/// <summary>
/// FNV-1a 32-bit hash implementation.
/// Deterministic across processes (unlike string.GetHashCode()).
/// Used for property name hashing in UseMetadata mode.
/// </summary>
public static class FnvHash
{
private const uint FnvOffsetBasis = 2166136261;
private const uint FnvPrime = 16777619;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int ComputeString(string value)
{
var hash = FnvOffsetBasis;
for (var i = 0; i < value.Length; i++)
{
hash ^= value[i];
hash *= FnvPrime;
}
return (int)hash;
}
}

View File

@ -75,6 +75,12 @@ public abstract class PropertyMetadataBase
/// </summary>
public bool IsComplexType { get; }
/// <summary>
/// FNV-1a hash of property name. Deterministic across processes.
/// Used for property matching in UseMetadata mode.
/// </summary>
public int PropertyNameHash { get; }
/// <summary>
/// The accessor type for fast typed getter/setter dispatch.
/// </summary>
@ -101,6 +107,8 @@ public abstract class PropertyMetadataBase
// Pre-compute: is this a complex type that needs recursive handling?
IsComplexType = !IsPrimitiveOrStringFast(PropertyType);
PropertyNameHash = FnvHash.ComputeString(Name);
AccessorType = DetermineAccessorType(PropertyType);
_dynamicGetter = AcSerializerCommon.CreateCompiledGetter(declaringType, prop);
}

View File

@ -35,6 +35,13 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
/// </summary>
internal readonly Func<object, Guid>? RefIdGetterGuid;
/// <summary>
/// UseMetadata: footer entry index for this type in the current serialization session.
/// -1 = not yet registered. Set by RegisterMetadataType, reset by ResetTracking.
/// Eliminates the need for Dictionary&lt;Type, int&gt; lookup in the serializer hot path.
/// </summary>
internal int MetadataFooterIndex = -1;
#region Typed IdentityMaps - No generic type checks in hot path!
/// <summary>
@ -110,6 +117,8 @@ public sealed class TypeMetadataWrapper<TMetadata> where TMetadata : TypeMetadat
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ResetTracking(bool preRentBuckets = false)
{
MetadataFooterIndex = -1;
if (SmallIdBitmap != null)
Array.Clear(SmallIdBitmap);