Adam
/
AyCode.Core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate UseMetadata to inline format, remove metadata footer 

Refactored AcBinarySerializer to write type property metadata inline
after the ObjectWithMetadata marker, eliminating the need for a
separate metadata footer section. Updated serialization, deserialization,
and diagnostic test logic to support the new inline metadata format.
Also updated settings.local.json to allow "Bash(git stash:*)" commands.
This commit is contained in:
Loretta 2026-02-04 16:04:53 +01:00
parent 18370879ec
commit 1410ee71f0
5 changed files with 107 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.claude/settings.local.json
View File

@ -30,7 +30,8 @@
"Bash(dotnet new:*)",
"Bash(Remove-Item \"H:\\\\Applications\\\\Aycode\\\\Source\\\\AyCode.Core\\\\AyCode.Core\\\\Serializers\\\\Toons\\\\AcToonSerializer.RelationshipDetection.cs\")",
"Bash(find:*)",
"Bash(dir:*)"
"Bash(dir:*)",
"Bash(git stash:*)"
]
}
}

90
AyCode.Core.Tests/Serialization/AcBinarySerializerDiagnosticTests.cs
View File

@ -262,20 +262,55 @@ public class AcBinarySerializerDiagnosticTests
// Skip any header data (strings interning, etc.)
// New format uses PropertyIndex directly - no metadata header with property names
// Find Object marker (0x19)
while (pos < binary.Length && binary[pos] != 0x19)
// Find Object marker (0x19) or ObjectWithMetadata marker (0x1F)
while (pos < binary.Length && binary[pos] != 0x19 && binary[pos] != 0x1F)
{
pos++;
}
Console.WriteLine($"\n=== BODY (starts at position {pos}) ===");
// The body should start with Object marker (0x19)
// The body should start with Object (0x19) or ObjectWithMetadata (0x1F) marker
var bodyStart = pos;
var objectMarker = binary[pos++];
Console.WriteLine($"Object marker: 0x{objectMarker:X2} (expected 0x19 for Object)");
Assert.AreEqual(0x19, objectMarker, "Object marker should be 0x19");
Console.WriteLine($"Object marker: 0x{objectMarker:X2} (0x19=Object, 0x1F=ObjectWithMetadata)");
Assert.IsTrue(objectMarker == 0x19 || objectMarker == 0x1F,
$"Object marker should be 0x19 or 0x1F, got 0x{objectMarker:X2}");
// If ObjectWithMetadata (0x1F), skip inline metadata
if (objectMarker == 0x1F)
{
// propNameHash (4 bytes)
var propNameHash = BitConverter.ToInt32(binary, pos);
pos += 4;
Console.WriteLine($"PropNameHash: 0x{propNameHash:X8}");
// First occurrence: propCount (VarUInt) + property hashes
// VarUInt: if top bit is set, continue reading
var propCountByte = binary[pos];
int inlinePropCount;
if ((propCountByte & 0x80) == 0)
{
inlinePropCount = propCountByte;
pos++;
}
else
{
// Multi-byte VarUInt - simplified 2-byte parsing
inlinePropCount = (propCountByte & 0x7F) | (binary[pos + 1] << 7);
pos += 2;
}
Console.WriteLine($"Inline metadata propCount: {inlinePropCount}");
// Skip property hashes (4 bytes each)
for (int h = 0; h < inlinePropCount; h++)
{
var hash = BitConverter.ToInt32(binary, pos);
Console.WriteLine($" Property hash [{h}]: 0x{hash:X8}");
pos += 4;
}
}
// Read ref ID (if reference handling is enabled)
// VarInt: if top bit is set, continue reading
var refIdByte = binary[pos];
@ -292,43 +327,50 @@ public class AcBinarySerializerDiagnosticTests
pos += 2; // Skip for now
}
Console.WriteLine($"RefId: {refId}");
// Read property count in body
var bodyPropCount = binary[pos++];
Console.WriteLine($"Property count in body: {bodyPropCount}");
Console.WriteLine($"\n=== BODY PROPERTIES (PropertyIndex format) ===");
Console.WriteLine($"\n=== BODY PROPERTIES ===");
for (int i = 0; i < bodyPropCount && pos < binary.Length; i++)
{
var propIndex = binary[pos++]; // This is now PropertyIndex (alphabetical order)
Console.WriteLine($" Body property [{i}]: PropertyIndex={propIndex}");
// Skip the value (simplified - just log)
// Log the value (no PropertyIndex in inline metadata mode — properties are in hash order)
var valueType = binary[pos];
if (valueType == 0x14) // DateTime
{
Console.WriteLine($" -> DateTime (9 bytes)");
Console.WriteLine($" Property [{i}]: DateTime (9 bytes)");
pos += 10; // type + 9 bytes
}
else if (valueType >= 0xD0 && valueType <= 0xE7) // TinyInt
else if (valueType >= 0xC0 && valueType <= 0xFF) // TinyInt (192-255)
{
var tinyValue = valueType - 0xD0;
Console.WriteLine($" -> TinyInt value: {tinyValue}");
var tinyValue = valueType - 192 - 16;
Console.WriteLine($" Property [{i}]: TinyInt value: {tinyValue}");
pos += 1;
}
else if (valueType == 0x03) // False
else if (valueType == 0x02) // False (BinaryTypeCode.False = 2)
{
Console.WriteLine($" -> Boolean: false");
Console.WriteLine($" Property [{i}]: Boolean: false");
pos += 1;
}
else if (valueType == 0x02) // True
else if (valueType == 0x01) // True (BinaryTypeCode.True = 1)
{
Console.WriteLine($" -> Boolean: true");
Console.WriteLine($" Property [{i}]: Boolean: true");
pos += 1;
}
else if (valueType == 0x00) // Null
{
Console.WriteLine($" Property [{i}]: Null");
pos += 1;
}
else if (valueType == 0xBF) // PropertySkip
{
Console.WriteLine($" Property [{i}]: PropertySkip (default/null)");
pos += 1;
}
else
{
Console.WriteLine($" -> Unknown type: 0x{valueType:X2}");
Console.WriteLine($" Property [{i}]: Unknown type: 0x{valueType:X2}");
break;
}
}

35
AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.BinaryDeserializationContext.cs
View File

@ -195,44 +195,13 @@ public static partial class AcBinaryDeserializer
_nextDupPosition = _dupData[0];
}
// Read UseMetadata footer section (per-type property hashes)
if (HasMetadata && _position < _buffer.Length)
{
ReadMetadataFooter();
}
// Metadata is now inline in the body (not in footer).
// No ReadMetadataFooter() call needed.
// Seek back to data position
_position = dataPosition;
}
/// <summary>
/// UseMetadata footer olvasása: entry-k flat array-be.
/// Formátum: [entryCount (VarUInt)]
/// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// Entry 0 = root, 1+ = nested object-ek.
/// </summary>
private void ReadMetadataFooter()
{
var entryCount = (int)ReadVarUInt();
for (var i = 0; i < entryCount; i++)
{
EnsureAvailable(4);
var propNameHash = Unsafe.ReadUnaligned<int>(ref Unsafe.AsRef(in _buffer[_position]));
_position += 4;
var propCount = (int)ReadVarUInt();
var hashes = new int[propCount];
for (var p = 0; p < propCount; p++)
{
EnsureAvailable(4);
hashes[p] = Unsafe.ReadUnaligned<int>(ref Unsafe.AsRef(in _buffer[_position]));
_position += 4;
}
ContextClass.RegisterFooterEntry(i, propNameHash, hashes);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte ReadByte() => ReadByteInternal();

95
AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs
View File

@ -74,14 +74,6 @@ public static partial class AcBinarySerializer
private const int PropertyIndexBufferMaxCache = 512;
private const int PropertyStateBufferMaxCache = 512;
private const int InitialInternCapacity = 32;
/// <summary>
/// UseMetadata footer bejegyzések sorrendben.
/// Minden bejegyzés egy (Type, propertyHashes) pár.
/// A 0. elem mindig a root object.
/// A body-ban a nested object-eknél az index (VarUInt) kerül kiírásra.
/// </summary>
private List<MetadataFooterEntry>? _metadataEntries;
private byte[] _buffer;
private int _position;
private int _initialBufferSize;
@ -129,7 +121,11 @@ public static partial class AcBinarySerializer
/// <summary>
/// True if we need footer position in header (string interning OR reference handling OR metadata).
/// </summary>
public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None || UseMetadata;
/// <summary>
/// True if we need footer position in header (string interning OR reference handling).
/// UseMetadata no longer uses footer — metadata is inline in the body.
/// </summary>
public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None;
public bool UseMetadata => Options.UseMetadata;
public byte MinStringInternLength => Options.MinStringInternLength;
public byte MaxStringInternLength => Options.MaxStringInternLength;
@ -180,7 +176,6 @@ public static partial class AcBinarySerializer
//_refTracker.Reset();
_stringInternMap?.Reset();
_metadataEntries?.Clear();
_nextCacheIndex = 0;
if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length > PropertyIndexBufferMaxCache)
@ -367,68 +362,37 @@ public static partial class AcBinarySerializer
#region UseMetadata Type Tracking
/// <summary>
/// Egy footer bejegyzés: típus propNameHash + property hash-ek.
/// </summary>
internal readonly struct MetadataFooterEntry
{
public readonly int PropNameHash; // FNV-1a hash a típus nevéből
public readonly int[] PropertyHashes; // property name hash-ek sorrendben
public MetadataFooterEntry(int propNameHash, int[] propertyHashes)
{
PropNameHash = propNameHash;
PropertyHashes = propertyHashes;
}
}
/// <summary>
/// Regisztrálja a típust a UseMetadata footer-be.
/// Visszaadja a footer index-et. Ha már regisztrálva van (wrapper.MetadataFooterIndex >= 0),
/// a meglévő index-et adja vissza. Nincs Dictionary lookup — a wrapper tárolja az indexet.
/// Regisztrálja a típust UseMetadata módban.
/// Visszaadja true-t ha ez az első előfordulás (inline hash-eket kell írni),
/// false-t ha ismételt (csak propNameHash kell).
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int RegisterMetadataType(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
public bool RegisterMetadataType(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
{
if (wrapper.MetadataFooterIndex >= 0)
return wrapper.MetadataFooterIndex;
return false; // ismételt
return RegisterMetadataTypeSlow(wrapper);
}
[MethodImpl(MethodImplOptions.NoInlining)]
private int RegisterMetadataTypeSlow(TypeMetadataWrapper<BinarySerializeTypeMetadata> wrapper)
{
_metadataEntries ??= new List<MetadataFooterEntry>();
// PropNameHash and MetadataPropertyHashes are lazy-computed once per type in metadata.
// Duplicate hash validation also happens once (in MetadataPropertyHashes getter).
var index = _metadataEntries.Count;
_metadataEntries.Add(new MetadataFooterEntry(
wrapper.Metadata.PropNameHash,
wrapper.Metadata.MetadataPropertyHashes));
wrapper.MetadataFooterIndex = index;
return index;
wrapper.MetadataFooterIndex = 0; // jelöljük hogy már regisztrálva
return true; // első előfordulás
}
/// <summary>
/// UseMetadata footer kiírása.
/// Formátum: [entryCount (VarUInt)]
/// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// Inline metadata kiírása az ObjectWithMetadata marker után.
/// Első előfordulás: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]...
/// Ismételt: [propNameHash (4b)]
/// </summary>
public void WriteMetadataFooter()
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void WriteInlineMetadata(BinarySerializeTypeMetadata metadata, bool isFirstOccurrence)
{
if (_metadataEntries == null || _metadataEntries.Count == 0) return;
WriteRaw(metadata.PropNameHash);
WriteVarUInt((uint)_metadataEntries.Count);
for (var i = 0; i < _metadataEntries.Count; i++)
if (isFirstOccurrence)
{
var entry = _metadataEntries[i];
WriteRaw(entry.PropNameHash);
WriteVarUInt((uint)entry.PropertyHashes.Length);
for (var j = 0; j < entry.PropertyHashes.Length; j++)
var hashes = metadata.MetadataPropertyHashes;
WriteVarUInt((uint)hashes.Length);
for (var i = 0; i < hashes.Length; i++)
{
WriteRaw(entry.PropertyHashes[j]);
WriteRaw(hashes[i]);
}
}
}
@ -1063,27 +1027,22 @@ public static partial class AcBinarySerializer
{
var dupCount = GetDupCount(); // Shared counter: string intern + ID tracking
var hasInternTable = dupCount > 0;
var hasMetadataFooter = UseMetadata && _metadataEntries is { Count: > 0 };
// Footer: write merged intern entries (string + ID)
// Metadata footer is no longer written here — metadata is inline in the body.
var footerPosition = 0;
if (hasInternTable || hasMetadataFooter)
if (hasInternTable)
{
footerPosition = _position;
// Intern footer
WriteVarUInt((uint)dupCount);
if (hasInternTable)
WriteInternedFooter();
// Metadata footer (per-type property hashes)
if (hasMetadataFooter)
WriteMetadataFooter();
WriteInternedFooter();
}
// Write header
var flags = BinaryTypeCode.HeaderFlagsBase;
if (hasMetadataFooter)
if (UseMetadata)
flags |= BinaryTypeCode.HeaderFlag_Metadata;
// Encode ReferenceHandlingMode using separate bits
if (ReferenceHandling == ReferenceHandlingMode.OnlyId)

26
AyCode.Core/Serializers/Binaries/AcBinarySerializer.cs
View File

@ -775,22 +775,16 @@ public static partial class AcBinarySerializer
var metadata = wrapper.Metadata;
// Wire format:
// - IId types: [Object/ObjectWithMetadata][props 0-tól...] - Id a props-ban, nincs extra
// - Non-IId + All: [Object/ObjectWithMetadata][props 0-tól...] - no hashcode prefix
// - Ref=Off: [Object/ObjectWithMetadata][props 0-tól...] - no prefix
// ObjectRef format:
// - IId: [ObjectRef][cacheIndex]
// - Non-IId: [ObjectRef][cacheIndex]
//
// UseMetadata:
// - Root (depth==0): [Object] marker, footer entry 0
// - Nested: [ObjectWithMetadata][footerIndex (VarUInt)]
// - UseMetadata=false: [Object][props...]
// - UseMetadata=true, első: [ObjectWithMetadata][propNameHash (4b)][propCount (VarUInt)][hash0..N][props...]
// - UseMetadata=true, ismételt: [ObjectWithMetadata][propNameHash (4b)][props...]
// ObjectRef: [ObjectRef][cacheIndex]
// UseMetadata: regisztráljuk a típust a footer-be (index kell a marker kiíráshoz)
var metadataFooterIndex = -1;
// UseMetadata: típus regisztrálása (első vs ismételt előfordulás tracking)
var isFirstMetadataOccurrence = false;
if (context.UseMetadata)
{
metadataFooterIndex = context.RegisterMetadataType(wrapper);
isFirstMetadataOccurrence = context.RegisterMetadataType(wrapper);
}
if (context.UseTypeReferenceHandling(metadata))
@ -840,11 +834,11 @@ public static partial class AcBinarySerializer
}
}
// Marker kiírása: UseMetadata nested → ObjectWithMetadata + footer index, egyébként Object
if (context.UseMetadata && isNested)
// Marker kiírása: UseMetadata → ObjectWithMetadata + inline metadata, egyébként Object
if (context.UseMetadata)
{
context.WriteByte(BinaryTypeCode.ObjectWithMetadata);
context.WriteVarUInt((uint)metadataFooterIndex);
context.WriteInlineMetadata(wrapper.Metadata, isFirstMetadataOccurrence);
}
else
{