diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 5afb158..a3e4284 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -30,7 +30,8 @@ "Bash(dotnet new:*)", "Bash(Remove-Item \"H:\\\\Applications\\\\Aycode\\\\Source\\\\AyCode.Core\\\\AyCode.Core\\\\Serializers\\\\Toons\\\\AcToonSerializer.RelationshipDetection.cs\")", "Bash(find:*)", - "Bash(dir:*)" + "Bash(dir:*)", + "Bash(git stash:*)" ] } } diff --git a/AyCode.Core.Tests/Serialization/AcBinarySerializerDiagnosticTests.cs b/AyCode.Core.Tests/Serialization/AcBinarySerializerDiagnosticTests.cs index 62317d5..ea446b6 100644 --- a/AyCode.Core.Tests/Serialization/AcBinarySerializerDiagnosticTests.cs +++ b/AyCode.Core.Tests/Serialization/AcBinarySerializerDiagnosticTests.cs @@ -262,20 +262,55 @@ public class AcBinarySerializerDiagnosticTests // Skip any header data (strings interning, etc.) // New format uses PropertyIndex directly - no metadata header with property names - // Find Object marker (0x19) - while (pos < binary.Length && binary[pos] != 0x19) + // Find Object marker (0x19) or ObjectWithMetadata marker (0x1F) + while (pos < binary.Length && binary[pos] != 0x19 && binary[pos] != 0x1F) { pos++; } - + Console.WriteLine($"\n=== BODY (starts at position {pos}) ==="); - - // The body should start with Object marker (0x19) + + // The body should start with Object (0x19) or ObjectWithMetadata (0x1F) marker var bodyStart = pos; var objectMarker = binary[pos++]; - Console.WriteLine($"Object marker: 0x{objectMarker:X2} (expected 0x19 for Object)"); - Assert.AreEqual(0x19, objectMarker, "Object marker should be 0x19"); - + Console.WriteLine($"Object marker: 0x{objectMarker:X2} (0x19=Object, 0x1F=ObjectWithMetadata)"); + Assert.IsTrue(objectMarker == 0x19 || objectMarker == 0x1F, + $"Object marker should be 0x19 or 0x1F, got 0x{objectMarker:X2}"); + + // If ObjectWithMetadata (0x1F), skip inline metadata + if (objectMarker == 0x1F) + { + // propNameHash (4 bytes) + var propNameHash = BitConverter.ToInt32(binary, pos); + pos += 4; + Console.WriteLine($"PropNameHash: 0x{propNameHash:X8}"); + + // First occurrence: propCount (VarUInt) + property hashes + // VarUInt: if top bit is set, continue reading + var propCountByte = binary[pos]; + int inlinePropCount; + if ((propCountByte & 0x80) == 0) + { + inlinePropCount = propCountByte; + pos++; + } + else + { + // Multi-byte VarUInt - simplified 2-byte parsing + inlinePropCount = (propCountByte & 0x7F) | (binary[pos + 1] << 7); + pos += 2; + } + Console.WriteLine($"Inline metadata propCount: {inlinePropCount}"); + + // Skip property hashes (4 bytes each) + for (int h = 0; h < inlinePropCount; h++) + { + var hash = BitConverter.ToInt32(binary, pos); + Console.WriteLine($" Property hash [{h}]: 0x{hash:X8}"); + pos += 4; + } + } + // Read ref ID (if reference handling is enabled) // VarInt: if top bit is set, continue reading var refIdByte = binary[pos]; @@ -292,43 +327,50 @@ public class AcBinarySerializerDiagnosticTests pos += 2; // Skip for now } Console.WriteLine($"RefId: {refId}"); - + // Read property count in body var bodyPropCount = binary[pos++]; Console.WriteLine($"Property count in body: {bodyPropCount}"); - - Console.WriteLine($"\n=== BODY PROPERTIES (PropertyIndex format) ==="); + + Console.WriteLine($"\n=== BODY PROPERTIES ==="); for (int i = 0; i < bodyPropCount && pos < binary.Length; i++) { - var propIndex = binary[pos++]; // This is now PropertyIndex (alphabetical order) - Console.WriteLine($" Body property [{i}]: PropertyIndex={propIndex}"); - - // Skip the value (simplified - just log) + // Log the value (no PropertyIndex in inline metadata mode — properties are in hash order) var valueType = binary[pos]; if (valueType == 0x14) // DateTime { - Console.WriteLine($" -> DateTime (9 bytes)"); + Console.WriteLine($" Property [{i}]: DateTime (9 bytes)"); pos += 10; // type + 9 bytes } - else if (valueType >= 0xD0 && valueType <= 0xE7) // TinyInt + else if (valueType >= 0xC0 && valueType <= 0xFF) // TinyInt (192-255) { - var tinyValue = valueType - 0xD0; - Console.WriteLine($" -> TinyInt value: {tinyValue}"); + var tinyValue = valueType - 192 - 16; + Console.WriteLine($" Property [{i}]: TinyInt value: {tinyValue}"); pos += 1; } - else if (valueType == 0x03) // False + else if (valueType == 0x02) // False (BinaryTypeCode.False = 2) { - Console.WriteLine($" -> Boolean: false"); + Console.WriteLine($" Property [{i}]: Boolean: false"); pos += 1; } - else if (valueType == 0x02) // True + else if (valueType == 0x01) // True (BinaryTypeCode.True = 1) { - Console.WriteLine($" -> Boolean: true"); + Console.WriteLine($" Property [{i}]: Boolean: true"); + pos += 1; + } + else if (valueType == 0x00) // Null + { + Console.WriteLine($" Property [{i}]: Null"); + pos += 1; + } + else if (valueType == 0xBF) // PropertySkip + { + Console.WriteLine($" Property [{i}]: PropertySkip (default/null)"); pos += 1; } else { - Console.WriteLine($" -> Unknown type: 0x{valueType:X2}"); + Console.WriteLine($" Property [{i}]: Unknown type: 0x{valueType:X2}"); break; } } diff --git a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.BinaryDeserializationContext.cs b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.BinaryDeserializationContext.cs index c7188d9..09a4182 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.BinaryDeserializationContext.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinaryDeserializer.BinaryDeserializationContext.cs @@ -195,44 +195,13 @@ public static partial class AcBinaryDeserializer _nextDupPosition = _dupData[0]; } - // Read UseMetadata footer section (per-type property hashes) - if (HasMetadata && _position < _buffer.Length) - { - ReadMetadataFooter(); - } + // Metadata is now inline in the body (not in footer). + // No ReadMetadataFooter() call needed. // Seek back to data position _position = dataPosition; } - /// - /// UseMetadata footer olvasása: entry-k flat array-be. - /// Formátum: [entryCount (VarUInt)] - /// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]... - /// Entry 0 = root, 1+ = nested object-ek. - /// - private void ReadMetadataFooter() - { - var entryCount = (int)ReadVarUInt(); - for (var i = 0; i < entryCount; i++) - { - EnsureAvailable(4); - var propNameHash = Unsafe.ReadUnaligned(ref Unsafe.AsRef(in _buffer[_position])); - _position += 4; - - var propCount = (int)ReadVarUInt(); - var hashes = new int[propCount]; - for (var p = 0; p < propCount; p++) - { - EnsureAvailable(4); - hashes[p] = Unsafe.ReadUnaligned(ref Unsafe.AsRef(in _buffer[_position])); - _position += 4; - } - - ContextClass.RegisterFooterEntry(i, propNameHash, hashes); - } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public byte ReadByte() => ReadByteInternal(); diff --git a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs index 8e5e105..adf53c5 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.BinarySerializationContext.cs @@ -74,14 +74,6 @@ public static partial class AcBinarySerializer private const int PropertyIndexBufferMaxCache = 512; private const int PropertyStateBufferMaxCache = 512; private const int InitialInternCapacity = 32; - /// - /// UseMetadata footer bejegyzések sorrendben. - /// Minden bejegyzés egy (Type, propertyHashes) pár. - /// A 0. elem mindig a root object. - /// A body-ban a nested object-eknél az index (VarUInt) kerül kiírásra. - /// - private List? _metadataEntries; - private byte[] _buffer; private int _position; private int _initialBufferSize; @@ -129,7 +121,11 @@ public static partial class AcBinarySerializer /// /// True if we need footer position in header (string interning OR reference handling OR metadata). /// - public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None || UseMetadata; + /// + /// True if we need footer position in header (string interning OR reference handling). + /// UseMetadata no longer uses footer — metadata is inline in the body. + /// + public bool HasFooter => UseStringInterning || ReferenceHandling != ReferenceHandlingMode.None; public bool UseMetadata => Options.UseMetadata; public byte MinStringInternLength => Options.MinStringInternLength; public byte MaxStringInternLength => Options.MaxStringInternLength; @@ -180,7 +176,6 @@ public static partial class AcBinarySerializer //_refTracker.Reset(); _stringInternMap?.Reset(); - _metadataEntries?.Clear(); _nextCacheIndex = 0; if (_propertyIndexBuffer != null && _propertyIndexBuffer.Length > PropertyIndexBufferMaxCache) @@ -367,68 +362,37 @@ public static partial class AcBinarySerializer #region UseMetadata Type Tracking /// - /// Egy footer bejegyzés: típus propNameHash + property hash-ek. - /// - internal readonly struct MetadataFooterEntry - { - public readonly int PropNameHash; // FNV-1a hash a típus nevéből - public readonly int[] PropertyHashes; // property name hash-ek sorrendben - - public MetadataFooterEntry(int propNameHash, int[] propertyHashes) - { - PropNameHash = propNameHash; - PropertyHashes = propertyHashes; - } - } - - /// - /// Regisztrálja a típust a UseMetadata footer-be. - /// Visszaadja a footer index-et. Ha már regisztrálva van (wrapper.MetadataFooterIndex >= 0), - /// a meglévő index-et adja vissza. Nincs Dictionary lookup — a wrapper tárolja az indexet. + /// Regisztrálja a típust UseMetadata módban. + /// Visszaadja true-t ha ez az első előfordulás (inline hash-eket kell írni), + /// false-t ha ismételt (csak propNameHash kell). /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int RegisterMetadataType(TypeMetadataWrapper wrapper) + public bool RegisterMetadataType(TypeMetadataWrapper wrapper) { if (wrapper.MetadataFooterIndex >= 0) - return wrapper.MetadataFooterIndex; + return false; // ismételt - return RegisterMetadataTypeSlow(wrapper); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private int RegisterMetadataTypeSlow(TypeMetadataWrapper wrapper) - { - _metadataEntries ??= new List(); - - // PropNameHash and MetadataPropertyHashes are lazy-computed once per type in metadata. - // Duplicate hash validation also happens once (in MetadataPropertyHashes getter). - var index = _metadataEntries.Count; - _metadataEntries.Add(new MetadataFooterEntry( - wrapper.Metadata.PropNameHash, - wrapper.Metadata.MetadataPropertyHashes)); - wrapper.MetadataFooterIndex = index; - return index; + wrapper.MetadataFooterIndex = 0; // jelöljük hogy már regisztrálva + return true; // első előfordulás } /// - /// UseMetadata footer kiírása. - /// Formátum: [entryCount (VarUInt)] - /// entry-nként: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]... + /// Inline metadata kiírása az ObjectWithMetadata marker után. + /// Első előfordulás: [propNameHash (4b)][propCount (VarUInt)][hash0 (4b)][hash1 (4b)]... + /// Ismételt: [propNameHash (4b)] /// - public void WriteMetadataFooter() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void WriteInlineMetadata(BinarySerializeTypeMetadata metadata, bool isFirstOccurrence) { - if (_metadataEntries == null || _metadataEntries.Count == 0) return; + WriteRaw(metadata.PropNameHash); - WriteVarUInt((uint)_metadataEntries.Count); - - for (var i = 0; i < _metadataEntries.Count; i++) + if (isFirstOccurrence) { - var entry = _metadataEntries[i]; - WriteRaw(entry.PropNameHash); - WriteVarUInt((uint)entry.PropertyHashes.Length); - for (var j = 0; j < entry.PropertyHashes.Length; j++) + var hashes = metadata.MetadataPropertyHashes; + WriteVarUInt((uint)hashes.Length); + for (var i = 0; i < hashes.Length; i++) { - WriteRaw(entry.PropertyHashes[j]); + WriteRaw(hashes[i]); } } } @@ -1063,27 +1027,22 @@ public static partial class AcBinarySerializer { var dupCount = GetDupCount(); // Shared counter: string intern + ID tracking var hasInternTable = dupCount > 0; - var hasMetadataFooter = UseMetadata && _metadataEntries is { Count: > 0 }; // Footer: write merged intern entries (string + ID) + // Metadata footer is no longer written here — metadata is inline in the body. var footerPosition = 0; - if (hasInternTable || hasMetadataFooter) + if (hasInternTable) { footerPosition = _position; // Intern footer WriteVarUInt((uint)dupCount); - if (hasInternTable) - WriteInternedFooter(); - - // Metadata footer (per-type property hashes) - if (hasMetadataFooter) - WriteMetadataFooter(); + WriteInternedFooter(); } // Write header var flags = BinaryTypeCode.HeaderFlagsBase; - if (hasMetadataFooter) + if (UseMetadata) flags |= BinaryTypeCode.HeaderFlag_Metadata; // Encode ReferenceHandlingMode using separate bits if (ReferenceHandling == ReferenceHandlingMode.OnlyId) diff --git a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.cs b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.cs index 72500bb..63bfc62 100644 --- a/AyCode.Core/Serializers/Binaries/AcBinarySerializer.cs +++ b/AyCode.Core/Serializers/Binaries/AcBinarySerializer.cs @@ -775,22 +775,16 @@ public static partial class AcBinarySerializer var metadata = wrapper.Metadata; // Wire format: - // - IId types: [Object/ObjectWithMetadata][props 0-tól...] - Id a props-ban, nincs extra - // - Non-IId + All: [Object/ObjectWithMetadata][props 0-tól...] - no hashcode prefix - // - Ref=Off: [Object/ObjectWithMetadata][props 0-tól...] - no prefix - // ObjectRef format: - // - IId: [ObjectRef][cacheIndex] - // - Non-IId: [ObjectRef][cacheIndex] - // - // UseMetadata: - // - Root (depth==0): [Object] marker, footer entry 0 - // - Nested: [ObjectWithMetadata][footerIndex (VarUInt)] + // - UseMetadata=false: [Object][props...] + // - UseMetadata=true, első: [ObjectWithMetadata][propNameHash (4b)][propCount (VarUInt)][hash0..N][props...] + // - UseMetadata=true, ismételt: [ObjectWithMetadata][propNameHash (4b)][props...] + // ObjectRef: [ObjectRef][cacheIndex] - // UseMetadata: regisztráljuk a típust a footer-be (index kell a marker kiíráshoz) - var metadataFooterIndex = -1; + // UseMetadata: típus regisztrálása (első vs ismételt előfordulás tracking) + var isFirstMetadataOccurrence = false; if (context.UseMetadata) { - metadataFooterIndex = context.RegisterMetadataType(wrapper); + isFirstMetadataOccurrence = context.RegisterMetadataType(wrapper); } if (context.UseTypeReferenceHandling(metadata)) @@ -840,11 +834,11 @@ public static partial class AcBinarySerializer } } - // Marker kiírása: UseMetadata nested → ObjectWithMetadata + footer index, egyébként Object - if (context.UseMetadata && isNested) + // Marker kiírása: UseMetadata → ObjectWithMetadata + inline metadata, egyébként Object + if (context.UseMetadata) { context.WriteByte(BinaryTypeCode.ObjectWithMetadata); - context.WriteVarUInt((uint)metadataFooterIndex); + context.WriteInlineMetadata(wrapper.Metadata, isFirstMetadataOccurrence); } else {