[LOADED_DOCS: 3 files, no new loads]
Refactor AcBinary streaming: multi-message protocol - Renamed framing flags to multiMessage for clarity in AsyncPipeReaderInput/AsyncPipeWriterOutput. - Multi-message mode ([202]=end-of-message) now auto-resets input for reuse; session end is explicit. - Updated framing state machine, buffer cycling, and sentinel logic. - Revised all serializer/deserializer entry points and tests for new protocol. - Expanded docs and XML comments to detail wire format and protocol constraints. - Updated benchmarks and tests for new streaming API and multi-message behavior. - Documented protocol limits and added security issue/TODO for type-name deserialization in SignalR binary protocol.
This commit is contained in:
parent
42b40a92c1
commit
204b361748
|
|
@ -57,22 +57,21 @@ public static class Program
|
|||
private const string ModeRuntime = "Runtime";
|
||||
private const string ModeHybrid = "Hybrid";
|
||||
|
||||
private const int JitSleep = 3000;
|
||||
|
||||
// OptionsPreset values are passed per-instance (constructor argument), not constants —
|
||||
// each CreateSerializers call line specifies its own preset name (e.g. "FastMode", "NoIntern").
|
||||
|
||||
private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false);
|
||||
|
||||
#if DEBUG
|
||||
private static int WarmupIterations = 0;
|
||||
private static int TestIterations = 1;
|
||||
private static int BenchmarkSamples = 1; // Debug: single sample, fast iteration
|
||||
private const int WarmupIterations = 0;
|
||||
private const int TestIterations = 1;
|
||||
private const int BenchmarkSamples = 1; // Debug: single sample, fast iteration
|
||||
#else
|
||||
private static int WarmupIterations = 5000;
|
||||
private static int TestIterations = 1000;
|
||||
private static int BenchmarkSamples = 5; // Release: 5-sample median for stability (~±5% variance vs. ~±15% single-sample)
|
||||
|
||||
//private static int WarmupIterations = 5000;
|
||||
//private static int TestIterations = 2000;
|
||||
private static int WarmupIterations = 5;
|
||||
private static int TestIterations = 1;
|
||||
private static int BenchmarkSamples = 1;
|
||||
#endif
|
||||
|
||||
public static void Main(string[] args)
|
||||
|
|
@ -87,7 +86,7 @@ public static class Program
|
|||
// Determine layer (which test data to run) and opMode (ser/des/all).
|
||||
// CLI args take precedence; if no args, show interactive menu.
|
||||
string layer;
|
||||
string opMode = "all";
|
||||
var opMode = "all";
|
||||
|
||||
if (args.Length == 0)
|
||||
{
|
||||
|
|
@ -175,7 +174,7 @@ public static class Program
|
|||
}
|
||||
}
|
||||
// Let background tiered-JIT compilation drain before we begin measuring.
|
||||
Thread.Sleep(3000);
|
||||
Thread.Sleep(JitSleep);
|
||||
System.Console.WriteLine("✓ Global pre-warmup complete.\n");
|
||||
}
|
||||
|
||||
|
|
@ -215,7 +214,7 @@ public static class Program
|
|||
var options = AcBinarySerializerOptions.WithoutReferenceHandling;
|
||||
options.UseStringInterning = StringInterningMode.None;
|
||||
|
||||
byte[] bytes = AcBinarySerializer.Serialize(order, options);
|
||||
var bytes = AcBinarySerializer.Serialize(order, options);
|
||||
// Warmup (fills caches)
|
||||
System.Console.WriteLine("Warming up (1000 iterations)...");
|
||||
for (var i = 0; i < 1000; i++)
|
||||
|
|
@ -261,6 +260,7 @@ public static class Program
|
|||
|
||||
// Round-trip correctness check — once per (cell × serializer), BEFORE warmup. Aborts the entire benchmark on failure.
|
||||
System.Console.WriteLine("Verifying round-trip correctness...");
|
||||
|
||||
foreach (var serializer in serializers)
|
||||
{
|
||||
if (!serializer.VerifyRoundTrip())
|
||||
|
|
@ -270,6 +270,7 @@ public static class Program
|
|||
Environment.Exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
System.Console.WriteLine("✓ All serializers passed round-trip verification.");
|
||||
|
||||
// Warmup all serializers
|
||||
|
|
@ -280,7 +281,7 @@ public static class Program
|
|||
}
|
||||
|
||||
// Wait for tiered JIT background compilation to complete
|
||||
Thread.Sleep(3000);
|
||||
Thread.Sleep(JitSleep);
|
||||
|
||||
// Run benchmarks
|
||||
System.Console.WriteLine($"Running benchmarks ({TestIterations} iterations × {BenchmarkSamples} samples median)...\n");
|
||||
|
|
@ -421,7 +422,7 @@ public static class Program
|
|||
}
|
||||
|
||||
var times = new double[samples];
|
||||
for (int s = 0; s < samples; s++)
|
||||
for (var s = 0; s < samples; s++)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var i = 0; i < iterations; i++) action();
|
||||
|
|
@ -812,41 +813,46 @@ public static class Program
|
|||
}
|
||||
|
||||
/// <summary>
|
||||
/// Benchmarks AcBinary over a long-lived NamedPipe IPC connection — pipe is set up ONCE in the constructor;
|
||||
/// each iteration only sends a length-prefixed payload through the existing pipe. Closer to a real SignalR-style
|
||||
/// scenario where the connection is established at process start and reused for many messages, rather than the
|
||||
/// pathological one-pipe-per-message setup overhead.
|
||||
/// Benchmarks AcBinary over a long-lived NamedPipe IPC connection using the AcBinary native streaming API
|
||||
/// (<see cref="AcBinarySerializer.SerializeChunked{T}(T, System.IO.Pipelines.PipeWriter, AcBinarySerializerOptions)"/>
|
||||
/// + <see cref="AsyncPipeReaderInput"/> + <see cref="AsyncPipeReaderInputExtensions.DrainFromAsync"/>).
|
||||
/// Mirrors what a real consumer (e.g. <c>DeserializeFromPipeReaderAsync</c>) does per message:
|
||||
/// fresh input + 2 background tasks (drain + deserialize) per iteration, on top of a long-lived NamedPipe.
|
||||
///
|
||||
/// <para><b>Architecture</b>:</para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Constructor: sets up <see cref="NamedPipeServerStream"/> + <see cref="NamedPipeClientStream"/>,
|
||||
/// waits for connection, starts a long-lived background drain task on the server side that reads length-prefixed
|
||||
/// messages and pushes deserialized results into a <see cref="System.Threading.Channels.Channel{T}"/>.</item>
|
||||
/// <item>Per-iteration <see cref="Serialize"/>: encodes the payload via the Byte[] API, writes a 4-byte length
|
||||
/// prefix + payload bytes to the pipe, then awaits the channel for the server-deserialized result.</item>
|
||||
/// <item><see cref="Deserialize"/> is a no-op (the round-trip happens inside Serialize); same IsRoundTripOnly contract
|
||||
/// as the previous one-shot variant.</item>
|
||||
/// <item>Constructor (NOT timed): sets up <see cref="NamedPipeServerStream"/> + <see cref="NamedPipeClientStream"/>,
|
||||
/// waits for connection, creates one long-lived <see cref="System.IO.Pipelines.PipeWriter"/> /
|
||||
/// <see cref="System.IO.Pipelines.PipeReader"/> pair on top of the streams.</item>
|
||||
/// <item>Per-iteration <see cref="Serialize"/> (timed): sender writes one message via
|
||||
/// <see cref="AcBinarySerializer.SerializeChunked{T}(T, System.IO.Pipelines.PipeWriter, AcBinarySerializerOptions)"/>
|
||||
/// + <c>FlushAsync</c> on the long-lived pipeWriter; receiver creates a per-message
|
||||
/// <see cref="AsyncPipeReaderInput"/>, spawns a drain Task and a deserialize Task, awaits the deserialize result,
|
||||
/// then cancels the drain (which calls <c>input.Complete()</c> in its <c>finally</c>) and disposes the input.</item>
|
||||
/// <item><see cref="Deserialize"/> is a no-op (full round-trip captured in <see cref="Serialize"/>);
|
||||
/// <see cref="IsRoundTripOnly"/>=true → Ser ms / SerAlloc oszlopok N/A, RT ms = full round-trip.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para><b>What this measures</b>: per-message Byte[] serialize + length-prefix framing + pipe write/read syscall +
|
||||
/// kernel context switch + Byte[] deserialize. NOT measured: pipe lifecycle (one-time setup amortized over all iterations
|
||||
/// and across all test data cells, since this benchmark runs against many cells).</para>
|
||||
/// <para><b>Why per-message tasks</b>: the current AcBinary streaming API does not allow long-lived
|
||||
/// <see cref="AsyncPipeReaderInput"/> reuse across multiple messages on a raw transport — see
|
||||
/// <c>BINARY_ISSUES.md#accore-bin-i-q4t8</c>. This is therefore the canonical pattern, mirrored after
|
||||
/// <c>AcBinaryDeserializer.DeserializeFromPipeReaderAsync</c>'s internals. The Task.Run pair + per-iter
|
||||
/// <c>AsyncPipeReaderInput</c> allocation are an intrinsic cost of the API today, NOT a benchmark artifact.</para>
|
||||
///
|
||||
/// <para><b>Approximation note</b>: this is a single-process loopback pipe. Real cross-process or cross-machine SignalR
|
||||
/// will add transport latency (TCP, WebSocket framing) on top of these numbers. The benchmark gives a lower bound for
|
||||
/// streaming/IPC scenarios.</para>
|
||||
/// <para><b>Approximation note</b>: single-process loopback NamedPipe. Real cross-process / cross-machine SignalR
|
||||
/// adds further transport latency (TCP, WebSocket framing) on top. The benchmark gives a lower bound.</para>
|
||||
/// </summary>
|
||||
private sealed class AcBinaryNamedPipeBenchmark : ISerializerBenchmark, IDisposable
|
||||
{
|
||||
private readonly TestOrder _order;
|
||||
private readonly AcBinarySerializerOptions _options;
|
||||
private readonly byte[] _serialized; // for SerializedSize reporting
|
||||
private readonly byte[] _serialized; // for SerializedSize reporting only
|
||||
|
||||
// Long-lived pipe + drain pump (set up once in ctor)
|
||||
// Long-lived pipe lifecycle (set up once in ctor — NOT timed).
|
||||
private readonly NamedPipeServerStream _pipeServer;
|
||||
private readonly NamedPipeClientStream _pipeClient;
|
||||
private readonly Task _drainTask;
|
||||
private readonly System.Threading.Channels.Channel<TestOrder?> _resultChannel;
|
||||
private readonly PipeWriter _pipeWriter;
|
||||
private readonly PipeReader _pipeReader;
|
||||
private bool _disposed;
|
||||
|
||||
public string Engine => EngineAcBinary;
|
||||
|
|
@ -856,76 +862,34 @@ public static class Program
|
|||
public int SerializedSize => _serialized.Length;
|
||||
public long SetupAllocBytes => 0;
|
||||
public bool IsRoundTripOnly => true; // Serialize() does the full per-message round-trip; Deserialize() is a no-op
|
||||
public string OptionsDescription => $"WireMode={_options.WireMode}, RefHandling={_options.ReferenceHandling}, Interning={_options.UseStringInterning}, Metadata={_options.UseMetadata}, SGen={_options.UseGeneratedCode}, Compression={_options.UseCompression}, BufferSize={_options.BufferWriterChunkSize}B, Transport=NamedPipe(long-lived,length-prefixed)";
|
||||
public string OptionsDescription => $"WireMode={_options.WireMode}, RefHandling={_options.ReferenceHandling}, Interning={_options.UseStringInterning}, Metadata={_options.UseMetadata}, SGen={_options.UseGeneratedCode}, Compression={_options.UseCompression}, BufferSize={_options.BufferWriterChunkSize}B, Transport=NamedPipe(long-lived,SerializeChunked+AsyncPipeReaderInput)";
|
||||
|
||||
public AcBinaryNamedPipeBenchmark(TestOrder order, AcBinarySerializerOptions options, string optionsPreset)
|
||||
{
|
||||
_order = order;
|
||||
_options = options;
|
||||
// SignalR-aligned 4 KB initial buffer for the Byte[] API — matches Kestrel slab + TCP MTU,
|
||||
// simulates the realistic per-message buffer profile the SignalR transport ends up with.
|
||||
// (The 65535 default is fine for big batch encoding but over-allocates on small messages.)
|
||||
// 4 KB chunk size for the AsyncPipeWriterOutput (raw mode — no [201][UINT16][data] framing).
|
||||
// Aligns with Kestrel slab + TCP MTU, the realistic SignalR-style profile.
|
||||
_options.BufferWriterChunkSize = 4096;
|
||||
OptionsPreset = optionsPreset;
|
||||
|
||||
_serialized = AcBinarySerializer.Serialize(order, _options);
|
||||
|
||||
// 1× setup — pipe persists for the lifetime of the benchmark instance.
|
||||
// Byte mode (not Message mode) — we frame messages ourselves with a 4-byte length prefix.
|
||||
// PipeOptions.Asynchronous → enables overlapped I/O on Windows; harmless on Linux/macOS.
|
||||
// 1× setup — long-lived NamedPipe + PipeWriter/PipeReader on top of the streams.
|
||||
// Byte mode (not Message mode) — AcBinary's chunked-stream wire format manages its own boundaries
|
||||
// via the deserializer's structural knowledge of when an object graph ends.
|
||||
var pipeName = $"AcBinaryBench-{Guid.NewGuid():N}";
|
||||
|
||||
_pipeServer = new NamedPipeServerStream(pipeName, PipeDirection.In, 1, PipeTransmissionMode.Byte, System.IO.Pipes.PipeOptions.Asynchronous);
|
||||
_pipeClient = new NamedPipeClientStream(".", pipeName, PipeDirection.Out, System.IO.Pipes.PipeOptions.Asynchronous);
|
||||
|
||||
// Establish the connection. Server async-wait + client connect happen in parallel.
|
||||
var serverWait = _pipeServer.WaitForConnectionAsync();
|
||||
|
||||
_pipeClient.Connect();
|
||||
serverWait.GetAwaiter().GetResult();
|
||||
|
||||
_resultChannel = System.Threading.Channels.Channel.CreateUnbounded<TestOrder?>();
|
||||
|
||||
// Long-lived drain loop on the server side. Reads length-prefixed messages until the pipe is closed.
|
||||
_drainTask = Task.Run(async () =>
|
||||
{
|
||||
var lenBuf = new byte[4];
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// Read 4-byte length prefix (handle short reads in a loop)
|
||||
if (!await ReadExactAsync(_pipeServer, lenBuf, 0, 4).ConfigureAwait(false))
|
||||
break;
|
||||
var len = BitConverter.ToInt32(lenBuf, 0);
|
||||
if (len <= 0) break; // sentinel / corruption guard
|
||||
var data = new byte[len];
|
||||
if (!await ReadExactAsync(_pipeServer, data, 0, len).ConfigureAwait(false))
|
||||
break;
|
||||
|
||||
var result = AcBinaryDeserializer.Deserialize<TestOrder>(data, _options);
|
||||
await _resultChannel.Writer.WriteAsync(result).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is System.IO.IOException or ObjectDisposedException)
|
||||
{
|
||||
// pipe closed — normal teardown path
|
||||
}
|
||||
finally
|
||||
{
|
||||
_resultChannel.Writer.TryComplete();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Reads exactly <paramref name="count"/> bytes; returns false if pipe closed before completion.</summary>
|
||||
private static async Task<bool> ReadExactAsync(System.IO.Stream stream, byte[] buffer, int offset, int count)
|
||||
{
|
||||
var read = 0;
|
||||
while (read < count)
|
||||
{
|
||||
var n = await stream.ReadAsync(buffer.AsMemory(offset + read, count - read)).ConfigureAwait(false);
|
||||
if (n == 0) return false; // EOF
|
||||
read += n;
|
||||
}
|
||||
return true;
|
||||
_pipeWriter = PipeWriter.Create(_pipeClient);
|
||||
_pipeReader = PipeReader.Create(_pipeServer);
|
||||
}
|
||||
|
||||
public void Warmup(int iterations)
|
||||
|
|
@ -939,22 +903,23 @@ public static class Program
|
|||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
public void Serialize()
|
||||
{
|
||||
// 1) Byte[] encode (same path as the IoByteArray benchmark)
|
||||
var payload = AcBinarySerializer.Serialize(_order, _options);
|
||||
using var input = new AsyncPipeReaderInput(_options.BufferWriterChunkSize * 2, multiMessage: false);
|
||||
using var cts = new CancellationTokenSource();
|
||||
|
||||
// 2) Length-prefix framing (4 bytes little-endian) — pure benchmark-side framing, not an AcBinary feature.
|
||||
// Stack-allocated to avoid per-iter heap traffic for the prefix.
|
||||
Span<byte> lenBuf = stackalloc byte[4];
|
||||
BitConverter.TryWriteBytes(lenBuf, payload.Length);
|
||||
// Receiver tasks must be ready BEFORE the sender flushes — otherwise the FlushAsync deadlocks
|
||||
// waiting for someone to drain the kernel pipe buffer (NamedPipe loopback flow control).
|
||||
var drainTask = input.DrainFromAsync(_pipeReader, cts.Token);
|
||||
var deserTask = Task.Run(() => AcBinaryDeserializer.Deserialize<TestOrder>(input, _options), cts.Token);
|
||||
|
||||
// 3) Sync write to the pipe — Stream.Write blocks until the OS accepts the bytes into the pipe buffer.
|
||||
_pipeClient.Write(lenBuf);
|
||||
_pipeClient.Write(payload, 0, payload.Length);
|
||||
_pipeClient.Flush();
|
||||
AcBinarySerializer.SerializeChunked(_order, _pipeWriter, _options);
|
||||
_pipeWriter.FlushAsync(cts.Token).AsTask().GetAwaiter().GetResult();
|
||||
|
||||
// 4) Wait for the server drain loop to deserialize and post the result. Sync wait via channel reader.
|
||||
// A console app has no SynchronizationContext, so .GetAwaiter().GetResult() is deadlock-safe.
|
||||
_resultChannel.Reader.ReadAsync().AsTask().GetAwaiter().GetResult();
|
||||
_ = deserTask.GetAwaiter().GetResult();
|
||||
|
||||
cts.Cancel();
|
||||
try { drainTask.GetAwaiter().GetResult(); }
|
||||
catch (OperationCanceledException) { }
|
||||
catch (AggregateException ae) when (ae.InnerException is OperationCanceledException) { }
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
|
|
@ -965,14 +930,23 @@ public static class Program
|
|||
|
||||
public bool VerifyRoundTrip()
|
||||
{
|
||||
// Round-trip a single message and compare structurally.
|
||||
var payload = AcBinarySerializer.Serialize(_order, _options);
|
||||
Span<byte> lenBuf = stackalloc byte[4];
|
||||
BitConverter.TryWriteBytes(lenBuf, payload.Length);
|
||||
_pipeClient.Write(lenBuf);
|
||||
_pipeClient.Write(payload, 0, payload.Length);
|
||||
_pipeClient.Flush();
|
||||
var result = _resultChannel.Reader.ReadAsync().AsTask().GetAwaiter().GetResult();
|
||||
// Single round-trip via the same path Serialize() uses, with the deserialized graph compared.
|
||||
using var input = new AsyncPipeReaderInput(_options.BufferWriterChunkSize * 2, multiMessage: false);
|
||||
using var cts = new CancellationTokenSource();
|
||||
|
||||
var drainTask = input.DrainFromAsync(_pipeReader, cts.Token);
|
||||
var deserTask = Task.Run(() => AcBinaryDeserializer.Deserialize<TestOrder>(input, _options), cts.Token);
|
||||
|
||||
AcBinarySerializer.SerializeChunked(_order, _pipeWriter, _options);
|
||||
_pipeWriter.FlushAsync(cts.Token).AsTask().GetAwaiter().GetResult();
|
||||
|
||||
var result = deserTask.GetAwaiter().GetResult();
|
||||
cts.Cancel();
|
||||
|
||||
try { drainTask.GetAwaiter().GetResult(); }
|
||||
catch (OperationCanceledException) { }
|
||||
catch (AggregateException ae) when (ae.InnerException is OperationCanceledException) { }
|
||||
|
||||
return result != null && DeepEqualsViaJson(_order, result);
|
||||
}
|
||||
|
||||
|
|
@ -980,10 +954,12 @@ public static class Program
|
|||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
// Closing the client triggers EOF on the server's ReadAsync → drain loop exits gracefully.
|
||||
// Complete the writer side first → the underlying pipe stream signals EOF, the reader sees it,
|
||||
// any in-flight DrainFromAsync exits cleanly. Then dispose the streams.
|
||||
try { _pipeWriter.CompleteAsync().AsTask().Wait(TimeSpan.FromSeconds(2)); } catch { /* swallow on teardown */ }
|
||||
try { _pipeReader.Complete(); } catch { /* swallow on teardown */ }
|
||||
try { _pipeClient.Dispose(); } catch { /* swallow on teardown */ }
|
||||
try { _pipeServer.Dispose(); } catch { /* swallow on teardown */ }
|
||||
try { _drainTask.Wait(TimeSpan.FromSeconds(5)); } catch { /* swallow on teardown */ }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1178,7 +1154,7 @@ public static class Program
|
|||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
ReferenceHandler = System.Text.Json.Serialization.ReferenceHandler.IgnoreCycles
|
||||
};
|
||||
_serialized = System.Text.Json.JsonSerializer.Serialize(order, _options);
|
||||
_serialized = JsonSerializer.Serialize(order, _options);
|
||||
_serializedUtf8 = Utf8NoBom.GetBytes(_serialized);
|
||||
}
|
||||
|
||||
|
|
@ -1192,15 +1168,15 @@ public static class Program
|
|||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
public void Serialize() => System.Text.Json.JsonSerializer.Serialize(_order, _options);
|
||||
public void Serialize() => JsonSerializer.Serialize(_order, _options);
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
public void Deserialize() => System.Text.Json.JsonSerializer.Deserialize<TestOrder>(_serialized, _options);
|
||||
public void Deserialize() => JsonSerializer.Deserialize<TestOrder>(_serialized, _options);
|
||||
|
||||
public bool VerifyRoundTrip()
|
||||
{
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(_order, _options);
|
||||
var roundTripped = System.Text.Json.JsonSerializer.Deserialize<TestOrder>(json, _options);
|
||||
var json = JsonSerializer.Serialize(_order, _options);
|
||||
var roundTripped = JsonSerializer.Deserialize<TestOrder>(json, _options);
|
||||
return DeepEqualsViaJson(_order, roundTripped);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ namespace AyCode.Core.Tests.Serialization;
|
|||
/// plus the real parallel pipeline test (Step 3, ACCORE-BIN-T-V7C9), plus runtime type-detect
|
||||
/// sanity pinning (Step 4).
|
||||
///
|
||||
/// <para>Tests run with <see cref="AsyncPipeReaderInput"/>'s default <c>stripChunkFraming = true</c> —
|
||||
/// <para>Tests run with <see cref="AsyncPipeReaderInput"/>'s default <c>multiMessage = true</c> —
|
||||
/// <see cref="AsyncPipeReaderInput.Feed"/> expects the AsyncSegment chunked wire format
|
||||
/// <c>[201][UINT16 LE size][data]</c> per chunk, tolerates <c>[200]</c> CHUNK_START prefix, and
|
||||
/// signals end-of-stream on <c>[202]</c> CHUNK_END. The <see cref="WrapInChunkFrame"/> helper
|
||||
|
|
@ -260,21 +260,68 @@ public class AcBinarySerializerPipeParallelTests
|
|||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Feed_ChunkEndMarker_SignalsCompletion()
|
||||
public void Feed_ChunkEndMarker_AutoResetsForNextMessage()
|
||||
{
|
||||
// [202] CHUNK_END alone (without external Complete()) should signal end-of-stream.
|
||||
// [202] CHUNK_END is end-of-MESSAGE, NOT end-of-session. The input auto-resets so the same
|
||||
// long-lived instance can deserialize the next message on the same stream — see
|
||||
// BINARY_ISSUES.md#accore-bin-i-q4t8 / R5K2 fix. Session end is signalled separately by
|
||||
// an external Complete() call (or stream-EOF on the underlying transport).
|
||||
using var input = new AsyncPipeReaderInput(64);
|
||||
|
||||
// Message 1
|
||||
input.Feed(WrapInChunkFrame([1, 2, 3]));
|
||||
input.Feed([202]); // CHUNK_END marker — auto-reset, NOT completion
|
||||
|
||||
// First message is consumable
|
||||
input.Initialize(out var buf1, out var pos1, out var bufLen1);
|
||||
Assert.AreEqual(3, bufLen1);
|
||||
Assert.AreEqual(1, buf1[0]);
|
||||
Assert.AreEqual(2, buf1[1]);
|
||||
Assert.AreEqual(3, buf1[2]);
|
||||
|
||||
// Consume the bytes (simulate deserializer): reports position = 3 to producer via TryAdvanceSegment.
|
||||
// Consumer NOT yet at end-of-session, so this should NOT immediately return false — but since the
|
||||
// [202] reset _readPos to _writePos (= 3), the next AppendToBuffer for message 2 will recycle to 0.
|
||||
|
||||
// Message 2 — same long-lived input, just keeps going
|
||||
input.Feed(WrapInChunkFrame([10, 20, 30, 40]));
|
||||
input.Feed([202]);
|
||||
|
||||
// Re-initialize for the next deserializer call — the buffer was recycled to 0 by the
|
||||
// sliding-window cycling triggered when AppendToBuffer saw _readPos == _writePos > 0.
|
||||
input.Initialize(out var buf2, out var pos2, out var bufLen2);
|
||||
Assert.AreEqual(4, bufLen2);
|
||||
Assert.AreEqual(10, buf2[0]);
|
||||
Assert.AreEqual(20, buf2[1]);
|
||||
Assert.AreEqual(30, buf2[2]);
|
||||
Assert.AreEqual(40, buf2[3]);
|
||||
|
||||
// Now signal end-of-session explicitly
|
||||
input.Complete();
|
||||
|
||||
// After Complete, TryAdvanceSegment returns false on empty — session truly ended
|
||||
var pos3 = bufLen2;
|
||||
var bufLen3 = bufLen2;
|
||||
var buf3 = buf2;
|
||||
var hasMore = input.TryAdvanceSegment(ref buf3, ref pos3, ref bufLen3, 1);
|
||||
Assert.IsFalse(hasMore);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Feed_ExternalComplete_SignalsEndOfSession()
|
||||
{
|
||||
// Explicit Complete() (or stream-EOF in the DrainFromAsync path) is the session-end signal —
|
||||
// distinct from per-message [202] markers which only auto-reset for the next message.
|
||||
using var input = new AsyncPipeReaderInput(64);
|
||||
|
||||
input.Feed(WrapInChunkFrame([1, 2, 3]));
|
||||
input.Feed([202]); // CHUNK_END marker only — no external Complete()
|
||||
input.Complete(); // external session-end
|
||||
|
||||
// Should observe completion: TryAdvanceSegment returns false on empty after consume
|
||||
input.Initialize(out var buffer, out var position, out var bufferLength);
|
||||
Assert.AreEqual(3, bufferLength);
|
||||
|
||||
position = bufferLength;
|
||||
var hasMore = input.TryAdvanceSegment(ref buffer, ref position, ref bufferLength, 1);
|
||||
|
||||
Assert.IsFalse(hasMore);
|
||||
}
|
||||
|
||||
|
|
@ -431,7 +478,7 @@ public class AcBinarySerializerPipeParallelTests
|
|||
{
|
||||
// SerializeChunkedFramed — writes [201][UINT16][data] per chunk on the wire.
|
||||
// AsyncPipeReaderInput.Feed strips framing internally on the receive side
|
||||
// (default stripChunkFraming = true).
|
||||
// (default multiMessage = true).
|
||||
AcBinarySerializer.SerializeChunkedFramed(original, pipe.Writer, opts);
|
||||
}
|
||||
finally
|
||||
|
|
|
|||
|
|
@ -350,11 +350,11 @@ public static partial class AcBinaryDeserializer
|
|||
|
||||
var opts = options ?? AcBinarySerializerOptions.Default;
|
||||
|
||||
// Raw mode (stripChunkFraming: false) — bytes drained from the PipeReader are forwarded
|
||||
// Single-message mode (multiMessage: false) — bytes drained from the PipeReader are forwarded
|
||||
// verbatim to the deserialization buffer. Pair with AcBinarySerializer.SerializeChunked
|
||||
// (raw byte stream) on the producer side; for chunked-framed wire formats the parser
|
||||
// (raw byte stream) on the producer side; for multi-message framed wire formats the parser
|
||||
// strips framing upstream and feeds only data bytes here.
|
||||
using var input = new AsyncPipeReaderInput(initialCapacity: opts.BufferWriterChunkSize * 2, stripChunkFraming: false);
|
||||
using var input = new AsyncPipeReaderInput(initialCapacity: opts.BufferWriterChunkSize * 2, multiMessage: false);
|
||||
var deserTask = Task.Run(() => Deserialize<T>(input, opts), ct);
|
||||
|
||||
await input.DrainFromAsync(reader, ct).ConfigureAwait(false);
|
||||
|
|
|
|||
|
|
@ -452,7 +452,7 @@ public static partial class AcBinarySerializer
|
|||
public static int SerializeChunked<T>(T value, System.IO.Pipelines.Pipe pipe, AcBinarySerializerOptions options, bool waitForFlush = true, TimeSpan? flushTimeout = null)
|
||||
{
|
||||
if (pipe is null) throw new ArgumentNullException(nameof(pipe));
|
||||
return SerializeToPipeWriterCore(value, pipe.Writer, options, waitForFlush, flushTimeout, emitChunkFraming: false);
|
||||
return SerializeToPipeWriterCore(value, pipe.Writer, options, waitForFlush, flushTimeout, multiMessage: false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -479,7 +479,7 @@ public static partial class AcBinarySerializer
|
|||
/// <param name="options">Serializer options (type wrappers, reference handling, interning, etc.).</param>
|
||||
/// <returns>Total serialized bytes written.</returns>
|
||||
public static int SerializeChunked<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options)
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush: true, flushTimeout: null, emitChunkFraming: false);
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush: true, flushTimeout: null, multiMessage: false);
|
||||
|
||||
/// <summary>
|
||||
/// Serialize a value into a chunked stream where each chunk carries a self-describing
|
||||
|
|
@ -511,7 +511,7 @@ public static partial class AcBinarySerializer
|
|||
public static int SerializeChunkedFramed<T>(T value, System.IO.Pipelines.Pipe pipe, AcBinarySerializerOptions options, bool waitForFlush = true, TimeSpan? flushTimeout = null)
|
||||
{
|
||||
if (pipe is null) throw new ArgumentNullException(nameof(pipe));
|
||||
return SerializeToPipeWriterCore(value, pipe.Writer, options, waitForFlush, flushTimeout, emitChunkFraming: true);
|
||||
return SerializeToPipeWriterCore(value, pipe.Writer, options, waitForFlush, flushTimeout, multiMessage: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -524,7 +524,7 @@ public static partial class AcBinarySerializer
|
|||
/// <see cref="SerializeChunked{T}(T, System.IO.Pipelines.PipeWriter, AcBinarySerializerOptions)"/>.</para>
|
||||
/// </summary>
|
||||
public static int SerializeChunkedFramed<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options)
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush: true, flushTimeout: null, emitChunkFraming: true);
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush: true, flushTimeout: null, multiMessage: true);
|
||||
|
||||
/// <summary>
|
||||
/// Internal flush-tunable framed PipeWriter overload — used by <c>AyCode.Services</c>
|
||||
|
|
@ -533,7 +533,7 @@ public static partial class AcBinarySerializer
|
|||
/// <paramref name="waitForFlush"/> on a guaranteed parallel-capable writer.
|
||||
/// </summary>
|
||||
internal static int SerializeChunkedFramed<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options, bool waitForFlush, TimeSpan? flushTimeout)
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush, flushTimeout, emitChunkFraming: true);
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush, flushTimeout, multiMessage: true);
|
||||
|
||||
/// <summary>
|
||||
/// Internal legacy alias for <see cref="SerializeChunkedFramed{T}(T, System.IO.Pipelines.PipeWriter, AcBinarySerializerOptions, bool, TimeSpan?)"/>
|
||||
|
|
@ -542,14 +542,14 @@ public static partial class AcBinarySerializer
|
|||
/// (framed wire format with <c>[201][UINT16][data]</c> per chunk + <c>[202]</c> end marker).
|
||||
/// </summary>
|
||||
internal static int Serialize<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options, bool waitForFlush, TimeSpan? flushTimeout)
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush, flushTimeout, emitChunkFraming: true);
|
||||
=> SerializeToPipeWriterCore(value, pipeWriter, options, waitForFlush, flushTimeout, multiMessage: true);
|
||||
|
||||
/// <summary>
|
||||
/// Common pipe-output serialization core. Same loop for both raw (<see cref="SerializeChunked{T}"/>)
|
||||
/// and framed (<see cref="SerializeChunkedFramed{T}"/>) modes — the only difference flows through
|
||||
/// <paramref name="emitChunkFraming"/> into the <see cref="AsyncPipeWriterOutput"/> ctor.
|
||||
/// <paramref name="multiMessage"/> into the <see cref="AsyncPipeWriterOutput"/> ctor.
|
||||
/// </summary>
|
||||
private static int SerializeToPipeWriterCore<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options, bool waitForFlush, TimeSpan? flushTimeout, bool emitChunkFraming)
|
||||
private static int SerializeToPipeWriterCore<T>(T value, System.IO.Pipelines.PipeWriter pipeWriter, AcBinarySerializerOptions options, bool waitForFlush, TimeSpan? flushTimeout, bool multiMessage)
|
||||
{
|
||||
if (value == null)
|
||||
{
|
||||
|
|
@ -563,7 +563,7 @@ public static partial class AcBinarySerializer
|
|||
var runtimeType = value.GetType();
|
||||
var context = BinarySerializationContextPool<AsyncPipeWriterOutput>.Get(options);
|
||||
|
||||
context.Output = new AsyncPipeWriterOutput(pipeWriter, options.BufferWriterChunkSize, emitChunkFraming, waitForFlush, flushTimeout);
|
||||
context.Output = new AsyncPipeWriterOutput(pipeWriter, options.BufferWriterChunkSize, multiMessage, waitForFlush, flushTimeout);
|
||||
context.Output.Initialize(out context._buffer, out context._position, out context._bufferEnd);
|
||||
|
||||
try
|
||||
|
|
|
|||
|
|
@ -15,11 +15,14 @@ namespace AyCode.Core.Serializers.Binaries;
|
|||
/// .NET BCL convention for type-level <c>Async</c> prefix (<c>AsyncEnumerable</c>,
|
||||
/// <c>IAsyncDisposable</c>, <c>AsyncLocal<T></c>, ...).
|
||||
///
|
||||
/// <para><see cref="Feed"/> behavior is driven by the <c>stripChunkFraming</c> ctor flag:
|
||||
/// <para><see cref="Feed"/> behavior is driven by the <c>multiMessage</c> ctor flag:
|
||||
/// <c>true</c> (default) — parses <c>[201][UINT16][data]</c> chunked frames + <c>[202]</c> end
|
||||
/// marker (matches <c>AsyncPipeWriterOutput</c> framed output and SignalR's AsyncSegment wire
|
||||
/// format); <c>false</c> — appends bytes verbatim (matches <c>AcBinarySerializer.SerializeChunked</c>
|
||||
/// raw output drained from a <see cref="System.IO.Pipelines.PipeReader"/>).</para>
|
||||
/// format); on every <c>[202]</c> the input <b>auto-resets</b> for the next message — multiple
|
||||
/// <see cref="AcBinaryDeserializer.Deserialize{T}(AsyncPipeReaderInput, AcBinarySerializerOptions)"/>
|
||||
/// calls can reuse the same long-lived input over a single transport. <c>false</c> — appends bytes
|
||||
/// verbatim (matches <c>AcBinarySerializer.SerializeChunked</c> raw output drained from a
|
||||
/// <see cref="System.IO.Pipelines.PipeReader"/>); single-message scenario, no auto-reset.</para>
|
||||
///
|
||||
/// <para>Usage modes:</para>
|
||||
/// <list type="bullet">
|
||||
|
|
@ -60,18 +63,22 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
private int _readPos; // consumer reports consumed position here
|
||||
private bool _completed;
|
||||
|
||||
// Whether Feed() should strip [201][UINT16][data] chunked framing (true, default — matches
|
||||
// SignalR-style multiplexed wire) or append bytes verbatim (false — matches the raw output
|
||||
// of SerializeChunked / single-shot byte[] over PipeWriter).
|
||||
private readonly bool _stripChunkFraming;
|
||||
// multi-message wire framing flag:
|
||||
// true (default): Feed() parses [201][UINT16][data] chunked framing + [202] CHUNK_END markers,
|
||||
// auto-resets the buffer cursor on every [202] for the next message.
|
||||
// Matches AsyncPipeWriterOutput multi-message wire and SignalR AsyncSegment.
|
||||
// false: Feed() appends bytes verbatim (no wire-format interpretation, single message
|
||||
// scenario). Matches AcBinarySerializer.SerializeChunked raw output drained
|
||||
// from a PipeReader.
|
||||
private readonly bool _multiMessage;
|
||||
|
||||
// Framing state machine — parses [201][UINT16 LE size][data] frames + [202] CHUNK_END.
|
||||
// [200] CHUNK_START tolerated (skipped). Wire format matches AsyncPipeWriterOutput's framed
|
||||
// output and SignalR's AsyncSegment chunked frame format. Only active when
|
||||
// _stripChunkFraming = true.
|
||||
// [200] CHUNK_START tolerated (skipped). Wire format matches AsyncPipeWriterOutput's
|
||||
// multi-message output and SignalR's AsyncSegment chunked frame format. Only active when
|
||||
// _multiMessage = true.
|
||||
private const byte ChunkStart = 200; // CHUNK_START — tolerated, skipped
|
||||
private const byte ChunkData = 201; // CHUNK_DATA — header followed by [UINT16 size][data]
|
||||
private const byte ChunkEnd = 202; // CHUNK_END — signals end-of-stream
|
||||
private const byte ChunkEnd = 202; // CHUNK_END — signals end-of-MESSAGE (auto-reset for next message)
|
||||
|
||||
private FramingState _framingState = FramingState.AwaitingHeader;
|
||||
|
||||
|
|
@ -84,7 +91,8 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
AwaitingSizeLow, // have [201], expect UINT16 LE low byte
|
||||
AwaitingSizeHigh, // have low, expect UINT16 LE high byte
|
||||
AwaitingData, // expect _bytesRemainingInChunk data bytes
|
||||
Done // saw [202], ignore further bytes
|
||||
// No "Done" state — [202] auto-resets to AwaitingHeader for next-message reuse.
|
||||
// Session-end is signalled by external Complete() / stream-EOF, NOT by framing-state.
|
||||
}
|
||||
|
||||
private readonly ManualResetEventSlim _dataAvailable;
|
||||
|
|
@ -116,20 +124,26 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
/// 4 KB chunk size, 128 KB for the standalone 64 KB default).
|
||||
/// </summary>
|
||||
/// <param name="initialCapacity">Initial buffer size. Rounded up by ArrayPool.</param>
|
||||
/// <param name="stripChunkFraming">
|
||||
/// <c>true</c> (default): <see cref="Feed"/> parses <c>[201][UINT16][data]</c> chunked frames +
|
||||
/// <c>[202]</c> end marker (matches <see cref="AsyncPipeWriterOutput"/> framed output and
|
||||
/// SignalR's AsyncSegment chunked wire format).
|
||||
/// <c>false</c>: <see cref="Feed"/> appends bytes verbatim — for raw byte streams (matches
|
||||
/// <c>AcBinarySerializer.SerializeChunked</c> output and the single-shot
|
||||
/// <c>byte[]</c> output).
|
||||
/// <param name="multiMessage">
|
||||
/// <c>true</c> (default): <see cref="Feed"/> parses the multi-message wire framing
|
||||
/// (<c>[201][UINT16][data]</c> chunks + <c>[202]</c> end-of-MESSAGE marker — matches
|
||||
/// <see cref="AsyncPipeWriterOutput"/> multi-message output and SignalR's AsyncSegment).
|
||||
/// On every <c>[202]</c> the input auto-resets the buffer cursor for the next message —
|
||||
/// the same long-lived input can be reused across many
|
||||
/// <see cref="AcBinaryDeserializer.Deserialize{T}(AsyncPipeReaderInput, AcBinarySerializerOptions)"/>
|
||||
/// calls without allocating a fresh instance per message. End of session is signalled by an
|
||||
/// external <see cref="Complete"/> call or stream-EOF, NOT by <c>[202]</c>.
|
||||
///
|
||||
/// <c>false</c>: <see cref="Feed"/> appends bytes verbatim — single-message scenario where the
|
||||
/// stream lifecycle equals the message lifecycle (matches <c>AcBinarySerializer.SerializeChunked</c>
|
||||
/// raw output, paired with <c>pipeWriter.CompleteAsync()</c> as the end-of-message signal).
|
||||
/// </param>
|
||||
public AsyncPipeReaderInput(int initialCapacity, bool stripChunkFraming = true)
|
||||
public AsyncPipeReaderInput(int initialCapacity, bool multiMessage = true)
|
||||
{
|
||||
if (initialCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(initialCapacity));
|
||||
|
||||
_buffer = ArrayPool<byte>.Shared.Rent(initialCapacity);
|
||||
_stripChunkFraming = stripChunkFraming;
|
||||
_multiMessage = multiMessage;
|
||||
_dataAvailable = new ManualResetEventSlim(false);
|
||||
}
|
||||
|
||||
|
|
@ -137,42 +151,42 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
|
||||
/// <summary>
|
||||
/// Feeds bytes into the consumer-visible buffer. Behavior is driven by the
|
||||
/// <c>stripChunkFraming</c> ctor flag:
|
||||
/// <c>multiMessage</c> ctor flag:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>stripChunkFraming = true</b> (default): expects the chunked wire format
|
||||
/// <c>[201][UINT16 LE size][data]</c> per chunk, tolerates <c>[200]</c>
|
||||
/// CHUNK_START prefix, and signals end-of-stream on <c>[202]</c> CHUNK_END. State
|
||||
/// is preserved across <c>Feed</c> calls — partial frame headers, mid-size boundaries,
|
||||
/// and mid-data boundaries all resume correctly. On <c>[202]</c>, sets the completion
|
||||
/// flag and signals waiting consumers — equivalent to an external <see cref="Complete"/>
|
||||
/// call. Bytes after <c>[202]</c> are ignored.</item>
|
||||
/// <item><b>stripChunkFraming = false</b>: appends bytes verbatim — no wire-format
|
||||
/// interpretation. The producer must pass only payload bytes (e.g. raw byte stream
|
||||
/// drained from a <see cref="System.IO.Pipelines.PipeReader"/> paired with
|
||||
/// <c>AcBinarySerializer.SerializeChunked</c>).</item>
|
||||
/// <item><b>multiMessage = true</b> (default): expects the multi-message wire format
|
||||
/// <c>[201][UINT16 LE size][data]</c> per chunk, tolerates <c>[200]</c> CHUNK_START
|
||||
/// prefix, treats <c>[202]</c> CHUNK_END as <b>end-of-MESSAGE</b>. State is preserved
|
||||
/// across <c>Feed</c> calls — partial frame headers, mid-size boundaries, and mid-data
|
||||
/// boundaries all resume correctly. On <c>[202]</c>, the input <b>auto-resets</b> the
|
||||
/// buffer cursor for the next message (signals the producer's sliding-window cycling
|
||||
/// to recycle the buffer on next <see cref="AppendToBuffer"/>) and resets the framing
|
||||
/// state machine to <c>AwaitingHeader</c> — the next bytes are expected to be a new
|
||||
/// <c>[201]...</c> frame. End-of-session is NOT signalled by <c>[202]</c>; only an
|
||||
/// external <see cref="Complete"/> call or stream-EOF marks the session as ended.</item>
|
||||
/// <item><b>multiMessage = false</b>: appends bytes verbatim — no wire-format interpretation.
|
||||
/// The producer passes only payload bytes (e.g. raw byte stream drained from a
|
||||
/// <see cref="System.IO.Pipelines.PipeReader"/> paired with
|
||||
/// <c>AcBinarySerializer.SerializeChunked</c>). Single-message scenario; end-of-message
|
||||
/// is the same as end-of-stream, signalled by external <see cref="Complete"/> call.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public void Feed(ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.IsEmpty) return;
|
||||
|
||||
if (!_stripChunkFraming)
|
||||
if (!_multiMessage)
|
||||
{
|
||||
// Raw mode: append verbatim, no framing interpretation.
|
||||
// Single-message mode: append verbatim, no framing interpretation.
|
||||
AppendToBuffer(data);
|
||||
return;
|
||||
}
|
||||
|
||||
// Framed mode: state machine parses [201][UINT16 LE size][data] frames + [202] end marker.
|
||||
// Multi-message mode: state machine parses [201][UINT16 LE size][data] frames + [202] end-of-message marker.
|
||||
var i = 0;
|
||||
while (i < data.Length)
|
||||
{
|
||||
switch (_framingState)
|
||||
{
|
||||
case FramingState.Done:
|
||||
EmitDiagnostic($"Feed: bytes after CHUNK_END ignored, count={data.Length - i}");
|
||||
return;
|
||||
|
||||
case FramingState.AwaitingHeader:
|
||||
{
|
||||
var marker = data[i++];
|
||||
|
|
@ -187,11 +201,18 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
}
|
||||
else if (marker == ChunkEnd)
|
||||
{
|
||||
EmitDiagnostic("Feed: CHUNK_END [202] received, signaling completion");
|
||||
_framingState = FramingState.Done;
|
||||
Volatile.Write(ref _completed, true);
|
||||
_dataAvailable.Set();
|
||||
return;
|
||||
// [202] = end of CURRENT message (NOT end of session). Two-step signal:
|
||||
// (a) reset framing state machine to AwaitingHeader for the next [201] header,
|
||||
// (b) write _readPos = -1 sentinel — picked up by the next AppendToBuffer's
|
||||
// sliding-window cycling, which resets the buffer to 0 for the new message.
|
||||
// _completed stays false — only external Complete() / stream-EOF marks session end.
|
||||
// The sentinel is wire-format intrinsic: TryAdvanceSegment / Initialize handle
|
||||
// _readPos < 0 defensively (treat as "fully consumed"), so the consumer never
|
||||
// observes the sentinel directly — by the time the consumer reaches the next
|
||||
// Initialize call, AppendToBuffer has already cycled _readPos back to 0.
|
||||
EmitDiagnostic("Feed: CHUNK_END [202] received — framing reset, _readPos sentinel armed");
|
||||
_framingState = FramingState.AwaitingHeader;
|
||||
Volatile.Write(ref _readPos, -1);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -241,13 +262,19 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
|
||||
/// <summary>
|
||||
/// Appends data bytes to the internal buffer with sliding-window cycling
|
||||
/// (reset to 0 when consumer has caught up) and grow-as-last-resort. Signals the consumer.
|
||||
/// (reset to 0 when consumer has caught up OR a [202] message-end sentinel was raised) and
|
||||
/// grow-as-last-resort. Signals the consumer.
|
||||
/// </summary>
|
||||
private void AppendToBuffer(ReadOnlySpan<byte> data)
|
||||
{
|
||||
// If consumer consumed everything → reset positions to 0 (sliding-window cycling)
|
||||
// Cycle the buffer to 0 if either:
|
||||
// (a) consumer has caught up to _writePos (classic sliding-window pattern), OR
|
||||
// (b) a [202] CHUNK_END marker was just parsed and armed _readPos = -1 (sentinel) —
|
||||
// the message is complete on the wire, the consumer (per wire-format guarantee)
|
||||
// has read or will read exactly _writePos bytes; the next bytes are the start of
|
||||
// a new message and belong at offset 0.
|
||||
var rp = Volatile.Read(ref _readPos);
|
||||
if (rp > 0 && rp == _writePos)
|
||||
if (rp < 0 || (rp > 0 && rp == _writePos))
|
||||
{
|
||||
EmitDiagnostic($"AppendToBuffer reset positions rp={rp} wp={_writePos} → 0");
|
||||
_writePos = 0;
|
||||
|
|
@ -314,8 +341,14 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
{
|
||||
EmitDiagnostic($"TryAdvanceSegment enter position={position} bufferLength={bufferLength} needed={needed}");
|
||||
|
||||
// Report how far we've consumed — enables producer to reset positions to 0
|
||||
Volatile.Write(ref _readPos, position);
|
||||
// Report how far we've consumed — enables producer to reset positions to 0.
|
||||
// Sentinel respect: if _readPos < 0 (a [202] CHUNK_END marker armed it), DO NOT overwrite
|
||||
// the sentinel — the next AppendToBuffer needs to see it to cycle the buffer to 0.
|
||||
// The local sentinel-defence below ensures correct logic during the transient race window.
|
||||
if (Volatile.Read(ref _readPos) >= 0)
|
||||
{
|
||||
Volatile.Write(ref _readPos, position);
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
|
|
@ -323,6 +356,11 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
int rp = Volatile.Read(ref _readPos);
|
||||
int wp = Volatile.Read(ref _writePos);
|
||||
|
||||
// Sentinel defence: if [202] armed _readPos = -1 while we were reading, treat the
|
||||
// sentinel as "use our local position" — the cycle hasn't fired yet (no AppendToBuffer
|
||||
// has run since [202]); we still consume from our own position into the existing buffer.
|
||||
if (rp < 0) rp = position;
|
||||
|
||||
if (wp - rp >= needed)
|
||||
{
|
||||
buffer = _buffer; // may be new array after grow
|
||||
|
|
@ -354,6 +392,7 @@ public sealed class AsyncPipeReaderInput : IBinaryInputBase, IDisposable
|
|||
_dataAvailable.Reset();
|
||||
|
||||
rp = Volatile.Read(ref _readPos);
|
||||
if (rp < 0) rp = position; // sentinel defence (same as the top of the loop)
|
||||
wp = Volatile.Read(ref _writePos);
|
||||
|
||||
if (wp - rp >= needed || Volatile.Read(ref _completed)) continue;
|
||||
|
|
|
|||
|
|
@ -9,23 +9,25 @@ namespace AyCode.Core.Serializers.Binaries;
|
|||
|
||||
/// <summary>
|
||||
/// Binary output that writes to a PipeWriter chunk-by-chunk via the PipeWriter's natural slabbing.
|
||||
/// Two wire-format modes are supported, selected by the <c>emitChunkFraming</c> ctor flag:
|
||||
/// Two wire-format modes are supported, selected by the <c>multiMessage</c> ctor flag:
|
||||
///
|
||||
/// <list type="bullet">
|
||||
/// <item><b><c>emitChunkFraming = false</c></b> (raw): pure AcBinary bytes are written into
|
||||
/// <item><b><c>multiMessage = false</c></b> (single-message): pure AcBinary bytes are written into
|
||||
/// the PipeWriter's slabs and committed via Advance — no per-chunk header bytes appear on the
|
||||
/// wire. Bit-compatible with the single-shot <c>Serialize(value, opts) → byte[]</c> output.
|
||||
/// The receiver can deserialize the byte stream as-is (e.g. via
|
||||
/// <c>AcBinaryDeserializer.Deserialize(byte[])</c> after collecting, or any raw
|
||||
/// <c>PipeReader</c>-based path).</item>
|
||||
/// <c>PipeReader</c>-based path). End-of-message = end-of-stream; caller signals it by closing
|
||||
/// the writer (<c>pipeWriter.CompleteAsync()</c>).</item>
|
||||
///
|
||||
/// <item><b><c>emitChunkFraming = true</c></b> (framed): each chunk gets a 3-byte header
|
||||
/// <item><b><c>multiMessage = true</c></b> (default): each chunk gets a 3-byte header
|
||||
/// <c>[201][UINT16 size][data]</c>. The header is reserved at the start of each acquired slab;
|
||||
/// the serializer writes data after it, and on commit the size is patched and the full chunk
|
||||
/// is Advanced (zero-copy). The protocol layer writes a single <c>[202]</c> byte after all
|
||||
/// chunks to signal end-of-stream. This is the multiplexed wire format used by SignalR's
|
||||
/// <c>BinaryProtocolMode.AsyncSegment</c> and any custom multiplexed protocol where the
|
||||
/// receiver needs incremental chunk-boundary detection.</item>
|
||||
/// is Advanced (zero-copy). At end-of-serialize, <see cref="Flush"/> writes a <c>[202]</c>
|
||||
/// CHUNK_END marker (symmetrical with <c>[201]</c>) — this signals end-of-MESSAGE to the
|
||||
/// receiver, which auto-resets its <see cref="AsyncPipeReaderInput"/> for the next message
|
||||
/// on the same long-lived stream. Used by SignalR's <c>BinaryProtocolMode.AsyncSegment</c>
|
||||
/// and any custom multi-message protocol over a long-lived transport.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para><b>Backpressure modes</b> (controlled by <c>waitForFlush</c>) — independent of framing:</para>
|
||||
|
|
@ -58,6 +60,9 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
/// <summary>MsgAsyncChunkData type marker (201).</summary>
|
||||
private const byte ChunkDataMarker = 201;
|
||||
|
||||
/// <summary>MsgAsyncChunkEnd marker (202) — written at end-of-serialize in framed mode.</summary>
|
||||
private const byte ChunkEndMarker = 202;
|
||||
|
||||
/// <summary>Header size: 1 byte type + 2 bytes UINT16 size.</summary>
|
||||
private const int HeaderSize = 3;
|
||||
|
||||
|
|
@ -79,7 +84,7 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
|
||||
private readonly PipeWriter _pipeWriter;
|
||||
private readonly int _chunkSize;
|
||||
private readonly bool _emitChunkFraming;
|
||||
private readonly bool _multiMessage;
|
||||
private readonly bool _waitForFlush;
|
||||
private readonly bool _serializeFlushAndAcquire;
|
||||
private readonly TimeSpan _flushTimeout;
|
||||
|
|
@ -106,22 +111,24 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
[Conditional("DEBUG")]
|
||||
private static void EmitDiagnostic(string message) => DiagnosticLog?.Invoke(message);
|
||||
|
||||
/// <summary>Creates an output bound to the given PipeWriter — framed or raw mode per <paramref name="emitChunkFraming"/>.</summary>
|
||||
/// <summary>Creates an output bound to the given PipeWriter — multi-message or single-message mode per <paramref name="multiMessage"/>.</summary>
|
||||
/// <param name="pipeWriter">Target pipe (typically Kestrel's transport output for SignalR, NamedPipe, FileStream, or any custom <see cref="PipeWriter"/>).</param>
|
||||
/// <param name="chunkSize">Per-chunk data size (max <see cref="MaxChunkSize"/>). Default 4 KB matches Kestrel's slab size.</param>
|
||||
/// <param name="emitChunkFraming"><c>true</c> → write <c>[201][UINT16][data]</c> per-chunk header (multiplexed wire format).
|
||||
/// <c>false</c> → raw AcBinary bytes only, byte-compatible with the single-shot <c>byte[]</c> output. See class summary.</param>
|
||||
/// <param name="multiMessage"><c>true</c> (default) → multi-message wire format: <c>[201][UINT16][data]</c> per-chunk header
|
||||
/// + <c>[202]</c> end-of-message marker on Flush. Receiver auto-resets between messages.
|
||||
/// <c>false</c> → single-message: raw AcBinary bytes only, byte-compatible with the single-shot <c>byte[]</c> output;
|
||||
/// caller signals end-of-message by closing the writer. See class summary.</param>
|
||||
/// <param name="waitForFlush">See class summary — pipeline parallelism (true) vs adaptive (false).</param>
|
||||
/// <param name="flushTimeout">Per-flush timeout. <c>null</c> → <see cref="System.Threading.Timeout.InfiniteTimeSpan"/>
|
||||
/// (wait forever — legacy behavior). Pass a positive value to fail fast on stuck consumers.</param>
|
||||
public AsyncPipeWriterOutput(PipeWriter pipeWriter, int chunkSize = 4096, bool emitChunkFraming = true, bool waitForFlush = true, TimeSpan? flushTimeout = null)
|
||||
public AsyncPipeWriterOutput(PipeWriter pipeWriter, int chunkSize = 4096, bool multiMessage = true, bool waitForFlush = true, TimeSpan? flushTimeout = null)
|
||||
{
|
||||
if (chunkSize > MaxChunkSize)
|
||||
throw new ArgumentOutOfRangeException(nameof(chunkSize), chunkSize, $"Chunk size cannot exceed {MaxChunkSize} (UINT16 max).");
|
||||
|
||||
_pipeWriter = pipeWriter;
|
||||
_chunkSize = chunkSize;
|
||||
_emitChunkFraming = emitChunkFraming;
|
||||
_multiMessage = multiMessage;
|
||||
_waitForFlush = waitForFlush;
|
||||
|
||||
// null → Timeout.InfiniteTimeSpan ("wait forever" — natively supported by Task.Wait as -1ms).
|
||||
|
|
@ -230,10 +237,25 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
public int GetTotalPosition(int currentPosition) => _committedBytes + (currentPosition - _currentChunkStart);
|
||||
|
||||
/// <summary>
|
||||
/// Commits the last (partial) chunk to the PipeWriter — in framed mode patches the
|
||||
/// <c>[201][UINT16 size]</c> header before Advance, in raw mode simply Advances the data.
|
||||
/// Zero-copy: no data copying. Does NOT flush to network — in framed mode the protocol writes
|
||||
/// <c>[202]</c> and flushes after.
|
||||
/// Commits the last (partial) chunk to the PipeWriter, writes the <c>[202]</c> CHUNK_END marker
|
||||
/// in framed mode, and flushes everything to the underlying writer. End-of-serialize is fully
|
||||
/// owned by this output: when <see cref="Flush"/> returns, every byte is downstream — the
|
||||
/// caller does NOT need a follow-up <c>pipeWriter.FlushAsync()</c>.
|
||||
///
|
||||
/// <para><b>Behaviour by mode</b>:</para>
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Raw mode</b> (<c>emitChunkFraming = false</c>): final chunk <c>Advance</c> →
|
||||
/// <c>FlushAsync</c>. No end marker (raw mode has no framing concept).</item>
|
||||
/// <item><b>Framed mode</b> (<c>emitChunkFraming = true</c>): final chunk <c>Advance</c> →
|
||||
/// <c>[202]</c> CHUNK_END marker (symmetric with the <c>[201]</c> header that
|
||||
/// <see cref="CommitCurrentChunk"/> writes per chunk) → <c>FlushAsync</c>. The end
|
||||
/// marker is written here so the wire-format contract is fully owned by this output;
|
||||
/// protocol layers above (e.g. <c>AcBinaryHubProtocol</c>) no longer need to inject
|
||||
/// their own <c>[202]</c> + flush.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>Zero-copy: no data copying in either mode. The pre-flush wait covers any in-flight
|
||||
/// fire-and-forget flush from <see cref="Grow"/> on the Pipe-based parallel path.</para>
|
||||
/// </summary>
|
||||
public void Flush(byte[] buffer, int position)
|
||||
{
|
||||
|
|
@ -242,6 +264,21 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
|
||||
CommitCurrentChunk(buffer, position);
|
||||
|
||||
// Framed mode: write the [202] CHUNK_END marker — symmetric with [201] header that
|
||||
// CommitCurrentChunk writes per chunk. Both ends of the framing contract are owned here.
|
||||
if (_multiMessage)
|
||||
{
|
||||
var span = _pipeWriter.GetSpan(1);
|
||||
span[0] = ChunkEndMarker;
|
||||
_pipeWriter.Advance(1);
|
||||
|
||||
EmitDiagnostic("Flush[framed]: wrote [202] CHUNK_END");
|
||||
}
|
||||
|
||||
// Always flush — final chunk (and end marker, if framed) → downstream. Caller does not
|
||||
// need a follow-up FlushAsync.
|
||||
SyncAwaitFlush(_pipeWriter.FlushAsync());
|
||||
|
||||
// End of serialize lifecycle — return the owned fallback buffer to ArrayPool exactly
|
||||
// once (NOT per chunk). The buffer was reused across all chunks in this lifecycle;
|
||||
// releasing it now avoids per-chunk rent/return churn even when the fallback path
|
||||
|
|
@ -285,7 +322,7 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
var dataBytes = position - _currentChunkStart;
|
||||
if (dataBytes <= 0) return;
|
||||
|
||||
if (_emitChunkFraming)
|
||||
if (_multiMessage)
|
||||
{
|
||||
var headerStart = _currentChunkStart - HeaderSize;
|
||||
buffer[headerStart] = ChunkDataMarker;
|
||||
|
|
@ -329,11 +366,11 @@ public struct AsyncPipeWriterOutput : IBinaryOutputBase
|
|||
|
||||
// Header reservation only in framed mode — raw mode skips it for byte-compat with the
|
||||
// single-shot byte[] output (each chunk holds pure AcBinary bytes, no markers).
|
||||
var headerOffset = _emitChunkFraming ? HeaderSize : 0;
|
||||
var headerOffset = _multiMessage ? HeaderSize : 0;
|
||||
var totalRequest = dataSize + headerOffset;
|
||||
var memory = _pipeWriter.GetMemory(totalRequest);
|
||||
|
||||
EmitDiagnostic($"AcquireChunk: framed={_emitChunkFraming} requestSize={requestSize} dataSize={dataSize} totalRequest={totalRequest} memory.Length={memory.Length} _committedBytes={_committedBytes}");
|
||||
EmitDiagnostic($"AcquireChunk: framed={_multiMessage} requestSize={requestSize} dataSize={dataSize} totalRequest={totalRequest} memory.Length={memory.Length} _committedBytes={_committedBytes}");
|
||||
|
||||
if (MemoryMarshal.TryGetArray(memory, out ArraySegment<byte> segment) && segment.Array != null)
|
||||
{
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -196,3 +196,36 @@ In `AsyncSegment` mode the total message size is unknown until `CHUNK_END`, so a
|
|||
### Acceptance criteria
|
||||
- API sketch.
|
||||
- Stack-with-encryption order decision documented (industry standard: **encrypt-then-MAC**, but evaluate trade-offs).
|
||||
|
||||
## ACCORE-SBP-T-S2H6: Replace `Type.GetType(wire-supplied-name)` with whitelist-based type resolution
|
||||
**Priority:** P0 — security blocker · **Type:** Security fix · **Related Issue:** [`SIGNALR_BINARY_PROTOCOL_ISSUES.md#accore-sbp-i-r8k3`](SIGNALR_BINARY_PROTOCOL_ISSUES.md#accore-sbp-i-r8k3-typegettypewire-supplied-name-enables-deserialization-gadget-attack) (full threat model, attack surface, and mitigation analysis there — **do not duplicate here**)
|
||||
|
||||
Replace the unsafe `Type.GetType(typeName)` lookup in `AyCodeBinaryHubProtocol`'s argument-binder header parsing with a registry-validated lookup. The deserialization-gadget attack surface this fix closes, the rationale for why the type-resolution mechanism is needed at all, and the four mitigation directions are covered in the issue — **this TODO tracks the chosen implementation path only**.
|
||||
|
||||
### Implementation steps
|
||||
|
||||
1. **Pick a mitigation direction** (the issue lists four: whitelist, tag-based, type-hash, audit-only stopgap). Recommended starting point: **whitelist registry**, because it can ship without a wire-format change and immediately closes the vulnerability for typical deployments. Tag-based dispatch can land as a follow-up wire-format evolution (separate TODO once direction is committed).
|
||||
|
||||
2. **Registry surface design** — likely on `AyCodeBinaryHubProtocolOptions`:
|
||||
- `Action<TypeWhitelistBuilder>? ConfigureTypeWhitelist { get; init; }` — fluent builder for explicit type registration
|
||||
- Auto-population path: `RegisterAcBinarySerializableTypesFrom(Assembly...)` — scans for `[AcBinarySerializable]`-marked types from declared assemblies (acceptable trust boundary for many apps; ALSO opt-in)
|
||||
- Defaulting: if no whitelist configured, the protocol either (a) refuses to start with a clear error message, or (b) uses an empty whitelist (rejecting all `Type.GetType` lookups). Decide based on backward-compat appetite.
|
||||
|
||||
3. **Replace the call site** (`AyCodeBinaryHubProtocol.cs` ~line 114):
|
||||
- Before: `resolvedType = Type.GetType(typeName);`
|
||||
- After: `resolvedType = _typeWhitelist.TryResolve(typeName);` — returns `null` if not registered → propagate as protocol error (reject the message, log with the offending type-name for security audit).
|
||||
|
||||
4. **Test plan**:
|
||||
- Unit: whitelist-registered type resolves; non-registered type returns `null` and the protocol throws / logs.
|
||||
- Security regression: pass `System.Diagnostics.Process`, `System.IO.File`-related descriptor types — all rejected.
|
||||
- Backward compat: existing benchmarks / integration tests continue to pass with their DTOs registered.
|
||||
- Versioning: a registered DTO whose assembly version changed (NuGet upgrade) still resolves, because the whitelist is registry-keyed (not assembly-version-keyed) — verify or design accordingly.
|
||||
|
||||
### Acceptance criteria
|
||||
- The vulnerability described in `R8K3` is no longer reachable: a wire-supplied non-whitelisted type-name results in a protocol error, NOT a `Type.GetType` BCL lookup.
|
||||
- The `R8K3` issue is updated with `Status: Fixed` (or `Mitigated` if a chosen direction only partially closes it).
|
||||
- A subsequent TODO is opened for tag-based dispatch (wire-format evolution) IF that direction is also pursued — this TODO covers the immediate whitelist-fix only.
|
||||
|
||||
### Out of scope (deliberately)
|
||||
- Wire-format changes (tag-based / type-hash dispatch). Those are a separate TODO once direction is committed and a wire-format-evolution PR plan exists.
|
||||
- Audit of every Hub-method `dataArg : object` use site in consumer projects. The fix is at the protocol level — consumers register their DTOs and the protocol enforces.
|
||||
|
|
|
|||
Loading…
Reference in New Issue