From 048b353fba0e21f3a2bbae5f10a7049841e9cbb8 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Sat, 14 Nov 2026 18:47:21 +0000 Subject: [PATCH] reactor: a stop leaves the shared receive slab to the last handler still holding a connection, so a read after it gets the bytes queued before it, not freed memory (#605) In the default shared receive mode what a connection queues, and what a reader takes from that queue, points into the reactor's one receive slab, and Teardown freed it unconditionally. A connection its handler still held kept its queue across the stop: one still in the table, and one its client had closed, which had left the table. A read after the stop served that queue, and a reader still held what it took before the stop, so both handed out slices of freed memory, and reading them faulted the process. The slab now has holders: the teardown and every handler that has not let go. Each accept adds its handler's hold, the handler's DecRef gives it up once per life, and the teardown gives up its own where it freed the slab. The last to let go frees it, on its own thread; the kernel stopped using it when the teardown unregistered the ring. A stop whose handlers all let go during it, as one parked in a read does, frees it where it did. Incremental mode counts its handlers too and has no shared slab. Tests in StopParkedTests (E2E): a read after the stop of bytes queued before it, for a connection open at the stop and one its client closed before it, and what a pipe reader took before the stop, each get intact bytes from a slab still allocated; the slab is freed once the handlers the stop woke let go, and outlives the stop until the last of two held connections' handlers lets go. An incremental-mode control reads intact bytes on main too. --- src/ioxide/Connection/Tcp/TcpConnection.cs | 1 + .../Reactor/Loop/Reactor.Loop.SharedRing.cs | 11 + src/ioxide/Reactor/Reactor.Runner.cs | 7 +- src/ioxide/Reactor/Reactor.cs | 3 + .../Reactor/Transport/Tcp/Reactor.Tcp.cs | 1 + .../Ioxide.Tests.E2E/Core/StopParkedTests.cs | 276 ++++++++++++++++++ 6 files changed, 294 insertions(+), 5 deletions(-) diff --git a/src/ioxide/Connection/Tcp/TcpConnection.cs b/src/ioxide/Connection/Tcp/TcpConnection.cs index 1be10e7d..595466a1 100644 --- a/src/ioxide/Connection/Tcp/TcpConnection.cs +++ b/src/ioxide/Connection/Tcp/TcpConnection.cs @@ -148,6 +148,7 @@ public void DecRef() } ReleaseRef(); + _reactor.ReleaseBufSlab(); // after a stop, the last handler to let go frees the receive slab } internal void ReleaseReactorRef() => ReleaseRef(); diff --git a/src/ioxide/Reactor/Loop/Reactor.Loop.SharedRing.cs b/src/ioxide/Reactor/Loop/Reactor.Loop.SharedRing.cs index a950561d..68d4cfe0 100644 --- a/src/ioxide/Reactor/Loop/Reactor.Loop.SharedRing.cs +++ b/src/ioxide/Reactor/Loop/Reactor.Loop.SharedRing.cs @@ -34,6 +34,17 @@ private void InitSharedRingBuffer() _bufRingTail = (ushort)_bufferRingEntries; } + // The last holder frees it, on whatever thread it lets go on: the kernel let go when the teardown unregistered the + // ring. Incremental mode counts its handlers too, and has no slab to free. + internal void ReleaseBufSlab() + { + if (Interlocked.Decrement(ref _bufSlabHolders) == 0) + { + NativeMemory.AlignedFree(_bufSlab); + _bufSlab = null; + } + } + private void DispatchSharedRing(in IoUringCqe cqe) { byte kind = (byte)(cqe.user_data >> KindShift); diff --git a/src/ioxide/Reactor/Reactor.Runner.cs b/src/ioxide/Reactor/Reactor.Runner.cs index 400ef75c..ed3c2ba2 100644 --- a/src/ioxide/Reactor/Reactor.Runner.cs +++ b/src/ioxide/Reactor/Reactor.Runner.cs @@ -199,11 +199,8 @@ private void Teardown(bool startFailed) NativeMemory.AlignedFree(_bufRing); _bufRing = null; } - if (_bufSlab != null) - { - NativeMemory.AlignedFree(_bufSlab); - _bufSlab = null; - } + // Not under a handler holding a connection, whose queue and reader may point into it: the last to let go frees it. + ReleaseBufSlab(); FreeUdpMemory(); // Last, so what is still queued, or posted from here on, fails its switch rather than waiting forever. diff --git a/src/ioxide/Reactor/Reactor.cs b/src/ioxide/Reactor/Reactor.cs index 37b93bea..cab92565 100644 --- a/src/ioxide/Reactor/Reactor.cs +++ b/src/ioxide/Reactor/Reactor.cs @@ -67,6 +67,9 @@ private static ulong Tag(byte kind, ushort gen, int fd) private uint _bufRingMask; private ushort _bufRingTail; + // The teardown and every handler yet to let go: what a connection queued, or its reader took, points into the slab. + private int _bufSlabHolders = 1; + // Transport settings resolved once. _tcp / _udp are always non-null so the sizing knobs stay // readable; _tcpEnabled records whether the config actually asked for TCP. private readonly TcpOptions _tcp; diff --git a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs index cf94b53d..a9fe8e93 100644 --- a/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs +++ b/src/ioxide/Reactor/Transport/Tcp/Reactor.Tcp.cs @@ -252,6 +252,7 @@ private void OnTcpAcceptCompletion(int listenFd, int res, bool more) conn.UseZc = _zeroCopySend; // config default; kTLS overrides to plain on handshake Track(clientFd, conn); conn.InitRefs(); + Interlocked.Increment(ref _bufSlabHolders); // the handler's, given up by its DecRef conn.ListenerPort = PortOf(listenFd); conn.BindSendOpFlags(_sendOpFlags); // TcpOptions.SendWaitAll; the kTLS handoff clears it SubmitRecvMultishot(clientFd, (ushort)conn.Generation, bgid); diff --git a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs index 008675b3..b6eb3836 100644 --- a/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs +++ b/tests/Ioxide.Tests.E2E/Core/StopParkedTests.cs @@ -1,3 +1,5 @@ +using System.Buffers; +using System.IO.Pipelines; using System.Net; using System.Net.Sockets; using System.Runtime.InteropServices; @@ -341,6 +343,164 @@ public static void Register(Runner runner) AfterTheStop((conn, _) => conn.ReadAsync().AsTask(), "ReadAsync"); }); + foreach (bool peerClosed in new[] { false, true }) + { + string state = peerClosed ? "its client closed before the stop" : "open at the stop"; + + runner.Test($"stop: a read from another thread after the stop gets the bytes queued before it, intact, from receive memory the stop kept ({state})", () => + { + QueuedAcrossTheStop(SharedRing, peerClosed); + }); + } + + runner.Test("stop: control: in incremental mode a read from another thread after the stop gets the bytes queued before it, intact", () => + { + QueuedAcrossTheStop(IncrementalRing, peerClosed: false); + }, skip: !TestServer.KernelAtLeast(6, 12)); + + runner.Test("stop: what a pipe reader took before the stop and still holds stays intact, in receive memory the stop kept", () => + { + var holding = new TaskCompletionSource<(TcpConnection, ReadOnlySequence)>(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + var reader = new TcpConnectionPipeReader(conn); + try + { + while (true) + { + ReadResult result = await reader.ReadAsync(); + if (result.Buffer.Length >= Payload.Length) + { + holding.TrySetResult((conn, result.Buffer)); // neither consumed nor completed + await release.Task; + return; + } + if (result.IsCompleted) + { + return; // the harness's listen probe + } + reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + } + } + finally + { + reader.Complete(); + conn.DecRef(); + } + }, SharedRing); + + try + { + using Socket client = Connect(port); + client.Send(Payload); + Assert.True(holding.Task.Wait(5_000), "the handler never read what its client sent"); + + StopAndJoin(reactor, thread); + + (TcpConnection conn, ReadOnlySequence held) = holding.Task.Result; + foreach (ReadOnlyMemory segment in held) + { + AssertInAllocatedSlab(reactor, conn, SharedRing, segment); + } + Assert.True(held.ToArray().AsSpan().SequenceEqual(Payload), "what the reader held changed across the stop"); + } + finally + { + release.TrySetResult(); + } + }); + + runner.Test("stop: the receive slab is freed once the handlers the stop woke let go", () => + { + var parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var letGo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + if (!await IsThisTestsConnection(conn)) + { + conn.DecRef(); + return; + } + + parked.TrySetResult(); + await conn.ReadAsync(); // until the stop closes it + conn.DecRef(); + letGo.TrySetResult(); + }, SharedRing); + + using Socket client = Connect(port); + client.Send("hi"u8); + Assert.True(parked.Task.Wait(5_000), "the handler never parked"); + + StopAndJoin(reactor, thread); + Assert.True(letGo.Task.Wait(ReturnsWithinMs), "the handler the stop woke never let go"); + Assert.True(!HasSlab(reactor), "the receive slab outlived a stop whose handlers have all let go"); + }); + + runner.Test("stop: the receive slab outlives the stop until the last handler still holding a connection lets go", () => + { + using var running = new CountdownEvent(2); + var release = new Dictionary(); + var letGo = new Dictionary(); + foreach (char role in "ab") + { + release[role] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + letGo[role] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + int role = await ReadRole(conn); + if (role < 0) + { + conn.DecRef(); + return; + } + + running.Signal(); + await release[(char)role].Task; // busy outside ioxide when the stop comes + conn.DecRef(); + letGo[(char)role].TrySetResult(); + }, SharedRing); + + var clients = new List(); + try + { + foreach (char role in "ab") + { + Socket client = Connect(port); + clients.Add(client); + client.Send([(byte)role]); + } + Assert.True(running.Wait(5_000), "the handlers never ran"); + + StopAndJoin(reactor, thread); + Assert.True(HasSlab(reactor), "the receive slab was freed at the stop, under two connections their handlers still hold"); + + release['a'].TrySetResult(); + Assert.True(letGo['a'].Task.Wait(ReturnsWithinMs), "the first handler never let go"); + Assert.True(HasSlab(reactor), "the receive slab was freed when the first of two handlers let go, under the other"); + + release['b'].TrySetResult(); + Assert.True(letGo['b'].Task.Wait(ReturnsWithinMs), "the second handler never let go"); + Assert.True(!HasSlab(reactor), "the receive slab outlived the last handler to let go"); + } + finally + { + foreach (TaskCompletionSource each in release.Values) + { + each.TrySetResult(); + } + foreach (Socket client in clients) + { + client.Dispose(); + } + } + }); + runner.Test("stop: a RingTimer wait from another thread after the reactor stopped returns", () => { AfterTheStop((_, reactor) => new RingTimer(reactor).DelayAsync(1).AsTask(), "DelayAsync"); @@ -472,6 +632,122 @@ private static async Task ReadRole(TcpConnection conn) } } + // Small, so a slice's bounds are easy to check. + private static readonly ServerConfig SharedRing = new() { RecvBufferSize = 4096, RecvSlots = 256, Tcp = new TcpOptions() }; + + private static readonly ServerConfig IncrementalRing = new() + { + Incremental = new IncrementalOptions { MaxConnections = 16 }, + Tcp = new TcpOptions(), + }; + + private static readonly byte[] Payload = [.. Enumerable.Range(0, 1000).Select(i => (byte)('a' + i % 26))]; + + // A handler waiting on something else while its client's bytes queue, the stop, and then a read from another thread. + private static void QueuedAcrossTheStop(ServerConfig config, bool peerClosed) + { + var running = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + (int port, Reactor reactor, Thread thread) = TestServer.StartConfigured(async (_, conn) => + { + if (await IsThisTestsConnection(conn)) + { + running.TrySetResult(conn); + await release.Task; // waiting on something else, and not on the connection + } + + conn.DecRef(); + }, config); + + try + { + using Socket client = Connect(port); + client.Send("hi"u8); + Assert.True(running.Task.Wait(5_000), "the handler never ran"); + TcpConnection conn = running.Task.Result; + + client.Send(Payload); + if (peerClosed) + { + client.Shutdown(SocketShutdown.Send); // the reactor closes the connection itself, before the stop + } + Assert.True(SpinWait.SpinUntil(() => peerClosed ? conn.IsClosed : !RecvOf(conn).IsEmpty(), 5_000), + peerClosed ? "the reactor never saw its client close" : "the client's bytes were never queued"); + + StopAndJoin(reactor, thread); + + Task read = Task.Run(() => ReadQueued(reactor, conn, config)); + Assert.True(((IAsyncResult)read).AsyncWaitHandle.WaitOne(ReturnsWithinMs), "the read after the stop never returned"); + byte[] received = read.GetAwaiter().GetResult(); + Assert.True(received.AsSpan().SequenceEqual(Payload), + $"the read after the stop got {received.Length} bytes, not the {Payload.Length} queued before it"); + } + finally + { + release.TrySetResult(); + } + } + + private static byte[] ReadQueued(Reactor reactor, TcpConnection conn, ServerConfig config) + { + ValueTask read = conn.ReadAsync(); + Assert.True(read.IsCompleted, "the read after the stop waited"); + + RecvSnapshot snapshot = read.Result; + var received = new List(); + while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) + { + AssertInAllocatedSlab(reactor, conn, config, in item); + received.AddRange(item.AsSpan()); + conn.ReturnBuffer(in item); + } + } + + return [.. received]; + } + + private static unsafe void AssertInAllocatedSlab(Reactor reactor, TcpConnection conn, ServerConfig config, in SpscRecvRing.Item item) + => AssertInAllocatedSlab(reactor, conn, config, item.Ptr, item.Len); + + private static unsafe void AssertInAllocatedSlab(Reactor reactor, TcpConnection conn, ServerConfig config, ReadOnlyMemory segment) + { + fixed (byte* ptr = segment.Span) + { + AssertInAllocatedSlab(reactor, conn, config, ptr, segment.Length); + } + } + + // Before a byte of it is read: one of a freed slab can fault the process. + private static unsafe void AssertInAllocatedSlab(Reactor reactor, TcpConnection conn, ServerConfig config, byte* ptr, int length) + { + byte* slab; + nuint size; + if (config.Incremental is { } incremental) + { + slab = BufSlabOf(conn); + size = (nuint)incremental.RecvSlots * (nuint)incremental.RecvBufferSize; + } + else + { + slab = SlabOf(reactor); + size = (nuint)config.RecvSlots * (nuint)config.RecvBufferSize; + } + + Assert.True(slab != null, "the receive slab the read points into was freed at the stop, under a connection its handler still holds"); + Assert.True(ptr >= slab && ptr + length <= slab + size, "the read handed out bytes from outside the receive slab"); + } + + private static unsafe bool HasSlab(Reactor reactor) => SlabOf(reactor) != null; + + [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_bufSlab")] + private static extern unsafe ref byte* SlabOf(Reactor reactor); + + [System.Runtime.CompilerServices.UnsafeAccessor(System.Runtime.CompilerServices.UnsafeAccessorKind.Field, Name = "_recv")] + private static extern ref SpscRecvRing RecvOf(TcpConnection connection); + private static unsafe bool HasWriteBuffer(TcpConnection connection) => WriteBufferOf(connection) != null; private static unsafe bool HasRing(TcpConnection connection) => BufRingOf(connection) != null && BufSlabOf(connection) != null;