diff --git a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs index b4d801b8..62ac9819 100644 --- a/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs +++ b/src/clients/ioxide.httpclient/Http1/HttpClientPool.cs @@ -34,6 +34,9 @@ public sealed class HttpClientPool : IDisposable private int _requests; // inside SendAsync: acquiring, waiting or sending private bool _disposed; + // What an open started behind the backoff gate waits on. One is enough: such an open starts only with none opening. + private readonly RingOpSource _backoff = new(); + // Warm: PoolSize connections opened up front and kept that way, the long-lived upstream shape. // Otherwise connections open as requests need them, which is how RingHttpClient's pools run. private readonly bool _warm; @@ -161,7 +164,7 @@ private async ValueTask AcquireAsync() // Honour the same backoff gate Sweep() does. Without it, a dead origin turns this loop // into a connect() storm: open, refuse, wake, reopen - thousands of syscalls a second - // on the reactor thread. Behind the gate the ticker paces the retries instead. One open + // on the reactor thread. Behind the gate the retry waits the backoff out, below. One open // in flight per waiting request, so a burst that finds no connection opens in parallel. if (_live < _options.PoolSize && _opening <= _waiters.Count && Native.MonotonicMs >= _reopenAtMs) { @@ -173,6 +176,12 @@ private async ValueTask AcquireAsync() throw new HttpClientException(AcquireTimeoutMessage()); } + // Nothing opening: the retry starts now, waiting the backoff out on the ring, so a stop ends this wait too. + if (_opening == 0 && _live < _options.PoolSize) + { + StartOpen(Math.Max(1, _reopenAtMs - Native.MonotonicMs)); + } + // No timer of its own: the sweep fails a waiter at its deadline. A Task.WhenAny with a // Task.Delay ran the hand-over through the thread pool and back for every request that // waited - most of them, in a pool that is the bottleneck. Completed from the reactor, @@ -223,7 +232,7 @@ private void Discard(HttpClientConnection connection) WakeWaiters(); // they must re-check: a replacement is opening } - private void StartOpen() + private void StartOpen(long backoffMs = 0) { if (_disposed) { @@ -232,13 +241,28 @@ private void StartOpen() _live++; _opening++; - _ = OpenOneAsync(); + _ = OpenOneAsync(backoffMs); } - private async Task OpenOneAsync() + private async Task OpenOneAsync(long backoffMs) { try { + if (backoffMs > 0) + { + ValueTask backoff = _backoff.Prepare(); + _host.SubmitTimeout(backoffMs * 1_000_000L, _backoff); + int waited = await backoff; + + // Not -ETIME, the wait running its course, but the stop: no connect, and no failure to report. + if (waited is not (-62 or 0) || _disposed) + { + _live--; + WakeWaiters(); + return; + } + } + HttpClientConnection connection = await HttpClientConnection.ConnectAsync(_host, _options); _reopenAtMs = 0; // healthy - allow immediate replenishment _lastOpenFailure = null; diff --git a/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs b/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs new file mode 100644 index 00000000..d6e0f6fc --- /dev/null +++ b/tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs @@ -0,0 +1,121 @@ +using System.Net; +using System.Net.Sockets; +using ioxide; +using ioxide.httpclient; +using ioxide.timer; + +namespace Ioxide.Tests; + +/// +/// A request that finds no connection while its pool backs off after a failed open waits behind a retry that waits +/// the backoff out on the ring. The retries keep the backoff's pace, and the reactor's stop ends the request, as it +/// ends one waiting on a connect or a response. +/// +internal static class HttpClientBackoffTests +{ + public static void Register(Runner runner) + { + runner.Test("httpclient h1: a request waiting out the reconnect backoff ends when its reactor stops", () => + { + string outcome = EndedByTheStop(reactor => HttpClientPool.Start(reactor, new HttpClientOptions + { Port = (ushort)TestServer.DeadPort(), PoolSize = 1, AcquireTimeoutMs = 30_000 }).GetAsync("/")); + + Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome); + }); + + runner.Test("ring client: a request waiting out the reconnect backoff ends when its reactor stops", () => + { + string outcome = EndedByTheStop(reactor => RingHttpClient.Start(reactor, new RingHttpClientOptions + { Pool = new HttpClientOptions { PoolSize = 1, AcquireTimeoutMs = 30_000 } }) + .GetAsync($"http://127.0.0.1:{TestServer.DeadPort()}/")); + + Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome); + }); + + runner.Test("httpclient h1: a request waiting on an origin whose every open fails retries at the backoff's pace", () => + { + // Accepted, then closed before the handshake: the open fails as a refused one does, and the origin counts it. + using var origin = new TcpListener(IPAddress.Loopback, 0); + origin.Start(); + int opens = 0; + _ = Task.Run(async () => + { + try + { + while (true) + { + using Socket accepted = await origin.AcceptSocketAsync(); + Interlocked.Increment(ref opens); + } + } + catch (Exception) + { + // The listener stopped. + } + }); + + TlsClientContext tls = TestServer.Own(TlsClientContext.Create(new TlsClientOptions { ServerName = "localhost" })); + string outcome = OnReactor.Run(async reactor => + { + HttpClientPool pool = HttpClientPool.Start(reactor, new HttpClientOptions + { + Port = (ushort)((IPEndPoint)origin.LocalEndpoint).Port, + Tls = tls, + PoolSize = 1, + AcquireTimeoutMs = 2_000, + }); + try + { + using HttpClientResponse response = await pool.GetAsync("/"); + return $"answered {response.Status}"; + } + catch (HttpClientException e) + { + return e.Message; + } + }); + int counted = Volatile.Read(ref opens); + + Assert.True(outcome.StartsWith("no connection to "), $"expected the acquire to time out, got: {outcome}"); + + // The first open, then one per backoff of 200 to 400 ms, while the request waits its 2 s. + Assert.True(counted is >= 3 and <= 12, $"{counted} opens in the request's 2 s, where its backoff allows 6 to 11"); + }); + } + + // Makes the request on a reactor of its own and stops that reactor once 50 ms have passed: the refused connect is + // long done, and the backoff after it, 200 ms at least, has the request waiting. How the request ended, once stopped. + private static string EndedByTheStop(Func> request) + { + var made = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + (_, Reactor reactor, Thread thread) = TestServer.StartConfigured( + (_, connection) => { connection.DecRef(); return Task.CompletedTask; }, + new ServerConfig + { + RecvBufferSize = 4096, RecvSlots = 256, + Tcp = new TcpOptions { WriteSlabSize = 16 * 1024, PoolMax = 64, RecvQueueEntries = 64 }, + }, + r => _ = MakeAsync(r)); + + Assert.True(made.Task.Wait(10_000), "the request was never made"); + reactor.Stop(); + Assert.True(thread.Join(10_000), "the reactor did not stop"); + + Task call = made.Task.Result; + try + { + return call.Wait(2_000) ? $"answered {call.Result.Status}" : "still waiting 2 s after its reactor stopped"; + } + catch (AggregateException e) + { + return $"{e.InnerException!.GetType().Name}: {e.InnerException.Message.ReplaceLineEndings(" ")}"; + } + + async Task MakeAsync(Reactor r) + { + Task pending = request(r).AsTask(); + await new RingTimer(r).DelayAsync(50); + made.SetResult(pending); + } + } +} diff --git a/tests/Ioxide.Tests.Http/Program.cs b/tests/Ioxide.Tests.Http/Program.cs index a44780b0..b47345c2 100644 --- a/tests/Ioxide.Tests.Http/Program.cs +++ b/tests/Ioxide.Tests.Http/Program.cs @@ -24,6 +24,7 @@ private static int Main() HttpClientResponseLimitTests.Register(runner); HttpClientHttp10Tests.Register(runner); HttpClientThreadTests.Register(runner); + HttpClientBackoffTests.Register(runner); TlsClientTests.Register(runner); // Areas reserved for the failing-test review pass; empty until one lands.