Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions src/clients/ioxide.httpclient/Http1/HttpClientPool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public sealed class HttpClientPool : IDisposable
private int _requests; // inside SendAsync: acquiring, waiting or sending
private bool _disposed;

// What an open started behind the backoff gate waits on. One is enough: such an open starts only with none opening.
private readonly RingOpSource _backoff = new();

// Warm: PoolSize connections opened up front and kept that way, the long-lived upstream shape.
// Otherwise connections open as requests need them, which is how RingHttpClient's pools run.
private readonly bool _warm;
Expand Down Expand Up @@ -161,7 +164,7 @@ private async ValueTask<HttpClientConnection> AcquireAsync()

// Honour the same backoff gate Sweep() does. Without it, a dead origin turns this loop
// into a connect() storm: open, refuse, wake, reopen - thousands of syscalls a second
// on the reactor thread. Behind the gate the ticker paces the retries instead. One open
// on the reactor thread. Behind the gate the retry waits the backoff out, below. One open
// in flight per waiting request, so a burst that finds no connection opens in parallel.
if (_live < _options.PoolSize && _opening <= _waiters.Count && Native.MonotonicMs >= _reopenAtMs)
{
Expand All @@ -173,6 +176,12 @@ private async ValueTask<HttpClientConnection> AcquireAsync()
throw new HttpClientException(AcquireTimeoutMessage());
}

// Nothing opening: the retry starts now, waiting the backoff out on the ring, so a stop ends this wait too.
if (_opening == 0 && _live < _options.PoolSize)
{
StartOpen(Math.Max(1, _reopenAtMs - Native.MonotonicMs));
}

// No timer of its own: the sweep fails a waiter at its deadline. A Task.WhenAny with a
// Task.Delay ran the hand-over through the thread pool and back for every request that
// waited - most of them, in a pool that is the bottleneck. Completed from the reactor,
Expand Down Expand Up @@ -223,7 +232,7 @@ private void Discard(HttpClientConnection connection)
WakeWaiters(); // they must re-check: a replacement is opening
}

private void StartOpen()
private void StartOpen(long backoffMs = 0)
{
if (_disposed)
{
Expand All @@ -232,13 +241,28 @@ private void StartOpen()

_live++;
_opening++;
_ = OpenOneAsync();
_ = OpenOneAsync(backoffMs);
}

private async Task OpenOneAsync()
private async Task OpenOneAsync(long backoffMs)
{
try
{
if (backoffMs > 0)
{
ValueTask<int> backoff = _backoff.Prepare();
_host.SubmitTimeout(backoffMs * 1_000_000L, _backoff);
int waited = await backoff;

// Not -ETIME, the wait running its course, but the stop: no connect, and no failure to report.
if (waited is not (-62 or 0) || _disposed)
{
_live--;
WakeWaiters();
return;
}
}

HttpClientConnection connection = await HttpClientConnection.ConnectAsync(_host, _options);
_reopenAtMs = 0; // healthy - allow immediate replenishment
_lastOpenFailure = null;
Expand Down
121 changes: 121 additions & 0 deletions tests/Ioxide.Tests.Http/HttpClientBackoffTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
using System.Net;
using System.Net.Sockets;
using ioxide;
using ioxide.httpclient;
using ioxide.timer;

namespace Ioxide.Tests;

/// <summary>
/// A request that finds no connection while its pool backs off after a failed open waits behind a retry that waits
/// the backoff out on the ring. The retries keep the backoff's pace, and the reactor's stop ends the request, as it
/// ends one waiting on a connect or a response.
/// </summary>
internal static class HttpClientBackoffTests
{
public static void Register(Runner runner)
{
runner.Test("httpclient h1: a request waiting out the reconnect backoff ends when its reactor stops", () =>
{
string outcome = EndedByTheStop(reactor => HttpClientPool.Start(reactor, new HttpClientOptions
{ Port = (ushort)TestServer.DeadPort(), PoolSize = 1, AcquireTimeoutMs = 30_000 }).GetAsync("/"));

Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome);
});

runner.Test("ring client: a request waiting out the reconnect backoff ends when its reactor stops", () =>
{
string outcome = EndedByTheStop(reactor => RingHttpClient.Start(reactor, new RingHttpClientOptions
{ Pool = new HttpClientOptions { PoolSize = 1, AcquireTimeoutMs = 30_000 } })
.GetAsync($"http://127.0.0.1:{TestServer.DeadPort()}/"));

Assert.Equal("ObjectDisposedException: The reactor has stopped. Object name: 'Reactor'.", outcome);
});

runner.Test("httpclient h1: a request waiting on an origin whose every open fails retries at the backoff's pace", () =>
{
// Accepted, then closed before the handshake: the open fails as a refused one does, and the origin counts it.
using var origin = new TcpListener(IPAddress.Loopback, 0);
origin.Start();
int opens = 0;
_ = Task.Run(async () =>
{
try
{
while (true)
{
using Socket accepted = await origin.AcceptSocketAsync();
Interlocked.Increment(ref opens);
}
}
catch (Exception)
{
// The listener stopped.
}
});

TlsClientContext tls = TestServer.Own(TlsClientContext.Create(new TlsClientOptions { ServerName = "localhost" }));
string outcome = OnReactor.Run(async reactor =>
{
HttpClientPool pool = HttpClientPool.Start(reactor, new HttpClientOptions
{
Port = (ushort)((IPEndPoint)origin.LocalEndpoint).Port,
Tls = tls,
PoolSize = 1,
AcquireTimeoutMs = 2_000,
});
try
{
using HttpClientResponse response = await pool.GetAsync("/");
return $"answered {response.Status}";
}
catch (HttpClientException e)
{
return e.Message;
}
});
int counted = Volatile.Read(ref opens);

Assert.True(outcome.StartsWith("no connection to "), $"expected the acquire to time out, got: {outcome}");

// The first open, then one per backoff of 200 to 400 ms, while the request waits its 2 s.
Assert.True(counted is >= 3 and <= 12, $"{counted} opens in the request's 2 s, where its backoff allows 6 to 11");
});
}

// Makes the request on a reactor of its own and stops that reactor once 50 ms have passed: the refused connect is
// long done, and the backoff after it, 200 ms at least, has the request waiting. How the request ended, once stopped.
private static string EndedByTheStop(Func<Reactor, ValueTask<HttpClientResponse>> request)
{
var made = new TaskCompletionSource<Task<HttpClientResponse>>(TaskCreationOptions.RunContinuationsAsynchronously);
(_, Reactor reactor, Thread thread) = TestServer.StartConfigured(
(_, connection) => { connection.DecRef(); return Task.CompletedTask; },
new ServerConfig
{
RecvBufferSize = 4096, RecvSlots = 256,
Tcp = new TcpOptions { WriteSlabSize = 16 * 1024, PoolMax = 64, RecvQueueEntries = 64 },
},
r => _ = MakeAsync(r));

Assert.True(made.Task.Wait(10_000), "the request was never made");
reactor.Stop();
Assert.True(thread.Join(10_000), "the reactor did not stop");

Task<HttpClientResponse> call = made.Task.Result;
try
{
return call.Wait(2_000) ? $"answered {call.Result.Status}" : "still waiting 2 s after its reactor stopped";
}
catch (AggregateException e)
{
return $"{e.InnerException!.GetType().Name}: {e.InnerException.Message.ReplaceLineEndings(" ")}";
}

async Task MakeAsync(Reactor r)
{
Task<HttpClientResponse> pending = request(r).AsTask();
await new RingTimer(r).DelayAsync(50);
made.SetResult(pending);
}
}
}
1 change: 1 addition & 0 deletions tests/Ioxide.Tests.Http/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private static int Main()
HttpClientResponseLimitTests.Register(runner);
HttpClientHttp10Tests.Register(runner);
HttpClientThreadTests.Register(runner);
HttpClientBackoffTests.Register(runner);
TlsClientTests.Register(runner);

// Areas reserved for the failing-test review pass; empty until one lands.
Expand Down
Loading