Feature/improve auth and logging

src/GovUK.Dfe.FlexForms.Api.Client/GovUK.Dfe.FlexForms.Api.Client.csproj

@@ -24,7 +24,7 @@
 		<PackageReference Include="Azure.Identity" Version="1.21.0" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Contracts" Version="1.0.112" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Http" Version="1.0.12" />
-		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.30" />
+		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.31" />
 		<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
 	</ItemGroup>
 

src/GovUK.Dfe.FlexForms.Application/GovUK.Dfe.FlexForms.Application.csproj

@@ -21,7 +21,7 @@
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Messaging.Contracts" Version="0.1.6" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Messaging.MassTransit" Version="0.1.3" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Notifications" Version="0.1.7" />
-		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.30" />
+		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.31" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Utilities" Version="1.0.16" />
 		<PackageReference Include="FluentValidation" Version="12.1.1" />
 		<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="12.1.1" />

src/GovUK.Dfe.FlexForms.Infrastructure/GovUK.Dfe.FlexForms.Infrastructure.csproj

@@ -11,7 +11,7 @@
 	<ItemGroup>
 		<PackageReference Include="Azure.Identity" Version="1.21.0" />
 		<PackageReference Include="GovUK.Dfe.CoreLibs.Caching" Version="1.0.13" />
-		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.30" />
+		<PackageReference Include="GovUK.Dfe.CoreLibs.Security" Version="1.1.31" />
 		<PackageReference Include="MassTransit.EntityFrameworkCore" Version="8.5.5" />
 		<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.11" />
 		<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="10.0.11" />

src/GovUK.Dfe.FlexForms.Infrastructure/Messaging/Outbox/TenantOutboxDeliveryService.cs

@@ -15,7 +15,8 @@ namespace GovUK.Dfe.FlexForms.Infrastructure.Messaging.Outbox;
 /// Runs one MassTransit <see cref="BusOutboxDeliveryService{TDbContext}"/> per distinct tenant EA
 /// database. The built-in delivery service is disabled because it has no tenant context and would
 /// only ever drain the first tenant's database. Tenants sharing a database share a loop.
-/// The set of loops is reconciled whenever tenant configuration is refreshed.
+/// The set of loops is reconciled whenever tenant configuration changes, and again after
+/// <see cref="StartRetryDelay"/> if any loop failed to start.
 /// </summary>
 public sealed class TenantOutboxDeliveryService(
     IServiceProvider provider,
@@ -26,6 +27,8 @@ public sealed class TenantOutboxDeliveryService(
 {
     private const string ConnectionStringName = "DefaultConnection";
 
+    internal static readonly TimeSpan StartRetryDelay = TimeSpan.FromMinutes(1);
+
     private readonly Dictionary<string, RunningDelivery> _running = new(StringComparer.OrdinalIgnoreCase);
 
     private readonly Channel<bool> _reconcileSignal = Channel.CreateBounded<bool>(
@@ -77,6 +80,8 @@ private async Task ReconcileAsync(CancellationToken cancellationToken)
         foreach (var key in _running.Keys.Where(k => !desired.ContainsKey(k)).ToList())
             await StopDeliveryAsync(key);
 
+        var anyStartFailed = false;
+
         foreach (var (key, tenant) in desired)
         {
             if (_running.ContainsKey(key))
@@ -88,9 +93,26 @@ private async Task ReconcileAsync(CancellationToken cancellationToken)
             }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
-                logger.LogError(ex, "Failed to start outbox delivery for tenant {TenantName}", tenant.Name);
+                anyStartFailed = true;
+                logger.LogError(ex, "Failed to start outbox delivery for tenant {TenantName}. Retrying in {RetryDelay}",
+                    tenant.Name, StartRetryDelay);
             }
         }
+
+        if (anyStartFailed)
+            _ = SignalReconcileAfterDelayAsync(cancellationToken);
+    }
+
+    private async Task SignalReconcileAfterDelayAsync(CancellationToken cancellationToken)
+    {
+        try
+        {
+            await Task.Delay(StartRetryDelay, cancellationToken);
+            _reconcileSignal.Writer.TryWrite(true);
+        }
+        catch (OperationCanceledException)
+        {
+        }
     }
 
     private async Task StartDeliveryAsync(string key, TenantConfiguration tenant, CancellationToken cancellationToken)

src/GovUK.Dfe.FlexForms.Infrastructure/Services/DatabaseTenantConfigurationProvider.cs

@@ -1,3 +1,5 @@
+using System.Security.Cryptography;
+using System.Text;
 using System.Text.Json;
 using GovUK.Dfe.FlexForms.Domain.Tenancy;
 using GovUK.Dfe.FlexForms.Infrastructure.Database;
@@ -13,6 +15,12 @@ namespace GovUK.Dfe.FlexForms.Infrastructure.Services;
 /// Loads tenant configuration from the tenant config database and caches it in memory.
 /// Implements IHostedService to load at startup and refresh periodically.
 /// Uses volatile dictionary swaps for lock-free reads.
+/// <para>
+/// The snapshot is only swapped, and <see cref="ITenantConfigurationChangedNotifier"/> only fired,
+/// when the loaded data differs from the current snapshot. Subscribers rebuild expensive state
+/// (OIDC metadata caches, auth registries, options) on every notification, so an unconditional
+/// notify on each periodic refresh would discard those caches every interval.
+/// </para>
 /// </summary>
 public class DatabaseTenantConfigurationProvider(
     IServiceScopeFactory scopeFactory,
@@ -30,6 +38,10 @@ public class DatabaseTenantConfigurationProvider(
 
     private DateTimeOffset? _lastRefreshedUtc;
 
+    private string? _snapshotFingerprint;
+
+    private readonly SemaphoreSlim _refreshGate = new(1, 1);
+
     private Timer? _refreshTimer;
     private bool _disposed;
 
@@ -56,6 +68,9 @@ public IReadOnlyCollection<TenantConfiguration> GetAllTenants()
     /// </summary>
     public async Task RefreshAsync(CancellationToken cancellationToken = default)
     {
+        // Serialised so the fingerprint comparison and snapshot swap cannot interleave between
+        // the timer and admin-triggered refreshes.
+        await _refreshGate.WaitAsync(cancellationToken);
         try
         {
             using var scope = scopeFactory.CreateScope();
@@ -72,8 +87,11 @@ public async Task RefreshAsync(CancellationToken cancellationToken = default)
 
             var newTenantsById = new Dictionary<Guid, TenantConfiguration>();
             var newTenantsByOrigin = new Dictionary<string, TenantConfiguration>(StringComparer.OrdinalIgnoreCase);
+            using var fingerprint = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
 
-            foreach (var entity in tenantEntities)
+            // Deterministic order: the query has no ORDER BY, and both the fingerprint and the
+            // last-wins origin index must not depend on whatever order the database returns.
+            foreach (var entity in tenantEntities.OrderBy(t => t.Id))
             {
                 var configPairs = new List<KeyValuePair<string, string?>>();
 
@@ -122,11 +140,24 @@ public async Task RefreshAsync(CancellationToken cancellationToken = default)
                 {
                     newTenantsByOrigin[origin] = tenantConfig;
                 }
+
+                AppendTenantToFingerprint(fingerprint, entity.Id, entity.Name, configPairs, origins);
+            }
+
+            var newFingerprint = Convert.ToHexString(fingerprint.GetHashAndReset());
+            _lastRefreshedUtc = DateTimeOffset.UtcNow;
+
+            if (string.Equals(newFingerprint, _snapshotFingerprint, StringComparison.Ordinal))
+            {
+                logger.LogDebug(
+                    "Tenant configuration unchanged ({TenantCount} active tenants for target '{Target}'); keeping current snapshot",
+                    newTenantsById.Count, targetApplication);
+                return;
             }
 
             _tenantsById = newTenantsById;
             _tenantsByOrigin = newTenantsByOrigin;
-            _lastRefreshedUtc = DateTimeOffset.UtcNow;
+            _snapshotFingerprint = newFingerprint;
 
             logger.LogInformation(
                 "Tenant configuration refreshed. Loaded {TenantCount} active tenants for target '{Target}'",
@@ -142,6 +173,56 @@ public async Task RefreshAsync(CancellationToken cancellationToken = default)
             logger.LogError(ex,
                 "Failed to refresh tenant configuration from database. Retaining previous configuration");
         }
+        finally
+        {
+            _refreshGate.Release();
+        }
+    }
+
+    /// <summary>
+    /// Feeds everything that ends up in a <see cref="TenantConfiguration"/> into the snapshot
+    /// fingerprint. Values are length-prefixed so adjacent fields cannot run into each other,
+    /// and null is distinguished from empty string.
+    /// </summary>
+    private static void AppendTenantToFingerprint(
+        IncrementalHash hash,
+        Guid tenantId,
+        string tenantName,
+        List<KeyValuePair<string, string?>> configPairs,
+        string[] origins)
+    {
+        AppendFingerprintValue(hash, tenantId.ToString("N"));
+        AppendFingerprintValue(hash, tenantName);
+
+        // Stable sort: duplicate keys keep their relative order, which decides last-wins.
+        AppendFingerprintValue(hash, configPairs.Count.ToString());
+        foreach (var pair in configPairs.OrderBy(p => p.Key, StringComparer.Ordinal))
+        {
+            AppendFingerprintValue(hash, pair.Key);
+            AppendFingerprintValue(hash, pair.Value);
+        }
+
+        AppendFingerprintValue(hash, origins.Length.ToString());
+        foreach (var origin in origins.Order(StringComparer.OrdinalIgnoreCase))
+        {
+            AppendFingerprintValue(hash, origin);
+        }
+    }
+
+    private static void AppendFingerprintValue(IncrementalHash hash, string? value)
+    {
+        Span<byte> length = stackalloc byte[sizeof(int)];
+        if (value is null)
+        {
+            BitConverter.TryWriteBytes(length, -1);
+            hash.AppendData(length);
+            return;
+        }
+
+        var bytes = Encoding.UTF8.GetBytes(value);
+        BitConverter.TryWriteBytes(length, bytes.Length);
+        hash.AppendData(length);
+        hash.AppendData(bytes);
     }
 
     /// <inheritdoc />