From 15a8a3fb76fde3a1d75df94a73900c7aefe248c1 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 15:51:07 +1000 Subject: [PATCH 1/9] Report audit instance environment data in the usage report The audit instance serves host and storage facts on a new anonymous GET /api/environment endpoint, gathered from audit-local environment data providers. The primary polls every remote alongside the daily audit verification pass through the new IConfigurationApi.GetRemoteEnvironments, stores the per-instance dictionaries as AuditEnvironmentMetadata in the licensing stores, and the usage report emits them under the Audit.* prefix aggregated across instances: agreement passes through, processor count and memory take the maximum, and disagreement reports Mixed. Remotes that predate the endpoint are omitted rather than guessed. --- .../AuditEnvironmentMetadata.cs | 8 ++ .../InMemoryLicensingDataStore.cs | 9 ++ .../ILicensingDataStore.cs | 4 + .../AuditQuery_Tests.cs | 6 ++ ...tThroughputCollectorHostedService_Tests.cs | 46 ++++++++ .../Infrastructure/FakeConfigurationApi.cs | 1 + .../MonitoringService_Tests.cs | 6 ++ ...tor_Report_EnvironmentInformation_Tests.cs | 53 ++++++++++ .../AuditEnvironmentDataAggregator.cs | 51 +++++++++ .../AuditThroughput/AuditQuery.cs | 19 ++++ .../AuditThroughputCollectorHostedService.cs | 7 ++ .../AuditThroughput/IAuditQuery.cs | 1 + .../ThroughputCollector.cs | 10 ++ .../Contracts/RemoteEnvironment.cs | 14 +++ src/ServiceControl.Api/IConfigurationApi.cs | 2 + .../When_environment_data_is_requested.cs | 45 ++++++++ .../RavenEnvironmentDataProvider.cs | 45 ++++++++ .../RavenPersistence.cs | 1 + .../EnvironmentData/DatabaseHostClassifier.cs | 77 ++++++++++++++ .../EnvironmentData/DatabaseHostingSource.cs | 21 ++++ .../EnvironmentData/EnvironmentDatum.cs | 38 +++++++ .../IEnvironmentDataProvider.cs | 13 +++ .../APIApprovals.HttpApiRoutes.approved.txt | 1 + .../EnvironmentControllerTests.cs | 100 ++++++++++++++++++ .../HostApplicationBuilderExtensions.cs | 1 + .../HostEnvironmentDataProvider.cs | 67 ++++++++++++ .../WebApi/EnvironmentController.cs | 64 +++++++++++ .../Implementation/LicensingDataStore.cs | 6 ++ .../Infrastructure/SettingKeys.cs | 1 + .../Throughput/LicensingDataStore.cs | 18 ++++ .../Infrastructure/Api/ConfigurationApi.cs | 52 +++++++++ 31 files changed, 787 insertions(+) create mode 100644 src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs create mode 100644 src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs create mode 100644 src/ServiceControl.Api/Contracts/RemoteEnvironment.cs create mode 100644 src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs create mode 100644 src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs create mode 100644 src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs create mode 100644 src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs create mode 100644 src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs create mode 100644 src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs create mode 100644 src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs create mode 100644 src/ServiceControl.Audit/HostEnvironmentDataProvider.cs create mode 100644 src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs diff --git a/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs new file mode 100644 index 0000000000..416fad3b72 --- /dev/null +++ b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs @@ -0,0 +1,8 @@ +namespace Particular.LicensingComponent.Contracts; + +/// +/// The environment data each audit remote served when last polled, one dictionary per remote that +/// responded. Never saved by versions that predate it, so a null read means unknown and the report +/// omits the keys. +/// +public record AuditEnvironmentMetadata(List> Instances); diff --git a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs index 257eef677a..611611fdee 100644 --- a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs @@ -14,6 +14,7 @@ public class InMemoryLicensingDataStore : ILicensingDataStore readonly Dictionary allThroughput = []; BrokerMetadata brokerMetadata = new(null, []); AuditServiceMetadata auditServiceMetadata = new([], []); + AuditEnvironmentMetadata? auditEnvironmentMetadata; List reportMasks = []; LicensedEndpointDetails? endpointDetails = null; @@ -167,6 +168,14 @@ public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, return Task.CompletedTask; } + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => Task.FromResult(auditEnvironmentMetadata); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) + { + this.auditEnvironmentMetadata = auditEnvironmentMetadata; + return Task.CompletedTask; + } + public Task> GetReportMasks(CancellationToken cancellationToken = default) => Task.FromResult(reportMasks); public Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default) diff --git a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs index 9dc65e1caf..bbf997ecf5 100644 --- a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs @@ -36,6 +36,10 @@ Task RecordEndpointThroughput(string endpointName, ThroughputSource throughputSo Task GetAuditServiceMetadata(CancellationToken cancellationToken = default); Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default); + + Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default); + + Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default); Task> GetReportMasks(CancellationToken cancellationToken = default); Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default); diff --git a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs index 082fe266ee..b64a0c12c8 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs @@ -192,6 +192,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati } public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi @@ -201,6 +203,8 @@ class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => Task.FromResult(remotes); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_Configurable : IConfigurationApi @@ -221,6 +225,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public bool ReturnAuditConfig { get; set; } public string RemoteStatus { get; set; } public string RemoteVersion { get; set; } diff --git a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs index a77b84783a..3174f5bea7 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs @@ -272,10 +272,40 @@ public async Task Should_store_how_many_audit_instances_are_configured_and_how_m } } + [Test] + public async Task Should_store_the_environment_data_of_every_responding_audit_instance() + { + var auditQuery = new AuditQuery_WithRemotes([new RemoteInstanceInformation { Status = "online" }]) + { + Environments = + [ + new Dictionary { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "4" }, + new Dictionary { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "8" } + ] + }; + + using var auditThroughputCollectorHostedService = new AuditThroughputCollectorHostedService( + NullLogger.Instance, configuration.ThroughputSettings, DataStore, + auditQuery, new FakeTimeProvider()) + { DelayStart = TimeSpan.Zero }; + + await auditThroughputCollectorHostedService.StartAsync(CancellationToken.None); + await auditQuery.KnownEndpointsRequested.WaitAsync(TimeSpan.FromSeconds(30)); + await auditThroughputCollectorHostedService.StopAsync(CancellationToken.None); + + var auditEnvironmentMetadata = await DataStore.GetAuditEnvironmentMetadata(); + + Assert.That(auditEnvironmentMetadata, Is.Not.Null); + Assert.That(auditEnvironmentMetadata.Instances, Is.EqualTo(auditQuery.Environments)); + } + class AuditQuery_NoAuditRemotes : IAuditQuery { public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -309,6 +339,9 @@ public AuditQuery_WithOneEndpoint(string endpointName, long throughputCount, Dat public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -341,6 +374,9 @@ class AuditQuery_ThrowingAnExceptionOnKnownEndpointsCall : IAuditQuery { public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -420,6 +456,10 @@ public AuditQuery_WithTwoEndpointsAndSelectiveCounts( } public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => _ => true; public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => @@ -456,6 +496,12 @@ class AuditQuery_WithRemotes(List remotes) : IAuditQu public Task KnownEndpointsRequested => knownEndpointsRequested.Task; public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + + public List> Environments { get; init; } = []; + + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult(Environments); + public Func ValidRemoteInstances => _ => true; public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => diff --git a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs index d7d0893be2..0e18e277ae 100644 --- a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs +++ b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs @@ -10,6 +10,7 @@ class FakeConfigurationApi : IConfigurationApi { public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); } } diff --git a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs index a4d39e1b1c..f6d1e5651c 100644 --- a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs @@ -252,6 +252,12 @@ public Task GetAuditServiceMetadata(CancellationToken canc public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => throw new NotSupportedException(); diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index b91bac763e..d651315558 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -228,4 +228,57 @@ await DataStore.SaveAuditServiceMetadata(new AuditServiceMetadata( Assert.That(environmentData, Does.Not.ContainKey("Audit.LiveInstances")); } } + + [Test] + public async Task Should_report_audit_environment_data_aggregated_across_instances() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = "6.2.1", + ["Host.ProcessorCount"] = "4", + ["Host.AvailableMemoryGB"] = "16", + ["Host.OSPlatform"] = "Linux" + }, + new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = "5.4.200", + ["Host.ProcessorCount"] = "8", + ["Host.AvailableMemoryGB"] = "Unknown" + } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB")); + Assert.That(environmentData, Does.ContainKey("Audit.Storage.ServerVersion").WithValue("Mixed")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.ProcessorCount").WithValue("8")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.AvailableMemoryGB").WithValue("Mixed")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.OSPlatform").WithValue("Linux")); + } + } + + [Test] + public async Task Should_leave_audit_environment_data_out_when_it_was_never_collected() + { + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); + } + + [Test] + public async Task Should_leave_audit_environment_data_out_when_no_instance_responded() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); + } } \ No newline at end of file diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs new file mode 100644 index 0000000000..de5111a5d5 --- /dev/null +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -0,0 +1,51 @@ +namespace Particular.LicensingComponent; + +using System.Globalization; +using Contracts; + +/// +/// Folds the environment data of every responding audit instance into one value per key: values +/// that agree pass through, capacity numbers take the maximum, and anything else that differs +/// reports Mixed rather than guessing which instance is representative. +/// +static class AuditEnvironmentDataAggregator +{ + static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB"]; + + public static IEnumerable> Aggregate(List> instances) + { + if (instances is null) + { + yield break; + } + + var keys = instances.SelectMany(instance => instance.Keys).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal); + + foreach (var key in keys) + { + var values = instances.Where(instance => instance.ContainsKey(key)).Select(instance => instance[key]).ToArray(); + + yield return new(key, Aggregate(key, values)); + } + } + + static string Aggregate(string key, string[] values) + { + if (values.Distinct(StringComparer.Ordinal).Count() == 1) + { + return values[0]; + } + + if (MaximumKeys.Contains(key, StringComparer.Ordinal)) + { + var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); + + if (parsed.All(value => value.Parsed)) + { + return parsed.Max(value => value.Number).ToString(CultureInfo.InvariantCulture); + } + } + + return "Mixed"; + } +} diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs index e048619e34..d3f633d55f 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs @@ -114,6 +114,25 @@ public async Task> GetAuditRemotes(CancellationT } } + public async Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) + { + try + { + var remoteEnvironments = await configurationApi.GetRemoteEnvironments(cancellationToken); + + return [.. remoteEnvironments.Where(remote => remote.EnvironmentData is not null).Select(remote => remote.EnvironmentData)]; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get Audit Environments"); + return []; + } + } + public async Task TestAuditConnection(CancellationToken cancellationToken = default) { var connectionTestResult = new ConnectionSettingsTestResult { ConnectionSuccessful = true }; diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs index 9e46eb2f4c..9d5398382f 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs @@ -119,6 +119,7 @@ async Task VerifyAuditInstances(CancellationToken cancellationToken) { var remotesInfo = await auditQuery.GetAuditRemotes(cancellationToken); await SaveAuditInstanceData(remotesInfo, cancellationToken); + await SaveAuditEnvironmentData(cancellationToken); foreach (var remote in remotesInfo) { @@ -164,4 +165,10 @@ async Task SaveAuditInstanceData(List? auditRemotes, await dataStore.SaveAuditServiceMetadata(auditServiceMetadata, cancellationToken); } } + + async Task SaveAuditEnvironmentData(CancellationToken cancellationToken) + { + var environments = await auditQuery.GetAuditEnvironments(cancellationToken); + await dataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(environments), cancellationToken); + } } \ No newline at end of file diff --git a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs index 66f03b7f35..cf0252dfd1 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs @@ -12,6 +12,7 @@ public interface IAuditQuery Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default); Task> GetAuditRemotes(CancellationToken cancellationToken = default); + Task>> GetAuditEnvironments(CancellationToken cancellationToken = default); Task TestAuditConnection(CancellationToken cancellationToken = default); } diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index e2186f6dda..d979c56bd1 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -197,6 +197,16 @@ public async Task GenerateThroughputReport(string spVersion, DateT report.EnvironmentInformation.EnvironmentData["Audit.LiveInstances"] = liveAuditInstances.ToString(CultureInfo.InvariantCulture); } + var auditEnvironmentMetadata = await dataStore.GetAuditEnvironmentMetadata(cancellationToken); + + if (auditEnvironmentMetadata is not null) + { + foreach (var (key, value) in AuditEnvironmentDataAggregator.Aggregate(auditEnvironmentMetadata.Instances)) + { + report.EnvironmentInformation.EnvironmentData["Audit." + key] = value; + } + } + foreach (var environmentDataProvider in environmentDataProviders) { EnvironmentDatum[] environmentData; diff --git a/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs b/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs new file mode 100644 index 0000000000..f449c326f1 --- /dev/null +++ b/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Api.Contracts +{ + using System.Collections.Generic; + + public class RemoteEnvironment + { + public string ApiUri { get; set; } + + /// + /// Null when the remote could not be reached or predates the environment endpoint. + /// + public Dictionary EnvironmentData { get; set; } + } +} diff --git a/src/ServiceControl.Api/IConfigurationApi.cs b/src/ServiceControl.Api/IConfigurationApi.cs index 6a2dc1ad7c..b16c977437 100644 --- a/src/ServiceControl.Api/IConfigurationApi.cs +++ b/src/ServiceControl.Api/IConfigurationApi.cs @@ -11,5 +11,7 @@ public interface IConfigurationApi Task GetConfig(CancellationToken cancellationToken = default); Task GetRemoteConfigs(CancellationToken cancellationToken = default); + + Task GetRemoteEnvironments(CancellationToken cancellationToken = default); } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs new file mode 100644 index 0000000000..7b842aa729 --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Audit.AcceptanceTests +{ + using System.Text.Json.Nodes; + using System.Threading.Tasks; + using AcceptanceTesting; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + + class When_environment_data_is_requested : AcceptanceTest + { + [Test] + public async Task Should_serve_host_and_storage_facts() + { + JsonObject environmentData = null; + + _ = await Define() + .Done(async ctx => + { + var response = await this.GetRaw("/api/environment"); + + if (!response.IsSuccessStatusCode) + { + return false; + } + + var body = await JsonNode.ParseAsync(await response.Content.ReadAsStreamAsync()); + environmentData = body?["environment_data"]?.AsObject(); + + return environmentData is not null; + }) + .Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That((string)environmentData["Storage.Type"], Is.Not.Null.And.Not.Empty); + Assert.That((string)environmentData["Storage.ServerVersion"], Is.Not.Null.And.Not.Empty); + Assert.That((string)environmentData["Storage.FullTextSearch"], Is.AnyOf("Enabled", "Disabled")); + Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); + Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); + } + } + + class Context : ScenarioContext; + } +} diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs new file mode 100644 index 0000000000..42f9ac2646 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Audit.Persistence.RavenDB +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.ServerWide.Operations; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class RavenEnvironmentDataProvider(DatabaseConfiguration databaseConfiguration, IRavenDocumentStoreProvider documentStoreProvider) : IEnvironmentDataProvider + { + public IEnumerable GetData() => + [ + Value("Storage.Type", () => "RavenDB"), + Value("Storage.RavenServer", () => databaseConfiguration.ServerConfiguration.UseEmbeddedServer ? "Embedded" : "External"), + Value("Storage.Hosting", () => Hosting().Hosting), + Deferred("Storage.ServerVersion", ServerVersion), + Value("Storage.HostingSource", () => Hosting().Source), + Value("Storage.FullTextSearch", () => databaseConfiguration.EnableFullTextSearch ? "Enabled" : "Disabled") + ]; + + (string Hosting, string Source) Hosting() + { + var serverConfiguration = databaseConfiguration.ServerConfiguration; + + // An embedded server runs in this process, so there is nothing to infer. + if (serverConfiguration.UseEmbeddedServer) + { + return (DatabaseHostClassifier.SelfHosted, DatabaseHostingSource.Configuration); + } + + return Uri.TryCreate(serverConfiguration.ConnectionString, UriKind.Absolute, out var url) + ? (DatabaseHostClassifier.Classify(url.Host), DatabaseHostingSource.ConnectionString) + : (DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); + } + + async ValueTask ServerVersion(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var buildNumber = await documentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + return buildNumber.ProductVersion ?? DatabaseHostClassifier.Unknown; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs index a02107642e..d60db4af94 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs @@ -30,6 +30,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); } diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs new file mode 100644 index 0000000000..3389a5d07f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs @@ -0,0 +1,77 @@ +#nullable enable +namespace ServiceControl.Audit.Persistence +{ + using System; + using System.Linq; + + /// + /// Classifies a database host name into a managed-service category. Only the category is ever + /// reported; the host name itself never leaves the process. Copied from the primary instance's + /// ServiceControl.Persistence on purpose: the audit persistence stack does not share projects + /// with the primary's. + /// + /// + /// This is the fallback for servers that cannot identify their own hosting. Where the engine can be + /// asked directly, as SQL Server can through EngineEdition, that answer wins over this one. + /// + public static class DatabaseHostClassifier + { + public const string Unknown = "Unknown"; + public const string SelfHosted = "SelfHosted"; + + /// + /// A host name is only ever evidence of a managed service or of a server on this machine. + /// Anything else, a private DNS name above all, is unknown rather than self-hosted: a customer + /// who fronts a managed database with their own DNS must not be counted as self-hosting it. + /// + public static string Classify(string? host) + { + if (string.IsNullOrWhiteSpace(host)) + { + return Unknown; + } + + var normalized = host.Trim().ToLowerInvariant(); + + foreach (var (suffix, hosting) in ManagedHostSuffixes) + { + if (normalized.EndsWith(suffix, StringComparison.Ordinal) || normalized.Contains($"{suffix},", StringComparison.Ordinal)) + { + return hosting; + } + } + + // Cloud SQL is reached either through a host name or through a unix socket directory named + // after the instance, so neither end of the value is a reliable place to look. + if (normalized.Contains("cloudsql", StringComparison.Ordinal)) + { + return "GoogleCloudSql"; + } + + return IsThisMachine(normalized) ? SelfHosted : Unknown; + } + + static bool IsThisMachine(string normalized) => + LocalHosts.Contains(normalized) || + normalized.StartsWith("(localdb)", StringComparison.Ordinal) || + normalized.StartsWith("np:", StringComparison.Ordinal) || + normalized.StartsWith("lpc:", StringComparison.Ordinal) || + normalized.StartsWith('/'); // a unix socket directory, which only a server on this machine listens on + + static readonly string[] LocalHosts = ["localhost", "127.0.0.1", "::1", "[::1]", ".", "(local)"]; + + // Ordered longest-suffix-first so that the more specific Azure services win over the shared + // .database.azure.com suffix. + static readonly (string Suffix, string Hosting)[] ManagedHostSuffixes = + [ + (".postgres.database.azure.com", "AzurePostgres"), + (".mysql.database.azure.com", "AzureMySql"), + (".database.windows.net", "AzureSql"), + (".database.azure.com", "AzureSql"), + (".rds.amazonaws.com", "AwsRds"), + (".ravendb.cloud", "RavenCloud"), + (".development.run", "RavenCloud"), + (".gcp.cloud", "GoogleCloudSql") + ]; + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs new file mode 100644 index 0000000000..fdc0adcddc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence +{ + /// + /// Where a reported database hosting classification came from, so that a guess can be told apart + /// from an answer when the reports are analysed. + /// + public static class DatabaseHostingSource + { + /// The server itself said so, and is therefore authoritative. + public const string Probe = "Probe"; + + /// The configuration says so outright, with nothing left to infer. + public const string Configuration = "Configuration"; + + /// Inferred from the configured host name because the server could not be asked. + public const string ConnectionString = "ConnectionString"; + + /// Nothing was available to classify. + public const string None = "None"; + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs new file mode 100644 index 0000000000..ed9697f588 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Audit.Persistence +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// One key of the environment data served to the primary instance, together with how to read + /// its value. + /// + /// + /// The value is deferred rather than supplied so that reading it can be isolated. A datum that + /// cannot be read costs only its own key, never a sibling's, and providers therefore carry no + /// error handling of their own. + /// + public sealed record EnvironmentDatum(string Key, Func> ReadValue) + { + /// + /// Served in place of a value whose read threw. Deliberately not a word that could pass for + /// a state the instance is legitimately in: it always means the read failed, never that the + /// thing being described is absent or switched off. + /// + public const string ReadFailed = "ReadFailed"; + + /// + /// A value that is already at hand, such as one read from configuration. Still deferred, so + /// that nothing is evaluated while a provider is listing what it offers. + /// + public static EnvironmentDatum Value(string key, Func readValue) => + new(key, _ => new ValueTask(readValue())); + + /// + /// A value that has to be fetched, such as one read from storage or from the database itself. + /// + public static EnvironmentDatum Deferred(string key, Func> readValue) => + new(key, readValue); + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs new file mode 100644 index 0000000000..f55677c0fb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence +{ + using System.Collections.Generic; + + /// + /// Provides environment data that the primary instance polls over the environment endpoint and + /// folds into its usage report. + /// + public interface IEnvironmentDataProvider + { + IEnumerable GetData(); + } +} diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 3e218d68a5..97509ec622 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -7,6 +7,7 @@ GET /endpoints/{endpoint}/messages => ServiceControl.Audit.Auditing.MessagesView GET /endpoints/{endpoint}/messages/search => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Search(PagingInfo pagingInfo, SortInfo sortInfo, String endpoint, String q, CancellationToken cancellationToken) GET /endpoints/{endpoint}/messages/search/{keyword} => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:SearchByKeyword(PagingInfo pagingInfo, SortInfo sortInfo, String endpoint, String keyword, CancellationToken cancellationToken) GET /endpoints/known => ServiceControl.Audit.Monitoring.KnownEndpointsController:GetAll(PagingInfo pagingInfo) +GET /environment => ServiceControl.Audit.Infrastructure.WebApi.EnvironmentController:Environment(CancellationToken cancellationToken) GET /instance-info => ServiceControl.Audit.Infrastructure.WebApi.RootController:Config() GET /messages => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, Boolean includeSystemMessages, CancellationToken cancellationToken) GET /messages/{id}/body => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Get(String id, CancellationToken cancellationToken) diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs new file mode 100644 index 0000000000..ef5fc92046 --- /dev/null +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -0,0 +1,100 @@ +namespace ServiceControl.Audit.UnitTests.Infrastructure +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Logging.Abstractions; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure.WebApi; + using ServiceControl.Audit.Persistence; + + [TestFixture] + class EnvironmentControllerTests + { + [Test] + public async Task Merges_the_data_of_every_provider() + { + var controller = new EnvironmentController( + [ + new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB")), + new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) + ], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Host.ProcessorCount"] = "8" + })); + } + + [Test] + public async Task A_datum_that_cannot_be_read_costs_only_its_own_key() + { + var controller = new EnvironmentController( + [ + new Provider( + EnvironmentDatum.Value("Storage.Type", () => "RavenDB"), + EnvironmentDatum.Deferred("Storage.ServerVersion", _ => throw new InvalidOperationException("server down"))) + ], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = EnvironmentDatum.ReadFailed + })); + } + + [Test] + public async Task A_provider_that_cannot_list_its_data_costs_only_its_own_keys() + { + var controller = new EnvironmentController( + [ + new ThrowingProvider(), + new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) + ], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Host.ProcessorCount"] = "8" + })); + } + + [Test] + public void Cancellation_propagates() + { + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + var controller = new EnvironmentController( + [ + new Provider(EnvironmentDatum.Deferred("Storage.ServerVersion", cancellationToken => + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask("unreached"); + })) + ], NullLogger.Instance); + + Assert.ThrowsAsync(() => controller.Environment(cancellationTokenSource.Token)); + } + + class Provider(params EnvironmentDatum[] data) : IEnvironmentDataProvider + { + public IEnumerable GetData() => data; + } + + class ThrowingProvider : IEnvironmentDataProvider + { + public IEnumerable GetData() => throw new InvalidOperationException("cannot list"); + } + } +} diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index ccd2c635cc..da57b3350f 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -51,6 +51,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, services.AddSingleton(configuration); services.AddSingleton(settings); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); // required by the ingestion custom check which is auto-loaded diff --git a/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs b/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs new file mode 100644 index 0000000000..31032d0773 --- /dev/null +++ b/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Audit +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Runtime.InteropServices; + using Microsoft.Extensions.Hosting.WindowsServices; + using ServiceControl.Audit.Persistence; + using ServiceControl.Configuration; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class HostEnvironmentDataProvider : IEnvironmentDataProvider + { + public IEnumerable GetData() => + [ + Value("Host.Model", HostModel), + Value("Host.Orchestrator", Orchestrator), + Value("Host.OSPlatform", OSPlatformName), + Value("Host.OSVersion", () => $"{Environment.OSVersion.Version.Major}.{Environment.OSVersion.Version.Minor}"), + Value("Host.Architecture", () => RuntimeInformation.ProcessArchitecture.ToString()), + Value("Host.RuntimeVersion", () => Environment.Version.ToString(3)), + Value("Host.ProcessorCount", () => Environment.ProcessorCount.ToString(CultureInfo.InvariantCulture)), + Value("Host.AvailableMemoryGB", AvailableMemoryGB) + ]; + + static string Orchestrator() => Environment.GetEnvironmentVariable(KubernetesServiceHostVariable) is not null ? "Kubernetes" : "None"; + + static string HostModel() + { + if (AppEnvironment.RunningInContainer) + { + return "Container"; + } + + return WindowsServiceHelpers.IsWindowsService() ? "WindowsService" : "Console"; + } + + static string OSPlatformName() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "Windows"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "Linux"; + } + + return RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "macOS" : "Unknown"; + } + + // TotalAvailableMemoryBytes honours the cgroup limit, so a container reports the memory it is + // limited to rather than the physical memory of the machine hosting it. + static string AvailableMemoryGB() + { + var totalAvailableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + return totalAvailableMemoryBytes <= 0 + ? "Unknown" + : Math.Round(totalAvailableMemoryBytes / (double)BytesPerGigabyte, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); + } + + const string KubernetesServiceHostVariable = "KUBERNETES_SERVICE_HOST"; + const long BytesPerGigabyte = 1024L * 1024 * 1024; + } +} diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs new file mode 100644 index 0000000000..217bfbb264 --- /dev/null +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Audit.Infrastructure.WebApi +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.AspNetCore.Authorization; + using Microsoft.AspNetCore.Mvc; + using Microsoft.Extensions.Logging; + using ServiceControl.Audit.Persistence; + + // the /api/environment endpoint is polled by the primary instance for its usage report. This currently needs to be anonymous + [AllowAnonymous] + [ApiController] + [Route("api")] + public class EnvironmentController(IEnumerable providers, ILogger logger) : ControllerBase + { + [Route("environment")] + [HttpGet] + public async Task Environment(CancellationToken cancellationToken = default) + { + var environmentData = new Dictionary(); + + foreach (var provider in providers) + { + EnvironmentDatum[] data; + + try + { + data = [.. provider.GetData()]; + } + catch (Exception e) + { + logger.LogWarning(e, "Environment data provider {EnvironmentDataProvider} could not list what it offers, so none of its data is served", provider.GetType().Name); + continue; + } + + foreach (var datum in data) + { + try + { + environmentData[datum.Key] = await datum.ReadValue(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogWarning(e, "Environment datum {EnvironmentDatum} could not be read", datum.Key); + environmentData[datum.Key] = EnvironmentDatum.ReadFailed; + } + } + } + + return Ok(new EnvironmentDataResponse { EnvironmentData = environmentData }); + } + + public class EnvironmentDataResponse + { + public Dictionary EnvironmentData { get; set; } + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index 2683c5069c..e57519e46b 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -303,6 +303,12 @@ public Task GetAuditServiceMetadata(CancellationToken canc public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default) => ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditServiceMetadata, auditServiceMetadata, token), cancellationToken); + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.GetSetting(SettingKeys.AuditEnvironmentMetadata, token), cancellationToken); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditEnvironmentMetadata, auditEnvironmentMetadata, token), cancellationToken); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => ExecuteWithDbContext(async (context, token) => await context.GetSetting>(SettingKeys.ReportMasks, token) ?? [], cancellationToken); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index 7a8e403170..5f01a2ddf9 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -6,6 +6,7 @@ static class SettingKeys public const string TrialEndDate = "TrialEndDate"; public const string BrokerMetadata = "BrokerMetadata"; public const string AuditServiceMetadata = "AuditServiceMetadata"; + public const string AuditEnvironmentMetadata = "AuditEnvironmentMetadata"; public const string ReportMasks = "ReportMasks"; public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string NotificationEmails = "NotificationEmails"; diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 6e14a20845..4508e85843 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -22,6 +22,7 @@ class LicensingDataStore( { internal const string ThroughputTimeSeriesName = "INC: throughput data"; const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; + const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; const string BrokerMetadataDocumentId = "BrokerMetadata"; const string ReportMasksDocumentId = "ReportMasks"; const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; @@ -318,6 +319,23 @@ public async Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMeta await session.SaveChangesAsync(cancellationToken); } + public async Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + return await session.LoadAsync(AuditEnvironmentMetadataDocumentId, cancellationToken); + } + + public async Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + await session.StoreAsync(auditEnvironmentMetadata, AuditEnvironmentMetadataDocumentId, cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + public async Task> GetReportMasks(CancellationToken cancellationToken = default) { var store = await storeProvider.GetDocumentStore(cancellationToken); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4d61e23ca8..4a91b06f6b 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Infrastructure.Api; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -139,4 +140,55 @@ public async Task GetRemoteConfigs(CancellationToken canc return results; } + + public async Task GetRemoteEnvironments(CancellationToken cancellationToken = default) + { + var tasks = settings.RemoteInstances + .Select(async remote => + { + Dictionary environmentData = null; + HttpClient httpClient = httpClientFactory.CreateClient(remote.InstanceId); + + try + { + using var response = await httpClient.GetAsync("/api/environment", cancellationToken); + + // An unreachable remote and one predating the endpoint both leave the data null, + // and the report omits their keys rather than guessing. + if (response.IsSuccessStatusCode) + { + await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); + var body = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); + + if (body?.AsObject().TryGetPropertyValue("environment_data", out var data) == true && data is JsonObject dataObject) + { + environmentData = []; + + foreach (var pair in dataObject) + { + if (pair.Value is not null) + { + environmentData[pair.Key] = pair.Value.GetValue(); + } + } + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + } + + return new RemoteEnvironment + { + ApiUri = remote.BaseAddress, + EnvironmentData = environmentData + }; + }); + + return await Task.WhenAll(tasks); + } } \ No newline at end of file From 1d60d48c81fef64a400588c936ebeab34197b51a Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 16:02:09 +1000 Subject: [PATCH 2/9] Detect whether audit instances share the primary's machine, database or server Each instance hashes what identifies the machine it runs on and the storage it writes to: the OS machine id on Linux, the machine name elsewhere, NotApplicable in containers, and per engine the RavenDB server URL and database, SQL Server's ServerName and database, or PostgreSQL's cluster system identifier and database, plus the schema. The audit instance serves the hashes on the environment endpoint, the primary compares them with its own during collection, and only the classification reaches the report: Audit.SameMachine as True, False, Mixed, NotApplicable or Unknown, and Audit.DatabaseSharing as SameSchema, SameDatabase, SameServer, SeparateServer, NotApplicable or Unknown, aggregated to the most shared class across instances. A side that cannot be identified reports Unknown rather than guessing from configuration the two sides may spell differently. --- ...tor_Report_EnvironmentInformation_Tests.cs | 20 ++++++ .../AuditEnvironmentDataAggregator.cs | 14 ++++ .../When_environment_data_is_requested.cs | 15 ++-- .../RavenPersistence.cs | 1 + .../RavenStorageIdentityProvider.cs | 20 ++++++ .../IStorageIdentityProvider.cs | 17 +++++ .../EnvironmentControllerTests.cs | 59 +++++++++++++-- .../WebApi/EnvironmentController.cs | 59 ++++++++++++++- .../IdentityHash.cs | 17 +++++ .../MachineIdentity.cs | 59 +++++++++++++++ .../PostgreSqlPersistence.cs | 1 + .../PostgreSqlStorageIdentityProvider.cs | 49 +++++++++++++ .../SqlServerPersistence.cs | 1 + .../SqlServerStorageIdentityProvider.cs | 47 ++++++++++++ .../RavenPersistence.cs | 1 + .../RavenStorageIdentityProvider.cs | 19 +++++ .../StorageIdentityProviderTests.cs | 26 +++++++ .../StorageIdentityProviderTests.cs | 26 +++++++ .../IStorageIdentityProvider.cs | 20 ++++++ .../API/APIApprovals.cs | 3 +- .../Licensing/AuditSharingClassifierTests.cs | 72 +++++++++++++++++++ .../Api/AuditSharingClassifier.cs | 71 ++++++++++++++++++ .../Infrastructure/Api/ConfigurationApi.cs | 54 +++++++++++++- 23 files changed, 659 insertions(+), 12 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Infrastructure/IdentityHash.cs create mode 100644 src/ServiceControl.Infrastructure/MachineIdentity.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs create mode 100644 src/ServiceControl.Persistence/IStorageIdentityProvider.cs create mode 100644 src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs create mode 100644 src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index d651315558..3744d652c4 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -264,6 +264,26 @@ await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( } } + [Test] + public async Task Should_report_the_most_shared_database_class_and_mixed_machines() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["DatabaseSharing"] = "SeparateServer", ["SameMachine"] = "True" }, + new Dictionary { ["DatabaseSharing"] = "SameServer", ["SameMachine"] = "False" }, + new Dictionary { ["DatabaseSharing"] = "Unknown", ["SameMachine"] = "True" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.DatabaseSharing").WithValue("SameServer")); + Assert.That(environmentData, Does.ContainKey("Audit.SameMachine").WithValue("Mixed")); + } + } + [Test] public async Task Should_leave_audit_environment_data_out_when_it_was_never_collected() { diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs index de5111a5d5..e0bb865b1a 100644 --- a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -11,6 +11,7 @@ namespace Particular.LicensingComponent; static class AuditEnvironmentDataAggregator { static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB"]; + static readonly string[] SharingPrecedence = ["SameSchema", "SameDatabase", "SameServer", "SeparateServer", "NotApplicable", "Unknown"]; public static IEnumerable> Aggregate(List> instances) { @@ -36,6 +37,19 @@ static string Aggregate(string key, string[] values) return values[0]; } + // The most-shared class across instances, because one shared database matters however many + // separate ones sit beside it. + if (key == "DatabaseSharing") + { + foreach (var sharing in SharingPrecedence) + { + if (values.Contains(sharing, StringComparer.Ordinal)) + { + return sharing; + } + } + } + if (MaximumKeys.Contains(key, StringComparer.Ordinal)) { var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 7b842aa729..6228193159 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -11,7 +11,7 @@ class When_environment_data_is_requested : AcceptanceTest [Test] public async Task Should_serve_host_and_storage_facts() { - JsonObject environmentData = null; + JsonObject body = null; _ = await Define() .Done(async ctx => @@ -23,13 +23,15 @@ public async Task Should_serve_host_and_storage_facts() return false; } - var body = await JsonNode.ParseAsync(await response.Content.ReadAsStreamAsync()); - environmentData = body?["environment_data"]?.AsObject(); + body = (await JsonNode.ParseAsync(await response.Content.ReadAsStreamAsync()))?.AsObject(); - return environmentData is not null; + return body?["environment_data"] is not null; }) .Run(); + var environmentData = body["environment_data"].AsObject(); + var storageIdentity = body["storage_identity"]?.AsObject(); + using (Assert.EnterMultipleScope()) { Assert.That((string)environmentData["Storage.Type"], Is.Not.Null.And.Not.Empty); @@ -37,6 +39,11 @@ public async Task Should_serve_host_and_storage_facts() Assert.That((string)environmentData["Storage.FullTextSearch"], Is.AnyOf("Enabled", "Disabled")); Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); + Assert.That((string)body["machine_id_hash"], Is.Not.Null.And.Not.Empty); + Assert.That(storageIdentity, Is.Not.Null); + Assert.That((string)storageIdentity["engine"], Is.Not.Null.And.Not.Empty); + Assert.That((string)storageIdentity["server_hash"], Has.Length.EqualTo(64)); + Assert.That((string)storageIdentity["database_hash"], Has.Length.EqualTo(64)); } } diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs index d60db4af94..553014885a 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs @@ -31,6 +31,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); } diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs new file mode 100644 index 0000000000..228a713b21 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.RavenDB +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + class RavenStorageIdentityProvider(DatabaseConfiguration databaseConfiguration) : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + var serverConfiguration = databaseConfiguration.ServerConfiguration; + var configuredServer = serverConfiguration.UseEmbeddedServer ? serverConfiguration.ServerUrl : serverConfiguration.ConnectionString; + + return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), databaseConfiguration.Name, null)); + } + + static string NormalizeServer(string configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) ? $"{url.Host}:{url.Port}" : configuredServer ?? ""; + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs new file mode 100644 index 0000000000..c928de711a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Audit.Persistence +{ + using System.Threading; + using System.Threading.Tasks; + + /// + /// Identifies the storage this instance writes to, so that the primary instance can tell + /// whether it shares a database or a database server with this one. The raw values never leave + /// the process; the environment endpoint serves hashes of them. + /// + public interface IStorageIdentityProvider + { + ValueTask GetIdentity(CancellationToken cancellationToken = default); + } + + public record StorageIdentity(string Engine, string Server, string Database, string Schema); +} diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs index ef5fc92046..13897866cd 100644 --- a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -8,6 +8,7 @@ namespace ServiceControl.Audit.UnitTests.Infrastructure using NUnit.Framework; using ServiceControl.Audit.Infrastructure.WebApi; using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure; [TestFixture] class EnvironmentControllerTests @@ -19,7 +20,7 @@ public async Task Merges_the_data_of_every_provider() [ new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB")), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -39,7 +40,7 @@ public async Task A_datum_that_cannot_be_read_costs_only_its_own_key() new Provider( EnvironmentDatum.Value("Storage.Type", () => "RavenDB"), EnvironmentDatum.Deferred("Storage.ServerVersion", _ => throw new InvalidOperationException("server down"))) - ], NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -58,7 +59,7 @@ public async Task A_provider_that_cannot_list_its_data_costs_only_its_own_keys() [ new ThrowingProvider(), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -82,16 +83,66 @@ public void Cancellation_propagates() cancellationToken.ThrowIfCancellationRequested(); return new ValueTask("unreached"); })) - ], NullLogger.Instance); + ], [], NullLogger.Instance); Assert.ThrowsAsync(() => controller.Environment(cancellationTokenSource.Token)); } + [Test] + public async Task Serves_the_hashed_storage_identity() + { + var controller = new EnvironmentController( + [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], + [new IdentityProvider(new StorageIdentity("RavenDB", "http://server:8080", "audit", null))], + NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + using (Assert.EnterMultipleScope()) + { + Assert.That(response.StorageIdentity.Engine, Is.EqualTo("RavenDB")); + Assert.That(response.StorageIdentity.ServerHash, Is.EqualTo(IdentityHash.Compute("http://server:8080"))); + Assert.That(response.StorageIdentity.DatabaseHash, Is.EqualTo(IdentityHash.Compute("audit"))); + Assert.That(response.StorageIdentity.SchemaHash, Is.Null); + Assert.That(response.MachineIdHash, Is.EqualTo(MachineIdentity.Hash)); + Assert.That(response.EnvironmentData.Keys, Has.None.Contains("Hash")); + } + } + + [Test] + public async Task A_storage_identity_that_cannot_be_read_is_not_served() + { + var controller = new EnvironmentController( + [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], + [new ThrowingIdentityProvider()], + NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + using (Assert.EnterMultipleScope()) + { + Assert.That(response.StorageIdentity, Is.Null); + Assert.That(response.EnvironmentData, Does.ContainKey("Storage.Type")); + } + } + class Provider(params EnvironmentDatum[] data) : IEnvironmentDataProvider { public IEnumerable GetData() => data; } + class IdentityProvider(StorageIdentity identity) : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) => new(identity); + } + + class ThrowingIdentityProvider : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) => throw new InvalidOperationException("storage down"); + } + class ThrowingProvider : IEnvironmentDataProvider { public IEnumerable GetData() => throw new InvalidOperationException("cannot list"); diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs index 217bfbb264..c13a838a74 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -6,14 +6,16 @@ namespace ServiceControl.Audit.Infrastructure.WebApi using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; + using System.Linq; using Microsoft.Extensions.Logging; using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure; // the /api/environment endpoint is polled by the primary instance for its usage report. This currently needs to be anonymous [AllowAnonymous] [ApiController] [Route("api")] - public class EnvironmentController(IEnumerable providers, ILogger logger) : ControllerBase + public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, ILogger logger) : ControllerBase { [Route("environment")] [HttpGet] @@ -53,12 +55,65 @@ public async Task Environment(CancellationToken cancellationToke } } - return Ok(new EnvironmentDataResponse { EnvironmentData = environmentData }); + return Ok(new EnvironmentDataResponse + { + EnvironmentData = environmentData, + MachineIdHash = MachineIdentity.Hash, + StorageIdentity = await ReadStorageIdentity(cancellationToken) + }); + } + + async Task ReadStorageIdentity(CancellationToken cancellationToken) + { + var provider = storageIdentityProviders.FirstOrDefault(); + + if (provider is null) + { + return null; + } + + try + { + var identity = await provider.GetIdentity(cancellationToken); + + if (identity is null) + { + return null; + } + + return new StorageIdentityResponse + { + Engine = identity.Engine, + ServerHash = IdentityHash.Compute(identity.Server), + DatabaseHash = IdentityHash.Compute(identity.Database), + SchemaHash = identity.Schema is null ? null : IdentityHash.Compute(identity.Schema) + }; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogWarning(e, "The storage identity could not be read, so it is not served"); + + return null; + } } public class EnvironmentDataResponse { public Dictionary EnvironmentData { get; set; } + public string MachineIdHash { get; set; } + public StorageIdentityResponse StorageIdentity { get; set; } + } + + public class StorageIdentityResponse + { + public string Engine { get; set; } + public string ServerHash { get; set; } + public string DatabaseHash { get; set; } + public string SchemaHash { get; set; } } } } diff --git a/src/ServiceControl.Infrastructure/IdentityHash.cs b/src/ServiceControl.Infrastructure/IdentityHash.cs new file mode 100644 index 0000000000..81f6a1242a --- /dev/null +++ b/src/ServiceControl.Infrastructure/IdentityHash.cs @@ -0,0 +1,17 @@ +#nullable enable +namespace ServiceControl.Infrastructure; + +using System; +using System.Security.Cryptography; +using System.Text; + +/// +/// Hashes an identity value so that two instances can compare what they are running on without +/// either side revealing the value itself. The hashes are exchanged between the customer's own +/// instances and never appear in a usage report; only the comparison result does. +/// +public static class IdentityHash +{ + public static string Compute(string value) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value.Trim().ToLowerInvariant()))); +} diff --git a/src/ServiceControl.Infrastructure/MachineIdentity.cs b/src/ServiceControl.Infrastructure/MachineIdentity.cs new file mode 100644 index 0000000000..b4665ceb26 --- /dev/null +++ b/src/ServiceControl.Infrastructure/MachineIdentity.cs @@ -0,0 +1,59 @@ +#nullable enable +namespace ServiceControl.Infrastructure; + +using System; +using System.IO; +using ServiceControl.Configuration; + +public static class MachineIdentity +{ + /// + /// Containers report this instead of a hash: every container has its own machine id, so two + /// containers on one host would never compare equal and the comparison would mislead. + /// + public const string NotApplicable = "NotApplicable"; + + /// + /// A stable hash of this machine's identity, in a container, or + /// null when nothing stable was available. The source is prefixed into the hashed value so that + /// identities from different sources can never compare equal by accident. + /// + public static string? Hash => hash.Value; + + static readonly Lazy hash = new(Compute); + + static string? Compute() + { + if (AppEnvironment.RunningInContainer) + { + return NotApplicable; + } + + if (OperatingSystem.IsLinux()) + { + foreach (var path in MachineIdPaths) + { + try + { + var machineId = File.ReadAllText(path).Trim(); + + if (machineId.Length > 0) + { + return IdentityHash.Compute("machine-id:" + machineId); + } + } + catch (Exception) + { + } + } + } + + // The machine name rather than the Windows MachineGuid, which only the registry serves and + // would cost this cross-platform assembly a Windows-only package dependency. + var machineName = Environment.MachineName; + + return string.IsNullOrWhiteSpace(machineName) ? null : IdentityHash.Compute("machine-name:" + machineName); + } + + static readonly string[] MachineIdPaths = ["/etc/machine-id", "/var/lib/dbus/machine-id"]; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 3cbe6c1549..51d31902be 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -22,6 +22,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs new file mode 100644 index 0000000000..79a275b2e8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence; +using ServiceControl.Persistence.EFCore.DbContexts; + +class PostgreSqlStorageIdentityProvider(PostgreSqlPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The cluster's initialisation identifier rather than an address: every connection to + // the same cluster sees the same value however the host is spelled or proxied. + command.CommandText = "SELECT system_identifier::text, current_database() FROM pg_control_system()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? DefaultSchema); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; + const string DefaultSchema = "public"; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index dead62d3cc..7063a20058 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -22,6 +22,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs new file mode 100644 index 0000000000..00016666ea --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence; +using ServiceControl.Persistence.EFCore.DbContexts; + +class SqlServerStorageIdentityProvider(SqlServerPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? DefaultSchema); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; + const string DefaultSchema = "dbo"; +} diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs index 95dc171a86..87cf50bc5e 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs @@ -40,6 +40,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(p => p.GetRequiredService()); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs new file mode 100644 index 0000000000..29f287d75a --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB; + +using System; +using System.Threading; +using System.Threading.Tasks; + +class RavenStorageIdentityProvider(RavenPersisterSettings settings) : IStorageIdentityProvider +{ + public ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + var configuredServer = settings.UseEmbeddedServer ? settings.ServerUrl : settings.ConnectionString; + + return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), settings.DatabaseName, null)); + } + + static string NormalizeServer(string configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) ? $"{url.Host}:{url.Port}" : configuredServer ?? ""; +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs new file mode 100644 index 0000000000..2bc2d5daac --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.PostgreSql; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageIdentityProviderTests : PersistenceTestBase +{ + [Test] + public async Task Identifies_the_cluster_database_and_schema() + { + var provider = ServiceProvider.GetRequiredService(); + + var identity = await provider.GetIdentity(); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo("PostgreSQL")); + Assert.That(identity.Server, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Database, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(((PostgreSqlPersisterSettings)PersistenceSettings).Schema)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs new file mode 100644 index 0000000000..74c7a53053 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.SqlServer; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageIdentityProviderTests : PersistenceTestBase +{ + [Test] + public async Task Identifies_the_server_database_and_schema() + { + var provider = ServiceProvider.GetRequiredService(); + + var identity = await provider.GetIdentity(); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo("SQLServer")); + Assert.That(identity.Server, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Database, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(((SqlServerPersisterSettings)PersistenceSettings).Schema)); + } + } +} diff --git a/src/ServiceControl.Persistence/IStorageIdentityProvider.cs b/src/ServiceControl.Persistence/IStorageIdentityProvider.cs new file mode 100644 index 0000000000..2c13a60a83 --- /dev/null +++ b/src/ServiceControl.Persistence/IStorageIdentityProvider.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Persistence; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// Identifies the storage this instance writes to, so that it can be compared with what an audit +/// remote reports over the environment endpoint. The raw values never leave the process; only +/// hashes are compared, and only the comparison result is reported. +/// +public interface IStorageIdentityProvider +{ + /// + /// Null when the storage could not be asked, which the comparison reports as Unknown rather + /// than guessing from configuration that the two sides may spell differently. + /// + ValueTask GetIdentity(CancellationToken cancellationToken = default); +} + +public record StorageIdentity(string Engine, string Server, string Database, string? Schema); diff --git a/src/ServiceControl.UnitTests/API/APIApprovals.cs b/src/ServiceControl.UnitTests/API/APIApprovals.cs index 04d17fc409..a7e3f298e1 100644 --- a/src/ServiceControl.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.UnitTests/API/APIApprovals.cs @@ -38,7 +38,8 @@ public async Task RootPathValue() new ActiveLicense(null, NullLogger.Instance) { IsValid = true }, new Settings(), null, - new MassTransitConnectorHeartbeatStatus()); + new MassTransitConnectorHeartbeatStatus(), + []); var controller = new RootController(configurationApi) { diff --git a/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs new file mode 100644 index 0000000000..2cc776bdad --- /dev/null +++ b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.Licensing; + +using NUnit.Framework; +using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Api; +using ServiceControl.Persistence; + +[TestFixture] +public class AuditSharingClassifierTests +{ + [TestCase(null, "a", "Unknown")] + [TestCase("a", null, "Unknown")] + [TestCase("NotApplicable", "a", "NotApplicable")] + [TestCase("a", "NotApplicable", "NotApplicable")] + [TestCase("a", "a", "True")] + [TestCase("a", "b", "False")] + public void Same_machine(string local, string remote, string expected) => + Assert.That(AuditSharingClassifier.SameMachine(local, remote), Is.EqualTo(expected)); + + [Test] + public void Sharing_is_unknown_without_both_identities() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(AuditSharingClassifier.DatabaseSharing(null, Identity()), Is.EqualTo("Unknown")); + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), null), Is.EqualTo("Unknown")); + } + } + + [Test] + public void Different_engines_cannot_share() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "PostgreSQL"), Identity(engine: "RavenDB")), Is.EqualTo("NotApplicable")); + + [Test] + public void Engine_comparison_ignores_case() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "SQLServer"), Identity(engine: "sqlserver")), Is.EqualTo("SameSchema")); + + [Test] + public void Different_servers_are_separate() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(server: "other")), Is.EqualTo("SeparateServer")); + + [Test] + public void Different_databases_on_one_server_share_the_server() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(database: "other")), Is.EqualTo("SameServer")); + + [Test] + public void Equal_schemas_share_the_schema() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity()), Is.EqualTo("SameSchema")); + + [Test] + public void Different_schemas_share_the_database() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(schema: "other")), Is.EqualTo("SameDatabase")); + + [Test] + public void Schemaless_engines_share_the_database_at_most() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "RavenDB", schema: null), Identity(engine: "RavenDB", schema: null)), Is.EqualTo("SameDatabase")); + + [Test] + public void Hashing_normalises_case_and_whitespace() + { + var hashed = AuditSharingClassifier.Hash(new StorageIdentity("SQLServer", " PRODSQL01 ", "ServiceControl", "dbo")); + + Assert.That(hashed.ServerHash, Is.EqualTo(IdentityHash.Compute("prodsql01"))); + } + + [Test] + public void Hashing_nothing_is_nothing() => + Assert.That(AuditSharingClassifier.Hash(null), Is.Null); + + static HashedStorageIdentity Identity(string engine = "SQLServer", string server = "server", string database = "database", string schema = "schema") => + new(engine, IdentityHash.Compute(server), IdentityHash.Compute(database), schema is null ? null : IdentityHash.Compute(schema)); +} diff --git a/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs new file mode 100644 index 0000000000..78aab5f92e --- /dev/null +++ b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs @@ -0,0 +1,71 @@ +namespace ServiceControl.Infrastructure.Api; + +using System; +using ServiceControl.Persistence; + +/// +/// Compares this instance's identity hashes with what an audit remote served and reduces them to +/// the classifications the usage report carries. Only these results are ever reported; the hashes +/// stay between the instances. +/// +static class AuditSharingClassifier +{ + public const string Unknown = "Unknown"; + public const string NotApplicable = "NotApplicable"; + + public static string SameMachine(string localMachineIdHash, string remoteMachineIdHash) + { + if (localMachineIdHash is null || remoteMachineIdHash is null) + { + return Unknown; + } + + if (localMachineIdHash == MachineIdentity.NotApplicable || remoteMachineIdHash == MachineIdentity.NotApplicable) + { + return NotApplicable; + } + + return string.Equals(localMachineIdHash, remoteMachineIdHash, StringComparison.Ordinal) ? "True" : "False"; + } + + public static string DatabaseSharing(HashedStorageIdentity local, HashedStorageIdentity remote) + { + if (local is null || remote is null) + { + return Unknown; + } + + if (!string.Equals(local.Engine, remote.Engine, StringComparison.OrdinalIgnoreCase)) + { + return NotApplicable; + } + + if (!string.Equals(local.ServerHash, remote.ServerHash, StringComparison.Ordinal)) + { + return "SeparateServer"; + } + + if (!string.Equals(local.DatabaseHash, remote.DatabaseHash, StringComparison.Ordinal)) + { + return "SameServer"; + } + + if (local.SchemaHash is not null && string.Equals(local.SchemaHash, remote.SchemaHash, StringComparison.Ordinal)) + { + return "SameSchema"; + } + + return "SameDatabase"; + } + + public static HashedStorageIdentity Hash(StorageIdentity identity) => + identity is null + ? null + : new HashedStorageIdentity( + identity.Engine, + IdentityHash.Compute(identity.Server), + IdentityHash.Compute(identity.Database), + identity.Schema is null ? null : IdentityHash.Compute(identity.Schema)); +} + +record HashedStorageIdentity(string Engine, string ServerHash, string DatabaseHash, string SchemaHash); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4a91b06f6b..4df58bfb86 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -14,8 +14,9 @@ using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api; using ServiceControl.Api.Contracts; +using ServiceControl.Persistence; -class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : IConfigurationApi +class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, IEnumerable storageIdentityProviders) : IConfigurationApi { public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) { @@ -143,6 +144,9 @@ public async Task GetRemoteConfigs(CancellationToken canc public async Task GetRemoteEnvironments(CancellationToken cancellationToken = default) { + var localMachineIdHash = MachineIdentity.Hash; + var localStorageIdentity = await LocalStorageIdentity(cancellationToken); + var tasks = settings.RemoteInstances .Select(async remote => { @@ -171,6 +175,9 @@ public async Task GetRemoteEnvironments(CancellationToken c environmentData[pair.Key] = pair.Value.GetValue(); } } + + environmentData["SameMachine"] = AuditSharingClassifier.SameMachine(localMachineIdHash, ReadString(body, "machine_id_hash")); + environmentData["DatabaseSharing"] = AuditSharingClassifier.DatabaseSharing(localStorageIdentity, ReadStorageIdentity(body)); } } } @@ -191,4 +198,49 @@ public async Task GetRemoteEnvironments(CancellationToken c return await Task.WhenAll(tasks); } + + async Task LocalStorageIdentity(CancellationToken cancellationToken) + { + var provider = storageIdentityProviders.FirstOrDefault(); + + if (provider is null) + { + return null; + } + + try + { + return AuditSharingClassifier.Hash(await provider.GetIdentity(cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + return null; + } + } + + static HashedStorageIdentity ReadStorageIdentity(JsonNode body) + { + if (body?.AsObject().TryGetPropertyValue("storage_identity", out var identity) != true || identity is not JsonObject identityObject) + { + return null; + } + + var engine = ReadString(identityObject, "engine"); + var serverHash = ReadString(identityObject, "server_hash"); + var databaseHash = ReadString(identityObject, "database_hash"); + + if (engine is null || serverHash is null || databaseHash is null) + { + return null; + } + + return new HashedStorageIdentity(engine, serverHash, databaseHash, ReadString(identityObject, "schema_hash")); + } + + static string ReadString(JsonNode node, string propertyName) => + node?.AsObject().TryGetPropertyValue(propertyName, out var value) == true && value is JsonValue ? value.GetValue() : null; } \ No newline at end of file From 3a61159950ab253447f5b0ce4f4af7b901bdd7d2 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 16:18:21 +1000 Subject: [PATCH 3/9] Report storage footprint, edition and service tier for the primary and audit instances Storage.SizeGB, Storage.MessageCount and Storage.UnresolvedFailedMessages come from catalog statistics scoped to the configured schema on SQL Server and PostgreSQL and from database and collection statistics on RavenDB, never from counting rows. Storage.ServerEdition carries SQL Server's own edition designation and Storage.ServiceObjective the Azure SQL service objective, both NotApplicable elsewhere. The audit instance serves the same keys over the environment endpoint, and the report sums sizes and counts across audit instances with instances sharing a database counted once, keyed by the identity hashes from the sharing detection. A catalog that has no answer, such as a row estimate on a never analysed PostgreSQL table, reports Unknown rather than a guess. --- .../AuditEnvironmentMetadata.cs | 10 ++- ...tor_Report_EnvironmentInformation_Tests.cs | 21 ++++++ .../AuditEnvironmentDataAggregator.cs | 60 +++++++++++++-- .../When_environment_data_is_requested.cs | 4 + .../RavenEnvironmentDataProvider.cs | 26 +++++++ .../PostgreSqlPersistence.cs | 1 + .../PostgreSqlStorageFootprintProbe.cs | 72 ++++++++++++++++++ .../SqlServerDatabaseHostingProbe.cs | 9 ++- .../SqlServerPersistence.cs | 1 + .../SqlServerStorageFootprintProbe.cs | 75 +++++++++++++++++++ .../EFEnvironmentDataProvider.cs | 36 ++++++++- .../Infrastructure/IDatabaseHostingProbe.cs | 6 +- .../Infrastructure/IStorageFootprintProbe.cs | 16 ++++ .../RavenEnvironmentDataProvider.cs | 43 ++++++++++- .../StorageFootprintProbeTests.cs | 38 ++++++++++ .../EnvironmentDataConfigurationTests.cs | 2 +- .../StorageFootprintProbeTests.cs | 38 ++++++++++ .../EnvironmentDataConfigurationTests.cs | 16 +++- .../EFCore/EnvironmentDataTests.cs | 16 +++- .../Infrastructure/Api/ConfigurationApi.cs | 10 ++- 20 files changed, 480 insertions(+), 20 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs diff --git a/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs index 416fad3b72..b0db55fe43 100644 --- a/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs +++ b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs @@ -5,4 +5,12 @@ namespace Particular.LicensingComponent.Contracts; /// responded. Never saved by versions that predate it, so a null read means unknown and the report /// omits the keys. /// -public record AuditEnvironmentMetadata(List> Instances); +public record AuditEnvironmentMetadata(List> Instances) +{ + /// + /// The key under which the primary stores which database an instance's data came from, so that + /// instances sharing a database are counted once when sizes and counts are summed. Underscore + /// prefixed keys are collection bookkeeping and never reach the report. + /// + public const string DatabaseKey = "_DatabaseKey"; +} diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index 3744d652c4..49ef827209 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -284,6 +284,27 @@ await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( } } + [Test] + public async Task Should_sum_sizes_and_counts_once_per_database() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["Storage.SizeGB"] = "10.0", ["Storage.MessageCount"] = "100", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "10.2", ["Storage.MessageCount"] = "90", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "5.0", ["Storage.MessageCount"] = "10" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.SizeGB").WithValue("15.2")); + Assert.That(environmentData, Does.ContainKey("Audit.Storage.MessageCount").WithValue("110")); + Assert.That(environmentData.Keys, Has.None.StartsWith("Audit._")); + } + } + [Test] public async Task Should_leave_audit_environment_data_out_when_it_was_never_collected() { diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs index e0bb865b1a..74b1c8c445 100644 --- a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -5,8 +5,10 @@ namespace Particular.LicensingComponent; /// /// Folds the environment data of every responding audit instance into one value per key: values -/// that agree pass through, capacity numbers take the maximum, and anything else that differs -/// reports Mixed rather than guessing which instance is representative. +/// that agree pass through, capacity numbers take the maximum, sizes and counts sum with instances +/// on one database counted once, the sharing class takes the most shared, and anything else that +/// differs reports Mixed rather than guessing which instance is representative. Keys starting with +/// an underscore carry collection bookkeeping and are never emitted. /// static class AuditEnvironmentDataAggregator { @@ -20,18 +22,47 @@ public static IEnumerable> Aggregate(List instance.Keys).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal); + var keys = instances + .SelectMany(instance => instance.Keys) + .Where(key => !key.StartsWith('_')) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal); foreach (var key in keys) { - var values = instances.Where(instance => instance.ContainsKey(key)).Select(instance => instance[key]).ToArray(); + var contributions = instances + .Where(instance => instance.ContainsKey(key)) + .Select((instance, index) => (Value: instance[key], DatabaseKey: instance.GetValueOrDefault(AuditEnvironmentMetadata.DatabaseKey) ?? $"_{index}")) + .ToArray(); - yield return new(key, Aggregate(key, values)); + yield return new(key, Aggregate(key, contributions)); } } - static string Aggregate(string key, string[] values) + static string Aggregate(string key, (string Value, string DatabaseKey)[] contributions) { + if (key == "Storage.SizeGB") + { + var sum = SumOncePerDatabase(contributions, value => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var size) ? size : null); + + if (sum is not null) + { + return sum.Value.ToString("F1", CultureInfo.InvariantCulture); + } + } + + if (key == "Storage.MessageCount") + { + var sum = SumOncePerDatabase(contributions, value => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) ? count : null); + + if (sum is not null) + { + return ((long)sum.Value).ToString(CultureInfo.InvariantCulture); + } + } + + var values = contributions.Select(contribution => contribution.Value).ToArray(); + if (values.Distinct(StringComparer.Ordinal).Count() == 1) { return values[0]; @@ -62,4 +93,21 @@ static string Aggregate(string key, string[] values) return "Mixed"; } + + static double? SumOncePerDatabase((string Value, string DatabaseKey)[] contributions, Func parse) + { + var parsed = contributions + .Select(contribution => (contribution.DatabaseKey, Number: parse(contribution.Value))) + .Where(contribution => contribution.Number is not null) + .ToArray(); + + if (parsed.Length == 0) + { + return null; + } + + return parsed + .GroupBy(contribution => contribution.DatabaseKey, StringComparer.Ordinal) + .Sum(group => group.Max(contribution => contribution.Number!.Value)); + } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 6228193159..7ae97bb9fa 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.AcceptanceTests { + using System.Globalization; using System.Text.Json.Nodes; using System.Threading.Tasks; using AcceptanceTesting; @@ -37,6 +38,9 @@ public async Task Should_serve_host_and_storage_facts() Assert.That((string)environmentData["Storage.Type"], Is.Not.Null.And.Not.Empty); Assert.That((string)environmentData["Storage.ServerVersion"], Is.Not.Null.And.Not.Empty); Assert.That((string)environmentData["Storage.FullTextSearch"], Is.AnyOf("Enabled", "Disabled")); + Assert.That(double.Parse((string)environmentData["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); + Assert.That((string)environmentData["Storage.MessageCount"], Is.EqualTo("0")); + Assert.That((string)environmentData["Storage.ServerEdition"], Is.EqualTo("NotApplicable")); Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); Assert.That((string)body["machine_id_hash"], Is.Not.Null.And.Not.Empty); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs index 42f9ac2646..b08333c65c 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -2,8 +2,10 @@ namespace ServiceControl.Audit.Persistence.RavenDB { using System; using System.Collections.Generic; + using System.Globalization; using System.Threading; using System.Threading.Tasks; + using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; using static ServiceControl.Audit.Persistence.EnvironmentDatum; @@ -16,9 +18,31 @@ public IEnumerable GetData() => Value("Storage.Hosting", () => Hosting().Hosting), Deferred("Storage.ServerVersion", ServerVersion), Value("Storage.HostingSource", () => Hosting().Source), + Value("Storage.ServerEdition", () => "NotApplicable"), + Value("Storage.ServiceObjective", () => "NotApplicable"), + Deferred("Storage.SizeGB", SizeGB), + Deferred("Storage.MessageCount", ProcessedMessageCount), Value("Storage.FullTextSearch", () => databaseConfiguration.EnableFullTextSearch ? "Enabled" : "Disabled") ]; + async ValueTask SizeGB(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.ForDatabase(databaseConfiguration.Name).SendAsync(new GetStatisticsOperation(), cancellationToken); + + return (statistics.SizeOnDisk.SizeInBytes / BytesPerGigabyte).ToString("F1", CultureInfo.InvariantCulture); + } + + async ValueTask ProcessedMessageCount(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.ForDatabase(databaseConfiguration.Name).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("ProcessedMessages", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + (string Hosting, string Source) Hosting() { var serverConfiguration = databaseConfiguration.ServerConfiguration; @@ -41,5 +65,7 @@ async ValueTask ServerVersion(CancellationToken cancellationToken) return buildNumber.ProductVersion ?? DatabaseHostClassifier.Unknown; } + + const double BytesPerGigabyte = 1024d * 1024 * 1024; } } diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 51d31902be..423f1d8ba8 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -23,6 +23,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs new file mode 100644 index 0000000000..aefc7119c3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class PostgreSqlStorageFootprintProbe(PostgreSqlPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); + var schema = settings.Schema ?? "public"; + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = """ + SELECT + (SELECT SUM(pg_total_relation_size(c.oid)) / 1073741824.0 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = @schema AND c.relkind IN ('r', 'm')), + (SELECT CASE WHEN c.reltuples < 0 THEN NULL ELSE c.reltuples::bigint END + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = @schema AND c.relname = @table) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.Value = schema; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = failedMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs index 5ca4450e2e..201a472522 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs @@ -24,7 +24,7 @@ public async Task Probe(CancellationToken cancellationToken = d await using var command = dbContext.Database.GetDbConnection().CreateCommand(); // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server // only and comes back null on Azure SQL Database, Managed Instance and Synapse. - command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin')"; + command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(SERVERPROPERTY('Edition') AS nvarchar(128)), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; command.CommandTimeout = ProbeTimeoutSeconds; await dbContext.Database.OpenConnectionAsync(cancellationToken); @@ -40,7 +40,12 @@ public async Task Probe(CancellationToken cancellationToken = d var rds = !reader.IsDBNull(2); - return new DatabaseHosting(HostingFor(engineEdition, rds, ConfiguredHost), MajorVersion(reader), DatabaseHostingSource.Probe); + return new DatabaseHosting( + HostingFor(engineEdition, rds, ConfiguredHost), + MajorVersion(reader), + DatabaseHostingSource.Probe, + reader.IsDBNull(3) ? DatabaseHosting.NotApplicable : reader.GetString(3), + reader.IsDBNull(4) ? DatabaseHosting.NotApplicable : reader.GetString(4)); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index 7063a20058..8f6a8f78dd 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -23,6 +23,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs new file mode 100644 index 0000000000..632a192e96 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SqlServerStorageFootprintProbe(SqlServerPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); + var schema = settings.Schema ?? "dbo"; + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = """ + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.tables t + JOIN sys.schemas s ON s.schema_id = t.schema_id + JOIN sys.partitions p ON p.object_id = t.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE s.name = @schema), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.schemas s ON s.schema_id = t.schema_id + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE s.name = @schema AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.Value = schema; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = failedMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs index f5ccfaf98a..153069b5ee 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -2,27 +2,41 @@ namespace ServiceControl.Persistence.EFCore.Implementation; using System.Globalization; using Abstractions; +using DbContexts; +using Entities; using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Particular.LicensingComponent.Contracts; using ServiceControl.Configuration; using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; using static Particular.LicensingComponent.Contracts.EnvironmentDatum; -class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe) : IEnvironmentDataProvider +class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe, IStorageFootprintProbe footprintProbe, IServiceScopeFactory scopeFactory) : IEnvironmentDataProvider { public IEnumerable GetData() { - // The three hosting keys share one probe, so the database is asked once per report and they - // stand or fall together, which is right because they have a single cause. + // The hosting keys share one probe, so the database is asked once per report and they + // stand or fall together, which is right because they have a single cause. The footprint + // keys share theirs for the same reason. Task? probe = null; Task Hosting(CancellationToken cancellationToken) => probe ??= hostingProbe.Probe(cancellationToken); + Task? footprint = null; + Task Footprint(CancellationToken cancellationToken) => footprint ??= footprintProbe.Probe(cancellationToken); + return [ Value("Storage.Type", () => hostingProbe.StorageName), Deferred("Storage.Hosting", async cancellationToken => (await Hosting(cancellationToken)).Hosting), Deferred("Storage.ServerVersion", async cancellationToken => (await Hosting(cancellationToken)).ServerVersion), Deferred("Storage.HostingSource", async cancellationToken => (await Hosting(cancellationToken)).Source), + Deferred("Storage.ServerEdition", async cancellationToken => (await Hosting(cancellationToken)).ServerEdition), + Deferred("Storage.ServiceObjective", async cancellationToken => (await Hosting(cancellationToken)).ServiceObjective), + Deferred("Storage.SizeGB", async cancellationToken => SizeGB((await Footprint(cancellationToken))?.SizeGB)), + Deferred("Storage.MessageCount", async cancellationToken => Count((await Footprint(cancellationToken))?.MessageCount)), + Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => BodyStorageType(settings.BodyStorage)), Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture)), @@ -35,6 +49,22 @@ public IEnumerable GetData() ]; } + async ValueTask UnresolvedFailedMessages(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var count = await dbContext.Set().CountAsync(m => m.Status == FailedMessageStatus.Unresolved, cancellationToken); + + return Count(count); + } + + static string SizeGB(double? sizeGB) => + sizeGB is null ? "Unknown" : sizeGB.Value.ToString("F1", CultureInfo.InvariantCulture); + + static string Count(long? count) => + count is null ? "Unknown" : count.Value.ToString(CultureInfo.InvariantCulture); + string FreeSpaceThreshold() => settings.BodyStorage is FileSystemBodyStorageSettings fileSystem ? WhenConfigured(EFPersistenceConfigurationBase.FileSystemDataSpaceRemainingThresholdKey, () => Number(fileSystem.DataSpaceRemainingThreshold)) : "NotApplicable"; diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs index 73c3b12051..ff4e956df5 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs @@ -23,8 +23,12 @@ public interface IDatabaseHostingProbe /// One of AzureSql, AzureSqlManagedInstance, AzureSqlEdge, AzurePostgres, AzureMySql, AwsRds, GoogleCloudSql, RavenCloud, SelfHosted or Unknown. /// The engine major version, or Unknown. /// A value. -public record DatabaseHosting(string Hosting, string ServerVersion, string Source) +/// The engine's own edition designation, such as SQL Server's Edition property. NotApplicable where the engine has no editions. +/// The Azure SQL service objective, such as S0 or GP_Gen5_2. NotApplicable everywhere else. +public record DatabaseHosting(string Hosting, string ServerVersion, string Source, string ServerEdition = DatabaseHosting.NotApplicable, string ServiceObjective = DatabaseHosting.NotApplicable) { + public const string NotApplicable = "NotApplicable"; + /// Nothing was available to classify the host with. A determination, not a failure. public static readonly DatabaseHosting Unclassified = new(DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs new file mode 100644 index 0000000000..c82d543097 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// Measures what this instance's data occupies, from catalog statistics scoped to the configured +/// schema, never by counting rows. Reported in usage telemetry as sizes and counts only. +/// +public interface IStorageFootprintProbe +{ + /// + /// Null when the database could not be asked. A field is null when the catalog had no answer, + /// such as a row estimate on a table that was never analysed. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +public record StorageFootprint(double? SizeGB, long? MessageCount); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs index 61c709b6d7..a88aaf1c33 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -3,15 +3,20 @@ namespace ServiceControl.Persistence.RavenDB; using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; using ServiceControl.Configuration; using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; using static Particular.LicensingComponent.Contracts.EnvironmentDatum; -class RavenEnvironmentDataProvider(RavenPersisterSettings settings, IRavenDocumentStoreProvider documentStoreProvider) : IEnvironmentDataProvider +class RavenEnvironmentDataProvider(RavenPersisterSettings settings, IRavenDocumentStoreProvider documentStoreProvider, IRavenSessionProvider sessionProvider) : IEnvironmentDataProvider { public IEnumerable GetData() => [ @@ -20,6 +25,11 @@ public IEnumerable GetData() => Value("Storage.Hosting", () => Hosting().Hosting), Deferred("Storage.ServerVersion", ServerVersion), Value("Storage.HostingSource", () => Hosting().Source), + Value("Storage.ServerEdition", () => "NotApplicable"), + Value("Storage.ServiceObjective", () => "NotApplicable"), + Deferred("Storage.SizeGB", SizeGB), + Deferred("Storage.MessageCount", FailedMessageCount), + Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => "RavenAttachments"), Value("Storage.LogLevel", () => settings.LogsMode), @@ -29,6 +39,35 @@ public IEnumerable GetData() => Value("Storage.ExpirationIntervalSeconds", () => WhenConfigured(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, () => Number(settings.ExpirationProcessTimerInSeconds))) ]; + async ValueTask SizeGB(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetStatisticsOperation(), cancellationToken); + + return (statistics.SizeOnDisk.SizeInBytes / BytesPerGigabyte).ToString("F1", CultureInfo.InvariantCulture); + } + + async ValueTask FailedMessageCount(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("FailedMessages", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + + async ValueTask UnresolvedFailedMessages(CancellationToken cancellationToken) + { + using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); + + var count = await session.Query() + .Where(m => m.Status == FailedMessageStatus.Unresolved) + .CountAsync(cancellationToken); + + return count.ToString(CultureInfo.InvariantCulture); + } + (string Hosting, string Source) Hosting() { // An embedded server runs in this process, so there is nothing to infer. @@ -56,4 +95,6 @@ static string WhenConfigured(string key, Func readValue) => static string Number(int value) => value.ToString(CultureInfo.InvariantCulture); static readonly SettingsRootNamespace SettingsNamespace = new("ServiceControl"); + + const double BytesPerGigabyte = 1024d * 1024 * 1024; } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs new file mode 100644 index 0000000000..a2314a2427 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageFootprintProbeTests : PersistenceTestBase +{ + [Test] + public async Task Measures_schema_size_and_message_rows() + { + var probe = ServiceProvider.GetRequiredService(); + + var footprint = await probe.Probe(); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + Assert.That(footprint.MessageCount ?? 0, Is.EqualTo(0), "A never analysed table has no row estimate, which reads back as null"); + } + } + + [Test] + public async Task Editions_do_not_apply() + { + var probe = ServiceProvider.GetRequiredService(); + + var hosting = await probe.Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs index cd7c23e4dc..8f1d0d0783 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs @@ -55,7 +55,7 @@ static async Task> GetData(RavenPersisterSettings set { var data = new Dictionary(); - foreach (var datum in new RavenEnvironmentDataProvider(settings, documentStoreProvider: null).GetData()) + foreach (var datum in new RavenEnvironmentDataProvider(settings, documentStoreProvider: null, sessionProvider: null).GetData()) { if (ConfigurationKeys.Contains(datum.Key)) { diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs new file mode 100644 index 0000000000..267c7f5596 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageFootprintProbeTests : PersistenceTestBase +{ + [Test] + public async Task Measures_schema_size_and_message_rows() + { + var probe = ServiceProvider.GetRequiredService(); + + var footprint = await probe.Probe(); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + Assert.That(footprint.MessageCount, Is.EqualTo(0)); + } + } + + [Test] + public async Task Reports_the_engine_edition() + { + var probe = ServiceProvider.GetRequiredService(); + + var hosting = await probe.Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.Not.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs index daab19d03a..12bda43295 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs @@ -86,9 +86,16 @@ static async Task> GetData(EFPersisterSettings settin { var data = new Dictionary(); - foreach (var datum in new EFEnvironmentDataProvider(settings, new StubHostingProbe()).GetData()) + foreach (var datum in new EFEnvironmentDataProvider(settings, new StubHostingProbe(), new StubFootprintProbe(), scopeFactory: null).GetData()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + try + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + catch (Exception) + { + data[datum.Key] = Particular.LicensingComponent.Contracts.EnvironmentDatum.ReadFailed; + } } return data; @@ -110,4 +117,9 @@ sealed class StubHostingProbe : IDatabaseHostingProbe public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(DatabaseHosting.Unclassified); } + + sealed class StubFootprintProbe : IStorageFootprintProbe + { + public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(null); + } } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs index 08c8e3e5a0..c899ad2df3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs @@ -102,17 +102,29 @@ public async Task Should_fall_back_to_unknown_when_the_hosting_probe_fails() static async Task> GetData(BodyStorageSettings bodyStorage, IDatabaseHostingProbe hostingProbe = null) { var settings = new TestPersisterSettings { ConnectionString = "Host=localhost", BodyStorage = bodyStorage }; - var provider = new EFEnvironmentDataProvider(settings, hostingProbe ?? new TestHostingProbe()); + var provider = new EFEnvironmentDataProvider(settings, hostingProbe ?? new TestHostingProbe(), new TestFootprintProbe(), null); var data = new Dictionary(); foreach (var datum in provider.GetData()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + try + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + catch (Exception) + { + data[datum.Key] = Particular.LicensingComponent.Contracts.EnvironmentDatum.ReadFailed; + } } return data; } + class TestFootprintProbe : IStorageFootprintProbe + { + public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(null); + } + class TestPersisterSettings : EFPersisterSettings; class TestHostingProbe : IDatabaseHostingProbe diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4df58bfb86..75755b8a89 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Configuration; using Monitoring.HeartbeatMonitoring; +using Particular.LicensingComponent.Contracts; using Particular.ServiceControl.Licensing; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api; @@ -176,8 +177,15 @@ public async Task GetRemoteEnvironments(CancellationToken c } } + var remoteStorageIdentity = ReadStorageIdentity(body); + environmentData["SameMachine"] = AuditSharingClassifier.SameMachine(localMachineIdHash, ReadString(body, "machine_id_hash")); - environmentData["DatabaseSharing"] = AuditSharingClassifier.DatabaseSharing(localStorageIdentity, ReadStorageIdentity(body)); + environmentData["DatabaseSharing"] = AuditSharingClassifier.DatabaseSharing(localStorageIdentity, remoteStorageIdentity); + + if (remoteStorageIdentity is not null) + { + environmentData[AuditEnvironmentMetadata.DatabaseKey] = $"{remoteStorageIdentity.ServerHash}/{remoteStorageIdentity.DatabaseHash}"; + } } } } From ab2d0b0686eb75000abd2a478aa64e870b1b19b4 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 16:32:16 +1000 Subject: [PATCH 4/9] Collect hourly ingestion rates and busy time for the error and audit sides Both ingestion pipelines keep running totals in memory: messages stored per completed batch, time spent processing batches, storage write time on the error side, and how far behind the endpoint's own timestamp each message arrived, bucketed at one, ten and sixty minutes. The audit instance serves its counters on the environment endpoint. A new hourly collector in the licensing component polls the primary's counters in process and every audit instance's over the API, folds the deltas into daily records in the licensing store with the audit side summed across instances, and keeps 400 days. A restarted process contributes its whole counters as the delta, a counter set seen for the first time only establishes a baseline, and a collector restart builds on what today's record already holds. The report emits per side the average daily messages, the busiest hour's messages and its busy share, plus storage milliseconds per message for the error side, from the days inside the report window. Known gap, accepted in the plan: --error-ingestion-only workers keep their own counters and nothing polls them. --- .../IngestionHistory.cs | 28 +++ .../InMemoryLicensingDataStore.cs | 9 + .../ILicensingDataStore.cs | 4 + .../AuditQuery_Tests.cs | 6 + ...tThroughputCollectorHostedService_Tests.cs | 15 ++ .../Infrastructure/FakeConfigurationApi.cs | 2 + ...tionHistoryCollectorHostedService_Tests.cs | 161 +++++++++++++ .../MonitoringService_Tests.cs | 6 + ...tor_Report_EnvironmentInformation_Tests.cs | 38 ++++ .../AuditThroughput/AuditIngestionSnapshot.cs | 9 + .../AuditThroughput/AuditQuery.cs | 33 +++ .../AuditThroughput/IAuditQuery.cs | 1 + .../IErrorIngestionSnapshotProvider.cs | 12 + .../IngestionHistoryCollectorHostedService.cs | 212 ++++++++++++++++++ ...LicensingComponentHostBuilderExtensions.cs | 1 + .../ThroughputCollector.cs | 35 +++ .../Contracts/RemoteIngestionCounters.cs | 26 +++ src/ServiceControl.Api/IConfigurationApi.cs | 2 + .../When_environment_data_is_requested.cs | 4 + .../Auditing/Metrics/IngestionMetricsTests.cs | 6 +- .../EnvironmentControllerTests.cs | 13 +- .../Auditing/Metrics/IngestionMetrics.cs | 32 ++- .../HostApplicationBuilderExtensions.cs | 1 + .../WebApi/EnvironmentController.cs | 31 ++- .../Ingestion/IngestionCounters.cs | 73 ++++++ .../Ingestion/Metrics/BatchMetrics.cs | 3 +- .../Ingestion/Metrics/DurationScope.cs | 8 +- .../Implementation/LicensingDataStore.cs | 6 + .../Infrastructure/SettingKeys.cs | 1 + .../Throughput/LicensingDataStore.cs | 18 ++ .../Infrastructure/IngestionCountersTests.cs | 72 ++++++ .../Metrics/IngestionMetricsTests.cs | 6 +- .../ErrorIngestionSnapshotProvider.cs | 9 + .../HostApplicationBuilderExtensions.cs | 2 + .../Infrastructure/Api/ConfigurationApi.cs | 51 +++++ .../Operations/Metrics/IngestionMetrics.cs | 34 ++- 36 files changed, 950 insertions(+), 20 deletions(-) create mode 100644 src/Particular.LicensingComponent.Contracts/IngestionHistory.cs create mode 100644 src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs create mode 100644 src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs create mode 100644 src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs create mode 100644 src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs create mode 100644 src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs create mode 100644 src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs create mode 100644 src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs create mode 100644 src/ServiceControl/ErrorIngestionSnapshotProvider.cs diff --git a/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs b/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs new file mode 100644 index 0000000000..1e32b4f457 --- /dev/null +++ b/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs @@ -0,0 +1,28 @@ +namespace Particular.LicensingComponent.Contracts; + +/// +/// Daily ingestion records kept by the hourly collector, per source (Error or Audit, the audit +/// side summed across instances). Never saved by versions that predate it, so a null read means +/// unknown and the report omits the keys. +/// +public record IngestionHistory(List Days) +{ + public const string ErrorSource = "Error"; + public const string AuditSource = "Audit"; +} + +/// The UTC day, at midnight. +/// The busiest hour's messages. +/// The time the ingestion loop spent processing batches during that busiest hour. +/// The time spent in storage writes during that busiest hour. Always zero for sources that do not measure storage separately. +public record IngestionDay( + DateTime Date, + string Source, + long Messages, + long PeakHourMessages, + double PeakHourBusySeconds, + double PeakHourStorageSeconds, + long LagOverOneMinuteMessages, + long LagOverTenMinutesMessages, + long LagOverSixtyMinutesMessages, + long LagKnownMessages); diff --git a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs index 611611fdee..95c9fdd4c8 100644 --- a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs @@ -15,6 +15,7 @@ public class InMemoryLicensingDataStore : ILicensingDataStore BrokerMetadata brokerMetadata = new(null, []); AuditServiceMetadata auditServiceMetadata = new([], []); AuditEnvironmentMetadata? auditEnvironmentMetadata; + IngestionHistory? ingestionHistory; List reportMasks = []; LicensedEndpointDetails? endpointDetails = null; @@ -176,6 +177,14 @@ public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironme return Task.CompletedTask; } + public Task GetIngestionHistory(CancellationToken cancellationToken = default) => Task.FromResult(ingestionHistory); + + public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) + { + this.ingestionHistory = ingestionHistory; + return Task.CompletedTask; + } + public Task> GetReportMasks(CancellationToken cancellationToken = default) => Task.FromResult(reportMasks); public Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default) diff --git a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs index bbf997ecf5..3f5e3c6def 100644 --- a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs @@ -40,6 +40,10 @@ Task RecordEndpointThroughput(string endpointName, ThroughputSource throughputSo Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default); Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default); + + Task GetIngestionHistory(CancellationToken cancellationToken = default); + + Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default); Task> GetReportMasks(CancellationToken cancellationToken = default); Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default); diff --git a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs index b64a0c12c8..b70b8dfed6 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs @@ -194,6 +194,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi @@ -205,6 +207,8 @@ class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_Configurable : IConfigurationApi @@ -227,6 +231,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public bool ReturnAuditConfig { get; set; } public string RemoteStatus { get; set; } public string RemoteVersion { get; set; } diff --git a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs index 3174f5bea7..c0178087c6 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs @@ -306,6 +306,9 @@ class AuditQuery_NoAuditRemotes : IAuditQuery public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -342,6 +345,9 @@ public AuditQuery_WithOneEndpoint(string endpointName, long throughputCount, Dat public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -377,6 +383,9 @@ class AuditQuery_ThrowingAnExceptionOnKnownEndpointsCall : IAuditQuery public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -460,6 +469,9 @@ public AuditQuery_WithTwoEndpointsAndSelectiveCounts( public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + public Func ValidRemoteInstances => _ => true; public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => @@ -502,6 +514,9 @@ class AuditQuery_WithRemotes(List remotes) : IAuditQu public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult(Environments); + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + public Func ValidRemoteInstances => _ => true; public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => diff --git a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs index 0e18e277ae..4fc290fc16 100644 --- a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs +++ b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs @@ -11,6 +11,8 @@ class FakeConfigurationApi : IConfigurationApi public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); } } diff --git a/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs new file mode 100644 index 0000000000..6ec6176b7e --- /dev/null +++ b/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs @@ -0,0 +1,161 @@ +namespace Particular.LicensingComponent.UnitTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AuditThroughput; +using Contracts; +using Infrastructure; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NuGet.Versioning; +using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; + +[TestFixture] +class IngestionHistoryCollectorHostedService_Tests : ThroughputCollectorTestFixture +{ + public override Task Setup() + { + SetExtraDependencies = d => { }; + + return base.Setup(); + } + + [Test] + public async Task Second_poll_stores_the_delta_summed_across_audit_instances() + { + var start = new DateTimeOffset(2026, 9, 30, 10, 0, 0, TimeSpan.Zero); + var fakeTimeProvider = new FakeTimeProvider(start); + var processStart = start.UtcDateTime.AddDays(-3); + var errorProvider = new QueuedErrorSnapshots( + Snapshot(processStart, messages: 100, busySeconds: 10), + Snapshot(processStart, messages: 160, busySeconds: 16)); + var auditQuery = new AuditQuery_WithIngestionSnapshots( + [ + new AuditIngestionSnapshot("a", Snapshot(processStart, messages: 1000, busySeconds: 100)), + new AuditIngestionSnapshot("b", Snapshot(processStart, messages: 2000, busySeconds: 200)) + ], + [ + new AuditIngestionSnapshot("a", Snapshot(processStart, messages: 1300, busySeconds: 130)), + new AuditIngestionSnapshot("b", Snapshot(processStart, messages: 2500, busySeconds: 260)) + ]); + + await RunTwoPolls(fakeTimeProvider, errorProvider, auditQuery); + + var history = await DataStore.GetIngestionHistory(); + + Assert.That(history, Is.Not.Null); + var errorDay = history.Days.Single(day => day.Source == IngestionHistory.ErrorSource); + var auditDay = history.Days.Single(day => day.Source == IngestionHistory.AuditSource); + using (Assert.EnterMultipleScope()) + { + Assert.That(errorDay.Date, Is.EqualTo(start.UtcDateTime.Date)); + Assert.That(errorDay.Messages, Is.EqualTo(60)); + Assert.That(errorDay.PeakHourMessages, Is.EqualTo(60)); + Assert.That(errorDay.PeakHourBusySeconds, Is.EqualTo(6).Within(0.001)); + Assert.That(auditDay.Messages, Is.EqualTo(800)); + Assert.That(auditDay.PeakHourMessages, Is.EqualTo(800)); + Assert.That(auditDay.PeakHourBusySeconds, Is.EqualTo(90).Within(0.001)); + } + } + + [Test] + public async Task A_restarted_process_contributes_its_whole_counters_as_the_delta() + { + var start = new DateTimeOffset(2026, 9, 30, 10, 0, 0, TimeSpan.Zero); + var fakeTimeProvider = new FakeTimeProvider(start); + var errorProvider = new QueuedErrorSnapshots( + Snapshot(start.UtcDateTime.AddDays(-3), messages: 100, busySeconds: 10), + Snapshot(start.UtcDateTime.AddMinutes(30), messages: 25, busySeconds: 5)); + var auditQuery = new AuditQuery_WithIngestionSnapshots([], []); + + await RunTwoPolls(fakeTimeProvider, errorProvider, auditQuery); + + var history = await DataStore.GetIngestionHistory(); + + Assert.That(history, Is.Not.Null); + var errorDay = history.Days.Single(day => day.Source == IngestionHistory.ErrorSource); + Assert.That(errorDay.Messages, Is.EqualTo(25)); + } + + async Task RunTwoPolls(FakeTimeProvider fakeTimeProvider, QueuedErrorSnapshots errorProvider, AuditQuery_WithIngestionSnapshots auditQuery) + { + using var service = new IngestionHistoryCollectorHostedService( + NullLogger.Instance, DataStore, auditQuery, fakeTimeProvider, errorProvider) + { DelayStart = TimeSpan.Zero }; + + await service.StartAsync(CancellationToken.None); + + await errorProvider.PolledOnce.Task.WaitAsync(TimeSpan.FromSeconds(30)); + fakeTimeProvider.Advance(TimeSpan.FromHours(1)); + await errorProvider.PolledTwice.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + var saveDeadline = DateTime.UtcNow.AddSeconds(30); + + while (await DataStore.GetIngestionHistory() is null && DateTime.UtcNow < saveDeadline) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)); + } + + await service.StopAsync(CancellationToken.None); + } + + static IngestionCountersSnapshot Snapshot(DateTime processStartUtc, long messages, double busySeconds) => + new(processStartUtc, messages, busySeconds, 0, 0, 0, 0, 0); + + class QueuedErrorSnapshots(params IngestionCountersSnapshot[] snapshots) : IErrorIngestionSnapshotProvider + { + public TaskCompletionSource PolledOnce { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource PolledTwice { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public IngestionCountersSnapshot GetSnapshot() + { + var index = Math.Min(calls, snapshots.Length - 1); + calls++; + + if (calls == 1) + { + PolledOnce.TrySetResult(); + } + + if (calls >= 2) + { + PolledTwice.TrySetResult(); + } + + return snapshots[index]; + } + + int calls; + } + + class AuditQuery_WithIngestionSnapshots(List first, List second) : IAuditQuery + { + public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + + public Func ValidRemoteInstances => _ => true; + + public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => + Task.FromResult(calls++ == 0 ? first : second); + + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + + public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + + public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => + Task.FromResult>([]); + + public Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default) => + Task.FromResult>([]); + + public Task TestAuditConnection(CancellationToken cancellationToken = default) => + Task.FromResult(new ConnectionSettingsTestResult { ConnectionSuccessful = true }); + + int calls; + } +} diff --git a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs index f6d1e5651c..fb7cbd3ab5 100644 --- a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs @@ -258,6 +258,12 @@ public Task GetAuditEnvironmentMetadata(CancellationTo public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task GetIngestionHistory(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => throw new NotSupportedException(); diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index 49ef827209..d152317494 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -1,5 +1,6 @@ namespace Particular.LicensingComponent.UnitTests; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -313,6 +314,43 @@ public async Task Should_leave_audit_environment_data_out_when_it_was_never_coll Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); } + [Test] + public async Task Should_report_ingestion_rates_from_days_inside_the_report_window() + { + var reportEndDate = new DateTime(2026, 9, 30, 0, 0, 0, DateTimeKind.Utc); + var insideWindow = reportEndDate.AddDays(-1); + var outsideWindow = reportEndDate.AddDays(-10); + await DataStore.SaveIngestionHistory(new IngestionHistory( + [ + new IngestionDay(insideWindow, IngestionHistory.ErrorSource, 2400, 600, 1800, 60, 0, 0, 0, 0), + new IngestionDay(insideWindow, IngestionHistory.AuditSource, 1200, 300, 360, 0, 0, 0, 0, 0), + new IngestionDay(outsideWindow, IngestionHistory.ErrorSource, 999999, 99999, 3600, 999, 0, 0, 0, 0) + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", reportEndDate); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Ingestion.Error.AvgDailyMessages").WithValue("2400")); + Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourMessages").WithValue("600")); + Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourBusyPercent").WithValue("50")); + Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourStorageMsPerMessage").WithValue("100")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.AvgDailyMessages").WithValue("1200")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourMessages").WithValue("300")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourBusyPercent").WithValue("10")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Ingestion.PeakHourStorageMsPerMessage")); + } + } + + [Test] + public async Task Should_leave_ingestion_rates_out_when_never_collected() + { + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Ingestion.")); + } + [Test] public async Task Should_leave_audit_environment_data_out_when_no_instance_responded() { diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs new file mode 100644 index 0000000000..50ffcc4255 --- /dev/null +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs @@ -0,0 +1,9 @@ +namespace Particular.LicensingComponent.AuditThroughput; + +using ServiceControl.Infrastructure.Ingestion; + +/// +/// One audit instance's running counters. The ApiUri only keys hourly deltas in memory and is +/// never stored or reported. +/// +public record AuditIngestionSnapshot(string ApiUri, IngestionCountersSnapshot Snapshot); diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs index d3f633d55f..01ba9d0725 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using NuGet.Versioning; using ServiceControl.Api; + using ServiceControl.Infrastructure.Ingestion; using AuditCount = Contracts.AuditCount; public class AuditQuery(ILogger logger, IEndpointsApi endpointsApi, IAuditCountApi auditCountApi, IConfigurationApi configurationApi) : IAuditQuery @@ -133,6 +134,38 @@ public async Task>> GetAuditEnvironments(Cancell } } + public async Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) + { + try + { + var remoteCounters = await configurationApi.GetRemoteIngestionCounters(cancellationToken); + + return + [ + .. remoteCounters + .Where(remote => remote.Counters is not null) + .Select(remote => new AuditIngestionSnapshot(remote.ApiUri, new IngestionCountersSnapshot( + remote.Counters.ProcessStartUtc, + remote.Counters.MessagesTotal, + remote.Counters.BusySecondsTotal, + remote.Counters.StorageSecondsTotal, + remote.Counters.LagOverOneMinuteMessages, + remote.Counters.LagOverTenMinutesMessages, + remote.Counters.LagOverSixtyMinutesMessages, + remote.Counters.LagKnownMessages))) + ]; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get Audit Ingestion Snapshots"); + return []; + } + } + public async Task TestAuditConnection(CancellationToken cancellationToken = default) { var connectionTestResult = new ConnectionSettingsTestResult { ConnectionSuccessful = true }; diff --git a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs index cf0252dfd1..65eb583d4a 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs @@ -13,6 +13,7 @@ public interface IAuditQuery Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default); Task> GetAuditRemotes(CancellationToken cancellationToken = default); Task>> GetAuditEnvironments(CancellationToken cancellationToken = default); + Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default); Task TestAuditConnection(CancellationToken cancellationToken = default); } diff --git a/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs b/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs new file mode 100644 index 0000000000..b22c659978 --- /dev/null +++ b/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs @@ -0,0 +1,12 @@ +namespace Particular.LicensingComponent; + +using ServiceControl.Infrastructure.Ingestion; + +/// +/// The host's own error ingestion counters, registered by the primary instance so that the +/// licensing collector can read them in process the way it polls audit instances over their API. +/// +public interface IErrorIngestionSnapshotProvider +{ + IngestionCountersSnapshot GetSnapshot(); +} diff --git a/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs b/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs new file mode 100644 index 0000000000..c655598771 --- /dev/null +++ b/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs @@ -0,0 +1,212 @@ +namespace Particular.LicensingComponent; + +using AuditThroughput; +using Contracts; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Persistence; +using ServiceControl.Infrastructure.Ingestion; + +/// +/// Polls the ingestion counters every hour, this instance's in process and every audit instance's +/// over its environment endpoint, and folds the deltas into daily records in the licensing store. +/// A counter set whose process start time moved belongs to a restarted process, so its whole value +/// is the delta; one seen for the first time only establishes a baseline, because its totals span +/// an unknown stretch of time. +/// +public class IngestionHistoryCollectorHostedService( + ILogger logger, + ILicensingDataStore dataStore, + IAuditQuery auditQuery, + TimeProvider timeProvider, + IErrorIngestionSnapshotProvider? errorSnapshotProvider = null) : BackgroundService +{ + public TimeSpan DelayStart { get; set; } = TimeSpan.FromSeconds(50); + + protected override async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + logger.LogInformation("Starting {ServiceName}", nameof(IngestionHistoryCollectorHostedService)); + + try + { + await Task.Delay(DelayStart, cancellationToken); + + using PeriodicTimer timer = new(TimeSpan.FromHours(1), timeProvider); + + do + { + try + { + await Collect(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to collect ingestion counters"); + } + } while (await timer.WaitForNextTickAsync(cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + logger.LogInformation("Stopping {ServiceName}", nameof(IngestionHistoryCollectorHostedService)); + } + } + + internal async Task Collect(CancellationToken cancellationToken = default) + { + var now = timeProvider.GetUtcNow().UtcDateTime; + var changed = false; + + if (errorSnapshotProvider is not null) + { + changed |= Accumulate(IngestionHistory.ErrorSource, now, [new AuditIngestionSnapshot(LocalKey, errorSnapshotProvider.GetSnapshot())]); + } + + var auditSnapshots = await auditQuery.GetAuditIngestionSnapshots(cancellationToken); + + if (auditSnapshots.Count > 0) + { + changed |= Accumulate(IngestionHistory.AuditSource, now, auditSnapshots); + } + + if (changed) + { + await Save(now, cancellationToken); + } + } + + bool Accumulate(string source, DateTime now, List snapshots) + { + if (!states.TryGetValue(source, out var state)) + { + states[source] = state = new SourceState(); + } + + if (state.Date != now.Date) + { + state.Reset(now.Date); + } + + var any = false; + + foreach (var (apiUri, snapshot) in snapshots) + { + if (state.Baselines.TryGetValue(apiUri, out var last)) + { + var delta = snapshot.ProcessStartUtc != last.ProcessStartUtc ? snapshot : Diff(snapshot, last); + + state.HourMessages[now.Hour] += delta.Messages; + state.HourBusySeconds[now.Hour] += delta.BusySeconds; + state.HourStorageSeconds[now.Hour] += delta.StorageSeconds; + state.LagOverOneMinute += delta.LagOverOneMinuteMessages; + state.LagOverTenMinutes += delta.LagOverTenMinutesMessages; + state.LagOverSixtyMinutes += delta.LagOverSixtyMinutesMessages; + state.LagKnown += delta.LagKnownMessages; + any = true; + } + + state.Baselines[apiUri] = snapshot; + } + + return any; + } + + static IngestionCountersSnapshot Diff(IngestionCountersSnapshot current, IngestionCountersSnapshot last) => current with + { + Messages = Math.Max(0, current.Messages - last.Messages), + BusySeconds = Math.Max(0, current.BusySeconds - last.BusySeconds), + StorageSeconds = Math.Max(0, current.StorageSeconds - last.StorageSeconds), + LagOverOneMinuteMessages = Math.Max(0, current.LagOverOneMinuteMessages - last.LagOverOneMinuteMessages), + LagOverTenMinutesMessages = Math.Max(0, current.LagOverTenMinutesMessages - last.LagOverTenMinutesMessages), + LagOverSixtyMinutesMessages = Math.Max(0, current.LagOverSixtyMinutesMessages - last.LagOverSixtyMinutesMessages), + LagKnownMessages = Math.Max(0, current.LagKnownMessages - last.LagKnownMessages) + }; + + async Task Save(DateTime now, CancellationToken cancellationToken) + { + var history = await dataStore.GetIngestionHistory(cancellationToken) ?? new IngestionHistory([]); + + foreach (var (source, state) in states) + { + if (state.Date != now.Date) + { + continue; + } + + // A record already stored for today survived a collector restart, so today's totals + // build on it rather than overwriting what the previous incarnation had accumulated. + state.Base ??= history.Days.FirstOrDefault(day => day.Date == state.Date && day.Source == source) + ?? new IngestionDay(state.Date, source, 0, 0, 0, 0, 0, 0, 0, 0); + + history.Days.RemoveAll(day => day.Date == state.Date && day.Source == source); + history.Days.Add(state.ToDay(source)); + } + + history.Days.RemoveAll(day => day.Date < now.Date.AddDays(-HistoryDays)); + + await dataStore.SaveIngestionHistory(history, cancellationToken); + } + + class SourceState + { + public DateTime Date { get; private set; } = DateTime.MinValue; + public long[] HourMessages { get; } = new long[24]; + public double[] HourBusySeconds { get; } = new double[24]; + public double[] HourStorageSeconds { get; } = new double[24]; + public long LagOverOneMinute { get; set; } + public long LagOverTenMinutes { get; set; } + public long LagOverSixtyMinutes { get; set; } + public long LagKnown { get; set; } + public Dictionary Baselines { get; } = []; + public IngestionDay? Base { get; set; } + + public void Reset(DateTime date) + { + Date = date; + Array.Clear(HourMessages); + Array.Clear(HourBusySeconds); + Array.Clear(HourStorageSeconds); + LagOverOneMinute = 0; + LagOverTenMinutes = 0; + LagOverSixtyMinutes = 0; + LagKnown = 0; + Base = null; + } + + public IngestionDay ToDay(string source) + { + var peakHour = 0; + + for (var hour = 1; hour < 24; hour++) + { + if (HourMessages[hour] > HourMessages[peakHour]) + { + peakHour = hour; + } + } + + var peakMessages = HourMessages[peakHour]; + var basePeakWins = Base!.PeakHourMessages >= peakMessages; + + return new IngestionDay( + Date, + source, + Base.Messages + HourMessages.Sum(), + basePeakWins ? Base.PeakHourMessages : peakMessages, + basePeakWins ? Base.PeakHourBusySeconds : HourBusySeconds[peakHour], + basePeakWins ? Base.PeakHourStorageSeconds : HourStorageSeconds[peakHour], + Base.LagOverOneMinuteMessages + LagOverOneMinute, + Base.LagOverTenMinutesMessages + LagOverTenMinutes, + Base.LagOverSixtyMinutesMessages + LagOverSixtyMinutes, + Base.LagKnownMessages + LagKnown); + } + } + + readonly Dictionary states = []; + + const string LocalKey = "local"; + const int HistoryDays = 400; +} diff --git a/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs b/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs index 9e8db1f5e4..275d36eb6a 100644 --- a/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs +++ b/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs @@ -17,6 +17,7 @@ public static IHostApplicationBuilder AddLicensingComponent(this IHostApplicatio var throughputSettings = new ThroughputSettings(serviceControlQueue, errorQueue, transportType, customerName, serviceControlVersion); services.AddSingleton(throughputSettings); services.AddHostedService(); + services.AddHostedService(); services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index d979c56bd1..cdd2865cda 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -207,6 +207,17 @@ public async Task GenerateThroughputReport(string spVersion, DateT } } + var ingestionHistory = await dataStore.GetIngestionHistory(cancellationToken); + + if (ingestionHistory is not null) + { + var windowStart = report.StartTime.UtcDateTime.Date; + var windowEnd = ((DateTime)reportEndDate).Date; + + EmitIngestion(report, "Ingestion.Error", IngestionHistory.ErrorSource, ingestionHistory, windowStart, windowEnd, includeStorage: true); + EmitIngestion(report, "Audit.Ingestion", IngestionHistory.AuditSource, ingestionHistory, windowStart, windowEnd, includeStorage: false); + } + foreach (var environmentDataProvider in environmentDataProviders) { EnvironmentDatum[] environmentData; @@ -243,6 +254,30 @@ public async Task GenerateThroughputReport(string spVersion, DateT return throughputReport; } + static void EmitIngestion(Report.Report report, string prefix, string source, IngestionHistory ingestionHistory, DateTime windowStart, DateTime windowEnd, bool includeStorage) + { + var days = ingestionHistory.Days + .Where(day => day.Source == source && day.Date >= windowStart && day.Date <= windowEnd) + .ToArray(); + + if (days.Length == 0) + { + return; + } + + var environmentData = report.EnvironmentInformation.EnvironmentData; + var peakDay = days.MaxBy(day => day.PeakHourMessages)!; + + environmentData[$"{prefix}.AvgDailyMessages"] = Math.Round(days.Average(day => day.Messages), MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); + environmentData[$"{prefix}.PeakHourMessages"] = peakDay.PeakHourMessages.ToString(CultureInfo.InvariantCulture); + environmentData[$"{prefix}.PeakHourBusyPercent"] = Math.Min(100, Math.Round(peakDay.PeakHourBusySeconds / 3600 * 100, MidpointRounding.AwayFromZero)).ToString("F0", CultureInfo.InvariantCulture); + + if (includeStorage && peakDay.PeakHourMessages > 0) + { + environmentData[$"{prefix}.PeakHourStorageMsPerMessage"] = Math.Round(peakDay.PeakHourStorageSeconds / peakDay.PeakHourMessages * 1000, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); + } + } + async IAsyncEnumerable GetDistinctEndpointData(DateOnly? throughputMaxDate, [EnumeratorCancellation] CancellationToken cancellationToken) { var endpoints = (await dataStore.GetAllEndpoints(false, cancellationToken)).ToArray(); diff --git a/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs b/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs new file mode 100644 index 0000000000..ba6aa4db3c --- /dev/null +++ b/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Api.Contracts +{ + using System; + + public class RemoteIngestionCounters + { + public string ApiUri { get; set; } + + /// + /// Null when the remote could not be reached or predates the environment endpoint. + /// + public RemoteIngestionCounterValues Counters { get; set; } + } + + public class RemoteIngestionCounterValues + { + public DateTime ProcessStartUtc { get; set; } + public long MessagesTotal { get; set; } + public double BusySecondsTotal { get; set; } + public double StorageSecondsTotal { get; set; } + public long LagOverOneMinuteMessages { get; set; } + public long LagOverTenMinutesMessages { get; set; } + public long LagOverSixtyMinutesMessages { get; set; } + public long LagKnownMessages { get; set; } + } +} diff --git a/src/ServiceControl.Api/IConfigurationApi.cs b/src/ServiceControl.Api/IConfigurationApi.cs index b16c977437..73b8c0ab59 100644 --- a/src/ServiceControl.Api/IConfigurationApi.cs +++ b/src/ServiceControl.Api/IConfigurationApi.cs @@ -13,5 +13,7 @@ public interface IConfigurationApi Task GetRemoteConfigs(CancellationToken cancellationToken = default); Task GetRemoteEnvironments(CancellationToken cancellationToken = default); + + Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default); } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 7ae97bb9fa..9ef31c0250 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.AcceptanceTests { + using System; using System.Globalization; using System.Text.Json.Nodes; using System.Threading.Tasks; @@ -48,6 +49,9 @@ public async Task Should_serve_host_and_storage_facts() Assert.That((string)storageIdentity["engine"], Is.Not.Null.And.Not.Empty); Assert.That((string)storageIdentity["server_hash"], Has.Length.EqualTo(64)); Assert.That((string)storageIdentity["database_hash"], Has.Length.EqualTo(64)); + Assert.That(body["ingestion_counters"], Is.Not.Null); + Assert.That((DateTime)body["ingestion_counters"]["process_start_utc"], Is.Not.EqualTo(default(DateTime))); + Assert.That((long)body["ingestion_counters"]["messages_total"], Is.GreaterThanOrEqualTo(0)); } } diff --git a/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs b/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs index 1357d7705b..745565a561 100644 --- a/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.UnitTests.Auditing.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; @@ -7,6 +8,7 @@ namespace ServiceControl.Audit.UnitTests.Auditing.Metrics; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Audit.Auditing.Metrics; +using ServiceControl.Infrastructure.Ingestion; /// /// Instrument names are what dashboards and alerts are built on, so they are a published contract @@ -39,7 +41,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() listener.Start(); - _ = new IngestionMetrics(MeterFactory); + _ = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); Assert.That(published.Order(), Is.EqualTo(new[] { @@ -53,7 +55,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() [Test] public void Concurrent_batch_failures_are_all_counted() { - var metrics = new IngestionMetrics(MeterFactory); + var metrics = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); const int failedBatches = 1000; diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs index 13897866cd..36f6aca7c4 100644 --- a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -9,6 +9,7 @@ namespace ServiceControl.Audit.UnitTests.Infrastructure using ServiceControl.Audit.Infrastructure.WebApi; using ServiceControl.Audit.Persistence; using ServiceControl.Infrastructure; + using ServiceControl.Infrastructure.Ingestion; [TestFixture] class EnvironmentControllerTests @@ -20,7 +21,7 @@ public async Task Merges_the_data_of_every_provider() [ new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB")), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], [], NullLogger.Instance); + ], [], Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -40,7 +41,7 @@ public async Task A_datum_that_cannot_be_read_costs_only_its_own_key() new Provider( EnvironmentDatum.Value("Storage.Type", () => "RavenDB"), EnvironmentDatum.Deferred("Storage.ServerVersion", _ => throw new InvalidOperationException("server down"))) - ], [], NullLogger.Instance); + ], [], Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -59,7 +60,7 @@ public async Task A_provider_that_cannot_list_its_data_costs_only_its_own_keys() [ new ThrowingProvider(), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], [], NullLogger.Instance); + ], [], Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -83,7 +84,7 @@ public void Cancellation_propagates() cancellationToken.ThrowIfCancellationRequested(); return new ValueTask("unreached"); })) - ], [], NullLogger.Instance); + ], [], Counters(), NullLogger.Instance); Assert.ThrowsAsync(() => controller.Environment(cancellationTokenSource.Token)); } @@ -94,6 +95,7 @@ public async Task Serves_the_hashed_storage_identity() var controller = new EnvironmentController( [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], [new IdentityProvider(new StorageIdentity("RavenDB", "http://server:8080", "audit", null))], + Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -116,6 +118,7 @@ public async Task A_storage_identity_that_cannot_be_read_is_not_served() var controller = new EnvironmentController( [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], [new ThrowingIdentityProvider()], + Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -128,6 +131,8 @@ [new ThrowingIdentityProvider()], } } + static IngestionCounters Counters() => new(TimeProvider.System); + class Provider(params EnvironmentDatum[] data) : IEnvironmentDataProvider { public IEnumerable GetData() => data; diff --git a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs index cdd6788eab..68009f7919 100644 --- a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs +++ b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.Auditing.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; @@ -8,6 +9,7 @@ using NServiceBus; using NServiceBus.Transport; using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Infrastructure.Ingestion.Metrics; public class IngestionMetrics @@ -17,8 +19,10 @@ public class IngestionMetrics public static readonly string BatchDurationInstrumentName = $"{InstrumentPrefix}.batch_duration_seconds"; public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration_seconds"; - public IngestionMetrics(IMeterFactory meterFactory) + public IngestionMetrics(IMeterFactory meterFactory, IngestionCounters counters) { + this.counters = counters; + var meter = meterFactory.Create(MeterName, MeterVersion); batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "seconds", "Message batch processing duration in seconds"); @@ -27,11 +31,32 @@ public IngestionMetrics(IMeterFactory meterFactory) failureCounter = meter.CreateCounter($"{InstrumentPrefix}.failures_total", description: "Audit ingestion failure count"); } - public MessageMetrics BeginIngestion(MessageContext messageContext) => new(GetMessageTags(messageContext.Headers), ingestionDuration); + public MessageMetrics BeginIngestion(MessageContext messageContext) + { + RecordLag(messageContext.Headers); + + return new(GetMessageTags(messageContext.Headers), ingestionDuration); + } public FailureMetrics BeginErrorHandling(ErrorContext errorContext) => new(GetMessageTags(errorContext.Headers), failureCounter); - public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome); + public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome, counters.RecordBatch); + + void RecordLag(Dictionary headers) + { + if (!headers.TryGetValue(Headers.ProcessingEnded, out var processingEnded)) + { + return; + } + + try + { + counters.RecordLag(DateTime.UtcNow - DateTimeOffsetHelper.ToDateTimeOffset(processingEnded).UtcDateTime); + } + catch (FormatException) + { + } + } static TagList GetMessageTags(Dictionary headers) { @@ -66,6 +91,7 @@ void RecordBatchOutcome(bool success) readonly Histogram batchDuration; readonly Histogram ingestionDuration; readonly Counter failureCounter; + readonly IngestionCounters counters; const string MeterVersion = "0.1.0"; const string InstrumentPrefix = "sc.audit.ingestion"; diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index da57b3350f..965edaafd8 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -52,6 +52,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, services.AddSingleton(settings); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); // required by the ingestion custom check which is auto-loaded diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs index c13a838a74..2a264fabb8 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -10,12 +10,13 @@ namespace ServiceControl.Audit.Infrastructure.WebApi using Microsoft.Extensions.Logging; using ServiceControl.Audit.Persistence; using ServiceControl.Infrastructure; + using ServiceControl.Infrastructure.Ingestion; // the /api/environment endpoint is polled by the primary instance for its usage report. This currently needs to be anonymous [AllowAnonymous] [ApiController] [Route("api")] - public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, ILogger logger) : ControllerBase + public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, IngestionCounters ingestionCounters, ILogger logger) : ControllerBase { [Route("environment")] [HttpGet] @@ -59,7 +60,8 @@ public async Task Environment(CancellationToken cancellationToke { EnvironmentData = environmentData, MachineIdHash = MachineIdentity.Hash, - StorageIdentity = await ReadStorageIdentity(cancellationToken) + StorageIdentity = await ReadStorageIdentity(cancellationToken), + IngestionCounters = IngestionCountersResponse.From(ingestionCounters.GetSnapshot()) }); } @@ -106,6 +108,31 @@ public class EnvironmentDataResponse public Dictionary EnvironmentData { get; set; } public string MachineIdHash { get; set; } public StorageIdentityResponse StorageIdentity { get; set; } + public IngestionCountersResponse IngestionCounters { get; set; } + } + + public class IngestionCountersResponse + { + public DateTime ProcessStartUtc { get; set; } + public long MessagesTotal { get; set; } + public double BusySecondsTotal { get; set; } + public double StorageSecondsTotal { get; set; } + public long LagOverOneMinuteMessages { get; set; } + public long LagOverTenMinutesMessages { get; set; } + public long LagOverSixtyMinutesMessages { get; set; } + public long LagKnownMessages { get; set; } + + public static IngestionCountersResponse From(IngestionCountersSnapshot snapshot) => new() + { + ProcessStartUtc = snapshot.ProcessStartUtc, + MessagesTotal = snapshot.Messages, + BusySecondsTotal = snapshot.BusySeconds, + StorageSecondsTotal = snapshot.StorageSeconds, + LagOverOneMinuteMessages = snapshot.LagOverOneMinuteMessages, + LagOverTenMinutesMessages = snapshot.LagOverTenMinutesMessages, + LagOverSixtyMinutesMessages = snapshot.LagOverSixtyMinutesMessages, + LagKnownMessages = snapshot.LagKnownMessages + }; } public class StorageIdentityResponse diff --git a/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs new file mode 100644 index 0000000000..6f918aea0c --- /dev/null +++ b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Infrastructure.Ingestion; + +using System; +using System.Threading; + +/// +/// Running totals since process start, kept in memory for the licensing collector to poll and turn +/// into hourly deltas. Restarts reset them, which the poller detects through the start time. +/// +public class IngestionCounters(TimeProvider timeProvider) +{ + public DateTime ProcessStartUtc { get; } = timeProvider.GetUtcNow().UtcDateTime; + + public void RecordBatch(int completedSize, TimeSpan duration) + { + if (completedSize > 0) + { + Interlocked.Add(ref messages, completedSize); + } + + Interlocked.Add(ref busyTicks, duration.Ticks); + } + + public void RecordStorage(TimeSpan duration) => Interlocked.Add(ref storageTicks, duration.Ticks); + + public void RecordLag(TimeSpan lag) + { + Interlocked.Increment(ref lagKnown); + + if (lag > TimeSpan.FromMinutes(1)) + { + Interlocked.Increment(ref lagOverOneMinute); + } + + if (lag > TimeSpan.FromMinutes(10)) + { + Interlocked.Increment(ref lagOverTenMinutes); + } + + if (lag > TimeSpan.FromMinutes(60)) + { + Interlocked.Increment(ref lagOverSixtyMinutes); + } + } + + public IngestionCountersSnapshot GetSnapshot() => new( + ProcessStartUtc, + Interlocked.Read(ref messages), + TimeSpan.FromTicks(Interlocked.Read(ref busyTicks)).TotalSeconds, + TimeSpan.FromTicks(Interlocked.Read(ref storageTicks)).TotalSeconds, + Interlocked.Read(ref lagOverOneMinute), + Interlocked.Read(ref lagOverTenMinutes), + Interlocked.Read(ref lagOverSixtyMinutes), + Interlocked.Read(ref lagKnown)); + + long messages; + long busyTicks; + long storageTicks; + long lagOverOneMinute; + long lagOverTenMinutes; + long lagOverSixtyMinutes; + long lagKnown; +} + +public record IngestionCountersSnapshot( + DateTime ProcessStartUtc, + long Messages, + double BusySeconds, + double StorageSeconds, + long LagOverOneMinuteMessages, + long LagOverTenMinutesMessages, + long LagOverSixtyMinutesMessages, + long LagKnownMessages); diff --git a/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs b/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs index 4ef08a1ed8..0b98dccaa2 100644 --- a/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs +++ b/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs @@ -8,7 +8,7 @@ namespace ServiceControl.Infrastructure.Ingestion.Metrics; /// One batch write. Leaving the scope without calling is what records the /// batch as failed, so nothing has to be told about the exception that ended it. /// -public sealed class BatchMetrics(int maxBatchSize, Histogram batchDuration, Action recordOutcome) : IDisposable +public sealed class BatchMetrics(int maxBatchSize, Histogram batchDuration, Action recordOutcome, Action recordCompletion = null) : IDisposable { public void Complete(int batchSize) => completedSize = batchSize; @@ -17,6 +17,7 @@ public void Dispose() var succeeded = completedSize > 0; recordOutcome(succeeded); + recordCompletion?.Invoke(succeeded ? completedSize : 0, stopwatch.Elapsed); var result = succeeded ? completedSize == maxBatchSize ? "full" : "partial" diff --git a/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs b/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs index 16535614ac..94c7365c79 100644 --- a/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs +++ b/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs @@ -7,9 +7,13 @@ namespace ServiceControl.Infrastructure.Ingestion.Metrics; /// /// How long the scope was open, in seconds, and nothing else. /// -public sealed class DurationScope(Histogram duration) : IDisposable +public sealed class DurationScope(Histogram duration, Action recordDuration = null) : IDisposable { - public void Dispose() => duration.Record(stopwatch.Elapsed.TotalSeconds); + public void Dispose() + { + duration.Record(stopwatch.Elapsed.TotalSeconds); + recordDuration?.Invoke(stopwatch.Elapsed); + } readonly Stopwatch stopwatch = Stopwatch.StartNew(); } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index e57519e46b..eed26c0b8d 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -309,6 +309,12 @@ public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditEnvironmentMetadata, auditEnvironmentMetadata, token), cancellationToken); + public Task GetIngestionHistory(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.GetSetting(SettingKeys.IngestionHistory, token), cancellationToken); + + public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.IngestionHistory, ingestionHistory, token), cancellationToken); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => ExecuteWithDbContext(async (context, token) => await context.GetSetting>(SettingKeys.ReportMasks, token) ?? [], cancellationToken); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index 5f01a2ddf9..6279d9a3c9 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -7,6 +7,7 @@ static class SettingKeys public const string BrokerMetadata = "BrokerMetadata"; public const string AuditServiceMetadata = "AuditServiceMetadata"; public const string AuditEnvironmentMetadata = "AuditEnvironmentMetadata"; + public const string IngestionHistory = "IngestionHistory"; public const string ReportMasks = "ReportMasks"; public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string NotificationEmails = "NotificationEmails"; diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 4508e85843..236dc4715e 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -23,6 +23,7 @@ class LicensingDataStore( internal const string ThroughputTimeSeriesName = "INC: throughput data"; const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; + const string IngestionHistoryDocumentId = "IngestionHistory"; const string BrokerMetadataDocumentId = "BrokerMetadata"; const string ReportMasksDocumentId = "ReportMasks"; const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; @@ -336,6 +337,23 @@ public async Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnv await session.SaveChangesAsync(cancellationToken); } + public async Task GetIngestionHistory(CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + return await session.LoadAsync(IngestionHistoryDocumentId, cancellationToken); + } + + public async Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + await session.StoreAsync(ingestionHistory, IngestionHistoryDocumentId, cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + public async Task> GetReportMasks(CancellationToken cancellationToken = default) { var store = await storeProvider.GetDocumentStore(cancellationToken); diff --git a/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs b/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs new file mode 100644 index 0000000000..956a86fad3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.Infrastructure; + +using System; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; + +[TestFixture] +public class IngestionCountersTests +{ + [Test] + public void Batches_add_their_messages_and_their_time() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordBatch(10, TimeSpan.FromSeconds(2)); + counters.RecordBatch(5, TimeSpan.FromSeconds(1)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.Messages, Is.EqualTo(15)); + Assert.That(snapshot.BusySeconds, Is.EqualTo(3).Within(0.001)); + } + } + + [Test] + public void A_failed_batch_costs_time_but_stores_no_messages() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordBatch(0, TimeSpan.FromSeconds(2)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.Messages, Is.EqualTo(0)); + Assert.That(snapshot.BusySeconds, Is.EqualTo(2).Within(0.001)); + } + } + + [Test] + public void Lag_lands_in_every_bucket_it_exceeds() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordLag(TimeSpan.FromSeconds(30)); + counters.RecordLag(TimeSpan.FromMinutes(5)); + counters.RecordLag(TimeSpan.FromMinutes(30)); + counters.RecordLag(TimeSpan.FromHours(2)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.LagKnownMessages, Is.EqualTo(4)); + Assert.That(snapshot.LagOverOneMinuteMessages, Is.EqualTo(3)); + Assert.That(snapshot.LagOverTenMinutesMessages, Is.EqualTo(2)); + Assert.That(snapshot.LagOverSixtyMinutesMessages, Is.EqualTo(1)); + } + } + + [Test] + public void Storage_time_accumulates() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordStorage(TimeSpan.FromMilliseconds(500)); + counters.RecordStorage(TimeSpan.FromMilliseconds(250)); + + Assert.That(counters.GetSnapshot().StorageSeconds, Is.EqualTo(0.75).Within(0.001)); + } +} diff --git a/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs b/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs index 6da39ae9a8..86560cd816 100644 --- a/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs +++ b/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs @@ -1,11 +1,13 @@ namespace ServiceControl.UnitTests.Operations.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Operations.Metrics; /// @@ -39,7 +41,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() listener.Start(); - _ = new IngestionMetrics(MeterFactory); + _ = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); Assert.That(published.Order(), Is.EqualTo(new[] { @@ -54,7 +56,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() [Test] public void Concurrent_batch_failures_are_all_counted() { - var metrics = new IngestionMetrics(MeterFactory); + var metrics = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); const int failedBatches = 1000; diff --git a/src/ServiceControl/ErrorIngestionSnapshotProvider.cs b/src/ServiceControl/ErrorIngestionSnapshotProvider.cs new file mode 100644 index 0000000000..2a0a81d182 --- /dev/null +++ b/src/ServiceControl/ErrorIngestionSnapshotProvider.cs @@ -0,0 +1,9 @@ +namespace Particular.ServiceControl; + +using global::ServiceControl.Infrastructure.Ingestion; +using Particular.LicensingComponent; + +class ErrorIngestionSnapshotProvider(IngestionCounters counters) : IErrorIngestionSnapshotProvider +{ + public IngestionCountersSnapshot GetSnapshot() => counters.GetSnapshot(); +} diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index 6b3ac7c6e0..ef4c4f250b 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -163,6 +163,8 @@ public static void AddServiceControlInstallers(this IHostApplicationBuilder host public static void AddTelemetry(this IHostApplicationBuilder hostBuilder, Settings settings) { hostBuilder.Services.TryAddSingleton(TimeProvider.System); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 75755b8a89..f1471f9ef0 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -207,6 +207,57 @@ public async Task GetRemoteEnvironments(CancellationToken c return await Task.WhenAll(tasks); } + public async Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) + { + var tasks = settings.RemoteInstances + .Select(async remote => + { + RemoteIngestionCounterValues counters = null; + HttpClient httpClient = httpClientFactory.CreateClient(remote.InstanceId); + + try + { + using var response = await httpClient.GetAsync("/api/environment", cancellationToken); + + if (response.IsSuccessStatusCode) + { + await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); + var body = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); + + if (body?.AsObject().TryGetPropertyValue("ingestion_counters", out var node) == true && node is JsonObject countersObject) + { + counters = new RemoteIngestionCounterValues + { + ProcessStartUtc = countersObject["process_start_utc"]!.GetValue(), + MessagesTotal = countersObject["messages_total"]!.GetValue(), + BusySecondsTotal = countersObject["busy_seconds_total"]!.GetValue(), + StorageSecondsTotal = countersObject["storage_seconds_total"]!.GetValue(), + LagOverOneMinuteMessages = countersObject["lag_over_one_minute_messages"]!.GetValue(), + LagOverTenMinutesMessages = countersObject["lag_over_ten_minutes_messages"]!.GetValue(), + LagOverSixtyMinutesMessages = countersObject["lag_over_sixty_minutes_messages"]!.GetValue(), + LagKnownMessages = countersObject["lag_known_messages"]!.GetValue() + }; + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + } + + return new RemoteIngestionCounters + { + ApiUri = remote.BaseAddress, + Counters = counters + }; + }); + + return await Task.WhenAll(tasks); + } + async Task LocalStorageIdentity(CancellationToken cancellationToken) { var provider = storageIdentityProviders.FirstOrDefault(); diff --git a/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs b/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs index ffbc5d63f7..32a1b138ee 100644 --- a/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs +++ b/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs @@ -1,11 +1,14 @@ namespace ServiceControl.Operations.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Threading; +using NServiceBus; using NServiceBus.Transport; using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Infrastructure.Ingestion.Metrics; public class IngestionMetrics @@ -16,8 +19,9 @@ public class IngestionMetrics public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration_seconds"; public static readonly string StorageDurationInstrumentName = $"{InstrumentPrefix}.storage_duration_seconds"; - public IngestionMetrics(IMeterFactory meterFactory) + public IngestionMetrics(IMeterFactory meterFactory, IngestionCounters counters) { + this.counters = counters; var meter = meterFactory.Create(MeterName, MeterVersion); batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "seconds", description: "Message batch processing duration in seconds"); @@ -27,17 +31,38 @@ public IngestionMetrics(IMeterFactory meterFactory) failureCounter = meter.CreateCounter($"{InstrumentPrefix}.failures_total", description: "Error ingestion failure count"); } - public MessageMetrics BeginIngestion(MessageContext messageContext) => new(GetMessageTags(messageContext.Headers), ingestionDuration); + public MessageMetrics BeginIngestion(MessageContext messageContext) + { + RecordLag(messageContext.Headers); + + return new(GetMessageTags(messageContext.Headers), ingestionDuration); + } public FailureMetrics BeginErrorHandling(ErrorContext errorContext) => new(GetMessageTags(errorContext.Headers), failureCounter); - public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome); + public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome, counters.RecordBatch); /// /// The storage write on its own, which is the part of a batch that is neither announcing nor /// forwarding. /// - public DurationScope MeasureStorageWrite() => new(storageDuration); + public DurationScope MeasureStorageWrite() => new(storageDuration, counters.RecordStorage); + + void RecordLag(Dictionary headers) + { + if (!headers.TryGetValue("NServiceBus.TimeOfFailure", out var timeOfFailure)) + { + return; + } + + try + { + counters.RecordLag(DateTime.UtcNow - DateTimeOffsetHelper.ToDateTimeOffset(timeOfFailure).UtcDateTime); + } + catch (FormatException) + { + } + } // The same split the ingestor makes: a retry acknowledgement resolves a message, everything // else records a failure, and they cost quite different amounts of work. @@ -70,6 +95,7 @@ void RecordBatchOutcome(bool success) readonly Histogram ingestionDuration; readonly Histogram storageDuration; readonly Counter failureCounter; + readonly IngestionCounters counters; const string MeterVersion = "0.1.0"; const string InstrumentPrefix = "sc.error.ingestion"; From da35a7e668d0c041650da952f72a4c9c40eb4430 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 16:36:56 +1000 Subject: [PATCH 5/9] Report health signals for the error and audit sides Health.Error.FailedImports and Health.Error.RetentionBehindHours come from the primary's persister: failed import rows counted from the store, and how many hours the oldest failed message sits past the retention window on the EF persisters, with RavenDB omitting the retention key because expiry is engine driven. The audit instance serves its failed audit import count on the environment endpoint, summed across instances with shared databases counted once. Restarts are the process start time changes the hourly collector observes between polls, and the lag buckets it accumulates become Health.*.LagOver1MinPercent, LagOver10MinPercent and LagOver60MinPercent over the report window, emitted only when any message carried a usable timestamp. --- .../IngestionHistory.cs | 8 ++++- ...tionHistoryCollectorHostedService_Tests.cs | 6 +++- ...tor_Report_EnvironmentInformation_Tests.cs | 15 +++++++--- .../AuditEnvironmentDataAggregator.cs | 2 +- .../IngestionHistoryCollectorHostedService.cs | 15 ++++++++-- .../ThroughputCollector.cs | 20 +++++++++++-- .../When_environment_data_is_requested.cs | 1 + .../RavenEnvironmentDataProvider.cs | 3 +- .../EFEnvironmentDataProvider.cs | 30 +++++++++++++++++++ .../RavenEnvironmentDataProvider.cs | 11 +++++++ .../HealthEnvironmentDataTests.cs | 30 +++++++++++++++++++ .../HealthEnvironmentDataTests.cs | 30 +++++++++++++++++++ 12 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs diff --git a/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs b/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs index 1e32b4f457..80e7d2f72e 100644 --- a/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs +++ b/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs @@ -25,4 +25,10 @@ public record IngestionDay( long LagOverOneMinuteMessages, long LagOverTenMinutesMessages, long LagOverSixtyMinutesMessages, - long LagKnownMessages); + long LagKnownMessages) +{ + /// + /// Process start time changes observed between polls, summed across a source's instances. + /// + public long Restarts { get; init; } +} diff --git a/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs index 6ec6176b7e..95c4c67d16 100644 --- a/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs @@ -78,7 +78,11 @@ public async Task A_restarted_process_contributes_its_whole_counters_as_the_delt Assert.That(history, Is.Not.Null); var errorDay = history.Days.Single(day => day.Source == IngestionHistory.ErrorSource); - Assert.That(errorDay.Messages, Is.EqualTo(25)); + using (Assert.EnterMultipleScope()) + { + Assert.That(errorDay.Messages, Is.EqualTo(25)); + Assert.That(errorDay.Restarts, Is.EqualTo(1)); + } } async Task RunTwoPolls(FakeTimeProvider fakeTimeProvider, QueuedErrorSnapshots errorProvider, AuditQuery_WithIngestionSnapshots auditQuery) diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index d152317494..4c73f1ced3 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -290,9 +290,9 @@ public async Task Should_sum_sizes_and_counts_once_per_database() { await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( [ - new Dictionary { ["Storage.SizeGB"] = "10.0", ["Storage.MessageCount"] = "100", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, - new Dictionary { ["Storage.SizeGB"] = "10.2", ["Storage.MessageCount"] = "90", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, - new Dictionary { ["Storage.SizeGB"] = "5.0", ["Storage.MessageCount"] = "10" } + new Dictionary { ["Storage.SizeGB"] = "10.0", ["Storage.MessageCount"] = "100", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "10.2", ["Storage.MessageCount"] = "90", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "5.0", ["Storage.MessageCount"] = "10", ["Health.FailedImports"] = "4" } ])); var report = await ThroughputCollector.GenerateThroughputReport("", null); @@ -302,6 +302,7 @@ await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( { Assert.That(environmentData, Does.ContainKey("Audit.Storage.SizeGB").WithValue("15.2")); Assert.That(environmentData, Does.ContainKey("Audit.Storage.MessageCount").WithValue("110")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.FailedImports").WithValue("7")); Assert.That(environmentData.Keys, Has.None.StartsWith("Audit._")); } } @@ -322,7 +323,7 @@ public async Task Should_report_ingestion_rates_from_days_inside_the_report_wind var outsideWindow = reportEndDate.AddDays(-10); await DataStore.SaveIngestionHistory(new IngestionHistory( [ - new IngestionDay(insideWindow, IngestionHistory.ErrorSource, 2400, 600, 1800, 60, 0, 0, 0, 0), + new IngestionDay(insideWindow, IngestionHistory.ErrorSource, 2400, 600, 1800, 60, 100, 50, 10, 1000) { Restarts = 2 }, new IngestionDay(insideWindow, IngestionHistory.AuditSource, 1200, 300, 360, 0, 0, 0, 0, 0), new IngestionDay(outsideWindow, IngestionHistory.ErrorSource, 999999, 99999, 3600, 999, 0, 0, 0, 0) ])); @@ -340,6 +341,12 @@ await DataStore.SaveIngestionHistory(new IngestionHistory( Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourMessages").WithValue("300")); Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourBusyPercent").WithValue("10")); Assert.That(environmentData, Does.Not.ContainKey("Audit.Ingestion.PeakHourStorageMsPerMessage")); + Assert.That(environmentData, Does.ContainKey("Health.Error.Restarts").WithValue("2")); + Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver1MinPercent").WithValue("10")); + Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver10MinPercent").WithValue("5")); + Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver60MinPercent").WithValue("1")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.Restarts").WithValue("0")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Health.LagOver1MinPercent")); } } diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs index 74b1c8c445..8ce6627e7d 100644 --- a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -51,7 +51,7 @@ static string Aggregate(string key, (string Value, string DatabaseKey)[] contrib } } - if (key == "Storage.MessageCount") + if (key is "Storage.MessageCount" or "Health.FailedImports") { var sum = SumOncePerDatabase(contributions, value => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) ? count : null); diff --git a/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs b/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs index c655598771..bd60e55f97 100644 --- a/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs +++ b/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs @@ -96,7 +96,13 @@ bool Accumulate(string source, DateTime now, List snapsh { if (state.Baselines.TryGetValue(apiUri, out var last)) { - var delta = snapshot.ProcessStartUtc != last.ProcessStartUtc ? snapshot : Diff(snapshot, last); + var restarted = snapshot.ProcessStartUtc != last.ProcessStartUtc; + var delta = restarted ? snapshot : Diff(snapshot, last); + + if (restarted) + { + state.Restarts++; + } state.HourMessages[now.Hour] += delta.Messages; state.HourBusySeconds[now.Hour] += delta.BusySeconds; @@ -160,6 +166,7 @@ class SourceState public long LagOverTenMinutes { get; set; } public long LagOverSixtyMinutes { get; set; } public long LagKnown { get; set; } + public long Restarts { get; set; } public Dictionary Baselines { get; } = []; public IngestionDay? Base { get; set; } @@ -173,6 +180,7 @@ public void Reset(DateTime date) LagOverTenMinutes = 0; LagOverSixtyMinutes = 0; LagKnown = 0; + Restarts = 0; Base = null; } @@ -201,7 +209,10 @@ public IngestionDay ToDay(string source) Base.LagOverOneMinuteMessages + LagOverOneMinute, Base.LagOverTenMinutesMessages + LagOverTenMinutes, Base.LagOverSixtyMinutesMessages + LagOverSixtyMinutes, - Base.LagKnownMessages + LagKnown); + Base.LagKnownMessages + LagKnown) + { + Restarts = Base.Restarts + Restarts + }; } } diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index cdd2865cda..4f30c7349b 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -214,8 +214,8 @@ public async Task GenerateThroughputReport(string spVersion, DateT var windowStart = report.StartTime.UtcDateTime.Date; var windowEnd = ((DateTime)reportEndDate).Date; - EmitIngestion(report, "Ingestion.Error", IngestionHistory.ErrorSource, ingestionHistory, windowStart, windowEnd, includeStorage: true); - EmitIngestion(report, "Audit.Ingestion", IngestionHistory.AuditSource, ingestionHistory, windowStart, windowEnd, includeStorage: false); + EmitIngestion(report, "Ingestion.Error", "Health.Error", IngestionHistory.ErrorSource, ingestionHistory, windowStart, windowEnd, includeStorage: true); + EmitIngestion(report, "Audit.Ingestion", "Audit.Health", IngestionHistory.AuditSource, ingestionHistory, windowStart, windowEnd, includeStorage: false); } foreach (var environmentDataProvider in environmentDataProviders) @@ -254,7 +254,7 @@ public async Task GenerateThroughputReport(string spVersion, DateT return throughputReport; } - static void EmitIngestion(Report.Report report, string prefix, string source, IngestionHistory ingestionHistory, DateTime windowStart, DateTime windowEnd, bool includeStorage) + static void EmitIngestion(Report.Report report, string prefix, string healthPrefix, string source, IngestionHistory ingestionHistory, DateTime windowStart, DateTime windowEnd, bool includeStorage) { var days = ingestionHistory.Days .Where(day => day.Source == source && day.Date >= windowStart && day.Date <= windowEnd) @@ -276,8 +276,22 @@ static void EmitIngestion(Report.Report report, string prefix, string source, In { environmentData[$"{prefix}.PeakHourStorageMsPerMessage"] = Math.Round(peakDay.PeakHourStorageSeconds / peakDay.PeakHourMessages * 1000, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); } + + environmentData[$"{healthPrefix}.Restarts"] = days.Sum(day => day.Restarts).ToString(CultureInfo.InvariantCulture); + + var lagKnown = days.Sum(day => day.LagKnownMessages); + + if (lagKnown > 0) + { + environmentData[$"{healthPrefix}.LagOver1MinPercent"] = LagPercent(days.Sum(day => day.LagOverOneMinuteMessages), lagKnown); + environmentData[$"{healthPrefix}.LagOver10MinPercent"] = LagPercent(days.Sum(day => day.LagOverTenMinutesMessages), lagKnown); + environmentData[$"{healthPrefix}.LagOver60MinPercent"] = LagPercent(days.Sum(day => day.LagOverSixtyMinutesMessages), lagKnown); + } } + static string LagPercent(long over, long known) => + Math.Round(over / (double)known * 100, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); + async IAsyncEnumerable GetDistinctEndpointData(DateOnly? throughputMaxDate, [EnumeratorCancellation] CancellationToken cancellationToken) { var endpoints = (await dataStore.GetAllEndpoints(false, cancellationToken)).ToArray(); diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 9ef31c0250..53de5f381f 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -42,6 +42,7 @@ public async Task Should_serve_host_and_storage_facts() Assert.That(double.Parse((string)environmentData["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); Assert.That((string)environmentData["Storage.MessageCount"], Is.EqualTo("0")); Assert.That((string)environmentData["Storage.ServerEdition"], Is.EqualTo("NotApplicable")); + Assert.That((string)environmentData["Health.FailedImports"], Is.EqualTo("0")); Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); Assert.That((string)body["machine_id_hash"], Is.Not.Null.And.Not.Empty); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs index b08333c65c..364d2bfa96 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -9,7 +9,7 @@ namespace ServiceControl.Audit.Persistence.RavenDB using Raven.Client.ServerWide.Operations; using static ServiceControl.Audit.Persistence.EnvironmentDatum; - class RavenEnvironmentDataProvider(DatabaseConfiguration databaseConfiguration, IRavenDocumentStoreProvider documentStoreProvider) : IEnvironmentDataProvider + class RavenEnvironmentDataProvider(DatabaseConfiguration databaseConfiguration, IRavenDocumentStoreProvider documentStoreProvider, IFailedAuditStorage failedAuditStorage) : IEnvironmentDataProvider { public IEnumerable GetData() => [ @@ -22,6 +22,7 @@ public IEnumerable GetData() => Value("Storage.ServiceObjective", () => "NotApplicable"), Deferred("Storage.SizeGB", SizeGB), Deferred("Storage.MessageCount", ProcessedMessageCount), + Deferred("Health.FailedImports", async cancellationToken => (await failedAuditStorage.GetFailedAuditsCount(cancellationToken)).ToString(CultureInfo.InvariantCulture)), Value("Storage.FullTextSearch", () => databaseConfiguration.EnableFullTextSearch ? "Enabled" : "Disabled") ]; diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs index 153069b5ee..8fd95aa661 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.Implementation; +using System; using System.Globalization; using Abstractions; using DbContexts; @@ -37,6 +38,8 @@ public IEnumerable GetData() Deferred("Storage.SizeGB", async cancellationToken => SizeGB((await Footprint(cancellationToken))?.SizeGB)), Deferred("Storage.MessageCount", async cancellationToken => Count((await Footprint(cancellationToken))?.MessageCount)), Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), + Deferred("Health.Error.FailedImports", FailedImports), + Deferred("Health.Error.RetentionBehindHours", RetentionBehindHours), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => BodyStorageType(settings.BodyStorage)), Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture)), @@ -59,6 +62,33 @@ async ValueTask UnresolvedFailedMessages(CancellationToken cancellationT return Count(count); } + async ValueTask FailedImports(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var count = await dbContext.Set().CountAsync(cancellationToken); + + return Count(count); + } + + async ValueTask RetentionBehindHours(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var oldest = await dbContext.Set().MinAsync(m => (DateTime?)m.StatusChangedAt, cancellationToken); + + if (oldest is null) + { + return Count(0); + } + + var behind = DateTime.UtcNow - settings.ErrorRetentionPeriod - oldest.Value; + + return Count(behind > TimeSpan.Zero ? (long)behind.TotalHours : 0); + } + static string SizeGB(double? sizeGB) => sizeGB is null ? "Unknown" : sizeGB.Value.ToString("F1", CultureInfo.InvariantCulture); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs index a88aaf1c33..41ba550345 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -30,6 +30,7 @@ public IEnumerable GetData() => Deferred("Storage.SizeGB", SizeGB), Deferred("Storage.MessageCount", FailedMessageCount), Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), + Deferred("Health.Error.FailedImports", FailedImports), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => "RavenAttachments"), Value("Storage.LogLevel", () => settings.LogsMode), @@ -57,6 +58,16 @@ async ValueTask FailedMessageCount(CancellationToken cancellationToken) : "0"; } + async ValueTask FailedImports(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("FailedErrorImports", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + async ValueTask UnresolvedFailedMessages(CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs new file mode 100644 index 0000000000..534be6c71d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; + +class HealthEnvironmentDataTests : PersistenceTestBase +{ + [Test] + public async Task An_empty_schema_is_healthy() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs new file mode 100644 index 0000000000..534be6c71d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; + +class HealthEnvironmentDataTests : PersistenceTestBase +{ + [Test] + public async Task An_empty_schema_is_healthy() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + } + } +} From 7a51ce95ef69fb64d177480c4d6416ebb2a3274b Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 2 Oct 2026 14:18:42 +1000 Subject: [PATCH 6/9] Replace hourly ingestion history collector with live counters served on the environment endpoint Instead of polling ingestion counters hourly, accumulating deltas into daily records in the licensing store, and reading those records back into the report, each instance now computes ingestion and health summaries directly from its running totals at report time. Both the audit and error sides expose their counters through IEnvironmentDataProvider implementations that call IngestionSummary.Describe, which yields keys like Ingestion.AvgDailyMessages, Ingestion.BusyPercent, Health.UptimeHours, and Health.LagOver*Percent relative to the process uptime. The audit instance serves these on the environment endpoint alongside its existing data; the primary aggregates them under the Audit.* prefix when building the report, summing daily message rates across instances and taking the worst saturation and lag values. This removes IngestionHistoryCollectorHostedService, IngestionHistory and IngestionDay contracts, IErrorIngestionSnapshotProvider, GetRemoteIngestionCounters from IConfigurationApi, and GetIngestionHistory/SaveIngestionHistory from ILicensingDataStore and both persistence backends. --- .../IngestionHistory.cs | 34 --- .../InMemoryLicensingDataStore.cs | 9 - .../ILicensingDataStore.cs | 4 - .../AuditQuery_Tests.cs | 6 - ...tThroughputCollectorHostedService_Tests.cs | 15 -- .../Infrastructure/FakeConfigurationApi.cs | 2 - ...tionHistoryCollectorHostedService_Tests.cs | 165 ------------- .../MonitoringService_Tests.cs | 6 - ...tor_Report_EnvironmentInformation_Tests.cs | 40 +--- .../AuditEnvironmentDataAggregator.cs | 28 ++- .../AuditThroughput/AuditIngestionSnapshot.cs | 9 - .../AuditThroughput/AuditQuery.cs | 33 --- .../AuditThroughput/IAuditQuery.cs | 1 - .../IErrorIngestionSnapshotProvider.cs | 12 - .../IngestionHistoryCollectorHostedService.cs | 223 ------------------ ...LicensingComponentHostBuilderExtensions.cs | 1 - .../ThroughputCollector.cs | 49 ---- .../When_reporting_the_environment.cs | 3 +- .../Contracts/RemoteIngestionCounters.cs | 26 -- src/ServiceControl.Api/IConfigurationApi.cs | 2 - .../When_environment_data_is_requested.cs | 5 +- .../EnvironmentControllerTests.cs | 13 +- .../HostApplicationBuilderExtensions.cs | 1 + .../WebApi/EnvironmentController.cs | 31 +-- .../IngestionEnvironmentDataProvider.cs | 16 ++ .../Ingestion/IngestionCounters.cs | 4 - .../Ingestion/IngestionSummary.cs | 46 ++++ .../MachineIdentity.cs | 29 +-- .../Implementation/LicensingDataStore.cs | 6 - .../Infrastructure/SettingKeys.cs | 1 - .../Throughput/LicensingDataStore.cs | 18 -- .../Infrastructure/IngestionSummaryTests.cs | 78 ++++++ .../ErrorIngestionEnvironmentDataProvider.cs | 15 ++ .../ErrorIngestionSnapshotProvider.cs | 9 - .../HostApplicationBuilderExtensions.cs | 2 +- .../Infrastructure/Api/ConfigurationApi.cs | 51 ---- 36 files changed, 198 insertions(+), 795 deletions(-) delete mode 100644 src/Particular.LicensingComponent.Contracts/IngestionHistory.cs delete mode 100644 src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs delete mode 100644 src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs delete mode 100644 src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs delete mode 100644 src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs delete mode 100644 src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs create mode 100644 src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs create mode 100644 src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs create mode 100644 src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs create mode 100644 src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs delete mode 100644 src/ServiceControl/ErrorIngestionSnapshotProvider.cs diff --git a/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs b/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs deleted file mode 100644 index 80e7d2f72e..0000000000 --- a/src/Particular.LicensingComponent.Contracts/IngestionHistory.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Particular.LicensingComponent.Contracts; - -/// -/// Daily ingestion records kept by the hourly collector, per source (Error or Audit, the audit -/// side summed across instances). Never saved by versions that predate it, so a null read means -/// unknown and the report omits the keys. -/// -public record IngestionHistory(List Days) -{ - public const string ErrorSource = "Error"; - public const string AuditSource = "Audit"; -} - -/// The UTC day, at midnight. -/// The busiest hour's messages. -/// The time the ingestion loop spent processing batches during that busiest hour. -/// The time spent in storage writes during that busiest hour. Always zero for sources that do not measure storage separately. -public record IngestionDay( - DateTime Date, - string Source, - long Messages, - long PeakHourMessages, - double PeakHourBusySeconds, - double PeakHourStorageSeconds, - long LagOverOneMinuteMessages, - long LagOverTenMinutesMessages, - long LagOverSixtyMinutesMessages, - long LagKnownMessages) -{ - /// - /// Process start time changes observed between polls, summed across a source's instances. - /// - public long Restarts { get; init; } -} diff --git a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs index 95c9fdd4c8..611611fdee 100644 --- a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs @@ -15,7 +15,6 @@ public class InMemoryLicensingDataStore : ILicensingDataStore BrokerMetadata brokerMetadata = new(null, []); AuditServiceMetadata auditServiceMetadata = new([], []); AuditEnvironmentMetadata? auditEnvironmentMetadata; - IngestionHistory? ingestionHistory; List reportMasks = []; LicensedEndpointDetails? endpointDetails = null; @@ -177,14 +176,6 @@ public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironme return Task.CompletedTask; } - public Task GetIngestionHistory(CancellationToken cancellationToken = default) => Task.FromResult(ingestionHistory); - - public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) - { - this.ingestionHistory = ingestionHistory; - return Task.CompletedTask; - } - public Task> GetReportMasks(CancellationToken cancellationToken = default) => Task.FromResult(reportMasks); public Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default) diff --git a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs index 3f5e3c6def..bbf997ecf5 100644 --- a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs @@ -40,10 +40,6 @@ Task RecordEndpointThroughput(string endpointName, ThroughputSource throughputSo Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default); Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default); - - Task GetIngestionHistory(CancellationToken cancellationToken = default); - - Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default); Task> GetReportMasks(CancellationToken cancellationToken = default); Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default); diff --git a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs index b70b8dfed6..b64a0c12c8 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs @@ -194,8 +194,6 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - - public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi @@ -207,8 +205,6 @@ class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - - public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_Configurable : IConfigurationApi @@ -231,8 +227,6 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public bool ReturnAuditConfig { get; set; } public string RemoteStatus { get; set; } public string RemoteVersion { get; set; } diff --git a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs index c0178087c6..3174f5bea7 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs @@ -306,9 +306,6 @@ class AuditQuery_NoAuditRemotes : IAuditQuery public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -345,9 +342,6 @@ public AuditQuery_WithOneEndpoint(string endpointName, long throughputCount, Dat public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -383,9 +377,6 @@ class AuditQuery_ThrowingAnExceptionOnKnownEndpointsCall : IAuditQuery public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -469,9 +460,6 @@ public AuditQuery_WithTwoEndpointsAndSelectiveCounts( public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult>>([]); - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - public Func ValidRemoteInstances => _ => true; public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => @@ -514,9 +502,6 @@ class AuditQuery_WithRemotes(List remotes) : IAuditQu public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => Task.FromResult(Environments); - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - public Func ValidRemoteInstances => _ => true; public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => diff --git a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs index 4fc290fc16..0e18e277ae 100644 --- a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs +++ b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs @@ -11,8 +11,6 @@ class FakeConfigurationApi : IConfigurationApi public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - - public Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); } } diff --git a/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs deleted file mode 100644 index 95c4c67d16..0000000000 --- a/src/Particular.LicensingComponent.UnitTests/IngestionHistoryCollectorHostedService_Tests.cs +++ /dev/null @@ -1,165 +0,0 @@ -namespace Particular.LicensingComponent.UnitTests; - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using AuditThroughput; -using Contracts; -using Infrastructure; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using NuGet.Versioning; -using NUnit.Framework; -using ServiceControl.Infrastructure.Ingestion; - -[TestFixture] -class IngestionHistoryCollectorHostedService_Tests : ThroughputCollectorTestFixture -{ - public override Task Setup() - { - SetExtraDependencies = d => { }; - - return base.Setup(); - } - - [Test] - public async Task Second_poll_stores_the_delta_summed_across_audit_instances() - { - var start = new DateTimeOffset(2026, 9, 30, 10, 0, 0, TimeSpan.Zero); - var fakeTimeProvider = new FakeTimeProvider(start); - var processStart = start.UtcDateTime.AddDays(-3); - var errorProvider = new QueuedErrorSnapshots( - Snapshot(processStart, messages: 100, busySeconds: 10), - Snapshot(processStart, messages: 160, busySeconds: 16)); - var auditQuery = new AuditQuery_WithIngestionSnapshots( - [ - new AuditIngestionSnapshot("a", Snapshot(processStart, messages: 1000, busySeconds: 100)), - new AuditIngestionSnapshot("b", Snapshot(processStart, messages: 2000, busySeconds: 200)) - ], - [ - new AuditIngestionSnapshot("a", Snapshot(processStart, messages: 1300, busySeconds: 130)), - new AuditIngestionSnapshot("b", Snapshot(processStart, messages: 2500, busySeconds: 260)) - ]); - - await RunTwoPolls(fakeTimeProvider, errorProvider, auditQuery); - - var history = await DataStore.GetIngestionHistory(); - - Assert.That(history, Is.Not.Null); - var errorDay = history.Days.Single(day => day.Source == IngestionHistory.ErrorSource); - var auditDay = history.Days.Single(day => day.Source == IngestionHistory.AuditSource); - using (Assert.EnterMultipleScope()) - { - Assert.That(errorDay.Date, Is.EqualTo(start.UtcDateTime.Date)); - Assert.That(errorDay.Messages, Is.EqualTo(60)); - Assert.That(errorDay.PeakHourMessages, Is.EqualTo(60)); - Assert.That(errorDay.PeakHourBusySeconds, Is.EqualTo(6).Within(0.001)); - Assert.That(auditDay.Messages, Is.EqualTo(800)); - Assert.That(auditDay.PeakHourMessages, Is.EqualTo(800)); - Assert.That(auditDay.PeakHourBusySeconds, Is.EqualTo(90).Within(0.001)); - } - } - - [Test] - public async Task A_restarted_process_contributes_its_whole_counters_as_the_delta() - { - var start = new DateTimeOffset(2026, 9, 30, 10, 0, 0, TimeSpan.Zero); - var fakeTimeProvider = new FakeTimeProvider(start); - var errorProvider = new QueuedErrorSnapshots( - Snapshot(start.UtcDateTime.AddDays(-3), messages: 100, busySeconds: 10), - Snapshot(start.UtcDateTime.AddMinutes(30), messages: 25, busySeconds: 5)); - var auditQuery = new AuditQuery_WithIngestionSnapshots([], []); - - await RunTwoPolls(fakeTimeProvider, errorProvider, auditQuery); - - var history = await DataStore.GetIngestionHistory(); - - Assert.That(history, Is.Not.Null); - var errorDay = history.Days.Single(day => day.Source == IngestionHistory.ErrorSource); - using (Assert.EnterMultipleScope()) - { - Assert.That(errorDay.Messages, Is.EqualTo(25)); - Assert.That(errorDay.Restarts, Is.EqualTo(1)); - } - } - - async Task RunTwoPolls(FakeTimeProvider fakeTimeProvider, QueuedErrorSnapshots errorProvider, AuditQuery_WithIngestionSnapshots auditQuery) - { - using var service = new IngestionHistoryCollectorHostedService( - NullLogger.Instance, DataStore, auditQuery, fakeTimeProvider, errorProvider) - { DelayStart = TimeSpan.Zero }; - - await service.StartAsync(CancellationToken.None); - - await errorProvider.PolledOnce.Task.WaitAsync(TimeSpan.FromSeconds(30)); - fakeTimeProvider.Advance(TimeSpan.FromHours(1)); - await errorProvider.PolledTwice.Task.WaitAsync(TimeSpan.FromSeconds(30)); - - var saveDeadline = DateTime.UtcNow.AddSeconds(30); - - while (await DataStore.GetIngestionHistory() is null && DateTime.UtcNow < saveDeadline) - { - await Task.Delay(TimeSpan.FromMilliseconds(20)); - } - - await service.StopAsync(CancellationToken.None); - } - - static IngestionCountersSnapshot Snapshot(DateTime processStartUtc, long messages, double busySeconds) => - new(processStartUtc, messages, busySeconds, 0, 0, 0, 0, 0); - - class QueuedErrorSnapshots(params IngestionCountersSnapshot[] snapshots) : IErrorIngestionSnapshotProvider - { - public TaskCompletionSource PolledOnce { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public TaskCompletionSource PolledTwice { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public IngestionCountersSnapshot GetSnapshot() - { - var index = Math.Min(calls, snapshots.Length - 1); - calls++; - - if (calls == 1) - { - PolledOnce.TrySetResult(); - } - - if (calls >= 2) - { - PolledTwice.TrySetResult(); - } - - return snapshots[index]; - } - - int calls; - } - - class AuditQuery_WithIngestionSnapshots(List first, List second) : IAuditQuery - { - public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); - - public Func ValidRemoteInstances => _ => true; - - public Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) => - Task.FromResult(calls++ == 0 ? first : second); - - public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => - Task.FromResult>>([]); - - public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - - public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => - Task.FromResult>([]); - - public Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default) => - Task.FromResult>([]); - - public Task TestAuditConnection(CancellationToken cancellationToken = default) => - Task.FromResult(new ConnectionSettingsTestResult { ConnectionSuccessful = true }); - - int calls; - } -} diff --git a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs index fb7cbd3ab5..f6d1e5651c 100644 --- a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs @@ -258,12 +258,6 @@ public Task GetAuditEnvironmentMetadata(CancellationTo public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => throw new NotSupportedException(); - public Task GetIngestionHistory(CancellationToken cancellationToken = default) => - throw new NotSupportedException(); - - public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) => - throw new NotSupportedException(); - public Task> GetReportMasks(CancellationToken cancellationToken = default) => throw new NotSupportedException(); diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index 4c73f1ced3..5ae951d3b3 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -316,48 +316,26 @@ public async Task Should_leave_audit_environment_data_out_when_it_was_never_coll } [Test] - public async Task Should_report_ingestion_rates_from_days_inside_the_report_window() + public async Task Should_sum_audit_ingestion_rates_and_keep_the_worst_saturation_lag_and_uptime() { - var reportEndDate = new DateTime(2026, 9, 30, 0, 0, 0, DateTimeKind.Utc); - var insideWindow = reportEndDate.AddDays(-1); - var outsideWindow = reportEndDate.AddDays(-10); - await DataStore.SaveIngestionHistory(new IngestionHistory( + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( [ - new IngestionDay(insideWindow, IngestionHistory.ErrorSource, 2400, 600, 1800, 60, 100, 50, 10, 1000) { Restarts = 2 }, - new IngestionDay(insideWindow, IngestionHistory.AuditSource, 1200, 300, 360, 0, 0, 0, 0, 0), - new IngestionDay(outsideWindow, IngestionHistory.ErrorSource, 999999, 99999, 3600, 999, 0, 0, 0, 0) + new Dictionary { ["Ingestion.AvgDailyMessages"] = "1000", ["Ingestion.BusyPercent"] = "10", ["Health.UptimeHours"] = "500", ["Health.LagOver1MinPercent"] = "1", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Ingestion.AvgDailyMessages"] = "500", ["Ingestion.BusyPercent"] = "40", ["Health.UptimeHours"] = "3", ["Health.LagOver1MinPercent"] = "7", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" } ])); - var report = await ThroughputCollector.GenerateThroughputReport("", reportEndDate); + var report = await ThroughputCollector.GenerateThroughputReport("", null); var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; using (Assert.EnterMultipleScope()) { - Assert.That(environmentData, Does.ContainKey("Ingestion.Error.AvgDailyMessages").WithValue("2400")); - Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourMessages").WithValue("600")); - Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourBusyPercent").WithValue("50")); - Assert.That(environmentData, Does.ContainKey("Ingestion.Error.PeakHourStorageMsPerMessage").WithValue("100")); - Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.AvgDailyMessages").WithValue("1200")); - Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourMessages").WithValue("300")); - Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.PeakHourBusyPercent").WithValue("10")); - Assert.That(environmentData, Does.Not.ContainKey("Audit.Ingestion.PeakHourStorageMsPerMessage")); - Assert.That(environmentData, Does.ContainKey("Health.Error.Restarts").WithValue("2")); - Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver1MinPercent").WithValue("10")); - Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver10MinPercent").WithValue("5")); - Assert.That(environmentData, Does.ContainKey("Health.Error.LagOver60MinPercent").WithValue("1")); - Assert.That(environmentData, Does.ContainKey("Audit.Health.Restarts").WithValue("0")); - Assert.That(environmentData, Does.Not.ContainKey("Audit.Health.LagOver1MinPercent")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.AvgDailyMessages").WithValue("1500")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.BusyPercent").WithValue("40")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.UptimeHours").WithValue("3")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.LagOver1MinPercent").WithValue("7")); } } - [Test] - public async Task Should_leave_ingestion_rates_out_when_never_collected() - { - var report = await ThroughputCollector.GenerateThroughputReport("", null); - - Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Ingestion.")); - } - [Test] public async Task Should_leave_audit_environment_data_out_when_no_instance_responded() { diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs index 8ce6627e7d..40929ff5a4 100644 --- a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -5,14 +5,16 @@ namespace Particular.LicensingComponent; /// /// Folds the environment data of every responding audit instance into one value per key: values -/// that agree pass through, capacity numbers take the maximum, sizes and counts sum with instances -/// on one database counted once, the sharing class takes the most shared, and anything else that -/// differs reports Mixed rather than guessing which instance is representative. Keys starting with -/// an underscore carry collection bookkeeping and are never emitted. +/// that agree pass through, capacity and saturation numbers take the maximum, uptime takes the +/// minimum, sizes and counts sum with instances on one database counted once, ingestion rates sum +/// across all instances, the sharing class takes the most shared, and anything else that differs +/// reports Mixed rather than guessing which instance is representative. Keys starting with an +/// underscore carry collection bookkeeping and are never emitted. /// static class AuditEnvironmentDataAggregator { - static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB"]; + static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB", "Ingestion.BusyPercent", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent"]; + static readonly string[] MinimumKeys = ["Health.UptimeHours"]; static readonly string[] SharingPrecedence = ["SameSchema", "SameDatabase", "SameServer", "SeparateServer", "NotApplicable", "Unknown"]; public static IEnumerable> Aggregate(List> instances) @@ -63,6 +65,16 @@ static string Aggregate(string key, (string Value, string DatabaseKey)[] contrib var values = contributions.Select(contribution => contribution.Value).ToArray(); + if (key == "Ingestion.AvgDailyMessages") + { + var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); + + if (parsed.All(value => value.Parsed)) + { + return parsed.Sum(value => value.Number).ToString(CultureInfo.InvariantCulture); + } + } + if (values.Distinct(StringComparer.Ordinal).Count() == 1) { return values[0]; @@ -81,13 +93,15 @@ static string Aggregate(string key, (string Value, string DatabaseKey)[] contrib } } - if (MaximumKeys.Contains(key, StringComparer.Ordinal)) + if (MaximumKeys.Contains(key, StringComparer.Ordinal) || MinimumKeys.Contains(key, StringComparer.Ordinal)) { var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); if (parsed.All(value => value.Parsed)) { - return parsed.Max(value => value.Number).ToString(CultureInfo.InvariantCulture); + var numbers = parsed.Select(value => value.Number); + + return (MinimumKeys.Contains(key, StringComparer.Ordinal) ? numbers.Min() : numbers.Max()).ToString(CultureInfo.InvariantCulture); } } diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs deleted file mode 100644 index 50ffcc4255..0000000000 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditIngestionSnapshot.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Particular.LicensingComponent.AuditThroughput; - -using ServiceControl.Infrastructure.Ingestion; - -/// -/// One audit instance's running counters. The ApiUri only keys hourly deltas in memory and is -/// never stored or reported. -/// -public record AuditIngestionSnapshot(string ApiUri, IngestionCountersSnapshot Snapshot); diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs index 01ba9d0725..d3f633d55f 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Logging; using NuGet.Versioning; using ServiceControl.Api; - using ServiceControl.Infrastructure.Ingestion; using AuditCount = Contracts.AuditCount; public class AuditQuery(ILogger logger, IEndpointsApi endpointsApi, IAuditCountApi auditCountApi, IConfigurationApi configurationApi) : IAuditQuery @@ -134,38 +133,6 @@ public async Task>> GetAuditEnvironments(Cancell } } - public async Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default) - { - try - { - var remoteCounters = await configurationApi.GetRemoteIngestionCounters(cancellationToken); - - return - [ - .. remoteCounters - .Where(remote => remote.Counters is not null) - .Select(remote => new AuditIngestionSnapshot(remote.ApiUri, new IngestionCountersSnapshot( - remote.Counters.ProcessStartUtc, - remote.Counters.MessagesTotal, - remote.Counters.BusySecondsTotal, - remote.Counters.StorageSecondsTotal, - remote.Counters.LagOverOneMinuteMessages, - remote.Counters.LagOverTenMinutesMessages, - remote.Counters.LagOverSixtyMinutesMessages, - remote.Counters.LagKnownMessages))) - ]; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to get Audit Ingestion Snapshots"); - return []; - } - } - public async Task TestAuditConnection(CancellationToken cancellationToken = default) { var connectionTestResult = new ConnectionSettingsTestResult { ConnectionSuccessful = true }; diff --git a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs index 65eb583d4a..cf0252dfd1 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs @@ -13,7 +13,6 @@ public interface IAuditQuery Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default); Task> GetAuditRemotes(CancellationToken cancellationToken = default); Task>> GetAuditEnvironments(CancellationToken cancellationToken = default); - Task> GetAuditIngestionSnapshots(CancellationToken cancellationToken = default); Task TestAuditConnection(CancellationToken cancellationToken = default); } diff --git a/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs b/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs deleted file mode 100644 index b22c659978..0000000000 --- a/src/Particular.LicensingComponent/IErrorIngestionSnapshotProvider.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Particular.LicensingComponent; - -using ServiceControl.Infrastructure.Ingestion; - -/// -/// The host's own error ingestion counters, registered by the primary instance so that the -/// licensing collector can read them in process the way it polls audit instances over their API. -/// -public interface IErrorIngestionSnapshotProvider -{ - IngestionCountersSnapshot GetSnapshot(); -} diff --git a/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs b/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs deleted file mode 100644 index bd60e55f97..0000000000 --- a/src/Particular.LicensingComponent/IngestionHistoryCollectorHostedService.cs +++ /dev/null @@ -1,223 +0,0 @@ -namespace Particular.LicensingComponent; - -using AuditThroughput; -using Contracts; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Persistence; -using ServiceControl.Infrastructure.Ingestion; - -/// -/// Polls the ingestion counters every hour, this instance's in process and every audit instance's -/// over its environment endpoint, and folds the deltas into daily records in the licensing store. -/// A counter set whose process start time moved belongs to a restarted process, so its whole value -/// is the delta; one seen for the first time only establishes a baseline, because its totals span -/// an unknown stretch of time. -/// -public class IngestionHistoryCollectorHostedService( - ILogger logger, - ILicensingDataStore dataStore, - IAuditQuery auditQuery, - TimeProvider timeProvider, - IErrorIngestionSnapshotProvider? errorSnapshotProvider = null) : BackgroundService -{ - public TimeSpan DelayStart { get; set; } = TimeSpan.FromSeconds(50); - - protected override async Task ExecuteAsync(CancellationToken cancellationToken = default) - { - logger.LogInformation("Starting {ServiceName}", nameof(IngestionHistoryCollectorHostedService)); - - try - { - await Task.Delay(DelayStart, cancellationToken); - - using PeriodicTimer timer = new(TimeSpan.FromHours(1), timeProvider); - - do - { - try - { - await Collect(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to collect ingestion counters"); - } - } while (await timer.WaitForNextTickAsync(cancellationToken)); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - logger.LogInformation("Stopping {ServiceName}", nameof(IngestionHistoryCollectorHostedService)); - } - } - - internal async Task Collect(CancellationToken cancellationToken = default) - { - var now = timeProvider.GetUtcNow().UtcDateTime; - var changed = false; - - if (errorSnapshotProvider is not null) - { - changed |= Accumulate(IngestionHistory.ErrorSource, now, [new AuditIngestionSnapshot(LocalKey, errorSnapshotProvider.GetSnapshot())]); - } - - var auditSnapshots = await auditQuery.GetAuditIngestionSnapshots(cancellationToken); - - if (auditSnapshots.Count > 0) - { - changed |= Accumulate(IngestionHistory.AuditSource, now, auditSnapshots); - } - - if (changed) - { - await Save(now, cancellationToken); - } - } - - bool Accumulate(string source, DateTime now, List snapshots) - { - if (!states.TryGetValue(source, out var state)) - { - states[source] = state = new SourceState(); - } - - if (state.Date != now.Date) - { - state.Reset(now.Date); - } - - var any = false; - - foreach (var (apiUri, snapshot) in snapshots) - { - if (state.Baselines.TryGetValue(apiUri, out var last)) - { - var restarted = snapshot.ProcessStartUtc != last.ProcessStartUtc; - var delta = restarted ? snapshot : Diff(snapshot, last); - - if (restarted) - { - state.Restarts++; - } - - state.HourMessages[now.Hour] += delta.Messages; - state.HourBusySeconds[now.Hour] += delta.BusySeconds; - state.HourStorageSeconds[now.Hour] += delta.StorageSeconds; - state.LagOverOneMinute += delta.LagOverOneMinuteMessages; - state.LagOverTenMinutes += delta.LagOverTenMinutesMessages; - state.LagOverSixtyMinutes += delta.LagOverSixtyMinutesMessages; - state.LagKnown += delta.LagKnownMessages; - any = true; - } - - state.Baselines[apiUri] = snapshot; - } - - return any; - } - - static IngestionCountersSnapshot Diff(IngestionCountersSnapshot current, IngestionCountersSnapshot last) => current with - { - Messages = Math.Max(0, current.Messages - last.Messages), - BusySeconds = Math.Max(0, current.BusySeconds - last.BusySeconds), - StorageSeconds = Math.Max(0, current.StorageSeconds - last.StorageSeconds), - LagOverOneMinuteMessages = Math.Max(0, current.LagOverOneMinuteMessages - last.LagOverOneMinuteMessages), - LagOverTenMinutesMessages = Math.Max(0, current.LagOverTenMinutesMessages - last.LagOverTenMinutesMessages), - LagOverSixtyMinutesMessages = Math.Max(0, current.LagOverSixtyMinutesMessages - last.LagOverSixtyMinutesMessages), - LagKnownMessages = Math.Max(0, current.LagKnownMessages - last.LagKnownMessages) - }; - - async Task Save(DateTime now, CancellationToken cancellationToken) - { - var history = await dataStore.GetIngestionHistory(cancellationToken) ?? new IngestionHistory([]); - - foreach (var (source, state) in states) - { - if (state.Date != now.Date) - { - continue; - } - - // A record already stored for today survived a collector restart, so today's totals - // build on it rather than overwriting what the previous incarnation had accumulated. - state.Base ??= history.Days.FirstOrDefault(day => day.Date == state.Date && day.Source == source) - ?? new IngestionDay(state.Date, source, 0, 0, 0, 0, 0, 0, 0, 0); - - history.Days.RemoveAll(day => day.Date == state.Date && day.Source == source); - history.Days.Add(state.ToDay(source)); - } - - history.Days.RemoveAll(day => day.Date < now.Date.AddDays(-HistoryDays)); - - await dataStore.SaveIngestionHistory(history, cancellationToken); - } - - class SourceState - { - public DateTime Date { get; private set; } = DateTime.MinValue; - public long[] HourMessages { get; } = new long[24]; - public double[] HourBusySeconds { get; } = new double[24]; - public double[] HourStorageSeconds { get; } = new double[24]; - public long LagOverOneMinute { get; set; } - public long LagOverTenMinutes { get; set; } - public long LagOverSixtyMinutes { get; set; } - public long LagKnown { get; set; } - public long Restarts { get; set; } - public Dictionary Baselines { get; } = []; - public IngestionDay? Base { get; set; } - - public void Reset(DateTime date) - { - Date = date; - Array.Clear(HourMessages); - Array.Clear(HourBusySeconds); - Array.Clear(HourStorageSeconds); - LagOverOneMinute = 0; - LagOverTenMinutes = 0; - LagOverSixtyMinutes = 0; - LagKnown = 0; - Restarts = 0; - Base = null; - } - - public IngestionDay ToDay(string source) - { - var peakHour = 0; - - for (var hour = 1; hour < 24; hour++) - { - if (HourMessages[hour] > HourMessages[peakHour]) - { - peakHour = hour; - } - } - - var peakMessages = HourMessages[peakHour]; - var basePeakWins = Base!.PeakHourMessages >= peakMessages; - - return new IngestionDay( - Date, - source, - Base.Messages + HourMessages.Sum(), - basePeakWins ? Base.PeakHourMessages : peakMessages, - basePeakWins ? Base.PeakHourBusySeconds : HourBusySeconds[peakHour], - basePeakWins ? Base.PeakHourStorageSeconds : HourStorageSeconds[peakHour], - Base.LagOverOneMinuteMessages + LagOverOneMinute, - Base.LagOverTenMinutesMessages + LagOverTenMinutes, - Base.LagOverSixtyMinutesMessages + LagOverSixtyMinutes, - Base.LagKnownMessages + LagKnown) - { - Restarts = Base.Restarts + Restarts - }; - } - } - - readonly Dictionary states = []; - - const string LocalKey = "local"; - const int HistoryDays = 400; -} diff --git a/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs b/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs index 275d36eb6a..9e8db1f5e4 100644 --- a/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs +++ b/src/Particular.LicensingComponent/LicensingComponentHostBuilderExtensions.cs @@ -17,7 +17,6 @@ public static IHostApplicationBuilder AddLicensingComponent(this IHostApplicatio var throughputSettings = new ThroughputSettings(serviceControlQueue, errorQueue, transportType, customerName, serviceControlVersion); services.AddSingleton(throughputSettings); services.AddHostedService(); - services.AddHostedService(); services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index 4f30c7349b..d979c56bd1 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -207,17 +207,6 @@ public async Task GenerateThroughputReport(string spVersion, DateT } } - var ingestionHistory = await dataStore.GetIngestionHistory(cancellationToken); - - if (ingestionHistory is not null) - { - var windowStart = report.StartTime.UtcDateTime.Date; - var windowEnd = ((DateTime)reportEndDate).Date; - - EmitIngestion(report, "Ingestion.Error", "Health.Error", IngestionHistory.ErrorSource, ingestionHistory, windowStart, windowEnd, includeStorage: true); - EmitIngestion(report, "Audit.Ingestion", "Audit.Health", IngestionHistory.AuditSource, ingestionHistory, windowStart, windowEnd, includeStorage: false); - } - foreach (var environmentDataProvider in environmentDataProviders) { EnvironmentDatum[] environmentData; @@ -254,44 +243,6 @@ public async Task GenerateThroughputReport(string spVersion, DateT return throughputReport; } - static void EmitIngestion(Report.Report report, string prefix, string healthPrefix, string source, IngestionHistory ingestionHistory, DateTime windowStart, DateTime windowEnd, bool includeStorage) - { - var days = ingestionHistory.Days - .Where(day => day.Source == source && day.Date >= windowStart && day.Date <= windowEnd) - .ToArray(); - - if (days.Length == 0) - { - return; - } - - var environmentData = report.EnvironmentInformation.EnvironmentData; - var peakDay = days.MaxBy(day => day.PeakHourMessages)!; - - environmentData[$"{prefix}.AvgDailyMessages"] = Math.Round(days.Average(day => day.Messages), MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); - environmentData[$"{prefix}.PeakHourMessages"] = peakDay.PeakHourMessages.ToString(CultureInfo.InvariantCulture); - environmentData[$"{prefix}.PeakHourBusyPercent"] = Math.Min(100, Math.Round(peakDay.PeakHourBusySeconds / 3600 * 100, MidpointRounding.AwayFromZero)).ToString("F0", CultureInfo.InvariantCulture); - - if (includeStorage && peakDay.PeakHourMessages > 0) - { - environmentData[$"{prefix}.PeakHourStorageMsPerMessage"] = Math.Round(peakDay.PeakHourStorageSeconds / peakDay.PeakHourMessages * 1000, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); - } - - environmentData[$"{healthPrefix}.Restarts"] = days.Sum(day => day.Restarts).ToString(CultureInfo.InvariantCulture); - - var lagKnown = days.Sum(day => day.LagKnownMessages); - - if (lagKnown > 0) - { - environmentData[$"{healthPrefix}.LagOver1MinPercent"] = LagPercent(days.Sum(day => day.LagOverOneMinuteMessages), lagKnown); - environmentData[$"{healthPrefix}.LagOver10MinPercent"] = LagPercent(days.Sum(day => day.LagOverTenMinutesMessages), lagKnown); - environmentData[$"{healthPrefix}.LagOver60MinPercent"] = LagPercent(days.Sum(day => day.LagOverSixtyMinutesMessages), lagKnown); - } - } - - static string LagPercent(long over, long known) => - Math.Round(over / (double)known * 100, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); - async IAsyncEnumerable GetDistinctEndpointData(DateOnly? throughputMaxDate, [EnumeratorCancellation] CancellationToken cancellationToken) { var endpoints = (await dataStore.GetAllEndpoints(false, cancellationToken)).ToArray(); diff --git a/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs b/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs index d5f23ea1e8..40873df82e 100644 --- a/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs +++ b/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs @@ -121,7 +121,8 @@ await Define() "Recoverability.RetryHistoryDepth", "Licensing.ReportMasks", "Retention.ErrorHours", - "Retention.EventsHours" + "Retention.EventsHours", + "Health.Error.UptimeHours" ]; [Test] diff --git a/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs b/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs deleted file mode 100644 index ba6aa4db3c..0000000000 --- a/src/ServiceControl.Api/Contracts/RemoteIngestionCounters.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace ServiceControl.Api.Contracts -{ - using System; - - public class RemoteIngestionCounters - { - public string ApiUri { get; set; } - - /// - /// Null when the remote could not be reached or predates the environment endpoint. - /// - public RemoteIngestionCounterValues Counters { get; set; } - } - - public class RemoteIngestionCounterValues - { - public DateTime ProcessStartUtc { get; set; } - public long MessagesTotal { get; set; } - public double BusySecondsTotal { get; set; } - public double StorageSecondsTotal { get; set; } - public long LagOverOneMinuteMessages { get; set; } - public long LagOverTenMinutesMessages { get; set; } - public long LagOverSixtyMinutesMessages { get; set; } - public long LagKnownMessages { get; set; } - } -} diff --git a/src/ServiceControl.Api/IConfigurationApi.cs b/src/ServiceControl.Api/IConfigurationApi.cs index 73b8c0ab59..b16c977437 100644 --- a/src/ServiceControl.Api/IConfigurationApi.cs +++ b/src/ServiceControl.Api/IConfigurationApi.cs @@ -13,7 +13,5 @@ public interface IConfigurationApi Task GetRemoteConfigs(CancellationToken cancellationToken = default); Task GetRemoteEnvironments(CancellationToken cancellationToken = default); - - Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default); } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 53de5f381f..79a17d155a 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Audit.AcceptanceTests { - using System; using System.Globalization; using System.Text.Json.Nodes; using System.Threading.Tasks; @@ -50,9 +49,7 @@ public async Task Should_serve_host_and_storage_facts() Assert.That((string)storageIdentity["engine"], Is.Not.Null.And.Not.Empty); Assert.That((string)storageIdentity["server_hash"], Has.Length.EqualTo(64)); Assert.That((string)storageIdentity["database_hash"], Has.Length.EqualTo(64)); - Assert.That(body["ingestion_counters"], Is.Not.Null); - Assert.That((DateTime)body["ingestion_counters"]["process_start_utc"], Is.Not.EqualTo(default(DateTime))); - Assert.That((long)body["ingestion_counters"]["messages_total"], Is.GreaterThanOrEqualTo(0)); + Assert.That(long.Parse((string)environmentData["Health.UptimeHours"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); } } diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs index 36f6aca7c4..13897866cd 100644 --- a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -9,7 +9,6 @@ namespace ServiceControl.Audit.UnitTests.Infrastructure using ServiceControl.Audit.Infrastructure.WebApi; using ServiceControl.Audit.Persistence; using ServiceControl.Infrastructure; - using ServiceControl.Infrastructure.Ingestion; [TestFixture] class EnvironmentControllerTests @@ -21,7 +20,7 @@ public async Task Merges_the_data_of_every_provider() [ new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB")), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], [], Counters(), NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -41,7 +40,7 @@ public async Task A_datum_that_cannot_be_read_costs_only_its_own_key() new Provider( EnvironmentDatum.Value("Storage.Type", () => "RavenDB"), EnvironmentDatum.Deferred("Storage.ServerVersion", _ => throw new InvalidOperationException("server down"))) - ], [], Counters(), NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -60,7 +59,7 @@ public async Task A_provider_that_cannot_list_its_data_costs_only_its_own_keys() [ new ThrowingProvider(), new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) - ], [], Counters(), NullLogger.Instance); + ], [], NullLogger.Instance); var result = await controller.Environment(); @@ -84,7 +83,7 @@ public void Cancellation_propagates() cancellationToken.ThrowIfCancellationRequested(); return new ValueTask("unreached"); })) - ], [], Counters(), NullLogger.Instance); + ], [], NullLogger.Instance); Assert.ThrowsAsync(() => controller.Environment(cancellationTokenSource.Token)); } @@ -95,7 +94,6 @@ public async Task Serves_the_hashed_storage_identity() var controller = new EnvironmentController( [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], [new IdentityProvider(new StorageIdentity("RavenDB", "http://server:8080", "audit", null))], - Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -118,7 +116,6 @@ public async Task A_storage_identity_that_cannot_be_read_is_not_served() var controller = new EnvironmentController( [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], [new ThrowingIdentityProvider()], - Counters(), NullLogger.Instance); var result = await controller.Environment(); @@ -131,8 +128,6 @@ [new ThrowingIdentityProvider()], } } - static IngestionCounters Counters() => new(TimeProvider.System); - class Provider(params EnvironmentDatum[] data) : IEnvironmentDataProvider { public IEnumerable GetData() => data; diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 965edaafd8..b7925ee4db 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -53,6 +53,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); // required by the ingestion custom check which is auto-loaded diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs index 2a264fabb8..c13a838a74 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -10,13 +10,12 @@ namespace ServiceControl.Audit.Infrastructure.WebApi using Microsoft.Extensions.Logging; using ServiceControl.Audit.Persistence; using ServiceControl.Infrastructure; - using ServiceControl.Infrastructure.Ingestion; // the /api/environment endpoint is polled by the primary instance for its usage report. This currently needs to be anonymous [AllowAnonymous] [ApiController] [Route("api")] - public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, IngestionCounters ingestionCounters, ILogger logger) : ControllerBase + public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, ILogger logger) : ControllerBase { [Route("environment")] [HttpGet] @@ -60,8 +59,7 @@ public async Task Environment(CancellationToken cancellationToke { EnvironmentData = environmentData, MachineIdHash = MachineIdentity.Hash, - StorageIdentity = await ReadStorageIdentity(cancellationToken), - IngestionCounters = IngestionCountersResponse.From(ingestionCounters.GetSnapshot()) + StorageIdentity = await ReadStorageIdentity(cancellationToken) }); } @@ -108,31 +106,6 @@ public class EnvironmentDataResponse public Dictionary EnvironmentData { get; set; } public string MachineIdHash { get; set; } public StorageIdentityResponse StorageIdentity { get; set; } - public IngestionCountersResponse IngestionCounters { get; set; } - } - - public class IngestionCountersResponse - { - public DateTime ProcessStartUtc { get; set; } - public long MessagesTotal { get; set; } - public double BusySecondsTotal { get; set; } - public double StorageSecondsTotal { get; set; } - public long LagOverOneMinuteMessages { get; set; } - public long LagOverTenMinutesMessages { get; set; } - public long LagOverSixtyMinutesMessages { get; set; } - public long LagKnownMessages { get; set; } - - public static IngestionCountersResponse From(IngestionCountersSnapshot snapshot) => new() - { - ProcessStartUtc = snapshot.ProcessStartUtc, - MessagesTotal = snapshot.Messages, - BusySecondsTotal = snapshot.BusySeconds, - StorageSecondsTotal = snapshot.StorageSeconds, - LagOverOneMinuteMessages = snapshot.LagOverOneMinuteMessages, - LagOverTenMinutesMessages = snapshot.LagOverTenMinutesMessages, - LagOverSixtyMinutesMessages = snapshot.LagOverSixtyMinutesMessages, - LagKnownMessages = snapshot.LagKnownMessages - }; } public class StorageIdentityResponse diff --git a/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs b/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs new file mode 100644 index 0000000000..0561521771 --- /dev/null +++ b/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit +{ + using System; + using System.Collections.Generic; + using System.Linq; + using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure.Ingestion; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class IngestionEnvironmentDataProvider(IngestionCounters counters, TimeProvider timeProvider) : IEnvironmentDataProvider + { + public IEnumerable GetData() => + IngestionSummary.Describe(counters.GetSnapshot(), timeProvider.GetUtcNow().UtcDateTime, "Ingestion", "Health", includeStorage: false) + .Select(pair => Value(pair.Key, () => pair.Value)); + } +} diff --git a/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs index 6f918aea0c..5fb9da7f6d 100644 --- a/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs +++ b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs @@ -3,10 +3,6 @@ namespace ServiceControl.Infrastructure.Ingestion; using System; using System.Threading; -/// -/// Running totals since process start, kept in memory for the licensing collector to poll and turn -/// into hourly deltas. Restarts reset them, which the poller detects through the start time. -/// public class IngestionCounters(TimeProvider timeProvider) { public DateTime ProcessStartUtc { get; } = timeProvider.GetUtcNow().UtcDateTime; diff --git a/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs b/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs new file mode 100644 index 0000000000..9d1eb7ba6a --- /dev/null +++ b/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Infrastructure.Ingestion; + +using System; +using System.Collections.Generic; +using System.Globalization; + +public static class IngestionSummary +{ + public static readonly TimeSpan MinimumUptime = TimeSpan.FromHours(1); + + public static IEnumerable> Describe(IngestionCountersSnapshot snapshot, DateTime nowUtc, string ingestionPrefix, string healthPrefix, bool includeStorage) + { + var uptime = nowUtc - snapshot.ProcessStartUtc; + + if (uptime < TimeSpan.Zero) + { + uptime = TimeSpan.Zero; + } + + yield return new($"{healthPrefix}.UptimeHours", ((long)uptime.TotalHours).ToString(CultureInfo.InvariantCulture)); + + if (uptime < MinimumUptime) + { + yield break; + } + + yield return new($"{ingestionPrefix}.AvgDailyMessages", Whole(snapshot.Messages / uptime.TotalDays)); + yield return new($"{ingestionPrefix}.BusyPercent", Whole(Math.Min(100, snapshot.BusySeconds / uptime.TotalSeconds * 100))); + + if (includeStorage && snapshot.Messages > 0) + { + yield return new($"{ingestionPrefix}.StorageMsPerMessage", Whole(snapshot.StorageSeconds / snapshot.Messages * 1000)); + } + + if (snapshot.LagKnownMessages > 0) + { + yield return new($"{healthPrefix}.LagOver1MinPercent", Percent(snapshot.LagOverOneMinuteMessages, snapshot.LagKnownMessages)); + yield return new($"{healthPrefix}.LagOver10MinPercent", Percent(snapshot.LagOverTenMinutesMessages, snapshot.LagKnownMessages)); + yield return new($"{healthPrefix}.LagOver60MinPercent", Percent(snapshot.LagOverSixtyMinutesMessages, snapshot.LagKnownMessages)); + } + } + + static string Percent(long part, long whole) => Whole(part / (double)whole * 100); + + static string Whole(double value) => Math.Round(value, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); +} diff --git a/src/ServiceControl.Infrastructure/MachineIdentity.cs b/src/ServiceControl.Infrastructure/MachineIdentity.cs index b4665ceb26..28cd48fba1 100644 --- a/src/ServiceControl.Infrastructure/MachineIdentity.cs +++ b/src/ServiceControl.Infrastructure/MachineIdentity.cs @@ -2,21 +2,19 @@ namespace ServiceControl.Infrastructure; using System; -using System.IO; using ServiceControl.Configuration; public static class MachineIdentity { /// - /// Containers report this instead of a hash: every container has its own machine id, so two + /// Containers report this instead of a hash: every container has its own hostname, so two /// containers on one host would never compare equal and the comparison would mislead. /// public const string NotApplicable = "NotApplicable"; /// /// A stable hash of this machine's identity, in a container, or - /// null when nothing stable was available. The source is prefixed into the hashed value so that - /// identities from different sources can never compare equal by accident. + /// null when nothing stable was available. /// public static string? Hash => hash.Value; @@ -29,31 +27,8 @@ public static class MachineIdentity return NotApplicable; } - if (OperatingSystem.IsLinux()) - { - foreach (var path in MachineIdPaths) - { - try - { - var machineId = File.ReadAllText(path).Trim(); - - if (machineId.Length > 0) - { - return IdentityHash.Compute("machine-id:" + machineId); - } - } - catch (Exception) - { - } - } - } - - // The machine name rather than the Windows MachineGuid, which only the registry serves and - // would cost this cross-platform assembly a Windows-only package dependency. var machineName = Environment.MachineName; return string.IsNullOrWhiteSpace(machineName) ? null : IdentityHash.Compute("machine-name:" + machineName); } - - static readonly string[] MachineIdPaths = ["/etc/machine-id", "/var/lib/dbus/machine-id"]; } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index eed26c0b8d..e57519e46b 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -309,12 +309,6 @@ public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditEnvironmentMetadata, auditEnvironmentMetadata, token), cancellationToken); - public Task GetIngestionHistory(CancellationToken cancellationToken = default) => - ExecuteWithDbContext((context, token) => context.GetSetting(SettingKeys.IngestionHistory, token), cancellationToken); - - public Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) => - ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.IngestionHistory, ingestionHistory, token), cancellationToken); - public Task> GetReportMasks(CancellationToken cancellationToken = default) => ExecuteWithDbContext(async (context, token) => await context.GetSetting>(SettingKeys.ReportMasks, token) ?? [], cancellationToken); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index 6279d9a3c9..5f01a2ddf9 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -7,7 +7,6 @@ static class SettingKeys public const string BrokerMetadata = "BrokerMetadata"; public const string AuditServiceMetadata = "AuditServiceMetadata"; public const string AuditEnvironmentMetadata = "AuditEnvironmentMetadata"; - public const string IngestionHistory = "IngestionHistory"; public const string ReportMasks = "ReportMasks"; public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string NotificationEmails = "NotificationEmails"; diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 236dc4715e..4508e85843 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -23,7 +23,6 @@ class LicensingDataStore( internal const string ThroughputTimeSeriesName = "INC: throughput data"; const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; - const string IngestionHistoryDocumentId = "IngestionHistory"; const string BrokerMetadataDocumentId = "BrokerMetadata"; const string ReportMasksDocumentId = "ReportMasks"; const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; @@ -337,23 +336,6 @@ public async Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnv await session.SaveChangesAsync(cancellationToken); } - public async Task GetIngestionHistory(CancellationToken cancellationToken = default) - { - var store = await storeProvider.GetDocumentStore(cancellationToken); - using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); - - return await session.LoadAsync(IngestionHistoryDocumentId, cancellationToken); - } - - public async Task SaveIngestionHistory(IngestionHistory ingestionHistory, CancellationToken cancellationToken = default) - { - var store = await storeProvider.GetDocumentStore(cancellationToken); - using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); - - await session.StoreAsync(ingestionHistory, IngestionHistoryDocumentId, cancellationToken); - await session.SaveChangesAsync(cancellationToken); - } - public async Task> GetReportMasks(CancellationToken cancellationToken = default) { var store = await storeProvider.GetDocumentStore(cancellationToken); diff --git a/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs b/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs new file mode 100644 index 0000000000..aa7a10cd9e --- /dev/null +++ b/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Infrastructure; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; + +[TestFixture] +public class IngestionSummaryTests +{ + static readonly DateTime Start = new(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc); + + [Test] + public void Reports_only_uptime_during_the_first_hour() + { + var snapshot = new IngestionCountersSnapshot(Start, 500, 100, 10, 5, 2, 1, 50); + + var data = Describe(snapshot, Start.AddMinutes(59)); + + Assert.That(data, Is.EqualTo(new Dictionary { ["Health.UptimeHours"] = "0" })); + } + + [Test] + public void Rates_are_per_day_and_percentages_are_of_uptime() + { + var snapshot = new IngestionCountersSnapshot(Start, 4800, 43200, 48, 100, 50, 10, 1000); + + var data = Describe(snapshot, Start.AddDays(2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.UptimeHours").WithValue("48")); + Assert.That(data, Does.ContainKey("Ingestion.AvgDailyMessages").WithValue("2400")); + Assert.That(data, Does.ContainKey("Ingestion.BusyPercent").WithValue("25")); + Assert.That(data, Does.ContainKey("Ingestion.StorageMsPerMessage").WithValue("10")); + Assert.That(data, Does.ContainKey("Health.LagOver1MinPercent").WithValue("10")); + Assert.That(data, Does.ContainKey("Health.LagOver10MinPercent").WithValue("5")); + Assert.That(data, Does.ContainKey("Health.LagOver60MinPercent").WithValue("1")); + } + } + + [Test] + public void Busy_percent_is_capped_when_batches_overlap() + { + var snapshot = new IngestionCountersSnapshot(Start, 10, 7200, 0, 0, 0, 0, 0); + + var data = Describe(snapshot, Start.AddHours(1)); + + Assert.That(data, Does.ContainKey("Ingestion.BusyPercent").WithValue("100")); + } + + [Test] + public void Lag_is_left_out_until_a_message_carried_the_header() + { + var snapshot = new IngestionCountersSnapshot(Start, 10, 1, 0, 0, 0, 0, 0); + + var data = Describe(snapshot, Start.AddHours(1)); + + Assert.That(data.Keys, Has.None.StartsWith("Health.LagOver")); + } + + [Test] + public void Storage_time_is_left_out_when_not_asked_for_or_nothing_was_ingested() + { + var withoutStorage = new IngestionCountersSnapshot(Start, 10, 1, 1, 0, 0, 0, 0); + var withoutMessages = new IngestionCountersSnapshot(Start, 0, 1, 1, 0, 0, 0, 0); + + using (Assert.EnterMultipleScope()) + { + Assert.That(Describe(withoutStorage, Start.AddHours(1), includeStorage: false), Does.Not.ContainKey("Ingestion.StorageMsPerMessage")); + Assert.That(Describe(withoutMessages, Start.AddHours(1)), Does.Not.ContainKey("Ingestion.StorageMsPerMessage")); + } + } + + static Dictionary Describe(IngestionCountersSnapshot snapshot, DateTime nowUtc, bool includeStorage = true) => + IngestionSummary.Describe(snapshot, nowUtc, "Ingestion", "Health", includeStorage).ToDictionary(pair => pair.Key, pair => pair.Value); +} diff --git a/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs b/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs new file mode 100644 index 0000000000..c0f9a21f9b --- /dev/null +++ b/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs @@ -0,0 +1,15 @@ +namespace Particular.ServiceControl; + +using System; +using System.Collections.Generic; +using System.Linq; +using global::ServiceControl.Infrastructure.Ingestion; +using Particular.LicensingComponent.Contracts; +using static Particular.LicensingComponent.Contracts.EnvironmentDatum; + +class ErrorIngestionEnvironmentDataProvider(IngestionCounters counters, TimeProvider timeProvider) : IEnvironmentDataProvider +{ + public IEnumerable GetData() => + IngestionSummary.Describe(counters.GetSnapshot(), timeProvider.GetUtcNow().UtcDateTime, "Ingestion.Error", "Health.Error", includeStorage: true) + .Select(pair => Value(pair.Key, () => pair.Value)); +} diff --git a/src/ServiceControl/ErrorIngestionSnapshotProvider.cs b/src/ServiceControl/ErrorIngestionSnapshotProvider.cs deleted file mode 100644 index 2a0a81d182..0000000000 --- a/src/ServiceControl/ErrorIngestionSnapshotProvider.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Particular.ServiceControl; - -using global::ServiceControl.Infrastructure.Ingestion; -using Particular.LicensingComponent; - -class ErrorIngestionSnapshotProvider(IngestionCounters counters) : IErrorIngestionSnapshotProvider -{ - public IngestionCountersSnapshot GetSnapshot() => counters.GetSnapshot(); -} diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index ef4c4f250b..647b549024 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -164,7 +164,7 @@ public static void AddTelemetry(this IHostApplicationBuilder hostBuilder, Settin { hostBuilder.Services.TryAddSingleton(TimeProvider.System); hostBuilder.Services.AddSingleton(); - hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddEnvironmentDataProvider(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index f1471f9ef0..75755b8a89 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -207,57 +207,6 @@ public async Task GetRemoteEnvironments(CancellationToken c return await Task.WhenAll(tasks); } - public async Task GetRemoteIngestionCounters(CancellationToken cancellationToken = default) - { - var tasks = settings.RemoteInstances - .Select(async remote => - { - RemoteIngestionCounterValues counters = null; - HttpClient httpClient = httpClientFactory.CreateClient(remote.InstanceId); - - try - { - using var response = await httpClient.GetAsync("/api/environment", cancellationToken); - - if (response.IsSuccessStatusCode) - { - await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); - var body = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); - - if (body?.AsObject().TryGetPropertyValue("ingestion_counters", out var node) == true && node is JsonObject countersObject) - { - counters = new RemoteIngestionCounterValues - { - ProcessStartUtc = countersObject["process_start_utc"]!.GetValue(), - MessagesTotal = countersObject["messages_total"]!.GetValue(), - BusySecondsTotal = countersObject["busy_seconds_total"]!.GetValue(), - StorageSecondsTotal = countersObject["storage_seconds_total"]!.GetValue(), - LagOverOneMinuteMessages = countersObject["lag_over_one_minute_messages"]!.GetValue(), - LagOverTenMinutesMessages = countersObject["lag_over_ten_minutes_messages"]!.GetValue(), - LagOverSixtyMinutesMessages = countersObject["lag_over_sixty_minutes_messages"]!.GetValue(), - LagKnownMessages = countersObject["lag_known_messages"]!.GetValue() - }; - } - } - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception) - { - } - - return new RemoteIngestionCounters - { - ApiUri = remote.BaseAddress, - Counters = counters - }; - }); - - return await Task.WhenAll(tasks); - } - async Task LocalStorageIdentity(CancellationToken cancellationToken) { var provider = storageIdentityProviders.FirstOrDefault(); From 52c53097e39c79baf85d2b908fae3565bf6efaf9 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 8 Oct 2026 09:13:45 +1000 Subject: [PATCH 7/9] Report database hosting, footprint and identity for EF Core audit persisters SQL Server and PostgreSQL audit persistence now probe how their database is hosted, how much it stores, and what identifies it, the same way the primary instance's EF persisters already did, so the report gets real Storage.* data for audit instances on these backends instead of nothing. SQL Server's edition and service objective are normalised to a small fixed set of categories instead of raw server strings, and both EF persisters resolve the configured schema the same way when none is set. AuditEnvironmentDataAggregator now passes through only an explicit whitelist of known keys, so a key served by an audit instance of another version can never reach the report unreviewed. ThroughputCollector copies broker metadata into a new dictionary before it reaches the report, so the stored metadata is never mutated. The environment endpoint's machine hash field is renamed from machine_id_hash to machine_name_hash to match what it actually hashes, and RavenDB's storage identity now treats a loopback address as this machine rather than a shared one. --- ...tor_Report_EnvironmentInformation_Tests.cs | 51 +++++ .../AuditEnvironmentDataAggregator.cs | 38 +++- .../ThroughputCollector.cs | 2 +- .../When_environment_data_is_requested.cs | 6 +- .../PostgreSqlDatabaseHostingProbe.cs | 119 ++++++++++++ .../PostgreSqlPersistence.cs | 7 + .../PostgreSqlStorageFootprintProbe.cs | 76 ++++++++ .../PostgreSqlStorageIdentityProvider.cs | 52 +++++ .../SqlServerDatabaseHostingProbe.cs | 183 ++++++++++++++++++ .../SqlServerPersistence.cs | 7 + .../SqlServerStorageFootprintProbe.cs | 79 ++++++++ .../SqlServerStorageIdentityProvider.cs | 50 +++++ .../Abstractions/EFPersistenceBase.cs | 4 + .../EFEnvironmentDataProvider.cs | 41 ++++ .../Infrastructure/IDatabaseHostingProbe.cs | 36 ++++ .../Infrastructure/IStorageFootprintProbe.cs | 16 ++ .../RavenStorageIdentityProvider.cs | 7 +- .../PostgreSqlEnvironmentDataTests.cs | 104 ++++++++++ .../RavenStorageIdentityProviderTests.cs | 20 ++ .../SqlServerEnvironmentDataTests.cs | 143 ++++++++++++++ .../EFCore/EFEnvironmentDataTests.cs | 50 +++++ .../IStorageIdentityProvider.cs | 2 +- .../EnvironmentControllerTests.cs | 2 +- .../WebApi/EnvironmentController.cs | 4 +- .../IdentityHash.cs | 7 +- .../PostgreSqlStorageFootprintProbe.cs | 9 +- .../PostgreSqlStorageIdentityProvider.cs | 6 +- .../SqlServerDatabaseHostingProbe.cs | 52 ++++- .../SqlServerStorageFootprintProbe.cs | 19 +- .../SqlServerStorageIdentityProvider.cs | 6 +- .../EFEnvironmentDataProvider.cs | 4 +- .../Infrastructure/IDatabaseHostingProbe.cs | 4 +- .../RavenStorageIdentityProvider.cs | 7 +- .../HealthEnvironmentDataTests.cs | 78 +++++++- .../StorageIdentityProviderTests.cs | 28 +++ .../RavenStorageIdentityProviderTests.cs | 19 ++ .../HealthEnvironmentDataTests.cs | 78 +++++++- .../SqlServerDatabaseHostingProbeTests.cs | 52 +++++ .../StorageFootprintProbeTests.cs | 8 +- .../Licensing/AuditSharingClassifierTests.cs | 8 + .../ConfigurationApiRemoteEnvironmentTests.cs | 48 +++++ .../Api/AuditSharingClassifier.cs | 17 +- .../Infrastructure/Api/ConfigurationApi.cs | 6 +- 43 files changed, 1490 insertions(+), 65 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs create mode 100644 src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs create mode 100644 src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs create mode 100644 src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs create mode 100644 src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs create mode 100644 src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index 5ae951d3b3..ce584e91b3 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -198,6 +199,17 @@ await DataStore.CreateBuilder() Assert.That(report.ReportData.ScopeType, Is.EqualTo(expectedScopeType), $"Invalid ScopeType on report"); } + [Test] + public async Task Should_leave_the_stored_broker_metadata_unchanged() + { + await DataStore.SaveBrokerMetadata(new BrokerMetadata("testingScope", new Dictionary { [EnvironmentDataType.BrokerVersion.ToString()] = "1.2" })); + + await ThroughputCollector.GenerateThroughputReport("", null); + + var stored = await DataStore.GetBrokerMetadata(); + Assert.That(stored.Data.Keys, Is.EquivalentTo(new[] { EnvironmentDataType.BrokerVersion.ToString() })); + } + [Test] public async Task Should_include_audit_instance_counts_in_environment_data() { @@ -345,4 +357,43 @@ public async Task Should_leave_audit_environment_data_out_when_no_instance_respo Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); } + + [Test] + public async Task Should_report_every_key_an_audit_instance_serves() + { + string[] served = + [ + "Host.Model", "Host.Orchestrator", "Host.OSPlatform", "Host.OSVersion", "Host.Architecture", "Host.RuntimeVersion", "Host.ProcessorCount", "Host.AvailableMemoryGB", + "Storage.Type", "Storage.RavenServer", "Storage.Hosting", "Storage.ServerVersion", "Storage.HostingSource", "Storage.ServerEdition", "Storage.ServiceObjective", + "Storage.SizeGB", "Storage.MessageCount", "Storage.FullTextSearch", + "Health.FailedImports", "Health.UptimeHours", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent", + "Ingestion.AvgDailyMessages", "Ingestion.BusyPercent", + "SameMachine", "DatabaseSharing" + ]; + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([served.ToDictionary(key => key, _ => "1")])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys.Where(key => key.StartsWith("Audit.") && key is not "Audit.ConfiguredInstances" and not "Audit.LiveInstances"), + Is.EquivalentTo(served.Select(key => "Audit." + key))); + } + + [Test] + public async Task Should_leave_out_audit_keys_the_primary_does_not_report() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["Storage.Type"] = "RavenDB", ["Security.Authentication"] = "Enabled", ["Storage.Auth"] = "Integrated" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Security.Authentication")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Storage.Auth")); + } + } } \ No newline at end of file diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs index 40929ff5a4..dff5e4da19 100644 --- a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -8,11 +8,43 @@ namespace Particular.LicensingComponent; /// that agree pass through, capacity and saturation numbers take the maximum, uptime takes the /// minimum, sizes and counts sum with instances on one database counted once, ingestion rates sum /// across all instances, the sharing class takes the most shared, and anything else that differs -/// reports Mixed rather than guessing which instance is representative. Keys starting with an -/// underscore carry collection bookkeeping and are never emitted. +/// reports Mixed rather than guessing which instance is representative. Only keys in +/// are emitted, so a key an audit instance of another version serves, or +/// the bookkeeping keys the primary adds, never reach the report unreviewed. /// static class AuditEnvironmentDataAggregator { + static readonly HashSet ReportedKeys = new(StringComparer.Ordinal) + { + "Host.Model", + "Host.Orchestrator", + "Host.OSPlatform", + "Host.OSVersion", + "Host.Architecture", + "Host.RuntimeVersion", + "Host.ProcessorCount", + "Host.AvailableMemoryGB", + "Storage.Type", + "Storage.RavenServer", + "Storage.Hosting", + "Storage.ServerVersion", + "Storage.HostingSource", + "Storage.ServerEdition", + "Storage.ServiceObjective", + "Storage.SizeGB", + "Storage.MessageCount", + "Storage.FullTextSearch", + "Health.FailedImports", + "Health.UptimeHours", + "Health.LagOver1MinPercent", + "Health.LagOver10MinPercent", + "Health.LagOver60MinPercent", + "Ingestion.AvgDailyMessages", + "Ingestion.BusyPercent", + "SameMachine", + "DatabaseSharing" + }; + static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB", "Ingestion.BusyPercent", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent"]; static readonly string[] MinimumKeys = ["Health.UptimeHours"]; static readonly string[] SharingPrecedence = ["SameSchema", "SameDatabase", "SameServer", "SeparateServer", "NotApplicable", "Unknown"]; @@ -26,7 +58,7 @@ public static IEnumerable> Aggregate(List instance.Keys) - .Where(key => !key.StartsWith('_')) + .Where(ReportedKeys.Contains) .Distinct(StringComparer.Ordinal) .Order(StringComparer.Ordinal); diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index d979c56bd1..e11853e87f 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -168,7 +168,7 @@ public async Task GenerateThroughputReport(string spVersion, DateT IgnoredQueues = [.. ignoredQueueNames], Queues = [.. queueThroughputs], TotalQueues = queueThroughputs.Count, - EnvironmentInformation = new EnvironmentInformation { AuditServicesData = new AuditServicesData(auditServiceMetadata.Versions, auditServiceMetadata.Transports), EnvironmentData = brokerMetaData.Data } + EnvironmentInformation = new EnvironmentInformation { AuditServicesData = new AuditServicesData(auditServiceMetadata.Versions, auditServiceMetadata.Transports), EnvironmentData = new Dictionary(brokerMetaData.Data) } }; var auditThroughput = queueThroughputs.SelectMany(w => w.DailyThroughputFromAudit); diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs index 79a17d155a..b1b32c4c5e 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -39,12 +39,12 @@ public async Task Should_serve_host_and_storage_facts() Assert.That((string)environmentData["Storage.ServerVersion"], Is.Not.Null.And.Not.Empty); Assert.That((string)environmentData["Storage.FullTextSearch"], Is.AnyOf("Enabled", "Disabled")); Assert.That(double.Parse((string)environmentData["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); - Assert.That((string)environmentData["Storage.MessageCount"], Is.EqualTo("0")); - Assert.That((string)environmentData["Storage.ServerEdition"], Is.EqualTo("NotApplicable")); + Assert.That((string)environmentData["Storage.MessageCount"], Is.AnyOf("0", "Unknown")); + Assert.That((string)environmentData["Storage.ServerEdition"], Is.AnyOf("Express", "Standard", "Enterprise", "NotApplicable")); Assert.That((string)environmentData["Health.FailedImports"], Is.EqualTo("0")); Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); - Assert.That((string)body["machine_id_hash"], Is.Not.Null.And.Not.Empty); + Assert.That((string)body["machine_name_hash"], Is.Not.Null.And.Not.Empty); Assert.That(storageIdentity, Is.Not.Null); Assert.That((string)storageIdentity["engine"], Is.Not.Null.And.Not.Empty); Assert.That((string)storageIdentity["server_hash"], Has.Length.EqualTo(64)); diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs new file mode 100644 index 0000000000..69365ba3c2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs @@ -0,0 +1,119 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using System.Data.Common; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Npgsql; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlDatabaseHostingProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IDatabaseHostingProbe +{ + public string StorageName => "PostgreSQL"; + + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = ProbeSql; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return HostingFromConnectionString(); + } + + var hosting = HostingFor(reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), ConfiguredHost); + + return new DatabaseHosting(hosting, MajorVersion(reader), DatabaseHostingSource.Probe); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not ask PostgreSQL how it is hosted, falling back to the connection string"); + + return HostingFromConnectionString(); + } + } + + /// + /// PostgreSQL has no equivalent of SQL Server's EngineEdition, but every managed offering + /// creates a distinctive administrative role that a self-hosted server does not have. A server + /// that answered and has none of them is self-hosted, unless its host name names a managed + /// service that does not announce itself this way. + /// + internal static string HostingFor(bool azure, bool rds, bool cloudSql, string? host) + { + if (azure) + { + return "AzurePostgres"; + } + + if (rds) + { + return "AwsRds"; + } + + if (cloudSql) + { + return "GoogleCloudSql"; + } + + var classified = DatabaseHostClassifier.Classify(host); + + return classified == DatabaseHostClassifier.Unknown ? DatabaseHostClassifier.SelfHosted : classified; + } + + static string MajorVersion(DbDataReader reader) => + reader.IsDBNull(0) ? DatabaseHostClassifier.Unknown : (reader.GetInt32(0) / 10000).ToString(CultureInfo.InvariantCulture); + + DatabaseHosting HostingFromConnectionString() + { + var host = ConfiguredHost; + + return host is null + ? DatabaseHosting.Unclassified + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString); + } + + string? ConfiguredHost + { + get + { + try + { + return new NpgsqlConnectionStringBuilder(settings.ConnectionString).Host; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the configured PostgreSQL host"); + + return null; + } + } + } + + const string ProbeSql = """ + SELECT current_setting('server_version_num')::int, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'azure_pg_admin'), + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rds_superuser'), + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'cloudsqlsuperuser') + """; + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 66d36b959f..bf8d700e80 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -59,6 +59,13 @@ protected override void AddQueryServices(IServiceCollection services) services.AddSingleton(); } + protected override void AddEnvironmentProbes(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } + protected override void AddCustomChecks(EndpointConfiguration endpointConfiguration) => endpointConfiguration.AddCustomCheck(); } diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs new file mode 100644 index 0000000000..c5abd8f862 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using System.Data; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlStorageFootprintProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The audit messages table is partitioned by day, and the parent carries no row estimate, + // so the estimate is summed over the partitions that have been analysed. + command.CommandText = """ + SELECT + (SELECT SUM(pg_total_relation_size(c.oid)) / 1073741824.0 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relkind IN ('r', 'm')), + (SELECT CASE WHEN bool_and(c.reltuples < 0) THEN NULL ELSE SUM(GREATEST(c.reltuples, 0))::bigint END + FROM pg_partition_tree(to_regclass(format('%I.%I', COALESCE(@schema, current_schema()), @table))) t + JOIN pg_class c ON c.oid = t.relid + WHERE t.isleaf) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = auditMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs new file mode 100644 index 0000000000..745dfb02b9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +// The query and the default schema must stay identical to the primary instance's provider, or the +// two sides hash different values for one database and DatabaseSharing never matches. +class PostgreSqlStorageIdentityProvider(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The cluster's initialisation identifier rather than an address: every connection to + // the same cluster sees the same value however the host is spelled or proxied. + // current_schema() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT system_identifier::text, current_database(), current_schema() FROM pg_control_system()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs new file mode 100644 index 0000000000..f107a47b8e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs @@ -0,0 +1,183 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using System.Data.Common; +using System.Globalization; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerDatabaseHostingProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IDatabaseHostingProbe +{ + public string StorageName => "SQLServer"; + + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server + // only and comes back null on Azure SQL Database, Managed Instance and Synapse. + command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0)) + { + return HostingFromConnectionString(); + } + + var engineEdition = Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture); + + var rds = !reader.IsDBNull(2); + + return new DatabaseHosting( + HostingFor(engineEdition, rds, ConfiguredHost), + MajorVersion(reader), + DatabaseHostingSource.Probe, + EditionFamily(engineEdition), + ServiceObjectiveTier(reader.IsDBNull(3) ? null : reader.GetString(3))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not ask SQL Server how it is hosted, falling back to the connection string"); + + return HostingFromConnectionString(); + } + } + + /// + /// Maps an EngineEdition to a hosting classification. The Azure editions identify their service + /// outright. The ordinary editions say only that this is a regular SQL Server, which still + /// leaves RDS and Cloud SQL in play, so the host name decides before self-hosted is concluded. + /// Synapse and Fabric are not mapped: ServiceControl does not run on them, and an edition we do + /// not recognise is not evidence of an ordinary SQL Server, so it falls to the host name. + /// + internal static string HostingFor(int engineEdition, bool rds, string? host) => engineEdition switch + { + AzureSqlDatabase => "AzureSql", + AzureSqlManagedInstance => "AzureSqlManagedInstance", + AzureSqlEdge => "AzureSqlEdge", + PersonalOrDesktop or Standard or Enterprise or Express => rds ? "AwsRds" : ManagedOrSelfHosted(host), + _ => DatabaseHostClassifier.Classify(host) + }; + + /// + /// Developer and Evaluation run the Enterprise engine and report Enterprise, so the report never + /// says which licence an instance runs under. The managed Azure services have no edition. + /// + internal static string EditionFamily(int engineEdition) => engineEdition switch + { + Standard => "Standard", + Enterprise => "Enterprise", + Express => "Express", + AzureSqlDatabase or AzureSqlManagedInstance => DatabaseHosting.NotApplicable, + _ => "Other" + }; + + /// + /// Keeps only the tier. The full service objective also encodes the hardware series, and a + /// DC-series name suggests Always Encrypted with secure enclaves. + /// + internal static string ServiceObjectiveTier(string? serviceObjective) + { + if (string.IsNullOrWhiteSpace(serviceObjective)) + { + return DatabaseHosting.NotApplicable; + } + + var objective = serviceObjective.Trim().ToUpperInvariant(); + + return objective switch + { + "BASIC" => "Basic", + "ELASTICPOOL" => "ElasticPool", + _ when objective.StartsWith("GP_", StringComparison.Ordinal) => "GeneralPurpose", + _ when objective.StartsWith("BC_", StringComparison.Ordinal) => "BusinessCritical", + _ when objective.StartsWith("HS_", StringComparison.Ordinal) => "Hyperscale", + _ when IsDtuObjective(objective, 'S') => "Standard", + _ when IsDtuObjective(objective, 'P') => "Premium", + _ => "Other" + }; + } + + static bool IsDtuObjective(string objective, char tier) => + objective.Length > 1 && objective[0] == tier && objective.AsSpan(1).IndexOfAnyExceptInRange('0', '9') < 0; + + static string ManagedOrSelfHosted(string? host) + { + var classified = DatabaseHostClassifier.Classify(host); + + return classified == DatabaseHostClassifier.Unknown ? DatabaseHostClassifier.SelfHosted : classified; + } + + static string MajorVersion(DbDataReader reader) + { + if (reader.IsDBNull(1)) + { + return DatabaseHostClassifier.Unknown; + } + + var productVersion = Convert.ToString(reader.GetValue(1), CultureInfo.InvariantCulture); + + if (string.IsNullOrWhiteSpace(productVersion)) + { + return DatabaseHostClassifier.Unknown; + } + + var major = productVersion.Split('.')[0]; + + return string.IsNullOrWhiteSpace(major) ? DatabaseHostClassifier.Unknown : major; + } + + DatabaseHosting HostingFromConnectionString() + { + var host = ConfiguredHost; + + return host is null + ? DatabaseHosting.Unclassified with { ServerEdition = DatabaseHostClassifier.Unknown, ServiceObjective = DatabaseHostClassifier.Unknown } + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString, DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown); + } + + string? ConfiguredHost + { + get + { + try + { + return new SqlConnectionStringBuilder(settings.ConnectionString).DataSource; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the configured SQL Server host"); + + return null; + } + } + } + + const int ProbeTimeoutSeconds = 5; + + // https://learn.microsoft.com/en-us/sql/t-sql/functions/serverproperty-transact-sql + const int PersonalOrDesktop = 1; + const int Standard = 2; + const int Enterprise = 3; + const int Express = 4; + const int AzureSqlDatabase = 5; + const int AzureSqlManagedInstance = 8; + const int AzureSqlEdge = 9; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index b7db7a8bbb..80b8be71b1 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -56,4 +56,11 @@ protected override void AddQueryServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); } + + protected override void AddEnvironmentProbes(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } } diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs new file mode 100644 index 0000000000..a1407c63d6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -0,0 +1,79 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using System.Data; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerStorageFootprintProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. + command.CommandText = """ + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.partitions p + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = auditMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs new file mode 100644 index 0000000000..47cfc5f4bf --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +// The query and the default schema must stay identical to the primary instance's provider, or the +// two sides hash different values for one database and DatabaseSharing never matches. +class SqlServerStorageIdentityProvider(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // SCHEMA_NAME() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME(), SCHEMA_NAME()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs index dea2331e99..90cb94678b 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs @@ -19,6 +19,8 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + AddEnvironmentProbes(services); services.AddSingleton(); services.AddHostedService(); @@ -53,6 +55,8 @@ void AddCommon(IServiceCollection services) protected abstract void AddQueryServices(IServiceCollection services); + protected abstract void AddEnvironmentProbes(IServiceCollection services); + protected virtual void AddCustomChecks(EndpointConfiguration endpointConfiguration) { } diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs new file mode 100644 index 0000000000..73b2728737 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using System.Globalization; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using static ServiceControl.Audit.Persistence.EnvironmentDatum; + +class EFEnvironmentDataProvider(IDatabaseHostingProbe hostingProbe, IStorageFootprintProbe footprintProbe, IFailedAuditStorage failedAuditStorage) : IEnvironmentDataProvider +{ + public IEnumerable GetData() + { + // The hosting keys share one probe, so the database is asked once per request and they + // stand or fall together. The footprint keys share theirs for the same reason. + Task? probe = null; + Task Hosting(CancellationToken cancellationToken) => probe ??= hostingProbe.Probe(cancellationToken); + + Task? footprint = null; + Task Footprint(CancellationToken cancellationToken) => footprint ??= footprintProbe.Probe(cancellationToken); + + return + [ + Value("Storage.Type", () => hostingProbe.StorageName), + Deferred("Storage.Hosting", async cancellationToken => (await Hosting(cancellationToken)).Hosting), + Deferred("Storage.ServerVersion", async cancellationToken => (await Hosting(cancellationToken)).ServerVersion), + Deferred("Storage.HostingSource", async cancellationToken => (await Hosting(cancellationToken)).Source), + Deferred("Storage.ServerEdition", async cancellationToken => (await Hosting(cancellationToken)).ServerEdition), + Deferred("Storage.ServiceObjective", async cancellationToken => (await Hosting(cancellationToken)).ServiceObjective), + Deferred("Storage.SizeGB", async cancellationToken => SizeGB((await Footprint(cancellationToken))?.SizeGB)), + Deferred("Storage.MessageCount", async cancellationToken => Count((await Footprint(cancellationToken))?.MessageCount)), + Deferred("Health.FailedImports", async cancellationToken => Count(await failedAuditStorage.GetFailedAuditsCount(cancellationToken))), + // The EF persisters always index bodies for full-text search; EnableFullTextSearchOnBodies is ignored. + Value("Storage.FullTextSearch", () => "Enabled") + ]; + } + + static string SizeGB(double? sizeGB) => + sizeGB is null ? DatabaseHostClassifier.Unknown : sizeGB.Value.ToString("F1", CultureInfo.InvariantCulture); + + static string Count(long? count) => + count is null ? DatabaseHostClassifier.Unknown : count.Value.ToString(CultureInfo.InvariantCulture); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs new file mode 100644 index 0000000000..b40f5de881 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using ServiceControl.Audit.Persistence; + +/// +/// How the database this instance stores its data in is hosted. Served to the primary instance for +/// its usage report, so every value is a fixed classification and never carries a host name, +/// database name or credential. Copied from the primary instance's EF persistence on purpose: the +/// audit persistence stack does not share projects with the primary's. +/// +public interface IDatabaseHostingProbe +{ + /// + /// The persistence name as it appears in the persistence manifest, for example SQLServer. + /// + string StorageName { get; } + + /// + /// Classifies the database host, asking the server itself where it can. Never throws; a server + /// that cannot be reached or does not answer is reported as unknown. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +/// One of AzureSql, AzureSqlManagedInstance, AzureSqlEdge, AzurePostgres, AwsRds, GoogleCloudSql, SelfHosted or Unknown. +/// The engine major version, or Unknown. +/// A value. +/// The SQL Server edition family: Express, Standard, Enterprise or Other. NotApplicable on the managed Azure services and on engines without editions. +/// The Azure SQL service tier: Basic, Standard, Premium, GeneralPurpose, BusinessCritical, Hyperscale, ElasticPool or Other. NotApplicable everywhere else. +public record DatabaseHosting(string Hosting, string ServerVersion, string Source, string ServerEdition = DatabaseHosting.NotApplicable, string ServiceObjective = DatabaseHosting.NotApplicable) +{ + public const string NotApplicable = "NotApplicable"; + + /// Nothing was available to classify the host with. A determination, not a failure. + public static readonly DatabaseHosting Unclassified = new(DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs new file mode 100644 index 0000000000..f1662c4917 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +/// +/// Measures what this instance's data occupies, from catalog statistics scoped to the configured +/// schema, never by counting rows. Served to the primary instance as sizes and counts only. +/// +public interface IStorageFootprintProbe +{ + /// + /// Null when the database could not be asked. A field is null when the catalog had no answer, + /// such as a row estimate on a table that was never analysed. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +public record StorageFootprint(double? SizeGB, long? MessageCount); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs index 228a713b21..7f1003e419 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -14,7 +14,10 @@ public ValueTask GetIdentity(CancellationToken cancellationToke return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), databaseConfiguration.Name, null)); } - static string NormalizeServer(string configuredServer) => - Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) ? $"{url.Host}:{url.Port}" : configuredServer ?? ""; + // A loopback address names a different server on every machine, embedded servers included. + internal static string NormalizeServer(string configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) + ? $"{(url.IsLoopback ? Environment.MachineName : url.Host)}:{url.Port}" + : configuredServer ?? ""; } } diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs new file mode 100644 index 0000000000..e14534561d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs @@ -0,0 +1,104 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; + using Npgsql; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.DbContexts; + using ServiceControl.Audit.Persistence.EFCore.Entities; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + + class PostgreSqlEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Reports_no_edition_or_service_objective() + { + var hosting = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } + + [Test] + public async Task Counts_messages_across_the_partitions_once_analysed() + { + await Ingest(MakeMessage(), MakeMessage()); + configuration.TimeProvider.Advance(TimeSpan.FromDays(1)); + await Ingest(MakeMessage()); + + var parentRowEstimate = await WithDbContext(async (dbContext, token) => + { + var table = $"{dbContext.Schema}.{dbContext.Model.FindEntityType(typeof(AuditMessageEntity))!.GetTableName()}"; + var leaves = await Query(dbContext, $"SELECT relid::regclass::text FROM pg_partition_tree('{table}'::regclass) WHERE isleaf", token); + + foreach (var leaf in leaves) + { + await Query(dbContext, $"ANALYZE {leaf}", token); + } + + return (await Query(dbContext, $"SELECT reltuples::text FROM pg_class WHERE oid = '{table}'::regclass", token)).Single(); + }); + + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(double.Parse(parentRowEstimate, CultureInfo.InvariantCulture), Is.LessThan(0), "The partitioned parent carries no row estimate of its own"); + Assert.That(footprint.MessageCount, Is.EqualTo(3)); + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + } + } + + [Test] + public async Task Measures_and_identifies_the_search_path_schema_when_none_is_configured() + { + var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = + new NpgsqlConnectionStringBuilder(configuration.ConnectionString) { SearchPath = configuration.Schema }.ConnectionString; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(configuration.TimeProvider); + configuration.CreateConfiguration().Create(settings).AddPersistence(services); + await using var unconfigured = services.BuildServiceProvider(); + + var identity = await unconfigured.GetRequiredService().GetIdentity(TestTimeoutCancellationToken); + var footprint = await unconfigured.GetRequiredService().Probe(TestTimeoutCancellationToken); + var expected = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(identity?.Schema, Is.EqualTo(configuration.Schema)); + Assert.That(footprint?.SizeGB, Is.Not.Null.And.EqualTo(expected?.SizeGB)); + } + } + + static async Task> Query(AuditDbContext dbContext, string sql, CancellationToken cancellationToken) + { + await dbContext.Database.OpenConnectionAsync(cancellationToken); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = sql; + + var values = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + while (await reader.ReadAsync(cancellationToken)) + { + values.Add(reader.GetString(0)); + } + + return values; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs new file mode 100644 index 0000000000..4ca76d809a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.RavenDB; + + [TestFixture] + class RavenStorageIdentityProviderTests + { + [TestCase("http://localhost:44445")] + [TestCase("http://127.0.0.1:44445")] + [TestCase("http://[::1]:44445")] + public void A_loopback_server_is_this_machine(string url) => + Assert.That(RavenStorageIdentityProvider.NormalizeServer(url), Is.EqualTo($"{Environment.MachineName}:44445")); + + [Test] + public void A_remote_server_is_its_host_and_port() => + Assert.That(RavenStorageIdentityProvider.NormalizeServer("https://raven.example.com:8080"), Is.EqualTo("raven.example.com:8080")); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs new file mode 100644 index 0000000000..303e7cf93c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs @@ -0,0 +1,143 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging.Abstractions; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.SqlServer; + + class SqlServerEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Reports_the_edition_family() + { + var hosting = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + var managed = hosting.Hosting is "AzureSql" or "AzureSqlManagedInstance"; + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, managed ? Is.EqualTo(DatabaseHosting.NotApplicable) : Is.AnyOf("Express", "Standard", "Enterprise")); + Assert.That(hosting.ServiceObjective, hosting.Hosting == "AzureSql" ? Is.AnyOf(ServiceTiers) : Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } + + [Test] + public async Task Counts_the_audit_messages_it_stores() + { + await Ingest(MakeMessage(), MakeMessage()); + + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.MessageCount, Is.EqualTo(2)); + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + } + } + + [TestCase(2, "Standard")] + [TestCase(3, "Enterprise")] + [TestCase(4, "Express")] + [TestCase(5, "NotApplicable")] + [TestCase(8, "NotApplicable")] + [TestCase(1, "Other")] + [TestCase(9, "Other")] + [TestCase(12, "Other")] + public void Should_report_the_edition_family(int engineEdition, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.EditionFamily(engineEdition), Is.EqualTo(expected)); + + [TestCase(null, "NotApplicable")] + [TestCase("Basic", "Basic")] + [TestCase("S0", "Standard")] + [TestCase("S12", "Standard")] + [TestCase("P15", "Premium")] + [TestCase("GP_Gen5_2", "GeneralPurpose")] + [TestCase("GP_S_Gen5_1", "GeneralPurpose")] + [TestCase("GP_DC_8", "GeneralPurpose")] + [TestCase("BC_Gen5_8", "BusinessCritical")] + [TestCase("HS_Gen5_4", "Hyperscale")] + [TestCase("ElasticPool", "ElasticPool")] + [TestCase("DW100c", "Other")] + [TestCase("System2", "Other")] + public void Should_report_only_the_service_objective_tier(string serviceObjective, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.ServiceObjectiveTier(serviceObjective), Is.EqualTo(expected)); + + [TestCase("Server=sc.database.windows.net;Database=sc")] + [TestCase("not a connection string")] + public async Task Should_report_edition_and_service_objective_as_unknown_when_the_server_cannot_be_asked(string connectionString) + { + var settings = new EFPersisterSettings { ConnectionString = connectionString, AuditRetentionPeriod = TimeSpan.FromDays(7), MaxBodySizeToStore = 1024 }; + var probe = new SqlServerDatabaseHostingProbe(settings, new UnreachableScopeFactory(), NullLogger.Instance); + + var hosting = await probe.Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo("Unknown")); + Assert.That(hosting.ServiceObjective, Is.EqualTo("Unknown")); + } + } + + [Test] + public async Task Counts_the_full_text_index_in_the_size() + { + await Ingest(MakeMessage(), MakeMessage()); + + var fullTextGB = await WithDbContext(async (dbContext, token) => + { + await dbContext.Database.OpenConnectionAsync(token); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = """ + SELECT COALESCE(SUM(a.total_pages), 0) * 8.0 / 1048576 + FROM sys.internal_tables it + JOIN sys.tables t ON t.object_id = it.parent_id + JOIN sys.partitions p ON p.object_id = it.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) AND it.internal_type_desc LIKE 'FULLTEXT%' + """; + var schema = command.CreateParameter(); + schema.ParameterName = "@schema"; + schema.Value = dbContext.Schema; + command.Parameters.Add(schema); + + return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); + }); + + var tablesGB = await WithDbContext(async (dbContext, token) => + { + await dbContext.Database.OpenConnectionAsync(token); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = """ + SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) + """; + var schema = command.CreateParameter(); + schema.ParameterName = "@schema"; + schema.Value = dbContext.Schema; + command.Parameters.Add(schema); + + return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); + }); + + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + Assert.That(fullTextGB, Is.GreaterThan(0), "The audit messages table always has a full-text index"); + Assert.That(footprint?.SizeGB, Is.EqualTo(tablesGB + fullTextGB).Within(1e-9)); + } + + static readonly object[] ServiceTiers = ["Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool"]; + + sealed class UnreachableScopeFactory : IServiceScopeFactory + { + public IServiceScope CreateScope() => throw new InvalidOperationException("The database cannot be reached"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs new file mode 100644 index 0000000000..27f8d907f7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Collections.Generic; + using System.Globalization; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using NUnit.Framework; + + class EFEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Serves_storage_facts_for_an_empty_schema() + { + var data = new Dictionary(); + + foreach (var datum in ServiceProvider.GetRequiredService().GetData()) + { + data[datum.Key] = await datum.ReadValue(TestTimeoutCancellationToken); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Storage.Type").WithValue(PersisterName)); + Assert.That(data, Does.ContainKey("Storage.HostingSource").WithValue("Probe")); + Assert.That(int.TryParse(data["Storage.ServerVersion"], NumberStyles.Integer, CultureInfo.InvariantCulture, out _), Is.True, data["Storage.ServerVersion"]); + Assert.That(data["Storage.ServerEdition"], Is.AnyOf("Express", "Standard", "Enterprise", "NotApplicable")); + Assert.That(data["Storage.ServiceObjective"], Is.AnyOf("NotApplicable", "Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool")); + Assert.That(double.Parse(data["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); + Assert.That(data["Storage.MessageCount"], Is.AnyOf("0", "Unknown")); + Assert.That(data, Does.ContainKey("Health.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.FullTextSearch").WithValue("Enabled")); + } + } + + [Test] + public async Task Identifies_the_database_and_schema_it_writes_to() + { + var identity = await ServiceProvider.GetRequiredService().GetIdentity(TestTimeoutCancellationToken); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo(PersisterName)); + Assert.That(identity.Server, Is.Not.Empty); + Assert.That(identity.Database, Is.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(configuration.Schema)); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs index c928de711a..dc38b24088 100644 --- a/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Audit.Persistence /// /// Identifies the storage this instance writes to, so that the primary instance can tell /// whether it shares a database or a database server with this one. The raw values never leave - /// the process; the environment endpoint serves hashes of them. + /// the process; the anonymous environment endpoint serves unsalted hashes of them. /// public interface IStorageIdentityProvider { diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs index 13897866cd..7d2d94eccb 100644 --- a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -105,7 +105,7 @@ [new IdentityProvider(new StorageIdentity("RavenDB", "http://server:8080", "audi Assert.That(response.StorageIdentity.ServerHash, Is.EqualTo(IdentityHash.Compute("http://server:8080"))); Assert.That(response.StorageIdentity.DatabaseHash, Is.EqualTo(IdentityHash.Compute("audit"))); Assert.That(response.StorageIdentity.SchemaHash, Is.Null); - Assert.That(response.MachineIdHash, Is.EqualTo(MachineIdentity.Hash)); + Assert.That(response.MachineNameHash, Is.EqualTo(MachineIdentity.Hash)); Assert.That(response.EnvironmentData.Keys, Has.None.Contains("Hash")); } } diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs index c13a838a74..4016209d4e 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -58,7 +58,7 @@ public async Task Environment(CancellationToken cancellationToke return Ok(new EnvironmentDataResponse { EnvironmentData = environmentData, - MachineIdHash = MachineIdentity.Hash, + MachineNameHash = MachineIdentity.Hash, StorageIdentity = await ReadStorageIdentity(cancellationToken) }); } @@ -104,7 +104,7 @@ async Task ReadStorageIdentity(CancellationToken cancel public class EnvironmentDataResponse { public Dictionary EnvironmentData { get; set; } - public string MachineIdHash { get; set; } + public string MachineNameHash { get; set; } public StorageIdentityResponse StorageIdentity { get; set; } } diff --git a/src/ServiceControl.Infrastructure/IdentityHash.cs b/src/ServiceControl.Infrastructure/IdentityHash.cs index 81f6a1242a..fb94bbae69 100644 --- a/src/ServiceControl.Infrastructure/IdentityHash.cs +++ b/src/ServiceControl.Infrastructure/IdentityHash.cs @@ -6,9 +6,10 @@ namespace ServiceControl.Infrastructure; using System.Text; /// -/// Hashes an identity value so that two instances can compare what they are running on without -/// either side revealing the value itself. The hashes are exchanged between the customer's own -/// instances and never appear in a usage report; only the comparison result does. +/// Hashes an identity value so that two instances can compare what they are running on. The hash is +/// unsalted and the audit instance serves it on its anonymous environment endpoint, so anyone who +/// can reach that endpoint can confirm a guessed value. It never appears in a usage report; only the +/// comparison result does. /// public static class IdentityHash { diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs index aefc7119c3..072681c132 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; +using System.Data; using System.Globalization; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; @@ -17,7 +18,6 @@ class PostgreSqlStorageFootprintProbe(PostgreSqlPersisterSettings settings, ISer await using var scope = scopeFactory.CreateAsyncScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); - var schema = settings.Schema ?? "public"; await using var command = dbContext.Database.GetDbConnection().CreateCommand(); command.CommandText = """ @@ -25,17 +25,18 @@ class PostgreSqlStorageFootprintProbe(PostgreSqlPersisterSettings settings, ISer (SELECT SUM(pg_total_relation_size(c.oid)) / 1073741824.0 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = @schema AND c.relkind IN ('r', 'm')), + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relkind IN ('r', 'm')), (SELECT CASE WHEN c.reltuples < 0 THEN NULL ELSE c.reltuples::bigint END FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = @schema AND c.relname = @table) + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relname = @table) """; command.CommandTimeout = ProbeTimeoutSeconds; var schemaParameter = command.CreateParameter(); schemaParameter.ParameterName = "@schema"; - schemaParameter.Value = schema; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; command.Parameters.Add(schemaParameter); var tableParameter = command.CreateParameter(); diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs index 79a275b2e8..45f477bfee 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs @@ -18,7 +18,8 @@ class PostgreSqlStorageIdentityProvider(PostgreSqlPersisterSettings settings, IS await using var command = dbContext.Database.GetDbConnection().CreateCommand(); // The cluster's initialisation identifier rather than an address: every connection to // the same cluster sees the same value however the host is spelled or proxied. - command.CommandText = "SELECT system_identifier::text, current_database() FROM pg_control_system()"; + // current_schema() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT system_identifier::text, current_database(), current_schema() FROM pg_control_system()"; command.CommandTimeout = ProbeTimeoutSeconds; await dbContext.Database.OpenConnectionAsync(cancellationToken); @@ -30,7 +31,7 @@ class PostgreSqlStorageIdentityProvider(PostgreSqlPersisterSettings settings, IS return null; } - return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? DefaultSchema); + return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -45,5 +46,4 @@ class PostgreSqlStorageIdentityProvider(PostgreSqlPersisterSettings settings, IS } const int ProbeTimeoutSeconds = 5; - const string DefaultSchema = "public"; } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs index 201a472522..c9b10a0c6c 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs @@ -24,7 +24,7 @@ public async Task Probe(CancellationToken cancellationToken = d await using var command = dbContext.Database.GetDbConnection().CreateCommand(); // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server // only and comes back null on Azure SQL Database, Managed Instance and Synapse. - command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(SERVERPROPERTY('Edition') AS nvarchar(128)), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; + command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; command.CommandTimeout = ProbeTimeoutSeconds; await dbContext.Database.OpenConnectionAsync(cancellationToken); @@ -44,8 +44,8 @@ public async Task Probe(CancellationToken cancellationToken = d HostingFor(engineEdition, rds, ConfiguredHost), MajorVersion(reader), DatabaseHostingSource.Probe, - reader.IsDBNull(3) ? DatabaseHosting.NotApplicable : reader.GetString(3), - reader.IsDBNull(4) ? DatabaseHosting.NotApplicable : reader.GetString(4)); + EditionFamily(engineEdition), + ServiceObjectiveTier(reader.IsDBNull(3) ? null : reader.GetString(3))); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -75,6 +75,48 @@ public async Task Probe(CancellationToken cancellationToken = d _ => DatabaseHostClassifier.Classify(host) }; + /// + /// Developer and Evaluation run the Enterprise engine and report Enterprise, so the report never + /// says which licence an instance runs under. The managed Azure services have no edition. + /// + internal static string EditionFamily(int engineEdition) => engineEdition switch + { + Standard => "Standard", + Enterprise => "Enterprise", + Express => "Express", + AzureSqlDatabase or AzureSqlManagedInstance => DatabaseHosting.NotApplicable, + _ => "Other" + }; + + /// + /// Keeps only the tier. The full service objective also encodes the hardware series, and a + /// DC-series name suggests Always Encrypted with secure enclaves. + /// + internal static string ServiceObjectiveTier(string? serviceObjective) + { + if (string.IsNullOrWhiteSpace(serviceObjective)) + { + return DatabaseHosting.NotApplicable; + } + + var objective = serviceObjective.Trim().ToUpperInvariant(); + + return objective switch + { + "BASIC" => "Basic", + "ELASTICPOOL" => "ElasticPool", + _ when objective.StartsWith("GP_", StringComparison.Ordinal) => "GeneralPurpose", + _ when objective.StartsWith("BC_", StringComparison.Ordinal) => "BusinessCritical", + _ when objective.StartsWith("HS_", StringComparison.Ordinal) => "Hyperscale", + _ when IsDtuObjective(objective, 'S') => "Standard", + _ when IsDtuObjective(objective, 'P') => "Premium", + _ => "Other" + }; + } + + static bool IsDtuObjective(string objective, char tier) => + objective.Length > 1 && objective[0] == tier && objective.AsSpan(1).IndexOfAnyExceptInRange('0', '9') < 0; + static string ManagedOrSelfHosted(string? host) { var classified = DatabaseHostClassifier.Classify(host); @@ -106,8 +148,8 @@ DatabaseHosting HostingFromConnectionString() var host = ConfiguredHost; return host is null - ? DatabaseHosting.Unclassified - : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString); + ? DatabaseHosting.Unclassified with { ServerEdition = DatabaseHostClassifier.Unknown, ServiceObjective = DatabaseHostClassifier.Unknown } + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString, DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown); } string? ConfiguredHost diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs index 632a192e96..6b0b87deee 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; +using System.Data; using System.Globalization; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; @@ -17,28 +18,30 @@ class SqlServerStorageFootprintProbe(SqlServerPersisterSettings settings, IServi await using var scope = scopeFactory.CreateAsyncScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); - var schema = settings.Schema ?? "dbo"; await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. command.CommandText = """ + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); SELECT (SELECT SUM(a.total_pages) * 8.0 / 1048576 - FROM sys.tables t - JOIN sys.schemas s ON s.schema_id = t.schema_id - JOIN sys.partitions p ON p.object_id = t.object_id + FROM sys.partitions p JOIN sys.allocation_units a ON a.container_id = p.partition_id - WHERE s.name = @schema), + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), (SELECT SUM(p.rows) FROM sys.tables t - JOIN sys.schemas s ON s.schema_id = t.schema_id JOIN sys.partitions p ON p.object_id = t.object_id - WHERE s.name = @schema AND t.name = @table AND p.index_id IN (0, 1)) + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) """; command.CommandTimeout = ProbeTimeoutSeconds; var schemaParameter = command.CreateParameter(); schemaParameter.ParameterName = "@schema"; - schemaParameter.Value = schema; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; command.Parameters.Add(schemaParameter); var tableParameter = command.CreateParameter(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs index 00016666ea..ae21a6c7df 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs @@ -16,7 +16,8 @@ class SqlServerStorageIdentityProvider(SqlServerPersisterSettings settings, ISer var dbContext = scope.ServiceProvider.GetRequiredService(); await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME()"; + // SCHEMA_NAME() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME(), SCHEMA_NAME()"; command.CommandTimeout = ProbeTimeoutSeconds; await dbContext.Database.OpenConnectionAsync(cancellationToken); @@ -28,7 +29,7 @@ class SqlServerStorageIdentityProvider(SqlServerPersisterSettings settings, ISer return null; } - return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? DefaultSchema); + return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -43,5 +44,4 @@ class SqlServerStorageIdentityProvider(SqlServerPersisterSettings settings, ISer } const int ProbeTimeoutSeconds = 5; - const string DefaultSchema = "dbo"; } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs index 8fd95aa661..0b3809c362 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -77,7 +77,9 @@ async ValueTask RetentionBehindHours(CancellationToken cancellationToken await using var scope = scopeFactory.CreateAsyncScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); - var oldest = await dbContext.Set().MinAsync(m => (DateTime?)m.StatusChangedAt, cancellationToken); + var oldest = await dbContext.Set() + .Where(m => m.Status == FailedMessageStatus.Resolved || m.Status == FailedMessageStatus.Archived) + .MinAsync(m => (DateTime?)m.StatusChangedAt, cancellationToken); if (oldest is null) { diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs index ff4e956df5..1c8088b7c8 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs @@ -23,8 +23,8 @@ public interface IDatabaseHostingProbe /// One of AzureSql, AzureSqlManagedInstance, AzureSqlEdge, AzurePostgres, AzureMySql, AwsRds, GoogleCloudSql, RavenCloud, SelfHosted or Unknown. /// The engine major version, or Unknown. /// A value. -/// The engine's own edition designation, such as SQL Server's Edition property. NotApplicable where the engine has no editions. -/// The Azure SQL service objective, such as S0 or GP_Gen5_2. NotApplicable everywhere else. +/// The SQL Server edition family: Express, Standard, Enterprise or Other. NotApplicable on the managed Azure services and on engines without editions. +/// The Azure SQL service tier: Basic, Standard, Premium, GeneralPurpose, BusinessCritical, Hyperscale, ElasticPool or Other. NotApplicable everywhere else. public record DatabaseHosting(string Hosting, string ServerVersion, string Source, string ServerEdition = DatabaseHosting.NotApplicable, string ServiceObjective = DatabaseHosting.NotApplicable) { public const string NotApplicable = "NotApplicable"; diff --git a/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs index 29f287d75a..0c75b444de 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -14,6 +14,9 @@ class RavenStorageIdentityProvider(RavenPersisterSettings settings) : IStorageId return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), settings.DatabaseName, null)); } - static string NormalizeServer(string configuredServer) => - Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) ? $"{url.Host}:{url.Port}" : configuredServer ?? ""; + // A loopback address names a different server on every machine, embedded servers included. + internal static string NormalizeServer(string? configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) + ? $"{(url.IsLoopback ? Environment.MachineName : url.Host)}:{url.Port}" + : configuredServer ?? ""; } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs index 534be6c71d..448658c141 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs @@ -1,30 +1,96 @@ namespace ServiceControl.Persistence.Tests; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using Particular.LicensingComponent.Contracts; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; class HealthEnvironmentDataTests : PersistenceTestBase { [Test] public async Task An_empty_schema_is_healthy() { - var provider = ServiceProvider.GetRequiredService(); - var data = new Dictionary(); + var data = await GetData(); - foreach (var datum in provider.GetData()) + using (Assert.EnterMultipleScope()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); } + } + + [Test] + public async Task An_old_unresolved_message_does_not_put_retention_behind() + { + var ancient = DateTime.UtcNow.AddDays(-400); + + await Store(FailedMessage(FailedMessageStatus.Unresolved, ancient)); + + var data = await GetData(); using (Assert.EnterMultipleScope()) { - Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); - Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("1")); } } + + [Test] + public async Task Retention_is_behind_by_the_oldest_resolved_or_archived_message_past_the_window() + { + var retention = PersistenceTestsContext.DefaultRetentionPeriod; + + await Store( + FailedMessage(FailedMessageStatus.Unresolved, DateTime.UtcNow.AddDays(-400)), + FailedMessage(FailedMessageStatus.Resolved, DateTime.UtcNow - retention - TimeSpan.FromHours(48)), + FailedMessage(FailedMessageStatus.Archived, DateTime.UtcNow - retention - TimeSpan.FromHours(24))); + + var data = await GetData(); + + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("48")); + } + + static FailedMessageEntity FailedMessage(FailedMessageStatus status, DateTime statusChangedAt) => new() + { + UniqueMessageId = Guid.NewGuid(), + Status = status, + StatusChangedAt = statusChangedAt, + LastModified = statusChangedAt, + NumberOfProcessingAttempts = 1, + FirstTimeOfFailure = statusChangedAt, + LastTimeOfFailure = statusChangedAt, + LastAttemptedAt = statusChangedAt, + HeadersJson = "{}", + FailingEndpointAddress = "Shipping" + }; + + async Task Store(params FailedMessageEntity[] messages) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.Set().AddRange(messages); + + await dbContext.SaveChangesAsync(CancellationToken.None); + } + + async Task> GetData() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + return data; + } } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs index 2bc2d5daac..f7a28ea2e2 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs @@ -1,8 +1,10 @@ namespace ServiceControl.Persistence.Tests; using System.Threading.Tasks; +using EFCore.Infrastructure; using EFCore.PostgreSql; using Microsoft.Extensions.DependencyInjection; +using Npgsql; using NUnit.Framework; class StorageIdentityProviderTests : PersistenceTestBase @@ -23,4 +25,30 @@ public async Task Identifies_the_cluster_database_and_schema() Assert.That(identity.Schema, Is.EqualTo(((PostgreSqlPersisterSettings)PersistenceSettings).Schema)); } } + + [Test] + public async Task Measures_the_search_path_schema_when_none_is_configured() + { + var configured = (PostgreSqlPersisterSettings)PersistenceSettings; + var services = new ServiceCollection(); + services.AddLogging(); + new PostgreSqlPersistenceConfiguration().Create(new PostgreSqlPersisterSettings + { + ConnectionString = new NpgsqlConnectionStringBuilder(configured.ConnectionString) { SearchPath = configured.Schema }.ConnectionString, + BodyStorage = configured.BodyStorage, + ErrorRetentionPeriod = configured.ErrorRetentionPeriod, + EventsRetentionPeriod = configured.EventsRetentionPeriod + }).AddPersistence(services); + await using var unconfigured = services.BuildServiceProvider(); + + var identity = await unconfigured.GetRequiredService().GetIdentity(); + var footprint = await unconfigured.GetRequiredService().Probe(); + var expected = await ServiceProvider.GetRequiredService().Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(identity?.Schema, Is.EqualTo(configured.Schema)); + Assert.That(footprint?.SizeGB, Is.Not.Null.And.EqualTo(expected?.SizeGB)); + } + } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs new file mode 100644 index 0000000000..cfb895b377 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.Tests.RavenDB; + +using System; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDB; + +[TestFixture] +class RavenStorageIdentityProviderTests +{ + [TestCase("http://localhost:33334")] + [TestCase("http://127.0.0.1:33334")] + [TestCase("http://[::1]:33334")] + public void A_loopback_server_is_this_machine(string url) => + Assert.That(RavenStorageIdentityProvider.NormalizeServer(url), Is.EqualTo($"{Environment.MachineName}:33334")); + + [Test] + public void A_remote_server_is_its_host_and_port() => + Assert.That(RavenStorageIdentityProvider.NormalizeServer("https://raven.example.com:8080"), Is.EqualTo("raven.example.com:8080")); +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs index 534be6c71d..448658c141 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs @@ -1,30 +1,96 @@ namespace ServiceControl.Persistence.Tests; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using Particular.LicensingComponent.Contracts; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; class HealthEnvironmentDataTests : PersistenceTestBase { [Test] public async Task An_empty_schema_is_healthy() { - var provider = ServiceProvider.GetRequiredService(); - var data = new Dictionary(); + var data = await GetData(); - foreach (var datum in provider.GetData()) + using (Assert.EnterMultipleScope()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); } + } + + [Test] + public async Task An_old_unresolved_message_does_not_put_retention_behind() + { + var ancient = DateTime.UtcNow.AddDays(-400); + + await Store(FailedMessage(FailedMessageStatus.Unresolved, ancient)); + + var data = await GetData(); using (Assert.EnterMultipleScope()) { - Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); - Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("1")); } } + + [Test] + public async Task Retention_is_behind_by_the_oldest_resolved_or_archived_message_past_the_window() + { + var retention = PersistenceTestsContext.DefaultRetentionPeriod; + + await Store( + FailedMessage(FailedMessageStatus.Unresolved, DateTime.UtcNow.AddDays(-400)), + FailedMessage(FailedMessageStatus.Resolved, DateTime.UtcNow - retention - TimeSpan.FromHours(48)), + FailedMessage(FailedMessageStatus.Archived, DateTime.UtcNow - retention - TimeSpan.FromHours(24))); + + var data = await GetData(); + + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("48")); + } + + static FailedMessageEntity FailedMessage(FailedMessageStatus status, DateTime statusChangedAt) => new() + { + UniqueMessageId = Guid.NewGuid(), + Status = status, + StatusChangedAt = statusChangedAt, + LastModified = statusChangedAt, + NumberOfProcessingAttempts = 1, + FirstTimeOfFailure = statusChangedAt, + LastTimeOfFailure = statusChangedAt, + LastAttemptedAt = statusChangedAt, + HeadersJson = "{}", + FailingEndpointAddress = "Shipping" + }; + + async Task Store(params FailedMessageEntity[] messages) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.Set().AddRange(messages); + + await dbContext.SaveChangesAsync(CancellationToken.None); + } + + async Task> GetData() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + return data; + } } diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs index 31d19aea3d..11780a2066 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs @@ -1,6 +1,11 @@ namespace ServiceControl.Persistence.Tests.SqlServer; +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.SqlServer; // EngineEdition values per @@ -50,4 +55,51 @@ public void Should_fall_back_to_the_host_for_an_unmapped_edition(int engineEditi public void Should_report_unknown_for_an_unmapped_edition_on_an_unrecognised_host(int engineEdition) => Assert.That(SqlServerDatabaseHostingProbe.HostingFor(engineEdition, false, "db01.corp.example"), Is.EqualTo("Unknown"), "An edition we do not map is not evidence of an ordinary SQL Server"); + + [TestCase(2, "Standard")] + [TestCase(3, "Enterprise")] + [TestCase(4, "Express")] + [TestCase(5, "NotApplicable")] + [TestCase(8, "NotApplicable")] + [TestCase(1, "Other")] + [TestCase(9, "Other")] + [TestCase(12, "Other")] + public void Should_report_the_edition_family(int engineEdition, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.EditionFamily(engineEdition), Is.EqualTo(expected)); + + [TestCase(null, "NotApplicable")] + [TestCase("Basic", "Basic")] + [TestCase("S0", "Standard")] + [TestCase("S12", "Standard")] + [TestCase("P15", "Premium")] + [TestCase("GP_Gen5_2", "GeneralPurpose")] + [TestCase("GP_S_Gen5_1", "GeneralPurpose")] + [TestCase("GP_DC_8", "GeneralPurpose")] + [TestCase("BC_Gen5_8", "BusinessCritical")] + [TestCase("HS_Gen5_4", "Hyperscale")] + [TestCase("ElasticPool", "ElasticPool")] + [TestCase("DW100c", "Other")] + [TestCase("System2", "Other")] + public void Should_report_only_the_service_objective_tier(string serviceObjective, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.ServiceObjectiveTier(serviceObjective), Is.EqualTo(expected)); + + [TestCase("Server=sc.database.windows.net;Database=sc")] + [TestCase("not a connection string")] + public async Task Should_report_edition_and_service_objective_as_unknown_when_the_server_cannot_be_asked(string connectionString) + { + var probe = new SqlServerDatabaseHostingProbe(new SqlServerPersisterSettings { ConnectionString = connectionString, BodyStorage = new FileSystemBodyStorageSettings { StoragePath = "/var/bodies" } }, new UnreachableScopeFactory(), NullLogger.Instance); + + var hosting = await probe.Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo("Unknown")); + Assert.That(hosting.ServiceObjective, Is.EqualTo("Unknown")); + } + } + + sealed class UnreachableScopeFactory : IServiceScopeFactory + { + public IServiceScope CreateScope() => throw new InvalidOperationException("The database cannot be reached"); + } } diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs index 267c7f5596..0a916cbc7e 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs @@ -29,10 +29,14 @@ public async Task Reports_the_engine_edition() var hosting = await probe.Probe(); + var managed = hosting.Hosting is "AzureSql" or "AzureSqlManagedInstance"; + using (Assert.EnterMultipleScope()) { - Assert.That(hosting.ServerEdition, Is.Not.EqualTo(DatabaseHosting.NotApplicable)); - Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServerEdition, managed ? Is.EqualTo(DatabaseHosting.NotApplicable) : Is.AnyOf("Express", "Standard", "Enterprise")); + Assert.That(hosting.ServiceObjective, hosting.Hosting == "AzureSql" ? Is.AnyOf(ServiceTiers) : Is.EqualTo(DatabaseHosting.NotApplicable)); } } + + static readonly object[] ServiceTiers = ["Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool"]; } diff --git a/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs index 2cc776bdad..e079f10d00 100644 --- a/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs +++ b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs @@ -55,6 +55,14 @@ public void Different_schemas_share_the_database() => public void Schemaless_engines_share_the_database_at_most() => Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "RavenDB", schema: null), Identity(engine: "RavenDB", schema: null)), Is.EqualTo("SameDatabase")); + [Test] + public void A_schema_is_a_store_of_its_own() => + Assert.That(AuditSharingClassifier.StoreKey(Identity()), Is.EqualTo($"{IdentityHash.Compute("server")}/{IdentityHash.Compute("database")}/{IdentityHash.Compute("schema")}")); + + [Test] + public void Schemaless_engines_are_one_store_per_database() => + Assert.That(AuditSharingClassifier.StoreKey(Identity(engine: "RavenDB", schema: null)), Is.EqualTo($"{IdentityHash.Compute("server")}/{IdentityHash.Compute("database")}")); + [Test] public void Hashing_normalises_case_and_whitespace() { diff --git a/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs b/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs new file mode 100644 index 0000000000..8aceb24c11 --- /dev/null +++ b/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.UnitTests.Licensing; + +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using Particular.ServiceControl.Licensing; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Api; +using ServiceControl.Monitoring.HeartbeatMonitoring; +using ServiceControl.UnitTests.ScatterGather; + +[TestFixture] +class ConfigurationApiRemoteEnvironmentTests +{ + [Test] + public async Task Reads_the_machine_hash_and_the_store_from_an_audit_response() + { + var remote = new RemoteInstanceSetting("http://audit.example:44444/api"); + var settings = new Settings { RemoteInstances = [remote] }; + var body = $$""" + { + "environment_data": { "Storage.Type": "SQLServer" }, + "machine_name_hash": "{{MachineIdentity.Hash}}", + "storage_identity": { "engine": "SQLServer", "server_hash": "s", "database_hash": "d", "schema_hash": "x" } + } + """; + var httpClientFactory = new FakeHttpClientFactory(); + httpClientFactory.Register(remote, (request, _) => Task.FromResult(request.RequestUri?.AbsolutePath == "/api/environment" + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") } + : new HttpResponseMessage(HttpStatusCode.NotFound))); + var api = new ConfigurationApi(new ActiveLicense(null, NullLogger.Instance), settings, httpClientFactory, new MassTransitConnectorHeartbeatStatus(), []); + + var environment = (await api.GetRemoteEnvironments()).Single().EnvironmentData; + + using (Assert.EnterMultipleScope()) + { + Assert.That(environment, Does.ContainKey("Storage.Type").WithValue("SQLServer")); + Assert.That(environment, Does.ContainKey("SameMachine").WithValue(MachineIdentity.Hash == MachineIdentity.NotApplicable ? "NotApplicable" : "True")); + Assert.That(environment, Does.ContainKey(AuditEnvironmentMetadata.DatabaseKey).WithValue("s/d/x")); + } + } +} diff --git a/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs index 78aab5f92e..263d8d914b 100644 --- a/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs +++ b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs @@ -13,19 +13,19 @@ static class AuditSharingClassifier public const string Unknown = "Unknown"; public const string NotApplicable = "NotApplicable"; - public static string SameMachine(string localMachineIdHash, string remoteMachineIdHash) + public static string SameMachine(string localMachineNameHash, string remoteMachineNameHash) { - if (localMachineIdHash is null || remoteMachineIdHash is null) + if (localMachineNameHash is null || remoteMachineNameHash is null) { return Unknown; } - if (localMachineIdHash == MachineIdentity.NotApplicable || remoteMachineIdHash == MachineIdentity.NotApplicable) + if (localMachineNameHash == MachineIdentity.NotApplicable || remoteMachineNameHash == MachineIdentity.NotApplicable) { return NotApplicable; } - return string.Equals(localMachineIdHash, remoteMachineIdHash, StringComparison.Ordinal) ? "True" : "False"; + return string.Equals(localMachineNameHash, remoteMachineNameHash, StringComparison.Ordinal) ? "True" : "False"; } public static string DatabaseSharing(HashedStorageIdentity local, HashedStorageIdentity remote) @@ -58,6 +58,15 @@ public static string DatabaseSharing(HashedStorageIdentity local, HashedStorageI return "SameDatabase"; } + /// + /// What the report counts once when summing sizes and counts. Instances writing to the same + /// schema share one store; instances in different schemas of one database each have their own. + /// + public static string StoreKey(HashedStorageIdentity identity) => + identity.SchemaHash is null + ? $"{identity.ServerHash}/{identity.DatabaseHash}" + : $"{identity.ServerHash}/{identity.DatabaseHash}/{identity.SchemaHash}"; + public static HashedStorageIdentity Hash(StorageIdentity identity) => identity is null ? null diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 75755b8a89..3873a445bc 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -145,7 +145,7 @@ public async Task GetRemoteConfigs(CancellationToken canc public async Task GetRemoteEnvironments(CancellationToken cancellationToken = default) { - var localMachineIdHash = MachineIdentity.Hash; + var localMachineNameHash = MachineIdentity.Hash; var localStorageIdentity = await LocalStorageIdentity(cancellationToken); var tasks = settings.RemoteInstances @@ -179,12 +179,12 @@ public async Task GetRemoteEnvironments(CancellationToken c var remoteStorageIdentity = ReadStorageIdentity(body); - environmentData["SameMachine"] = AuditSharingClassifier.SameMachine(localMachineIdHash, ReadString(body, "machine_id_hash")); + environmentData["SameMachine"] = AuditSharingClassifier.SameMachine(localMachineNameHash, ReadString(body, "machine_name_hash")); environmentData["DatabaseSharing"] = AuditSharingClassifier.DatabaseSharing(localStorageIdentity, remoteStorageIdentity); if (remoteStorageIdentity is not null) { - environmentData[AuditEnvironmentMetadata.DatabaseKey] = $"{remoteStorageIdentity.ServerHash}/{remoteStorageIdentity.DatabaseHash}"; + environmentData[AuditEnvironmentMetadata.DatabaseKey] = AuditSharingClassifier.StoreKey(remoteStorageIdentity); } } } From 4495376d5cb5c9f632e01da6a45e32a1b11c71ae Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 8 Oct 2026 12:51:37 +1000 Subject: [PATCH 8/9] Fix flaky full-text index size assertion in SQL Server audit footprint test WaitForFullTextIndex is now exposed as a public method with a default cancellation token so the environment data test can reuse the same wait the search tests rely on, instead of racing the index population. The test now waits for the full-text index to catch up before probing storage footprint, and asserts the footprint is only greater than the tables size rather than an exact sum, since the full-text index size can still drift slightly after the wait. The duplicated inline SQL for computing table and full-text sizes is consolidated into a single SizeGB helper. --- .../PersistenceTestsConfiguration.cs | 2 +- .../SqlServerEnvironmentDataTests.cs | 72 ++++++++----------- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs index 85143e7ba9..226175c015 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs @@ -27,7 +27,7 @@ class PersistenceTestsConfiguration : EFPersistenceTestsConfiguration protected override IAuditMessagesViewDataStore DecorateMessagesViewStore(IAuditMessagesViewDataStore store) => new SearchesWaitForFullTextIndex(store, WaitForFullTextIndex); - async Task WaitForFullTextIndex(CancellationToken cancellationToken) + public async Task WaitForFullTextIndex(CancellationToken cancellationToken = default) { await using var connection = new SqlConnection(ConnectionString); await connection.OpenAsync(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs index 303e7cf93c..348189f06e 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs @@ -87,52 +87,42 @@ public async Task Should_report_edition_and_service_objective_as_unknown_when_th public async Task Counts_the_full_text_index_in_the_size() { await Ingest(MakeMessage(), MakeMessage()); - - var fullTextGB = await WithDbContext(async (dbContext, token) => - { - await dbContext.Database.OpenConnectionAsync(token); - await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - command.CommandText = """ - SELECT COALESCE(SUM(a.total_pages), 0) * 8.0 / 1048576 - FROM sys.internal_tables it - JOIN sys.tables t ON t.object_id = it.parent_id - JOIN sys.partitions p ON p.object_id = it.object_id - JOIN sys.allocation_units a ON a.container_id = p.partition_id - WHERE t.schema_id = SCHEMA_ID(@schema) AND it.internal_type_desc LIKE 'FULLTEXT%' - """; - var schema = command.CreateParameter(); - schema.ParameterName = "@schema"; - schema.Value = dbContext.Schema; - command.Parameters.Add(schema); - - return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); - }); - - var tablesGB = await WithDbContext(async (dbContext, token) => - { - await dbContext.Database.OpenConnectionAsync(token); - await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - command.CommandText = """ - SELECT SUM(a.total_pages) * 8.0 / 1048576 - FROM sys.tables t - JOIN sys.partitions p ON p.object_id = t.object_id - JOIN sys.allocation_units a ON a.container_id = p.partition_id - WHERE t.schema_id = SCHEMA_ID(@schema) - """; - var schema = command.CreateParameter(); - schema.ParameterName = "@schema"; - schema.Value = dbContext.Schema; - command.Parameters.Add(schema); - - return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); - }); - + await configuration.WaitForFullTextIndex(TestTimeoutCancellationToken); + + var tablesGB = await SizeGB(""" + SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) + """); var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + var fullTextGB = await SizeGB(""" + SELECT COALESCE(SUM(a.total_pages), 0) * 8.0 / 1048576 + FROM sys.internal_tables it + JOIN sys.tables t ON t.object_id = it.parent_id + JOIN sys.partitions p ON p.object_id = it.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) AND it.internal_type_desc LIKE 'FULLTEXT%' + """); Assert.That(fullTextGB, Is.GreaterThan(0), "The audit messages table always has a full-text index"); - Assert.That(footprint?.SizeGB, Is.EqualTo(tablesGB + fullTextGB).Within(1e-9)); + Assert.That(footprint?.SizeGB, Is.GreaterThan(tablesGB), "The size counts the full-text index on top of the tables"); } + Task SizeGB(string sql) => WithDbContext(async (dbContext, token) => + { + await dbContext.Database.OpenConnectionAsync(token); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = sql; + var schema = command.CreateParameter(); + schema.ParameterName = "@schema"; + schema.Value = dbContext.Schema; + command.Parameters.Add(schema); + + return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); + }); + static readonly object[] ServiceTiers = ["Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool"]; sealed class UnreachableScopeFactory : IServiceScopeFactory From eed79073b4842743defaf21cdf310aa772b4efe7 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 8 Oct 2026 14:58:15 +1000 Subject: [PATCH 9/9] Retry SQL Server storage footprint probe on deadlock or lost scan position Catalog reads under READ COMMITTED can deadlock with concurrent DDL, so the probe now runs under READ UNCOMMITTED instead. That isolation level can let a scan lose its place when pages move, so both error codes are now retried up to three times with a short backoff before the probe gives up and returns null. Applied to both the primary and audit EF Core SQL Server persisters, and the audit environment data test's inline SQL is updated to match the new isolation level. --- .../SqlServerStorageFootprintProbe.cs | 115 ++++++++++-------- .../SqlServerEnvironmentDataTests.cs | 2 +- .../SqlServerStorageFootprintProbe.cs | 115 ++++++++++-------- 3 files changed, 135 insertions(+), 97 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs index a1407c63d6..3456371456 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -2,6 +2,7 @@ namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; using System.Data; using System.Globalization; +using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -14,66 +15,84 @@ class SqlServerStorageFootprintProbe(EFPersisterSettings settings, IServiceScope { public async Task Probe(CancellationToken cancellationToken = default) { - try + for (var attempt = 1; ; attempt++) { - await using var scope = scopeFactory.CreateAsyncScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); + try + { + return await Read(cancellationToken); + } + catch (SqlException e) when (attempt < MaxAttempts && e.Number is DeadlockVictim or ScanLostItsPlace) + { + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); - await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - // Full-text index fragments live in internal tables under the indexed table, so they are counted too. - command.CommandText = """ - DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); - SELECT - (SELECT SUM(a.total_pages) * 8.0 / 1048576 - FROM sys.partitions p - JOIN sys.allocation_units a ON a.container_id = p.partition_id - WHERE p.object_id IN ( - SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) - UNION ALL - SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), - (SELECT SUM(p.rows) - FROM sys.tables t - JOIN sys.partitions p ON p.object_id = t.object_id - WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) - """; - command.CommandTimeout = ProbeTimeoutSeconds; + return null; + } + } + } - var schemaParameter = command.CreateParameter(); - schemaParameter.ParameterName = "@schema"; - schemaParameter.DbType = DbType.String; - schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; - command.Parameters.Add(schemaParameter); + async Task Read(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); - var tableParameter = command.CreateParameter(); - tableParameter.ParameterName = "@table"; - tableParameter.Value = auditMessagesTable ?? ""; - command.Parameters.Add(tableParameter); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. + // Catalog reads under READ COMMITTED deadlock with concurrent DDL. Under READ UNCOMMITTED a scan can + // instead lose its place when pages move, so Probe retries both errors. + command.CommandText = """ + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.partitions p + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; - await dbContext.Database.OpenConnectionAsync(cancellationToken); + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); - await using var reader = await command.ExecuteReaderAsync(cancellationToken); + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = auditMessagesTable ?? ""; + command.Parameters.Add(tableParameter); - if (!await reader.ReadAsync(cancellationToken)) - { - return null; - } + await dbContext.Database.OpenConnectionAsync(cancellationToken); - return new StorageFootprint( - reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), - reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception e) - { - logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { return null; } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); } const int ProbeTimeoutSeconds = 30; + const int MaxAttempts = 3; + const int DeadlockVictim = 1205; + const int ScanLostItsPlace = 601; } diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs index 348189f06e..1ce23546f8 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs @@ -114,7 +114,7 @@ Task SizeGB(string sql) => WithDbContext(async (dbContext, token) => { await dbContext.Database.OpenConnectionAsync(token); await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - command.CommandText = sql; + command.CommandText = "SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; " + sql; var schema = command.CreateParameter(); schema.ParameterName = "@schema"; schema.Value = dbContext.Schema; diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs index 6b0b87deee..c8c965bdfc 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -2,6 +2,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using System.Data; using System.Globalization; +using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -13,66 +14,84 @@ class SqlServerStorageFootprintProbe(SqlServerPersisterSettings settings, IServi { public async Task Probe(CancellationToken cancellationToken = default) { - try + for (var attempt = 1; ; attempt++) { - await using var scope = scopeFactory.CreateAsyncScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); + try + { + return await Read(cancellationToken); + } + catch (SqlException e) when (attempt < MaxAttempts && e.Number is DeadlockVictim or ScanLostItsPlace) + { + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); - await using var command = dbContext.Database.GetDbConnection().CreateCommand(); - // Full-text index fragments live in internal tables under the indexed table, so they are counted too. - command.CommandText = """ - DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); - SELECT - (SELECT SUM(a.total_pages) * 8.0 / 1048576 - FROM sys.partitions p - JOIN sys.allocation_units a ON a.container_id = p.partition_id - WHERE p.object_id IN ( - SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) - UNION ALL - SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), - (SELECT SUM(p.rows) - FROM sys.tables t - JOIN sys.partitions p ON p.object_id = t.object_id - WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) - """; - command.CommandTimeout = ProbeTimeoutSeconds; + return null; + } + } + } - var schemaParameter = command.CreateParameter(); - schemaParameter.ParameterName = "@schema"; - schemaParameter.DbType = DbType.String; - schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; - command.Parameters.Add(schemaParameter); + async Task Read(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); - var tableParameter = command.CreateParameter(); - tableParameter.ParameterName = "@table"; - tableParameter.Value = failedMessagesTable ?? ""; - command.Parameters.Add(tableParameter); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. + // Catalog reads under READ COMMITTED deadlock with concurrent DDL. Under READ UNCOMMITTED a scan can + // instead lose its place when pages move, so Probe retries both errors. + command.CommandText = """ + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.partitions p + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; - await dbContext.Database.OpenConnectionAsync(cancellationToken); + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); - await using var reader = await command.ExecuteReaderAsync(cancellationToken); + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = failedMessagesTable ?? ""; + command.Parameters.Add(tableParameter); - if (!await reader.ReadAsync(cancellationToken)) - { - return null; - } + await dbContext.Database.OpenConnectionAsync(cancellationToken); - return new StorageFootprint( - reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), - reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception e) - { - logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { return null; } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); } const int ProbeTimeoutSeconds = 30; + const int MaxAttempts = 3; + const int DeadlockVictim = 1205; + const int ScanLostItsPlace = 601; }