diff --git a/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs new file mode 100644 index 0000000000..b0db55fe43 --- /dev/null +++ b/src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs @@ -0,0 +1,16 @@ +namespace Particular.LicensingComponent.Contracts; + +/// +/// The environment data each audit remote served when last polled, one dictionary per remote that +/// responded. Never saved by versions that predate it, so a null read means unknown and the report +/// omits the keys. +/// +public record AuditEnvironmentMetadata(List> Instances) +{ + /// + /// The key under which the primary stores which database an instance's data came from, so that + /// instances sharing a database are counted once when sizes and counts are summed. Underscore + /// prefixed keys are collection bookkeeping and never reach the report. + /// + public const string DatabaseKey = "_DatabaseKey"; +} diff --git a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs index 257eef677a..611611fdee 100644 --- a/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs @@ -14,6 +14,7 @@ public class InMemoryLicensingDataStore : ILicensingDataStore readonly Dictionary allThroughput = []; BrokerMetadata brokerMetadata = new(null, []); AuditServiceMetadata auditServiceMetadata = new([], []); + AuditEnvironmentMetadata? auditEnvironmentMetadata; List reportMasks = []; LicensedEndpointDetails? endpointDetails = null; @@ -167,6 +168,14 @@ public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, return Task.CompletedTask; } + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => Task.FromResult(auditEnvironmentMetadata); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) + { + this.auditEnvironmentMetadata = auditEnvironmentMetadata; + return Task.CompletedTask; + } + public Task> GetReportMasks(CancellationToken cancellationToken = default) => Task.FromResult(reportMasks); public Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default) diff --git a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs index 9dc65e1caf..bbf997ecf5 100644 --- a/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs +++ b/src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs @@ -36,6 +36,10 @@ Task RecordEndpointThroughput(string endpointName, ThroughputSource throughputSo Task GetAuditServiceMetadata(CancellationToken cancellationToken = default); Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default); + + Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default); + + Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default); Task> GetReportMasks(CancellationToken cancellationToken = default); Task SaveReportMasks(List reportMasks, CancellationToken cancellationToken = default); diff --git a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs index 082fe266ee..b64a0c12c8 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs @@ -192,6 +192,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati } public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi @@ -201,6 +203,8 @@ class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => Task.FromResult(remotes); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); } class ConfigurationApi_Configurable : IConfigurationApi @@ -221,6 +225,8 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public bool ReturnAuditConfig { get; set; } public string RemoteStatus { get; set; } public string RemoteVersion { get; set; } diff --git a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs index a77b84783a..3174f5bea7 100644 --- a/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs @@ -272,10 +272,40 @@ public async Task Should_store_how_many_audit_instances_are_configured_and_how_m } } + [Test] + public async Task Should_store_the_environment_data_of_every_responding_audit_instance() + { + var auditQuery = new AuditQuery_WithRemotes([new RemoteInstanceInformation { Status = "online" }]) + { + Environments = + [ + new Dictionary { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "4" }, + new Dictionary { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "8" } + ] + }; + + using var auditThroughputCollectorHostedService = new AuditThroughputCollectorHostedService( + NullLogger.Instance, configuration.ThroughputSettings, DataStore, + auditQuery, new FakeTimeProvider()) + { DelayStart = TimeSpan.Zero }; + + await auditThroughputCollectorHostedService.StartAsync(CancellationToken.None); + await auditQuery.KnownEndpointsRequested.WaitAsync(TimeSpan.FromSeconds(30)); + await auditThroughputCollectorHostedService.StopAsync(CancellationToken.None); + + var auditEnvironmentMetadata = await DataStore.GetAuditEnvironmentMetadata(); + + Assert.That(auditEnvironmentMetadata, Is.Not.Null); + Assert.That(auditEnvironmentMetadata.Instances, Is.EqualTo(auditQuery.Environments)); + } + class AuditQuery_NoAuditRemotes : IAuditQuery { public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -309,6 +339,9 @@ public AuditQuery_WithOneEndpoint(string endpointName, long throughputCount, Dat public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -341,6 +374,9 @@ class AuditQuery_ThrowingAnExceptionOnKnownEndpointsCall : IAuditQuery { public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => r => true; public Task> GetAuditCountForEndpoint(string endpointUrlName, @@ -420,6 +456,10 @@ public AuditQuery_WithTwoEndpointsAndSelectiveCounts( } public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult>>([]); + public Func ValidRemoteInstances => _ => true; public Task> GetKnownEndpoints(CancellationToken cancellationToken = default) => @@ -456,6 +496,12 @@ class AuditQuery_WithRemotes(List remotes) : IAuditQu public Task KnownEndpointsRequested => knownEndpointsRequested.Task; public SemanticVersion MinAuditCountsVersion => new(4, 29, 0); + + public List> Environments { get; init; } = []; + + public Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) => + Task.FromResult(Environments); + public Func ValidRemoteInstances => _ => true; public Task> GetAuditRemotes(CancellationToken cancellationToken = default) => diff --git a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs index d7d0893be2..0e18e277ae 100644 --- a/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs +++ b/src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs @@ -10,6 +10,7 @@ class FakeConfigurationApi : IConfigurationApi { public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetRemoteConfigs(CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException(); public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException(); } } diff --git a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs index a4d39e1b1c..f6d1e5651c 100644 --- a/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs @@ -252,6 +252,12 @@ public Task GetAuditServiceMetadata(CancellationToken canc public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => throw new NotSupportedException(); diff --git a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs index b91bac763e..ce584e91b3 100644 --- a/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs +++ b/src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs @@ -1,6 +1,8 @@ namespace Particular.LicensingComponent.UnitTests; +using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -197,6 +199,17 @@ await DataStore.CreateBuilder() Assert.That(report.ReportData.ScopeType, Is.EqualTo(expectedScopeType), $"Invalid ScopeType on report"); } + [Test] + public async Task Should_leave_the_stored_broker_metadata_unchanged() + { + await DataStore.SaveBrokerMetadata(new BrokerMetadata("testingScope", new Dictionary { [EnvironmentDataType.BrokerVersion.ToString()] = "1.2" })); + + await ThroughputCollector.GenerateThroughputReport("", null); + + var stored = await DataStore.GetBrokerMetadata(); + Assert.That(stored.Data.Keys, Is.EquivalentTo(new[] { EnvironmentDataType.BrokerVersion.ToString() })); + } + [Test] public async Task Should_include_audit_instance_counts_in_environment_data() { @@ -228,4 +241,159 @@ await DataStore.SaveAuditServiceMetadata(new AuditServiceMetadata( Assert.That(environmentData, Does.Not.ContainKey("Audit.LiveInstances")); } } + + [Test] + public async Task Should_report_audit_environment_data_aggregated_across_instances() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = "6.2.1", + ["Host.ProcessorCount"] = "4", + ["Host.AvailableMemoryGB"] = "16", + ["Host.OSPlatform"] = "Linux" + }, + new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = "5.4.200", + ["Host.ProcessorCount"] = "8", + ["Host.AvailableMemoryGB"] = "Unknown" + } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB")); + Assert.That(environmentData, Does.ContainKey("Audit.Storage.ServerVersion").WithValue("Mixed")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.ProcessorCount").WithValue("8")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.AvailableMemoryGB").WithValue("Mixed")); + Assert.That(environmentData, Does.ContainKey("Audit.Host.OSPlatform").WithValue("Linux")); + } + } + + [Test] + public async Task Should_report_the_most_shared_database_class_and_mixed_machines() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["DatabaseSharing"] = "SeparateServer", ["SameMachine"] = "True" }, + new Dictionary { ["DatabaseSharing"] = "SameServer", ["SameMachine"] = "False" }, + new Dictionary { ["DatabaseSharing"] = "Unknown", ["SameMachine"] = "True" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.DatabaseSharing").WithValue("SameServer")); + Assert.That(environmentData, Does.ContainKey("Audit.SameMachine").WithValue("Mixed")); + } + } + + [Test] + public async Task Should_sum_sizes_and_counts_once_per_database() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["Storage.SizeGB"] = "10.0", ["Storage.MessageCount"] = "100", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "10.2", ["Storage.MessageCount"] = "90", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Storage.SizeGB"] = "5.0", ["Storage.MessageCount"] = "10", ["Health.FailedImports"] = "4" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.SizeGB").WithValue("15.2")); + Assert.That(environmentData, Does.ContainKey("Audit.Storage.MessageCount").WithValue("110")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.FailedImports").WithValue("7")); + Assert.That(environmentData.Keys, Has.None.StartsWith("Audit._")); + } + } + + [Test] + public async Task Should_leave_audit_environment_data_out_when_it_was_never_collected() + { + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); + } + + [Test] + public async Task Should_sum_audit_ingestion_rates_and_keep_the_worst_saturation_lag_and_uptime() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["Ingestion.AvgDailyMessages"] = "1000", ["Ingestion.BusyPercent"] = "10", ["Health.UptimeHours"] = "500", ["Health.LagOver1MinPercent"] = "1", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }, + new Dictionary { ["Ingestion.AvgDailyMessages"] = "500", ["Ingestion.BusyPercent"] = "40", ["Health.UptimeHours"] = "3", ["Health.LagOver1MinPercent"] = "7", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.AvgDailyMessages").WithValue("1500")); + Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.BusyPercent").WithValue("40")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.UptimeHours").WithValue("3")); + Assert.That(environmentData, Does.ContainKey("Audit.Health.LagOver1MinPercent").WithValue("7")); + } + } + + [Test] + public async Task Should_leave_audit_environment_data_out_when_no_instance_responded() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage.")); + } + + [Test] + public async Task Should_report_every_key_an_audit_instance_serves() + { + string[] served = + [ + "Host.Model", "Host.Orchestrator", "Host.OSPlatform", "Host.OSVersion", "Host.Architecture", "Host.RuntimeVersion", "Host.ProcessorCount", "Host.AvailableMemoryGB", + "Storage.Type", "Storage.RavenServer", "Storage.Hosting", "Storage.ServerVersion", "Storage.HostingSource", "Storage.ServerEdition", "Storage.ServiceObjective", + "Storage.SizeGB", "Storage.MessageCount", "Storage.FullTextSearch", + "Health.FailedImports", "Health.UptimeHours", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent", + "Ingestion.AvgDailyMessages", "Ingestion.BusyPercent", + "SameMachine", "DatabaseSharing" + ]; + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([served.ToDictionary(key => key, _ => "1")])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys.Where(key => key.StartsWith("Audit.") && key is not "Audit.ConfiguredInstances" and not "Audit.LiveInstances"), + Is.EquivalentTo(served.Select(key => "Audit." + key))); + } + + [Test] + public async Task Should_leave_out_audit_keys_the_primary_does_not_report() + { + await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata( + [ + new Dictionary { ["Storage.Type"] = "RavenDB", ["Security.Authentication"] = "Enabled", ["Storage.Auth"] = "Integrated" } + ])); + + var report = await ThroughputCollector.GenerateThroughputReport("", null); + + var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData; + using (Assert.EnterMultipleScope()) + { + Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Security.Authentication")); + Assert.That(environmentData, Does.Not.ContainKey("Audit.Storage.Auth")); + } + } } \ No newline at end of file diff --git a/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs new file mode 100644 index 0000000000..dff5e4da19 --- /dev/null +++ b/src/Particular.LicensingComponent/AuditEnvironmentDataAggregator.cs @@ -0,0 +1,159 @@ +namespace Particular.LicensingComponent; + +using System.Globalization; +using Contracts; + +/// +/// Folds the environment data of every responding audit instance into one value per key: values +/// that agree pass through, capacity and saturation numbers take the maximum, uptime takes the +/// minimum, sizes and counts sum with instances on one database counted once, ingestion rates sum +/// across all instances, the sharing class takes the most shared, and anything else that differs +/// reports Mixed rather than guessing which instance is representative. Only keys in +/// are emitted, so a key an audit instance of another version serves, or +/// the bookkeeping keys the primary adds, never reach the report unreviewed. +/// +static class AuditEnvironmentDataAggregator +{ + static readonly HashSet ReportedKeys = new(StringComparer.Ordinal) + { + "Host.Model", + "Host.Orchestrator", + "Host.OSPlatform", + "Host.OSVersion", + "Host.Architecture", + "Host.RuntimeVersion", + "Host.ProcessorCount", + "Host.AvailableMemoryGB", + "Storage.Type", + "Storage.RavenServer", + "Storage.Hosting", + "Storage.ServerVersion", + "Storage.HostingSource", + "Storage.ServerEdition", + "Storage.ServiceObjective", + "Storage.SizeGB", + "Storage.MessageCount", + "Storage.FullTextSearch", + "Health.FailedImports", + "Health.UptimeHours", + "Health.LagOver1MinPercent", + "Health.LagOver10MinPercent", + "Health.LagOver60MinPercent", + "Ingestion.AvgDailyMessages", + "Ingestion.BusyPercent", + "SameMachine", + "DatabaseSharing" + }; + + static readonly string[] MaximumKeys = ["Host.ProcessorCount", "Host.AvailableMemoryGB", "Ingestion.BusyPercent", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent"]; + static readonly string[] MinimumKeys = ["Health.UptimeHours"]; + static readonly string[] SharingPrecedence = ["SameSchema", "SameDatabase", "SameServer", "SeparateServer", "NotApplicable", "Unknown"]; + + public static IEnumerable> Aggregate(List> instances) + { + if (instances is null) + { + yield break; + } + + var keys = instances + .SelectMany(instance => instance.Keys) + .Where(ReportedKeys.Contains) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal); + + foreach (var key in keys) + { + var contributions = instances + .Where(instance => instance.ContainsKey(key)) + .Select((instance, index) => (Value: instance[key], DatabaseKey: instance.GetValueOrDefault(AuditEnvironmentMetadata.DatabaseKey) ?? $"_{index}")) + .ToArray(); + + yield return new(key, Aggregate(key, contributions)); + } + } + + static string Aggregate(string key, (string Value, string DatabaseKey)[] contributions) + { + if (key == "Storage.SizeGB") + { + var sum = SumOncePerDatabase(contributions, value => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var size) ? size : null); + + if (sum is not null) + { + return sum.Value.ToString("F1", CultureInfo.InvariantCulture); + } + } + + if (key is "Storage.MessageCount" or "Health.FailedImports") + { + var sum = SumOncePerDatabase(contributions, value => long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) ? count : null); + + if (sum is not null) + { + return ((long)sum.Value).ToString(CultureInfo.InvariantCulture); + } + } + + var values = contributions.Select(contribution => contribution.Value).ToArray(); + + if (key == "Ingestion.AvgDailyMessages") + { + var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); + + if (parsed.All(value => value.Parsed)) + { + return parsed.Sum(value => value.Number).ToString(CultureInfo.InvariantCulture); + } + } + + if (values.Distinct(StringComparer.Ordinal).Count() == 1) + { + return values[0]; + } + + // The most-shared class across instances, because one shared database matters however many + // separate ones sit beside it. + if (key == "DatabaseSharing") + { + foreach (var sharing in SharingPrecedence) + { + if (values.Contains(sharing, StringComparer.Ordinal)) + { + return sharing; + } + } + } + + if (MaximumKeys.Contains(key, StringComparer.Ordinal) || MinimumKeys.Contains(key, StringComparer.Ordinal)) + { + var parsed = values.Select(value => (Parsed: long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var number), Number: number)).ToArray(); + + if (parsed.All(value => value.Parsed)) + { + var numbers = parsed.Select(value => value.Number); + + return (MinimumKeys.Contains(key, StringComparer.Ordinal) ? numbers.Min() : numbers.Max()).ToString(CultureInfo.InvariantCulture); + } + } + + return "Mixed"; + } + + static double? SumOncePerDatabase((string Value, string DatabaseKey)[] contributions, Func parse) + { + var parsed = contributions + .Select(contribution => (contribution.DatabaseKey, Number: parse(contribution.Value))) + .Where(contribution => contribution.Number is not null) + .ToArray(); + + if (parsed.Length == 0) + { + return null; + } + + return parsed + .GroupBy(contribution => contribution.DatabaseKey, StringComparer.Ordinal) + .Sum(group => group.Max(contribution => contribution.Number!.Value)); + } +} diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs index e048619e34..d3f633d55f 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs @@ -114,6 +114,25 @@ public async Task> GetAuditRemotes(CancellationT } } + public async Task>> GetAuditEnvironments(CancellationToken cancellationToken = default) + { + try + { + var remoteEnvironments = await configurationApi.GetRemoteEnvironments(cancellationToken); + + return [.. remoteEnvironments.Where(remote => remote.EnvironmentData is not null).Select(remote => remote.EnvironmentData)]; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get Audit Environments"); + return []; + } + } + public async Task TestAuditConnection(CancellationToken cancellationToken = default) { var connectionTestResult = new ConnectionSettingsTestResult { ConnectionSuccessful = true }; diff --git a/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs b/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs index 9e46eb2f4c..9d5398382f 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs @@ -119,6 +119,7 @@ async Task VerifyAuditInstances(CancellationToken cancellationToken) { var remotesInfo = await auditQuery.GetAuditRemotes(cancellationToken); await SaveAuditInstanceData(remotesInfo, cancellationToken); + await SaveAuditEnvironmentData(cancellationToken); foreach (var remote in remotesInfo) { @@ -164,4 +165,10 @@ async Task SaveAuditInstanceData(List? auditRemotes, await dataStore.SaveAuditServiceMetadata(auditServiceMetadata, cancellationToken); } } + + async Task SaveAuditEnvironmentData(CancellationToken cancellationToken) + { + var environments = await auditQuery.GetAuditEnvironments(cancellationToken); + await dataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(environments), cancellationToken); + } } \ No newline at end of file diff --git a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs index 66f03b7f35..cf0252dfd1 100644 --- a/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs +++ b/src/Particular.LicensingComponent/AuditThroughput/IAuditQuery.cs @@ -12,6 +12,7 @@ public interface IAuditQuery Task> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default); Task> GetAuditRemotes(CancellationToken cancellationToken = default); + Task>> GetAuditEnvironments(CancellationToken cancellationToken = default); Task TestAuditConnection(CancellationToken cancellationToken = default); } diff --git a/src/Particular.LicensingComponent/ThroughputCollector.cs b/src/Particular.LicensingComponent/ThroughputCollector.cs index e2186f6dda..e11853e87f 100644 --- a/src/Particular.LicensingComponent/ThroughputCollector.cs +++ b/src/Particular.LicensingComponent/ThroughputCollector.cs @@ -168,7 +168,7 @@ public async Task GenerateThroughputReport(string spVersion, DateT IgnoredQueues = [.. ignoredQueueNames], Queues = [.. queueThroughputs], TotalQueues = queueThroughputs.Count, - EnvironmentInformation = new EnvironmentInformation { AuditServicesData = new AuditServicesData(auditServiceMetadata.Versions, auditServiceMetadata.Transports), EnvironmentData = brokerMetaData.Data } + EnvironmentInformation = new EnvironmentInformation { AuditServicesData = new AuditServicesData(auditServiceMetadata.Versions, auditServiceMetadata.Transports), EnvironmentData = new Dictionary(brokerMetaData.Data) } }; var auditThroughput = queueThroughputs.SelectMany(w => w.DailyThroughputFromAudit); @@ -197,6 +197,16 @@ public async Task GenerateThroughputReport(string spVersion, DateT report.EnvironmentInformation.EnvironmentData["Audit.LiveInstances"] = liveAuditInstances.ToString(CultureInfo.InvariantCulture); } + var auditEnvironmentMetadata = await dataStore.GetAuditEnvironmentMetadata(cancellationToken); + + if (auditEnvironmentMetadata is not null) + { + foreach (var (key, value) in AuditEnvironmentDataAggregator.Aggregate(auditEnvironmentMetadata.Instances)) + { + report.EnvironmentInformation.EnvironmentData["Audit." + key] = value; + } + } + foreach (var environmentDataProvider in environmentDataProviders) { EnvironmentDatum[] environmentData; diff --git a/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs b/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs index d5f23ea1e8..40873df82e 100644 --- a/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs +++ b/src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs @@ -121,7 +121,8 @@ await Define() "Recoverability.RetryHistoryDepth", "Licensing.ReportMasks", "Retention.ErrorHours", - "Retention.EventsHours" + "Retention.EventsHours", + "Health.Error.UptimeHours" ]; [Test] diff --git a/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs b/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs new file mode 100644 index 0000000000..f449c326f1 --- /dev/null +++ b/src/ServiceControl.Api/Contracts/RemoteEnvironment.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Api.Contracts +{ + using System.Collections.Generic; + + public class RemoteEnvironment + { + public string ApiUri { get; set; } + + /// + /// Null when the remote could not be reached or predates the environment endpoint. + /// + public Dictionary EnvironmentData { get; set; } + } +} diff --git a/src/ServiceControl.Api/IConfigurationApi.cs b/src/ServiceControl.Api/IConfigurationApi.cs index 6a2dc1ad7c..b16c977437 100644 --- a/src/ServiceControl.Api/IConfigurationApi.cs +++ b/src/ServiceControl.Api/IConfigurationApi.cs @@ -11,5 +11,7 @@ public interface IConfigurationApi Task GetConfig(CancellationToken cancellationToken = default); Task GetRemoteConfigs(CancellationToken cancellationToken = default); + + Task GetRemoteEnvironments(CancellationToken cancellationToken = default); } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs new file mode 100644 index 0000000000..b1b32c4c5e --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests/When_environment_data_is_requested.cs @@ -0,0 +1,58 @@ +namespace ServiceControl.Audit.AcceptanceTests +{ + using System.Globalization; + using System.Text.Json.Nodes; + using System.Threading.Tasks; + using AcceptanceTesting; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + + class When_environment_data_is_requested : AcceptanceTest + { + [Test] + public async Task Should_serve_host_and_storage_facts() + { + JsonObject body = null; + + _ = await Define() + .Done(async ctx => + { + var response = await this.GetRaw("/api/environment"); + + if (!response.IsSuccessStatusCode) + { + return false; + } + + body = (await JsonNode.ParseAsync(await response.Content.ReadAsStreamAsync()))?.AsObject(); + + return body?["environment_data"] is not null; + }) + .Run(); + + var environmentData = body["environment_data"].AsObject(); + var storageIdentity = body["storage_identity"]?.AsObject(); + + using (Assert.EnterMultipleScope()) + { + Assert.That((string)environmentData["Storage.Type"], Is.Not.Null.And.Not.Empty); + Assert.That((string)environmentData["Storage.ServerVersion"], Is.Not.Null.And.Not.Empty); + Assert.That((string)environmentData["Storage.FullTextSearch"], Is.AnyOf("Enabled", "Disabled")); + Assert.That(double.Parse((string)environmentData["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); + Assert.That((string)environmentData["Storage.MessageCount"], Is.AnyOf("0", "Unknown")); + Assert.That((string)environmentData["Storage.ServerEdition"], Is.AnyOf("Express", "Standard", "Enterprise", "NotApplicable")); + Assert.That((string)environmentData["Health.FailedImports"], Is.EqualTo("0")); + Assert.That((string)environmentData["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console")); + Assert.That((string)environmentData["Host.ProcessorCount"], Is.Not.Null.And.Not.Empty); + Assert.That((string)body["machine_name_hash"], Is.Not.Null.And.Not.Empty); + Assert.That(storageIdentity, Is.Not.Null); + Assert.That((string)storageIdentity["engine"], Is.Not.Null.And.Not.Empty); + Assert.That((string)storageIdentity["server_hash"], Has.Length.EqualTo(64)); + Assert.That((string)storageIdentity["database_hash"], Has.Length.EqualTo(64)); + Assert.That(long.Parse((string)environmentData["Health.UptimeHours"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); + } + } + + class Context : ScenarioContext; + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs new file mode 100644 index 0000000000..69365ba3c2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlDatabaseHostingProbe.cs @@ -0,0 +1,119 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using System.Data.Common; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Npgsql; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlDatabaseHostingProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IDatabaseHostingProbe +{ + public string StorageName => "PostgreSQL"; + + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = ProbeSql; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return HostingFromConnectionString(); + } + + var hosting = HostingFor(reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), ConfiguredHost); + + return new DatabaseHosting(hosting, MajorVersion(reader), DatabaseHostingSource.Probe); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not ask PostgreSQL how it is hosted, falling back to the connection string"); + + return HostingFromConnectionString(); + } + } + + /// + /// PostgreSQL has no equivalent of SQL Server's EngineEdition, but every managed offering + /// creates a distinctive administrative role that a self-hosted server does not have. A server + /// that answered and has none of them is self-hosted, unless its host name names a managed + /// service that does not announce itself this way. + /// + internal static string HostingFor(bool azure, bool rds, bool cloudSql, string? host) + { + if (azure) + { + return "AzurePostgres"; + } + + if (rds) + { + return "AwsRds"; + } + + if (cloudSql) + { + return "GoogleCloudSql"; + } + + var classified = DatabaseHostClassifier.Classify(host); + + return classified == DatabaseHostClassifier.Unknown ? DatabaseHostClassifier.SelfHosted : classified; + } + + static string MajorVersion(DbDataReader reader) => + reader.IsDBNull(0) ? DatabaseHostClassifier.Unknown : (reader.GetInt32(0) / 10000).ToString(CultureInfo.InvariantCulture); + + DatabaseHosting HostingFromConnectionString() + { + var host = ConfiguredHost; + + return host is null + ? DatabaseHosting.Unclassified + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString); + } + + string? ConfiguredHost + { + get + { + try + { + return new NpgsqlConnectionStringBuilder(settings.ConnectionString).Host; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the configured PostgreSQL host"); + + return null; + } + } + } + + const string ProbeSql = """ + SELECT current_setting('server_version_num')::int, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'azure_pg_admin'), + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rds_superuser'), + EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'cloudsqlsuperuser') + """; + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 66d36b959f..bf8d700e80 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -59,6 +59,13 @@ protected override void AddQueryServices(IServiceCollection services) services.AddSingleton(); } + protected override void AddEnvironmentProbes(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } + protected override void AddCustomChecks(EndpointConfiguration endpointConfiguration) => endpointConfiguration.AddCustomCheck(); } diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs new file mode 100644 index 0000000000..c5abd8f862 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using System.Data; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlStorageFootprintProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The audit messages table is partitioned by day, and the parent carries no row estimate, + // so the estimate is summed over the partitions that have been analysed. + command.CommandText = """ + SELECT + (SELECT SUM(pg_total_relation_size(c.oid)) / 1073741824.0 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relkind IN ('r', 'm')), + (SELECT CASE WHEN bool_and(c.reltuples < 0) THEN NULL ELSE SUM(GREATEST(c.reltuples, 0))::bigint END + FROM pg_partition_tree(to_regclass(format('%I.%I', COALESCE(@schema, current_schema()), @table))) t + JOIN pg_class c ON c.oid = t.relid + WHERE t.isleaf) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = auditMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs new file mode 100644 index 0000000000..745dfb02b9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +// The query and the default schema must stay identical to the primary instance's provider, or the +// two sides hash different values for one database and DatabaseSharing never matches. +class PostgreSqlStorageIdentityProvider(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The cluster's initialisation identifier rather than an address: every connection to + // the same cluster sees the same value however the host is spelled or proxied. + // current_schema() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT system_identifier::text, current_database(), current_schema() FROM pg_control_system()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs new file mode 100644 index 0000000000..f107a47b8e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs @@ -0,0 +1,183 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using System.Data.Common; +using System.Globalization; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerDatabaseHostingProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IDatabaseHostingProbe +{ + public string StorageName => "SQLServer"; + + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server + // only and comes back null on Azure SQL Database, Managed Instance and Synapse. + command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0)) + { + return HostingFromConnectionString(); + } + + var engineEdition = Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture); + + var rds = !reader.IsDBNull(2); + + return new DatabaseHosting( + HostingFor(engineEdition, rds, ConfiguredHost), + MajorVersion(reader), + DatabaseHostingSource.Probe, + EditionFamily(engineEdition), + ServiceObjectiveTier(reader.IsDBNull(3) ? null : reader.GetString(3))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not ask SQL Server how it is hosted, falling back to the connection string"); + + return HostingFromConnectionString(); + } + } + + /// + /// Maps an EngineEdition to a hosting classification. The Azure editions identify their service + /// outright. The ordinary editions say only that this is a regular SQL Server, which still + /// leaves RDS and Cloud SQL in play, so the host name decides before self-hosted is concluded. + /// Synapse and Fabric are not mapped: ServiceControl does not run on them, and an edition we do + /// not recognise is not evidence of an ordinary SQL Server, so it falls to the host name. + /// + internal static string HostingFor(int engineEdition, bool rds, string? host) => engineEdition switch + { + AzureSqlDatabase => "AzureSql", + AzureSqlManagedInstance => "AzureSqlManagedInstance", + AzureSqlEdge => "AzureSqlEdge", + PersonalOrDesktop or Standard or Enterprise or Express => rds ? "AwsRds" : ManagedOrSelfHosted(host), + _ => DatabaseHostClassifier.Classify(host) + }; + + /// + /// Developer and Evaluation run the Enterprise engine and report Enterprise, so the report never + /// says which licence an instance runs under. The managed Azure services have no edition. + /// + internal static string EditionFamily(int engineEdition) => engineEdition switch + { + Standard => "Standard", + Enterprise => "Enterprise", + Express => "Express", + AzureSqlDatabase or AzureSqlManagedInstance => DatabaseHosting.NotApplicable, + _ => "Other" + }; + + /// + /// Keeps only the tier. The full service objective also encodes the hardware series, and a + /// DC-series name suggests Always Encrypted with secure enclaves. + /// + internal static string ServiceObjectiveTier(string? serviceObjective) + { + if (string.IsNullOrWhiteSpace(serviceObjective)) + { + return DatabaseHosting.NotApplicable; + } + + var objective = serviceObjective.Trim().ToUpperInvariant(); + + return objective switch + { + "BASIC" => "Basic", + "ELASTICPOOL" => "ElasticPool", + _ when objective.StartsWith("GP_", StringComparison.Ordinal) => "GeneralPurpose", + _ when objective.StartsWith("BC_", StringComparison.Ordinal) => "BusinessCritical", + _ when objective.StartsWith("HS_", StringComparison.Ordinal) => "Hyperscale", + _ when IsDtuObjective(objective, 'S') => "Standard", + _ when IsDtuObjective(objective, 'P') => "Premium", + _ => "Other" + }; + } + + static bool IsDtuObjective(string objective, char tier) => + objective.Length > 1 && objective[0] == tier && objective.AsSpan(1).IndexOfAnyExceptInRange('0', '9') < 0; + + static string ManagedOrSelfHosted(string? host) + { + var classified = DatabaseHostClassifier.Classify(host); + + return classified == DatabaseHostClassifier.Unknown ? DatabaseHostClassifier.SelfHosted : classified; + } + + static string MajorVersion(DbDataReader reader) + { + if (reader.IsDBNull(1)) + { + return DatabaseHostClassifier.Unknown; + } + + var productVersion = Convert.ToString(reader.GetValue(1), CultureInfo.InvariantCulture); + + if (string.IsNullOrWhiteSpace(productVersion)) + { + return DatabaseHostClassifier.Unknown; + } + + var major = productVersion.Split('.')[0]; + + return string.IsNullOrWhiteSpace(major) ? DatabaseHostClassifier.Unknown : major; + } + + DatabaseHosting HostingFromConnectionString() + { + var host = ConfiguredHost; + + return host is null + ? DatabaseHosting.Unclassified with { ServerEdition = DatabaseHostClassifier.Unknown, ServiceObjective = DatabaseHostClassifier.Unknown } + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString, DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown); + } + + string? ConfiguredHost + { + get + { + try + { + return new SqlConnectionStringBuilder(settings.ConnectionString).DataSource; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the configured SQL Server host"); + + return null; + } + } + } + + const int ProbeTimeoutSeconds = 5; + + // https://learn.microsoft.com/en-us/sql/t-sql/functions/serverproperty-transact-sql + const int PersonalOrDesktop = 1; + const int Standard = 2; + const int Enterprise = 3; + const int Express = 4; + const int AzureSqlDatabase = 5; + const int AzureSqlManagedInstance = 8; + const int AzureSqlEdge = 9; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index b7db7a8bbb..80b8be71b1 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -56,4 +56,11 @@ protected override void AddQueryServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); } + + protected override void AddEnvironmentProbes(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } } diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs new file mode 100644 index 0000000000..3456371456 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -0,0 +1,98 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using System.Data; +using System.Globalization; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerStorageFootprintProbe(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + for (var attempt = 1; ; attempt++) + { + try + { + return await Read(cancellationToken); + } + catch (SqlException e) when (attempt < MaxAttempts && e.Number is DeadlockVictim or ScanLostItsPlace) + { + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + + return null; + } + } + } + + async Task Read(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var auditMessagesTable = dbContext.Model.FindEntityType(typeof(AuditMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. + // Catalog reads under READ COMMITTED deadlock with concurrent DDL. Under READ UNCOMMITTED a scan can + // instead lose its place when pages move, so Probe retries both errors. + command.CommandText = """ + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.partitions p + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = auditMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + + const int ProbeTimeoutSeconds = 30; + const int MaxAttempts = 3; + const int DeadlockVictim = 1205; + const int ScanLostItsPlace = 601; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs new file mode 100644 index 0000000000..47cfc5f4bf --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +// The query and the default schema must stay identical to the primary instance's provider, or the +// two sides hash different values for one database and DatabaseSharing never matches. +class SqlServerStorageIdentityProvider(EFPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // SCHEMA_NAME() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME(), SCHEMA_NAME()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs index dea2331e99..90cb94678b 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs @@ -19,6 +19,8 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + AddEnvironmentProbes(services); services.AddSingleton(); services.AddHostedService(); @@ -53,6 +55,8 @@ void AddCommon(IServiceCollection services) protected abstract void AddQueryServices(IServiceCollection services); + protected abstract void AddEnvironmentProbes(IServiceCollection services); + protected virtual void AddCustomChecks(EndpointConfiguration endpointConfiguration) { } diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs new file mode 100644 index 0000000000..73b2728737 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using System.Globalization; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using static ServiceControl.Audit.Persistence.EnvironmentDatum; + +class EFEnvironmentDataProvider(IDatabaseHostingProbe hostingProbe, IStorageFootprintProbe footprintProbe, IFailedAuditStorage failedAuditStorage) : IEnvironmentDataProvider +{ + public IEnumerable GetData() + { + // The hosting keys share one probe, so the database is asked once per request and they + // stand or fall together. The footprint keys share theirs for the same reason. + Task? probe = null; + Task Hosting(CancellationToken cancellationToken) => probe ??= hostingProbe.Probe(cancellationToken); + + Task? footprint = null; + Task Footprint(CancellationToken cancellationToken) => footprint ??= footprintProbe.Probe(cancellationToken); + + return + [ + Value("Storage.Type", () => hostingProbe.StorageName), + Deferred("Storage.Hosting", async cancellationToken => (await Hosting(cancellationToken)).Hosting), + Deferred("Storage.ServerVersion", async cancellationToken => (await Hosting(cancellationToken)).ServerVersion), + Deferred("Storage.HostingSource", async cancellationToken => (await Hosting(cancellationToken)).Source), + Deferred("Storage.ServerEdition", async cancellationToken => (await Hosting(cancellationToken)).ServerEdition), + Deferred("Storage.ServiceObjective", async cancellationToken => (await Hosting(cancellationToken)).ServiceObjective), + Deferred("Storage.SizeGB", async cancellationToken => SizeGB((await Footprint(cancellationToken))?.SizeGB)), + Deferred("Storage.MessageCount", async cancellationToken => Count((await Footprint(cancellationToken))?.MessageCount)), + Deferred("Health.FailedImports", async cancellationToken => Count(await failedAuditStorage.GetFailedAuditsCount(cancellationToken))), + // The EF persisters always index bodies for full-text search; EnableFullTextSearchOnBodies is ignored. + Value("Storage.FullTextSearch", () => "Enabled") + ]; + } + + static string SizeGB(double? sizeGB) => + sizeGB is null ? DatabaseHostClassifier.Unknown : sizeGB.Value.ToString("F1", CultureInfo.InvariantCulture); + + static string Count(long? count) => + count is null ? DatabaseHostClassifier.Unknown : count.Value.ToString(CultureInfo.InvariantCulture); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs new file mode 100644 index 0000000000..b40f5de881 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using ServiceControl.Audit.Persistence; + +/// +/// How the database this instance stores its data in is hosted. Served to the primary instance for +/// its usage report, so every value is a fixed classification and never carries a host name, +/// database name or credential. Copied from the primary instance's EF persistence on purpose: the +/// audit persistence stack does not share projects with the primary's. +/// +public interface IDatabaseHostingProbe +{ + /// + /// The persistence name as it appears in the persistence manifest, for example SQLServer. + /// + string StorageName { get; } + + /// + /// Classifies the database host, asking the server itself where it can. Never throws; a server + /// that cannot be reached or does not answer is reported as unknown. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +/// One of AzureSql, AzureSqlManagedInstance, AzureSqlEdge, AzurePostgres, AwsRds, GoogleCloudSql, SelfHosted or Unknown. +/// The engine major version, or Unknown. +/// A value. +/// The SQL Server edition family: Express, Standard, Enterprise or Other. NotApplicable on the managed Azure services and on engines without editions. +/// The Azure SQL service tier: Basic, Standard, Premium, GeneralPurpose, BusinessCritical, Hyperscale, ElasticPool or Other. NotApplicable everywhere else. +public record DatabaseHosting(string Hosting, string ServerVersion, string Source, string ServerEdition = DatabaseHosting.NotApplicable, string ServiceObjective = DatabaseHosting.NotApplicable) +{ + public const string NotApplicable = "NotApplicable"; + + /// Nothing was available to classify the host with. A determination, not a failure. + public static readonly DatabaseHosting Unclassified = new(DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs new file mode 100644 index 0000000000..f1662c4917 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +/// +/// Measures what this instance's data occupies, from catalog statistics scoped to the configured +/// schema, never by counting rows. Served to the primary instance as sizes and counts only. +/// +public interface IStorageFootprintProbe +{ + /// + /// Null when the database could not be asked. A field is null when the catalog had no answer, + /// such as a row estimate on a table that was never analysed. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +public record StorageFootprint(double? SizeGB, long? MessageCount); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs new file mode 100644 index 0000000000..364d2bfa96 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.Audit.Persistence.RavenDB +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents.Operations; + using Raven.Client.ServerWide.Operations; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class RavenEnvironmentDataProvider(DatabaseConfiguration databaseConfiguration, IRavenDocumentStoreProvider documentStoreProvider, IFailedAuditStorage failedAuditStorage) : IEnvironmentDataProvider + { + public IEnumerable GetData() => + [ + Value("Storage.Type", () => "RavenDB"), + Value("Storage.RavenServer", () => databaseConfiguration.ServerConfiguration.UseEmbeddedServer ? "Embedded" : "External"), + Value("Storage.Hosting", () => Hosting().Hosting), + Deferred("Storage.ServerVersion", ServerVersion), + Value("Storage.HostingSource", () => Hosting().Source), + Value("Storage.ServerEdition", () => "NotApplicable"), + Value("Storage.ServiceObjective", () => "NotApplicable"), + Deferred("Storage.SizeGB", SizeGB), + Deferred("Storage.MessageCount", ProcessedMessageCount), + Deferred("Health.FailedImports", async cancellationToken => (await failedAuditStorage.GetFailedAuditsCount(cancellationToken)).ToString(CultureInfo.InvariantCulture)), + Value("Storage.FullTextSearch", () => databaseConfiguration.EnableFullTextSearch ? "Enabled" : "Disabled") + ]; + + async ValueTask SizeGB(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.ForDatabase(databaseConfiguration.Name).SendAsync(new GetStatisticsOperation(), cancellationToken); + + return (statistics.SizeOnDisk.SizeInBytes / BytesPerGigabyte).ToString("F1", CultureInfo.InvariantCulture); + } + + async ValueTask ProcessedMessageCount(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.ForDatabase(databaseConfiguration.Name).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("ProcessedMessages", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + + (string Hosting, string Source) Hosting() + { + var serverConfiguration = databaseConfiguration.ServerConfiguration; + + // An embedded server runs in this process, so there is nothing to infer. + if (serverConfiguration.UseEmbeddedServer) + { + return (DatabaseHostClassifier.SelfHosted, DatabaseHostingSource.Configuration); + } + + return Uri.TryCreate(serverConfiguration.ConnectionString, UriKind.Absolute, out var url) + ? (DatabaseHostClassifier.Classify(url.Host), DatabaseHostingSource.ConnectionString) + : (DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); + } + + async ValueTask ServerVersion(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var buildNumber = await documentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + return buildNumber.ProductVersion ?? DatabaseHostClassifier.Unknown; + } + + const double BytesPerGigabyte = 1024d * 1024 * 1024; + } +} diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs index a02107642e..553014885a 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs @@ -30,6 +30,8 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); } diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs new file mode 100644 index 0000000000..7f1003e419 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Audit.Persistence.RavenDB +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + class RavenStorageIdentityProvider(DatabaseConfiguration databaseConfiguration) : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + var serverConfiguration = databaseConfiguration.ServerConfiguration; + var configuredServer = serverConfiguration.UseEmbeddedServer ? serverConfiguration.ServerUrl : serverConfiguration.ConnectionString; + + return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), databaseConfiguration.Name, null)); + } + + // A loopback address names a different server on every machine, embedded servers included. + internal static string NormalizeServer(string configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) + ? $"{(url.IsLoopback ? Environment.MachineName : url.Host)}:{url.Port}" + : configuredServer ?? ""; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs new file mode 100644 index 0000000000..e14534561d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PostgreSqlEnvironmentDataTests.cs @@ -0,0 +1,104 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; + using Npgsql; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.DbContexts; + using ServiceControl.Audit.Persistence.EFCore.Entities; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + + class PostgreSqlEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Reports_no_edition_or_service_objective() + { + var hosting = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } + + [Test] + public async Task Counts_messages_across_the_partitions_once_analysed() + { + await Ingest(MakeMessage(), MakeMessage()); + configuration.TimeProvider.Advance(TimeSpan.FromDays(1)); + await Ingest(MakeMessage()); + + var parentRowEstimate = await WithDbContext(async (dbContext, token) => + { + var table = $"{dbContext.Schema}.{dbContext.Model.FindEntityType(typeof(AuditMessageEntity))!.GetTableName()}"; + var leaves = await Query(dbContext, $"SELECT relid::regclass::text FROM pg_partition_tree('{table}'::regclass) WHERE isleaf", token); + + foreach (var leaf in leaves) + { + await Query(dbContext, $"ANALYZE {leaf}", token); + } + + return (await Query(dbContext, $"SELECT reltuples::text FROM pg_class WHERE oid = '{table}'::regclass", token)).Single(); + }); + + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(double.Parse(parentRowEstimate, CultureInfo.InvariantCulture), Is.LessThan(0), "The partitioned parent carries no row estimate of its own"); + Assert.That(footprint.MessageCount, Is.EqualTo(3)); + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + } + } + + [Test] + public async Task Measures_and_identifies_the_search_path_schema_when_none_is_configured() + { + var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = + new NpgsqlConnectionStringBuilder(configuration.ConnectionString) { SearchPath = configuration.Schema }.ConnectionString; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(configuration.TimeProvider); + configuration.CreateConfiguration().Create(settings).AddPersistence(services); + await using var unconfigured = services.BuildServiceProvider(); + + var identity = await unconfigured.GetRequiredService().GetIdentity(TestTimeoutCancellationToken); + var footprint = await unconfigured.GetRequiredService().Probe(TestTimeoutCancellationToken); + var expected = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(identity?.Schema, Is.EqualTo(configuration.Schema)); + Assert.That(footprint?.SizeGB, Is.Not.Null.And.EqualTo(expected?.SizeGB)); + } + } + + static async Task> Query(AuditDbContext dbContext, string sql, CancellationToken cancellationToken) + { + await dbContext.Database.OpenConnectionAsync(cancellationToken); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = sql; + + var values = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + while (await reader.ReadAsync(cancellationToken)) + { + values.Add(reader.GetString(0)); + } + + return values; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs new file mode 100644 index 0000000000..4ca76d809a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.RavenDB; + + [TestFixture] + class RavenStorageIdentityProviderTests + { + [TestCase("http://localhost:44445")] + [TestCase("http://127.0.0.1:44445")] + [TestCase("http://[::1]:44445")] + public void A_loopback_server_is_this_machine(string url) => + Assert.That(RavenStorageIdentityProvider.NormalizeServer(url), Is.EqualTo($"{Environment.MachineName}:44445")); + + [Test] + public void A_remote_server_is_its_host_and_port() => + Assert.That(RavenStorageIdentityProvider.NormalizeServer("https://raven.example.com:8080"), Is.EqualTo("raven.example.com:8080")); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs index 85143e7ba9..226175c015 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs @@ -27,7 +27,7 @@ class PersistenceTestsConfiguration : EFPersistenceTestsConfiguration protected override IAuditMessagesViewDataStore DecorateMessagesViewStore(IAuditMessagesViewDataStore store) => new SearchesWaitForFullTextIndex(store, WaitForFullTextIndex); - async Task WaitForFullTextIndex(CancellationToken cancellationToken) + public async Task WaitForFullTextIndex(CancellationToken cancellationToken = default) { await using var connection = new SqlConnection(ConnectionString); await connection.OpenAsync(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs new file mode 100644 index 0000000000..1ce23546f8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/SqlServerEnvironmentDataTests.cs @@ -0,0 +1,133 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging.Abstractions; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.SqlServer; + + class SqlServerEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Reports_the_edition_family() + { + var hosting = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + var managed = hosting.Hosting is "AzureSql" or "AzureSqlManagedInstance"; + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, managed ? Is.EqualTo(DatabaseHosting.NotApplicable) : Is.AnyOf("Express", "Standard", "Enterprise")); + Assert.That(hosting.ServiceObjective, hosting.Hosting == "AzureSql" ? Is.AnyOf(ServiceTiers) : Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } + + [Test] + public async Task Counts_the_audit_messages_it_stores() + { + await Ingest(MakeMessage(), MakeMessage()); + + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.MessageCount, Is.EqualTo(2)); + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + } + } + + [TestCase(2, "Standard")] + [TestCase(3, "Enterprise")] + [TestCase(4, "Express")] + [TestCase(5, "NotApplicable")] + [TestCase(8, "NotApplicable")] + [TestCase(1, "Other")] + [TestCase(9, "Other")] + [TestCase(12, "Other")] + public void Should_report_the_edition_family(int engineEdition, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.EditionFamily(engineEdition), Is.EqualTo(expected)); + + [TestCase(null, "NotApplicable")] + [TestCase("Basic", "Basic")] + [TestCase("S0", "Standard")] + [TestCase("S12", "Standard")] + [TestCase("P15", "Premium")] + [TestCase("GP_Gen5_2", "GeneralPurpose")] + [TestCase("GP_S_Gen5_1", "GeneralPurpose")] + [TestCase("GP_DC_8", "GeneralPurpose")] + [TestCase("BC_Gen5_8", "BusinessCritical")] + [TestCase("HS_Gen5_4", "Hyperscale")] + [TestCase("ElasticPool", "ElasticPool")] + [TestCase("DW100c", "Other")] + [TestCase("System2", "Other")] + public void Should_report_only_the_service_objective_tier(string serviceObjective, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.ServiceObjectiveTier(serviceObjective), Is.EqualTo(expected)); + + [TestCase("Server=sc.database.windows.net;Database=sc")] + [TestCase("not a connection string")] + public async Task Should_report_edition_and_service_objective_as_unknown_when_the_server_cannot_be_asked(string connectionString) + { + var settings = new EFPersisterSettings { ConnectionString = connectionString, AuditRetentionPeriod = TimeSpan.FromDays(7), MaxBodySizeToStore = 1024 }; + var probe = new SqlServerDatabaseHostingProbe(settings, new UnreachableScopeFactory(), NullLogger.Instance); + + var hosting = await probe.Probe(TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo("Unknown")); + Assert.That(hosting.ServiceObjective, Is.EqualTo("Unknown")); + } + } + + [Test] + public async Task Counts_the_full_text_index_in_the_size() + { + await Ingest(MakeMessage(), MakeMessage()); + await configuration.WaitForFullTextIndex(TestTimeoutCancellationToken); + + var tablesGB = await SizeGB(""" + SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) + """); + var footprint = await ServiceProvider.GetRequiredService().Probe(TestTimeoutCancellationToken); + var fullTextGB = await SizeGB(""" + SELECT COALESCE(SUM(a.total_pages), 0) * 8.0 / 1048576 + FROM sys.internal_tables it + JOIN sys.tables t ON t.object_id = it.parent_id + JOIN sys.partitions p ON p.object_id = it.object_id + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE t.schema_id = SCHEMA_ID(@schema) AND it.internal_type_desc LIKE 'FULLTEXT%' + """); + + Assert.That(fullTextGB, Is.GreaterThan(0), "The audit messages table always has a full-text index"); + Assert.That(footprint?.SizeGB, Is.GreaterThan(tablesGB), "The size counts the full-text index on top of the tables"); + } + + Task SizeGB(string sql) => WithDbContext(async (dbContext, token) => + { + await dbContext.Database.OpenConnectionAsync(token); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = "SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; " + sql; + var schema = command.CreateParameter(); + schema.ParameterName = "@schema"; + schema.Value = dbContext.Schema; + command.Parameters.Add(schema); + + return Convert.ToDouble(await command.ExecuteScalarAsync(token), System.Globalization.CultureInfo.InvariantCulture); + }); + + static readonly object[] ServiceTiers = ["Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool"]; + + sealed class UnreachableScopeFactory : IServiceScopeFactory + { + public IServiceScope CreateScope() => throw new InvalidOperationException("The database cannot be reached"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs new file mode 100644 index 0000000000..27f8d907f7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFEnvironmentDataTests.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Collections.Generic; + using System.Globalization; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using NUnit.Framework; + + class EFEnvironmentDataTests : EFPersistenceTestFixture + { + [Test] + public async Task Serves_storage_facts_for_an_empty_schema() + { + var data = new Dictionary(); + + foreach (var datum in ServiceProvider.GetRequiredService().GetData()) + { + data[datum.Key] = await datum.ReadValue(TestTimeoutCancellationToken); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Storage.Type").WithValue(PersisterName)); + Assert.That(data, Does.ContainKey("Storage.HostingSource").WithValue("Probe")); + Assert.That(int.TryParse(data["Storage.ServerVersion"], NumberStyles.Integer, CultureInfo.InvariantCulture, out _), Is.True, data["Storage.ServerVersion"]); + Assert.That(data["Storage.ServerEdition"], Is.AnyOf("Express", "Standard", "Enterprise", "NotApplicable")); + Assert.That(data["Storage.ServiceObjective"], Is.AnyOf("NotApplicable", "Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool")); + Assert.That(double.Parse(data["Storage.SizeGB"], CultureInfo.InvariantCulture), Is.GreaterThanOrEqualTo(0)); + Assert.That(data["Storage.MessageCount"], Is.AnyOf("0", "Unknown")); + Assert.That(data, Does.ContainKey("Health.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.FullTextSearch").WithValue("Enabled")); + } + } + + [Test] + public async Task Identifies_the_database_and_schema_it_writes_to() + { + var identity = await ServiceProvider.GetRequiredService().GetIdentity(TestTimeoutCancellationToken); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo(PersisterName)); + Assert.That(identity.Server, Is.Not.Empty); + Assert.That(identity.Database, Is.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(configuration.Schema)); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs new file mode 100644 index 0000000000..3389a5d07f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostClassifier.cs @@ -0,0 +1,77 @@ +#nullable enable +namespace ServiceControl.Audit.Persistence +{ + using System; + using System.Linq; + + /// + /// Classifies a database host name into a managed-service category. Only the category is ever + /// reported; the host name itself never leaves the process. Copied from the primary instance's + /// ServiceControl.Persistence on purpose: the audit persistence stack does not share projects + /// with the primary's. + /// + /// + /// This is the fallback for servers that cannot identify their own hosting. Where the engine can be + /// asked directly, as SQL Server can through EngineEdition, that answer wins over this one. + /// + public static class DatabaseHostClassifier + { + public const string Unknown = "Unknown"; + public const string SelfHosted = "SelfHosted"; + + /// + /// A host name is only ever evidence of a managed service or of a server on this machine. + /// Anything else, a private DNS name above all, is unknown rather than self-hosted: a customer + /// who fronts a managed database with their own DNS must not be counted as self-hosting it. + /// + public static string Classify(string? host) + { + if (string.IsNullOrWhiteSpace(host)) + { + return Unknown; + } + + var normalized = host.Trim().ToLowerInvariant(); + + foreach (var (suffix, hosting) in ManagedHostSuffixes) + { + if (normalized.EndsWith(suffix, StringComparison.Ordinal) || normalized.Contains($"{suffix},", StringComparison.Ordinal)) + { + return hosting; + } + } + + // Cloud SQL is reached either through a host name or through a unix socket directory named + // after the instance, so neither end of the value is a reliable place to look. + if (normalized.Contains("cloudsql", StringComparison.Ordinal)) + { + return "GoogleCloudSql"; + } + + return IsThisMachine(normalized) ? SelfHosted : Unknown; + } + + static bool IsThisMachine(string normalized) => + LocalHosts.Contains(normalized) || + normalized.StartsWith("(localdb)", StringComparison.Ordinal) || + normalized.StartsWith("np:", StringComparison.Ordinal) || + normalized.StartsWith("lpc:", StringComparison.Ordinal) || + normalized.StartsWith('/'); // a unix socket directory, which only a server on this machine listens on + + static readonly string[] LocalHosts = ["localhost", "127.0.0.1", "::1", "[::1]", ".", "(local)"]; + + // Ordered longest-suffix-first so that the more specific Azure services win over the shared + // .database.azure.com suffix. + static readonly (string Suffix, string Hosting)[] ManagedHostSuffixes = + [ + (".postgres.database.azure.com", "AzurePostgres"), + (".mysql.database.azure.com", "AzureMySql"), + (".database.windows.net", "AzureSql"), + (".database.azure.com", "AzureSql"), + (".rds.amazonaws.com", "AwsRds"), + (".ravendb.cloud", "RavenCloud"), + (".development.run", "RavenCloud"), + (".gcp.cloud", "GoogleCloudSql") + ]; + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs new file mode 100644 index 0000000000..fdc0adcddc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/DatabaseHostingSource.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence +{ + /// + /// Where a reported database hosting classification came from, so that a guess can be told apart + /// from an answer when the reports are analysed. + /// + public static class DatabaseHostingSource + { + /// The server itself said so, and is therefore authoritative. + public const string Probe = "Probe"; + + /// The configuration says so outright, with nothing left to infer. + public const string Configuration = "Configuration"; + + /// Inferred from the configured host name because the server could not be asked. + public const string ConnectionString = "ConnectionString"; + + /// Nothing was available to classify. + public const string None = "None"; + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs new file mode 100644 index 0000000000..ed9697f588 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/EnvironmentDatum.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Audit.Persistence +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// One key of the environment data served to the primary instance, together with how to read + /// its value. + /// + /// + /// The value is deferred rather than supplied so that reading it can be isolated. A datum that + /// cannot be read costs only its own key, never a sibling's, and providers therefore carry no + /// error handling of their own. + /// + public sealed record EnvironmentDatum(string Key, Func> ReadValue) + { + /// + /// Served in place of a value whose read threw. Deliberately not a word that could pass for + /// a state the instance is legitimately in: it always means the read failed, never that the + /// thing being described is absent or switched off. + /// + public const string ReadFailed = "ReadFailed"; + + /// + /// A value that is already at hand, such as one read from configuration. Still deferred, so + /// that nothing is evaluated while a provider is listing what it offers. + /// + public static EnvironmentDatum Value(string key, Func readValue) => + new(key, _ => new ValueTask(readValue())); + + /// + /// A value that has to be fetched, such as one read from storage or from the database itself. + /// + public static EnvironmentDatum Deferred(string key, Func> readValue) => + new(key, readValue); + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs new file mode 100644 index 0000000000..f55677c0fb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/IEnvironmentDataProvider.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence +{ + using System.Collections.Generic; + + /// + /// Provides environment data that the primary instance polls over the environment endpoint and + /// folds into its usage report. + /// + public interface IEnvironmentDataProvider + { + IEnumerable GetData(); + } +} diff --git a/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs new file mode 100644 index 0000000000..dc38b24088 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/EnvironmentData/IStorageIdentityProvider.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Audit.Persistence +{ + using System.Threading; + using System.Threading.Tasks; + + /// + /// Identifies the storage this instance writes to, so that the primary instance can tell + /// whether it shares a database or a database server with this one. The raw values never leave + /// the process; the anonymous environment endpoint serves unsalted hashes of them. + /// + public interface IStorageIdentityProvider + { + ValueTask GetIdentity(CancellationToken cancellationToken = default); + } + + public record StorageIdentity(string Engine, string Server, string Database, string Schema); +} diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 3e218d68a5..97509ec622 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -7,6 +7,7 @@ GET /endpoints/{endpoint}/messages => ServiceControl.Audit.Auditing.MessagesView GET /endpoints/{endpoint}/messages/search => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Search(PagingInfo pagingInfo, SortInfo sortInfo, String endpoint, String q, CancellationToken cancellationToken) GET /endpoints/{endpoint}/messages/search/{keyword} => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:SearchByKeyword(PagingInfo pagingInfo, SortInfo sortInfo, String endpoint, String keyword, CancellationToken cancellationToken) GET /endpoints/known => ServiceControl.Audit.Monitoring.KnownEndpointsController:GetAll(PagingInfo pagingInfo) +GET /environment => ServiceControl.Audit.Infrastructure.WebApi.EnvironmentController:Environment(CancellationToken cancellationToken) GET /instance-info => ServiceControl.Audit.Infrastructure.WebApi.RootController:Config() GET /messages => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, Boolean includeSystemMessages, CancellationToken cancellationToken) GET /messages/{id}/body => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Get(String id, CancellationToken cancellationToken) diff --git a/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs b/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs index 1357d7705b..745565a561 100644 --- a/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs +++ b/src/ServiceControl.Audit.UnitTests/Auditing/Metrics/IngestionMetricsTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.UnitTests.Auditing.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; @@ -7,6 +8,7 @@ namespace ServiceControl.Audit.UnitTests.Auditing.Metrics; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Audit.Auditing.Metrics; +using ServiceControl.Infrastructure.Ingestion; /// /// Instrument names are what dashboards and alerts are built on, so they are a published contract @@ -39,7 +41,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() listener.Start(); - _ = new IngestionMetrics(MeterFactory); + _ = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); Assert.That(published.Order(), Is.EqualTo(new[] { @@ -53,7 +55,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() [Test] public void Concurrent_batch_failures_are_all_counted() { - var metrics = new IngestionMetrics(MeterFactory); + var metrics = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); const int failedBatches = 1000; diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs new file mode 100644 index 0000000000..7d2d94eccb --- /dev/null +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/EnvironmentControllerTests.cs @@ -0,0 +1,151 @@ +namespace ServiceControl.Audit.UnitTests.Infrastructure +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Logging.Abstractions; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure.WebApi; + using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure; + + [TestFixture] + class EnvironmentControllerTests + { + [Test] + public async Task Merges_the_data_of_every_provider() + { + var controller = new EnvironmentController( + [ + new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB")), + new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) + ], [], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Host.ProcessorCount"] = "8" + })); + } + + [Test] + public async Task A_datum_that_cannot_be_read_costs_only_its_own_key() + { + var controller = new EnvironmentController( + [ + new Provider( + EnvironmentDatum.Value("Storage.Type", () => "RavenDB"), + EnvironmentDatum.Deferred("Storage.ServerVersion", _ => throw new InvalidOperationException("server down"))) + ], [], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Storage.Type"] = "RavenDB", + ["Storage.ServerVersion"] = EnvironmentDatum.ReadFailed + })); + } + + [Test] + public async Task A_provider_that_cannot_list_its_data_costs_only_its_own_keys() + { + var controller = new EnvironmentController( + [ + new ThrowingProvider(), + new Provider(EnvironmentDatum.Value("Host.ProcessorCount", () => "8")) + ], [], NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + Assert.That(response.EnvironmentData, Is.EqualTo(new Dictionary + { + ["Host.ProcessorCount"] = "8" + })); + } + + [Test] + public void Cancellation_propagates() + { + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + var controller = new EnvironmentController( + [ + new Provider(EnvironmentDatum.Deferred("Storage.ServerVersion", cancellationToken => + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask("unreached"); + })) + ], [], NullLogger.Instance); + + Assert.ThrowsAsync(() => controller.Environment(cancellationTokenSource.Token)); + } + + [Test] + public async Task Serves_the_hashed_storage_identity() + { + var controller = new EnvironmentController( + [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], + [new IdentityProvider(new StorageIdentity("RavenDB", "http://server:8080", "audit", null))], + NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + using (Assert.EnterMultipleScope()) + { + Assert.That(response.StorageIdentity.Engine, Is.EqualTo("RavenDB")); + Assert.That(response.StorageIdentity.ServerHash, Is.EqualTo(IdentityHash.Compute("http://server:8080"))); + Assert.That(response.StorageIdentity.DatabaseHash, Is.EqualTo(IdentityHash.Compute("audit"))); + Assert.That(response.StorageIdentity.SchemaHash, Is.Null); + Assert.That(response.MachineNameHash, Is.EqualTo(MachineIdentity.Hash)); + Assert.That(response.EnvironmentData.Keys, Has.None.Contains("Hash")); + } + } + + [Test] + public async Task A_storage_identity_that_cannot_be_read_is_not_served() + { + var controller = new EnvironmentController( + [new Provider(EnvironmentDatum.Value("Storage.Type", () => "RavenDB"))], + [new ThrowingIdentityProvider()], + NullLogger.Instance); + + var result = await controller.Environment(); + + var response = (EnvironmentController.EnvironmentDataResponse)result.Value; + using (Assert.EnterMultipleScope()) + { + Assert.That(response.StorageIdentity, Is.Null); + Assert.That(response.EnvironmentData, Does.ContainKey("Storage.Type")); + } + } + + class Provider(params EnvironmentDatum[] data) : IEnvironmentDataProvider + { + public IEnumerable GetData() => data; + } + + class IdentityProvider(StorageIdentity identity) : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) => new(identity); + } + + class ThrowingIdentityProvider : IStorageIdentityProvider + { + public ValueTask GetIdentity(CancellationToken cancellationToken = default) => throw new InvalidOperationException("storage down"); + } + + class ThrowingProvider : IEnvironmentDataProvider + { + public IEnumerable GetData() => throw new InvalidOperationException("cannot list"); + } + } +} diff --git a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs index cdd6788eab..68009f7919 100644 --- a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs +++ b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.Auditing.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; @@ -8,6 +9,7 @@ using NServiceBus; using NServiceBus.Transport; using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Infrastructure.Ingestion.Metrics; public class IngestionMetrics @@ -17,8 +19,10 @@ public class IngestionMetrics public static readonly string BatchDurationInstrumentName = $"{InstrumentPrefix}.batch_duration_seconds"; public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration_seconds"; - public IngestionMetrics(IMeterFactory meterFactory) + public IngestionMetrics(IMeterFactory meterFactory, IngestionCounters counters) { + this.counters = counters; + var meter = meterFactory.Create(MeterName, MeterVersion); batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "seconds", "Message batch processing duration in seconds"); @@ -27,11 +31,32 @@ public IngestionMetrics(IMeterFactory meterFactory) failureCounter = meter.CreateCounter($"{InstrumentPrefix}.failures_total", description: "Audit ingestion failure count"); } - public MessageMetrics BeginIngestion(MessageContext messageContext) => new(GetMessageTags(messageContext.Headers), ingestionDuration); + public MessageMetrics BeginIngestion(MessageContext messageContext) + { + RecordLag(messageContext.Headers); + + return new(GetMessageTags(messageContext.Headers), ingestionDuration); + } public FailureMetrics BeginErrorHandling(ErrorContext errorContext) => new(GetMessageTags(errorContext.Headers), failureCounter); - public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome); + public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome, counters.RecordBatch); + + void RecordLag(Dictionary headers) + { + if (!headers.TryGetValue(Headers.ProcessingEnded, out var processingEnded)) + { + return; + } + + try + { + counters.RecordLag(DateTime.UtcNow - DateTimeOffsetHelper.ToDateTimeOffset(processingEnded).UtcDateTime); + } + catch (FormatException) + { + } + } static TagList GetMessageTags(Dictionary headers) { @@ -66,6 +91,7 @@ void RecordBatchOutcome(bool success) readonly Histogram batchDuration; readonly Histogram ingestionDuration; readonly Counter failureCounter; + readonly IngestionCounters counters; const string MeterVersion = "0.1.0"; const string InstrumentPrefix = "sc.audit.ingestion"; diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index ccd2c635cc..b7925ee4db 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -51,6 +51,9 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, services.AddSingleton(configuration); services.AddSingleton(settings); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); // required by the ingestion custom check which is auto-loaded diff --git a/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs b/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs new file mode 100644 index 0000000000..31032d0773 --- /dev/null +++ b/src/ServiceControl.Audit/HostEnvironmentDataProvider.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Audit +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Runtime.InteropServices; + using Microsoft.Extensions.Hosting.WindowsServices; + using ServiceControl.Audit.Persistence; + using ServiceControl.Configuration; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class HostEnvironmentDataProvider : IEnvironmentDataProvider + { + public IEnumerable GetData() => + [ + Value("Host.Model", HostModel), + Value("Host.Orchestrator", Orchestrator), + Value("Host.OSPlatform", OSPlatformName), + Value("Host.OSVersion", () => $"{Environment.OSVersion.Version.Major}.{Environment.OSVersion.Version.Minor}"), + Value("Host.Architecture", () => RuntimeInformation.ProcessArchitecture.ToString()), + Value("Host.RuntimeVersion", () => Environment.Version.ToString(3)), + Value("Host.ProcessorCount", () => Environment.ProcessorCount.ToString(CultureInfo.InvariantCulture)), + Value("Host.AvailableMemoryGB", AvailableMemoryGB) + ]; + + static string Orchestrator() => Environment.GetEnvironmentVariable(KubernetesServiceHostVariable) is not null ? "Kubernetes" : "None"; + + static string HostModel() + { + if (AppEnvironment.RunningInContainer) + { + return "Container"; + } + + return WindowsServiceHelpers.IsWindowsService() ? "WindowsService" : "Console"; + } + + static string OSPlatformName() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "Windows"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "Linux"; + } + + return RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "macOS" : "Unknown"; + } + + // TotalAvailableMemoryBytes honours the cgroup limit, so a container reports the memory it is + // limited to rather than the physical memory of the machine hosting it. + static string AvailableMemoryGB() + { + var totalAvailableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + + return totalAvailableMemoryBytes <= 0 + ? "Unknown" + : Math.Round(totalAvailableMemoryBytes / (double)BytesPerGigabyte, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); + } + + const string KubernetesServiceHostVariable = "KUBERNETES_SERVICE_HOST"; + const long BytesPerGigabyte = 1024L * 1024 * 1024; + } +} diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs new file mode 100644 index 0000000000..4016209d4e --- /dev/null +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/EnvironmentController.cs @@ -0,0 +1,119 @@ +namespace ServiceControl.Audit.Infrastructure.WebApi +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.AspNetCore.Authorization; + using Microsoft.AspNetCore.Mvc; + using System.Linq; + using Microsoft.Extensions.Logging; + using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure; + + // the /api/environment endpoint is polled by the primary instance for its usage report. This currently needs to be anonymous + [AllowAnonymous] + [ApiController] + [Route("api")] + public class EnvironmentController(IEnumerable providers, IEnumerable storageIdentityProviders, ILogger logger) : ControllerBase + { + [Route("environment")] + [HttpGet] + public async Task Environment(CancellationToken cancellationToken = default) + { + var environmentData = new Dictionary(); + + foreach (var provider in providers) + { + EnvironmentDatum[] data; + + try + { + data = [.. provider.GetData()]; + } + catch (Exception e) + { + logger.LogWarning(e, "Environment data provider {EnvironmentDataProvider} could not list what it offers, so none of its data is served", provider.GetType().Name); + continue; + } + + foreach (var datum in data) + { + try + { + environmentData[datum.Key] = await datum.ReadValue(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogWarning(e, "Environment datum {EnvironmentDatum} could not be read", datum.Key); + environmentData[datum.Key] = EnvironmentDatum.ReadFailed; + } + } + } + + return Ok(new EnvironmentDataResponse + { + EnvironmentData = environmentData, + MachineNameHash = MachineIdentity.Hash, + StorageIdentity = await ReadStorageIdentity(cancellationToken) + }); + } + + async Task ReadStorageIdentity(CancellationToken cancellationToken) + { + var provider = storageIdentityProviders.FirstOrDefault(); + + if (provider is null) + { + return null; + } + + try + { + var identity = await provider.GetIdentity(cancellationToken); + + if (identity is null) + { + return null; + } + + return new StorageIdentityResponse + { + Engine = identity.Engine, + ServerHash = IdentityHash.Compute(identity.Server), + DatabaseHash = IdentityHash.Compute(identity.Database), + SchemaHash = identity.Schema is null ? null : IdentityHash.Compute(identity.Schema) + }; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogWarning(e, "The storage identity could not be read, so it is not served"); + + return null; + } + } + + public class EnvironmentDataResponse + { + public Dictionary EnvironmentData { get; set; } + public string MachineNameHash { get; set; } + public StorageIdentityResponse StorageIdentity { get; set; } + } + + public class StorageIdentityResponse + { + public string Engine { get; set; } + public string ServerHash { get; set; } + public string DatabaseHash { get; set; } + public string SchemaHash { get; set; } + } + } +} diff --git a/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs b/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs new file mode 100644 index 0000000000..0561521771 --- /dev/null +++ b/src/ServiceControl.Audit/IngestionEnvironmentDataProvider.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit +{ + using System; + using System.Collections.Generic; + using System.Linq; + using ServiceControl.Audit.Persistence; + using ServiceControl.Infrastructure.Ingestion; + using static ServiceControl.Audit.Persistence.EnvironmentDatum; + + class IngestionEnvironmentDataProvider(IngestionCounters counters, TimeProvider timeProvider) : IEnvironmentDataProvider + { + public IEnumerable GetData() => + IngestionSummary.Describe(counters.GetSnapshot(), timeProvider.GetUtcNow().UtcDateTime, "Ingestion", "Health", includeStorage: false) + .Select(pair => Value(pair.Key, () => pair.Value)); + } +} diff --git a/src/ServiceControl.Infrastructure/IdentityHash.cs b/src/ServiceControl.Infrastructure/IdentityHash.cs new file mode 100644 index 0000000000..fb94bbae69 --- /dev/null +++ b/src/ServiceControl.Infrastructure/IdentityHash.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace ServiceControl.Infrastructure; + +using System; +using System.Security.Cryptography; +using System.Text; + +/// +/// Hashes an identity value so that two instances can compare what they are running on. The hash is +/// unsalted and the audit instance serves it on its anonymous environment endpoint, so anyone who +/// can reach that endpoint can confirm a guessed value. It never appears in a usage report; only the +/// comparison result does. +/// +public static class IdentityHash +{ + public static string Compute(string value) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value.Trim().ToLowerInvariant()))); +} diff --git a/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs new file mode 100644 index 0000000000..5fb9da7f6d --- /dev/null +++ b/src/ServiceControl.Infrastructure/Ingestion/IngestionCounters.cs @@ -0,0 +1,69 @@ +namespace ServiceControl.Infrastructure.Ingestion; + +using System; +using System.Threading; + +public class IngestionCounters(TimeProvider timeProvider) +{ + public DateTime ProcessStartUtc { get; } = timeProvider.GetUtcNow().UtcDateTime; + + public void RecordBatch(int completedSize, TimeSpan duration) + { + if (completedSize > 0) + { + Interlocked.Add(ref messages, completedSize); + } + + Interlocked.Add(ref busyTicks, duration.Ticks); + } + + public void RecordStorage(TimeSpan duration) => Interlocked.Add(ref storageTicks, duration.Ticks); + + public void RecordLag(TimeSpan lag) + { + Interlocked.Increment(ref lagKnown); + + if (lag > TimeSpan.FromMinutes(1)) + { + Interlocked.Increment(ref lagOverOneMinute); + } + + if (lag > TimeSpan.FromMinutes(10)) + { + Interlocked.Increment(ref lagOverTenMinutes); + } + + if (lag > TimeSpan.FromMinutes(60)) + { + Interlocked.Increment(ref lagOverSixtyMinutes); + } + } + + public IngestionCountersSnapshot GetSnapshot() => new( + ProcessStartUtc, + Interlocked.Read(ref messages), + TimeSpan.FromTicks(Interlocked.Read(ref busyTicks)).TotalSeconds, + TimeSpan.FromTicks(Interlocked.Read(ref storageTicks)).TotalSeconds, + Interlocked.Read(ref lagOverOneMinute), + Interlocked.Read(ref lagOverTenMinutes), + Interlocked.Read(ref lagOverSixtyMinutes), + Interlocked.Read(ref lagKnown)); + + long messages; + long busyTicks; + long storageTicks; + long lagOverOneMinute; + long lagOverTenMinutes; + long lagOverSixtyMinutes; + long lagKnown; +} + +public record IngestionCountersSnapshot( + DateTime ProcessStartUtc, + long Messages, + double BusySeconds, + double StorageSeconds, + long LagOverOneMinuteMessages, + long LagOverTenMinutesMessages, + long LagOverSixtyMinutesMessages, + long LagKnownMessages); diff --git a/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs b/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs new file mode 100644 index 0000000000..9d1eb7ba6a --- /dev/null +++ b/src/ServiceControl.Infrastructure/Ingestion/IngestionSummary.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Infrastructure.Ingestion; + +using System; +using System.Collections.Generic; +using System.Globalization; + +public static class IngestionSummary +{ + public static readonly TimeSpan MinimumUptime = TimeSpan.FromHours(1); + + public static IEnumerable> Describe(IngestionCountersSnapshot snapshot, DateTime nowUtc, string ingestionPrefix, string healthPrefix, bool includeStorage) + { + var uptime = nowUtc - snapshot.ProcessStartUtc; + + if (uptime < TimeSpan.Zero) + { + uptime = TimeSpan.Zero; + } + + yield return new($"{healthPrefix}.UptimeHours", ((long)uptime.TotalHours).ToString(CultureInfo.InvariantCulture)); + + if (uptime < MinimumUptime) + { + yield break; + } + + yield return new($"{ingestionPrefix}.AvgDailyMessages", Whole(snapshot.Messages / uptime.TotalDays)); + yield return new($"{ingestionPrefix}.BusyPercent", Whole(Math.Min(100, snapshot.BusySeconds / uptime.TotalSeconds * 100))); + + if (includeStorage && snapshot.Messages > 0) + { + yield return new($"{ingestionPrefix}.StorageMsPerMessage", Whole(snapshot.StorageSeconds / snapshot.Messages * 1000)); + } + + if (snapshot.LagKnownMessages > 0) + { + yield return new($"{healthPrefix}.LagOver1MinPercent", Percent(snapshot.LagOverOneMinuteMessages, snapshot.LagKnownMessages)); + yield return new($"{healthPrefix}.LagOver10MinPercent", Percent(snapshot.LagOverTenMinutesMessages, snapshot.LagKnownMessages)); + yield return new($"{healthPrefix}.LagOver60MinPercent", Percent(snapshot.LagOverSixtyMinutesMessages, snapshot.LagKnownMessages)); + } + } + + static string Percent(long part, long whole) => Whole(part / (double)whole * 100); + + static string Whole(double value) => Math.Round(value, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture); +} diff --git a/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs b/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs index 4ef08a1ed8..0b98dccaa2 100644 --- a/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs +++ b/src/ServiceControl.Infrastructure/Ingestion/Metrics/BatchMetrics.cs @@ -8,7 +8,7 @@ namespace ServiceControl.Infrastructure.Ingestion.Metrics; /// One batch write. Leaving the scope without calling is what records the /// batch as failed, so nothing has to be told about the exception that ended it. /// -public sealed class BatchMetrics(int maxBatchSize, Histogram batchDuration, Action recordOutcome) : IDisposable +public sealed class BatchMetrics(int maxBatchSize, Histogram batchDuration, Action recordOutcome, Action recordCompletion = null) : IDisposable { public void Complete(int batchSize) => completedSize = batchSize; @@ -17,6 +17,7 @@ public void Dispose() var succeeded = completedSize > 0; recordOutcome(succeeded); + recordCompletion?.Invoke(succeeded ? completedSize : 0, stopwatch.Elapsed); var result = succeeded ? completedSize == maxBatchSize ? "full" : "partial" diff --git a/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs b/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs index 16535614ac..94c7365c79 100644 --- a/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs +++ b/src/ServiceControl.Infrastructure/Ingestion/Metrics/DurationScope.cs @@ -7,9 +7,13 @@ namespace ServiceControl.Infrastructure.Ingestion.Metrics; /// /// How long the scope was open, in seconds, and nothing else. /// -public sealed class DurationScope(Histogram duration) : IDisposable +public sealed class DurationScope(Histogram duration, Action recordDuration = null) : IDisposable { - public void Dispose() => duration.Record(stopwatch.Elapsed.TotalSeconds); + public void Dispose() + { + duration.Record(stopwatch.Elapsed.TotalSeconds); + recordDuration?.Invoke(stopwatch.Elapsed); + } readonly Stopwatch stopwatch = Stopwatch.StartNew(); } \ No newline at end of file diff --git a/src/ServiceControl.Infrastructure/MachineIdentity.cs b/src/ServiceControl.Infrastructure/MachineIdentity.cs new file mode 100644 index 0000000000..28cd48fba1 --- /dev/null +++ b/src/ServiceControl.Infrastructure/MachineIdentity.cs @@ -0,0 +1,34 @@ +#nullable enable +namespace ServiceControl.Infrastructure; + +using System; +using ServiceControl.Configuration; + +public static class MachineIdentity +{ + /// + /// Containers report this instead of a hash: every container has its own hostname, so two + /// containers on one host would never compare equal and the comparison would mislead. + /// + public const string NotApplicable = "NotApplicable"; + + /// + /// A stable hash of this machine's identity, in a container, or + /// null when nothing stable was available. + /// + public static string? Hash => hash.Value; + + static readonly Lazy hash = new(Compute); + + static string? Compute() + { + if (AppEnvironment.RunningInContainer) + { + return NotApplicable; + } + + var machineName = Environment.MachineName; + + return string.IsNullOrWhiteSpace(machineName) ? null : IdentityHash.Compute("machine-name:" + machineName); + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 3cbe6c1549..423f1d8ba8 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -22,6 +22,8 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs new file mode 100644 index 0000000000..072681c132 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageFootprintProbe.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using System.Data; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class PostgreSqlStorageFootprintProbe(PostgreSqlPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = """ + SELECT + (SELECT SUM(pg_total_relation_size(c.oid)) / 1073741824.0 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relkind IN ('r', 'm')), + (SELECT CASE WHEN c.reltuples < 0 THEN NULL ELSE c.reltuples::bigint END + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = COALESCE(@schema, current_schema()) AND c.relname = @table) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = failedMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage footprint"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 30; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs new file mode 100644 index 0000000000..45f477bfee --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageIdentityProvider.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence; +using ServiceControl.Persistence.EFCore.DbContexts; + +class PostgreSqlStorageIdentityProvider(PostgreSqlPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // The cluster's initialisation identifier rather than an address: every connection to + // the same cluster sees the same value however the host is spelled or proxied. + // current_schema() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT system_identifier::text, current_database(), current_schema() FROM pg_control_system()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("PostgreSQL", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the PostgreSQL storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs index 5ca4450e2e..c9b10a0c6c 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs @@ -24,7 +24,7 @@ public async Task Probe(CancellationToken cancellationToken = d await using var command = dbContext.Database.GetDbConnection().CreateCommand(); // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server // only and comes back null on Azure SQL Database, Managed Instance and Synapse. - command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin')"; + command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin'), CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(64))"; command.CommandTimeout = ProbeTimeoutSeconds; await dbContext.Database.OpenConnectionAsync(cancellationToken); @@ -40,7 +40,12 @@ public async Task Probe(CancellationToken cancellationToken = d var rds = !reader.IsDBNull(2); - return new DatabaseHosting(HostingFor(engineEdition, rds, ConfiguredHost), MajorVersion(reader), DatabaseHostingSource.Probe); + return new DatabaseHosting( + HostingFor(engineEdition, rds, ConfiguredHost), + MajorVersion(reader), + DatabaseHostingSource.Probe, + EditionFamily(engineEdition), + ServiceObjectiveTier(reader.IsDBNull(3) ? null : reader.GetString(3))); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -70,6 +75,48 @@ public async Task Probe(CancellationToken cancellationToken = d _ => DatabaseHostClassifier.Classify(host) }; + /// + /// Developer and Evaluation run the Enterprise engine and report Enterprise, so the report never + /// says which licence an instance runs under. The managed Azure services have no edition. + /// + internal static string EditionFamily(int engineEdition) => engineEdition switch + { + Standard => "Standard", + Enterprise => "Enterprise", + Express => "Express", + AzureSqlDatabase or AzureSqlManagedInstance => DatabaseHosting.NotApplicable, + _ => "Other" + }; + + /// + /// Keeps only the tier. The full service objective also encodes the hardware series, and a + /// DC-series name suggests Always Encrypted with secure enclaves. + /// + internal static string ServiceObjectiveTier(string? serviceObjective) + { + if (string.IsNullOrWhiteSpace(serviceObjective)) + { + return DatabaseHosting.NotApplicable; + } + + var objective = serviceObjective.Trim().ToUpperInvariant(); + + return objective switch + { + "BASIC" => "Basic", + "ELASTICPOOL" => "ElasticPool", + _ when objective.StartsWith("GP_", StringComparison.Ordinal) => "GeneralPurpose", + _ when objective.StartsWith("BC_", StringComparison.Ordinal) => "BusinessCritical", + _ when objective.StartsWith("HS_", StringComparison.Ordinal) => "Hyperscale", + _ when IsDtuObjective(objective, 'S') => "Standard", + _ when IsDtuObjective(objective, 'P') => "Premium", + _ => "Other" + }; + } + + static bool IsDtuObjective(string objective, char tier) => + objective.Length > 1 && objective[0] == tier && objective.AsSpan(1).IndexOfAnyExceptInRange('0', '9') < 0; + static string ManagedOrSelfHosted(string? host) { var classified = DatabaseHostClassifier.Classify(host); @@ -101,8 +148,8 @@ DatabaseHosting HostingFromConnectionString() var host = ConfiguredHost; return host is null - ? DatabaseHosting.Unclassified - : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString); + ? DatabaseHosting.Unclassified with { ServerEdition = DatabaseHostClassifier.Unknown, ServiceObjective = DatabaseHostClassifier.Unknown } + : new DatabaseHosting(DatabaseHostClassifier.Classify(host), DatabaseHostClassifier.Unknown, DatabaseHostingSource.ConnectionString, DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown); } string? ConfiguredHost diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index dead62d3cc..8f6a8f78dd 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -22,6 +22,8 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs new file mode 100644 index 0000000000..c8c965bdfc --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageFootprintProbe.cs @@ -0,0 +1,97 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using System.Data; +using System.Globalization; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SqlServerStorageFootprintProbe(SqlServerPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageFootprintProbe +{ + public async Task Probe(CancellationToken cancellationToken = default) + { + for (var attempt = 1; ; attempt++) + { + try + { + return await Read(cancellationToken); + } + catch (SqlException e) when (attempt < MaxAttempts && e.Number is DeadlockVictim or ScanLostItsPlace) + { + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage footprint"); + + return null; + } + } + } + + async Task Read(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var failedMessagesTable = dbContext.Model.FindEntityType(typeof(FailedMessageEntity))?.GetTableName(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // Full-text index fragments live in internal tables under the indexed table, so they are counted too. + // Catalog reads under READ COMMITTED deadlock with concurrent DDL. Under READ UNCOMMITTED a scan can + // instead lose its place when pages move, so Probe retries both errors. + command.CommandText = """ + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + DECLARE @resolved sysname = COALESCE(@schema, SCHEMA_NAME()); + SELECT + (SELECT SUM(a.total_pages) * 8.0 / 1048576 + FROM sys.partitions p + JOIN sys.allocation_units a ON a.container_id = p.partition_id + WHERE p.object_id IN ( + SELECT t.object_id FROM sys.tables t WHERE t.schema_id = SCHEMA_ID(@resolved) + UNION ALL + SELECT it.object_id FROM sys.internal_tables it JOIN sys.tables t ON t.object_id = it.parent_id WHERE t.schema_id = SCHEMA_ID(@resolved))), + (SELECT SUM(p.rows) + FROM sys.tables t + JOIN sys.partitions p ON p.object_id = t.object_id + WHERE t.schema_id = SCHEMA_ID(@resolved) AND t.name = @table AND p.index_id IN (0, 1)) + """; + command.CommandTimeout = ProbeTimeoutSeconds; + + var schemaParameter = command.CreateParameter(); + schemaParameter.ParameterName = "@schema"; + schemaParameter.DbType = DbType.String; + schemaParameter.Value = (object?)settings.Schema ?? DBNull.Value; + command.Parameters.Add(schemaParameter); + + var tableParameter = command.CreateParameter(); + tableParameter.ParameterName = "@table"; + tableParameter.Value = failedMessagesTable ?? ""; + command.Parameters.Add(tableParameter); + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new StorageFootprint( + reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + + const int ProbeTimeoutSeconds = 30; + const int MaxAttempts = 3; + const int DeadlockVictim = 1205; + const int ScanLostItsPlace = 601; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs new file mode 100644 index 0000000000..ae21a6c7df --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageIdentityProvider.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence; +using ServiceControl.Persistence.EFCore.DbContexts; + +class SqlServerStorageIdentityProvider(SqlServerPersisterSettings settings, IServiceScopeFactory scopeFactory, ILogger logger) : IStorageIdentityProvider +{ + public async ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + // SCHEMA_NAME() is where unqualified tables go when no schema is configured. + command.CommandText = "SELECT CAST(SERVERPROPERTY('ServerName') AS nvarchar(256)), DB_NAME(), SCHEMA_NAME()"; + command.CommandTimeout = ProbeTimeoutSeconds; + + await dbContext.Database.OpenConnectionAsync(cancellationToken); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0) || reader.IsDBNull(1)) + { + return null; + } + + return new StorageIdentity("SQLServer", reader.GetString(0), reader.GetString(1), settings.Schema ?? (reader.IsDBNull(2) ? null : reader.GetString(2))); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + logger.LogDebug(e, "Could not read the SQL Server storage identity"); + + return null; + } + } + + const int ProbeTimeoutSeconds = 5; +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs index f5ccfaf98a..0b3809c362 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs @@ -1,28 +1,45 @@ namespace ServiceControl.Persistence.EFCore.Implementation; +using System; using System.Globalization; using Abstractions; +using DbContexts; +using Entities; using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Particular.LicensingComponent.Contracts; using ServiceControl.Configuration; using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; using static Particular.LicensingComponent.Contracts.EnvironmentDatum; -class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe) : IEnvironmentDataProvider +class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe, IStorageFootprintProbe footprintProbe, IServiceScopeFactory scopeFactory) : IEnvironmentDataProvider { public IEnumerable GetData() { - // The three hosting keys share one probe, so the database is asked once per report and they - // stand or fall together, which is right because they have a single cause. + // The hosting keys share one probe, so the database is asked once per report and they + // stand or fall together, which is right because they have a single cause. The footprint + // keys share theirs for the same reason. Task? probe = null; Task Hosting(CancellationToken cancellationToken) => probe ??= hostingProbe.Probe(cancellationToken); + Task? footprint = null; + Task Footprint(CancellationToken cancellationToken) => footprint ??= footprintProbe.Probe(cancellationToken); + return [ Value("Storage.Type", () => hostingProbe.StorageName), Deferred("Storage.Hosting", async cancellationToken => (await Hosting(cancellationToken)).Hosting), Deferred("Storage.ServerVersion", async cancellationToken => (await Hosting(cancellationToken)).ServerVersion), Deferred("Storage.HostingSource", async cancellationToken => (await Hosting(cancellationToken)).Source), + Deferred("Storage.ServerEdition", async cancellationToken => (await Hosting(cancellationToken)).ServerEdition), + Deferred("Storage.ServiceObjective", async cancellationToken => (await Hosting(cancellationToken)).ServiceObjective), + Deferred("Storage.SizeGB", async cancellationToken => SizeGB((await Footprint(cancellationToken))?.SizeGB)), + Deferred("Storage.MessageCount", async cancellationToken => Count((await Footprint(cancellationToken))?.MessageCount)), + Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), + Deferred("Health.Error.FailedImports", FailedImports), + Deferred("Health.Error.RetentionBehindHours", RetentionBehindHours), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => BodyStorageType(settings.BodyStorage)), Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture)), @@ -35,6 +52,51 @@ public IEnumerable GetData() ]; } + async ValueTask UnresolvedFailedMessages(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var count = await dbContext.Set().CountAsync(m => m.Status == FailedMessageStatus.Unresolved, cancellationToken); + + return Count(count); + } + + async ValueTask FailedImports(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var count = await dbContext.Set().CountAsync(cancellationToken); + + return Count(count); + } + + async ValueTask RetentionBehindHours(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var oldest = await dbContext.Set() + .Where(m => m.Status == FailedMessageStatus.Resolved || m.Status == FailedMessageStatus.Archived) + .MinAsync(m => (DateTime?)m.StatusChangedAt, cancellationToken); + + if (oldest is null) + { + return Count(0); + } + + var behind = DateTime.UtcNow - settings.ErrorRetentionPeriod - oldest.Value; + + return Count(behind > TimeSpan.Zero ? (long)behind.TotalHours : 0); + } + + static string SizeGB(double? sizeGB) => + sizeGB is null ? "Unknown" : sizeGB.Value.ToString("F1", CultureInfo.InvariantCulture); + + static string Count(long? count) => + count is null ? "Unknown" : count.Value.ToString(CultureInfo.InvariantCulture); + string FreeSpaceThreshold() => settings.BodyStorage is FileSystemBodyStorageSettings fileSystem ? WhenConfigured(EFPersistenceConfigurationBase.FileSystemDataSpaceRemainingThresholdKey, () => Number(fileSystem.DataSpaceRemainingThreshold)) : "NotApplicable"; diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index 2683c5069c..e57519e46b 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -303,6 +303,12 @@ public Task GetAuditServiceMetadata(CancellationToken canc public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default) => ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditServiceMetadata, auditServiceMetadata, token), cancellationToken); + public Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.GetSetting(SettingKeys.AuditEnvironmentMetadata, token), cancellationToken); + + public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((context, token) => context.StoreSetting(SettingKeys.AuditEnvironmentMetadata, auditEnvironmentMetadata, token), cancellationToken); + public Task> GetReportMasks(CancellationToken cancellationToken = default) => ExecuteWithDbContext(async (context, token) => await context.GetSetting>(SettingKeys.ReportMasks, token) ?? [], cancellationToken); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs index 73c3b12051..1c8088b7c8 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IDatabaseHostingProbe.cs @@ -23,8 +23,12 @@ public interface IDatabaseHostingProbe /// One of AzureSql, AzureSqlManagedInstance, AzureSqlEdge, AzurePostgres, AzureMySql, AwsRds, GoogleCloudSql, RavenCloud, SelfHosted or Unknown. /// The engine major version, or Unknown. /// A value. -public record DatabaseHosting(string Hosting, string ServerVersion, string Source) +/// The SQL Server edition family: Express, Standard, Enterprise or Other. NotApplicable on the managed Azure services and on engines without editions. +/// The Azure SQL service tier: Basic, Standard, Premium, GeneralPurpose, BusinessCritical, Hyperscale, ElasticPool or Other. NotApplicable everywhere else. +public record DatabaseHosting(string Hosting, string ServerVersion, string Source, string ServerEdition = DatabaseHosting.NotApplicable, string ServiceObjective = DatabaseHosting.NotApplicable) { + public const string NotApplicable = "NotApplicable"; + /// Nothing was available to classify the host with. A determination, not a failure. public static readonly DatabaseHosting Unclassified = new(DatabaseHostClassifier.Unknown, DatabaseHostClassifier.Unknown, DatabaseHostingSource.None); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs new file mode 100644 index 0000000000..c82d543097 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IStorageFootprintProbe.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// Measures what this instance's data occupies, from catalog statistics scoped to the configured +/// schema, never by counting rows. Reported in usage telemetry as sizes and counts only. +/// +public interface IStorageFootprintProbe +{ + /// + /// Null when the database could not be asked. A field is null when the catalog had no answer, + /// such as a row estimate on a table that was never analysed. + /// + Task Probe(CancellationToken cancellationToken = default); +} + +public record StorageFootprint(double? SizeGB, long? MessageCount); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index 7a8e403170..5f01a2ddf9 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -6,6 +6,7 @@ static class SettingKeys public const string TrialEndDate = "TrialEndDate"; public const string BrokerMetadata = "BrokerMetadata"; public const string AuditServiceMetadata = "AuditServiceMetadata"; + public const string AuditEnvironmentMetadata = "AuditEnvironmentMetadata"; public const string ReportMasks = "ReportMasks"; public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string NotificationEmails = "NotificationEmails"; diff --git a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs index 61c709b6d7..41ba550345 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs @@ -3,15 +3,20 @@ namespace ServiceControl.Persistence.RavenDB; using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Operations; using Raven.Client.ServerWide.Operations; using ServiceControl.Configuration; using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; using static Particular.LicensingComponent.Contracts.EnvironmentDatum; -class RavenEnvironmentDataProvider(RavenPersisterSettings settings, IRavenDocumentStoreProvider documentStoreProvider) : IEnvironmentDataProvider +class RavenEnvironmentDataProvider(RavenPersisterSettings settings, IRavenDocumentStoreProvider documentStoreProvider, IRavenSessionProvider sessionProvider) : IEnvironmentDataProvider { public IEnumerable GetData() => [ @@ -20,6 +25,12 @@ public IEnumerable GetData() => Value("Storage.Hosting", () => Hosting().Hosting), Deferred("Storage.ServerVersion", ServerVersion), Value("Storage.HostingSource", () => Hosting().Source), + Value("Storage.ServerEdition", () => "NotApplicable"), + Value("Storage.ServiceObjective", () => "NotApplicable"), + Deferred("Storage.SizeGB", SizeGB), + Deferred("Storage.MessageCount", FailedMessageCount), + Deferred("Storage.UnresolvedFailedMessages", UnresolvedFailedMessages), + Deferred("Health.Error.FailedImports", FailedImports), Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"), Value("Storage.BodyStorage.Type", () => "RavenAttachments"), Value("Storage.LogLevel", () => settings.LogsMode), @@ -29,6 +40,45 @@ public IEnumerable GetData() => Value("Storage.ExpirationIntervalSeconds", () => WhenConfigured(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, () => Number(settings.ExpirationProcessTimerInSeconds))) ]; + async ValueTask SizeGB(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetStatisticsOperation(), cancellationToken); + + return (statistics.SizeOnDisk.SizeInBytes / BytesPerGigabyte).ToString("F1", CultureInfo.InvariantCulture); + } + + async ValueTask FailedMessageCount(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("FailedMessages", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + + async ValueTask FailedImports(CancellationToken cancellationToken) + { + var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); + var statistics = await documentStore.Maintenance.SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + + return statistics.Collections.TryGetValue("FailedErrorImports", out var count) + ? count.ToString(CultureInfo.InvariantCulture) + : "0"; + } + + async ValueTask UnresolvedFailedMessages(CancellationToken cancellationToken) + { + using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); + + var count = await session.Query() + .Where(m => m.Status == FailedMessageStatus.Unresolved) + .CountAsync(cancellationToken); + + return count.ToString(CultureInfo.InvariantCulture); + } + (string Hosting, string Source) Hosting() { // An embedded server runs in this process, so there is nothing to infer. @@ -56,4 +106,6 @@ static string WhenConfigured(string key, Func readValue) => static string Number(int value) => value.ToString(CultureInfo.InvariantCulture); static readonly SettingsRootNamespace SettingsNamespace = new("ServiceControl"); + + const double BytesPerGigabyte = 1024d * 1024 * 1024; } diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs index 95dc171a86..87cf50bc5e 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs @@ -40,6 +40,7 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(p => p.GetRequiredService()); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs new file mode 100644 index 0000000000..0c75b444de --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/RavenStorageIdentityProvider.cs @@ -0,0 +1,22 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB; + +using System; +using System.Threading; +using System.Threading.Tasks; + +class RavenStorageIdentityProvider(RavenPersisterSettings settings) : IStorageIdentityProvider +{ + public ValueTask GetIdentity(CancellationToken cancellationToken = default) + { + var configuredServer = settings.UseEmbeddedServer ? settings.ServerUrl : settings.ConnectionString; + + return new ValueTask(new StorageIdentity("RavenDB", NormalizeServer(configuredServer), settings.DatabaseName, null)); + } + + // A loopback address names a different server on every machine, embedded servers included. + internal static string NormalizeServer(string? configuredServer) => + Uri.TryCreate(configuredServer, UriKind.Absolute, out var url) + ? $"{(url.IsLoopback ? Environment.MachineName : url.Host)}:{url.Port}" + : configuredServer ?? ""; +} diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 6e14a20845..4508e85843 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -22,6 +22,7 @@ class LicensingDataStore( { internal const string ThroughputTimeSeriesName = "INC: throughput data"; const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; + const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; const string BrokerMetadataDocumentId = "BrokerMetadata"; const string ReportMasksDocumentId = "ReportMasks"; const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; @@ -318,6 +319,23 @@ public async Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMeta await session.SaveChangesAsync(cancellationToken); } + public async Task GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + return await session.LoadAsync(AuditEnvironmentMetadataDocumentId, cancellationToken); + } + + public async Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) + { + var store = await storeProvider.GetDocumentStore(cancellationToken); + using IAsyncDocumentSession session = store.OpenAsyncSession(databaseConfiguration.Name); + + await session.StoreAsync(auditEnvironmentMetadata, AuditEnvironmentMetadataDocumentId, cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + public async Task> GetReportMasks(CancellationToken cancellationToken = default) { var store = await storeProvider.GetDocumentStore(cancellationToken); diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs new file mode 100644 index 0000000000..448658c141 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/HealthEnvironmentDataTests.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class HealthEnvironmentDataTests : PersistenceTestBase +{ + [Test] + public async Task An_empty_schema_is_healthy() + { + var data = await GetData(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + } + } + + [Test] + public async Task An_old_unresolved_message_does_not_put_retention_behind() + { + var ancient = DateTime.UtcNow.AddDays(-400); + + await Store(FailedMessage(FailedMessageStatus.Unresolved, ancient)); + + var data = await GetData(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("1")); + } + } + + [Test] + public async Task Retention_is_behind_by_the_oldest_resolved_or_archived_message_past_the_window() + { + var retention = PersistenceTestsContext.DefaultRetentionPeriod; + + await Store( + FailedMessage(FailedMessageStatus.Unresolved, DateTime.UtcNow.AddDays(-400)), + FailedMessage(FailedMessageStatus.Resolved, DateTime.UtcNow - retention - TimeSpan.FromHours(48)), + FailedMessage(FailedMessageStatus.Archived, DateTime.UtcNow - retention - TimeSpan.FromHours(24))); + + var data = await GetData(); + + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("48")); + } + + static FailedMessageEntity FailedMessage(FailedMessageStatus status, DateTime statusChangedAt) => new() + { + UniqueMessageId = Guid.NewGuid(), + Status = status, + StatusChangedAt = statusChangedAt, + LastModified = statusChangedAt, + NumberOfProcessingAttempts = 1, + FirstTimeOfFailure = statusChangedAt, + LastTimeOfFailure = statusChangedAt, + LastAttemptedAt = statusChangedAt, + HeadersJson = "{}", + FailingEndpointAddress = "Shipping" + }; + + async Task Store(params FailedMessageEntity[] messages) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.Set().AddRange(messages); + + await dbContext.SaveChangesAsync(CancellationToken.None); + } + + async Task> GetData() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + return data; + } +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs new file mode 100644 index 0000000000..a2314a2427 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageFootprintProbeTests.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageFootprintProbeTests : PersistenceTestBase +{ + [Test] + public async Task Measures_schema_size_and_message_rows() + { + var probe = ServiceProvider.GetRequiredService(); + + var footprint = await probe.Probe(); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + Assert.That(footprint.MessageCount ?? 0, Is.EqualTo(0), "A never analysed table has no row estimate, which reads back as null"); + } + } + + [Test] + public async Task Editions_do_not_apply() + { + var probe = ServiceProvider.GetRequiredService(); + + var hosting = await probe.Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo(DatabaseHosting.NotApplicable)); + Assert.That(hosting.ServiceObjective, Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs new file mode 100644 index 0000000000..f7a28ea2e2 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/StorageIdentityProviderTests.cs @@ -0,0 +1,54 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.Infrastructure; +using EFCore.PostgreSql; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; +using NUnit.Framework; + +class StorageIdentityProviderTests : PersistenceTestBase +{ + [Test] + public async Task Identifies_the_cluster_database_and_schema() + { + var provider = ServiceProvider.GetRequiredService(); + + var identity = await provider.GetIdentity(); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo("PostgreSQL")); + Assert.That(identity.Server, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Database, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(((PostgreSqlPersisterSettings)PersistenceSettings).Schema)); + } + } + + [Test] + public async Task Measures_the_search_path_schema_when_none_is_configured() + { + var configured = (PostgreSqlPersisterSettings)PersistenceSettings; + var services = new ServiceCollection(); + services.AddLogging(); + new PostgreSqlPersistenceConfiguration().Create(new PostgreSqlPersisterSettings + { + ConnectionString = new NpgsqlConnectionStringBuilder(configured.ConnectionString) { SearchPath = configured.Schema }.ConnectionString, + BodyStorage = configured.BodyStorage, + ErrorRetentionPeriod = configured.ErrorRetentionPeriod, + EventsRetentionPeriod = configured.EventsRetentionPeriod + }).AddPersistence(services); + await using var unconfigured = services.BuildServiceProvider(); + + var identity = await unconfigured.GetRequiredService().GetIdentity(); + var footprint = await unconfigured.GetRequiredService().Probe(); + var expected = await ServiceProvider.GetRequiredService().Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(identity?.Schema, Is.EqualTo(configured.Schema)); + Assert.That(footprint?.SizeGB, Is.Not.Null.And.EqualTo(expected?.SizeGB)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs index cd7c23e4dc..8f1d0d0783 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs @@ -55,7 +55,7 @@ static async Task> GetData(RavenPersisterSettings set { var data = new Dictionary(); - foreach (var datum in new RavenEnvironmentDataProvider(settings, documentStoreProvider: null).GetData()) + foreach (var datum in new RavenEnvironmentDataProvider(settings, documentStoreProvider: null, sessionProvider: null).GetData()) { if (ConfigurationKeys.Contains(datum.Key)) { diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs new file mode 100644 index 0000000000..cfb895b377 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/RavenStorageIdentityProviderTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.Tests.RavenDB; + +using System; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDB; + +[TestFixture] +class RavenStorageIdentityProviderTests +{ + [TestCase("http://localhost:33334")] + [TestCase("http://127.0.0.1:33334")] + [TestCase("http://[::1]:33334")] + public void A_loopback_server_is_this_machine(string url) => + Assert.That(RavenStorageIdentityProvider.NormalizeServer(url), Is.EqualTo($"{Environment.MachineName}:33334")); + + [Test] + public void A_remote_server_is_its_host_and_port() => + Assert.That(RavenStorageIdentityProvider.NormalizeServer("https://raven.example.com:8080"), Is.EqualTo("raven.example.com:8080")); +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs new file mode 100644 index 0000000000..448658c141 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/HealthEnvironmentDataTests.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class HealthEnvironmentDataTests : PersistenceTestBase +{ + [Test] + public async Task An_empty_schema_is_healthy() + { + var data = await GetData(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.FailedImports").WithValue("0")); + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("0")); + } + } + + [Test] + public async Task An_old_unresolved_message_does_not_put_retention_behind() + { + var ancient = DateTime.UtcNow.AddDays(-400); + + await Store(FailedMessage(FailedMessageStatus.Unresolved, ancient)); + + var data = await GetData(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("0")); + Assert.That(data, Does.ContainKey("Storage.UnresolvedFailedMessages").WithValue("1")); + } + } + + [Test] + public async Task Retention_is_behind_by_the_oldest_resolved_or_archived_message_past_the_window() + { + var retention = PersistenceTestsContext.DefaultRetentionPeriod; + + await Store( + FailedMessage(FailedMessageStatus.Unresolved, DateTime.UtcNow.AddDays(-400)), + FailedMessage(FailedMessageStatus.Resolved, DateTime.UtcNow - retention - TimeSpan.FromHours(48)), + FailedMessage(FailedMessageStatus.Archived, DateTime.UtcNow - retention - TimeSpan.FromHours(24))); + + var data = await GetData(); + + Assert.That(data, Does.ContainKey("Health.Error.RetentionBehindHours").WithValue("48")); + } + + static FailedMessageEntity FailedMessage(FailedMessageStatus status, DateTime statusChangedAt) => new() + { + UniqueMessageId = Guid.NewGuid(), + Status = status, + StatusChangedAt = statusChangedAt, + LastModified = statusChangedAt, + NumberOfProcessingAttempts = 1, + FirstTimeOfFailure = statusChangedAt, + LastTimeOfFailure = statusChangedAt, + LastAttemptedAt = statusChangedAt, + HeadersJson = "{}", + FailingEndpointAddress = "Shipping" + }; + + async Task Store(params FailedMessageEntity[] messages) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.Set().AddRange(messages); + + await dbContext.SaveChangesAsync(CancellationToken.None); + } + + async Task> GetData() + { + var provider = ServiceProvider.GetRequiredService(); + var data = new Dictionary(); + + foreach (var datum in provider.GetData()) + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + + return data; + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs index 31d19aea3d..11780a2066 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/SqlServerDatabaseHostingProbeTests.cs @@ -1,6 +1,11 @@ namespace ServiceControl.Persistence.Tests.SqlServer; +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.SqlServer; // EngineEdition values per @@ -50,4 +55,51 @@ public void Should_fall_back_to_the_host_for_an_unmapped_edition(int engineEditi public void Should_report_unknown_for_an_unmapped_edition_on_an_unrecognised_host(int engineEdition) => Assert.That(SqlServerDatabaseHostingProbe.HostingFor(engineEdition, false, "db01.corp.example"), Is.EqualTo("Unknown"), "An edition we do not map is not evidence of an ordinary SQL Server"); + + [TestCase(2, "Standard")] + [TestCase(3, "Enterprise")] + [TestCase(4, "Express")] + [TestCase(5, "NotApplicable")] + [TestCase(8, "NotApplicable")] + [TestCase(1, "Other")] + [TestCase(9, "Other")] + [TestCase(12, "Other")] + public void Should_report_the_edition_family(int engineEdition, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.EditionFamily(engineEdition), Is.EqualTo(expected)); + + [TestCase(null, "NotApplicable")] + [TestCase("Basic", "Basic")] + [TestCase("S0", "Standard")] + [TestCase("S12", "Standard")] + [TestCase("P15", "Premium")] + [TestCase("GP_Gen5_2", "GeneralPurpose")] + [TestCase("GP_S_Gen5_1", "GeneralPurpose")] + [TestCase("GP_DC_8", "GeneralPurpose")] + [TestCase("BC_Gen5_8", "BusinessCritical")] + [TestCase("HS_Gen5_4", "Hyperscale")] + [TestCase("ElasticPool", "ElasticPool")] + [TestCase("DW100c", "Other")] + [TestCase("System2", "Other")] + public void Should_report_only_the_service_objective_tier(string serviceObjective, string expected) => + Assert.That(SqlServerDatabaseHostingProbe.ServiceObjectiveTier(serviceObjective), Is.EqualTo(expected)); + + [TestCase("Server=sc.database.windows.net;Database=sc")] + [TestCase("not a connection string")] + public async Task Should_report_edition_and_service_objective_as_unknown_when_the_server_cannot_be_asked(string connectionString) + { + var probe = new SqlServerDatabaseHostingProbe(new SqlServerPersisterSettings { ConnectionString = connectionString, BodyStorage = new FileSystemBodyStorageSettings { StoragePath = "/var/bodies" } }, new UnreachableScopeFactory(), NullLogger.Instance); + + var hosting = await probe.Probe(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, Is.EqualTo("Unknown")); + Assert.That(hosting.ServiceObjective, Is.EqualTo("Unknown")); + } + } + + sealed class UnreachableScopeFactory : IServiceScopeFactory + { + public IServiceScope CreateScope() => throw new InvalidOperationException("The database cannot be reached"); + } } diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs new file mode 100644 index 0000000000..0a916cbc7e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/StorageFootprintProbeTests.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageFootprintProbeTests : PersistenceTestBase +{ + [Test] + public async Task Measures_schema_size_and_message_rows() + { + var probe = ServiceProvider.GetRequiredService(); + + var footprint = await probe.Probe(); + + Assert.That(footprint, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(footprint.SizeGB, Is.GreaterThan(0)); + Assert.That(footprint.MessageCount, Is.EqualTo(0)); + } + } + + [Test] + public async Task Reports_the_engine_edition() + { + var probe = ServiceProvider.GetRequiredService(); + + var hosting = await probe.Probe(); + + var managed = hosting.Hosting is "AzureSql" or "AzureSqlManagedInstance"; + + using (Assert.EnterMultipleScope()) + { + Assert.That(hosting.ServerEdition, managed ? Is.EqualTo(DatabaseHosting.NotApplicable) : Is.AnyOf("Express", "Standard", "Enterprise")); + Assert.That(hosting.ServiceObjective, hosting.Hosting == "AzureSql" ? Is.AnyOf(ServiceTiers) : Is.EqualTo(DatabaseHosting.NotApplicable)); + } + } + + static readonly object[] ServiceTiers = ["Basic", "Standard", "Premium", "GeneralPurpose", "BusinessCritical", "Hyperscale", "ElasticPool"]; +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs new file mode 100644 index 0000000000..74c7a53053 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/StorageIdentityProviderTests.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using EFCore.SqlServer; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +class StorageIdentityProviderTests : PersistenceTestBase +{ + [Test] + public async Task Identifies_the_server_database_and_schema() + { + var provider = ServiceProvider.GetRequiredService(); + + var identity = await provider.GetIdentity(); + + Assert.That(identity, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(identity.Engine, Is.EqualTo("SQLServer")); + Assert.That(identity.Server, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Database, Is.Not.Null.And.Not.Empty); + Assert.That(identity.Schema, Is.EqualTo(((SqlServerPersisterSettings)PersistenceSettings).Schema)); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs index daab19d03a..12bda43295 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataConfigurationTests.cs @@ -86,9 +86,16 @@ static async Task> GetData(EFPersisterSettings settin { var data = new Dictionary(); - foreach (var datum in new EFEnvironmentDataProvider(settings, new StubHostingProbe()).GetData()) + foreach (var datum in new EFEnvironmentDataProvider(settings, new StubHostingProbe(), new StubFootprintProbe(), scopeFactory: null).GetData()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + try + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + catch (Exception) + { + data[datum.Key] = Particular.LicensingComponent.Contracts.EnvironmentDatum.ReadFailed; + } } return data; @@ -110,4 +117,9 @@ sealed class StubHostingProbe : IDatabaseHostingProbe public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(DatabaseHosting.Unclassified); } + + sealed class StubFootprintProbe : IStorageFootprintProbe + { + public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(null); + } } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs index 08c8e3e5a0..c899ad2df3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/EnvironmentDataTests.cs @@ -102,17 +102,29 @@ public async Task Should_fall_back_to_unknown_when_the_hosting_probe_fails() static async Task> GetData(BodyStorageSettings bodyStorage, IDatabaseHostingProbe hostingProbe = null) { var settings = new TestPersisterSettings { ConnectionString = "Host=localhost", BodyStorage = bodyStorage }; - var provider = new EFEnvironmentDataProvider(settings, hostingProbe ?? new TestHostingProbe()); + var provider = new EFEnvironmentDataProvider(settings, hostingProbe ?? new TestHostingProbe(), new TestFootprintProbe(), null); var data = new Dictionary(); foreach (var datum in provider.GetData()) { - data[datum.Key] = await datum.ReadValue(CancellationToken.None); + try + { + data[datum.Key] = await datum.ReadValue(CancellationToken.None); + } + catch (Exception) + { + data[datum.Key] = Particular.LicensingComponent.Contracts.EnvironmentDatum.ReadFailed; + } } return data; } + class TestFootprintProbe : IStorageFootprintProbe + { + public Task Probe(CancellationToken cancellationToken = default) => Task.FromResult(null); + } + class TestPersisterSettings : EFPersisterSettings; class TestHostingProbe : IDatabaseHostingProbe diff --git a/src/ServiceControl.Persistence/IStorageIdentityProvider.cs b/src/ServiceControl.Persistence/IStorageIdentityProvider.cs new file mode 100644 index 0000000000..2c13a60a83 --- /dev/null +++ b/src/ServiceControl.Persistence/IStorageIdentityProvider.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Persistence; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// Identifies the storage this instance writes to, so that it can be compared with what an audit +/// remote reports over the environment endpoint. The raw values never leave the process; only +/// hashes are compared, and only the comparison result is reported. +/// +public interface IStorageIdentityProvider +{ + /// + /// Null when the storage could not be asked, which the comparison reports as Unknown rather + /// than guessing from configuration that the two sides may spell differently. + /// + ValueTask GetIdentity(CancellationToken cancellationToken = default); +} + +public record StorageIdentity(string Engine, string Server, string Database, string? Schema); diff --git a/src/ServiceControl.UnitTests/API/APIApprovals.cs b/src/ServiceControl.UnitTests/API/APIApprovals.cs index 04d17fc409..a7e3f298e1 100644 --- a/src/ServiceControl.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.UnitTests/API/APIApprovals.cs @@ -38,7 +38,8 @@ public async Task RootPathValue() new ActiveLicense(null, NullLogger.Instance) { IsValid = true }, new Settings(), null, - new MassTransitConnectorHeartbeatStatus()); + new MassTransitConnectorHeartbeatStatus(), + []); var controller = new RootController(configurationApi) { diff --git a/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs b/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs new file mode 100644 index 0000000000..956a86fad3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Infrastructure/IngestionCountersTests.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.Infrastructure; + +using System; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; + +[TestFixture] +public class IngestionCountersTests +{ + [Test] + public void Batches_add_their_messages_and_their_time() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordBatch(10, TimeSpan.FromSeconds(2)); + counters.RecordBatch(5, TimeSpan.FromSeconds(1)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.Messages, Is.EqualTo(15)); + Assert.That(snapshot.BusySeconds, Is.EqualTo(3).Within(0.001)); + } + } + + [Test] + public void A_failed_batch_costs_time_but_stores_no_messages() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordBatch(0, TimeSpan.FromSeconds(2)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.Messages, Is.EqualTo(0)); + Assert.That(snapshot.BusySeconds, Is.EqualTo(2).Within(0.001)); + } + } + + [Test] + public void Lag_lands_in_every_bucket_it_exceeds() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordLag(TimeSpan.FromSeconds(30)); + counters.RecordLag(TimeSpan.FromMinutes(5)); + counters.RecordLag(TimeSpan.FromMinutes(30)); + counters.RecordLag(TimeSpan.FromHours(2)); + + var snapshot = counters.GetSnapshot(); + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot.LagKnownMessages, Is.EqualTo(4)); + Assert.That(snapshot.LagOverOneMinuteMessages, Is.EqualTo(3)); + Assert.That(snapshot.LagOverTenMinutesMessages, Is.EqualTo(2)); + Assert.That(snapshot.LagOverSixtyMinutesMessages, Is.EqualTo(1)); + } + } + + [Test] + public void Storage_time_accumulates() + { + var counters = new IngestionCounters(new FakeTimeProvider()); + + counters.RecordStorage(TimeSpan.FromMilliseconds(500)); + counters.RecordStorage(TimeSpan.FromMilliseconds(250)); + + Assert.That(counters.GetSnapshot().StorageSeconds, Is.EqualTo(0.75).Within(0.001)); + } +} diff --git a/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs b/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs new file mode 100644 index 0000000000..aa7a10cd9e --- /dev/null +++ b/src/ServiceControl.UnitTests/Infrastructure/IngestionSummaryTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Infrastructure; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; + +[TestFixture] +public class IngestionSummaryTests +{ + static readonly DateTime Start = new(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc); + + [Test] + public void Reports_only_uptime_during_the_first_hour() + { + var snapshot = new IngestionCountersSnapshot(Start, 500, 100, 10, 5, 2, 1, 50); + + var data = Describe(snapshot, Start.AddMinutes(59)); + + Assert.That(data, Is.EqualTo(new Dictionary { ["Health.UptimeHours"] = "0" })); + } + + [Test] + public void Rates_are_per_day_and_percentages_are_of_uptime() + { + var snapshot = new IngestionCountersSnapshot(Start, 4800, 43200, 48, 100, 50, 10, 1000); + + var data = Describe(snapshot, Start.AddDays(2)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(data, Does.ContainKey("Health.UptimeHours").WithValue("48")); + Assert.That(data, Does.ContainKey("Ingestion.AvgDailyMessages").WithValue("2400")); + Assert.That(data, Does.ContainKey("Ingestion.BusyPercent").WithValue("25")); + Assert.That(data, Does.ContainKey("Ingestion.StorageMsPerMessage").WithValue("10")); + Assert.That(data, Does.ContainKey("Health.LagOver1MinPercent").WithValue("10")); + Assert.That(data, Does.ContainKey("Health.LagOver10MinPercent").WithValue("5")); + Assert.That(data, Does.ContainKey("Health.LagOver60MinPercent").WithValue("1")); + } + } + + [Test] + public void Busy_percent_is_capped_when_batches_overlap() + { + var snapshot = new IngestionCountersSnapshot(Start, 10, 7200, 0, 0, 0, 0, 0); + + var data = Describe(snapshot, Start.AddHours(1)); + + Assert.That(data, Does.ContainKey("Ingestion.BusyPercent").WithValue("100")); + } + + [Test] + public void Lag_is_left_out_until_a_message_carried_the_header() + { + var snapshot = new IngestionCountersSnapshot(Start, 10, 1, 0, 0, 0, 0, 0); + + var data = Describe(snapshot, Start.AddHours(1)); + + Assert.That(data.Keys, Has.None.StartsWith("Health.LagOver")); + } + + [Test] + public void Storage_time_is_left_out_when_not_asked_for_or_nothing_was_ingested() + { + var withoutStorage = new IngestionCountersSnapshot(Start, 10, 1, 1, 0, 0, 0, 0); + var withoutMessages = new IngestionCountersSnapshot(Start, 0, 1, 1, 0, 0, 0, 0); + + using (Assert.EnterMultipleScope()) + { + Assert.That(Describe(withoutStorage, Start.AddHours(1), includeStorage: false), Does.Not.ContainKey("Ingestion.StorageMsPerMessage")); + Assert.That(Describe(withoutMessages, Start.AddHours(1)), Does.Not.ContainKey("Ingestion.StorageMsPerMessage")); + } + } + + static Dictionary Describe(IngestionCountersSnapshot snapshot, DateTime nowUtc, bool includeStorage = true) => + IngestionSummary.Describe(snapshot, nowUtc, "Ingestion", "Health", includeStorage).ToDictionary(pair => pair.Key, pair => pair.Value); +} diff --git a/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs new file mode 100644 index 0000000000..e079f10d00 --- /dev/null +++ b/src/ServiceControl.UnitTests/Licensing/AuditSharingClassifierTests.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.UnitTests.Licensing; + +using NUnit.Framework; +using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Api; +using ServiceControl.Persistence; + +[TestFixture] +public class AuditSharingClassifierTests +{ + [TestCase(null, "a", "Unknown")] + [TestCase("a", null, "Unknown")] + [TestCase("NotApplicable", "a", "NotApplicable")] + [TestCase("a", "NotApplicable", "NotApplicable")] + [TestCase("a", "a", "True")] + [TestCase("a", "b", "False")] + public void Same_machine(string local, string remote, string expected) => + Assert.That(AuditSharingClassifier.SameMachine(local, remote), Is.EqualTo(expected)); + + [Test] + public void Sharing_is_unknown_without_both_identities() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(AuditSharingClassifier.DatabaseSharing(null, Identity()), Is.EqualTo("Unknown")); + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), null), Is.EqualTo("Unknown")); + } + } + + [Test] + public void Different_engines_cannot_share() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "PostgreSQL"), Identity(engine: "RavenDB")), Is.EqualTo("NotApplicable")); + + [Test] + public void Engine_comparison_ignores_case() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "SQLServer"), Identity(engine: "sqlserver")), Is.EqualTo("SameSchema")); + + [Test] + public void Different_servers_are_separate() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(server: "other")), Is.EqualTo("SeparateServer")); + + [Test] + public void Different_databases_on_one_server_share_the_server() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(database: "other")), Is.EqualTo("SameServer")); + + [Test] + public void Equal_schemas_share_the_schema() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity()), Is.EqualTo("SameSchema")); + + [Test] + public void Different_schemas_share_the_database() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(), Identity(schema: "other")), Is.EqualTo("SameDatabase")); + + [Test] + public void Schemaless_engines_share_the_database_at_most() => + Assert.That(AuditSharingClassifier.DatabaseSharing(Identity(engine: "RavenDB", schema: null), Identity(engine: "RavenDB", schema: null)), Is.EqualTo("SameDatabase")); + + [Test] + public void A_schema_is_a_store_of_its_own() => + Assert.That(AuditSharingClassifier.StoreKey(Identity()), Is.EqualTo($"{IdentityHash.Compute("server")}/{IdentityHash.Compute("database")}/{IdentityHash.Compute("schema")}")); + + [Test] + public void Schemaless_engines_are_one_store_per_database() => + Assert.That(AuditSharingClassifier.StoreKey(Identity(engine: "RavenDB", schema: null)), Is.EqualTo($"{IdentityHash.Compute("server")}/{IdentityHash.Compute("database")}")); + + [Test] + public void Hashing_normalises_case_and_whitespace() + { + var hashed = AuditSharingClassifier.Hash(new StorageIdentity("SQLServer", " PRODSQL01 ", "ServiceControl", "dbo")); + + Assert.That(hashed.ServerHash, Is.EqualTo(IdentityHash.Compute("prodsql01"))); + } + + [Test] + public void Hashing_nothing_is_nothing() => + Assert.That(AuditSharingClassifier.Hash(null), Is.Null); + + static HashedStorageIdentity Identity(string engine = "SQLServer", string server = "server", string database = "database", string schema = "schema") => + new(engine, IdentityHash.Compute(server), IdentityHash.Compute(database), schema is null ? null : IdentityHash.Compute(schema)); +} diff --git a/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs b/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs new file mode 100644 index 0000000000..8aceb24c11 --- /dev/null +++ b/src/ServiceControl.UnitTests/Licensing/ConfigurationApiRemoteEnvironmentTests.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.UnitTests.Licensing; + +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using Particular.ServiceControl.Licensing; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Api; +using ServiceControl.Monitoring.HeartbeatMonitoring; +using ServiceControl.UnitTests.ScatterGather; + +[TestFixture] +class ConfigurationApiRemoteEnvironmentTests +{ + [Test] + public async Task Reads_the_machine_hash_and_the_store_from_an_audit_response() + { + var remote = new RemoteInstanceSetting("http://audit.example:44444/api"); + var settings = new Settings { RemoteInstances = [remote] }; + var body = $$""" + { + "environment_data": { "Storage.Type": "SQLServer" }, + "machine_name_hash": "{{MachineIdentity.Hash}}", + "storage_identity": { "engine": "SQLServer", "server_hash": "s", "database_hash": "d", "schema_hash": "x" } + } + """; + var httpClientFactory = new FakeHttpClientFactory(); + httpClientFactory.Register(remote, (request, _) => Task.FromResult(request.RequestUri?.AbsolutePath == "/api/environment" + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") } + : new HttpResponseMessage(HttpStatusCode.NotFound))); + var api = new ConfigurationApi(new ActiveLicense(null, NullLogger.Instance), settings, httpClientFactory, new MassTransitConnectorHeartbeatStatus(), []); + + var environment = (await api.GetRemoteEnvironments()).Single().EnvironmentData; + + using (Assert.EnterMultipleScope()) + { + Assert.That(environment, Does.ContainKey("Storage.Type").WithValue("SQLServer")); + Assert.That(environment, Does.ContainKey("SameMachine").WithValue(MachineIdentity.Hash == MachineIdentity.NotApplicable ? "NotApplicable" : "True")); + Assert.That(environment, Does.ContainKey(AuditEnvironmentMetadata.DatabaseKey).WithValue("s/d/x")); + } + } +} diff --git a/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs b/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs index 6da39ae9a8..86560cd816 100644 --- a/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs +++ b/src/ServiceControl.UnitTests/Operations/Metrics/IngestionMetricsTests.cs @@ -1,11 +1,13 @@ namespace ServiceControl.UnitTests.Operations.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Operations.Metrics; /// @@ -39,7 +41,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() listener.Start(); - _ = new IngestionMetrics(MeterFactory); + _ = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); Assert.That(published.Order(), Is.EqualTo(new[] { @@ -54,7 +56,7 @@ public void The_meter_publishes_the_instruments_it_is_named_for() [Test] public void Concurrent_batch_failures_are_all_counted() { - var metrics = new IngestionMetrics(MeterFactory); + var metrics = new IngestionMetrics(MeterFactory, new IngestionCounters(TimeProvider.System)); const int failedBatches = 1000; diff --git a/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs b/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs new file mode 100644 index 0000000000..c0f9a21f9b --- /dev/null +++ b/src/ServiceControl/ErrorIngestionEnvironmentDataProvider.cs @@ -0,0 +1,15 @@ +namespace Particular.ServiceControl; + +using System; +using System.Collections.Generic; +using System.Linq; +using global::ServiceControl.Infrastructure.Ingestion; +using Particular.LicensingComponent.Contracts; +using static Particular.LicensingComponent.Contracts.EnvironmentDatum; + +class ErrorIngestionEnvironmentDataProvider(IngestionCounters counters, TimeProvider timeProvider) : IEnvironmentDataProvider +{ + public IEnumerable GetData() => + IngestionSummary.Describe(counters.GetSnapshot(), timeProvider.GetUtcNow().UtcDateTime, "Ingestion.Error", "Health.Error", includeStorage: true) + .Select(pair => Value(pair.Key, () => pair.Value)); +} diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index 6b3ac7c6e0..647b549024 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -163,6 +163,8 @@ public static void AddServiceControlInstallers(this IHostApplicationBuilder host public static void AddTelemetry(this IHostApplicationBuilder hostBuilder, Settings settings) { hostBuilder.Services.TryAddSingleton(TimeProvider.System); + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddEnvironmentDataProvider(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); diff --git a/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs new file mode 100644 index 0000000000..263d8d914b --- /dev/null +++ b/src/ServiceControl/Infrastructure/Api/AuditSharingClassifier.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Infrastructure.Api; + +using System; +using ServiceControl.Persistence; + +/// +/// Compares this instance's identity hashes with what an audit remote served and reduces them to +/// the classifications the usage report carries. Only these results are ever reported; the hashes +/// stay between the instances. +/// +static class AuditSharingClassifier +{ + public const string Unknown = "Unknown"; + public const string NotApplicable = "NotApplicable"; + + public static string SameMachine(string localMachineNameHash, string remoteMachineNameHash) + { + if (localMachineNameHash is null || remoteMachineNameHash is null) + { + return Unknown; + } + + if (localMachineNameHash == MachineIdentity.NotApplicable || remoteMachineNameHash == MachineIdentity.NotApplicable) + { + return NotApplicable; + } + + return string.Equals(localMachineNameHash, remoteMachineNameHash, StringComparison.Ordinal) ? "True" : "False"; + } + + public static string DatabaseSharing(HashedStorageIdentity local, HashedStorageIdentity remote) + { + if (local is null || remote is null) + { + return Unknown; + } + + if (!string.Equals(local.Engine, remote.Engine, StringComparison.OrdinalIgnoreCase)) + { + return NotApplicable; + } + + if (!string.Equals(local.ServerHash, remote.ServerHash, StringComparison.Ordinal)) + { + return "SeparateServer"; + } + + if (!string.Equals(local.DatabaseHash, remote.DatabaseHash, StringComparison.Ordinal)) + { + return "SameServer"; + } + + if (local.SchemaHash is not null && string.Equals(local.SchemaHash, remote.SchemaHash, StringComparison.Ordinal)) + { + return "SameSchema"; + } + + return "SameDatabase"; + } + + /// + /// What the report counts once when summing sizes and counts. Instances writing to the same + /// schema share one store; instances in different schemas of one database each have their own. + /// + public static string StoreKey(HashedStorageIdentity identity) => + identity.SchemaHash is null + ? $"{identity.ServerHash}/{identity.DatabaseHash}" + : $"{identity.ServerHash}/{identity.DatabaseHash}/{identity.SchemaHash}"; + + public static HashedStorageIdentity Hash(StorageIdentity identity) => + identity is null + ? null + : new HashedStorageIdentity( + identity.Engine, + IdentityHash.Compute(identity.Server), + IdentityHash.Compute(identity.Database), + identity.Schema is null ? null : IdentityHash.Compute(identity.Schema)); +} + +record HashedStorageIdentity(string Engine, string ServerHash, string DatabaseHash, string SchemaHash); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4d61e23ca8..3873a445bc 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Infrastructure.Api; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -9,12 +10,14 @@ using System.Threading.Tasks; using Configuration; using Monitoring.HeartbeatMonitoring; +using Particular.LicensingComponent.Contracts; using Particular.ServiceControl.Licensing; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api; using ServiceControl.Api.Contracts; +using ServiceControl.Persistence; -class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : IConfigurationApi +class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, IEnumerable storageIdentityProviders) : IConfigurationApi { public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) { @@ -139,4 +142,113 @@ public async Task GetRemoteConfigs(CancellationToken canc return results; } + + public async Task GetRemoteEnvironments(CancellationToken cancellationToken = default) + { + var localMachineNameHash = MachineIdentity.Hash; + var localStorageIdentity = await LocalStorageIdentity(cancellationToken); + + var tasks = settings.RemoteInstances + .Select(async remote => + { + Dictionary environmentData = null; + HttpClient httpClient = httpClientFactory.CreateClient(remote.InstanceId); + + try + { + using var response = await httpClient.GetAsync("/api/environment", cancellationToken); + + // An unreachable remote and one predating the endpoint both leave the data null, + // and the report omits their keys rather than guessing. + if (response.IsSuccessStatusCode) + { + await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); + var body = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); + + if (body?.AsObject().TryGetPropertyValue("environment_data", out var data) == true && data is JsonObject dataObject) + { + environmentData = []; + + foreach (var pair in dataObject) + { + if (pair.Value is not null) + { + environmentData[pair.Key] = pair.Value.GetValue(); + } + } + + var remoteStorageIdentity = ReadStorageIdentity(body); + + environmentData["SameMachine"] = AuditSharingClassifier.SameMachine(localMachineNameHash, ReadString(body, "machine_name_hash")); + environmentData["DatabaseSharing"] = AuditSharingClassifier.DatabaseSharing(localStorageIdentity, remoteStorageIdentity); + + if (remoteStorageIdentity is not null) + { + environmentData[AuditEnvironmentMetadata.DatabaseKey] = AuditSharingClassifier.StoreKey(remoteStorageIdentity); + } + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + } + + return new RemoteEnvironment + { + ApiUri = remote.BaseAddress, + EnvironmentData = environmentData + }; + }); + + return await Task.WhenAll(tasks); + } + + async Task LocalStorageIdentity(CancellationToken cancellationToken) + { + var provider = storageIdentityProviders.FirstOrDefault(); + + if (provider is null) + { + return null; + } + + try + { + return AuditSharingClassifier.Hash(await provider.GetIdentity(cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + return null; + } + } + + static HashedStorageIdentity ReadStorageIdentity(JsonNode body) + { + if (body?.AsObject().TryGetPropertyValue("storage_identity", out var identity) != true || identity is not JsonObject identityObject) + { + return null; + } + + var engine = ReadString(identityObject, "engine"); + var serverHash = ReadString(identityObject, "server_hash"); + var databaseHash = ReadString(identityObject, "database_hash"); + + if (engine is null || serverHash is null || databaseHash is null) + { + return null; + } + + return new HashedStorageIdentity(engine, serverHash, databaseHash, ReadString(identityObject, "schema_hash")); + } + + static string ReadString(JsonNode node, string propertyName) => + node?.AsObject().TryGetPropertyValue(propertyName, out var value) == true && value is JsonValue ? value.GetValue() : null; } \ No newline at end of file diff --git a/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs b/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs index ffbc5d63f7..32a1b138ee 100644 --- a/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs +++ b/src/ServiceControl/Operations/Metrics/IngestionMetrics.cs @@ -1,11 +1,14 @@ namespace ServiceControl.Operations.Metrics; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Threading; +using NServiceBus; using NServiceBus.Transport; using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Ingestion; using ServiceControl.Infrastructure.Ingestion.Metrics; public class IngestionMetrics @@ -16,8 +19,9 @@ public class IngestionMetrics public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration_seconds"; public static readonly string StorageDurationInstrumentName = $"{InstrumentPrefix}.storage_duration_seconds"; - public IngestionMetrics(IMeterFactory meterFactory) + public IngestionMetrics(IMeterFactory meterFactory, IngestionCounters counters) { + this.counters = counters; var meter = meterFactory.Create(MeterName, MeterVersion); batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "seconds", description: "Message batch processing duration in seconds"); @@ -27,17 +31,38 @@ public IngestionMetrics(IMeterFactory meterFactory) failureCounter = meter.CreateCounter($"{InstrumentPrefix}.failures_total", description: "Error ingestion failure count"); } - public MessageMetrics BeginIngestion(MessageContext messageContext) => new(GetMessageTags(messageContext.Headers), ingestionDuration); + public MessageMetrics BeginIngestion(MessageContext messageContext) + { + RecordLag(messageContext.Headers); + + return new(GetMessageTags(messageContext.Headers), ingestionDuration); + } public FailureMetrics BeginErrorHandling(ErrorContext errorContext) => new(GetMessageTags(errorContext.Headers), failureCounter); - public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome); + public BatchMetrics BeginBatch(int maxBatchSize) => new(maxBatchSize, batchDuration, RecordBatchOutcome, counters.RecordBatch); /// /// The storage write on its own, which is the part of a batch that is neither announcing nor /// forwarding. /// - public DurationScope MeasureStorageWrite() => new(storageDuration); + public DurationScope MeasureStorageWrite() => new(storageDuration, counters.RecordStorage); + + void RecordLag(Dictionary headers) + { + if (!headers.TryGetValue("NServiceBus.TimeOfFailure", out var timeOfFailure)) + { + return; + } + + try + { + counters.RecordLag(DateTime.UtcNow - DateTimeOffsetHelper.ToDateTimeOffset(timeOfFailure).UtcDateTime); + } + catch (FormatException) + { + } + } // The same split the ingestor makes: a retry acknowledgement resolves a message, everything // else records a failure, and they cost quite different amounts of work. @@ -70,6 +95,7 @@ void RecordBatchOutcome(bool success) readonly Histogram ingestionDuration; readonly Histogram storageDuration; readonly Counter failureCounter; + readonly IngestionCounters counters; const string MeterVersion = "0.1.0"; const string InstrumentPrefix = "sc.error.ingestion";