Report audit instance environment, storage footprint and ingestion health in the usage report

src/Particular.LicensingComponent.Contracts/AuditEnvironmentMetadata.cs

@@ -0,0 +1,16 @@
+namespace Particular.LicensingComponent.Contracts;
+
+/// <summary>
+/// The environment data each audit remote served when last polled, one dictionary per remote that
+/// responded. Never saved by versions that predate it, so a null read means unknown and the report
+/// omits the keys.
+/// </summary>
+public record AuditEnvironmentMetadata(List<Dictionary<string, string>> Instances)
+{
+    /// <summary>
+    /// The key under which the primary stores which database an instance's data came from, so that
+    /// instances sharing a database are counted once when sizes and counts are summed. Underscore
+    /// prefixed keys are collection bookkeeping and never reach the report.
+    /// </summary>
+    public const string DatabaseKey = "_DatabaseKey";
+}

src/Particular.LicensingComponent.Persistence.InMemory/InMemoryLicensingDataStore.cs

@@ -14,6 +14,7 @@ public class InMemoryLicensingDataStore : ILicensingDataStore
     readonly Dictionary<EndpointIdentifier, ThroughputData> allThroughput = [];
     BrokerMetadata brokerMetadata = new(null, []);
     AuditServiceMetadata auditServiceMetadata = new([], []);
+    AuditEnvironmentMetadata? auditEnvironmentMetadata;
     List<string> reportMasks = [];
     LicensedEndpointDetails? endpointDetails = null;
 
@@ -167,6 +168,14 @@ public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata,
         return Task.CompletedTask;
     }
 
+    public Task<AuditEnvironmentMetadata?> GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) => Task.FromResult(auditEnvironmentMetadata);
+
+    public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default)
+    {
+        this.auditEnvironmentMetadata = auditEnvironmentMetadata;
+        return Task.CompletedTask;
+    }
+
     public Task<List<string>> GetReportMasks(CancellationToken cancellationToken = default) => Task.FromResult(reportMasks);
 
     public Task SaveReportMasks(List<string> reportMasks, CancellationToken cancellationToken = default)

src/Particular.LicensingComponent.Persistence/ILicensingDataStore.cs

@@ -36,6 +36,10 @@ Task RecordEndpointThroughput(string endpointName, ThroughputSource throughputSo
     Task<AuditServiceMetadata> GetAuditServiceMetadata(CancellationToken cancellationToken = default);
 
     Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default);
+
+    Task<AuditEnvironmentMetadata?> GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default);
+
+    Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default);
     Task<List<string>> GetReportMasks(CancellationToken cancellationToken = default);
     Task SaveReportMasks(List<string> reportMasks, CancellationToken cancellationToken = default);
 

src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs

@@ -192,6 +192,8 @@ public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellati
         }
 
         public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
+
+        public Task<RemoteEnvironment[]> GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException();
     }
 
     class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi
@@ -201,6 +203,8 @@ class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) :
         public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellationToken = default) => Task.FromResult(remotes);
 
         public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
+
+        public Task<RemoteEnvironment[]> GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException();
     }
 
     class ConfigurationApi_Configurable : IConfigurationApi
@@ -221,6 +225,8 @@ public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellati
 
         public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
 
+        public Task<RemoteEnvironment[]> GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException();
+
         public bool ReturnAuditConfig { get; set; }
         public string RemoteStatus { get; set; }
         public string RemoteVersion { get; set; }

src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs

@@ -272,10 +272,40 @@ public async Task Should_store_how_many_audit_instances_are_configured_and_how_m
         }
     }
 
+    [Test]
+    public async Task Should_store_the_environment_data_of_every_responding_audit_instance()
+    {
+        var auditQuery = new AuditQuery_WithRemotes([new RemoteInstanceInformation { Status = "online" }])
+        {
+            Environments =
+            [
+                new Dictionary<string, string> { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "4" },
+                new Dictionary<string, string> { ["Storage.Type"] = "RavenDB", ["Host.ProcessorCount"] = "8" }
+            ]
+        };
+
+        using var auditThroughputCollectorHostedService = new AuditThroughputCollectorHostedService(
+            NullLogger<AuditThroughputCollectorHostedService>.Instance, configuration.ThroughputSettings, DataStore,
+            auditQuery, new FakeTimeProvider())
+        { DelayStart = TimeSpan.Zero };
+
+        await auditThroughputCollectorHostedService.StartAsync(CancellationToken.None);
+        await auditQuery.KnownEndpointsRequested.WaitAsync(TimeSpan.FromSeconds(30));
+        await auditThroughputCollectorHostedService.StopAsync(CancellationToken.None);
+
+        var auditEnvironmentMetadata = await DataStore.GetAuditEnvironmentMetadata();
+
+        Assert.That(auditEnvironmentMetadata, Is.Not.Null);
+        Assert.That(auditEnvironmentMetadata.Instances, Is.EqualTo(auditQuery.Environments));
+    }
+
     class AuditQuery_NoAuditRemotes : IAuditQuery
     {
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
 
+        public Task<List<Dictionary<string, string>>> GetAuditEnvironments(CancellationToken cancellationToken = default) =>
+            Task.FromResult<List<Dictionary<string, string>>>([]);
+
         public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => r => true;
 
         public Task<IEnumerable<AuditCount>> GetAuditCountForEndpoint(string endpointUrlName,
@@ -309,6 +339,9 @@ public AuditQuery_WithOneEndpoint(string endpointName, long throughputCount, Dat
 
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
 
+        public Task<List<Dictionary<string, string>>> GetAuditEnvironments(CancellationToken cancellationToken = default) =>
+            Task.FromResult<List<Dictionary<string, string>>>([]);
+
         public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => r => true;
 
         public Task<IEnumerable<AuditCount>> GetAuditCountForEndpoint(string endpointUrlName,
@@ -341,6 +374,9 @@ class AuditQuery_ThrowingAnExceptionOnKnownEndpointsCall : IAuditQuery
     {
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
 
+        public Task<List<Dictionary<string, string>>> GetAuditEnvironments(CancellationToken cancellationToken = default) =>
+            Task.FromResult<List<Dictionary<string, string>>>([]);
+
         public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => r => true;
 
         public Task<IEnumerable<AuditCount>> GetAuditCountForEndpoint(string endpointUrlName,
@@ -420,6 +456,10 @@ public AuditQuery_WithTwoEndpointsAndSelectiveCounts(
         }
 
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
+
+        public Task<List<Dictionary<string, string>>> GetAuditEnvironments(CancellationToken cancellationToken = default) =>
+            Task.FromResult<List<Dictionary<string, string>>>([]);
+
         public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => _ => true;
 
         public Task<IEnumerable<ServiceControlEndpoint>> GetKnownEndpoints(CancellationToken cancellationToken = default) =>
@@ -456,6 +496,12 @@ class AuditQuery_WithRemotes(List<RemoteInstanceInformation> remotes) : IAuditQu
         public Task KnownEndpointsRequested => knownEndpointsRequested.Task;
 
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
+
+        public List<Dictionary<string, string>> Environments { get; init; } = [];
+
+        public Task<List<Dictionary<string, string>>> GetAuditEnvironments(CancellationToken cancellationToken = default) =>
+            Task.FromResult(Environments);
+
         public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => _ => true;
 
         public Task<List<RemoteInstanceInformation>> GetAuditRemotes(CancellationToken cancellationToken = default) =>

src/Particular.LicensingComponent.UnitTests/Infrastructure/FakeConfigurationApi.cs

@@ -10,6 +10,7 @@ class FakeConfigurationApi : IConfigurationApi
     {
         public Task<object> GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException();
         public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellationToken = default) => throw new NotImplementedException();
+        public Task<RemoteEnvironment[]> GetRemoteEnvironments(CancellationToken cancellationToken = default) => throw new NotImplementedException();
         public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
     }
 }

src/Particular.LicensingComponent.UnitTests/MonitoringService_Tests.cs

@@ -252,6 +252,12 @@ public Task<AuditServiceMetadata> GetAuditServiceMetadata(CancellationToken canc
         public Task SaveAuditServiceMetadata(AuditServiceMetadata auditServiceMetadata, CancellationToken cancellationToken = default) =>
             throw new NotSupportedException();
 
+        public Task<AuditEnvironmentMetadata> GetAuditEnvironmentMetadata(CancellationToken cancellationToken = default) =>
+            throw new NotSupportedException();
+
+        public Task SaveAuditEnvironmentMetadata(AuditEnvironmentMetadata auditEnvironmentMetadata, CancellationToken cancellationToken = default) =>
+            throw new NotSupportedException();
+
         public Task<List<string>> GetReportMasks(CancellationToken cancellationToken = default) =>
             throw new NotSupportedException();
 

src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs

@@ -1,6 +1,8 @@
 namespace Particular.LicensingComponent.UnitTests;
 
+using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using NUnit.Framework;
@@ -197,6 +199,17 @@ await DataStore.CreateBuilder()
         Assert.That(report.ReportData.ScopeType, Is.EqualTo(expectedScopeType), $"Invalid ScopeType on report");
     }
 
+    [Test]
+    public async Task Should_leave_the_stored_broker_metadata_unchanged()
+    {
+        await DataStore.SaveBrokerMetadata(new BrokerMetadata("testingScope", new Dictionary<string, string> { [EnvironmentDataType.BrokerVersion.ToString()] = "1.2" }));
+
+        await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var stored = await DataStore.GetBrokerMetadata();
+        Assert.That(stored.Data.Keys, Is.EquivalentTo(new[] { EnvironmentDataType.BrokerVersion.ToString() }));
+    }
+
     [Test]
     public async Task Should_include_audit_instance_counts_in_environment_data()
     {
@@ -228,4 +241,159 @@ await DataStore.SaveAuditServiceMetadata(new AuditServiceMetadata(
             Assert.That(environmentData, Does.Not.ContainKey("Audit.LiveInstances"));
         }
     }
+
+    [Test]
+    public async Task Should_report_audit_environment_data_aggregated_across_instances()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(
+        [
+            new Dictionary<string, string>
+            {
+                ["Storage.Type"] = "RavenDB",
+                ["Storage.ServerVersion"] = "6.2.1",
+                ["Host.ProcessorCount"] = "4",
+                ["Host.AvailableMemoryGB"] = "16",
+                ["Host.OSPlatform"] = "Linux"
+            },
+            new Dictionary<string, string>
+            {
+                ["Storage.Type"] = "RavenDB",
+                ["Storage.ServerVersion"] = "5.4.200",
+                ["Host.ProcessorCount"] = "8",
+                ["Host.AvailableMemoryGB"] = "Unknown"
+            }
+        ]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Storage.ServerVersion").WithValue("Mixed"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Host.ProcessorCount").WithValue("8"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Host.AvailableMemoryGB").WithValue("Mixed"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Host.OSPlatform").WithValue("Linux"));
+        }
+    }
+
+    [Test]
+    public async Task Should_report_the_most_shared_database_class_and_mixed_machines()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(
+        [
+            new Dictionary<string, string> { ["DatabaseSharing"] = "SeparateServer", ["SameMachine"] = "True" },
+            new Dictionary<string, string> { ["DatabaseSharing"] = "SameServer", ["SameMachine"] = "False" },
+            new Dictionary<string, string> { ["DatabaseSharing"] = "Unknown", ["SameMachine"] = "True" }
+        ]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.DatabaseSharing").WithValue("SameServer"));
+            Assert.That(environmentData, Does.ContainKey("Audit.SameMachine").WithValue("Mixed"));
+        }
+    }
+
+    [Test]
+    public async Task Should_sum_sizes_and_counts_once_per_database()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(
+        [
+            new Dictionary<string, string> { ["Storage.SizeGB"] = "10.0", ["Storage.MessageCount"] = "100", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" },
+            new Dictionary<string, string> { ["Storage.SizeGB"] = "10.2", ["Storage.MessageCount"] = "90", ["Health.FailedImports"] = "3", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" },
+            new Dictionary<string, string> { ["Storage.SizeGB"] = "5.0", ["Storage.MessageCount"] = "10", ["Health.FailedImports"] = "4" }
+        ]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.Storage.SizeGB").WithValue("15.2"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Storage.MessageCount").WithValue("110"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Health.FailedImports").WithValue("7"));
+            Assert.That(environmentData.Keys, Has.None.StartsWith("Audit._"));
+        }
+    }
+
+    [Test]
+    public async Task Should_leave_audit_environment_data_out_when_it_was_never_collected()
+    {
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage."));
+    }
+
+    [Test]
+    public async Task Should_sum_audit_ingestion_rates_and_keep_the_worst_saturation_lag_and_uptime()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(
+        [
+            new Dictionary<string, string> { ["Ingestion.AvgDailyMessages"] = "1000", ["Ingestion.BusyPercent"] = "10", ["Health.UptimeHours"] = "500", ["Health.LagOver1MinPercent"] = "1", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" },
+            new Dictionary<string, string> { ["Ingestion.AvgDailyMessages"] = "500", ["Ingestion.BusyPercent"] = "40", ["Health.UptimeHours"] = "3", ["Health.LagOver1MinPercent"] = "7", [AuditEnvironmentMetadata.DatabaseKey] = "server-a/db-1" }
+        ]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.AvgDailyMessages").WithValue("1500"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Ingestion.BusyPercent").WithValue("40"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Health.UptimeHours").WithValue("3"));
+            Assert.That(environmentData, Does.ContainKey("Audit.Health.LagOver1MinPercent").WithValue("7"));
+        }
+    }
+
+    [Test]
+    public async Task Should_leave_audit_environment_data_out_when_no_instance_responded()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys, Has.None.StartsWith("Audit.Storage."));
+    }
+
+    [Test]
+    public async Task Should_report_every_key_an_audit_instance_serves()
+    {
+        string[] served =
+        [
+            "Host.Model", "Host.Orchestrator", "Host.OSPlatform", "Host.OSVersion", "Host.Architecture", "Host.RuntimeVersion", "Host.ProcessorCount", "Host.AvailableMemoryGB",
+            "Storage.Type", "Storage.RavenServer", "Storage.Hosting", "Storage.ServerVersion", "Storage.HostingSource", "Storage.ServerEdition", "Storage.ServiceObjective",
+            "Storage.SizeGB", "Storage.MessageCount", "Storage.FullTextSearch",
+            "Health.FailedImports", "Health.UptimeHours", "Health.LagOver1MinPercent", "Health.LagOver10MinPercent", "Health.LagOver60MinPercent",
+            "Ingestion.AvgDailyMessages", "Ingestion.BusyPercent",
+            "SameMachine", "DatabaseSharing"
+        ];
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata([served.ToDictionary(key => key, _ => "1")]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        Assert.That(report.ReportData.EnvironmentInformation.EnvironmentData.Keys.Where(key => key.StartsWith("Audit.") && key is not "Audit.ConfiguredInstances" and not "Audit.LiveInstances"),
+            Is.EquivalentTo(served.Select(key => "Audit." + key)));
+    }
+
+    [Test]
+    public async Task Should_leave_out_audit_keys_the_primary_does_not_report()
+    {
+        await DataStore.SaveAuditEnvironmentMetadata(new AuditEnvironmentMetadata(
+        [
+            new Dictionary<string, string> { ["Storage.Type"] = "RavenDB", ["Security.Authentication"] = "Enabled", ["Storage.Auth"] = "Integrated" }
+        ]));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.Storage.Type").WithValue("RavenDB"));
+            Assert.That(environmentData, Does.Not.ContainKey("Audit.Security.Authentication"));
+            Assert.That(environmentData, Does.Not.ContainKey("Audit.Storage.Auth"));
+        }
+    }
 }