From 91735c5d88957d75a766f6389976ec79008c3f12 Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Wed, 30 Sep 2026 15:04:10 -0400 Subject: [PATCH 1/8] first pass --- docs/README.md | 1 + docs/platform-health.md | 11 +++ .../When_custom_checks_are_classified.cs | 10 +- .../Contracts/PlatformHealthView.cs | 23 +++++ src/ServiceControl.Api/Contracts/RootUrls.cs | 1 + .../APIApprovals.HttpApiRoutes.approved.txt | 1 + .../APIApprovals.RootPathValue.approved.txt | 3 +- .../PlatformHealthStateTests.cs | 91 +++++++++++++++++++ .../CustomCheckResultProcessor.cs | 7 +- .../CustomChecks/CustomChecksComponent.cs | 2 + .../Infrastructure/Api/ConfigurationApi.cs | 1 + .../PlatformHealthController.cs | 17 ++++ .../PlatformHealth/PlatformHealthState.cs | 75 +++++++++++++++ 13 files changed, 240 insertions(+), 3 deletions(-) create mode 100644 docs/platform-health.md create mode 100644 src/ServiceControl.Api/Contracts/PlatformHealthView.cs create mode 100644 src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs create mode 100644 src/ServiceControl/PlatformHealth/PlatformHealthController.cs create mode 100644 src/ServiceControl/PlatformHealth/PlatformHealthState.cs diff --git a/docs/README.md b/docs/README.md index f421eb3b1c..bbc7b5bb98 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,7 @@ This section points to sources that explain why ServiceControl is designed the w - [Retries over Azure Storage Queues transport](retries-asq-transport.md) — transport-specific retry handling - [Data versioning design](data-versioning-design.md) — the cache-versioning invariant for API responses - [Event log design](eventlog-design.md) — what the event log is and what it records +- [Platform health API](platform-health.md) — how ServicePulse reads internal health independently from customer custom checks - [Multiple ServiceControl instances communication](multipleservicecontrolinstancescommunication.md) — how primary, audit, and monitoring instances talk to each other - [Handling unavailable runtime dependencies](handling-unavailable-runtime-dependencies.md) — how instances react when a dependency is unavailable - [Telemetry](telemetry.md) — telemetry configuration and emitted metrics diff --git a/docs/platform-health.md b/docs/platform-health.md new file mode 100644 index 0000000000..a1f887100e --- /dev/null +++ b/docs/platform-health.md @@ -0,0 +1,11 @@ +# Platform Health API + +ServiceControl exposes `GET /api/platform-health` for ServicePulse to read ServiceControl's internal health signals without treating them as customer custom checks. The API root response advertises the endpoint through `PlatformHealth`. + +The response contains an overall `status` and `severity`, plus an `alerts` array for currently failing internal checks. Each alert includes the check id, category, failure message, report time, instance name, host, and host id. Status is `unknown` before any internal check has reported, `healthy` when internal checks have reported and none are failing, and `unhealthy` when one or more are failing. Severity is `unknown`, `none`, or `error` for those respective states. + +Health state is held in memory by the ServiceControl process. Each new report replaces the current state for that check, so a successful report clears its alert. The endpoint does not currently age out checks that stop reporting; a previously reported failure remains until that check reports again or the process restarts. ServicePulse should retain its custom-check fallback for older ServiceControl versions during rollout. + +This initial response does not include version or license/upgrade information. It also does not replace `/api/customchecks` or the existing custom-check integration events. Internally, the transitional implementation still recognizes shipped checks through `InternalCustomCheckClassification`; removing that classification requires a coordinated contract and migration for audit-originated health reports. + +The endpoint uses the existing `error:customchecks:view` authorization policy. Container liveness and readiness remain separate at `/health` and `/health/ready`. \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs index 8b1995386e..8c3e1bde6a 100644 --- a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs +++ b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs @@ -11,6 +11,7 @@ namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks using NServiceBus.CustomChecks; using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Api.Contracts; using CustomCheckView = global::ServiceControl.Contracts.CustomChecks.CustomCheckView; using CheckStatus = global::ServiceControl.Persistence.Status; @@ -28,6 +29,7 @@ public async Task Internal_checks_are_flagged_internal_and_endpoint_checks_are_n CustomCheckView internalCheck = null; CustomCheckView endpointCheck = null; + PlatformHealthView platformHealth = null; string wireBody = null; await Define() @@ -47,7 +49,12 @@ await Define() wireBody = await raw.Content.ReadAsStringAsync(); } - return internalCheck != null && endpointCheck != null && wireBody != null; + if (internalCheck != null && endpointCheck != null && platformHealth == null) + { + platformHealth = await this.TryGet("/api/platform-health"); + } + + return internalCheck != null && endpointCheck != null && wireBody != null && platformHealth != null; }) .Run(); @@ -58,6 +65,7 @@ await Define() Assert.That(endpointCheck, Is.Not.Null); Assert.That(endpointCheck.Internal, Is.False); + Assert.That(platformHealth.Alerts, Has.None.Matches(alert => alert.CheckId == "MyCustomCheckId")); // What the wire actually carries: Assert.That(wireBody, Does.Contain("\"internal\":true"), "internal checks must render internal:true on the wire"); diff --git a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs new file mode 100644 index 0000000000..ecab8e0ab0 --- /dev/null +++ b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Api.Contracts +{ + using System; + + public class PlatformHealthView + { + public string Status { get; set; } + public string Severity { get; set; } + public PlatformHealthAlert[] Alerts { get; set; } + } + + public class PlatformHealthAlert + { + public Guid Id { get; set; } + public string CheckId { get; set; } + public string Category { get; set; } + public string Message { get; set; } + public DateTime ReportedAt { get; set; } + public string InstanceName { get; set; } + public string Host { get; set; } + public Guid HostId { get; set; } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Api/Contracts/RootUrls.cs b/src/ServiceControl.Api/Contracts/RootUrls.cs index 2e9a2aeb32..f5b3df37ae 100644 --- a/src/ServiceControl.Api/Contracts/RootUrls.cs +++ b/src/ServiceControl.Api/Contracts/RootUrls.cs @@ -21,5 +21,6 @@ public class RootUrls public string ArchivedGroupsUrl { get; set; } public string GetArchiveGroup { get; set; } public string MyRoutesUrl { get; set; } + public string PlatformHealth { get; set; } } } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 69498d4d04..92748b3504 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -57,6 +57,7 @@ PATCH /pendingretries/queues/resolve => ServiceControl.MessageFailures.Api.Resol POST /pendingretries/queues/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(PendingRetryRequest request, CancellationToken cancellationToken) PATCH /pendingretries/resolve => ServiceControl.MessageFailures.Api.ResolveMessagesController:ResolveBy(UniqueMessageIdsModel request, CancellationToken cancellationToken) POST /pendingretries/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(String[] ids, CancellationToken cancellationToken) +GET /platform-health => ServiceControl.PlatformHealth.PlatformHealthController:Get() GET /recoverability/classifiers => ServiceControl.Recoverability.API.FailureGroupsController:GetSupportedClassifiers() GET /recoverability/groups/{classifier?} => ServiceControl.Recoverability.API.FailureGroupsController:GetAllGroups(String classifier, String classifierFilter, CancellationToken cancellationToken) DELETE /recoverability/groups/{groupId:required:minlength(1)}/comment => ServiceControl.Recoverability.API.FailureGroupsController:DeleteComment(String groupId, CancellationToken cancellationToken) diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt index b73f751bba..7dd5575f82 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt @@ -17,5 +17,6 @@ "EventLogItems": "http://localhost/eventlogitems", "ArchivedGroupsUrl": "http://localhost/errors/groups/{classifier?}", "GetArchiveGroup": "http://localhost/archive/groups/id/{groupId}", - "MyRoutesUrl": "http://localhost/my/routes" + "MyRoutesUrl": "http://localhost/my/routes", + "PlatformHealth": "http://localhost/platform-health" } \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs new file mode 100644 index 0000000000..808ed74060 --- /dev/null +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs @@ -0,0 +1,91 @@ +namespace ServiceControl.UnitTests.PlatformHealth +{ + using System; + using NUnit.Framework; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Operations; + using ServiceControl.PlatformHealth; + + [TestFixture] + class PlatformHealthStateTests + { + [Test] + public void Health_is_unknown_until_an_internal_check_reports() + { + var health = new PlatformHealthState().GetHealth(); + + Assert.That(health.Status, Is.EqualTo("unknown")); + Assert.That(health.Severity, Is.EqualTo("unknown")); + Assert.That(health.Alerts, Is.Empty); + } + + [Test] + public void Failed_internal_check_is_returned_as_an_alert_and_cleared_on_recovery() + { + var state = new PlatformHealthState(); + var detail = Detail("ServiceControl Primary Instance", hasFailed: true); + + state.Record(detail); + + var failingHealth = state.GetHealth(); + Assert.That(failingHealth.Status, Is.EqualTo("unhealthy")); + Assert.That(failingHealth.Severity, Is.EqualTo("error")); + Assert.That(failingHealth.Alerts, Has.Length.EqualTo(1)); + Assert.That(failingHealth.Alerts[0].CheckId, Is.EqualTo(detail.CustomCheckId)); + Assert.That(failingHealth.Alerts[0].Message, Is.EqualTo(detail.FailureReason)); + Assert.That(failingHealth.Alerts[0].InstanceName, Is.EqualTo(detail.OriginatingEndpoint.Name)); + Assert.That(failingHealth.Alerts[0].HostId, Is.EqualTo(detail.OriginatingEndpoint.HostId)); + + detail.HasFailed = false; + detail.FailureReason = null; + state.Record(detail); + + var recoveredHealth = state.GetHealth(); + Assert.That(recoveredHealth.Status, Is.EqualTo("healthy")); + Assert.That(recoveredHealth.Severity, Is.EqualTo("none")); + Assert.That(recoveredHealth.Alerts, Is.Empty); + } + + [Test] + public void Customer_custom_checks_do_not_affect_platform_health() + { + var state = new PlatformHealthState(); + state.Record(Detail("Customer check", hasFailed: true)); + + var health = state.GetHealth(); + + Assert.That(health.Status, Is.EqualTo("unknown")); + Assert.That(health.Alerts, Is.Empty); + } + + [Test] + public void Audit_internal_checks_are_included_with_their_originating_instance() + { + var state = new PlatformHealthState(); + var detail = Detail("Audit Message Ingestion", hasFailed: true); + detail.OriginatingEndpoint.Name = "ServiceControl.Audit"; + + state.Record(detail); + + var health = state.GetHealth(); + Assert.That(health.Status, Is.EqualTo("unhealthy")); + Assert.That(health.Alerts, Has.Length.EqualTo(1)); + Assert.That(health.Alerts[0].InstanceName, Is.EqualTo("ServiceControl.Audit")); + } + + static CustomCheckDetail Detail(string checkId, bool hasFailed) => new() + { + CustomCheckId = checkId, + Category = "ServiceControl Health", + HasFailed = hasFailed, + FailureReason = hasFailed ? "Check failed" : null, + ReportedAt = new DateTime(2026, 9, 30, 12, 0, 0, DateTimeKind.Utc), + OriginatingEndpoint = new EndpointDetails + { + Name = "ServiceControl", + Host = "localhost", + HostId = Guid.Parse("82E379F4-A5BD-4D83-8B64-70488BC6ED3A") + } + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs index a4677006fb..f768593b63 100644 --- a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs +++ b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs @@ -6,19 +6,23 @@ namespace ServiceControl.CustomChecks using Contracts.CustomChecks; using Infrastructure.DomainEvents; using Microsoft.Extensions.Logging; + using PlatformHealth; using ServiceControl.Persistence; class CustomCheckResultProcessor { - public CustomCheckResultProcessor(IDomainEvents domainEvents, ICustomChecksDataStore store, ILogger logger) + public CustomCheckResultProcessor(IDomainEvents domainEvents, ICustomChecksDataStore store, ILogger logger, PlatformHealthState platformHealthState = null) { this.domainEvents = domainEvents; this.store = store; this.logger = logger; + this.platformHealthState = platformHealthState; } public async Task ProcessResult(CustomCheckDetail checkDetail, CancellationToken cancellationToken = default) { + platformHealthState?.Record(checkDetail); + try { var statusChange = await store.UpdateCustomCheckStatus(checkDetail, cancellationToken); @@ -84,5 +88,6 @@ await domainEvents.Raise(new CustomCheckSucceeded int lastCount; readonly ILogger logger; + readonly PlatformHealthState platformHealthState; } } \ No newline at end of file diff --git a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs index 5e4b20e731..4fb8a175fc 100644 --- a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs +++ b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Particular.ServiceControl; + using PlatformHealth; using ServiceBus.Management.Infrastructure.Settings; using Transports; @@ -34,6 +35,7 @@ public override void Configure(Settings settings, ITransportCustomization transp hostBuilder.Services.AddPlatformConnectionProvider(); } hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); } } } \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4d61e23ca8..7dcc876e5d 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -44,6 +44,7 @@ public Task GetUrls(string baseUrl, CancellationToken cancellationToke ArchivedGroupsUrl = baseUrl + "errors/groups/{classifier?}", GetArchiveGroup = baseUrl + "archive/groups/id/{groupId}", MyRoutesUrl = baseUrl + "my/routes", + PlatformHealth = baseUrl + "platform-health", }; return Task.FromResult(model); diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthController.cs b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs new file mode 100644 index 0000000000..d04382ed4f --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.PlatformHealth +{ + using Microsoft.AspNetCore.Authorization; + using Microsoft.AspNetCore.Mvc; + using ServiceControl.Api.Contracts; + using ServiceControl.Infrastructure.Auth; + + [ApiController] + [Route("api")] + public class PlatformHealthController(PlatformHealthState platformHealthState) : ControllerBase + { + [Authorize(Policy = Permissions.ErrorCustomChecksView)] + [Route("platform-health")] + [HttpGet] + public PlatformHealthView Get() => platformHealthState.GetHealth(); + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthState.cs b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs new file mode 100644 index 0000000000..161494d1ca --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.PlatformHealth +{ + using System; + using System.Collections.Concurrent; + using System.Linq; + using ServiceControl.Api.Contracts; + using ServiceControl.Contracts.CustomChecks; + + public class PlatformHealthState + { + internal void Record(CustomCheckDetail detail) + { + if (!InternalCustomCheckClassification.IsInternal(detail.CustomCheckId)) + { + return; + } + + var id = detail.GetDeterministicId(); + checks[id] = new CheckState + { + Id = id, + CheckId = detail.CustomCheckId, + Category = detail.Category, + HasFailed = detail.HasFailed, + Message = detail.FailureReason, + ReportedAt = detail.ReportedAt, + InstanceName = detail.OriginatingEndpoint.Name, + Host = detail.OriginatingEndpoint.Host, + HostId = detail.OriginatingEndpoint.HostId + }; + } + + public PlatformHealthView GetHealth() + { + var currentChecks = checks.Values.ToArray(); + var failedChecks = currentChecks + .Where(check => check.HasFailed) + .OrderBy(check => check.InstanceName, StringComparer.OrdinalIgnoreCase) + .ThenBy(check => check.CheckId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return new PlatformHealthView + { + Status = currentChecks.Length == 0 ? "unknown" : failedChecks.Length == 0 ? "healthy" : "unhealthy", + Severity = currentChecks.Length == 0 ? "unknown" : failedChecks.Length == 0 ? "none" : "error", + Alerts = failedChecks.Select(check => new PlatformHealthAlert + { + Id = check.Id, + CheckId = check.CheckId, + Category = check.Category, + Message = check.Message, + ReportedAt = check.ReportedAt, + InstanceName = check.InstanceName, + Host = check.Host, + HostId = check.HostId + }).ToArray() + }; + } + + readonly ConcurrentDictionary checks = new(); + + class CheckState + { + public Guid Id { get; init; } + public string CheckId { get; init; } + public string Category { get; init; } + public bool HasFailed { get; init; } + public string Message { get; init; } + public DateTime ReportedAt { get; init; } + public string InstanceName { get; init; } + public string Host { get; init; } + public Guid HostId { get; init; } + } + } +} \ No newline at end of file From 8ecd9bba071490ad84e032ccae79ff686820ed6b Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Mon, 5 Oct 2026 12:13:00 -0400 Subject: [PATCH 2/8] Separate internal checks from the custom checks --- docs/platform-health.md | 76 +++- .../When_custom_checks_are_classified.cs | 27 +- .../When_authentication_is_enabled.cs | 35 +- .../When_the_configuration_page_is_read.cs | 5 + .../Contracts/PlatformHealthView.cs | 43 ++ src/ServiceControl.Api/IPlatformHealthApi.cs | 11 + .../API/APIApprovals.cs | 20 +- .../Infrastructure/WebApi/RootController.cs | 7 +- .../When_inspecting_platform_health.cs | 99 +++++ .../API/APIApprovals.cs | 22 +- .../APIApprovals.HttpApiRoutes.approved.txt | 2 +- .../Licensing/ActiveLicenseTests.cs | 45 ++ .../PlatformHealth/PlatformHealthApiTests.cs | 420 ++++++++++++++++++ .../PlatformHealthStateTests.cs | 116 +++++ .../RemoteInstanceHttpClientTests.cs | 106 +++++ .../CustomChecks/CustomChecksComponent.cs | 2 + .../Infrastructure/Api/ConfigurationApi.cs | 23 +- ...moteInstanceServiceCollectionExtensions.cs | 2 +- .../Licensing/LicenseController.cs | 33 +- .../Licensing/LicenseInfoProvider.cs | 44 ++ src/ServiceControl/PlatformHealth.http | 14 + .../PlatformHealth/PlatformHealthApi.cs | 277 ++++++++++++ .../PlatformHealthController.cs | 11 +- .../PlatformHealth/PlatformHealthState.cs | 19 +- .../ServiceControlApiHostBuilderExtensions.cs | 2 + 25 files changed, 1399 insertions(+), 62 deletions(-) create mode 100644 src/ServiceControl.Api/IPlatformHealthApi.cs create mode 100644 src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs create mode 100644 src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs create mode 100644 src/ServiceControl/Licensing/LicenseInfoProvider.cs create mode 100644 src/ServiceControl/PlatformHealth.http create mode 100644 src/ServiceControl/PlatformHealth/PlatformHealthApi.cs diff --git a/docs/platform-health.md b/docs/platform-health.md index a1f887100e..737d019ca6 100644 --- a/docs/platform-health.md +++ b/docs/platform-health.md @@ -1,11 +1,77 @@ # Platform Health API -ServiceControl exposes `GET /api/platform-health` for ServicePulse to read ServiceControl's internal health signals without treating them as customer custom checks. The API root response advertises the endpoint through `PlatformHealth`. +ServiceControl exposes `GET /api/platform-health` for the ServiceControl-owned data on ServicePulse's Platform Health page. The API root advertises its URL in `platform_health`. The response uses the existing snake_case JSON convention and omits unknown nullable fields. -The response contains an overall `status` and `severity`, plus an `alerts` array for currently failing internal checks. Each alert includes the check id, category, failure message, report time, instance name, host, and host id. Status is `unknown` before any internal check has reported, `healthy` when internal checks have reported and none are failing, and `unhealthy` when one or more are failing. Severity is `unknown`, `none`, or `error` for those respective states. +The public motivation is [ServiceControl #5860](https://github.com/Particular/ServiceControl/issues/5860). The consumer data requirements were checked against [ServicePulse's Platform Health store](https://github.com/Particular/ServicePulse/blob/e2688743d23fe5a4b835d4cff128ad12da87d34c/src/Frontend/src/stores/PlatformHealthStore.ts) and [platform model](https://github.com/Particular/ServicePulse/blob/e2688743d23fe5a4b835d4cff128ad12da87d34c/src/Frontend/src/resources/PlatformModel.ts). -Health state is held in memory by the ServiceControl process. Each new report replaces the current state for that check, so a successful report clears its alert. The endpoint does not currently age out checks that stop reporting; a previously reported failure remains until that check reports again or the process restarts. ServicePulse should retain its custom-check fallback for older ServiceControl versions during rollout. +## Response -This initial response does not include version or license/upgrade information. It also does not replace `/api/customchecks` or the existing custom-check integration events. Internally, the transitional implementation still recognizes shipped checks through `InternalCustomCheckClassification`; removing that classification requires a coordinated contract and migration for audit-originated health reports. +The existing `status`, `severity`, and `alerts` fields remain, with additive `instances` and `license` sections. -The endpoint uses the existing `error:customchecks:view` authorization policy. Container liveness and readiness remain separate at `/health` and `/health/ready`. \ No newline at end of file +### Instances + +`instances` contains the primary followed by every distinct configured remote, even when no check has reported or a remote cannot be reached. Remotes are ordered by stable ID, not by the order that their requests complete. + +| Field | Meaning and source | +| --- | --- | +| `id` | Existing URL-derived ServiceControl instance ID; independent of display name and row position | +| `name` | Configured instance name; a never-observed remote falls back to its URI hostname | +| `kind`, `role` | `error` / `primary-error`, `error` / `remote-error`, `audit` / `remote-audit`, or `unknown` / `remote-unknown` | +| `api_url` | Request-facing primary URL, honoring forwarded scheme, host and prefix; configured remote URL with its virtual directory preserved | +| `version` | Installed local version or remote `X-Particular-Version`; absent when unknown, never replaced with the primary's version | +| `host_id` | Actual reporting host identity from the local NServiceBus host or remote configuration; absent on older remotes | +| `health` | `healthy` for reachable instances without an associated failure, `degraded` for reachable instances with failures, `unavailable` for failed probes | +| `observed_at` | UTC timestamp for the current refresh, from the injected clock | +| `metadata_observed_at` | Timestamp of the last successful metadata observation; differs from `observed_at` during an outage | +| `health_signals_status` | `reported`, `unreported`, `disabled`, or `ambiguous`; not a guarantee that every possible check has run | +| `last_reported_at` | Latest associated check timestamp, including successful reports; distinct from HTTP observation time | +| `issues` | Associated failed internal checks, with the same fields as root alerts | +| `transport_type`, `error_queue`, `error_log_queue`, `forward_error_messages` | Available transport configuration; a known `false` forwarding setting is preserved | +| `audit_queue`, `audit_log_queue`, `forward_audit_messages` | Available audit transport configuration | +| `error_retention_period`, `audit_retention_period` | Available retention durations in the existing TimeSpan JSON format, for example `14.00:00:00` | + +Primary and audit `/api/configuration` (also `/api/instance-info`) include `instance_type` and `host.host_id`. Primary configuration additionally reports `health_checks_enabled`. Older remotes without `instance_type` are identified only when their retention configuration establishes the type. A never-observed, unreachable remote is explicitly unknown, not assumed to be an audit instance. + +Remote probes use the registered named HTTP clients and their query timeout. Non-success status codes, empty or malformed configuration, and connection failures do not produce healthy rows. An outage retains the last successful metadata in memory, clearly dated by `metadata_observed_at`. Other rows and the license section still return. Caller cancellation propagates instead of returning partial success. No recursive platform-health requests are made to other primaries. + +### Issues and summary + +Each failed check has `id`, `check_id`, `category`, `message`, `reported_at`, `instance_name`, `host`, and `host_id`. An associated issue also has `instance_id`. + +Association uses case-insensitive instance name plus reporting host ID. A legacy remote without a host ID can use a name match only when there is one matching inventory row and one reporting host with that name. Ambiguous or unmatched reports remain in root `alerts` without `instance_id`; they are never assigned to several rows. Consumers should retain a place to display those unassigned alerts. + +The legacy summary describes captured checks, not the whole browser-visible platform: `status` is `unknown` before any internal report, `healthy` when none are failing, and `unhealthy` when at least one is failing. Its corresponding `severity` values are `unknown`, `none`, and `error`. ServicePulse should use per-instance health for page severity and combine it with its independently observed monitoring state. The legacy summary does not account for monitoring, browser connectivity, license expiry, or available upgrades. + +Check state is process-local. Reports older than a check's latest `reported_at` are ignored; a newer successful report clears that failure. Reports do not expire: different checks have different schedules, including one-shot checks. After restart, check observations and last-known remote metadata are initially empty. `healthy` therefore means reachable without a known associated failure, not proof of complete or fresh check coverage. An unreachable process cannot report its own browser-facing unavailability in a successful response. + +### License + +`license.availability` is `available` after a successful refresh and `unavailable` when license details cannot be refreshed. An unavailable license never claims to be valid and does not suppress instance health. + +The available summary includes `status`, `license_status`, `license_type`, `trial_license`, optional `expiration_date` and `upgrade_protection_expiration`, and `license_extension_url`. It preserves the existing license status values for subscription, trial and upgrade-protection gates. Renewal URLs share the `/api/license` mapping with `clientName=servicepulse`, including MassTransit evaluation/subscription links. `has_mass_transit_connector` reports connector presence. Customer registration, licensed products and endpoint-license metadata are not included. + +## ServicePulse integration + +The endpoint supplies primary/remote inventory, installed versions, configuration, issues, and the license summary. Updating this endpoint does not update the ServicePulse consumer automatically; the consumer must map `instances` and `license` into its stores and support unknown instance types and unassigned alerts. + +ServicePulse continues to own: + +- Its running frontend version and ServicePulse row. +- The browser-selected monitoring URL, monitoring requests, and monitoring row. +- Browser-to-primary connectivity failures, including when this endpoint cannot be reached. +- Release-feed requests, latest-version comparison, release links, upgrade badges, and outdated-only navigation state. Installed version and license validity are not a guarantee that an upgrade path is supported. +- The customer-check fetch for the support export. Export combines this response, browser-owned rows, and the existing custom-check results. Customer checks never affect platform health. + +Keep the legacy consumer fallback for supported ServiceControl versions without the advertised capability. Do not interpret `401`, `403`, a timeout, or a failed response as an absent capability. The existing custom-check API, classification, notifications, and integration events remain unchanged. Audit health still arrives through the current custom-check reporting transport; this increment does not remove that dependency or introduce replacement events. + +## Access + +The endpoint retains `error:customchecks:view`, granted by the existing reader, writer and admin roles. No permission or authentication behavior is changed. With authentication disabled it is anonymous. With authentication and RBAC enabled, anonymous callers receive `401` and authenticated callers without a read role receive `403`. + +Known shared-policy limitation: authentication enabled with RBAC disabled currently resolves named permissions to allow-all, so the expanded response, including the license summary, can be accessed anonymously. Fixing that policy is separate work. Container liveness and readiness remain separate at `/health` and `/health/ready`. + +## Verification + +For manual requests, use [PlatformHealth.http](../src/ServiceControl/PlatformHealth.http). Its authenticated request reads an existing bearer token from `SERVICECONTROL_ACCESS_TOKEN`; do not store credentials in the request file. + +`PlatformHealthStateTests` and `PlatformHealthApiTests` cover snapshot ordering, delayed reports, serialization, source projection, identity ambiguity, offline metadata, partial failures, license mapping and cancellation. Remote-client tests cover HTTP status, malformed responses and prefixed URLs. Shared acceptance scenarios exercise the real root/configuration/health responses and preserve custom-check behavior. The multi-instance `When_inspecting_platform_health` scenario exercises real audit check delivery, issue ownership, recovery and unavailable inventory. OIDC acceptance scenarios cover the existing read-role policy. \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs index 8c3e1bde6a..da388165c1 100644 --- a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs +++ b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs @@ -2,6 +2,7 @@ namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks { using System; using System.Linq; + using System.Text.Json; using System.Threading; using System.Threading.Tasks; using AcceptanceTesting; @@ -12,6 +13,8 @@ namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api.Contracts; + using ServiceControl.Infrastructure; + using ApiSerializerOptions = global::ServiceControl.Infrastructure.WebApi.SerializerOptions; using CustomCheckView = global::ServiceControl.Contracts.CustomChecks.CustomCheckView; using CheckStatus = global::ServiceControl.Persistence.Status; @@ -30,7 +33,9 @@ public async Task Internal_checks_are_flagged_internal_and_endpoint_checks_are_n CustomCheckView internalCheck = null; CustomCheckView endpointCheck = null; PlatformHealthView platformHealth = null; + RootUrls urls = null; string wireBody = null; + string healthWireBody = null; await Define() .WithEndpoint() @@ -51,13 +56,20 @@ await Define() if (internalCheck != null && endpointCheck != null && platformHealth == null) { - platformHealth = await this.TryGet("/api/platform-health"); + urls = await this.TryGet("/api"); + using var response = await this.GetRaw("/api/platform-health"); + healthWireBody = await response.Content.ReadAsStringAsync(); + platformHealth = JsonSerializer.Deserialize(healthWireBody, ApiSerializerOptions.Default); } return internalCheck != null && endpointCheck != null && wireBody != null && platformHealth != null; }) .Run(); + using var healthJson = JsonDocument.Parse(healthWireBody); + var instanceJson = healthJson.RootElement.GetProperty("instances")[0]; + var instance = platformHealth.Instances.Single(item => item.Role == "primary-error"); + using (Assert.EnterMultipleScope()) { Assert.That(internalCheck, Is.Not.Null, "primary internal checks report at startup; nothing was found"); @@ -66,6 +78,19 @@ await Define() Assert.That(endpointCheck, Is.Not.Null); Assert.That(endpointCheck.Internal, Is.False); Assert.That(platformHealth.Alerts, Has.None.Matches(alert => alert.CheckId == "MyCustomCheckId")); + Assert.That(urls.PlatformHealth, Does.EndWith("/api/platform-health")); + Assert.That(instance.Id, Is.EqualTo(Settings.InstanceId)); + Assert.That(instance.Name, Is.EqualTo(Settings.InstanceName)); + Assert.That(instance.HostId, Is.EqualTo(internalCheck.OriginatingEndpoint.HostId)); + Assert.That(instance.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(instance.Version, Is.EqualTo(ServiceControlVersion.GetFileVersion())); + Assert.That(instance.ApiUrl.TrimEnd('/'), Is.EqualTo(urls.PlatformHealth[..^"/platform-health".Length])); + Assert.That(instance.ErrorQueue, Is.EqualTo(Settings.ErrorQueue)); + Assert.That(instance.ErrorRetentionPeriod, Is.EqualTo(Settings.ErrorRetentionPeriod)); + Assert.That(instanceJson.GetProperty("health_signals_status").GetString(), Is.EqualTo("reported")); + Assert.That(instanceJson.GetProperty("forward_error_messages").GetBoolean(), Is.EqualTo(Settings.ForwardErrorMessages)); + Assert.That(healthJson.RootElement.GetProperty("license").GetProperty("availability").GetString(), Is.EqualTo("available")); + Assert.That(platformHealth.License.LicenseStatus, Is.Not.Null.And.Not.Empty); // What the wire actually carries: Assert.That(wireBody, Does.Contain("\"internal\":true"), "internal checks must render internal:true on the wire"); diff --git a/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs b/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs index 3795913478..16e252c37c 100644 --- a/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs +++ b/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs @@ -82,20 +82,19 @@ await OpenIdConnectAssertions.AssertAuthConfigurationResponse( expectedRoleBasedAuthorizationEnabled: true); } - [Test] - public async Task Should_reject_requests_without_bearer_token() + [TestCase("/api/errors")] + [TestCase("/api/platform-health")] + public async Task Should_reject_requests_without_bearer_token(string path) { HttpResponseMessage response = null; _ = await Define() .Done(async ctx => { - // Use /api/errors which does NOT have [AllowAnonymous] so it should require authentication - // Note: /api is marked [AllowAnonymous] for server-to-server configuration fetching response = await OpenIdConnectAssertions.SendRequestWithoutAuth( HttpClient, HttpMethod.Get, - "/api/errors"); + path); return response != null; }) .Run(); @@ -123,22 +122,21 @@ public async Task Should_reject_requests_with_invalid_bearer_token() OpenIdConnectAssertions.AssertUnauthorized(response); } - [Test] - public async Task Should_accept_requests_with_valid_bearer_token() + [TestCase("/api/errors")] + [TestCase("/api/platform-health")] + public async Task Should_accept_requests_with_valid_bearer_token(string path) { HttpResponseMessage response = null; _ = await Define() .Done(async ctx => { - // The "reader" role grants every :view permission, including error:messages:view - // required by /api/errors. Without a role-bearing claim the request would be 403. var validToken = mockOidcServer.GenerateToken( additionalClaims: new[] { new Claim("roles", "reader") }); response = await OpenIdConnectAssertions.SendRequestWithBearerToken( HttpClient, HttpMethod.Get, - "/api/errors", + path, validToken); return response != null; }) @@ -147,6 +145,23 @@ public async Task Should_accept_requests_with_valid_bearer_token() OpenIdConnectAssertions.AssertAuthenticated(response); } + [Test] + public async Task Should_forbid_platform_health_without_a_read_role() + { + HttpResponseMessage response = null; + + await Define() + .Done(async _ => + { + response = await OpenIdConnectAssertions.SendRequestWithBearerToken( + HttpClient, HttpMethod.Get, "/api/platform-health", mockOidcServer.GenerateToken()); + return response != null; + }) + .Run(); + + OpenIdConnectAssertions.AssertForbidden(response); + } + [Test] public async Task Should_reject_requests_with_expired_token() { diff --git a/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs b/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs index f9a6ffe0b1..64e39fb9e0 100644 --- a/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs +++ b/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs @@ -1,5 +1,6 @@ namespace ServiceControl.AcceptanceTests.WebApi { + using System; using System.IO; using System.IO.Compression; using System.Net; @@ -29,6 +30,7 @@ await Define() }) .Run(); + using var json = JsonDocument.Parse(configuration); using (Assert.EnterMultipleScope()) { Assert.That(configuration, Is.EqualTo(instanceInfo), @@ -36,6 +38,9 @@ await Define() Assert.That(configuration, Does.Contain(Settings.InstanceName), "The configuration page names the instance it is describing"); + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("error")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.Not.EqualTo(Guid.Empty)); + Assert.That(json.RootElement.GetProperty("health_checks_enabled").GetBoolean(), Is.False); } } diff --git a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs index ecab8e0ab0..20fc1708dd 100644 --- a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs +++ b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs @@ -7,11 +7,14 @@ public class PlatformHealthView public string Status { get; set; } public string Severity { get; set; } public PlatformHealthAlert[] Alerts { get; set; } + public PlatformHealthInstance[] Instances { get; set; } = []; + public PlatformHealthLicense License { get; set; } = new(); } public class PlatformHealthAlert { public Guid Id { get; set; } + public string InstanceId { get; set; } public string CheckId { get; set; } public string Category { get; set; } public string Message { get; set; } @@ -20,4 +23,44 @@ public class PlatformHealthAlert public string Host { get; set; } public Guid HostId { get; set; } } + +#nullable enable + public sealed record PlatformHealthInstance + { + public required string Id { get; init; } + public required string Name { get; init; } + public required string ApiUrl { get; init; } + public string Kind { get; init; } = "unknown"; + public string Role { get; init; } = "remote-unknown"; + public string? Version { get; init; } + public Guid? HostId { get; init; } + public string Health { get; init; } = "unavailable"; + public DateTimeOffset ObservedAt { get; init; } + public DateTimeOffset? MetadataObservedAt { get; init; } + public string HealthSignalsStatus { get; init; } = "unreported"; + public DateTimeOffset? LastReportedAt { get; init; } + public PlatformHealthAlert[] Issues { get; init; } = []; + public string? TransportType { get; init; } + public string? ErrorQueue { get; init; } + public string? ErrorLogQueue { get; init; } + public bool? ForwardErrorMessages { get; init; } + public string? AuditQueue { get; init; } + public string? AuditLogQueue { get; init; } + public bool? ForwardAuditMessages { get; init; } + public TimeSpan? ErrorRetentionPeriod { get; init; } + public TimeSpan? AuditRetentionPeriod { get; init; } + } + + public sealed record PlatformHealthLicense + { + public string Availability { get; init; } = "unavailable"; + public string? Status { get; init; } + public string? LicenseStatus { get; init; } + public string? LicenseType { get; init; } + public bool? TrialLicense { get; init; } + public DateTimeOffset? ExpirationDate { get; init; } + public DateTimeOffset? UpgradeProtectionExpiration { get; init; } + public string? LicenseExtensionUrl { get; init; } + public bool HasMassTransitConnector { get; init; } + } } \ No newline at end of file diff --git a/src/ServiceControl.Api/IPlatformHealthApi.cs b/src/ServiceControl.Api/IPlatformHealthApi.cs new file mode 100644 index 0000000000..aad4036471 --- /dev/null +++ b/src/ServiceControl.Api/IPlatformHealthApi.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Api +{ + using System.Threading; + using System.Threading.Tasks; + using Contracts; + + public interface IPlatformHealthApi + { + Task GetHealth(string baseUrl, CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs b/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs index 94e2302234..a545839986 100644 --- a/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Reflection; using System.Text; + using System.Text.Json; using Audit.Infrastructure.Settings; using Audit.Infrastructure.WebApi; using Microsoft.AspNetCore.Authorization; @@ -13,6 +14,7 @@ using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Mvc.Routing; using Microsoft.AspNetCore.Routing; + using NServiceBus.Hosting; using NUnit.Framework; using Particular.Approvals; using ServiceControl.Hosting.Auth; @@ -30,7 +32,7 @@ public void RootPathValue() var settings = CreateTestSettings(); - var controller = new RootController(settings) + var controller = new RootController(settings, new HostInformation(Guid.Empty, "localhost")) { ControllerContext = controllerContext, Url = new UrlHelper(actionContext) @@ -41,6 +43,22 @@ public void RootPathValue() Approver.Verify(result.Value); } + [Test] + public void Configuration_reports_the_instance_type_and_reporting_host_id() + { + var hostId = Guid.NewGuid(); + var controller = new RootController(CreateTestSettings(), new HostInformation(hostId, "localhost")); + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(controller.Config().Value, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("audit")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.EqualTo(hostId)); + } + } + [Test] public void HttpApiRoutes() { diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs index 6334d612a6..8a6f72d379 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http.Extensions; using Microsoft.AspNetCore.Mvc; + using NServiceBus.Hosting; using Settings; // the /api endpoint is used for service-to-service communication. This currently needs to be anonymous @@ -12,9 +13,10 @@ [Route("api")] public class RootController : ControllerBase { - public RootController(Settings settings) + public RootController(Settings settings, HostInformation hostInformation) { this.settings = settings; + this.hostInformation = hostInformation; } [Route("")] @@ -50,9 +52,11 @@ public OkObjectResult Config() { object content = new { + InstanceType = "audit", Host = new { settings.InstanceName, + hostInformation.HostId, Logging = new { settings.LoggingSettings.LogPath, @@ -87,6 +91,7 @@ public OkObjectResult Config() } readonly Settings settings; + readonly HostInformation hostInformation; public class RootUrls { diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs new file mode 100644 index 0000000000..de4cd93809 --- /dev/null +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs @@ -0,0 +1,99 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.Infrastructure; + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using AcceptanceTesting; +using Api.Contracts; +using Audit.Auditing; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.AcceptanceTesting; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using TestSupport; + +class When_inspecting_platform_health : AcceptanceTest +{ + [Test] + public async Task Should_show_audit_issues_recovery_and_unavailable_configured_instances() + { + const string checkId = "Audit Message Ingestion Process"; + const string failure = "Audit ingestion interrupted for the platform health scenario"; + var auditState = new AuditIngestionCustomCheck.State(); + auditState.ReportError(failure); + var offline = new RemoteInstanceSetting("http://offline:12121"); + + AuditHostBuilderCustomization = builder => builder.Services.AddSingleton(auditState); + CustomServiceControlPrimarySettings = settings => settings.RemoteInstances = [.. settings.RemoteInstances, offline]; + PrimaryHostBuilderCustomization = builder => builder.Services.AddKeyedSingleton>( + offline.InstanceId, () => new UnavailableHandler()); + + PlatformHealthView failing = null; + PlatformHealthView recovered = null; + + await Define() + .Do("Read the platform inventory and failing audit report", async context => + { + failing = await this.TryGet("/api/platform-health", instanceName: ServiceControlInstanceName); + context.LastAlerts = string.Join(", ", failing?.Alerts.Select(alert => $"{alert.InstanceName}: {alert.CheckId}") ?? []); + return failing?.Alerts.Any(alert => alert.CheckId == checkId) == true; + }) + .Do("Observe audit recovery in platform health", async context => + { + if (!context.RecoveryRequested) + { + auditState.Clear(); + context.RecoveryRequested = true; + } + + recovered = await this.TryGet("/api/platform-health", instanceName: ServiceControlInstanceName); + context.LastAlerts = string.Join(", ", recovered?.Alerts.Select(alert => $"{alert.InstanceName}: {alert.CheckId}") ?? []); + return recovered != null && recovered.Alerts.All(alert => alert.CheckId != checkId); + }) + .Done(_ => true) + .Run(); + + var primary = failing.Instances.Single(instance => instance.Role == "primary-error"); + var audit = failing.Instances.Single(instance => instance.Role == "remote-audit"); + var unavailable = failing.Instances.Single(instance => instance.Id == offline.InstanceId); + var issue = failing.Alerts.Single(alert => alert.CheckId == checkId); + var recoveredAudit = recovered.Instances.Single(instance => instance.Id == audit.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(failing.Instances, Has.Length.EqualTo(3)); + Assert.That(primary.Name, Is.EqualTo(ServiceControlInstanceName)); + Assert.That(primary.Issues, Has.None.Matches(alert => alert.CheckId == checkId)); + Assert.That(audit.Name, Is.EqualTo(ServiceControlAuditInstanceName)); + Assert.That(audit.HostId, Is.EqualTo(issue.HostId)); + Assert.That(audit.Health, Is.EqualTo("degraded")); + Assert.That(audit.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(audit.Version, Is.Not.Null.And.Not.Empty); + Assert.That(audit.AuditRetentionPeriod, Is.Not.Null); + Assert.That(audit.Issues, Has.Some.Matches(alert => alert.Id == issue.Id)); + Assert.That(issue.InstanceId, Is.EqualTo(audit.Id)); + Assert.That(issue.Message, Is.EqualTo(failure)); + Assert.That(unavailable.Kind, Is.EqualTo("unknown")); + Assert.That(unavailable.Health, Is.EqualTo("unavailable")); + Assert.That(unavailable.Version, Is.Null); + Assert.That(recoveredAudit.Health, Is.EqualTo("healthy")); + Assert.That(recoveredAudit.Issues, Is.Empty); + Assert.That(recovered.License.Availability, Is.EqualTo("available")); + } + } + + sealed class UnavailableHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) => + throw new HttpRequestException(HttpRequestError.ConnectionError); + } + + class Context : ScenarioContext, ISequenceContext + { + public int Step { get; set; } + public string LastAlerts { get; set; } + public bool RecoveryRequested { get; set; } + } +} \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/API/APIApprovals.cs b/src/ServiceControl.UnitTests/API/APIApprovals.cs index 04d17fc409..d103264a2e 100644 --- a/src/ServiceControl.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.UnitTests/API/APIApprovals.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Reflection; using System.Text; + using System.Text.Json; using System.Threading.Tasks; using Api.Contracts; using Microsoft.AspNetCore.Authorization; @@ -15,6 +16,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Logging.Abstractions; using NServiceBus.CustomChecks; + using NServiceBus.Hosting; using NUnit.Framework; using Particular.Approvals; using Particular.ServiceControl.Licensing; @@ -38,7 +40,8 @@ public async Task RootPathValue() new ActiveLicense(null, NullLogger.Instance) { IsValid = true }, new Settings(), null, - new MassTransitConnectorHeartbeatStatus()); + new MassTransitConnectorHeartbeatStatus(), + new HostInformation(Guid.Empty, "localhost")); var controller = new RootController(configurationApi) { @@ -51,6 +54,23 @@ public async Task RootPathValue() Approver.Verify(result); } + [Test] + public async Task Configuration_reports_the_instance_type_and_reporting_host_id() + { + var hostId = Guid.NewGuid(); + var configuration = new ConfigurationApi(null, new Settings { DisableHealthChecks = true }, null, + new MassTransitConnectorHeartbeatStatus(), new HostInformation(hostId, "localhost")); + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(await configuration.GetConfig(), SerializerOptions.Default)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("error")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.EqualTo(hostId)); + Assert.That(json.RootElement.GetProperty("health_checks_enabled").GetBoolean(), Is.False); + } + } + [Test] public void HttpApiRoutes() { diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 92748b3504..994be1bead 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -57,7 +57,7 @@ PATCH /pendingretries/queues/resolve => ServiceControl.MessageFailures.Api.Resol POST /pendingretries/queues/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(PendingRetryRequest request, CancellationToken cancellationToken) PATCH /pendingretries/resolve => ServiceControl.MessageFailures.Api.ResolveMessagesController:ResolveBy(UniqueMessageIdsModel request, CancellationToken cancellationToken) POST /pendingretries/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(String[] ids, CancellationToken cancellationToken) -GET /platform-health => ServiceControl.PlatformHealth.PlatformHealthController:Get() +GET /platform-health => ServiceControl.PlatformHealth.PlatformHealthController:Get(CancellationToken cancellationToken) GET /recoverability/classifiers => ServiceControl.Recoverability.API.FailureGroupsController:GetSupportedClassifiers() GET /recoverability/groups/{classifier?} => ServiceControl.Recoverability.API.FailureGroupsController:GetAllGroups(String classifier, String classifierFilter, CancellationToken cancellationToken) DELETE /recoverability/groups/{groupId:required:minlength(1)}/comment => ServiceControl.Recoverability.API.FailureGroupsController:DeleteComment(String groupId, CancellationToken cancellationToken) diff --git a/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs b/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs index 00a93ffc04..7ce334c736 100644 --- a/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs +++ b/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs @@ -4,9 +4,14 @@ using System.Threading; using System.Threading.Tasks; using LicenseManagement; + using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; using Particular.ServiceControl.Licensing; using Persistence; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Connector.MassTransit; + using ServiceControl.Licensing; + using ServiceControl.Monitoring.HeartbeatMonitoring; [TestFixture] public class ActiveLicenseTests @@ -60,6 +65,46 @@ public async Task Accepts_license_base_on_the_db_value_only() Assert.That(checkedDetails.HasLicenseExpired, Is.False); } + [TestCase(false, true, "https://particular.net/extend-your-trial?p=servicepulse")] + [TestCase(true, true, "https://particular.net/license/mt?p=servicepulse&t=0")] + [TestCase(true, false, "https://particular.net/license/mt?p=servicepulse&t=1")] + public async Task License_information_preserves_the_existing_mapping_and_renewal_links(bool massTransit, bool evaluation, string expectedUrl) + { + var details = LicenseDetails.TrialFromEndDate(new DateOnly(2026, 9, 30)); + var active = new ActiveLicense(null, NullLogger.Instance) + { + Details = details, + IsValid = false, + IsEvaluation = evaluation + }; + var connector = new MassTransitConnectorHeartbeatStatus(); + if (massTransit) + { + connector.Update(new MassTransitConnectorHeartbeat + { + Version = "1.0.0", + ErrorQueues = [], + Logs = [], + SentDateTimeOffset = DateTimeOffset.MinValue + }); + } + + var provider = new LicenseInfoProvider(active, new Settings { InstanceName = "Primary" }, connector); + var result = await provider.GetLicense(false, "servicepulse"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.LicenseExtensionUrl, Is.EqualTo(expectedUrl)); + Assert.That(result.LicenseStatus, Is.EqualTo(details.Status)); + Assert.That(result.Status, Is.EqualTo("invalid")); + Assert.That(result.TrialLicense, Is.True); + Assert.That(result.LicenseType, Is.EqualTo(details.LicenseType)); + Assert.That(result.ExpirationDate, Is.EqualTo(details.ExpirationDate?.ToString("O"))); + Assert.That(result.UpgradeProtectionExpiration, Is.Empty); + Assert.That(result.InstanceName, Is.EqualTo("Primary")); + } + } + class FakeDataProvider : ITrialLicenseDataProvider { TrialMetadata metadata; diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs new file mode 100644 index 0000000000..b2bee5bde4 --- /dev/null +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs @@ -0,0 +1,420 @@ +namespace ServiceControl.UnitTests.PlatformHealth; + +using System; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Time.Testing; +using NServiceBus.Hosting; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Api; +using ServiceControl.Api.Contracts; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Infrastructure; +using ServiceControl.Licensing; +using ServiceControl.Monitoring.HeartbeatMonitoring; +using ServiceControl.Operations; +using ServiceControl.PlatformHealth; + +[TestFixture] +class PlatformHealthApiTests +{ + [SetUp] + public void SetUp() + { + settings = new Settings + { + InstanceName = "Primary", + TransportType = "RabbitMQ", + ErrorQueue = "error", + ErrorLogQueue = "error.log", + ForwardErrorMessages = false, + RemoteInstances = [] + }; + state = new PlatformHealthState(); + configuration = new FakeConfigurationApi(); + licensing = new FakeLicenseInfoProvider(); + clock = new FakeTimeProvider(new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero)); + api = new PlatformHealthApi(settings, new HostInformation(PrimaryHostId, "primary-host"), state, + configuration, licensing, new MassTransitConnectorHeartbeatStatus(), clock); + } + + [Test] + public async Task Includes_primary_inventory_configuration_and_version_before_checks_report() + { + var result = await api.GetHealth("https://public/servicecontrol/api"); + var primary = result.Instances.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Status, Is.EqualTo("unknown")); + Assert.That(primary.Id, Is.EqualTo(settings.InstanceId)); + Assert.That(primary.Name, Is.EqualTo("Primary")); + Assert.That(primary.Kind, Is.EqualTo("error")); + Assert.That(primary.Role, Is.EqualTo("primary-error")); + Assert.That(primary.ApiUrl, Is.EqualTo("https://public/servicecontrol/api/")); + Assert.That(primary.Version, Is.EqualTo(ServiceControlVersion.GetFileVersion())); + Assert.That(primary.Health, Is.EqualTo("healthy")); + Assert.That(primary.HealthSignalsStatus, Is.EqualTo("unreported")); + Assert.That(primary.LastReportedAt, Is.Null); + Assert.That(primary.ObservedAt, Is.EqualTo(clock.GetUtcNow())); + Assert.That(primary.HostId, Is.EqualTo(PrimaryHostId)); + Assert.That(primary.TransportType, Is.EqualTo("RabbitMQ")); + Assert.That(primary.ErrorQueue, Is.EqualTo("error")); + Assert.That(primary.ErrorLogQueue, Is.EqualTo("error.log")); + Assert.That(primary.ForwardErrorMessages, Is.False); + Assert.That(primary.ErrorRetentionPeriod, Is.EqualTo(settings.ErrorRetentionPeriod)); + Assert.That(licensing.Refresh, Is.True); + Assert.That(licensing.ClientName, Is.EqualTo("servicepulse")); + } + } + + [Test] + public async Task Controller_uses_the_public_scheme_host_and_proxy_prefix() + { + var context = new DefaultHttpContext(); + context.Request.Scheme = "https"; + context.Request.Host = new HostString("public.example", 8443); + context.Request.PathBase = "/servicecontrol"; + var controller = new PlatformHealthController(api) { ControllerContext = new ControllerContext { HttpContext = context } }; + + var result = await controller.Get(); + + Assert.That(result.Instances[0].ApiUrl, Is.EqualTo("https://public.example:8443/servicecontrol/api/")); + } + + [Test] + public async Task Correlates_issues_by_host_identity_and_clears_them_after_recovery() + { + var remote = new RemoteInstanceSetting("https://audit"); + settings.RemoteInstances = [remote]; + configuration.Remotes = [Remote(remote, "Primary", AuditHostId)]; + var report = Report("Primary", AuditHostId); + state.Record(report); + + var failing = await api.GetHealth("https://primary/api/"); + var audit = failing.Instances.Single(instance => instance.Id == remote.InstanceId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(failing.Instances[0].Issues, Is.Empty); + Assert.That(audit.Kind, Is.EqualTo("audit")); + Assert.That(audit.Role, Is.EqualTo("remote-audit")); + Assert.That(audit.Health, Is.EqualTo("degraded")); + Assert.That(audit.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(audit.AuditRetentionPeriod, Is.EqualTo(TimeSpan.FromDays(7))); + Assert.That(audit.Issues, Has.Length.EqualTo(1)); + Assert.That(audit.Issues[0].InstanceId, Is.EqualTo(remote.InstanceId)); + Assert.That(failing.Alerts[0].InstanceId, Is.EqualTo(remote.InstanceId)); + } + + report.HasFailed = false; + report.ReportedAt = report.ReportedAt.AddMinutes(1); + state.Record(report); + var recovered = await api.GetHealth("https://primary/api/"); + + Assert.That(recovered.Instances.Single(instance => instance.Id == remote.InstanceId).Health, Is.EqualTo("healthy")); + Assert.That(recovered.Alerts, Is.Empty); + } + + [Test] + public async Task Keeps_configured_offline_instances_and_last_observed_metadata() + { + var remote = new RemoteInstanceSetting("https://audit/prefix"); + settings.RemoteInstances = [remote]; + var first = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Instances, Has.Length.EqualTo(2)); + Assert.That(first.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(first.Instances[1].Kind, Is.EqualTo("unknown")); + Assert.That(first.Instances[1].Version, Is.Null); + } + + configuration.Remotes = [Remote(remote, "Audit", AuditHostId)]; + var online = await api.GetHealth("https://primary/api/"); + configuration.Remotes = []; + clock.Advance(TimeSpan.FromMinutes(1)); + var offline = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(offline.Instances[1].Name, Is.EqualTo("Audit")); + Assert.That(offline.Instances[1].Kind, Is.EqualTo("audit")); + Assert.That(offline.Instances[1].Version, Is.EqualTo("6.10.0")); + Assert.That(offline.Instances[1].ApiUrl, Is.EqualTo("https://audit/prefix/api/")); + Assert.That(offline.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(offline.Instances[1].MetadataObservedAt, Is.EqualTo(online.Instances[1].ObservedAt)); + Assert.That(offline.Instances[1].ObservedAt, Is.EqualTo(clock.GetUtcNow())); + } + } + + [Test] + public async Task Remote_error_instances_keep_configuration_and_recover_connectivity() + { + var remote = new RemoteInstanceSetting("https://remote-error/prefix"); + settings.RemoteInstances = [remote]; + var metadata = Remote(remote, "Remote error", AuditHostId); + metadata.Configuration["instance_type"] = "error"; + metadata.Configuration["data_retention"] = JsonNode.Parse("""{"error_retention_period":"21.00:00:00"}"""); + metadata.Configuration["transport"] = JsonNode.Parse(""" + {"transport_type":"RabbitMQ","error_queue":"remote.error","error_log_queue":"remote.log","forward_error_messages":false} + """); + configuration.Remotes = [metadata]; + state.Record(Report("Remote error", AuditHostId)); + + var result = await api.GetHealth("https://primary/api/"); + var instance = result.Instances[1]; + using (Assert.EnterMultipleScope()) + { + Assert.That(instance.Kind, Is.EqualTo("error")); + Assert.That(instance.Role, Is.EqualTo("remote-error")); + Assert.That(instance.Health, Is.EqualTo("degraded")); + Assert.That(instance.ErrorQueue, Is.EqualTo("remote.error")); + Assert.That(instance.ErrorLogQueue, Is.EqualTo("remote.log")); + Assert.That(instance.ForwardErrorMessages, Is.False); + Assert.That(instance.ErrorRetentionPeriod, Is.EqualTo(TimeSpan.FromDays(21))); + Assert.That(instance.AuditRetentionPeriod, Is.Null); + } + + metadata.Status = "unavailable"; + Assert.That((await api.GetHealth("https://primary/api/")).Instances[1].Health, Is.EqualTo("unavailable")); + metadata.Status = "online"; + metadata.Version = "6.11.0"; + clock.Advance(TimeSpan.FromMinutes(1)); + var recovered = (await api.GetHealth("https://primary/api/")).Instances[1]; + using (Assert.EnterMultipleScope()) + { + Assert.That(recovered.Health, Is.EqualTo("degraded")); + Assert.That(recovered.Version, Is.EqualTo("6.11.0")); + Assert.That(recovered.MetadataObservedAt, Is.EqualTo(clock.GetUtcNow())); + } + } + + [Test] + public async Task Malformed_optional_remote_metadata_does_not_hide_other_instances() + { + var malformed = new RemoteInstanceSetting("https://malformed"); + var valid = new RemoteInstanceSetting("https://valid"); + settings.RemoteInstances = [malformed, valid]; + var badMetadata = Remote(malformed, "Malformed audit", AuditHostId); + badMetadata.Configuration["data_retention"]["audit_retention_period"] = "not a duration"; + configuration.Remotes = [badMetadata, Remote(valid, "Valid audit", Guid.NewGuid())]; + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances, Has.Length.EqualTo(3)); + Assert.That(result.Instances.Single(instance => instance.Id == malformed.InstanceId).Health, Is.EqualTo("unavailable")); + Assert.That(result.Instances.Single(instance => instance.Id == valid.InstanceId).Health, Is.EqualTo("healthy")); + Assert.That(result.License.Availability, Is.EqualTo("available")); + } + } + + [TestCase(false)] + [TestCase(true)] + public async Task Legacy_name_matching_requires_unique_inventory_and_reporting_host(bool duplicateInventory) + { + var first = new RemoteInstanceSetting("https://first"); + var second = new RemoteInstanceSetting("https://second"); + settings.RemoteInstances = duplicateInventory ? [first, second] : [first]; + configuration.Remotes = duplicateInventory ? [Remote(first, "Audit"), Remote(second, "Audit")] : [Remote(first, "Audit")]; + state.Record(Report("Audit", AuditHostId)); + if (!duplicateInventory) + { + state.Record(Report("Audit", Guid.NewGuid())); + } + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances.Skip(1).Select(instance => instance.HealthSignalsStatus), Is.All.EqualTo("ambiguous")); + Assert.That(result.Instances.SelectMany(instance => instance.Issues), Is.Empty); + Assert.That(result.Alerts.Select(alert => alert.InstanceId), Is.All.Null); + } + } + + [Test] + public async Task Legacy_configuration_infers_kind_and_matches_unique_names_without_inventing_versions() + { + var remote = new RemoteInstanceSetting("https://audit"); + settings.RemoteInstances = [remote]; + var legacy = Remote(remote, "Audit"); + legacy.Configuration.AsObject().Remove("instance_type"); + legacy.Version = "Unknown"; + configuration.Remotes = [legacy]; + state.Record(Report("audit", AuditHostId)); + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances[1].Kind, Is.EqualTo("audit")); + Assert.That(result.Instances[1].Version, Is.Null); + Assert.That(result.Instances[1].Health, Is.EqualTo("degraded")); + Assert.That(result.Alerts[0].InstanceId, Is.EqualTo(remote.InstanceId)); + } + } + + [Test] + public async Task License_failure_and_remote_failure_do_not_hide_local_health() + { + settings.RemoteInstances = [new RemoteInstanceSetting("https://offline")]; + configuration.Failure = new InvalidOperationException("Remote unavailable"); + licensing.Failure = new InvalidOperationException("License unavailable"); + state.Record(Report("Primary", PrimaryHostId)); + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances, Has.Length.EqualTo(2)); + Assert.That(result.Instances[0].Health, Is.EqualTo("degraded")); + Assert.That(result.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(result.License.Availability, Is.EqualTo("unavailable")); + Assert.That(result.License.Status, Is.Null); + Assert.That(result.License.LicenseStatus, Is.Null); + } + } + + [TestCase("Valid")] + [TestCase("ValidWithExpiringTrial")] + [TestCase("InvalidDueToExpiredTrial")] + [TestCase("InvalidDueToExpiredSubscription")] + [TestCase("InvalidDueToExpiredUpgradeProtection")] + public async Task License_status_and_renewal_fields_preserve_the_license_api_values(string licenseStatus) + { + licensing.Info.LicenseStatus = licenseStatus; + licensing.Info.ExpirationDate = "2026-10-01T00:00:00.0000000Z"; + licensing.Info.UpgradeProtectionExpiration = ""; + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.License.Availability, Is.EqualTo("available")); + Assert.That(result.License.LicenseStatus, Is.EqualTo(licenseStatus)); + Assert.That(result.License.LicenseType, Is.EqualTo(licensing.Info.LicenseType)); + Assert.That(result.License.TrialLicense, Is.EqualTo(licensing.Info.TrialLicense)); + Assert.That(result.License.LicenseExtensionUrl, Is.EqualTo(licensing.Info.LicenseExtensionUrl)); + Assert.That(result.License.ExpirationDate, Is.EqualTo(new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero))); + Assert.That(result.License.UpgradeProtectionExpiration, Is.Null); + } + } + + [Test] + public async Task Disabled_checks_are_explicit_and_not_mistaken_for_reports() + { + settings.DisableHealthChecks = true; + + var result = await api.GetHealth("https://primary/api/"); + + Assert.That(result.Instances[0].HealthSignalsStatus, Is.EqualTo("disabled")); + } + + [Test] + public void Caller_cancellation_is_not_converted_to_partial_success() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.That(async () => await api.GetHealth("https://primary/api/", cancellation.Token), Throws.InstanceOf()); + } + + [Test] + public void Cancellation_is_forwarded_to_dependencies_and_propagated_during_refresh() + { + settings.RemoteInstances = [new RemoteInstanceSetting("https://audit")]; + using var cancellation = new CancellationTokenSource(); + licensing.BeforeRead = cancellation.Cancel; + licensing.Failure = new OperationCanceledException(cancellation.Token); + + Assert.That(async () => await api.GetHealth("https://primary/api/", cancellation.Token), Throws.InstanceOf()); + using (Assert.EnterMultipleScope()) + { + Assert.That(configuration.CancellationToken, Is.EqualTo(cancellation.Token)); + Assert.That(licensing.CancellationToken, Is.EqualTo(cancellation.Token)); + } + } + + static RemoteConfiguration Remote(RemoteInstanceSetting setting, string name, Guid? hostId = null) + { + var configuration = JsonNode.Parse(""" + {"instance_type":"audit","host":{},"data_retention":{"audit_retention_period":"7.00:00:00"}} + """); + configuration["host"]["instance_name"] = name; + if (hostId.HasValue) + { + configuration["host"]["host_id"] = hostId.Value.ToString(); + } + return new RemoteConfiguration { ApiUri = setting.BaseAddress, Configuration = configuration, Status = "online", Version = "6.10.0" }; + } + + CustomCheckDetail Report(string name, Guid hostId) => new() + { + CustomCheckId = "Audit Message Ingestion", + Category = "ServiceControl Health", + HasFailed = true, + FailureReason = "Ingestion failed", + ReportedAt = clock.GetUtcNow().UtcDateTime, + OriginatingEndpoint = new EndpointDetails { Name = name, Host = "host", HostId = hostId } + }; + + sealed class FakeConfigurationApi : IConfigurationApi + { + public RemoteConfiguration[] Remotes { get; set; } = []; + public Exception Failure { get; set; } + public CancellationToken CancellationToken { get; private set; } + + public Task GetRemoteConfigs(CancellationToken cancellationToken = default) + { + CancellationToken = cancellationToken; + return Failure == null ? Task.FromResult(Remotes) : Task.FromException(Failure); + } + + public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + sealed class FakeLicenseInfoProvider : ILicenseInfoProvider + { + public LicenseInfo Info { get; } = new() + { + Status = "valid", + LicenseStatus = "Valid", + LicenseType = "Trial", + TrialLicense = true, + LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" + }; + public Exception Failure { get; set; } + public bool Refresh { get; private set; } + public string ClientName { get; private set; } + public CancellationToken CancellationToken { get; private set; } + public Action BeforeRead { get; set; } + + public Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default) + { + Refresh = refresh; + ClientName = clientName; + CancellationToken = cancellationToken; + BeforeRead?.Invoke(); + return Failure == null ? Task.FromResult(Info) : Task.FromException(Failure); + } + } + + Settings settings; + PlatformHealthState state; + FakeConfigurationApi configuration; + FakeLicenseInfoProvider licensing; + FakeTimeProvider clock; + PlatformHealthApi api; + static readonly Guid PrimaryHostId = Guid.Parse("BD444A23-93E3-42E5-A9C2-15CD7436756E"); + static readonly Guid AuditHostId = Guid.Parse("627A66F4-F7C5-4D18-8793-0D8C385A5744"); +} \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs index 808ed74060..11418d14e2 100644 --- a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs @@ -1,8 +1,11 @@ namespace ServiceControl.UnitTests.PlatformHealth { using System; + using System.Text.Json; using NUnit.Framework; + using ServiceControl.Api.Contracts; using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Infrastructure.WebApi; using ServiceControl.Operations; using ServiceControl.PlatformHealth; @@ -73,6 +76,119 @@ public void Audit_internal_checks_are_included_with_their_originating_instance() Assert.That(health.Alerts[0].InstanceName, Is.EqualTo("ServiceControl.Audit")); } + [Test] + public void Delayed_failure_does_not_replace_a_newer_recovery() + { + var state = new PlatformHealthState(); + var recovery = Detail("ServiceControl Primary Instance", hasFailed: false); + recovery.ReportedAt = recovery.ReportedAt.AddMinutes(1); + + state.Record(recovery); + state.Record(Detail("ServiceControl Primary Instance", hasFailed: true)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(state.GetHealth().Status, Is.EqualTo("healthy")); + Assert.That(state.GetChecks()[0].ReportedAt, Is.EqualTo(recovery.ReportedAt)); + } + } + + [Test] + public void Snapshots_include_passes_and_do_not_change_with_later_reports() + { + var state = new PlatformHealthState(); + var detail = Detail("ServiceControl Primary Instance", hasFailed: false); + state.Record(detail); + var snapshot = state.GetChecks(); + + detail.HasFailed = true; + detail.FailureReason = "Later failure"; + detail.ReportedAt = detail.ReportedAt.AddMinutes(1); + state.Record(detail); + + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot, Has.Length.EqualTo(1)); + Assert.That(snapshot[0].HasFailed, Is.False); + Assert.That(snapshot[0].Message, Is.Null); + Assert.That(state.GetChecks()[0].HasFailed, Is.True); + Assert.That(state.GetChecks()[0].Id, Is.EqualTo(snapshot[0].Id)); + } + } + + [Test] + public void Same_named_checks_on_different_hosts_have_distinct_stably_ordered_ids() + { + var state = new PlatformHealthState(); + var first = Detail("Audit Message Ingestion", hasFailed: true); + var second = Detail("Audit Message Ingestion", hasFailed: false); + second.OriginatingEndpoint.HostId = Guid.Parse("627A66F4-F7C5-4D18-8793-0D8C385A5744"); + + state.Record(second); + state.Record(first); + var snapshot = state.GetChecks(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot, Has.Length.EqualTo(2)); + Assert.That(snapshot[0].Id, Is.Not.EqualTo(snapshot[1].Id)); + Assert.That(snapshot, Is.Ordered.By(nameof(PlatformHealthState.CheckState.Id))); + Assert.That(state.GetHealth().Alerts, Has.Length.EqualTo(1)); + } + } + + [Test] + public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_unknown_fields() + { + var health = new PlatformHealthView + { + Status = "unknown", + Severity = "unknown", + Alerts = [], + Instances = [new PlatformHealthInstance + { + Id = "primary", + Name = "ServiceControl", + ApiUrl = "https://localhost/servicecontrol/api/", + Kind = "error", + Role = "primary-error", + Version = "6.10.0", + Health = "healthy", + ObservedAt = new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero), + ForwardErrorMessages = false, + ErrorRetentionPeriod = TimeSpan.FromDays(14) + }], + License = new PlatformHealthLicense + { + Availability = "available", + LicenseStatus = "InvalidDueToExpiredSubscription", + TrialLicense = false, + LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" + } + }; + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(health, SerializerOptions.Default)); + var instance = json.RootElement.GetProperty("instances")[0]; + var license = json.RootElement.GetProperty("license"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("status").GetString(), Is.EqualTo("unknown")); + Assert.That(json.RootElement.GetProperty("alerts").GetArrayLength(), Is.Zero); + Assert.That(instance.GetProperty("api_url").GetString(), Is.EqualTo("https://localhost/servicecontrol/api/")); + Assert.That(instance.GetProperty("version").GetString(), Is.EqualTo("6.10.0")); + Assert.That(instance.GetProperty("forward_error_messages").GetBoolean(), Is.False); + Assert.That(instance.GetProperty("error_retention_period").GetString(), Is.EqualTo("14.00:00:00")); + Assert.That(instance.GetProperty("observed_at").GetDateTimeOffset(), Is.EqualTo(health.Instances[0].ObservedAt)); + Assert.That(instance.GetProperty("health_signals_status").GetString(), Is.EqualTo("unreported")); + Assert.That(instance.TryGetProperty("audit_retention_period", out _), Is.False); + Assert.That(instance.TryGetProperty("last_reported_at", out _), Is.False); + Assert.That(license.GetProperty("license_status").GetString(), Is.EqualTo("InvalidDueToExpiredSubscription")); + Assert.That(license.GetProperty("trial_license").GetBoolean(), Is.False); + Assert.That(license.TryGetProperty("expiration_date", out _), Is.False); + } + } + static CustomCheckDetail Detail(string checkId, bool hasFailed) => new() { CustomCheckId = checkId, diff --git a/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs b/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs index fbe124508d..e3064ff5ae 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs @@ -1,11 +1,17 @@ namespace ServiceControl.UnitTests.ScatterGather; using System; +using System.Net; using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using NServiceBus.Hosting; using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure.Api; using ServiceControl.Infrastructure.WebApi; +using ServiceControl.Monitoring.HeartbeatMonitoring; using ServiceControl.Persistence; [TestFixture] @@ -28,5 +34,105 @@ public void The_remote_client_waits_no_longer_than_the_query_time_limit() Assert.That(client.Timeout, Is.EqualTo(TimeSpan.FromMinutes(5))); } + [Test] + public async Task Configuration_requests_preserve_virtual_directory_prefixes() + { + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit/servicecontrol/api")] }; + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"host":{"instance_name":"Audit"}}""") + })); + var services = new ServiceCollection(); + services.AddSingleton(new TestPersistenceSettings()); + services.AddRemoteInstancesHttpClients(settings); + services.AddHttpClient(settings.RemoteInstances[0].InstanceId).ConfigurePrimaryHttpMessageHandler(() => handler); + using var provider = services.BuildServiceProvider(); + var api = CreateApi(settings, provider.GetRequiredService()); + + var remotes = await api.GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(handler.RequestUri, Is.EqualTo(new Uri("https://audit/servicecontrol/api/configuration"))); + Assert.That(remotes[0].Status, Is.EqualTo("online")); + } + } + + [TestCase(HttpStatusCode.Unauthorized)] + [TestCase(HttpStatusCode.Forbidden)] + [TestCase(HttpStatusCode.InternalServerError)] + public async Task Failed_http_responses_cannot_be_reported_as_online(HttpStatusCode statusCode) + { + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(statusCode) + { + Content = new StringContent("""{"host":{"instance_name":"Audit"}}""") + })); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://audit/") }; + + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }; + var remotes = await CreateApi(settings, new StubClientFactory(client)).GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(remotes[0].Status, Is.Not.EqualTo("online")); + Assert.That(remotes[0].Configuration, Is.Null); + } + } + + [TestCase("null")] + [TestCase("[]")] + [TestCase("{}")] + [TestCase("{broken}")] + public async Task Malformed_or_missing_configuration_is_not_online(string body) + { + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + })); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://audit/") }; + + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }; + var remotes = await CreateApi(settings, new StubClientFactory(client)).GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(remotes[0].Status, Is.EqualTo("error")); + Assert.That(remotes[0].Configuration, Is.Null); + } + } + + [Test] + public async Task A_timeout_returns_an_unavailable_remote_but_caller_cancellation_propagates() + { + using var client = new HttpClient(new StubHandler(_ => Task.FromException(new TaskCanceledException()))) + { BaseAddress = new Uri("https://audit/") }; + var api = CreateApi(new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }, new StubClientFactory(client)); + + Assert.That((await api.GetRemoteConfigs())[0].Status, Is.EqualTo("unavailable")); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.That(async () => await api.GetRemoteConfigs(cancellation.Token), Throws.InstanceOf()); + } + + static ConfigurationApi CreateApi(Settings settings, IHttpClientFactory clientFactory) => + new(null, settings, clientFactory, new MassTransitConnectorHeartbeatStatus(), new HostInformation(Guid.Empty, "localhost")); + + class StubClientFactory(HttpClient client) : IHttpClientFactory + { + public HttpClient CreateClient(string name) => client; + } + + class StubHandler(Func> respond) : HttpMessageHandler + { + public Uri RequestUri { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) + { + RequestUri = request.RequestUri; + return respond(cancellationToken); + } + } + class TestPersistenceSettings : PersistenceSettings; } diff --git a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs index 4fb8a175fc..4f83f3b098 100644 --- a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs +++ b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs @@ -8,6 +8,7 @@ using Particular.ServiceControl; using PlatformHealth; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Api; using Transports; class CustomChecksComponent : ServiceControlComponent @@ -33,6 +34,7 @@ public override void Configure(Settings settings, ITransportCustomization transp if (!settings.ErrorIngestionOnly) { hostBuilder.Services.AddPlatformConnectionProvider(); + hostBuilder.Services.AddSingleton(); } hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 7dcc876e5d..793c561525 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -4,17 +4,19 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Configuration; using Monitoring.HeartbeatMonitoring; +using NServiceBus.Hosting; using Particular.ServiceControl.Licensing; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api; using ServiceControl.Api.Contracts; -class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : IConfigurationApi +class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, HostInformation hostInformation) : IConfigurationApi { public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) { @@ -55,9 +57,12 @@ public Task GetConfig(CancellationToken cancellationToken = default) { object content = new { + InstanceType = "error", + HealthChecksEnabled = !settings.DisableHealthChecks, Host = new { settings.InstanceName, + hostInformation.HostId, Logging = new { settings.LoggingSettings.LogPath, @@ -104,7 +109,8 @@ public async Task GetRemoteConfigs(CancellationToken canc try { - using var response = await httpClient.GetAsync("/api/configuration", cancellationToken); + using var response = await httpClient.GetAsync("api/configuration", cancellationToken); + response.EnsureSuccessStatusCode(); if (response.Headers.TryGetValues("X-Particular-Version", out var values)) { @@ -113,6 +119,13 @@ public async Task GetRemoteConfigs(CancellationToken canc await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); config = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); + if (config is not JsonObject configuration || + configuration["host"] is not JsonObject host || + host["instance_name"] is not JsonValue instanceName || + !instanceName.TryGetValue(out var name) || string.IsNullOrWhiteSpace(name)) + { + throw new JsonException("Remote response is not an instance configuration."); + } } catch (HttpRequestException ex) { @@ -122,6 +135,10 @@ public async Task GetRemoteConfigs(CancellationToken canc { throw; } + catch (OperationCanceledException) + { + status = "unavailable"; + } catch (Exception) { status = "error"; @@ -132,7 +149,7 @@ public async Task GetRemoteConfigs(CancellationToken canc ApiUri = remote.BaseAddress, Version = version, Status = status, - Configuration = config + Configuration = status == "online" ? config : null }; }); diff --git a/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs b/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs index 6daf32d9a8..145afbd037 100644 --- a/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs +++ b/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs @@ -37,7 +37,7 @@ public static void AddRemoteInstancesHttpClients(this IServiceCollection service { client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); // Application settings might contain remote URLs with /api. We strip that away to be a real base address. - client.BaseAddress = new Uri(remoteInstance.BaseAddress); + client.BaseAddress = new Uri(remoteInstance.BaseAddress.TrimEnd('/') + "/"); // This instance's query time limit bounds the whole composite: a remote that has not answered by then // is reported as missing, whatever its own limit is. Its own limit still ends the query on its side. client.Timeout = serviceProvider.GetRequiredService().QueryTimeout; diff --git a/src/ServiceControl/Licensing/LicenseController.cs b/src/ServiceControl/Licensing/LicenseController.cs index a2c734b261..666a4b9669 100644 --- a/src/ServiceControl/Licensing/LicenseController.cs +++ b/src/ServiceControl/Licensing/LicenseController.cs @@ -10,46 +10,19 @@ namespace ServiceControl.Licensing using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; - using Monitoring.HeartbeatMonitoring; using Particular.LicensingComponent.Contracts; using Particular.LicensingComponent.Persistence; using Particular.ServiceControl.Licensing; - using ServiceBus.Management.Infrastructure.Settings; [ApiController] [Route("api")] - public class LicenseController(ActiveLicense activeLicense, Settings settings, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, ILicensingDataStore dataStore) : ControllerBase + public class LicenseController(ActiveLicense activeLicense, ILicenseInfoProvider licenseInfoProvider, ILicensingDataStore dataStore) : ControllerBase { [Authorize(Policy = Permissions.ErrorLicensingView)] [HttpGet] [Route("license")] - public async Task> License(bool refresh, string clientName, CancellationToken cancellationToken = default) - { - if (refresh) - { - await activeLicense.Refresh(cancellationToken); - } - - var licenseInfo = new LicenseInfo - { - TrialLicense = activeLicense.Details.IsTrialLicense, - Edition = activeLicense.Details.Edition ?? string.Empty, - RegisteredTo = activeLicense.Details.RegisteredTo ?? string.Empty, - UpgradeProtectionExpiration = activeLicense.Details.UpgradeProtectionExpiration?.ToString("O") ?? string.Empty, - ExpirationDate = activeLicense.Details.ExpirationDate?.ToString("O") ?? string.Empty, - Status = activeLicense.IsValid ? "valid" : "invalid", - LicenseType = activeLicense.Details.LicenseType ?? string.Empty, - InstanceName = settings.InstanceName ?? string.Empty, - LicenseStatus = activeLicense.Details.Status, - Products = activeLicense.Details.Products, - HasEndpointMetadata = activeLicense.Details.HasEndpointMetadata, - LicenseExtensionUrl = connectorHeartbeatStatus.LastHeartbeat == null - ? $"https://particular.net/extend-your-trial?p={clientName}" - : $"https://particular.net/license/mt?p={clientName}&t={(activeLicense.IsEvaluation ? 0 : 1)}" - }; - - return licenseInfo; - } + public async Task> License(bool refresh, string clientName, CancellationToken cancellationToken = default) => + await licenseInfoProvider.GetLicense(refresh, clientName, cancellationToken); [Authorize(Policy = Permissions.ErrorThroughputView)] [HttpGet] diff --git a/src/ServiceControl/Licensing/LicenseInfoProvider.cs b/src/ServiceControl/Licensing/LicenseInfoProvider.cs new file mode 100644 index 0000000000..e1b88f7afd --- /dev/null +++ b/src/ServiceControl/Licensing/LicenseInfoProvider.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Licensing; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Monitoring.HeartbeatMonitoring; +using Particular.ServiceControl.Licensing; +using ServiceBus.Management.Infrastructure.Settings; + +public interface ILicenseInfoProvider +{ + Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default); +} + +sealed class LicenseInfoProvider(ActiveLicense activeLicense, Settings settings, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : ILicenseInfoProvider +{ + public async Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default) + { + if (refresh) + { + await activeLicense.Refresh(cancellationToken); + } + + var details = activeLicense.Details ?? throw new InvalidOperationException("License details are unavailable."); + + return new LicenseInfo + { + TrialLicense = details.IsTrialLicense, + Edition = details.Edition ?? string.Empty, + RegisteredTo = details.RegisteredTo ?? string.Empty, + UpgradeProtectionExpiration = details.UpgradeProtectionExpiration?.ToString("O") ?? string.Empty, + ExpirationDate = details.ExpirationDate?.ToString("O") ?? string.Empty, + Status = activeLicense.IsValid ? "valid" : "invalid", + LicenseType = details.LicenseType ?? string.Empty, + InstanceName = settings.InstanceName ?? string.Empty, + LicenseStatus = details.Status, + Products = details.Products, + HasEndpointMetadata = details.HasEndpointMetadata, + LicenseExtensionUrl = connectorHeartbeatStatus.LastHeartbeat == null + ? $"https://particular.net/extend-your-trial?p={clientName}" + : $"https://particular.net/license/mt?p={clientName}&t={(activeLicense.IsEvaluation ? 0 : 1)}" + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth.http b/src/ServiceControl/PlatformHealth.http new file mode 100644 index 0000000000..72fbdab048 --- /dev/null +++ b/src/ServiceControl/PlatformHealth.http @@ -0,0 +1,14 @@ +@baseUrl = http://localhost:33333/api + +### Discover platform health +GET {{baseUrl}} +Accept: application/json + +### Read platform health with authentication disabled +GET {{baseUrl}}/platform-health +Accept: application/json + +### Read platform health with an existing bearer token +GET {{baseUrl}}/platform-health +Accept: application/json +Authorization: Bearer {{$processEnv SERVICECONTROL_ACCESS_TOKEN}} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs new file mode 100644 index 0000000000..cf9bc03121 --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs @@ -0,0 +1,277 @@ +namespace ServiceControl.PlatformHealth; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Api; +using Api.Contracts; +using Infrastructure; +using Infrastructure.WebApi; +using Licensing; +using Monitoring.HeartbeatMonitoring; +using NServiceBus.Hosting; +using NServiceBus.Logging; +using ServiceBus.Management.Infrastructure.Settings; + +sealed class PlatformHealthApi( + Settings settings, + HostInformation hostInformation, + PlatformHealthState state, + IConfigurationApi configurationApi, + ILicenseInfoProvider licenseInfoProvider, + MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, + TimeProvider timeProvider) : IPlatformHealthApi +{ + public async Task GetHealth(string baseUrl, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var observedAt = timeProvider.GetUtcNow(); + var remoteTask = GetRemoteConfigurations(cancellationToken); + var licenseTask = GetLicense(cancellationToken); + await Task.WhenAll(remoteTask, licenseTask); + cancellationToken.ThrowIfCancellationRequested(); + + var remotes = await remoteTask; + var instances = new List + { + new() + { + Id = settings.InstanceId, + Name = settings.InstanceName, + Kind = "error", + Role = "primary-error", + ApiUrl = baseUrl.TrimEnd('/') + "/", + Version = ServiceControlVersion.GetFileVersion(), + HostId = hostInformation.HostId, + Health = "healthy", + ObservedAt = observedAt, + MetadataObservedAt = observedAt, + HealthSignalsStatus = settings.DisableHealthChecks ? "disabled" : "unreported", + TransportType = settings.TransportType, + ErrorQueue = settings.ErrorQueue, + ErrorLogQueue = settings.ErrorLogQueue, + ForwardErrorMessages = settings.ForwardErrorMessages, + ErrorRetentionPeriod = settings.ErrorRetentionPeriod, + AuditRetentionPeriod = settings.AuditRetentionPeriod + } + }; + + foreach (var remote in settings.RemoteInstances.DistinctBy(remote => remote.InstanceId).OrderBy(remote => remote.InstanceId, StringComparer.Ordinal)) + { + var configuration = remotes.FirstOrDefault(candidate => candidate.ApiUri == remote.BaseAddress); + instances.Add(GetRemoteInstance(remote, configuration, observedAt)); + } + + var reports = state.GetChecks(); + var response = PlatformHealthState.GetHealth(reports); + var assignments = new Dictionary(); + var ambiguousInstances = new HashSet(StringComparer.Ordinal); + foreach (var report in reports) + { + var namedInstances = instances.Where(instance => string.Equals(instance.Name, report.InstanceName, StringComparison.OrdinalIgnoreCase)).ToArray(); + var exactMatches = namedInstances.Where(instance => instance.HostId == report.HostId).ToArray(); + if (exactMatches.Length == 1) + { + assignments[report.Id] = exactMatches[0].Id; + continue; + } + + var reportingHosts = reports.Where(candidate => string.Equals(candidate.InstanceName, report.InstanceName, StringComparison.OrdinalIgnoreCase)) + .Select(candidate => candidate.HostId).Distinct().Take(2).Count(); + if (exactMatches.Length == 0 && namedInstances.Length == 1 && namedInstances[0].HostId is null && reportingHosts == 1) + { + assignments[report.Id] = namedInstances[0].Id; + continue; + } + + foreach (var instance in namedInstances) + { + ambiguousInstances.Add(instance.Id); + } + } + + foreach (var alert in response.Alerts) + { + alert.InstanceId = assignments.GetValueOrDefault(alert.Id); + } + + response.Instances = instances.Select(instance => + { + var associated = reports.Where(report => assignments.GetValueOrDefault(report.Id) == instance.Id).ToArray(); + var issues = response.Alerts.Where(alert => alert.InstanceId == instance.Id).ToArray(); + return instance with + { + Health = instance.Health == "unavailable" ? "unavailable" : issues.Length == 0 ? "healthy" : "degraded", + HealthSignalsStatus = instance.HealthSignalsStatus == "disabled" ? "disabled" : + ambiguousInstances.Contains(instance.Id) ? "ambiguous" : associated.Length == 0 ? "unreported" : "reported", + LastReportedAt = associated.Length == 0 ? null : new DateTimeOffset(DateTime.SpecifyKind(associated.Max(report => report.ReportedAt), DateTimeKind.Utc)), + Issues = issues + }; + }).ToArray(); + response.License = await licenseTask; + return response; + } + + PlatformHealthInstance GetRemoteInstance(RemoteInstanceSetting remote, RemoteConfiguration configuration, DateTimeOffset observedAt) + { + if (configuration?.Status == "online" && configuration.Configuration != null) + { + try + { + var metadata = configuration.Configuration.Deserialize(SerializerOptions.Default); + if (!string.IsNullOrWhiteSpace(metadata?.Host?.InstanceName)) + { + var kind = metadata.InstanceType switch + { + "error" => "error", + "audit" => "audit", + null when metadata.DataRetention?.ErrorRetentionPeriod != null => "error", + null when metadata.DataRetention?.AuditRetentionPeriod != null => "audit", + _ => "unknown" + }; + var instance = new PlatformHealthInstance + { + Id = remote.InstanceId, + Name = metadata.Host.InstanceName, + Kind = kind, + Role = "remote-" + kind, + ApiUrl = remote.BaseAddress.TrimEnd('/') + "/api/", + Version = configuration.Version is null or "Unknown" or "Missing" or "" ? null : configuration.Version, + HostId = metadata.Host.HostId is null || metadata.Host.HostId == Guid.Empty ? null : metadata.Host.HostId, + Health = "healthy", + ObservedAt = observedAt, + MetadataObservedAt = observedAt, + HealthSignalsStatus = metadata.HealthChecksEnabled == false ? "disabled" : "unreported", + TransportType = metadata.Transport?.TransportType, + ErrorQueue = metadata.Transport?.ErrorQueue, + ErrorLogQueue = metadata.Transport?.ErrorLogQueue, + ForwardErrorMessages = metadata.Transport?.ForwardErrorMessages, + AuditQueue = metadata.Transport?.AuditQueue, + AuditLogQueue = metadata.Transport?.AuditLogQueue, + ForwardAuditMessages = metadata.Transport?.ForwardAuditMessages, + ErrorRetentionPeriod = metadata.DataRetention?.ErrorRetentionPeriod, + AuditRetentionPeriod = metadata.DataRetention?.AuditRetentionPeriod + }; + return lastKnownRemotes.AddOrUpdate(remote.InstanceId, instance, + (_, previous) => instance.MetadataObservedAt >= previous.MetadataObservedAt ? instance : previous); + } + } + catch (JsonException exception) + { + log.Warn("Unable to read remote instance metadata for platform health.", exception); + } + } + + if (lastKnownRemotes.TryGetValue(remote.InstanceId, out var lastKnown)) + { + return lastKnown with { Health = "unavailable", ObservedAt = observedAt }; + } + + return new PlatformHealthInstance + { + Id = remote.InstanceId, + Name = new Uri(remote.BaseAddress).Host, + ApiUrl = remote.BaseAddress.TrimEnd('/') + "/api/", + ObservedAt = observedAt + }; + } + + async Task GetRemoteConfigurations(CancellationToken cancellationToken) + { + if (settings.RemoteInstances.Length == 0) + { + return []; + } + + try + { + return await configurationApi.GetRemoteConfigs(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + log.Warn("Unable to refresh remote configurations for platform health.", exception); + return []; + } + } + + async Task GetLicense(CancellationToken cancellationToken) + { + var hasConnector = connectorHeartbeatStatus.LastHeartbeat != null; + try + { + var license = await licenseInfoProvider.GetLicense(true, "servicepulse", cancellationToken); + if (!string.IsNullOrWhiteSpace(license?.LicenseStatus)) + { + return new PlatformHealthLicense + { + Availability = "available", + Status = license.Status, + LicenseStatus = license.LicenseStatus, + LicenseType = license.LicenseType, + TrialLicense = license.TrialLicense, + ExpirationDate = ParseDate(license.ExpirationDate), + UpgradeProtectionExpiration = ParseDate(license.UpgradeProtectionExpiration), + LicenseExtensionUrl = license.LicenseExtensionUrl, + HasMassTransitConnector = hasConnector + }; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + log.Warn("Unable to refresh license information for platform health.", exception); + } + + return new PlatformHealthLicense { HasMassTransitConnector = hasConnector }; + } + + static DateTimeOffset? ParseDate(string value) => + DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var date) ? date : null; + + readonly ConcurrentDictionary lastKnownRemotes = new(StringComparer.Ordinal); + static readonly ILog log = LogManager.GetLogger(); + + sealed class InstanceConfiguration + { + public string InstanceType { get; init; } + public bool? HealthChecksEnabled { get; init; } + public HostConfiguration Host { get; init; } + public RetentionConfiguration DataRetention { get; init; } + public TransportConfiguration Transport { get; init; } + } + + sealed class HostConfiguration + { + public string InstanceName { get; init; } + public Guid? HostId { get; init; } + } + + sealed class RetentionConfiguration + { + public TimeSpan? ErrorRetentionPeriod { get; init; } + public TimeSpan? AuditRetentionPeriod { get; init; } + } + + sealed class TransportConfiguration + { + public string TransportType { get; init; } + public string ErrorQueue { get; init; } + public string ErrorLogQueue { get; init; } + public bool? ForwardErrorMessages { get; init; } + public string AuditQueue { get; init; } + public string AuditLogQueue { get; init; } + public bool? ForwardAuditMessages { get; init; } + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthController.cs b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs index d04382ed4f..9d289a48ae 100644 --- a/src/ServiceControl/PlatformHealth/PlatformHealthController.cs +++ b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs @@ -1,17 +1,24 @@ namespace ServiceControl.PlatformHealth { + using System.Threading; + using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; + using Microsoft.AspNetCore.Http; + using Microsoft.AspNetCore.Http.Extensions; using Microsoft.AspNetCore.Mvc; + using ServiceControl.Api; using ServiceControl.Api.Contracts; using ServiceControl.Infrastructure.Auth; [ApiController] [Route("api")] - public class PlatformHealthController(PlatformHealthState platformHealthState) : ControllerBase + public class PlatformHealthController(IPlatformHealthApi platformHealthApi) : ControllerBase { [Authorize(Policy = Permissions.ErrorCustomChecksView)] [Route("platform-health")] [HttpGet] - public PlatformHealthView Get() => platformHealthState.GetHealth(); + [ProducesResponseType(typeof(PlatformHealthView), StatusCodes.Status200OK)] + public Task Get(CancellationToken cancellationToken = default) => + platformHealthApi.GetHealth(UriHelper.BuildAbsolute(Request.Scheme, Request.Host, Request.PathBase, "/api/"), cancellationToken); } } \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthState.cs b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs index 161494d1ca..8f363bfc22 100644 --- a/src/ServiceControl/PlatformHealth/PlatformHealthState.cs +++ b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs @@ -16,7 +16,7 @@ internal void Record(CustomCheckDetail detail) } var id = detail.GetDeterministicId(); - checks[id] = new CheckState + var report = new CheckState { Id = id, CheckId = detail.CustomCheckId, @@ -28,15 +28,22 @@ internal void Record(CustomCheckDetail detail) Host = detail.OriginatingEndpoint.Host, HostId = detail.OriginatingEndpoint.HostId }; + + checks.AddOrUpdate(id, report, (_, previous) => report.ReportedAt >= previous.ReportedAt ? report : previous); } - public PlatformHealthView GetHealth() + internal CheckState[] GetChecks() => checks.Values + .OrderBy(check => check.InstanceName, StringComparer.OrdinalIgnoreCase) + .ThenBy(check => check.CheckId, StringComparer.OrdinalIgnoreCase) + .ThenBy(check => check.Id) + .ToArray(); + + public PlatformHealthView GetHealth() => GetHealth(GetChecks()); + + internal static PlatformHealthView GetHealth(CheckState[] currentChecks) { - var currentChecks = checks.Values.ToArray(); var failedChecks = currentChecks .Where(check => check.HasFailed) - .OrderBy(check => check.InstanceName, StringComparer.OrdinalIgnoreCase) - .ThenBy(check => check.CheckId, StringComparer.OrdinalIgnoreCase) .ToArray(); return new PlatformHealthView @@ -59,7 +66,7 @@ public PlatformHealthView GetHealth() readonly ConcurrentDictionary checks = new(); - class CheckState + internal sealed record CheckState { public Guid Id { get; init; } public string CheckId { get; init; } diff --git a/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs b/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs index ad678bcef7..f08814c940 100644 --- a/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs +++ b/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs @@ -2,6 +2,7 @@ namespace Particular.ServiceControl { using global::ServiceControl.Api; using global::ServiceControl.Infrastructure.Api; + using global::ServiceControl.Licensing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -10,6 +11,7 @@ static class ServiceControlApiHostBuilderExtensions public static void AddServiceControlApis(this IHostApplicationBuilder hostBuilder) { hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); From e1ceaa999ab54be706982e1e9471214153fe729b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:55:30 +0000 Subject: [PATCH 3/8] Pin Azure CLI on Windows CI Co-authored-by: hazel-bohon <2416062+hazel-bohon@users.noreply.github.com> --- .github/workflows/ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9632f42a..c8bdc397d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,18 @@ jobs: # once the issue is resolved it should be able to be re-floated # https://github.com/Azure/azure-cli/issues/32980. # This can be removed once https://github.com/Azure/azure-cli/issues/32869 is supported. + - name: Set Python 3.13 (Windows) + if: matrix.os-name == 'Windows' && matrix.test-category == 'AzureServiceBus' + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install pinned Azure CLI on Python 3.13 (Windows) + if: matrix.os-name == 'Windows' && matrix.test-category == 'AzureServiceBus' + run: | + python -m pip install --upgrade pip + python -m pip install --user "azure-cli==2.64.0" + $userScripts = python -c "import sysconfig; print(sysconfig.get_path('scripts', 'nt_user'))" + echo $userScripts >> $Env:GITHUB_PATH - name: Set Python 3.13 (Linux) if: matrix.os-name == 'Linux' && matrix.test-category == 'AzureServiceBus' uses: actions/setup-python@v5 From 7e2106921fc27446c6fcc7921119fc08275ff9f3 Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Wed, 7 Oct 2026 06:23:09 -0400 Subject: [PATCH 4/8] Remvoe license checking information from the API. --- docs/platform-health.md | 20 ++--- .../When_custom_checks_are_classified.cs | 3 +- .../Contracts/PlatformHealthView.cs | 13 ---- .../When_inspecting_platform_health.cs | 1 - .../PlatformHealth/PlatformHealthApiTests.cs | 78 ++----------------- .../PlatformHealthStateTests.cs | 14 +--- .../PlatformHealth/PlatformHealthApi.cs | 48 +----------- 7 files changed, 18 insertions(+), 159 deletions(-) diff --git a/docs/platform-health.md b/docs/platform-health.md index 737d019ca6..05f502705e 100644 --- a/docs/platform-health.md +++ b/docs/platform-health.md @@ -6,7 +6,7 @@ The public motivation is [ServiceControl #5860](https://github.com/Particular/Se ## Response -The existing `status`, `severity`, and `alerts` fields remain, with additive `instances` and `license` sections. +The existing `status`, `severity`, and `alerts` fields remain, with an additive `instances` section. ### Instances @@ -32,7 +32,7 @@ The existing `status`, `severity`, and `alerts` fields remain, with additive `in Primary and audit `/api/configuration` (also `/api/instance-info`) include `instance_type` and `host.host_id`. Primary configuration additionally reports `health_checks_enabled`. Older remotes without `instance_type` are identified only when their retention configuration establishes the type. A never-observed, unreachable remote is explicitly unknown, not assumed to be an audit instance. -Remote probes use the registered named HTTP clients and their query timeout. Non-success status codes, empty or malformed configuration, and connection failures do not produce healthy rows. An outage retains the last successful metadata in memory, clearly dated by `metadata_observed_at`. Other rows and the license section still return. Caller cancellation propagates instead of returning partial success. No recursive platform-health requests are made to other primaries. +Remote probes use the registered named HTTP clients and their query timeout. Non-success status codes, empty or malformed configuration, and connection failures do not produce healthy rows. An outage retains the last successful metadata in memory, clearly dated by `metadata_observed_at`. Other rows still return. Caller cancellation propagates instead of returning partial success. No recursive platform-health requests are made to other primaries. ### Issues and summary @@ -40,26 +40,20 @@ Each failed check has `id`, `check_id`, `category`, `message`, `reported_at`, `i Association uses case-insensitive instance name plus reporting host ID. A legacy remote without a host ID can use a name match only when there is one matching inventory row and one reporting host with that name. Ambiguous or unmatched reports remain in root `alerts` without `instance_id`; they are never assigned to several rows. Consumers should retain a place to display those unassigned alerts. -The legacy summary describes captured checks, not the whole browser-visible platform: `status` is `unknown` before any internal report, `healthy` when none are failing, and `unhealthy` when at least one is failing. Its corresponding `severity` values are `unknown`, `none`, and `error`. ServicePulse should use per-instance health for page severity and combine it with its independently observed monitoring state. The legacy summary does not account for monitoring, browser connectivity, license expiry, or available upgrades. +The legacy summary describes captured checks, not the whole browser-visible platform: `status` is `unknown` before any internal report, `healthy` when none are failing, and `unhealthy` when at least one is failing. Its corresponding `severity` values are `unknown`, `none`, and `error`. ServicePulse should use per-instance health for page severity and combine it with its independently observed monitoring state. The legacy summary does not account for monitoring, browser connectivity, or available upgrades. Check state is process-local. Reports older than a check's latest `reported_at` are ignored; a newer successful report clears that failure. Reports do not expire: different checks have different schedules, including one-shot checks. After restart, check observations and last-known remote metadata are initially empty. `healthy` therefore means reachable without a known associated failure, not proof of complete or fresh check coverage. An unreachable process cannot report its own browser-facing unavailability in a successful response. -### License - -`license.availability` is `available` after a successful refresh and `unavailable` when license details cannot be refreshed. An unavailable license never claims to be valid and does not suppress instance health. - -The available summary includes `status`, `license_status`, `license_type`, `trial_license`, optional `expiration_date` and `upgrade_protection_expiration`, and `license_extension_url`. It preserves the existing license status values for subscription, trial and upgrade-protection gates. Renewal URLs share the `/api/license` mapping with `clientName=servicepulse`, including MassTransit evaluation/subscription links. `has_mass_transit_connector` reports connector presence. Customer registration, licensed products and endpoint-license metadata are not included. - ## ServicePulse integration -The endpoint supplies primary/remote inventory, installed versions, configuration, issues, and the license summary. Updating this endpoint does not update the ServicePulse consumer automatically; the consumer must map `instances` and `license` into its stores and support unknown instance types and unassigned alerts. +The endpoint supplies primary/remote inventory, installed versions, configuration, and issues. Updating this endpoint does not update the ServicePulse consumer automatically; the consumer must map `instances` into its stores and support unknown instance types and unassigned alerts. ServicePulse continues to own: - Its running frontend version and ServicePulse row. - The browser-selected monitoring URL, monitoring requests, and monitoring row. - Browser-to-primary connectivity failures, including when this endpoint cannot be reached. -- Release-feed requests, latest-version comparison, release links, upgrade badges, and outdated-only navigation state. Installed version and license validity are not a guarantee that an upgrade path is supported. +- Release-feed requests, latest-version comparison, release links, upgrade badges, and outdated-only navigation state. An installed version is not a guarantee that an upgrade path is supported. - The customer-check fetch for the support export. Export combines this response, browser-owned rows, and the existing custom-check results. Customer checks never affect platform health. Keep the legacy consumer fallback for supported ServiceControl versions without the advertised capability. Do not interpret `401`, `403`, a timeout, or a failed response as an absent capability. The existing custom-check API, classification, notifications, and integration events remain unchanged. Audit health still arrives through the current custom-check reporting transport; this increment does not remove that dependency or introduce replacement events. @@ -68,10 +62,10 @@ Keep the legacy consumer fallback for supported ServiceControl versions without The endpoint retains `error:customchecks:view`, granted by the existing reader, writer and admin roles. No permission or authentication behavior is changed. With authentication disabled it is anonymous. With authentication and RBAC enabled, anonymous callers receive `401` and authenticated callers without a read role receive `403`. -Known shared-policy limitation: authentication enabled with RBAC disabled currently resolves named permissions to allow-all, so the expanded response, including the license summary, can be accessed anonymously. Fixing that policy is separate work. Container liveness and readiness remain separate at `/health` and `/health/ready`. +Known shared-policy limitation: authentication enabled with RBAC disabled currently resolves named permissions to allow-all, so the expanded response can be accessed anonymously. Fixing that policy is separate work. Container liveness and readiness remain separate at `/health` and `/health/ready`. ## Verification For manual requests, use [PlatformHealth.http](../src/ServiceControl/PlatformHealth.http). Its authenticated request reads an existing bearer token from `SERVICECONTROL_ACCESS_TOKEN`; do not store credentials in the request file. -`PlatformHealthStateTests` and `PlatformHealthApiTests` cover snapshot ordering, delayed reports, serialization, source projection, identity ambiguity, offline metadata, partial failures, license mapping and cancellation. Remote-client tests cover HTTP status, malformed responses and prefixed URLs. Shared acceptance scenarios exercise the real root/configuration/health responses and preserve custom-check behavior. The multi-instance `When_inspecting_platform_health` scenario exercises real audit check delivery, issue ownership, recovery and unavailable inventory. OIDC acceptance scenarios cover the existing read-role policy. \ No newline at end of file +`PlatformHealthStateTests` and `PlatformHealthApiTests` cover snapshot ordering, delayed reports, serialization, source projection, identity ambiguity, offline metadata, partial failures and cancellation. Remote-client tests cover HTTP status, malformed responses and prefixed URLs. Shared acceptance scenarios exercise the real root/configuration/health responses and preserve custom-check behavior. The multi-instance `When_inspecting_platform_health` scenario exercises real audit check delivery, issue ownership, recovery and unavailable inventory. OIDC acceptance scenarios cover the existing read-role policy. \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs index da388165c1..f1ee739d20 100644 --- a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs +++ b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs @@ -89,8 +89,7 @@ await Define() Assert.That(instance.ErrorRetentionPeriod, Is.EqualTo(Settings.ErrorRetentionPeriod)); Assert.That(instanceJson.GetProperty("health_signals_status").GetString(), Is.EqualTo("reported")); Assert.That(instanceJson.GetProperty("forward_error_messages").GetBoolean(), Is.EqualTo(Settings.ForwardErrorMessages)); - Assert.That(healthJson.RootElement.GetProperty("license").GetProperty("availability").GetString(), Is.EqualTo("available")); - Assert.That(platformHealth.License.LicenseStatus, Is.Not.Null.And.Not.Empty); + Assert.That(healthJson.RootElement.TryGetProperty("license", out _), Is.False); // What the wire actually carries: Assert.That(wireBody, Does.Contain("\"internal\":true"), "internal checks must render internal:true on the wire"); diff --git a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs index 20fc1708dd..aaba40b72a 100644 --- a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs +++ b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs @@ -8,7 +8,6 @@ public class PlatformHealthView public string Severity { get; set; } public PlatformHealthAlert[] Alerts { get; set; } public PlatformHealthInstance[] Instances { get; set; } = []; - public PlatformHealthLicense License { get; set; } = new(); } public class PlatformHealthAlert @@ -51,16 +50,4 @@ public sealed record PlatformHealthInstance public TimeSpan? AuditRetentionPeriod { get; init; } } - public sealed record PlatformHealthLicense - { - public string Availability { get; init; } = "unavailable"; - public string? Status { get; init; } - public string? LicenseStatus { get; init; } - public string? LicenseType { get; init; } - public bool? TrialLicense { get; init; } - public DateTimeOffset? ExpirationDate { get; init; } - public DateTimeOffset? UpgradeProtectionExpiration { get; init; } - public string? LicenseExtensionUrl { get; init; } - public bool HasMassTransitConnector { get; init; } - } } \ No newline at end of file diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs index de4cd93809..07ef2ae6d8 100644 --- a/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs @@ -80,7 +80,6 @@ await Define() Assert.That(unavailable.Version, Is.Null); Assert.That(recoveredAudit.Health, Is.EqualTo("healthy")); Assert.That(recoveredAudit.Issues, Is.Empty); - Assert.That(recovered.License.Availability, Is.EqualTo("available")); } } diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs index b2bee5bde4..5a3f0ec7b9 100644 --- a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs @@ -15,8 +15,6 @@ namespace ServiceControl.UnitTests.PlatformHealth; using ServiceControl.Api.Contracts; using ServiceControl.Contracts.CustomChecks; using ServiceControl.Infrastructure; -using ServiceControl.Licensing; -using ServiceControl.Monitoring.HeartbeatMonitoring; using ServiceControl.Operations; using ServiceControl.PlatformHealth; @@ -37,10 +35,9 @@ public void SetUp() }; state = new PlatformHealthState(); configuration = new FakeConfigurationApi(); - licensing = new FakeLicenseInfoProvider(); clock = new FakeTimeProvider(new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero)); api = new PlatformHealthApi(settings, new HostInformation(PrimaryHostId, "primary-host"), state, - configuration, licensing, new MassTransitConnectorHeartbeatStatus(), clock); + configuration, clock); } [Test] @@ -68,8 +65,6 @@ public async Task Includes_primary_inventory_configuration_and_version_before_ch Assert.That(primary.ErrorLogQueue, Is.EqualTo("error.log")); Assert.That(primary.ForwardErrorMessages, Is.False); Assert.That(primary.ErrorRetentionPeriod, Is.EqualTo(settings.ErrorRetentionPeriod)); - Assert.That(licensing.Refresh, Is.True); - Assert.That(licensing.ClientName, Is.EqualTo("servicepulse")); } } @@ -213,7 +208,6 @@ public async Task Malformed_optional_remote_metadata_does_not_hide_other_instanc Assert.That(result.Instances, Has.Length.EqualTo(3)); Assert.That(result.Instances.Single(instance => instance.Id == malformed.InstanceId).Health, Is.EqualTo("unavailable")); Assert.That(result.Instances.Single(instance => instance.Id == valid.InstanceId).Health, Is.EqualTo("healthy")); - Assert.That(result.License.Availability, Is.EqualTo("available")); } } @@ -264,11 +258,10 @@ public async Task Legacy_configuration_infers_kind_and_matches_unique_names_with } [Test] - public async Task License_failure_and_remote_failure_do_not_hide_local_health() + public async Task Remote_failure_does_not_hide_local_health() { settings.RemoteInstances = [new RemoteInstanceSetting("https://offline")]; configuration.Failure = new InvalidOperationException("Remote unavailable"); - licensing.Failure = new InvalidOperationException("License unavailable"); state.Record(Report("Primary", PrimaryHostId)); var result = await api.GetHealth("https://primary/api/"); @@ -278,34 +271,6 @@ public async Task License_failure_and_remote_failure_do_not_hide_local_health() Assert.That(result.Instances, Has.Length.EqualTo(2)); Assert.That(result.Instances[0].Health, Is.EqualTo("degraded")); Assert.That(result.Instances[1].Health, Is.EqualTo("unavailable")); - Assert.That(result.License.Availability, Is.EqualTo("unavailable")); - Assert.That(result.License.Status, Is.Null); - Assert.That(result.License.LicenseStatus, Is.Null); - } - } - - [TestCase("Valid")] - [TestCase("ValidWithExpiringTrial")] - [TestCase("InvalidDueToExpiredTrial")] - [TestCase("InvalidDueToExpiredSubscription")] - [TestCase("InvalidDueToExpiredUpgradeProtection")] - public async Task License_status_and_renewal_fields_preserve_the_license_api_values(string licenseStatus) - { - licensing.Info.LicenseStatus = licenseStatus; - licensing.Info.ExpirationDate = "2026-10-01T00:00:00.0000000Z"; - licensing.Info.UpgradeProtectionExpiration = ""; - - var result = await api.GetHealth("https://primary/api/"); - - using (Assert.EnterMultipleScope()) - { - Assert.That(result.License.Availability, Is.EqualTo("available")); - Assert.That(result.License.LicenseStatus, Is.EqualTo(licenseStatus)); - Assert.That(result.License.LicenseType, Is.EqualTo(licensing.Info.LicenseType)); - Assert.That(result.License.TrialLicense, Is.EqualTo(licensing.Info.TrialLicense)); - Assert.That(result.License.LicenseExtensionUrl, Is.EqualTo(licensing.Info.LicenseExtensionUrl)); - Assert.That(result.License.ExpirationDate, Is.EqualTo(new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero))); - Assert.That(result.License.UpgradeProtectionExpiration, Is.Null); } } @@ -333,15 +298,11 @@ public void Cancellation_is_forwarded_to_dependencies_and_propagated_during_refr { settings.RemoteInstances = [new RemoteInstanceSetting("https://audit")]; using var cancellation = new CancellationTokenSource(); - licensing.BeforeRead = cancellation.Cancel; - licensing.Failure = new OperationCanceledException(cancellation.Token); + configuration.BeforeRead = cancellation.Cancel; + configuration.Failure = new OperationCanceledException(cancellation.Token); Assert.That(async () => await api.GetHealth("https://primary/api/", cancellation.Token), Throws.InstanceOf()); - using (Assert.EnterMultipleScope()) - { - Assert.That(configuration.CancellationToken, Is.EqualTo(cancellation.Token)); - Assert.That(licensing.CancellationToken, Is.EqualTo(cancellation.Token)); - } + Assert.That(configuration.CancellationToken, Is.EqualTo(cancellation.Token)); } static RemoteConfiguration Remote(RemoteInstanceSetting setting, string name, Guid? hostId = null) @@ -372,10 +333,12 @@ sealed class FakeConfigurationApi : IConfigurationApi public RemoteConfiguration[] Remotes { get; set; } = []; public Exception Failure { get; set; } public CancellationToken CancellationToken { get; private set; } + public Action BeforeRead { get; set; } public Task GetRemoteConfigs(CancellationToken cancellationToken = default) { CancellationToken = cancellationToken; + BeforeRead?.Invoke(); return Failure == null ? Task.FromResult(Remotes) : Task.FromException(Failure); } @@ -383,36 +346,9 @@ public Task GetRemoteConfigs(CancellationToken cancellati public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotSupportedException(); } - sealed class FakeLicenseInfoProvider : ILicenseInfoProvider - { - public LicenseInfo Info { get; } = new() - { - Status = "valid", - LicenseStatus = "Valid", - LicenseType = "Trial", - TrialLicense = true, - LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" - }; - public Exception Failure { get; set; } - public bool Refresh { get; private set; } - public string ClientName { get; private set; } - public CancellationToken CancellationToken { get; private set; } - public Action BeforeRead { get; set; } - - public Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default) - { - Refresh = refresh; - ClientName = clientName; - CancellationToken = cancellationToken; - BeforeRead?.Invoke(); - return Failure == null ? Task.FromResult(Info) : Task.FromException(Failure); - } - } - Settings settings; PlatformHealthState state; FakeConfigurationApi configuration; - FakeLicenseInfoProvider licensing; FakeTimeProvider clock; PlatformHealthApi api; static readonly Guid PrimaryHostId = Guid.Parse("BD444A23-93E3-42E5-A9C2-15CD7436756E"); diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs index 11418d14e2..f2669cf1e0 100644 --- a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs @@ -157,19 +157,11 @@ public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_un ObservedAt = new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero), ForwardErrorMessages = false, ErrorRetentionPeriod = TimeSpan.FromDays(14) - }], - License = new PlatformHealthLicense - { - Availability = "available", - LicenseStatus = "InvalidDueToExpiredSubscription", - TrialLicense = false, - LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" - } + }] }; using var json = JsonDocument.Parse(JsonSerializer.Serialize(health, SerializerOptions.Default)); var instance = json.RootElement.GetProperty("instances")[0]; - var license = json.RootElement.GetProperty("license"); using (Assert.EnterMultipleScope()) { @@ -183,9 +175,7 @@ public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_un Assert.That(instance.GetProperty("health_signals_status").GetString(), Is.EqualTo("unreported")); Assert.That(instance.TryGetProperty("audit_retention_period", out _), Is.False); Assert.That(instance.TryGetProperty("last_reported_at", out _), Is.False); - Assert.That(license.GetProperty("license_status").GetString(), Is.EqualTo("InvalidDueToExpiredSubscription")); - Assert.That(license.GetProperty("trial_license").GetBoolean(), Is.False); - Assert.That(license.TryGetProperty("expiration_date", out _), Is.False); + Assert.That(json.RootElement.TryGetProperty("license", out _), Is.False); } } diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs index cf9bc03121..40b9924c45 100644 --- a/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs +++ b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs @@ -3,7 +3,6 @@ namespace ServiceControl.PlatformHealth; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Text.Json; using System.Threading; @@ -12,8 +11,6 @@ namespace ServiceControl.PlatformHealth; using Api.Contracts; using Infrastructure; using Infrastructure.WebApi; -using Licensing; -using Monitoring.HeartbeatMonitoring; using NServiceBus.Hosting; using NServiceBus.Logging; using ServiceBus.Management.Infrastructure.Settings; @@ -23,20 +20,15 @@ sealed class PlatformHealthApi( HostInformation hostInformation, PlatformHealthState state, IConfigurationApi configurationApi, - ILicenseInfoProvider licenseInfoProvider, - MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, TimeProvider timeProvider) : IPlatformHealthApi { public async Task GetHealth(string baseUrl, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); var observedAt = timeProvider.GetUtcNow(); - var remoteTask = GetRemoteConfigurations(cancellationToken); - var licenseTask = GetLicense(cancellationToken); - await Task.WhenAll(remoteTask, licenseTask); + var remotes = await GetRemoteConfigurations(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - var remotes = await remoteTask; var instances = new List { new() @@ -113,7 +105,6 @@ public async Task GetHealth(string baseUrl, CancellationToke Issues = issues }; }).ToArray(); - response.License = await licenseTask; return response; } @@ -203,43 +194,6 @@ async Task GetRemoteConfigurations(CancellationToken canc } } - async Task GetLicense(CancellationToken cancellationToken) - { - var hasConnector = connectorHeartbeatStatus.LastHeartbeat != null; - try - { - var license = await licenseInfoProvider.GetLicense(true, "servicepulse", cancellationToken); - if (!string.IsNullOrWhiteSpace(license?.LicenseStatus)) - { - return new PlatformHealthLicense - { - Availability = "available", - Status = license.Status, - LicenseStatus = license.LicenseStatus, - LicenseType = license.LicenseType, - TrialLicense = license.TrialLicense, - ExpirationDate = ParseDate(license.ExpirationDate), - UpgradeProtectionExpiration = ParseDate(license.UpgradeProtectionExpiration), - LicenseExtensionUrl = license.LicenseExtensionUrl, - HasMassTransitConnector = hasConnector - }; - } - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception exception) - { - log.Warn("Unable to refresh license information for platform health.", exception); - } - - return new PlatformHealthLicense { HasMassTransitConnector = hasConnector }; - } - - static DateTimeOffset? ParseDate(string value) => - DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var date) ? date : null; - readonly ConcurrentDictionary lastKnownRemotes = new(StringComparer.Ordinal); static readonly ILog log = LogManager.GetLogger(); From 2fc4229a9525741c705e5b281f01be807b1c0913 Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Wed, 7 Oct 2026 06:25:36 -0400 Subject: [PATCH 5/8] Update docs/README.md Co-authored-by: Michelle Chamberlin --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index bbc7b5bb98..cf5579964b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,7 @@ This section points to sources that explain why ServiceControl is designed the w - [Retries over Azure Storage Queues transport](retries-asq-transport.md) — transport-specific retry handling - [Data versioning design](data-versioning-design.md) — the cache-versioning invariant for API responses - [Event log design](eventlog-design.md) — what the event log is and what it records -- [Platform health API](platform-health.md) — how ServicePulse reads internal health independently from customer custom checks +- [Platform health API](platform-health.md) — how ServicePulse can read internal health independently from customer custom checks - [Multiple ServiceControl instances communication](multipleservicecontrolinstancescommunication.md) — how primary, audit, and monitoring instances talk to each other - [Handling unavailable runtime dependencies](handling-unavailable-runtime-dependencies.md) — how instances react when a dependency is unavailable - [Telemetry](telemetry.md) — telemetry configuration and emitted metrics From c80678f896f89129e79b188e44194ae1b5226f18 Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Wed, 7 Oct 2026 06:52:35 -0400 Subject: [PATCH 6/8] Refactor PlatformHealthView to enforce required properties and update tests for instance ID handling --- .../Contracts/PlatformHealthView.cs | 21 ++++++++------- .../PlatformHealthStateTests.cs | 27 ++++++++++++++++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs index aaba40b72a..581fa0ea63 100644 --- a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs +++ b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs @@ -1,29 +1,30 @@ +#nullable enable + namespace ServiceControl.Api.Contracts { using System; public class PlatformHealthView { - public string Status { get; set; } - public string Severity { get; set; } - public PlatformHealthAlert[] Alerts { get; set; } + public required string Status { get; set; } + public required string Severity { get; set; } + public PlatformHealthAlert[] Alerts { get; set; } = []; public PlatformHealthInstance[] Instances { get; set; } = []; } public class PlatformHealthAlert { public Guid Id { get; set; } - public string InstanceId { get; set; } - public string CheckId { get; set; } - public string Category { get; set; } - public string Message { get; set; } + public string? InstanceId { get; set; } + public required string CheckId { get; set; } + public required string Category { get; set; } + public required string Message { get; set; } public DateTime ReportedAt { get; set; } - public string InstanceName { get; set; } - public string Host { get; set; } + public required string InstanceName { get; set; } + public required string Host { get; set; } public Guid HostId { get; set; } } -#nullable enable public sealed record PlatformHealthInstance { public required string Id { get; init; } diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs index f2669cf1e0..5f6ee9eba8 100644 --- a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs @@ -144,7 +144,6 @@ public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_un { Status = "unknown", Severity = "unknown", - Alerts = [], Instances = [new PlatformHealthInstance { Id = "primary", @@ -179,6 +178,32 @@ public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_un } } + [TestCase(null)] + [TestCase("primary")] + public void Alerts_round_trip_with_optional_instance_id(string instanceId) + { + var state = new PlatformHealthState(); + var detail = Detail("ServiceControl Primary Instance", hasFailed: true); + state.Record(detail); + var health = state.GetHealth(); + health.Alerts[0].InstanceId = instanceId; + + var body = JsonSerializer.Serialize(health, SerializerOptions.Default); + using var json = JsonDocument.Parse(body); + var deserialized = JsonSerializer.Deserialize(body, SerializerOptions.Default); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("alerts")[0].TryGetProperty("instance_id", out _), Is.EqualTo(instanceId is not null)); + Assert.That(deserialized.Status, Is.EqualTo(health.Status)); + Assert.That(deserialized.Severity, Is.EqualTo(health.Severity)); + Assert.That(deserialized.Alerts, Has.Length.EqualTo(1)); + Assert.That(deserialized.Alerts[0].InstanceId, Is.EqualTo(instanceId)); + Assert.That(deserialized.Alerts[0].CheckId, Is.EqualTo(detail.CustomCheckId)); + Assert.That(deserialized.Alerts[0].Message, Is.EqualTo(detail.FailureReason)); + } + } + static CustomCheckDetail Detail(string checkId, bool hasFailed) => new() { CustomCheckId = checkId, From dacd3089cfd649846af532662ecad2db9ba6b2e5 Mon Sep 17 00:00:00 2001 From: Hazel Bohon Date: Wed, 7 Oct 2026 07:06:07 -0400 Subject: [PATCH 7/8] Move `.record` call into `try` block. --- src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs index f768593b63..f940628588 100644 --- a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs +++ b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs @@ -21,10 +21,9 @@ public CustomCheckResultProcessor(IDomainEvents domainEvents, ICustomChecksDataS public async Task ProcessResult(CustomCheckDetail checkDetail, CancellationToken cancellationToken = default) { - platformHealthState?.Record(checkDetail); - try { + platformHealthState?.Record(checkDetail); var statusChange = await store.UpdateCustomCheckStatus(checkDetail, cancellationToken); await RaiseEvents(statusChange, checkDetail, cancellationToken); From 5ca6de3e17f3a70e202f01f6b7b916ed0b052722 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:10:52 +0000 Subject: [PATCH 8/8] Wait for current index during range unarchive Co-authored-by: hazel-bohon <2416062+hazel-bohon@users.noreply.github.com> --- .../ErrorMessagesDataStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs index 52bb6f3b5e..4dd45d9332 100644 --- a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs @@ -431,7 +431,7 @@ public async Task UnArchiveMessagesByRange(DateTime from, DateTime to, var patch = new PatchByQueryOperation(query, new QueryOperationOptions { - AllowStale = true, + AllowStale = false, RetrieveDetails = true });