Add default-on PIT traversal for semantic searches

.env.example

@@ -5,6 +5,10 @@ OpenSearchDomainEndpoint=
 SearchOpenSearchRequestTimeoutMs=20000
 SearchRequestDeadlineMs=30000
 SearchReadinessTimeoutMs=5000
+# PIT traversal is enabled by default; set false to restore bounded semantic searches.
+SearchPitEnabled=true
+SearchPitBatchSize=1000
+SearchPitKeepAliveMs=60000
 RequestLogSuccessSampleRate=0.1
 RequestLogHealthChecks=false
 WebSearchEnabled=true

api-docs/openapi.json

@@ -21,7 +21,7 @@
           "CVE Record"
         ],
         "summary": "Retrieves all matching CVE records after applying the JSON request body as filters",
-        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves all matching CVE records after applying the JSON request body as filters</p>  <h2>Result Counts and Pagination</h2>  <p>Direct searches report the exact number of matching records, including totals greater than 10,000, but requested pages must remain within the first 10,000 results.</p>  <p>CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. These searches evaluate at most 10,000 search candidates, so their reported totals and pages include only matches found within that candidate set. A warning is returned when that limit is reached.</p>",
+        "description": "  <h2>Expected Behavior</h2>  <p>Retrieves all matching CVE records after applying the JSON request body as filters</p>  <h2>Result Counts and Pagination</h2>  <p>Direct searches report the exact number of matching records, including totals greater than 10,000, but requested pages must remain within the first 10,000 results.</p>  <p>CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. PIT is enabled by default on the server. With PIT explicitly disabled using SearchPitEnabled=false, these searches evaluate at most 10,000 search candidates, so their reported totals and pages include only matches found within that candidate set. Reaching that limit returns SEMANTIC_CANDIDATE_LIMIT_REACHED and pagination.totalIsExact=false.</p>  <p>With point-in-time (PIT) traversal enabled, semantic searches evaluate all candidate batches before returning exact totals and pagination.totalIsExact=true. Requested pages must still fit entirely within the first 10,000 matching results. Filling the requested page does not stop the scan because the total must also be computed. PIT is a server-side setting, not a request field; clients continue to use pageNumber and resultsPerPage, with no public cursor.</p>  <p>Each PIT semantic request uses one snapshot for its candidates and returned records. Separate page requests use separate snapshots, so changes between requests can still shift results. Incomplete traversal returns a sanitized error, not partial results, and does not restart or fall back to a bounded search. Warnings remain before records in successful responses. Direct searches and /webSearch do not use PIT.</p>",
         "operationId": "search",
         "responses": {
           "200": {
@@ -83,7 +83,7 @@
             }
           },
           "502": {
-            "description": "The search backend returned an invalid or unexpected response",
+            "description": "UPSTREAM_SEARCH_ERROR: The search backend returned an invalid or unexpected response. This includes malformed or incomplete PIT batches or full-page responses; no partial PIT result is returned.",
             "content": {
               "application/json": {
                 "schema": {
@@ -93,7 +93,7 @@
             }
           },
           "503": {
-            "description": "The search service is temporarily unavailable",
+            "description": "SEARCH_UNAVAILABLE: The search service is temporarily unavailable, including a missing or expired PIT search context. PIT traversal is not silently restarted. Retry-After specifies the suggested delay.",
             "content": {
               "application/json": {
                 "schema": {
@@ -111,7 +111,7 @@
             }
           },
           "504": {
-            "description": "Search exceeded its processing deadline",
+            "description": "SEARCH_TIMEOUT: Search exceeded its operation timeout or overall processing deadline, or a PIT search reported a timeout. No partial PIT result is returned. PIT cleanup has a separate bounded timeout and can add up to five seconds before the error response.",
             "content": {
               "application/json": {
                 "schema": {
@@ -456,6 +456,7 @@
       },
       "SearchWarning": {
         "type": "object",
+        "description": "Non-fatal search interpretation information. SEMANTIC_CANDIDATE_LIMIT_REACHED applies only to bounded semantic searches with PIT disabled. BROAD_VIRTUAL_MATCH_STRING can occur in either mode; its message describes the candidate limit when PIT is disabled and increased processing time when PIT is enabled.",
         "additionalProperties": false,
         "required": [
           "code",
@@ -527,16 +528,16 @@
           "totalResults": {
             "type": "integer",
             "minimum": 0,
-            "description": "For direct searches, the exact number of matching records, which may be greater than 10,000. For semantic post-filtered searches, the number of matches found within the bounded 10,000-candidate set."
+            "description": "For direct searches, the exact number of matching records, which may be greater than 10,000. For semantic searches with PIT enabled (the default), the exact number of matches across the completed snapshot traversal. With PIT explicitly disabled, the number of matches found within the bounded 10,000-candidate set. Separate page requests use separate snapshots."
           },
           "totalPages": {
             "type": "integer",
             "minimum": 0,
-            "description": "Pages derived from totalResults and pageSize. Semantic post-filtered searches derive this value from matches found within the bounded 10,000-candidate set."
+            "description": "Pages derived from totalResults and pageSize, using the same bounded or exact total as totalResults. This value can exceed the number of accessible pages: each requested page must still fit entirely within the first 10,000 matching results in either mode."
           },
           "totalIsExact": {
             "type": "boolean",
-            "description": "True when totalResults represents every match. False when a semantic search reaches the 10,000-candidate limit and additional matches may exist."
+            "description": "True for direct searches and successful, fully completed PIT semantic searches (the default mode). With PIT explicitly disabled, false when a semantic search reaches the 10,000-candidate limit and additional matches may exist; true otherwise. A failed PIT traversal returns an error rather than a success with a partial total."
           }
         }
       },
@@ -571,7 +572,7 @@
           },
           "warnings": {
             "type": "array",
-            "description": "Non-fatal conditions that may affect how the results should be interpreted.",
+            "description": "Non-fatal conditions that may affect how the results should be interpreted. This field is emitted before records. Successful PIT traversal does not return a semantic candidate-limit warning, but other interpretation warnings still apply.",
             "items": {
               "$ref": "#/components/schemas/SearchWarning"
             }
@@ -646,7 +647,7 @@
       },
       "SearchRequest": {
         "type": "object",
-        "description": "Optional filters and pagination controls for a CVE record search. Enum-valued fields are accepted case-insensitively and normalized to the documented canonical values. When both boundaries of a date range are supplied, the start must be before or equal to the end. Direct searches report exact totals even when they exceed 10,000, although requested pages must remain inside that window. CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. They evaluate at most 10,000 search candidates, so reported totals and pages include only matches found within that candidate set.",
+        "description": "Optional filters and pagination controls for a CVE record search. Enum-valued fields are accepted case-insensitively and normalized to the documented canonical values. When both boundaries of a date range are supplied, the start must be before or equal to the end. Direct searches report exact totals even when they exceed 10,000, although requested pages must remain inside that window. CVSS vector filters, SSVC filters, date-bounded positive knownExploited searches, cpeName, and virtualMatchString use semantic post-filtering. PIT is enabled by default on the server. All candidate batches must complete before a successful response reports exact totals; traversal failures return an error instead of partial results. With PIT explicitly disabled using SearchPitEnabled=false, these searches evaluate at most 10,000 search candidates, so reported totals and pages include only matches found within that candidate set. Reaching that limit returns SEMANTIC_CANDIDATE_LIMIT_REACHED and totalIsExact=false. Requested pages must still remain within the first 10,000 matching results. PIT is a server-side setting, not a request field.",
         "additionalProperties": false,
         "properties": {
           "scope": {

api-docs/openapi.test.js

@@ -171,6 +171,23 @@ describe('OpenAPI contract', () => {
         expect(operation.responses['410'].description).toContain('disabled by service configuration');
     });
 
+    test('distinguishes bounded semantic searches from internal PIT traversal without adding client controls', () => {
+        const operation = openapi.paths['/search'].post;
+        const request = jsonRequestBodySchema('/search', 'post');
+        for (const description of [operation.description, request.description]) {
+            expect(description).toContain('PIT is enabled by default on the server');
+            expect(description).toContain('With PIT explicitly disabled using SearchPitEnabled=false');
+            expect(description).toContain('SEMANTIC_CANDIDATE_LIMIT_REACHED');
+            expect(description).toContain('first 10,000 matching results');
+            expect(description).toContain('PIT is a server-side setting, not a request field');
+        }
+        expect(operation.description).toContain('Separate page requests use separate snapshots');
+        expect(operation.description).toContain('does not restart or fall back');
+        expect(operation.description).toContain('no public cursor');
+        expect(operation.description).toContain('Filling the requested page does not stop the scan');
+        expect(openapi.components.schemas.SearchWarning.description).toContain('can occur in either mode');
+    });
+
     test('documents cpeName as a CPE Match Criteria filter with required components', () => {
         const cpeName = openapi.components.schemas.SearchRequest.properties.cpeName;
 
@@ -199,7 +216,11 @@ describe('OpenAPI contract', () => {
         expect(responses['400'].description).toContain('Malformed JSON');
         expect(responses['415'].description).toContain('application/json');
         expect(responses['502'].description).toContain('invalid or unexpected response');
+        expect(responses['502'].description).toContain('no partial PIT result');
         expect(responses['503'].description).toContain('search service');
+        expect(responses['503'].description).toContain('missing or expired PIT search context');
+        expect(responses['504'].description).toContain('SEARCH_TIMEOUT');
+        expect(responses['504'].description).toContain('separate bounded timeout');
         expect(responses['503'].headers['Retry-After']).toStrictEqual({
             description: 'Seconds before retrying the unavailable search service',
             schema: {
@@ -250,7 +271,11 @@ describe('OpenAPI contract', () => {
             'exact number of matching records'
         );
         expect(schemas.SearchPagination.properties.totalResults.description).toContain('10,000-candidate set');
+        expect(schemas.SearchPagination.properties.totalResults.description).toContain('completed snapshot traversal');
+        expect(schemas.SearchPagination.properties.totalPages.description).toContain('number of accessible pages');
         expect(schemas.SearchPagination.properties.totalIsExact.description).toContain('additional matches may exist');
+        expect(schemas.SearchPagination.properties.totalIsExact.description).toContain('error rather than a success');
+        expect(searchResponse.properties.warnings.description).toContain('emitted before records');
         expect(schemas.CveRecord.description).toContain('without search-backend transport metadata');
         expect(schemas.ApiError.required).toStrictEqual(['error', 'message']);
         expect(schemas.ApiErrorResponse.oneOf).toHaveLength(2);

config/custom-environment-variables.jsonc

@@ -10,14 +10,17 @@
             "allowUnknownSslCerts": "OpenSearchAllowUnknownSslCerts",
             "openSearchRequestTimeoutMs": "SearchOpenSearchRequestTimeoutMs",
             "requestDeadlineMs": "SearchRequestDeadlineMs",
-            "readinessTimeoutMs": "SearchReadinessTimeoutMs"
+            "readinessTimeoutMs": "SearchReadinessTimeoutMs",
+            "pitBatchSize": "SearchPitBatchSize",
+            "pitKeepAliveMs": "SearchPitKeepAliveMs"
         },
         "requestLogging": {
             "successSampleRate": "RequestLogSuccessSampleRate",
             "healthChecks": "RequestLogHealthChecks"
         },
         "features": {
-            "webSearchEnabled": "WebSearchEnabled"
+            "webSearchEnabled": "WebSearchEnabled",
+            "searchPitEnabled": "SearchPitEnabled"
         }
     }
 }

config/default.jsonc

@@ -16,7 +16,10 @@
             // Limits total processing time for a /search request.
             "requestDeadlineMs": "30000",
             // Overall /api/ready-check deadline and per-operation timeout for field capabilities and search.
-            "readinessTimeoutMs": "5000"
+            "readinessTimeoutMs": "5000",
+            // Semantic PIT traversal settings; validate capacity for each deployment.
+            "pitBatchSize": "1000",
+            "pitKeepAliveMs": "60000"
         },
         // Access logging settings. Error responses are always logged.
         "requestLogging": {
@@ -25,10 +28,12 @@
             // Set to TRUE to include successful health checks in the same successSampleRate sampling.
             "healthChecks": "FALSE"
         },
-        // Feature flags used for operator-controlled endpoint availability.
+        // Operator-controlled feature flags.
         "features": {
             // Set to FALSE only when an environment must disable POST /webSearch.
-            "webSearchEnabled": "TRUE"
+            "webSearchEnabled": "TRUE",
+            // Set to FALSE to restore bounded semantic searches without PIT.
+            "searchPitEnabled": "TRUE"
         }
     }
 }

config/runtime-config.test.js

@@ -91,9 +91,24 @@ describe('runtime container and compose configuration', () => {
         );
         expect(customEnvironmentVariables).toContain('"requestDeadlineMs": "SearchRequestDeadlineMs"');
         expect(customEnvironmentVariables).toContain('"readinessTimeoutMs": "SearchReadinessTimeoutMs"');
+        expect(customEnvironmentVariables).toContain('"pitBatchSize": "SearchPitBatchSize"');
+        expect(customEnvironmentVariables).toContain('"pitKeepAliveMs": "SearchPitKeepAliveMs"');
         expect(customEnvironmentVariables).toContain('"successSampleRate": "RequestLogSuccessSampleRate"');
         expect(customEnvironmentVariables).toContain('"healthChecks": "RequestLogHealthChecks"');
         expect(customEnvironmentVariables).toContain('"webSearchEnabled": "WebSearchEnabled"');
+        expect(customEnvironmentVariables).toContain('"searchPitEnabled": "SearchPitEnabled"');
+    });
+
+    test('enables PIT in defaults and the environment example with bounded batch tuning', () => {
+        const defaults = readProjectFile('config/default.jsonc');
+        const envExample = readProjectFile('.env.example');
+
+        expect(defaults).toContain('"searchPitEnabled": "TRUE"');
+        expect(defaults).toContain('"pitBatchSize": "1000"');
+        expect(defaults).toContain('"pitKeepAliveMs": "60000"');
+        expect(envExample).toMatch(/^SearchPitEnabled=true$/m);
+        expect(envExample).toMatch(/^SearchPitBatchSize=1000$/m);
+        expect(envExample).toMatch(/^SearchPitKeepAliveMs=60000$/m);
     });
 
     test.each([['config/default.jsonc'], ['config/devel.jsonc'], ['config/prod.jsonc']])(

controllers/searchController.js

@@ -1,6 +1,8 @@
 const constants = require('../utils/constants');
 const errors = require('../utils/errors');
+const featureFlags = require('../utils/featureFlags');
 const searchDeadline = require('../utils/searchDeadline');
+const searchPitTraversal = require('../utils/searchPitTraversal');
 const searchProvider = require('../utils/searchProvider');
 const searchQueryBuilder = require('../utils/searchQueryBuilder');
 const searchPostFilters = require('../utils/searchPostFilters');
@@ -133,11 +135,19 @@ async function executePostFilterSearch(searchOptions, queryStringObjects, postFi
     };
 }
 
-async function executeSearch(searchOptions, queryStringObjects, postFilters, responseOptions, signal) {
+async function executeSearch(searchOptions, queryStringObjects, postFilters, responseOptions, lifecycle) {
+    const { signal, getRemainingTimeMs } = lifecycle;
     if (!searchPostFilters.hasSearchPostFilters(postFilters)) {
         return executeStandardSearch(searchOptions, queryStringObjects, signal);
     }
 
+    if (featureFlags.isSearchPitEnabled()) {
+        return searchPitTraversal.executePitSearch(
+            { searchOptions, queryStringObjects, postFilters, responseOptions },
+            { signal, getRemainingTimeMs }
+        );
+    }
+
     return executePostFilterSearch(searchOptions, queryStringObjects, postFilters, responseOptions, signal);
 }
 
@@ -158,11 +168,14 @@ async function searchHandler(req, res, next) {
             searchRequest.queryStringObjects,
             postFilters,
             responseOptions,
-            lifecycle.signal
+            lifecycle
         );
         lifecycle.throwIfAborted();
         const pagingOptions = responseOptions ?? searchOptions;
-        const warnings = [...searchResponseBuilder.getRequestWarnings(params), ...searchResult.warnings];
+        const warnings = [
+            ...searchResponseBuilder.getRequestWarnings(params, { usesPit: searchResult.usesPit }),
+            ...searchResult.warnings
+        ];
 
         return res.status(200).json(
             searchResponseBuilder.buildSearchResponse({
@@ -180,7 +193,7 @@ async function searchHandler(req, res, next) {
             return res.status(504).json(error.searchTimeoutError());
         }
 
-        if (isOpenSearchUnavailableError(err)) {
+        if (err?.code === 'SEARCH_PIT_UNAVAILABLE' || isOpenSearchUnavailableError(err)) {
             if (typeof res.set === 'function') res.set('Retry-After', '5');
 
             return res.status(503).json(error.searchUnavailableError());