feat(jobs): show the background job queue health, failures by job, and two admin actions

README.md

@@ -136,6 +136,35 @@ as far as open_basedir lets it check, one written before the last reboot or
 on another host, one older than 26 hours, and one dated more than five
 minutes in the future (a clock problem).
 
+## Background jobs
+
+The **Background jobs** page shows how background jobs run: the cron mode and
+its last run, the queue (queued jobs, jobs reserved now, jobs stuck for more
+than 12 hours, how long ago the queued job that waited longest last ran), the
+jobs cron has reserved, failed and crashed runs grouped by job with their
+count and latest failure, the latest runs, the job kinds with the most queued
+jobs and the slowest jobs. The queue is read from the `jobs` table only while
+the page is shown, and is cached for a minute.
+
+Admins can act on a single job. Both actions ask for the password, are limited
+to 20 per user in 10 minutes and are logged at info level with the admin, the
+job id and its class. Neither runs job code in the web request; cron runs the
+job.
+
+- **Release reservation**, on a job reserved for more than 12 hours: clears
+  the reservation, as `IJobList::unlockJob()` does, and keeps the last run, so
+  a timed job keeps its schedule. Cron already ignores a reservation that old
+  and takes the job again, so this clears the leftover reservation, which
+  otherwise counts as a running job.
+- **Run at next cron**, on a stuck job, or on a failing job when its class has
+  exactly one job queued: clears the last run and the reservation, as
+  `IJobList::resetBackgroundJob()` does, and puts the job first in line, so the
+  next cron run starts it. A job reserved within the last 12 hours may be
+  running and is left alone. Outside a configured `maintenance_window_start`,
+  cron runs only time-sensitive jobs, so other jobs wait for the window.
+
+Removing jobs is left to `occ background-job:delete`.
+
 ## API
 
 The API provides a lot of information about a running Nextcloud

appinfo/routes.php

@@ -27,6 +27,9 @@
 		['name' => 'database#settings', 'url' => '/database/settings', 'verb' => 'GET'],
 		['name' => 'database#updateSettings', 'url' => '/database/settings', 'verb' => 'PUT'],
 		['name' => 'database#testConnection', 'url' => '/database/test-connection', 'verb' => 'POST'],
+		['name' => 'jobs#queue', 'url' => '/jobs', 'verb' => 'GET'],
+		['name' => 'jobs#release', 'url' => '/jobs/{id}/release', 'verb' => 'POST', 'requirements' => ['id' => '\d+']],
+		['name' => 'jobs#runAtNextCron', 'url' => '/jobs/{id}/run', 'verb' => 'POST', 'requirements' => ['id' => '\d+']],
 		['name' => 'os#status', 'url' => '/os', 'verb' => 'GET'],
 		['name' => 'os#refresh', 'url' => '/os/refresh', 'verb' => 'POST'],
 		['name' => 'os#updateSettings', 'url' => '/os/settings', 'verb' => 'PUT'],

css/serverinfo-main.css

@@ -1,2 +1,2 @@
 /* extracted by css-entry-points-plugin */
-@import './main-C-zeP7Uf.chunk.css';
+@import './main-DTpL9qUd.chunk.css';

js/toast-BFv9rUfa.chunk.mjs.license

@@ -0,0 +1,12 @@
+SPDX-License-Identifier: AGPL-3.0-or-later
+SPDX-License-Identifier: Apache-2.0
+SPDX-FileCopyrightText: @nextcloud/dialogs developers
+SPDX-FileCopyrightText: Austin Andrews
+
+This file is generated from multiple sources. Included packages:
+- @mdi/js
+	- version: 7.4.47
+	- license: Apache-2.0
+- @nextcloud/dialogs
+	- version: 7.5.0
+	- license: AGPL-3.0-or-later

js/toast-BFv9rUfa.chunk.mjs.map.license

@@ -0,0 +1,12 @@
+SPDX-License-Identifier: AGPL-3.0-or-later
+SPDX-License-Identifier: Apache-2.0
+SPDX-FileCopyrightText: @nextcloud/dialogs developers
+SPDX-FileCopyrightText: Austin Andrews
+
+This file is generated from multiple sources. Included packages:
+- @mdi/js
+	- version: 7.4.47
+	- license: Apache-2.0
+- @nextcloud/dialogs
+	- version: 7.5.0
+	- license: AGPL-3.0-or-later

lib/Collector/BackgroundJobs.php

@@ -18,6 +18,7 @@
 use OCP\IAppConfig;
 use OCP\IConfig;
 use OCP\IDBConnection;
+use OCP\Snowflake\ISnowflakeDecoder;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -38,27 +39,30 @@
  *     maxDuration: int,
  *     memoryPeak: int
  * }
+ * @psalm-type ServerInfoFailedJob = array{
+ *     className: string,
+ *     failures: int,
+ *     lastFailedAt: int
+ * }
  */
 class BackgroundJobs {
 	private const LIMIT = 10;
 	private const SLOWEST_LIMIT = 5;
+	private const FAILED_CLASSES_LIMIT = 20;
 
 	public function __construct(
 		private IJobRuns $jobRuns,
 		private IConfig $config,
 		private IAppConfig $appConfig,
 		private IDBConnection $connection,
+		private ISnowflakeDecoder $snowflakeDecoder,
 		private LoggerInterface $logger,
 	) {
 	}
 
 	/**
-	 * Failures are queried separately: completedJobs() is ordered by run id, so on
-	 * a busy instance a failure scrolls out of the newest rows within seconds.
-	 *
 	 * @return array{
 	 *     recent: list<ServerInfoJobRun>,
-	 *     failures: list<ServerInfoJobRun>,
 	 *     slowest: list<ServerInfoSlowJob>,
 	 *     retentionDays: int
 	 * }
@@ -69,14 +73,50 @@ public function get(): array {
 
 		return [
 			'recent' => $this->collect($this->jobRuns->completedJobs(limit: self::LIMIT)),
-			'failures' => $this->collect(
-				$this->jobRuns->completedJobs([JobStatus::FAILED, JobStatus::CRASHED], limit: self::LIMIT),
-			),
 			'slowest' => $slowest,
 			'retentionDays' => $this->config->getSystemValueInt('background_jobs_expiration_days', 60),
 		];
 	}
 
+	/**
+	 * Failed and crashed runs per job class, most recent failure first.
+	 *
+	 * Grouped rather than listed: one job failing every five minutes would
+	 * otherwise push every other failure out of view. The status index keeps
+	 * this to the failed rows.
+	 *
+	 * @return list<ServerInfoFailedJob>
+	 */
+	public function failuresByClass(): array {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('c.class_name')
+			->selectAlias($query->func()->count(), 'failures')
+			// Run ids are snowflakes, so the highest is the latest
+			->selectAlias($query->func()->max('r.run_id'), 'last_run_id')
+			->from('job_runs', 'r')
+			->innerJoin('r', 'job_classes_registry', 'c', $query->expr()->eq('r.class_id', 'c.class_id'))
+			->where($query->expr()->in('r.status', $query->createNamedParameter(
+				[JobStatus::FAILED->value, JobStatus::CRASHED->value],
+				IQueryBuilder::PARAM_INT_ARRAY,
+			)))
+			->groupBy('r.class_id', 'c.class_name');
+
+		$result = $query->executeQuery();
+		$failed = [];
+		while ($row = $result->fetch()) {
+			$failed[] = [
+				'className' => (string)$row['class_name'],
+				'failures' => (int)$row['failures'],
+				'lastFailedAt' => $this->snowflakeDecoder->decode((string)$row['last_run_id'])->getCreatedAt()->getTimestamp(),
+			];
+		}
+		$result->closeCursor();
+
+		usort($failed, static fn (array $a, array $b): int => $b['lastFailedAt'] <=> $a['lastFailedAt']);
+
+		return array_slice($failed, 0, self::FAILED_CLASSES_LIMIT);
+	}
+
 	/**
 	 * Aggregates every retained run per job class: too expensive for a request, so
 	 * a background job caches the result and get() only reads it back.