From d2f3b08605aa4b8d2da1cfe8acec3219fed01515 Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 20 Oct 2026 12:15:52 +1300 Subject: [PATCH 1/3] Retry deadlocks and snapshot isolation conflicts per request set, #331 (backport of #348 and #349) --- CHANGELOG.md | 3 + Queue/Processor.php | 56 +++++++++++++ Queue/Processor/Handler.php | 54 +++++++++++- plugin.json | 2 +- tests/Framework/Mock/Tracker.php | 25 +++++- .../Queue/Processor/HandlerTest.php | 50 +++++++++++ tests/Integration/Queue/ProcessorTest.php | 83 +++++++++++++++++++ 7 files changed, 267 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1310295..0d8b70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## Changelog +5.2.1 +- Fixed queue processing failing with a fatal error, and some hits being counted twice, after a database deadlock or a MariaDB snapshot isolation conflict; requests hit by a snapshot isolation conflict are now retried + 5.2.0 - 2025-11-25 - No longer disable Provider plugin automatically. If performance issues occur, please disable it manually or install an ASN geolocation database (https://matomo.org/faq/how-to/setting-up-accurate-visitors-geolocation/). diff --git a/Queue/Processor.php b/Queue/Processor.php index e69a327..eace7c2 100644 --- a/Queue/Processor.php +++ b/Queue/Processor.php @@ -11,6 +11,8 @@ namespace Piwik\Plugins\QueuedTracking\Queue; use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Log\LoggerInterface; use Piwik\Tracker; use Piwik\Tracker\RequestSet; use Piwik\Plugins\QueuedTracking\Queue; @@ -62,6 +64,12 @@ class Processor */ private $numMinQueueIterations = 2; + /** + * How many transaction errors a request set may hit while a batch is retried. A set that reaches it is dropped, so + * that it does not block its queue, and the rest of the batch is retried without it. + */ + private const MAX_TRANSACTION_ERRORS_PER_REQUEST_SET = 3; + public function __construct(Queue\Manager $queueManager) { $this->queueManager = $queueManager; @@ -146,6 +154,54 @@ public function process(Tracker $tracker = null) } $failedRequestSets = $this->processRequestSets($tracker, $requestSetsToRetry); + + // the retry is rolled back as a whole after a transaction error, so repeat it while we still + // hold the lock, rather than leave the batch queued for a run that would count afresh + + // keyed by spl_object_id(), as the sets stay referenced until the batch is done + $numTransactionErrors = []; + $numDroppedRequestSets = 0; + for ($retry = 1; !empty($failedRequestSets) && $this->handler->hasTransactionErrors(); $retry++) { + foreach ($this->handler->getRequestSetsWithTransactionErrors() as $requestSet) { + $id = spl_object_id($requestSet); + $numTransactionErrors[$id] = ($numTransactionErrors[$id] ?? 0) + 1; + } + + // the previous pass already left out, or truncated, the sets that failed for other reasons. + // Each pass counts an error against at least one set, and a set is dropped once it reaches + // the limit, so this ends + $requestSetsToRetry = []; + foreach ($failedRequestSets as $requestSet) { + if (($numTransactionErrors[spl_object_id($requestSet)] ?? 0) >= self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET) { + $numDroppedRequestSets++; + } else { + $requestSetsToRetry[] = $requestSet; + } + } + + if (!empty($requestSetsToRetry)) { + // give the conflicting transaction time to finish + usleep(100000 * min($retry, self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET)); + } + + $failedRequestSets = $this->processRequestSets($tracker, $requestSetsToRetry); + } + + if ($numDroppedRequestSets > 0) { + $message = sprintf( + 'Dropping %d request sets from queue %s after repeated transaction errors.', + $numDroppedRequestSets, + $queue->getId() + ); + if (!empty($failedRequestSets)) { + $message .= sprintf( + ' %d more are dropped because their last retry was rolled back after another request set failed.', + count($failedRequestSets) + ); + } + StaticContainer::get(LoggerInterface::class)->warning($message); + } + if (!empty($failedRequestSets)) { Common::printDebug('Failed processing ' . count($failedRequestSets) . ' request sets.'); } diff --git a/Queue/Processor/Handler.php b/Queue/Processor/Handler.php index f0cdb7b..d595df9 100644 --- a/Queue/Processor/Handler.php +++ b/Queue/Processor/Handler.php @@ -12,7 +12,6 @@ use Piwik\Common; use Piwik\Container\StaticContainer; -use Piwik\Db; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Plugins\QueuedTracking\Configuration; use Piwik\Tracker; @@ -24,6 +23,7 @@ class Handler protected $transactionId; private $hasError = false; + private $requestSetsWithTransactionErrors = []; private $requestSetsToRetry = array(); private $count = 0; private $numTrackedRequestsBeginning = 0; @@ -32,6 +32,7 @@ public function init(Tracker $tracker) { $this->requestSetsToRetry = array(); $this->hasError = false; + $this->requestSetsWithTransactionErrors = []; $this->numTrackedRequestsBeginning = $tracker->getCountOfLoggedRequests(); $this->transactionId = $this->getDb()->beginTransaction(); } @@ -83,10 +84,11 @@ public function onException(RequestSet $requestSet, Exception $e) Common::printDebug('Got exception: ' . $e->getMessage()); - // retry if a deadlock or a lock wait timeout happened - if (Db::get()->isErrNo($e, 1213) || Db::get()->isErrNo($e, 1205)) { + if ($this->isRetryableTransactionError($e)) { + $this->requestSetsWithTransactionErrors[] = $requestSet; $this->requestSetsToRetry[] = $requestSet; - Common::printDebug('Added deadlocked requestSet to requestSetsToRetry'); + Common::printDebug('Added requestSet that hit a transaction error to requestSetsToRetry'); + $this->restartTransaction(); return; } @@ -103,6 +105,23 @@ public function hasErrors() return $this->hasError; } + /** + * Whether a deadlock, lock wait timeout or snapshot isolation conflict happened since init(). + */ + public function hasTransactionErrors(): bool + { + return !empty($this->requestSetsWithTransactionErrors); + } + + /** + * @return RequestSet[] the request sets that hit a deadlock, lock wait timeout or snapshot isolation conflict + * since init() + */ + public function getRequestSetsWithTransactionErrors(): array + { + return $this->requestSetsWithTransactionErrors; + } + public function rollBack(Tracker $tracker) { $tracker->setCountOfLoggedRequests($this->numTrackedRequestsBeginning); @@ -123,6 +142,33 @@ public function commit() $this->requestSetsToRetry = array(); } + /** + * Deadlock (1213), lock wait timeout (1205) and MariaDB's snapshot isolation conflict (1020). + */ + private function isRetryableTransactionError(Exception $e): bool + { + $db = $this->getDb(); + + return $db->isErrNo($e, 1213) || $db->isErrNo($e, 1205) || $db->isErrNo($e, 1020); + } + + /** + * These errors can end the transaction on the server, after which PDO runs every later query in autocommit + * mode, so the remaining request sets would be stored even though the batch is rolled back and retried. + * Every request set processed so far is already queued for retry, so discard their work and start over. + */ + private function restartTransaction(): void + { + try { + $this->getDb()->rollBack($this->transactionId); + } catch (Exception $e) { + // PDO throws "There is no active transaction" once the server has rolled back. + Common::printDebug('Rollback after a transaction error failed: ' . $e->getMessage()); + } + + $this->transactionId = $this->getDb()->beginTransaction(); + } + protected function getDb() { return Tracker::getDatabase(); diff --git a/plugin.json b/plugin.json index d932228..f32e29e 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "QueuedTracking", - "version": "5.2.0", + "version": "5.2.1", "description": "Scale your large traffic Matomo service by queuing tracking requests in Redis or MySQL for better performance and reliability when experiencing peaks.", "theme": false, "keywords": ["tracker", "tracking", "queue", "redis"], diff --git a/tests/Framework/Mock/Tracker.php b/tests/Framework/Mock/Tracker.php index 1b695b4..7b0bfd1 100644 --- a/tests/Framework/Mock/Tracker.php +++ b/tests/Framework/Mock/Tracker.php @@ -13,13 +13,36 @@ class Tracker extends \Piwik\Tests\Framework\Mock\Tracker { + public $numDeadlocks = 0; + + /** + * Keyed by the request's params, as a request is restored from the queue as a new object. + * @var int[] + */ + private $numDeadlocksPerRequest = []; + public function trackRequest(Request $request) { $allParams = $request->getRawParams(); - if (!empty($allParams['forceThrow'])) { + $key = json_encode($allParams); + $numOwnDeadlocks = $this->numDeadlocksPerRequest[$key] ?? 0; + + if (!empty($allParams['forceThrow']) || (!empty($allParams['forceThrowAfterDeadlock']) && $numOwnDeadlocks)) { throw new ForcedException("forced exception"); } + if ( + !empty($allParams['forceDeadlock']) + || (!empty($allParams['forceDeadlockOnce']) && !$numOwnDeadlocks) + || (!empty($allParams['forceDeadlockUntil']) && $numOwnDeadlocks < $allParams['forceDeadlockUntil']) + || (!empty($allParams['forceDeadlockOnceAfterDeadlocks']) && !$numOwnDeadlocks + && $this->numDeadlocks >= $allParams['forceDeadlockOnceAfterDeadlocks']) + ) { + $this->numDeadlocks++; + $this->numDeadlocksPerRequest[$key] = $numOwnDeadlocks + 1; + throw new ForcedException('SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction'); + } + return parent::trackRequest($request); } } diff --git a/tests/Integration/Queue/Processor/HandlerTest.php b/tests/Integration/Queue/Processor/HandlerTest.php index 953ba2c..4102b1c 100644 --- a/tests/Integration/Queue/Processor/HandlerTest.php +++ b/tests/Integration/Queue/Processor/HandlerTest.php @@ -9,6 +9,8 @@ namespace Piwik\Plugins\QueuedTracking\tests\Integration\Queue\Processor; +use Piwik\Common; +use Piwik\Db; use Piwik\Plugins\QueuedTracking\Queue\Processor\Handler; use Piwik\Plugins\QueuedTracking\tests\Framework\Mock\ForcedException; use Piwik\Plugins\QueuedTracking\tests\Framework\Mock\Tracker; @@ -100,10 +102,14 @@ public function test_onException_shouldMarkTheHandlerAsFaulty() public function test_init_shouldResetAFaultyHandler() { $this->handler->onException(new RequestSet(), new \Exception()); + $this->handler->onException(new RequestSet(), new \Exception('SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction')); $this->assertTrue($this->handler->hasErrors()); + $this->assertTrue($this->handler->hasTransactionErrors()); + $this->handler->rollBack($this->tracker); $this->handler->init($this->tracker); $this->assertFalse($this->handler->hasErrors()); + $this->assertFalse($this->handler->hasTransactionErrors()); } public function test_process_ShouldForwardTheRequestToTheTracker() @@ -285,6 +291,50 @@ public function test_process_ShouldRestoreTheEnvironmentOfARequest() $_COOKIE = $cookieBackup; } + /** + * @dataProvider getRetryableTransactionErrors + */ + public function testOnExceptionShouldRetryTheRequestSetAndDiscardWorkDoneInTheTransaction(string $message, bool $serverEndsTransaction) + { + $this->insertOption('QueuedTrackingTest_beforeError'); + + if ($serverEndsTransaction) { + Tracker::getDatabase()->query('ROLLBACK'); + } + + $requestSet = $this->buildRequestSet(3); + $this->handler->onException($requestSet, new \Exception($message)); + + // without a new transaction PDO would store this in autocommit mode and the rollBack below would throw + $this->insertOption('QueuedTrackingTest_afterError'); + + $this->handler->rollBack($this->tracker); + + $this->assertEquals([$requestSet], $this->handler->getRequestSetsToRetry()); + $this->assertSame(3, $requestSet->getNumberOfRequests()); + $this->assertSame(0, (int) Db::fetchOne( + 'SELECT COUNT(*) FROM `' . Common::prefixTable('option') . '` WHERE option_name LIKE ?', + ['QueuedTrackingTest_%'] + )); + } + + public function getRetryableTransactionErrors(): array + { + return [ + 'deadlock' => ['SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction', true], + 'snapshot isolation' => ["SQLSTATE[HY000]: General error: 1020 Record has changed since last read in table 'log_visit'; try restarting transaction", true], + 'lock wait timeout' => ['SQLSTATE[HY000]: General error: 1205 Lock wait timeout exceeded; try restarting transaction', false], + ]; + } + + private function insertOption(string $name): void + { + Tracker::getDatabase()->query( + 'INSERT INTO `' . Common::prefixTable('option') . '` (option_name, option_value, autoload) VALUES (?, ?, 0)', + [$name, '1'] + ); + } + private function createHandler() { return new TestHandler(); diff --git a/tests/Integration/Queue/ProcessorTest.php b/tests/Integration/Queue/ProcessorTest.php index 1661fdb..36ba5d9 100644 --- a/tests/Integration/Queue/ProcessorTest.php +++ b/tests/Integration/Queue/ProcessorTest.php @@ -11,6 +11,8 @@ use Piwik\Plugins\QueuedTracking\tests\Framework\TestCase\IntegrationTestCase; use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\Mock\Tracker\RequestSet; +use Piwik\Tracker\Request; use Piwik\Tracker\TrackerConfig; use Piwik\Tracker; use Piwik\Plugins\QueuedTracking\Queue; @@ -354,6 +356,87 @@ public function test_process_ShouldActuallyRetryProcessingAllRequestsWithoutTheF $mock->process($this->createTracker()); } + public function testProcessShouldTrackEveryRequestOnceIfTheRetrySucceedsAfterATransactionError() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(3); + + $deadlockingRequestSet = new RequestSet(); + $deadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockOnce' => 1])]); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + $this->queue->addRequestSetToQueues($deadlockingRequestSet); + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertNumberOfRequestSetsLeftInQueue(0); + $this->assertSame(3, $tracker->getCountOfLoggedRequests()); + } + + public function testProcessShouldDropOnlyTheConflictingRequestSetsIfTheBatchKeepsHittingTransactionErrors() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(3); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + $deadlockingRequestSet = new RequestSet(); + $deadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlock' => 1])]); + $this->queue->addRequestSetToQueues($deadlockingRequestSet); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(2)); + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertSame(4, $tracker->numDeadlocks); + $this->assertSame(3, $tracker->getCountOfLoggedRequests()); + $this->assertNumberOfRequestSetsLeftInQueue(0); + } + + public function testProcessShouldLeaveOutRequestSetsThatFailedInThePreviousRetryWhenRetryingAgainAfterATransactionError() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(3); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + // deadlocks in the first pass, then fails for another reason in the second + $failingRequestSet = new RequestSet(); + $failingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockOnce' => 1, 'forceThrowAfterDeadlock' => 1])]); + $this->queue->addRequestSetToQueues($failingRequestSet); + + // deadlocks in the first two passes + $deadlockingRequestSet = new RequestSet(); + $deadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockUntil' => 2])]); + $this->queue->addRequestSetToQueues($deadlockingRequestSet); + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertSame(3, $tracker->numDeadlocks); + $this->assertSame(2, $tracker->getCountOfLoggedRequests()); + $this->assertNumberOfRequestSetsLeftInQueue(0); + } + + public function testProcessShouldNotDropARequestSetThatHitsItsFirstTransactionErrorAfterAnotherSetWasDropped() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(3); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + $deadlockingRequestSet = new RequestSet(); + $deadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlock' => 1])]); + $this->queue->addRequestSetToQueues($deadlockingRequestSet); + + // deadlocks once, in the pass after which the set above reaches the limit and is dropped + $lateDeadlockingRequestSet = new RequestSet(); + $lateDeadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockOnceAfterDeadlocks' => 4])]); + $this->queue->addRequestSetToQueues($lateDeadlockingRequestSet); + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertSame(5, $tracker->numDeadlocks); + $this->assertSame(2, $tracker->getCountOfLoggedRequests()); + $this->assertNumberOfRequestSetsLeftInQueue(0); + } + public function test_process_shouldRestoreEnvironmentAfterTrackingRequests() { $serverBackup = $_SERVER; From 6fabeda9aea9923dd051a3956caad677cd522f77 Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 20 Oct 2026 12:41:50 +1300 Subject: [PATCH 2/3] Keep retrying a batch after any error, #331 --- CHANGELOG.md | 1 + Queue/Processor.php | 27 +++++++---------------- Queue/Processor/Handler.php | 4 ++-- tests/Integration/Queue/ProcessorTest.php | 18 +++++++++++++++ 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d8b70b..bd16ea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ 5.2.1 - Fixed queue processing failing with a fatal error, and some hits being counted twice, after a database deadlock or a MariaDB snapshot isolation conflict; requests hit by a snapshot isolation conflict are now retried +- Fixed requests being lost when the retry after a database deadlock or snapshot isolation conflict failed because of a different request 5.2.0 - 2025-11-25 - No longer disable Provider plugin automatically. If performance issues occur, please disable it manually or install an ASN geolocation database (https://matomo.org/faq/how-to/setting-up-accurate-visitors-geolocation/). diff --git a/Queue/Processor.php b/Queue/Processor.php index eace7c2..2c18c1c 100644 --- a/Queue/Processor.php +++ b/Queue/Processor.php @@ -155,21 +155,21 @@ public function process(Tracker $tracker = null) $failedRequestSets = $this->processRequestSets($tracker, $requestSetsToRetry); - // the retry is rolled back as a whole after a transaction error, so repeat it while we still - // hold the lock, rather than leave the batch queued for a run that would count afresh + // the retry is rolled back as a whole after any error, so repeat it while we still hold the + // lock, rather than leave the batch queued for a run that would count afresh // keyed by spl_object_id(), as the sets stay referenced until the batch is done $numTransactionErrors = []; $numDroppedRequestSets = 0; - for ($retry = 1; !empty($failedRequestSets) && $this->handler->hasTransactionErrors(); $retry++) { + for ($retry = 1; !empty($failedRequestSets); $retry++) { foreach ($this->handler->getRequestSetsWithTransactionErrors() as $requestSet) { $id = spl_object_id($requestSet); $numTransactionErrors[$id] = ($numTransactionErrors[$id] ?? 0) + 1; } // the previous pass already left out, or truncated, the sets that failed for other reasons. - // Each pass counts an error against at least one set, and a set is dropped once it reaches - // the limit, so this ends + // Each failed pass either counts an error against a set, which is dropped once it reaches + // the limit, or shortens or leaves out a set, so this ends $requestSetsToRetry = []; foreach ($failedRequestSets as $requestSet) { if (($numTransactionErrors[spl_object_id($requestSet)] ?? 0) >= self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET) { @@ -179,7 +179,7 @@ public function process(Tracker $tracker = null) } } - if (!empty($requestSetsToRetry)) { + if (!empty($requestSetsToRetry) && $this->handler->hasTransactionErrors()) { // give the conflicting transaction time to finish usleep(100000 * min($retry, self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET)); } @@ -188,22 +188,11 @@ public function process(Tracker $tracker = null) } if ($numDroppedRequestSets > 0) { - $message = sprintf( + StaticContainer::get(LoggerInterface::class)->warning(sprintf( 'Dropping %d request sets from queue %s after repeated transaction errors.', $numDroppedRequestSets, $queue->getId() - ); - if (!empty($failedRequestSets)) { - $message .= sprintf( - ' %d more are dropped because their last retry was rolled back after another request set failed.', - count($failedRequestSets) - ); - } - StaticContainer::get(LoggerInterface::class)->warning($message); - } - - if (!empty($failedRequestSets)) { - Common::printDebug('Failed processing ' . count($failedRequestSets) . ' request sets.'); + )); } $queue->markRequestSetsAsProcessed(); diff --git a/Queue/Processor/Handler.php b/Queue/Processor/Handler.php index d595df9..643c272 100644 --- a/Queue/Processor/Handler.php +++ b/Queue/Processor/Handler.php @@ -39,10 +39,10 @@ public function init(Tracker $tracker) public function process(Tracker $tracker, RequestSet $requestSet) { - $requestSet->restoreEnvironment(); - $this->count = 0; + $requestSet->restoreEnvironment(); + foreach ($requestSet->getRequests() as $request) { try { $startMs = round(microtime(true) * 1000); diff --git a/tests/Integration/Queue/ProcessorTest.php b/tests/Integration/Queue/ProcessorTest.php index 36ba5d9..f4c5e0d 100644 --- a/tests/Integration/Queue/ProcessorTest.php +++ b/tests/Integration/Queue/ProcessorTest.php @@ -437,6 +437,24 @@ public function testProcessShouldNotDropARequestSetThatHitsItsFirstTransactionEr $this->assertNumberOfRequestSetsLeftInQueue(0); } + public function testProcessShouldRetryAgainIfTheRetryFailsForAnotherReasonAfterATransactionError() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(2); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + // deadlocks in the first pass, then fails for another reason in the second + $failingRequestSet = new RequestSet(); + $failingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockOnce' => 1, 'forceThrowAfterDeadlock' => 1])]); + $this->queue->addRequestSetToQueues($failingRequestSet); + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertSame(1, $tracker->numDeadlocks); + $this->assertSame(1, $tracker->getCountOfLoggedRequests()); + $this->assertNumberOfRequestSetsLeftInQueue(0); + } + public function test_process_shouldRestoreEnvironmentAfterTrackingRequests() { $serverBackup = $_SERVER; From 496d35b60bbe3c590358d1c923e8c1f58af6e6d7 Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 20 Oct 2026 17:33:35 +1300 Subject: [PATCH 3/3] Drop sets that keep conflicting once a batch has been retried six times, #331 Sets that conflict in turn could otherwise keep the queue locked for up to three passes per set. Also tests that a set whose environment cannot be restored is not cut down using the previous set's count. --- Queue/Processor.php | 14 ++++++++++- tests/Framework/Mock/Tracker.php | 11 ++++++++- .../Queue/Processor/HandlerTest.php | 23 ++++++++++++++++++ tests/Integration/Queue/ProcessorTest.php | 24 +++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/Queue/Processor.php b/Queue/Processor.php index 2c18c1c..e3af985 100644 --- a/Queue/Processor.php +++ b/Queue/Processor.php @@ -70,6 +70,12 @@ class Processor */ private const MAX_TRANSACTION_ERRORS_PER_REQUEST_SET = 3; + /** + * After this many retries of a batch, a request set that hits a transaction error is dropped straight away, so that + * sets which conflict in turn cannot keep the queue locked for long. + */ + private const MAX_RETRIES_PER_BATCH = 2 * self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET; + public function __construct(Queue\Manager $queueManager) { $this->queueManager = $queueManager; @@ -162,9 +168,11 @@ public function process(Tracker $tracker = null) $numTransactionErrors = []; $numDroppedRequestSets = 0; for ($retry = 1; !empty($failedRequestSets); $retry++) { + $hasTransactionErrorInLatestPass = []; foreach ($this->handler->getRequestSetsWithTransactionErrors() as $requestSet) { $id = spl_object_id($requestSet); $numTransactionErrors[$id] = ($numTransactionErrors[$id] ?? 0) + 1; + $hasTransactionErrorInLatestPass[$id] = true; } // the previous pass already left out, or truncated, the sets that failed for other reasons. @@ -172,7 +180,11 @@ public function process(Tracker $tracker = null) // the limit, or shortens or leaves out a set, so this ends $requestSetsToRetry = []; foreach ($failedRequestSets as $requestSet) { - if (($numTransactionErrors[spl_object_id($requestSet)] ?? 0) >= self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET) { + $id = spl_object_id($requestSet); + if ( + ($numTransactionErrors[$id] ?? 0) >= self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET + || ($retry >= self::MAX_RETRIES_PER_BATCH && isset($hasTransactionErrorInLatestPass[$id])) + ) { $numDroppedRequestSets++; } else { $requestSetsToRetry[] = $requestSet; diff --git a/tests/Framework/Mock/Tracker.php b/tests/Framework/Mock/Tracker.php index 7b0bfd1..8d57856 100644 --- a/tests/Framework/Mock/Tracker.php +++ b/tests/Framework/Mock/Tracker.php @@ -16,7 +16,14 @@ class Tracker extends \Piwik\Tests\Framework\Mock\Tracker public $numDeadlocks = 0; /** - * Keyed by the request's params, as a request is restored from the queue as a new object. + * Counted by TestProcessor over the tracker's lifetime, so in a test that processes a single batch a pass's index is + * one less: 0 for its first processing, n for its nth retry. + */ + public $numPasses = 0; + + /** + * Keyed by the request's params, as a request is restored from the queue as a new object. Requests with identical + * params therefore share a count, so give each one a param of its own when a test needs them counted apart. * @var int[] */ private $numDeadlocksPerRequest = []; @@ -37,6 +44,8 @@ public function trackRequest(Request $request) || (!empty($allParams['forceDeadlockUntil']) && $numOwnDeadlocks < $allParams['forceDeadlockUntil']) || (!empty($allParams['forceDeadlockOnceAfterDeadlocks']) && !$numOwnDeadlocks && $this->numDeadlocks >= $allParams['forceDeadlockOnceAfterDeadlocks']) + || (isset($allParams['forceDeadlockInPasses']) + && in_array((string) ($this->numPasses - 1), explode(',', $allParams['forceDeadlockInPasses']), true)) ) { $this->numDeadlocks++; $this->numDeadlocksPerRequest[$key] = $numOwnDeadlocks + 1; diff --git a/tests/Integration/Queue/Processor/HandlerTest.php b/tests/Integration/Queue/Processor/HandlerTest.php index 4102b1c..8d7229b 100644 --- a/tests/Integration/Queue/Processor/HandlerTest.php +++ b/tests/Integration/Queue/Processor/HandlerTest.php @@ -168,6 +168,29 @@ public function test_onException_shouldRemoveAllInvalidRequestsFromValidRequests $this->assertEquals(array('idsite' => '1', 'index' => '3'), $requests[3]->getParams()); } + public function testOnExceptionShouldNotKeepRequestsOfASetWhoseEnvironmentCouldNotBeRestored() + { + $requestSet = $this->buildRequestSet(5); + $this->handler->process($this->tracker, $requestSet); + + $failingRequestSet = new class extends RequestSet { + public function restoreEnvironment() + { + throw new ForcedException('forced exception'); + } + }; + $failingRequestSet->setRequests($this->buildRequestSet(3)->getRequests()); + + try { + $this->handler->process($this->tracker, $failingRequestSet); + $this->fail('An expected exception was not triggered'); + } catch (ForcedException $e) { + $this->handler->onException($failingRequestSet, $e); + } + + $this->assertEquals([$requestSet], $this->handler->getRequestSetsToRetry()); + } + public function test_rollBack_shouldAddTheSuccessfullyProcessedRequestSetToAllValidRequestSets() { $requestSet1 = $this->buildRequestSet(5); diff --git a/tests/Integration/Queue/ProcessorTest.php b/tests/Integration/Queue/ProcessorTest.php index f4c5e0d..ce40bf0 100644 --- a/tests/Integration/Queue/ProcessorTest.php +++ b/tests/Integration/Queue/ProcessorTest.php @@ -22,6 +22,10 @@ class TestProcessor extends Processor { public function processRequestSets(Tracker $tracker, $queuedRequestSets) { + if (!empty($queuedRequestSets) && $tracker instanceof \Piwik\Plugins\QueuedTracking\tests\Framework\Mock\Tracker) { + $tracker->numPasses++; + } + return parent::processRequestSets($tracker, $queuedRequestSets); } } @@ -455,6 +459,26 @@ public function testProcessShouldRetryAgainIfTheRetryFailsForAnotherReasonAfterA $this->assertNumberOfRequestSetsLeftInQueue(0); } + public function testProcessShouldDropAConflictingRequestSetBelowItsLimitOnceTheBatchHasBeenRetriedTooOften() + { + $this->queue->setNumberOfRequestsToProcessAtSameTime(4); + + $this->queue->addRequestSetToQueues($this->buildRequestSet(1)); + + // the sets conflict in turn, so none reaches three errors by the sixth retry, which the first one fails again + foreach (['0,3,6', '1,4', '2,5'] as $passes) { + $deadlockingRequestSet = new RequestSet(); + $deadlockingRequestSet->setRequests([new Request(['idsite' => '1', 'forceDeadlockInPasses' => $passes])]); + $this->queue->addRequestSetToQueues($deadlockingRequestSet); + } + + $tracker = $this->processor->process($this->createTracker()); + + $this->assertSame(7, $tracker->numDeadlocks); + $this->assertSame(3, $tracker->getCountOfLoggedRequests()); + $this->assertNumberOfRequestSetsLeftInQueue(0); + } + public function test_process_shouldRestoreEnvironmentAfterTrackingRequests() { $serverBackup = $_SERVER;