Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
## Changelog

5.2.1
- Fixed queue processing failing with a fatal error, and some hits being counted twice, after a database deadlock or a MariaDB snapshot isolation conflict; requests hit by a snapshot isolation conflict are now retried
- Fixed requests being lost when the retry after a database deadlock or snapshot isolation conflict failed because of a different request

5.2.0 - 2025-11-25
- No longer disable Provider plugin automatically. If performance issues occur, please disable it manually or install an ASN geolocation database (https://matomo.org/faq/how-to/setting-up-accurate-visitors-geolocation/).

Expand Down
61 changes: 59 additions & 2 deletions Queue/Processor.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
namespace Piwik\Plugins\QueuedTracking\Queue;

use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\Log\LoggerInterface;
use Piwik\Tracker;
use Piwik\Tracker\RequestSet;
use Piwik\Plugins\QueuedTracking\Queue;
Expand Down Expand Up @@ -62,6 +64,18 @@ class Processor
*/
private $numMinQueueIterations = 2;

/**
* How many transaction errors a request set may hit while a batch is retried. A set that reaches it is dropped, so
* that it does not block its queue, and the rest of the batch is retried without it.
*/
private const MAX_TRANSACTION_ERRORS_PER_REQUEST_SET = 3;

/**
* After this many retries of a batch, a request set that hits a transaction error is dropped straight away, so that
* sets which conflict in turn cannot keep the queue locked for long.
*/
private const MAX_RETRIES_PER_BATCH = 2 * self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET;

public function __construct(Queue\Manager $queueManager)
{
$this->queueManager = $queueManager;
Expand Down Expand Up @@ -146,8 +160,51 @@ public function process(Tracker $tracker = null)
}

$failedRequestSets = $this->processRequestSets($tracker, $requestSetsToRetry);
if (!empty($failedRequestSets)) {
Common::printDebug('Failed processing ' . count($failedRequestSets) . ' request sets.');

// the retry is rolled back as a whole after any error, so repeat it while we still hold the
// lock, rather than leave the batch queued for a run that would count afresh

// keyed by spl_object_id(), as the sets stay referenced until the batch is done
$numTransactionErrors = [];
$numDroppedRequestSets = 0;
for ($retry = 1; !empty($failedRequestSets); $retry++) {
$hasTransactionErrorInLatestPass = [];
foreach ($this->handler->getRequestSetsWithTransactionErrors() as $requestSet) {
$id = spl_object_id($requestSet);
$numTransactionErrors[$id] = ($numTransactionErrors[$id] ?? 0) + 1;
$hasTransactionErrorInLatestPass[$id] = true;
}

// the previous pass already left out, or truncated, the sets that failed for other reasons.
// Each failed pass either counts an error against a set, which is dropped once it reaches
// the limit, or shortens or leaves out a set, so this ends
$requestSetsToRetry = [];
foreach ($failedRequestSets as $requestSet) {
$id = spl_object_id($requestSet);
if (
($numTransactionErrors[$id] ?? 0) >= self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET
|| ($retry >= self::MAX_RETRIES_PER_BATCH && isset($hasTransactionErrorInLatestPass[$id]))
) {
$numDroppedRequestSets++;
} else {
$requestSetsToRetry[] = $requestSet;
}
}

if (!empty($requestSetsToRetry) && $this->handler->hasTransactionErrors()) {
// give the conflicting transaction time to finish
usleep(100000 * min($retry, self::MAX_TRANSACTION_ERRORS_PER_REQUEST_SET));
}

$failedRequestSets = $this->processRequestSets($tracker, $requestSetsToRetry);
}

if ($numDroppedRequestSets > 0) {
StaticContainer::get(LoggerInterface::class)->warning(sprintf(
'Dropping %d request sets from queue %s after repeated transaction errors.',
$numDroppedRequestSets,
$queue->getId()
));
}

$queue->markRequestSetsAsProcessed();
Expand Down
58 changes: 52 additions & 6 deletions Queue/Processor/Handler.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\Exception\UnexpectedWebsiteFoundException;
use Piwik\Plugins\QueuedTracking\Configuration;
use Piwik\Tracker;
Expand All @@ -24,6 +23,7 @@ class Handler
protected $transactionId;

private $hasError = false;
private $requestSetsWithTransactionErrors = [];
private $requestSetsToRetry = array();
private $count = 0;
private $numTrackedRequestsBeginning = 0;
Expand All @@ -32,16 +32,17 @@ public function init(Tracker $tracker)
{
$this->requestSetsToRetry = array();
$this->hasError = false;
$this->requestSetsWithTransactionErrors = [];
$this->numTrackedRequestsBeginning = $tracker->getCountOfLoggedRequests();
$this->transactionId = $this->getDb()->beginTransaction();
}

public function process(Tracker $tracker, RequestSet $requestSet)
{
$requestSet->restoreEnvironment();

$this->count = 0;

$requestSet->restoreEnvironment();

foreach ($requestSet->getRequests() as $request) {
try {
$startMs = round(microtime(true) * 1000);
Expand Down Expand Up @@ -83,10 +84,11 @@ public function onException(RequestSet $requestSet, Exception $e)

Common::printDebug('Got exception: ' . $e->getMessage());

// retry if a deadlock or a lock wait timeout happened
if (Db::get()->isErrNo($e, 1213) || Db::get()->isErrNo($e, 1205)) {
if ($this->isRetryableTransactionError($e)) {
$this->requestSetsWithTransactionErrors[] = $requestSet;
$this->requestSetsToRetry[] = $requestSet;
Common::printDebug('Added deadlocked requestSet to requestSetsToRetry');
Common::printDebug('Added requestSet that hit a transaction error to requestSetsToRetry');
$this->restartTransaction();
return;
}

Expand All @@ -103,6 +105,23 @@ public function hasErrors()
return $this->hasError;
}

/**
* Whether a deadlock, lock wait timeout or snapshot isolation conflict happened since init().
*/
public function hasTransactionErrors(): bool
{
return !empty($this->requestSetsWithTransactionErrors);
}

/**
* @return RequestSet[] the request sets that hit a deadlock, lock wait timeout or snapshot isolation conflict
* since init()
*/
public function getRequestSetsWithTransactionErrors(): array
{
return $this->requestSetsWithTransactionErrors;
}

public function rollBack(Tracker $tracker)
{
$tracker->setCountOfLoggedRequests($this->numTrackedRequestsBeginning);
Expand All @@ -123,6 +142,33 @@ public function commit()
$this->requestSetsToRetry = array();
}

/**
* Deadlock (1213), lock wait timeout (1205) and MariaDB's snapshot isolation conflict (1020).
*/
private function isRetryableTransactionError(Exception $e): bool
{
$db = $this->getDb();

return $db->isErrNo($e, 1213) || $db->isErrNo($e, 1205) || $db->isErrNo($e, 1020);
}

/**
* These errors can end the transaction on the server, after which PDO runs every later query in autocommit
* mode, so the remaining request sets would be stored even though the batch is rolled back and retried.
* Every request set processed so far is already queued for retry, so discard their work and start over.
*/
private function restartTransaction(): void
{
try {
$this->getDb()->rollBack($this->transactionId);
} catch (Exception $e) {
// PDO throws "There is no active transaction" once the server has rolled back.
Common::printDebug('Rollback after a transaction error failed: ' . $e->getMessage());
}

$this->transactionId = $this->getDb()->beginTransaction();
}

protected function getDb()
{
return Tracker::getDatabase();
Expand Down
2 changes: 1 addition & 1 deletion plugin.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "QueuedTracking",
"version": "5.2.0",
"version": "5.2.1",
"description": "Scale your large traffic Matomo service by queuing tracking requests in Redis or MySQL for better performance and reliability when experiencing peaks.",
"theme": false,
"keywords": ["tracker", "tracking", "queue", "redis"],
Expand Down
34 changes: 33 additions & 1 deletion tests/Framework/Mock/Tracker.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,45 @@

class Tracker extends \Piwik\Tests\Framework\Mock\Tracker
{
public $numDeadlocks = 0;

/**
* Counted by TestProcessor over the tracker's lifetime, so in a test that processes a single batch a pass's index is
* one less: 0 for its first processing, n for its nth retry.
*/
public $numPasses = 0;

/**
* Keyed by the request's params, as a request is restored from the queue as a new object. Requests with identical
* params therefore share a count, so give each one a param of its own when a test needs them counted apart.
* @var int[]
*/
private $numDeadlocksPerRequest = [];

public function trackRequest(Request $request)
{
$allParams = $request->getRawParams();
if (!empty($allParams['forceThrow'])) {
$key = json_encode($allParams);
$numOwnDeadlocks = $this->numDeadlocksPerRequest[$key] ?? 0;

if (!empty($allParams['forceThrow']) || (!empty($allParams['forceThrowAfterDeadlock']) && $numOwnDeadlocks)) {
throw new ForcedException("forced exception");
}

if (
!empty($allParams['forceDeadlock'])
|| (!empty($allParams['forceDeadlockOnce']) && !$numOwnDeadlocks)
|| (!empty($allParams['forceDeadlockUntil']) && $numOwnDeadlocks < $allParams['forceDeadlockUntil'])
|| (!empty($allParams['forceDeadlockOnceAfterDeadlocks']) && !$numOwnDeadlocks
&& $this->numDeadlocks >= $allParams['forceDeadlockOnceAfterDeadlocks'])
|| (isset($allParams['forceDeadlockInPasses'])
&& in_array((string) ($this->numPasses - 1), explode(',', $allParams['forceDeadlockInPasses']), true))
) {
$this->numDeadlocks++;
$this->numDeadlocksPerRequest[$key] = $numOwnDeadlocks + 1;
throw new ForcedException('SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction');
}

return parent::trackRequest($request);
}
}
73 changes: 73 additions & 0 deletions tests/Integration/Queue/Processor/HandlerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@

namespace Piwik\Plugins\QueuedTracking\tests\Integration\Queue\Processor;

use Piwik\Common;
use Piwik\Db;
use Piwik\Plugins\QueuedTracking\Queue\Processor\Handler;
use Piwik\Plugins\QueuedTracking\tests\Framework\Mock\ForcedException;
use Piwik\Plugins\QueuedTracking\tests\Framework\Mock\Tracker;
Expand Down Expand Up @@ -100,10 +102,14 @@ public function test_onException_shouldMarkTheHandlerAsFaulty()
public function test_init_shouldResetAFaultyHandler()
{
$this->handler->onException(new RequestSet(), new \Exception());
$this->handler->onException(new RequestSet(), new \Exception('SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction'));
$this->assertTrue($this->handler->hasErrors());
$this->assertTrue($this->handler->hasTransactionErrors());

$this->handler->rollBack($this->tracker);
$this->handler->init($this->tracker);
$this->assertFalse($this->handler->hasErrors());
$this->assertFalse($this->handler->hasTransactionErrors());
}

public function test_process_ShouldForwardTheRequestToTheTracker()
Expand Down Expand Up @@ -162,6 +168,29 @@ public function test_onException_shouldRemoveAllInvalidRequestsFromValidRequests
$this->assertEquals(array('idsite' => '1', 'index' => '3'), $requests[3]->getParams());
}

public function testOnExceptionShouldNotKeepRequestsOfASetWhoseEnvironmentCouldNotBeRestored()
{
$requestSet = $this->buildRequestSet(5);
$this->handler->process($this->tracker, $requestSet);

$failingRequestSet = new class extends RequestSet {
public function restoreEnvironment()
{
throw new ForcedException('forced exception');
}
};
$failingRequestSet->setRequests($this->buildRequestSet(3)->getRequests());

try {
$this->handler->process($this->tracker, $failingRequestSet);
$this->fail('An expected exception was not triggered');
} catch (ForcedException $e) {
$this->handler->onException($failingRequestSet, $e);
}

$this->assertEquals([$requestSet], $this->handler->getRequestSetsToRetry());
}

public function test_rollBack_shouldAddTheSuccessfullyProcessedRequestSetToAllValidRequestSets()
{
$requestSet1 = $this->buildRequestSet(5);
Expand Down Expand Up @@ -285,6 +314,50 @@ public function test_process_ShouldRestoreTheEnvironmentOfARequest()
$_COOKIE = $cookieBackup;
}

/**
* @dataProvider getRetryableTransactionErrors
*/
public function testOnExceptionShouldRetryTheRequestSetAndDiscardWorkDoneInTheTransaction(string $message, bool $serverEndsTransaction)
{
$this->insertOption('QueuedTrackingTest_beforeError');

if ($serverEndsTransaction) {
Tracker::getDatabase()->query('ROLLBACK');
}

$requestSet = $this->buildRequestSet(3);
$this->handler->onException($requestSet, new \Exception($message));

// without a new transaction PDO would store this in autocommit mode and the rollBack below would throw
$this->insertOption('QueuedTrackingTest_afterError');

$this->handler->rollBack($this->tracker);

$this->assertEquals([$requestSet], $this->handler->getRequestSetsToRetry());
$this->assertSame(3, $requestSet->getNumberOfRequests());
$this->assertSame(0, (int) Db::fetchOne(
'SELECT COUNT(*) FROM `' . Common::prefixTable('option') . '` WHERE option_name LIKE ?',
['QueuedTrackingTest_%']
));
}

public function getRetryableTransactionErrors(): array
{
return [
'deadlock' => ['SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction', true],
'snapshot isolation' => ["SQLSTATE[HY000]: General error: 1020 Record has changed since last read in table 'log_visit'; try restarting transaction", true],
'lock wait timeout' => ['SQLSTATE[HY000]: General error: 1205 Lock wait timeout exceeded; try restarting transaction', false],
];
}

private function insertOption(string $name): void
{
Tracker::getDatabase()->query(
'INSERT INTO `' . Common::prefixTable('option') . '` (option_name, option_value, autoload) VALUES (?, ?, 0)',
[$name, '1']
);
}

private function createHandler()
{
return new TestHandler();
Expand Down
Loading
Loading