From 21bf253ac9daa1b5e3c28459b3523a4fb3acf316 Mon Sep 17 00:00:00 2001 From: Eric Cobos Date: Mon, 16 Nov 2026 21:54:19 +0100 Subject: [PATCH] Add redisPersistentConnection setting to reuse the Redis connection per PHP process (#369) * Add redisPersistentConnection setting to reuse the Redis connection per PHP process Every tracking request opened a new connection to Redis (Queue\Backend\Redis::connect). With the setting on, the standalone Redis backend uses pconnect, with one persistent id per database so a client sharing the connection cannot select() another database. Off by default; configurable in config.ini.php only, like redisTimeout. Closes #55 Co-Authored-By: Claude Opus 5.5 * Bump to 6.0.5 and assert the persistent connection id in tests The Factory and database-separation tests passed without the feature. They now check getPersistentID(), so they fail if the setting is not wired or the database is missing from the id. Co-Authored-By: Claude Fable 5.2 * Address review: drop the restart caveat, test database separation without pooling phpredis reconnects a dropped persistent connection by itself, so the FAQ caveat about a failed first request after a Redis restart did not hold. With connection pooling on, two clients never share a socket, so the separation test passed even with a constant persistent id; it now runs with pooling off. queuedtracking:test marks the setting as standalone Redis only. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: Eric Cobos Co-authored-by: Claude Opus 5.5 (cherry picked from commit 8e5dc00b4bbe23ffebfe949cd87c4f5c53726ad6) --- CHANGELOG.md | 3 ++ Commands/Test.php | 1 + Queue/Backend/Redis.php | 19 +++++++- Queue/Factory.php | 1 + SystemSettings.php | 19 ++++++++ docs/faq.md | 14 ++++++ lang/en.json | 2 + plugin.json | 2 +- tests/Integration/Queue/Backend/RedisTest.php | 45 +++++++++++++++++++ .../Queue/Backend/SentinelTest.php | 5 +++ tests/Integration/Queue/FactoryTest.php | 20 +++++++++ tests/Integration/SettingsTest.php | 17 +++++++ 12 files changed, 146 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 545f2f0..f937d2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## Changelog +5.2.3 +- Added the `redisPersistentConnection` setting to reuse one Redis connection per PHP process instead of connecting on every tracking request + 5.2.2 - 2026-10-26 - Fixed the Redis Cluster backend failing with a syntax error on PHP 7.2 and 7.3 - Fixed the Redis Cluster backend failing with a fatal error when the numbers of configured hosts and ports do not match diff --git a/Commands/Test.php b/Commands/Test.php index c9861f4..eb79d58 100644 --- a/Commands/Test.php +++ b/Commands/Test.php @@ -75,6 +75,7 @@ protected function doExecute(): int $output->writeln('Timeout: ' . $settings->redisTimeout->getValue()); $output->writeln('Password: ' . $settings->redisPassword->getValue()); $output->writeln('Database: ' . $settings->redisDatabase->getValue()); + $output->writeln('PersistentConnection (standalone Redis only): ' . (int) $settings->redisPersistentConnection->getValue()); $output->writeln('RedisBackendType: ' . $settings->getRedisType()); $output->writeln('SentinelMasterName: ' . $settings->sentinelMasterName->getValue()); diff --git a/Queue/Backend/Redis.php b/Queue/Backend/Redis.php index 06f829b..f7757d4 100644 --- a/Queue/Backend/Redis.php +++ b/Queue/Backend/Redis.php @@ -29,6 +29,11 @@ class Redis implements Backend */ protected $database; + /** + * @var bool + */ + protected $persistent = false; + public function testConnection() { try { @@ -258,7 +263,14 @@ private function connectIfNeeded() protected function connect() { $this->redis = new \Redis(); - $success = $this->redis->connect($this->host, $this->port, $this->timeout, null, 100); + + if ($this->persistent) { + // One persistent connection per database: a phpredis client sharing it could select() another database. + $persistentId = 'queuedtracking-' . (int) $this->database; + $success = $this->redis->pconnect($this->host, $this->port, $this->timeout, $persistentId, 100); + } else { + $success = $this->redis->connect($this->host, $this->port, $this->timeout, null, 100); + } if ($success && !empty($this->password)) { $success = $this->redis->auth($this->password); @@ -307,4 +319,9 @@ public function setDatabase($database) { $this->database = $database; } + + public function setUsePersistentConnection($persistent) + { + $this->persistent = (bool) $persistent; + } } diff --git a/Queue/Factory.php b/Queue/Factory.php index e2d4c28..32e29ac 100644 --- a/Queue/Factory.php +++ b/Queue/Factory.php @@ -81,6 +81,7 @@ public static function makeBackendFromSettings(SystemSettings $settings) } else { $redis = new Queue\Backend\Redis(); $redis->setDatabase($database); + $redis->setUsePersistentConnection($settings->redisPersistentConnection->getValue()); } $redis->setConfig($host, $port, $timeout, $password); diff --git a/SystemSettings.php b/SystemSettings.php index 80f43bd..aec827c 100644 --- a/SystemSettings.php +++ b/SystemSettings.php @@ -39,6 +39,9 @@ class SystemSettings extends \Piwik\Settings\Plugin\SystemSettings /** @var Setting */ public $redisDatabase; + /** @var Setting */ + public $redisPersistentConnection; + /** @var Setting */ public $redisPassword; @@ -91,6 +94,7 @@ protected function init() $this->redisPort = $this->createRedisPortSetting(); $this->redisTimeout = $this->createRedisTimeoutSetting(); $this->redisDatabase = $this->createRedisDatabaseSetting(); + $this->redisPersistentConnection = $this->createRedisPersistentConnectionSetting(); $this->redisPassword = $this->createRedisPasswordSetting(); $this->queueEnabled = $this->createQueueEnabledSetting(); $this->numQueueWorkers = $this->createNumberOfQueueWorkerSetting(); @@ -247,6 +251,21 @@ private function createRedisPasswordSetting() }); } + private function createRedisPersistentConnectionSetting() + { + $setting = $this->makeSetting('redisPersistentConnection', $default = false, FieldConfig::TYPE_BOOL, function (FieldConfig $field) { + $field->title = Piwik::translate('QueuedTracking_RedisPersistentConnectionFieldTitle'); + $field->condition = 'backend=="redis"'; + $field->uiControl = FieldConfig::UI_CONTROL_CHECKBOX; + $field->inlineHelp = Piwik::translate('QueuedTracking_RedisPersistentConnectionFieldHelp') . '
'; + }); + + // like redisTimeout, only configurable in config.ini.php for now + $setting->setIsWritableByCurrentUser(false); + + return $setting; + } + private function createRedisDatabaseSetting() { return $this->makeSetting('redisDatabase', $default = 0, FieldConfig::TYPE_INT, function (FieldConfig $field) { diff --git a/docs/faq.md b/docs/faq.md index cae107e..46f79d4 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -170,6 +170,20 @@ notify_queue_threshold_emails[] = example@example.org notify_queue_threshold_single_queue = 250000 ``` +__Can the tracker reuse its Redis connection instead of opening one per request?__ + +Yes, for a standalone Redis (not Sentinel or Cluster) using the `phpredis` extension. By default, every tracking request +opens a new connection to Redis, which on a busy server means thousands of new connections per minute, each with a +TLS handshake when Redis is reached over TLS. To keep one connection open per PHP process instead, set the following in +your `config/config.ini.php`: + +``` +[QueuedTracking] +redisPersistentConnection = 1 +``` + +Each PHP-FPM worker then keeps its own connection, so Redis sees up to `pm.max_children` connections per server. + __Are there any known issues?__ * In case you are using bulk tracking the bulk tracking response varies compared to the regular one. We will always return diff --git a/lang/en.json b/lang/en.json index 1b2f7df..9a451d0 100644 --- a/lang/en.json +++ b/lang/en.json @@ -20,6 +20,8 @@ "RedisPasswordFieldHelp": "Password set on the Redis server, if any. Redis can be instructed to require a password before allowing clients to execute commands.", "RedisDatabaseFieldTitle": "Redis database", "RedisDatabaseFieldHelp": "In case you are using Redis for caching make sure to use a different database.", + "RedisPersistentConnectionFieldTitle": "Use a persistent Redis connection", + "RedisPersistentConnectionFieldHelp": "Reuse one Redis connection per PHP process instead of opening a new one for every tracking request. Saves a connection (and a TLS handshake when connecting over TLS) per request. Standalone Redis only, requires the phpredis extension.", "QueueEnabledFieldTitle": "Queue enabled", "QueueEnabledFieldHelp": "If enabled, all tracking requests will be written into a queue instead of the directly into the database. Requires a Redis server and phpredis PHP extension if using Redis as a backend.", "NumRequestsToProcessFieldTitle": "Number of requests that are processed in one batch", diff --git a/plugin.json b/plugin.json index 8467f90..f5ad3d7 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "QueuedTracking", - "version": "5.2.2", + "version": "5.2.3", "description": "Scale your large traffic Matomo service by queuing tracking requests in Redis or MySQL for better performance and reliability when experiencing peaks.", "theme": false, "keywords": ["tracker", "tracking", "queue", "redis"], diff --git a/tests/Integration/Queue/Backend/RedisTest.php b/tests/Integration/Queue/Backend/RedisTest.php index 85cc2ca..510cab1 100644 --- a/tests/Integration/Queue/Backend/RedisTest.php +++ b/tests/Integration/Queue/Backend/RedisTest.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\QueuedTracking\tests\Integration\Queue\Backend; use Piwik\Plugins\QueuedTracking\Queue\Backend\Redis; +use Piwik\Plugins\QueuedTracking\Queue\Factory; use Piwik\Plugins\QueuedTracking\tests\Framework\TestCase\IntegrationTestCase; /** @@ -330,4 +331,48 @@ public function test_getKeysMatchingPattern_shouldReturnAnEmptyArrayIfNothingMat $keys = $backend->getKeysMatchingPattern('*fere*'); $this->assertEquals(array(), $keys); } + + public function test_persistentConnection_shouldKeepEachDatabaseSeparate() + { + $settings = Factory::getSettings(); + $database = (int) $settings->redisDatabase->getValue(); + $otherDatabase = $database === 14 ? 13 : 14; + + // Without pooling, phpredis clients with the same persistent id share one socket. + $pooling = ini_set('redis.pconnect.pooling_enabled', '0'); + + try { + $backend = $this->createPersistentRedisBackend($database); + $other = $this->createPersistentRedisBackend($otherDatabase); + + $other->delete($this->key); + $backend->setIfNotExists($this->key, 'inThisDatabase', 100); + $other->setIfNotExists($this->key, 'inTheOtherDatabase', 100); + + $this->assertSame('inThisDatabase', $backend->get($this->key)); + $this->assertSame('inTheOtherDatabase', $other->get($this->key)); + $this->assertSame('queuedtracking-' . $database, $backend->getConnection()->getPersistentID()); + $this->assertSame('queuedtracking-' . $otherDatabase, $other->getConnection()->getPersistentID()); + + $other->delete($this->key); + } finally { + ini_set('redis.pconnect.pooling_enabled', $pooling); + } + } + + private function createPersistentRedisBackend($database) + { + $settings = Factory::getSettings(); + $backend = new Redis(); + $backend->setConfig( + $settings->redisHost->getValue(), + $settings->redisPort->getValue(), + $settings->redisTimeout->getValue(), + $settings->redisPassword->getValue() + ); + $backend->setDatabase($database); + $backend->setUsePersistentConnection(true); + + return $backend; + } } diff --git a/tests/Integration/Queue/Backend/SentinelTest.php b/tests/Integration/Queue/Backend/SentinelTest.php index a7f3ca1..43e64a1 100644 --- a/tests/Integration/Queue/Backend/SentinelTest.php +++ b/tests/Integration/Queue/Backend/SentinelTest.php @@ -44,6 +44,11 @@ protected function createRedisBackend() return $sentinel; } + public function test_persistentConnection_shouldKeepEachDatabaseSeparate() + { + $this->markTestSkipped('Persistent connections are only supported by the standalone Redis backend'); + } + public function test_canCreateInstanceWithMultipleSentinelAndFallback() { $settings = Factory::getSettings(); diff --git a/tests/Integration/Queue/FactoryTest.php b/tests/Integration/Queue/FactoryTest.php index f7267c1..07519ef 100644 --- a/tests/Integration/Queue/FactoryTest.php +++ b/tests/Integration/Queue/FactoryTest.php @@ -100,6 +100,26 @@ public function test_makeBackend_shouldConfigureRedis() $this->assertTrue($success); } + public function test_makeBackend_shouldNotConnectWithAPersistentConnection_ByDefault() + { + $backend = Factory::makeBackend(); + + $this->assertTrue($backend->testConnection()); + $this->assertNull($backend->getConnection()->getPersistentID()); + } + + public function test_makeBackend_shouldConnectWithAPersistentConnection_IfConfigured() + { + Config::getInstance()->QueuedTracking = array('redisPersistentConnection' => '1'); + $backend = Factory::makeBackend(); + Config::getInstance()->QueuedTracking = array(); + + $database = (int) Factory::getSettings()->redisDatabase->getValue(); + + $this->assertTrue($backend->testConnection()); + $this->assertSame('queuedtracking-' . $database, $backend->getConnection()->getPersistentID()); + } + public function test_getSettings_shouldReturnARedisInstance() { $settings = Factory::getSettings(); diff --git a/tests/Integration/SettingsTest.php b/tests/Integration/SettingsTest.php index 09b5d38..8aef775 100644 --- a/tests/Integration/SettingsTest.php +++ b/tests/Integration/SettingsTest.php @@ -205,6 +205,23 @@ public function test_redisDatabase_ShouldBeEmptyByDefault() $this->assertEmpty($this->settings->redisDatabase->getValue()); } + public function test_redisPersistentConnection_ShouldBeDisabledByDefault() + { + $this->assertFalse($this->settings->redisPersistentConnection->getValue()); + } + + public function test_redisPersistentConnection_ShouldNotBeWritableByDefault() + { + $this->assertFalse($this->settings->redisPersistentConnection->isWritableByCurrentUser()); + } + + public function test_redisPersistentConnection_ShouldConvertAValueToBool() + { + $this->settings->redisPersistentConnection->setIsWritableByCurrentUser(true); + $this->settings->redisPersistentConnection->setValue('1'); + $this->assertTrue($this->settings->redisPersistentConnection->getValue()); + } + public function test_redisTimeout_ShouldBeNotUnlimitedByDefault() { $this->assertSame(0.0, $this->settings->redisTimeout->getValue());