diff --git a/CHANGELOG.md b/CHANGELOG.md index cf9a57d..ae5f326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ ## Changelog +6.0.5 +- Added the `redisPersistentConnection` setting to reuse one Redis connection per PHP process instead of connecting on every tracking request + 6.0.4 - 2026-11-09 - Fixed a queue worker crashing and keeping its queue locked when a database deadlock was caught while tracking a request; the request set is now retried diff --git a/Commands/Test.php b/Commands/Test.php index e67e06f..eb6a601 100644 --- a/Commands/Test.php +++ b/Commands/Test.php @@ -76,6 +76,7 @@ protected function doExecute(): int $output->writeln('Timeout: ' . $settings->redisTimeout->getValue()); $output->writeln('Password: ' . $settings->redisPassword->getValue()); $output->writeln('Database: ' . $settings->redisDatabase->getValue()); + $output->writeln('PersistentConnection (standalone Redis only): ' . (int) $settings->redisPersistentConnection->getValue()); $output->writeln('RedisBackendType: ' . $settings->getRedisType()); $output->writeln('SentinelMasterName: ' . $settings->sentinelMasterName->getValue()); diff --git a/Queue/Backend/Redis.php b/Queue/Backend/Redis.php index 26b943f..0d303b6 100644 --- a/Queue/Backend/Redis.php +++ b/Queue/Backend/Redis.php @@ -29,6 +29,11 @@ class Redis implements Backend */ protected $database; + /** + * @var bool + */ + protected $persistent = false; + public function testConnection() { try { @@ -260,7 +265,14 @@ private function connectIfNeeded() protected function connect() { $this->redis = new \Redis(); - $success = $this->redis->connect($this->host, $this->port, $this->timeout, null, 100); + + if ($this->persistent) { + // One persistent connection per database: a phpredis client sharing it could select() another database. + $persistentId = 'queuedtracking-' . (int) $this->database; + $success = $this->redis->pconnect($this->host, $this->port, $this->timeout, $persistentId, 100); + } else { + $success = $this->redis->connect($this->host, $this->port, $this->timeout, null, 100); + } if ($success && !empty($this->password)) { $success = $this->redis->auth($this->password); @@ -309,4 +321,9 @@ public function setDatabase($database) { $this->database = $database; } + + public function setUsePersistentConnection($persistent) + { + $this->persistent = (bool) $persistent; + } } diff --git a/Queue/Factory.php b/Queue/Factory.php index e2d4c28..32e29ac 100644 --- a/Queue/Factory.php +++ b/Queue/Factory.php @@ -81,6 +81,7 @@ public static function makeBackendFromSettings(SystemSettings $settings) } else { $redis = new Queue\Backend\Redis(); $redis->setDatabase($database); + $redis->setUsePersistentConnection($settings->redisPersistentConnection->getValue()); } $redis->setConfig($host, $port, $timeout, $password); diff --git a/SystemSettings.php b/SystemSettings.php index 05ca214..68dfc67 100644 --- a/SystemSettings.php +++ b/SystemSettings.php @@ -39,6 +39,9 @@ class SystemSettings extends \Piwik\Settings\Plugin\SystemSettings /** @var Setting */ public $redisDatabase; + /** @var Setting */ + public $redisPersistentConnection; + /** @var Setting */ public $redisPassword; @@ -91,6 +94,7 @@ protected function init() $this->redisPort = $this->createRedisPortSetting(); $this->redisTimeout = $this->createRedisTimeoutSetting(); $this->redisDatabase = $this->createRedisDatabaseSetting(); + $this->redisPersistentConnection = $this->createRedisPersistentConnectionSetting(); $this->redisPassword = $this->createRedisPasswordSetting(); $this->queueEnabled = $this->createQueueEnabledSetting(); $this->numQueueWorkers = $this->createNumberOfQueueWorkerSetting(); @@ -247,6 +251,21 @@ private function createRedisPasswordSetting() }); } + private function createRedisPersistentConnectionSetting() + { + $setting = $this->makeSetting('redisPersistentConnection', $default = false, FieldConfig::TYPE_BOOL, function (FieldConfig $field) { + $field->title = Piwik::translate('QueuedTracking_RedisPersistentConnectionFieldTitle'); + $field->condition = 'backend=="redis"'; + $field->uiControl = FieldConfig::UI_CONTROL_CHECKBOX; + $field->inlineHelp = Piwik::translate('QueuedTracking_RedisPersistentConnectionFieldHelp') . '
'; + }); + + // like redisTimeout, only configurable in config.ini.php for now + $setting->setIsWritableByCurrentUser(false); + + return $setting; + } + private function createRedisDatabaseSetting() { return $this->makeSetting('redisDatabase', $default = 0, FieldConfig::TYPE_INT, function (FieldConfig $field) { diff --git a/docs/faq.md b/docs/faq.md index cae107e..46f79d4 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -170,6 +170,20 @@ notify_queue_threshold_emails[] = example@example.org notify_queue_threshold_single_queue = 250000 ``` +__Can the tracker reuse its Redis connection instead of opening one per request?__ + +Yes, for a standalone Redis (not Sentinel or Cluster) using the `phpredis` extension. By default, every tracking request +opens a new connection to Redis, which on a busy server means thousands of new connections per minute, each with a +TLS handshake when Redis is reached over TLS. To keep one connection open per PHP process instead, set the following in +your `config/config.ini.php`: + +``` +[QueuedTracking] +redisPersistentConnection = 1 +``` + +Each PHP-FPM worker then keeps its own connection, so Redis sees up to `pm.max_children` connections per server. + __Are there any known issues?__ * In case you are using bulk tracking the bulk tracking response varies compared to the regular one. We will always return diff --git a/lang/en.json b/lang/en.json index 1b2f7df..9a451d0 100644 --- a/lang/en.json +++ b/lang/en.json @@ -20,6 +20,8 @@ "RedisPasswordFieldHelp": "Password set on the Redis server, if any. Redis can be instructed to require a password before allowing clients to execute commands.", "RedisDatabaseFieldTitle": "Redis database", "RedisDatabaseFieldHelp": "In case you are using Redis for caching make sure to use a different database.", + "RedisPersistentConnectionFieldTitle": "Use a persistent Redis connection", + "RedisPersistentConnectionFieldHelp": "Reuse one Redis connection per PHP process instead of opening a new one for every tracking request. Saves a connection (and a TLS handshake when connecting over TLS) per request. Standalone Redis only, requires the phpredis extension.", "QueueEnabledFieldTitle": "Queue enabled", "QueueEnabledFieldHelp": "If enabled, all tracking requests will be written into a queue instead of the directly into the database. Requires a Redis server and phpredis PHP extension if using Redis as a backend.", "NumRequestsToProcessFieldTitle": "Number of requests that are processed in one batch", diff --git a/plugin.json b/plugin.json index a795768..8e9c644 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "QueuedTracking", - "version": "6.0.4", + "version": "6.0.5", "description": "Scale your large traffic Matomo service by queuing tracking requests in Redis or MySQL for better performance and reliability when experiencing peaks.", "theme": false, "keywords": ["tracker", "tracking", "queue", "redis"], diff --git a/tests/Integration/Queue/Backend/RedisTest.php b/tests/Integration/Queue/Backend/RedisTest.php index 85cc2ca..510cab1 100644 --- a/tests/Integration/Queue/Backend/RedisTest.php +++ b/tests/Integration/Queue/Backend/RedisTest.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\QueuedTracking\tests\Integration\Queue\Backend; use Piwik\Plugins\QueuedTracking\Queue\Backend\Redis; +use Piwik\Plugins\QueuedTracking\Queue\Factory; use Piwik\Plugins\QueuedTracking\tests\Framework\TestCase\IntegrationTestCase; /** @@ -330,4 +331,48 @@ public function test_getKeysMatchingPattern_shouldReturnAnEmptyArrayIfNothingMat $keys = $backend->getKeysMatchingPattern('*fere*'); $this->assertEquals(array(), $keys); } + + public function test_persistentConnection_shouldKeepEachDatabaseSeparate() + { + $settings = Factory::getSettings(); + $database = (int) $settings->redisDatabase->getValue(); + $otherDatabase = $database === 14 ? 13 : 14; + + // Without pooling, phpredis clients with the same persistent id share one socket. + $pooling = ini_set('redis.pconnect.pooling_enabled', '0'); + + try { + $backend = $this->createPersistentRedisBackend($database); + $other = $this->createPersistentRedisBackend($otherDatabase); + + $other->delete($this->key); + $backend->setIfNotExists($this->key, 'inThisDatabase', 100); + $other->setIfNotExists($this->key, 'inTheOtherDatabase', 100); + + $this->assertSame('inThisDatabase', $backend->get($this->key)); + $this->assertSame('inTheOtherDatabase', $other->get($this->key)); + $this->assertSame('queuedtracking-' . $database, $backend->getConnection()->getPersistentID()); + $this->assertSame('queuedtracking-' . $otherDatabase, $other->getConnection()->getPersistentID()); + + $other->delete($this->key); + } finally { + ini_set('redis.pconnect.pooling_enabled', $pooling); + } + } + + private function createPersistentRedisBackend($database) + { + $settings = Factory::getSettings(); + $backend = new Redis(); + $backend->setConfig( + $settings->redisHost->getValue(), + $settings->redisPort->getValue(), + $settings->redisTimeout->getValue(), + $settings->redisPassword->getValue() + ); + $backend->setDatabase($database); + $backend->setUsePersistentConnection(true); + + return $backend; + } } diff --git a/tests/Integration/Queue/Backend/SentinelTest.php b/tests/Integration/Queue/Backend/SentinelTest.php index a7f3ca1..43e64a1 100644 --- a/tests/Integration/Queue/Backend/SentinelTest.php +++ b/tests/Integration/Queue/Backend/SentinelTest.php @@ -44,6 +44,11 @@ protected function createRedisBackend() return $sentinel; } + public function test_persistentConnection_shouldKeepEachDatabaseSeparate() + { + $this->markTestSkipped('Persistent connections are only supported by the standalone Redis backend'); + } + public function test_canCreateInstanceWithMultipleSentinelAndFallback() { $settings = Factory::getSettings(); diff --git a/tests/Integration/Queue/FactoryTest.php b/tests/Integration/Queue/FactoryTest.php index 020c4d8..edaf27d 100644 --- a/tests/Integration/Queue/FactoryTest.php +++ b/tests/Integration/Queue/FactoryTest.php @@ -81,6 +81,26 @@ public function test_makeBackend_shouldConfigureRedis() $this->assertTrue($success); } + public function test_makeBackend_shouldNotConnectWithAPersistentConnection_ByDefault() + { + $backend = Factory::makeBackend(); + + $this->assertTrue($backend->testConnection()); + $this->assertNull($backend->getConnection()->getPersistentID()); + } + + public function test_makeBackend_shouldConnectWithAPersistentConnection_IfConfigured() + { + Config::getInstance()->QueuedTracking = array('redisPersistentConnection' => '1'); + $backend = Factory::makeBackend(); + Config::getInstance()->QueuedTracking = array(); + + $database = (int) Factory::getSettings()->redisDatabase->getValue(); + + $this->assertTrue($backend->testConnection()); + $this->assertSame('queuedtracking-' . $database, $backend->getConnection()->getPersistentID()); + } + public function test_getSettings_shouldReturnARedisInstance() { $settings = Factory::getSettings(); diff --git a/tests/Integration/SettingsTest.php b/tests/Integration/SettingsTest.php index 09b5d38..8aef775 100644 --- a/tests/Integration/SettingsTest.php +++ b/tests/Integration/SettingsTest.php @@ -205,6 +205,23 @@ public function test_redisDatabase_ShouldBeEmptyByDefault() $this->assertEmpty($this->settings->redisDatabase->getValue()); } + public function test_redisPersistentConnection_ShouldBeDisabledByDefault() + { + $this->assertFalse($this->settings->redisPersistentConnection->getValue()); + } + + public function test_redisPersistentConnection_ShouldNotBeWritableByDefault() + { + $this->assertFalse($this->settings->redisPersistentConnection->isWritableByCurrentUser()); + } + + public function test_redisPersistentConnection_ShouldConvertAValueToBool() + { + $this->settings->redisPersistentConnection->setIsWritableByCurrentUser(true); + $this->settings->redisPersistentConnection->setValue('1'); + $this->assertTrue($this->settings->redisPersistentConnection->getValue()); + } + public function test_redisTimeout_ShouldBeNotUnlimitedByDefault() { $this->assertSame(0.0, $this->settings->redisTimeout->getValue());