diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2b7d4600..948cd7c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,10 +30,16 @@ jobs: - 5432:5432 env: - # The hostname used to communicate with the Redis service container + # The hostname used to communicate with the Redis/Sentinel service containers REDIS_HOST: redis + # The hostname for redis sentinel + REDIS_SENTINEL_HOST: redis-sentinel # The default Redis port REDIS_PORT: 6379 + # The default Redis Sentinel port + REDIS_SENTINEL_PORT: 26379 + # The default Redis Sentinel primary + REDIS_SENTINEL_SERVICE: myprimary # MySQL DB_DATABASE: test DB_USER: root @@ -70,6 +76,20 @@ jobs: with: redis-version: ${{ matrix.redis-version }} + - name: Generate Redis Sentinel conf compatible with redis 5 assuming 127.0.0.1 (no resolve hostname) + run: | + REDIS_SENTINEL_IP=127.0.0.1 + cat < sentinel5.conf + port ${{ env.REDIS_SENTINEL_PORT }} + sentinel monitor ${{ env.REDIS_SENTINEL_SERVICE }} $REDIS_SENTINEL_IP ${{ env.REDIS_PORT }} 2 + sentinel down-after-milliseconds ${{ env.REDIS_SENTINEL_SERVICE }} 10000 + sentinel failover-timeout ${{ env.REDIS_SENTINEL_SERVICE }} 180000 + sentinel parallel-syncs ${{ env.REDIS_SENTINEL_SERVICE }} 2 + EOF + + - name: Start Redis Sentinel + run: docker run -d --name ${{env.REDIS_SENTINEL_HOST}} -p 26379:26379 --link redis:redis -v $PWD/sentinel5.conf:/data/sentinel.conf redis:${{ matrix.redis-version }} redis-server sentinel.conf --sentinel + - name: Execute tests (PDO with Sqlite) run: vendor/bin/phpunit diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..a04f8bd1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "github-actions.workflows.pinned.workflows": [], + "githubPullRequests.ignoredPullRequestBranches": [ + "main" + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 925a2843..3ee05b8c 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,20 @@ Change the Redis options (the example shows the defaults): 'timeout' => 0.1, // in seconds 'read_timeout' => '10', // in seconds 'persistent_connections' => false + 'sentinel' => [ // sentinel options + 'enable' => false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis, default is the same as redis host if empty + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ] ] ); ``` diff --git a/docker-compose.yml b/docker-compose.yml index c6a794b0..b448f69d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,12 +14,21 @@ services: - redis environment: - REDIS_HOST=redis + - REDIS_SENTINEL_HOST=redis-sentinel redis: image: redis ports: - 6379:6379 + redis-sentinel: + image: redis + volumes: + - ./sentinel.conf://etc/sentinel.conf + ports: + - 26379:26379 + command: redis-sentinel /etc/sentinel.conf + phpunit: build: php-fpm/ volumes: @@ -29,3 +38,4 @@ services: - nginx environment: - REDIS_HOST=redis + - REDIS_SENTINEL_HOST=redis-sentinel diff --git a/examples/flush_adapter.php b/examples/flush_adapter.php index 712e2bec..56640d43 100644 --- a/examples/flush_adapter.php +++ b/examples/flush_adapter.php @@ -8,8 +8,9 @@ if ($adapterName === 'redis') { define('REDIS_HOST', $_SERVER['REDIS_HOST'] ?? '127.0.0.1'); + define('REDIS_SENTINEL_HOST', $_SERVER['REDIS_SENTINEL_HOST'] ?? '127.0.0.1'); - $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST]); + $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST, 'sentinel' => ['host' => REDIS_SENTINEL_HOST]]); } elseif ($adapterName === 'predis') { $adapter = new Prometheus\Storage\Predis(['host' => $_SERVER['REDIS_HOST'] ?? '127.0.0.1']); } elseif ($adapterName === 'apc') { diff --git a/sentinel.conf b/sentinel.conf new file mode 100644 index 00000000..0197f209 --- /dev/null +++ b/sentinel.conf @@ -0,0 +1,7 @@ +bind 0.0.0.0 +port 26379 +sentinel monitor myprimary redis 6379 2 +sentinel resolve-hostnames yes +sentinel down-after-milliseconds myprimary 10000 +sentinel failover-timeout myprimary 10000 +sentinel parallel-syncs myprimary 1 \ No newline at end of file diff --git a/src/Prometheus/Storage/Redis.php b/src/Prometheus/Storage/Redis.php index 78b0e375..d7517ef1 100644 --- a/src/Prometheus/Storage/Redis.php +++ b/src/Prometheus/Storage/Redis.php @@ -20,6 +20,20 @@ class Redis extends AbstractRedis 'persistent_connections' => false, 'password' => null, 'user' => null, + 'sentinel' => [ // sentinel options + 'enable' => false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis, default is the same as redis host if empty + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ], ]; /** @@ -27,6 +41,11 @@ class Redis extends AbstractRedis */ private $options = []; + /** + * @var RedisSentinel|null + */ + private $sentinel = null; + /** * Redis constructor. * @@ -34,15 +53,48 @@ class Redis extends AbstractRedis */ public function __construct(array $options = []) { - $this->options = array_merge(self::$defaultOptions, $options); - $this->redis = PHPRedis::create($this->options); + $this->options = [...self::$defaultOptions, ...$options]; + $this->options['sentinel'] = [...self::$defaultOptions['sentinel'] ?? [], ...$options['sentinel'] ?? []]; + if (boolval($this->options['sentinel']['enable'])) { + $sentinelOptions = $options['sentinel'] ?? []; + $sentinelOptions['host'] = $sentinelOptions['host'] ?? $this->options['host']; + $this->sentinel = new RedisSentinel($sentinelOptions); + } + $this->redis = PHPRedis::create($this->options, $this->sentinel); + } + + /** + * Sentinels discoverMaster + * @return void + */ + public function updateSentinelPrimary(): void + { + if ($this->redis instanceof PHPRedis) { + $this->redis->updateSentinelPrimary(); + } + } + + /** + * @return \RedisSentinel + */ + public function getRedisSentinel(): \RedisSentinel + { + if ($this->sentinel === null) { + throw new StorageException('Redis Sentinel is not enabled'); + } + + return $this->sentinel->getSentinel(); } /** * @throws StorageException */ - public static function fromExistingConnection(\Redis $redis): self + public static function fromExistingConnection(\Redis $redis, ?\RedisSentinel $redisSentinel = null): self { + if (isset($redisSentinel)) { + RedisSentinel::fromExistingConnection($redisSentinel); + } + if ($redis->isConnected() === false) { throw new StorageException('Connection to Redis server not established'); } diff --git a/src/Prometheus/Storage/RedisClients/PHPRedis.php b/src/Prometheus/Storage/RedisClients/PHPRedis.php index 35c7b014..7a80e425 100644 --- a/src/Prometheus/Storage/RedisClients/PHPRedis.php +++ b/src/Prometheus/Storage/RedisClients/PHPRedis.php @@ -5,9 +5,27 @@ namespace Prometheus\Storage\RedisClients; use Prometheus\Exception\StorageException; +use Prometheus\Storage\RedisSentinel; class PHPRedis implements RedisClient { + // The following array contains all exception message parts which are interpreted as a connection loss or + // another unavailability of Redis. + private const ERROR_MESSAGES_INDICATING_UNAVAILABILITY = [ + 'connection closed', + 'connection refused', + 'connection lost', + 'failed while reconnecting', + 'is loading the dataset in memory', + 'php_network_getaddresses', + 'read error on connection', + 'socket', + 'went away', + 'loading', + 'readonly', + "can't write against a read only replica", + ]; + /** * @var \Redis */ @@ -18,6 +36,11 @@ class PHPRedis implements RedisClient */ private $options = []; + /** + * @var RedisSentinel|null + */ + private $sentinel = null; + /** * @var bool */ @@ -26,20 +49,39 @@ class PHPRedis implements RedisClient /** * @param mixed[] $options */ - public function __construct(\Redis $redis, array $options) + public function __construct(\Redis $redis, array $options, ?RedisSentinel $sentinel = null) { $this->redis = $redis; $this->options = $options; + $this->sentinel = $sentinel; } /** * @param mixed[] $options */ - public static function create(array $options): self + public static function create(array $options, ?RedisSentinel $sentinel = null): self { $redis = new \Redis(); - return new self($redis, $options); + return new self($redis, $options, $sentinel); + } + + /** + * Sentinels discoverMaster + * @return void + */ + public function updateSentinelPrimary(): void + { + if ($this->sentinel === null) { + return; + } + + $master = $this->sentinel->getMaster(); + + if (is_array($master)) { + $this->options['host'] = $master['ip']; + $this->options['port'] = $master['port']; + } } /** @@ -117,7 +159,12 @@ public function ensureOpenConnection(): void return; } - $this->connectToServer(); + if ($this->sentinel !== null) { + $this->connectThroughSentinel(); + } else { + $this->connectToServer(); + } + $authParams = []; if (isset($this->options['user']) && $this->options['user'] !== '') { @@ -141,6 +188,53 @@ public function ensureOpenConnection(): void $this->connectionInitialized = true; } + /** + * Asks Sentinel for the current primary and connects to it. Retries up to `sentinel.reconnect` times + * (indefinitely if -1) while the error indicates that Redis is unavailable or readonly, which may happen + * during a Sentinel failover; any other error, or running out of retries, is thrown. + * + * @throws StorageException + */ + private function connectThroughSentinel(): void + { + $reconnect = (int) ($this->options['sentinel']['reconnect'] ?? 0); + $retries = 0; + while (true) { + try { + $this->updateSentinelPrimary(); + $this->connectToServer(); + + return; + } catch (StorageException $e) { + $cause = $e->getPrevious(); + $retryable = $cause instanceof \RedisException && $this->isRedisUnavailableOrReadonly($cause); + if (! $retryable || ($reconnect >= 0 && $retries >= $reconnect)) { + throw $e; + } + } + $retries++; + } + } + + /** + * Inspects the given exception and tells whether the reported error indicates that the server + * went away or is in readonly mode, which may happen in case of a Redis Sentinel failover. + */ + private function isRedisUnavailableOrReadonly(\RedisException $exception): bool + { + // We convert the exception message to lower-case in order to perform case-insensitive comparison. + $exceptionMessage = strtolower($exception->getMessage()); + + // Because we also match only partial exception messages, we cannot use in_array() at this point. + foreach (self::ERROR_MESSAGES_INDICATING_UNAVAILABILITY as $errorMessage) { + if (str_contains($exceptionMessage, $errorMessage)) { + return true; + } + } + + return false; + } + /** * @throws StorageException */ diff --git a/src/Prometheus/Storage/RedisSentinel.php b/src/Prometheus/Storage/RedisSentinel.php new file mode 100644 index 00000000..b1a957b4 --- /dev/null +++ b/src/Prometheus/Storage/RedisSentinel.php @@ -0,0 +1,159 @@ + false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ]; + + /** + * @param \RedisSentinel $redisSentinel + * @return self + * @throws StorageException + */ + public static function fromExistingConnection(\RedisSentinel $redisSentinel): self + { + $sentinel = new self(); + $sentinel->sentinel = $redisSentinel; + $sentinel->getMaster(); + return $sentinel; + } + + /** + * Redis constructor. + * @param mixed[] $options + */ + public function __construct(array $options = []) + { + $this->options = [...self::$defaultOptions, ...$options]; + $this->sentinel = $this->initSentinel(); + } + + /** + * {@inheritdoc} + * @return mixed[]|bool + * @throws StorageException|\RedisException + */ + public function getMaster(): array|bool + { + $service = $this->options['service']; + + try { + $master = $this->sentinel->master($service); + } catch (\RedisException $e) { + throw new StorageException( + sprintf("Can't connect to RedisSentinel server. %s", $e->getMessage()), + $e->getCode(), + $e + ); + } + + if (! $this->isValidMaster($master)) { + throw new StorageException(sprintf("No master found for service '%s'.", $service)); + } + + return $master; + } + + /** + * Check whether master is valid or not. + * @param mixed[]|bool $master + * @return bool + */ + protected function isValidMaster(array|bool $master): bool + { + return is_array($master) && isset($master['ip']) && isset($master['port']); + } + + /** + * Connect to the configured Redis Sentinel instance. + * @return \RedisSentinel + * @throws StorageException + */ + private function initSentinel(): \RedisSentinel + { + $host = $this->options['host'] ?? ''; + $port = $this->options['port'] ?? 26379; + $timeout = $this->options['timeout'] ?? 0.2; + $persistent = $this->options['persistent'] ?? null; + $retryInterval = $this->options['retry_interval'] ?? 0; + $readTimeout = $this->options['read_timeout'] ?? 0; + $username = $this->options['username'] ?? ''; + $password = $this->options['password'] ?? ''; + $ssl = $this->options['ssl'] ?? null; + + if (strlen(trim($host)) === 0) { + throw new StorageException('No host has been specified for the Redis Sentinel connection.'); + } + + $auth = null; + if (strlen(trim($username)) !== 0 && strlen(trim($password)) !== 0) { + $auth = [$username, $password]; + } elseif (strlen(trim($password)) !== 0) { + $auth = $password; + } + + if (version_compare((string)phpversion('redis'), '6.0', '>=')) { + $options = [ + 'host' => $host, + 'port' => $port, + 'connectTimeout' => $timeout, + 'persistent' => $persistent, + 'retryInterval' => $retryInterval, + 'readTimeout' => $readTimeout, + ]; + + if ($auth !== null) { + $options['auth'] = $auth; + } + + if (version_compare((string)phpversion('redis'), '6.1', '>=') && $ssl !== null) { + $options['ssl'] = $ssl; + } + // @phpstan-ignore arguments.count, argument.type + return new \RedisSentinel($options); + } + + if ($auth !== null) { + /** + * @phpstan-ignore arguments.count + **/ + return new \RedisSentinel($host, $port, $timeout, $persistent, $retryInterval, $readTimeout, $auth); + } + return new \RedisSentinel($host, $port, $timeout, $persistent, $retryInterval, $readTimeout); + } + + public function getSentinel(): \RedisSentinel + { + return $this->sentinel; + } +} diff --git a/tests/Test/Prometheus/Storage/RedisSentinelTest.php b/tests/Test/Prometheus/Storage/RedisSentinelTest.php new file mode 100644 index 00000000..c5636900 --- /dev/null +++ b/tests/Test/Prometheus/Storage/RedisSentinelTest.php @@ -0,0 +1,129 @@ +redisConnection = new \Redis(); + $this->redisConnection->connect(REDIS_HOST); + $this->redisConnection->flushAll(); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnConnectionFailureWithRedisSentinelNotEnabled(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null']]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to Redis server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnConnectionFailure(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null', 'enable' => true]]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to RedisSentinel server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldThrowExceptionWhenInjectedRedisIsNotConnected(): void + { + $connection = new \Redis(); + + $sentinel = version_compare((string)phpversion('redis'), '6.0', '>=') ? + new \RedisSentinel(['host' => '/dev/null']) : // @phpstan-ignore arguments.count, argument.type + new \RedisSentinel('/dev/null'); // @phpstan-ignore arguments.count + + self::expectException(StorageException::class); + self::expectExceptionMessageMatches("/Can't connect to RedisSentinel server\\..*/"); + + Redis::fromExistingConnection($connection, $sentinel); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnPrimaryFailure(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null', 'enable' => true, 'service' => 'dummy']]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to RedisSentinel server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldGetMasterWhenRetryingIndefinitely(): void + { + $redis = new Redis(['host' => REDIS_HOST, + 'sentinel' => ['host' => REDIS_SENTINEL_HOST, 'enable' => true, 'service' => 'myprimary', 'reconnect' => -1] + ]); + + $redis->collect(); + $redis->wipeStorage(); + $this->expectNotToPerformAssertions(); + } + + /** + * @test + */ + public function itShouldThrowAfterExhaustingReconnects(): void + { + $redis = new Redis(['host' => REDIS_HOST, + 'sentinel' => ['host' => REDIS_HOST, 'port' => 1, 'enable' => true, 'service' => 'myprimary', 'reconnect' => 2] + ]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to RedisSentinel server"); + + $redis->collect(); + } + + /** + * @test + */ + public function itShouldGetMaster(): void + { + $redis = new Redis(['host' => REDIS_HOST, + 'sentinel' => ['host' => REDIS_SENTINEL_HOST, 'enable' => true, 'service' => 'myprimary'] + ]); + + $redis->collect(); + $redis->wipeStorage(); + $this->expectNotToPerformAssertions(); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 12456d29..3523cbdb 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -17,3 +17,4 @@ $loader->add('Test\\Performance', __DIR__); define('REDIS_HOST', isset($_ENV['REDIS_HOST']) ? $_ENV['REDIS_HOST'] : '127.0.0.1'); +define('REDIS_SENTINEL_HOST', isset($_ENV['REDIS_SENTINEL_HOST']) ? $_ENV['REDIS_SENTINEL_HOST'] : '127.0.0.1');