[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * Redis client connection pooling manager.
   4   *
   5   * This program is free software; you can redistribute it and/or modify
   6   * it under the terms of the GNU General Public License as published by
   7   * the Free Software Foundation; either version 2 of the License, or
   8   * (at your option) any later version.
   9   *
  10   * This program is distributed in the hope that it will be useful,
  11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13   * GNU General Public License for more details.
  14   *
  15   * You should have received a copy of the GNU General Public License along
  16   * with this program; if not, write to the Free Software Foundation, Inc.,
  17   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18   * http://www.gnu.org/copyleft/gpl.html
  19   *
  20   * @file
  21   * @defgroup Redis Redis
  22   * @author Aaron Schulz
  23   */
  24  
  25  /**
  26   * Helper class to manage Redis connections.
  27   *
  28   * This can be used to get handle wrappers that free the handle when the wrapper
  29   * leaves scope. The maximum number of free handles (connections) is configurable.
  30   * This provides an easy way to cache connection handles that may also have state,
  31   * such as a handle does between multi() and exec(), and without hoarding connections.
  32   * The wrappers use PHP magic methods so that calling functions on them calls the
  33   * function of the actual Redis object handle.
  34   *
  35   * @ingroup Redis
  36   * @since 1.21
  37   */
  38  class RedisConnectionPool {
  39      /**
  40       * @name Pool settings.
  41       * Settings there are shared for any connection made in this pool.
  42       * See the singleton() method documentation for more details.
  43       * @{
  44       */
  45      /** @var string Connection timeout in seconds */
  46      protected $connectTimeout;
  47      /** @var string Read timeout in seconds */
  48      protected $readTimeout;
  49      /** @var string Plaintext auth password */
  50      protected $password;
  51      /** @var bool Whether connections persist */
  52      protected $persistent;
  53      /** @var int Serializer to use (Redis::SERIALIZER_*) */
  54      protected $serializer;
  55      /** @} */
  56  
  57      /** @var int Current idle pool size */
  58      protected $idlePoolSize = 0;
  59  
  60      /** @var array (server name => ((connection info array),...) */
  61      protected $connections = array();
  62      /** @var array (server name => UNIX timestamp) */
  63      protected $downServers = array();
  64  
  65      /** @var array (pool ID => RedisConnectionPool) */
  66      protected static $instances = array();
  67  
  68      /** integer; seconds to cache servers as "down". */
  69      const SERVER_DOWN_TTL = 30;
  70  
  71      /**
  72       * @param array $options
  73       * @throws MWException
  74       */
  75  	protected function __construct( array $options ) {
  76          if ( !class_exists( 'Redis' ) ) {
  77              throw new MWException( __CLASS__ . ' requires a Redis client library. ' .
  78                  'See https://www.mediawiki.org/wiki/Redis#Setup' );
  79          }
  80          $this->connectTimeout = $options['connectTimeout'];
  81          $this->readTimeout = $options['readTimeout'];
  82          $this->persistent = $options['persistent'];
  83          $this->password = $options['password'];
  84          if ( !isset( $options['serializer'] ) || $options['serializer'] === 'php' ) {
  85              $this->serializer = Redis::SERIALIZER_PHP;
  86          } elseif ( $options['serializer'] === 'igbinary' ) {
  87              $this->serializer = Redis::SERIALIZER_IGBINARY;
  88          } elseif ( $options['serializer'] === 'none' ) {
  89              $this->serializer = Redis::SERIALIZER_NONE;
  90          } else {
  91              throw new MWException( "Invalid serializer specified." );
  92          }
  93      }
  94  
  95      /**
  96       * @param array $options
  97       * @return array
  98       */
  99  	protected static function applyDefaultConfig( array $options ) {
 100          if ( !isset( $options['connectTimeout'] ) ) {
 101              $options['connectTimeout'] = 1;
 102          }
 103          if ( !isset( $options['readTimeout'] ) ) {
 104              $options['readTimeout'] = 1;
 105          }
 106          if ( !isset( $options['persistent'] ) ) {
 107              $options['persistent'] = false;
 108          }
 109          if ( !isset( $options['password'] ) ) {
 110              $options['password'] = null;
 111          }
 112  
 113          return $options;
 114      }
 115  
 116      /**
 117       * @param array $options
 118       * $options include:
 119       *   - connectTimeout : The timeout for new connections, in seconds.
 120       *                      Optional, default is 1 second.
 121       *   - readTimeout    : The timeout for operation reads, in seconds.
 122       *                      Commands like BLPOP can fail if told to wait longer than this.
 123       *                      Optional, default is 1 second.
 124       *   - persistent     : Set this to true to allow connections to persist across
 125       *                      multiple web requests. False by default.
 126       *   - password       : The authentication password, will be sent to Redis in clear text.
 127       *                      Optional, if it is unspecified, no AUTH command will be sent.
 128       *   - serializer     : Set to "php", "igbinary", or "none". Default is "php".
 129       * @return RedisConnectionPool
 130       */
 131  	public static function singleton( array $options ) {
 132          $options = self::applyDefaultConfig( $options );
 133          // Map the options to a unique hash...
 134          ksort( $options ); // normalize to avoid pool fragmentation
 135          $id = sha1( serialize( $options ) );
 136          // Initialize the object at the hash as needed...
 137          if ( !isset( self::$instances[$id] ) ) {
 138              self::$instances[$id] = new self( $options );
 139              wfDebug( "Creating a new " . __CLASS__ . " instance with id $id.\n" );
 140          }
 141  
 142          return self::$instances[$id];
 143      }
 144  
 145      /**
 146       * Get a connection to a redis server. Based on code in RedisBagOStuff.php.
 147       *
 148       * @param string $server A hostname/port combination or the absolute path of a UNIX socket.
 149       *                       If a hostname is specified but no port, port 6379 will be used.
 150       * @return RedisConnRef|bool Returns false on failure
 151       * @throws MWException
 152       */
 153  	public function getConnection( $server ) {
 154          // Check the listing "dead" servers which have had a connection errors.
 155          // Servers are marked dead for a limited period of time, to
 156          // avoid excessive overhead from repeated connection timeouts.
 157          if ( isset( $this->downServers[$server] ) ) {
 158              $now = time();
 159              if ( $now > $this->downServers[$server] ) {
 160                  // Dead time expired
 161                  unset( $this->downServers[$server] );
 162              } else {
 163                  // Server is dead
 164                  wfDebug( "server $server is marked down for another " .
 165                      ( $this->downServers[$server] - $now ) . " seconds, can't get connection\n" );
 166  
 167                  return false;
 168              }
 169          }
 170  
 171          // Check if a connection is already free for use
 172          if ( isset( $this->connections[$server] ) ) {
 173              foreach ( $this->connections[$server] as &$connection ) {
 174                  if ( $connection['free'] ) {
 175                      $connection['free'] = false;
 176                      --$this->idlePoolSize;
 177  
 178                      return new RedisConnRef( $this, $server, $connection['conn'] );
 179                  }
 180              }
 181          }
 182  
 183          if ( substr( $server, 0, 1 ) === '/' ) {
 184              // UNIX domain socket
 185              // These are required by the redis extension to start with a slash, but
 186              // we still need to set the port to a special value to make it work.
 187              $host = $server;
 188              $port = 0;
 189          } else {
 190              // TCP connection
 191              $hostPort = IP::splitHostAndPort( $server );
 192              if ( !$hostPort ) {
 193                  throw new MWException( __CLASS__ . ": invalid configured server \"$server\"" );
 194              }
 195              list( $host, $port ) = $hostPort;
 196              if ( $port === false ) {
 197                  $port = 6379;
 198              }
 199          }
 200  
 201          $conn = new Redis();
 202          try {
 203              if ( $this->persistent ) {
 204                  $result = $conn->pconnect( $host, $port, $this->connectTimeout );
 205              } else {
 206                  $result = $conn->connect( $host, $port, $this->connectTimeout );
 207              }
 208              if ( !$result ) {
 209                  wfDebugLog( 'redis', "Could not connect to server $server" );
 210                  // Mark server down for some time to avoid further timeouts
 211                  $this->downServers[$server] = time() + self::SERVER_DOWN_TTL;
 212  
 213                  return false;
 214              }
 215              if ( $this->password !== null ) {
 216                  if ( !$conn->auth( $this->password ) ) {
 217                      wfDebugLog( 'redis', "Authentication error connecting to $server" );
 218                  }
 219              }
 220          } catch ( RedisException $e ) {
 221              $this->downServers[$server] = time() + self::SERVER_DOWN_TTL;
 222              wfDebugLog( 'redis', "Redis exception connecting to $server: " . $e->getMessage() );
 223  
 224              return false;
 225          }
 226  
 227          if ( $conn ) {
 228              $conn->setOption( Redis::OPT_READ_TIMEOUT, $this->readTimeout );
 229              $conn->setOption( Redis::OPT_SERIALIZER, $this->serializer );
 230              $this->connections[$server][] = array( 'conn' => $conn, 'free' => false );
 231  
 232              return new RedisConnRef( $this, $server, $conn );
 233          } else {
 234              return false;
 235          }
 236      }
 237  
 238      /**
 239       * Mark a connection to a server as free to return to the pool
 240       *
 241       * @param string $server
 242       * @param Redis $conn
 243       * @return bool
 244       */
 245  	public function freeConnection( $server, Redis $conn ) {
 246          $found = false;
 247  
 248          foreach ( $this->connections[$server] as &$connection ) {
 249              if ( $connection['conn'] === $conn && !$connection['free'] ) {
 250                  $connection['free'] = true;
 251                  ++$this->idlePoolSize;
 252                  break;
 253              }
 254          }
 255  
 256          $this->closeExcessIdleConections();
 257  
 258          return $found;
 259      }
 260  
 261      /**
 262       * Close any extra idle connections if there are more than the limit
 263       */
 264  	protected function closeExcessIdleConections() {
 265          if ( $this->idlePoolSize <= count( $this->connections ) ) {
 266              return; // nothing to do (no more connections than servers)
 267          }
 268  
 269          foreach ( $this->connections as &$serverConnections ) {
 270              foreach ( $serverConnections as $key => &$connection ) {
 271                  if ( $connection['free'] ) {
 272                      unset( $serverConnections[$key] );
 273                      if ( --$this->idlePoolSize <= count( $this->connections ) ) {
 274                          return; // done (no more connections than servers)
 275                      }
 276                  }
 277              }
 278          }
 279      }
 280  
 281      /**
 282       * The redis extension throws an exception in response to various read, write
 283       * and protocol errors. Sometimes it also closes the connection, sometimes
 284       * not. The safest response for us is to explicitly destroy the connection
 285       * object and let it be reopened during the next request.
 286       *
 287       * @param string $server
 288       * @param RedisConnRef $cref
 289       * @param RedisException $e
 290       * @deprecated since 1.23
 291       */
 292  	public function handleException( $server, RedisConnRef $cref, RedisException $e ) {
 293          return $this->handleError( $cref, $e );
 294      }
 295  
 296      /**
 297       * The redis extension throws an exception in response to various read, write
 298       * and protocol errors. Sometimes it also closes the connection, sometimes
 299       * not. The safest response for us is to explicitly destroy the connection
 300       * object and let it be reopened during the next request.
 301       *
 302       * @param RedisConnRef $cref
 303       * @param RedisException $e
 304       */
 305  	public function handleError( RedisConnRef $cref, RedisException $e ) {
 306          $server = $cref->getServer();
 307          wfDebugLog( 'redis', "Redis exception on server $server: " . $e->getMessage() . "\n" );
 308          foreach ( $this->connections[$server] as $key => $connection ) {
 309              if ( $cref->isConnIdentical( $connection['conn'] ) ) {
 310                  $this->idlePoolSize -= $connection['free'] ? 1 : 0;
 311                  unset( $this->connections[$server][$key] );
 312                  break;
 313              }
 314          }
 315      }
 316  
 317      /**
 318       * Re-send an AUTH request to the redis server (useful after disconnects).
 319       *
 320       * This works around an upstream bug in phpredis. phpredis hides disconnects by transparently
 321       * reconnecting, but it neglects to re-authenticate the new connection. To the user of the
 322       * phpredis client API this manifests as a seemingly random tendency of connections to lose
 323       * their authentication status.
 324       *
 325       * This method is for internal use only.
 326       *
 327       * @see https://github.com/nicolasff/phpredis/issues/403
 328       *
 329       * @param string $server
 330       * @param Redis $conn
 331       * @return bool Success
 332       */
 333  	public function reauthenticateConnection( $server, Redis $conn ) {
 334          if ( $this->password !== null ) {
 335              if ( !$conn->auth( $this->password ) ) {
 336                  wfDebugLog( 'redis', "Authentication error connecting to $server" );
 337  
 338                  return false;
 339              }
 340          }
 341  
 342          return true;
 343      }
 344  
 345      /**
 346       * Adjust or reset the connection handle read timeout value
 347       *
 348       * @param Redis $conn
 349       * @param int $timeout Optional
 350       */
 351  	public function resetTimeout( Redis $conn, $timeout = null ) {
 352          $conn->setOption( Redis::OPT_READ_TIMEOUT, $timeout ?: $this->readTimeout );
 353      }
 354  
 355      /**
 356       * Make sure connections are closed for sanity
 357       */
 358  	function __destruct() {
 359          foreach ( $this->connections as $server => &$serverConnections ) {
 360              foreach ( $serverConnections as $key => &$connection ) {
 361                  $connection['conn']->close();
 362              }
 363          }
 364      }
 365  }
 366  
 367  /**
 368   * Helper class to handle automatically marking connectons as reusable (via RAII pattern)
 369   *
 370   * This class simply wraps the Redis class and can be used the same way
 371   *
 372   * @ingroup Redis
 373   * @since 1.21
 374   */
 375  class RedisConnRef {
 376      /** @var RedisConnectionPool */
 377      protected $pool;
 378      /** @var Redis */
 379      protected $conn;
 380  
 381      protected $server; // string
 382      protected $lastError; // string
 383  
 384      /**
 385       * @param RedisConnectionPool $pool
 386       * @param string $server
 387       * @param Redis $conn
 388       */
 389  	public function __construct( RedisConnectionPool $pool, $server, Redis $conn ) {
 390          $this->pool = $pool;
 391          $this->server = $server;
 392          $this->conn = $conn;
 393      }
 394  
 395      /**
 396       * @return string
 397       * @since 1.23
 398       */
 399  	public function getServer() {
 400          return $this->server;
 401      }
 402  
 403  	public function getLastError() {
 404          return $this->lastError;
 405      }
 406  
 407  	public function clearLastError() {
 408          $this->lastError = null;
 409      }
 410  
 411  	public function __call( $name, $arguments ) {
 412          $conn = $this->conn; // convenience
 413  
 414          // Work around https://github.com/nicolasff/phpredis/issues/70
 415          $lname = strtolower( $name );
 416          if ( ( $lname === 'blpop' || $lname == 'brpop' )
 417              && is_array( $arguments[0] ) && isset( $arguments[1] )
 418          ) {
 419              $this->pool->resetTimeout( $conn, $arguments[1] + 1 );
 420          } elseif ( $lname === 'brpoplpush' && isset( $arguments[2] ) ) {
 421              $this->pool->resetTimeout( $conn, $arguments[2] + 1 );
 422          }
 423  
 424          $conn->clearLastError();
 425          try {
 426              $res = call_user_func_array( array( $conn, $name ), $arguments );
 427              if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) {
 428                  $this->pool->reauthenticateConnection( $this->server, $conn );
 429                  $conn->clearLastError();
 430                  $res = call_user_func_array( array( $conn, $name ), $arguments );
 431                  wfDebugLog( 'redis', "Used automatic re-authentication for method '$name'." );
 432              }
 433          } catch ( RedisException $e ) {
 434              $this->pool->resetTimeout( $conn ); // restore
 435              throw $e;
 436          }
 437  
 438          $this->lastError = $conn->getLastError() ?: $this->lastError;
 439  
 440          $this->pool->resetTimeout( $conn ); // restore
 441  
 442          return $res;
 443      }
 444  
 445      /**
 446       * @param string $script
 447       * @param array $params
 448       * @param int $numKeys
 449       * @return mixed
 450       * @throws RedisException
 451       */
 452  	public function luaEval( $script, array $params, $numKeys ) {
 453          $sha1 = sha1( $script ); // 40 char hex
 454          $conn = $this->conn; // convenience
 455          $server = $this->server; // convenience
 456  
 457          // Try to run the server-side cached copy of the script
 458          $conn->clearLastError();
 459          $res = $conn->evalSha( $sha1, $params, $numKeys );
 460          // If we got a permission error reply that means that (a) we are not in
 461          // multi()/pipeline() and (b) some connection problem likely occurred. If
 462          // the password the client gave was just wrong, an exception should have
 463          // been thrown back in getConnection() previously.
 464          if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) {
 465              $this->pool->reauthenticateConnection( $server, $conn );
 466              $conn->clearLastError();
 467              $res = $conn->eval( $script, $params, $numKeys );
 468              wfDebugLog( 'redis', "Used automatic re-authentication for Lua script $sha1." );
 469          }
 470          // If the script is not in cache, use eval() to retry and cache it
 471          if ( preg_match( '/^NOSCRIPT/', $conn->getLastError() ) ) {
 472              $conn->clearLastError();
 473              $res = $conn->eval( $script, $params, $numKeys );
 474              wfDebugLog( 'redis', "Used eval() for Lua script $sha1." );
 475          }
 476  
 477          if ( $conn->getLastError() ) { // script bug?
 478              wfDebugLog( 'redis', "Lua script error on server $server: " . $conn->getLastError() );
 479          }
 480  
 481          $this->lastError = $conn->getLastError() ?: $this->lastError;
 482  
 483          return $res;
 484      }
 485  
 486      /**
 487       * @param Redis $conn
 488       * @return bool
 489       */
 490  	public function isConnIdentical( Redis $conn ) {
 491          return $this->conn === $conn;
 492      }
 493  
 494  	function __destruct() {
 495          $this->pool->freeConnection( $this->server, $this->conn );
 496      }
 497  }