[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * External storage in SQL database.
   4   *
   5   * This program is free software; you can redistribute it and/or modify
   6   * it under the terms of the GNU General Public License as published by
   7   * the Free Software Foundation; either version 2 of the License, or
   8   * (at your option) any later version.
   9   *
  10   * This program is distributed in the hope that it will be useful,
  11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13   * GNU General Public License for more details.
  14   *
  15   * You should have received a copy of the GNU General Public License along
  16   * with this program; if not, write to the Free Software Foundation, Inc.,
  17   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18   * http://www.gnu.org/copyleft/gpl.html
  19   *
  20   * @file
  21   */
  22  
  23  /**
  24   * DB accessable external objects.
  25   *
  26   * In this system, each store "location" maps to a database "cluster".
  27   * The clusters must be defined in the normal LBFactory configuration.
  28   *
  29   * @ingroup ExternalStorage
  30   */
  31  class ExternalStoreDB extends ExternalStoreMedium {
  32      /**
  33       * The provided URL is in the form of DB://cluster/id
  34       * or DB://cluster/id/itemid for concatened storage.
  35       *
  36       * @see ExternalStoreMedium::fetchFromURL()
  37       */
  38  	public function fetchFromURL( $url ) {
  39          list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  40          $ret = $this->fetchBlob( $cluster, $id, $itemID );
  41  
  42          if ( $itemID !== false && $ret !== false ) {
  43              return $ret->getItem( $itemID );
  44          }
  45  
  46          return $ret;
  47      }
  48  
  49      /**
  50       * Fetch data from given external store URLs.
  51       * The provided URLs are in the form of DB://cluster/id
  52       * or DB://cluster/id/itemid for concatened storage.
  53       *
  54       * @param array $urls An array of external store URLs
  55       * @return array A map from url to stored content. Failed results
  56       *     are not represented.
  57       */
  58  	public function batchFetchFromURLs( array $urls ) {
  59          $batched = $inverseUrlMap = array();
  60          foreach ( $urls as $url ) {
  61              list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  62              $batched[$cluster][$id][] = $itemID;
  63              // false $itemID gets cast to int, but should be ok
  64              // since we do === from the $itemID in $batched
  65              $inverseUrlMap[$cluster][$id][$itemID] = $url;
  66          }
  67          $ret = array();
  68          foreach ( $batched as $cluster => $batchByCluster ) {
  69              $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
  70              /** @var HistoryBlob $blob */
  71              foreach ( $res as $id => $blob ) {
  72                  foreach ( $batchByCluster[$id] as $itemID ) {
  73                      $url = $inverseUrlMap[$cluster][$id][$itemID];
  74                      if ( $itemID === false ) {
  75                          $ret[$url] = $blob;
  76                      } else {
  77                          $ret[$url] = $blob->getItem( $itemID );
  78                      }
  79                  }
  80              }
  81          }
  82  
  83          return $ret;
  84      }
  85  
  86      /**
  87       * @see ExternalStoreMedium::store()
  88       */
  89  	public function store( $cluster, $data ) {
  90          $dbw = $this->getMaster( $cluster );
  91          $id = $dbw->nextSequenceValue( 'blob_blob_id_seq' );
  92          $dbw->insert( $this->getTable( $dbw ),
  93              array( 'blob_id' => $id, 'blob_text' => $data ),
  94              __METHOD__ );
  95          $id = $dbw->insertId();
  96          if ( !$id ) {
  97              throw new MWException( __METHOD__ . ': no insert ID' );
  98          }
  99  
 100          return "DB://$cluster/$id";
 101      }
 102  
 103      /**
 104       * Get a LoadBalancer for the specified cluster
 105       *
 106       * @param string $cluster Cluster name
 107       * @return LoadBalancer
 108       */
 109  	function getLoadBalancer( $cluster ) {
 110          $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
 111  
 112          return wfGetLBFactory()->getExternalLB( $cluster, $wiki );
 113      }
 114  
 115      /**
 116       * Get a slave database connection for the specified cluster
 117       *
 118       * @param string $cluster Cluster name
 119       * @return DatabaseBase
 120       */
 121  	function getSlave( $cluster ) {
 122          global $wgDefaultExternalStore;
 123  
 124          $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
 125          $lb = $this->getLoadBalancer( $cluster );
 126  
 127          if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) {
 128              wfDebug( "read only external store\n" );
 129              $lb->allowLagged( true );
 130          } else {
 131              wfDebug( "writable external store\n" );
 132          }
 133  
 134          $db = $lb->getConnection( DB_SLAVE, array(), $wiki );
 135          $db->clearFlag( DBO_TRX ); // sanity
 136  
 137          return $db;
 138      }
 139  
 140      /**
 141       * Get a master database connection for the specified cluster
 142       *
 143       * @param string $cluster Cluster name
 144       * @return DatabaseBase
 145       */
 146  	function getMaster( $cluster ) {
 147          $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
 148          $lb = $this->getLoadBalancer( $cluster );
 149  
 150          $db = $lb->getConnection( DB_MASTER, array(), $wiki );
 151          $db->clearFlag( DBO_TRX ); // sanity
 152  
 153          return $db;
 154      }
 155  
 156      /**
 157       * Get the 'blobs' table name for this database
 158       *
 159       * @param DatabaseBase $db
 160       * @return string Table name ('blobs' by default)
 161       */
 162  	function getTable( $db ) {
 163          $table = $db->getLBInfo( 'blobs table' );
 164          if ( is_null( $table ) ) {
 165              $table = 'blobs';
 166          }
 167  
 168          return $table;
 169      }
 170  
 171      /**
 172       * Fetch a blob item out of the database; a cache of the last-loaded
 173       * blob will be kept so that multiple loads out of a multi-item blob
 174       * can avoid redundant database access and decompression.
 175       * @param string $cluster
 176       * @param string $id
 177       * @param string $itemID
 178       * @return mixed
 179       * @private
 180       */
 181  	function fetchBlob( $cluster, $id, $itemID ) {
 182          /**
 183           * One-step cache variable to hold base blobs; operations that
 184           * pull multiple revisions may often pull multiple times from
 185           * the same blob. By keeping the last-used one open, we avoid
 186           * redundant unserialization and decompression overhead.
 187           */
 188          static $externalBlobCache = array();
 189  
 190          $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
 191          if ( isset( $externalBlobCache[$cacheID] ) ) {
 192              wfDebugLog( 'ExternalStoreDB-cache',
 193                  "ExternalStoreDB::fetchBlob cache hit on $cacheID" );
 194  
 195              return $externalBlobCache[$cacheID];
 196          }
 197  
 198          wfDebugLog( 'ExternalStoreDB-cache',
 199              "ExternalStoreDB::fetchBlob cache miss on $cacheID" );
 200  
 201          $dbr = $this->getSlave( $cluster );
 202          $ret = $dbr->selectField( $this->getTable( $dbr ),
 203              'blob_text', array( 'blob_id' => $id ), __METHOD__ );
 204          if ( $ret === false ) {
 205              wfDebugLog( 'ExternalStoreDB',
 206                  "ExternalStoreDB::fetchBlob master fallback on $cacheID" );
 207              // Try the master
 208              $dbw = $this->getMaster( $cluster );
 209              $ret = $dbw->selectField( $this->getTable( $dbw ),
 210                  'blob_text', array( 'blob_id' => $id ), __METHOD__ );
 211              if ( $ret === false ) {
 212                  wfDebugLog( 'ExternalStoreDB',
 213                      "ExternalStoreDB::fetchBlob master failed to find $cacheID" );
 214              }
 215          }
 216          if ( $itemID !== false && $ret !== false ) {
 217              // Unserialise object; caller extracts item
 218              $ret = unserialize( $ret );
 219          }
 220  
 221          $externalBlobCache = array( $cacheID => $ret );
 222  
 223          return $ret;
 224      }
 225  
 226      /**
 227       * Fetch multiple blob items out of the database
 228       *
 229       * @param string $cluster A cluster name valid for use with LBFactory
 230       * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs
 231       * @return array A map from the blob_id's requested to their content.
 232       *   Unlocated ids are not represented
 233       */
 234  	function batchFetchBlobs( $cluster, array $ids ) {
 235          $dbr = $this->getSlave( $cluster );
 236          $res = $dbr->select( $this->getTable( $dbr ),
 237              array( 'blob_id', 'blob_text' ), array( 'blob_id' => array_keys( $ids ) ), __METHOD__ );
 238          $ret = array();
 239          if ( $res !== false ) {
 240              $this->mergeBatchResult( $ret, $ids, $res );
 241          }
 242          if ( $ids ) {
 243              wfDebugLog( __CLASS__, __METHOD__ .
 244                  " master fallback on '$cluster' for: " .
 245                  implode( ',', array_keys( $ids ) ) );
 246              // Try the master
 247              $dbw = $this->getMaster( $cluster );
 248              $res = $dbw->select( $this->getTable( $dbr ),
 249                  array( 'blob_id', 'blob_text' ),
 250                  array( 'blob_id' => array_keys( $ids ) ),
 251                  __METHOD__ );
 252              if ( $res === false ) {
 253                  wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'" );
 254              } else {
 255                  $this->mergeBatchResult( $ret, $ids, $res );
 256              }
 257          }
 258          if ( $ids ) {
 259              wfDebugLog( __CLASS__, __METHOD__ .
 260                  " master on '$cluster' failed locating items: " .
 261                  implode( ',', array_keys( $ids ) ) );
 262          }
 263  
 264          return $ret;
 265      }
 266  
 267      /**
 268       * Helper function for self::batchFetchBlobs for merging master/slave results
 269       * @param array &$ret Current self::batchFetchBlobs return value
 270       * @param array &$ids Map from blob_id to requested itemIDs
 271       * @param mixed $res DB result from DatabaseBase::select
 272       */
 273  	private function mergeBatchResult( array &$ret, array &$ids, $res ) {
 274          foreach ( $res as $row ) {
 275              $id = $row->blob_id;
 276              $itemIDs = $ids[$id];
 277              unset( $ids[$id] ); // to track if everything is found
 278              if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
 279                  // single result stored per blob
 280                  $ret[$id] = $row->blob_text;
 281              } else {
 282                  // multi result stored per blob
 283                  $ret[$id] = unserialize( $row->blob_text );
 284              }
 285          }
 286      }
 287  
 288      /**
 289       * @param string $url
 290       * @return array
 291       */
 292  	protected function parseURL( $url ) {
 293          $path = explode( '/', $url );
 294  
 295          return array(
 296              $path[2], // cluster
 297              $path[3], // id
 298              isset( $path[4] ) ? $path[4] : false // itemID
 299          );
 300      }
 301  }