[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * @defgroup ExternalStorage ExternalStorage
   4   */
   5  
   6  /**
   7   * Interface for data storage in external repositories.
   8   *
   9   * This program is free software; you can redistribute it and/or modify
  10   * it under the terms of the GNU General Public License as published by
  11   * the Free Software Foundation; either version 2 of the License, or
  12   * (at your option) any later version.
  13   *
  14   * This program is distributed in the hope that it will be useful,
  15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17   * GNU General Public License for more details.
  18   *
  19   * You should have received a copy of the GNU General Public License along
  20   * with this program; if not, write to the Free Software Foundation, Inc.,
  21   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22   * http://www.gnu.org/copyleft/gpl.html
  23   *
  24   * @file
  25   */
  26  
  27  /**
  28   * Constructor class for key/value blob data kept in external repositories.
  29   *
  30   * Objects in external stores are defined by a special URL. The URL is of
  31   * the form "<store protocol>://<location>/<object name>". The protocol is used
  32   * to determine what ExternalStoreMedium class is used. The location identifies
  33   * particular storage instances or database clusters for store class to use.
  34   *
  35   * When an object is inserted into a store, the calling code uses a partial URL of
  36   * the form "<store protocol>://<location>" and receives the full object URL on success.
  37   * This is useful since object names can be sequential IDs, UUIDs, or hashes.
  38   * Callers are not responsible for unique name generation.
  39   *
  40   * External repositories might be populated by maintenance/async
  41   * scripts, thus partial moving of data may be possible, as well
  42   * as the possibility to have any storage format (i.e. for archives).
  43   *
  44   * @ingroup ExternalStorage
  45   */
  46  class ExternalStore {
  47      /**
  48       * Get an external store object of the given type, with the given parameters
  49       *
  50       * @param string $proto Type of external storage, should be a value in $wgExternalStores
  51       * @param array $params Associative array of ExternalStoreMedium parameters
  52       * @return ExternalStoreMedium|bool The store class or false on error
  53       */
  54  	public static function getStoreObject( $proto, array $params = array() ) {
  55          global $wgExternalStores;
  56  
  57          if ( !$wgExternalStores || !in_array( $proto, $wgExternalStores ) ) {
  58              return false; // protocol not enabled
  59          }
  60  
  61          $class = 'ExternalStore' . ucfirst( $proto );
  62  
  63          // Any custom modules should be added to $wgAutoLoadClasses for on-demand loading
  64          return class_exists( $class ) ? new $class( $params ) : false;
  65      }
  66  
  67      /**
  68       * Fetch data from given URL
  69       *
  70       * @param string $url The URL of the text to get
  71       * @param array $params Associative array of ExternalStoreMedium parameters
  72       * @return string|bool The text stored or false on error
  73       * @throws MWException
  74       */
  75  	public static function fetchFromURL( $url, array $params = array() ) {
  76          $parts = explode( '://', $url, 2 );
  77          if ( count( $parts ) != 2 ) {
  78              return false; // invalid URL
  79          }
  80  
  81          list( $proto, $path ) = $parts;
  82          if ( $path == '' ) { // bad URL
  83              return false;
  84          }
  85  
  86          $store = self::getStoreObject( $proto, $params );
  87          if ( $store === false ) {
  88              return false;
  89          }
  90  
  91          return $store->fetchFromURL( $url );
  92      }
  93  
  94      /**
  95       * Fetch data from multiple URLs with a minimum of round trips
  96       *
  97       * @param array $urls The URLs of the text to get
  98       * @return array Map from url to its data.  Data is either string when found
  99       *     or false on failure.
 100       */
 101  	public static function batchFetchFromURLs( array $urls ) {
 102          $batches = array();
 103          foreach ( $urls as $url ) {
 104              $scheme = parse_url( $url, PHP_URL_SCHEME );
 105              if ( $scheme ) {
 106                  $batches[$scheme][] = $url;
 107              }
 108          }
 109          $retval = array();
 110          foreach ( $batches as $proto => $batchedUrls ) {
 111              $store = self::getStoreObject( $proto );
 112              if ( $store === false ) {
 113                  continue;
 114              }
 115              $retval += $store->batchFetchFromURLs( $batchedUrls );
 116          }
 117          // invalid, not found, db dead, etc.
 118          $missing = array_diff( $urls, array_keys( $retval ) );
 119          if ( $missing ) {
 120              foreach ( $missing as $url ) {
 121                  $retval[$url] = false;
 122              }
 123          }
 124  
 125          return $retval;
 126      }
 127  
 128      /**
 129       * Store a data item to an external store, identified by a partial URL
 130       * The protocol part is used to identify the class, the rest is passed to the
 131       * class itself as a parameter.
 132       *
 133       * @param string $url A partial external store URL ("<store type>://<location>")
 134       * @param string $data
 135       * @param array $params Associative array of ExternalStoreMedium parameters
 136       * @return string|bool The URL of the stored data item, or false on error
 137       * @throws MWException
 138       */
 139  	public static function insert( $url, $data, array $params = array() ) {
 140          $parts = explode( '://', $url, 2 );
 141          if ( count( $parts ) != 2 ) {
 142              return false; // invalid URL
 143          }
 144  
 145          list( $proto, $path ) = $parts;
 146          if ( $path == '' ) { // bad URL
 147              return false;
 148          }
 149  
 150          $store = self::getStoreObject( $proto, $params );
 151          if ( $store === false ) {
 152              return false;
 153          } else {
 154              return $store->store( $path, $data );
 155          }
 156      }
 157  
 158      /**
 159       * Like insert() above, but does more of the work for us.
 160       * This function does not need a url param, it builds it by
 161       * itself. It also fails-over to the next possible clusters
 162       * provided by $wgDefaultExternalStore.
 163       *
 164       * @param string $data
 165       * @param array $params Associative array of ExternalStoreMedium parameters
 166       * @return string|bool The URL of the stored data item, or false on error
 167       * @throws MWException
 168       */
 169  	public static function insertToDefault( $data, array $params = array() ) {
 170          global $wgDefaultExternalStore;
 171  
 172          return self::insertWithFallback( (array)$wgDefaultExternalStore, $data, $params );
 173      }
 174  
 175      /**
 176       * Like insert() above, but does more of the work for us.
 177       * This function does not need a url param, it builds it by
 178       * itself. It also fails-over to the next possible clusters
 179       * as provided in the first parameter.
 180       *
 181       * @param array $tryStores Refer to $wgDefaultExternalStore
 182       * @param string $data
 183       * @param array $params Associative array of ExternalStoreMedium parameters
 184       * @return string|bool The URL of the stored data item, or false on error
 185       * @throws MWException
 186       */
 187  	public static function insertWithFallback( array $tryStores, $data, array $params = array() ) {
 188          $error = false;
 189          while ( count( $tryStores ) > 0 ) {
 190              $index = mt_rand( 0, count( $tryStores ) - 1 );
 191              $storeUrl = $tryStores[$index];
 192              wfDebug( __METHOD__ . ": trying $storeUrl\n" );
 193              list( $proto, $path ) = explode( '://', $storeUrl, 2 );
 194              $store = self::getStoreObject( $proto, $params );
 195              if ( $store === false ) {
 196                  throw new MWException( "Invalid external storage protocol - $storeUrl" );
 197              }
 198              try {
 199                  $url = $store->store( $path, $data ); // Try to save the object
 200              } catch ( MWException $error ) {
 201                  $url = false;
 202              }
 203              if ( strlen( $url ) ) {
 204                  return $url; // Done!
 205              } else {
 206                  unset( $tryStores[$index] ); // Don't try this one again!
 207                  $tryStores = array_values( $tryStores ); // Must have consecutive keys
 208                  wfDebugLog( 'ExternalStorage',
 209                      "Unable to store text to external storage $storeUrl" );
 210              }
 211          }
 212          // All stores failed
 213          if ( $error ) {
 214              throw $error; // rethrow the last error
 215          } else {
 216              throw new MWException( "Unable to store text to external storage" );
 217          }
 218      }
 219  
 220      /**
 221       * @param string $data
 222       * @param string $wiki
 223       * @return string|bool The URL of the stored data item, or false on error
 224       * @throws MWException
 225       */
 226  	public static function insertToForeignDefault( $data, $wiki ) {
 227          return self::insertToDefault( $data, array( 'wiki' => $wiki ) );
 228      }
 229  }