[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * Foreign repository accessible through api.php requests.
   4   *
   5   * This program is free software; you can redistribute it and/or modify
   6   * it under the terms of the GNU General Public License as published by
   7   * the Free Software Foundation; either version 2 of the License, or
   8   * (at your option) any later version.
   9   *
  10   * This program is distributed in the hope that it will be useful,
  11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13   * GNU General Public License for more details.
  14   *
  15   * You should have received a copy of the GNU General Public License along
  16   * with this program; if not, write to the Free Software Foundation, Inc.,
  17   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18   * http://www.gnu.org/copyleft/gpl.html
  19   *
  20   * @file
  21   * @ingroup FileRepo
  22   */
  23  
  24  /**
  25   * A foreign repository with a remote MediaWiki with an API thingy
  26   *
  27   * Example config:
  28   *
  29   * $wgForeignFileRepos[] = array(
  30   *   'class'                  => 'ForeignAPIRepo',
  31   *   'name'                   => 'shared',
  32   *   'apibase'                => 'http://en.wikipedia.org/w/api.php',
  33   *   'fetchDescription'       => true, // Optional
  34   *   'descriptionCacheExpiry' => 3600,
  35   * );
  36   *
  37   * @ingroup FileRepo
  38   */
  39  class ForeignAPIRepo extends FileRepo {
  40      /* This version string is used in the user agent for requests and will help
  41       * server maintainers in identify ForeignAPI usage.
  42       * Update the version every time you make breaking or significant changes. */
  43      const VERSION = "2.1";
  44  
  45      /**
  46       * List of iiprop values for the thumbnail fetch queries.
  47       * @since 1.23
  48       */
  49      protected static $imageInfoProps = array(
  50          'url',
  51          'thumbnail',
  52          'timestamp',
  53      );
  54  
  55      protected $fileFactory = array( 'ForeignAPIFile', 'newFromTitle' );
  56      /** @var int Check back with Commons after a day (24*60*60) */
  57      protected $apiThumbCacheExpiry = 86400;
  58  
  59      /** @var int Redownload thumbnail files after a month (86400*30) */
  60      protected $fileCacheExpiry = 2592000;
  61  
  62      /** @var array */
  63      protected $mFileExists = array();
  64  
  65      /** @var array */
  66      private $mQueryCache = array();
  67  
  68      /**
  69       * @param array|null $info
  70       */
  71  	function __construct( $info ) {
  72          global $wgLocalFileRepo;
  73          parent::__construct( $info );
  74  
  75          // http://commons.wikimedia.org/w/api.php
  76          $this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null;
  77  
  78          if ( isset( $info['apiThumbCacheExpiry'] ) ) {
  79              $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
  80          }
  81          if ( isset( $info['fileCacheExpiry'] ) ) {
  82              $this->fileCacheExpiry = $info['fileCacheExpiry'];
  83          }
  84          if ( !$this->scriptDirUrl ) {
  85              // hack for description fetches
  86              $this->scriptDirUrl = dirname( $this->mApiBase );
  87          }
  88          // If we can cache thumbs we can guess sane defaults for these
  89          if ( $this->canCacheThumbs() && !$this->url ) {
  90              $this->url = $wgLocalFileRepo['url'];
  91          }
  92          if ( $this->canCacheThumbs() && !$this->thumbUrl ) {
  93              $this->thumbUrl = $this->url . '/thumb';
  94          }
  95      }
  96  
  97      /**
  98       * @return string
  99       * @since 1.22
 100       */
 101  	function getApiUrl() {
 102          return $this->mApiBase;
 103      }
 104  
 105      /**
 106       * Per docs in FileRepo, this needs to return false if we don't support versioned
 107       * files. Well, we don't.
 108       *
 109       * @param Title $title
 110       * @param string|bool $time
 111       * @return File
 112       */
 113  	function newFile( $title, $time = false ) {
 114          if ( $time ) {
 115              return false;
 116          }
 117  
 118          return parent::newFile( $title, $time );
 119      }
 120  
 121      /**
 122       * @param array $files
 123       * @return array
 124       */
 125  	function fileExistsBatch( array $files ) {
 126          $results = array();
 127          foreach ( $files as $k => $f ) {
 128              if ( isset( $this->mFileExists[$f] ) ) {
 129                  $results[$k] = $this->mFileExists[$f];
 130                  unset( $files[$k] );
 131              } elseif ( self::isVirtualUrl( $f ) ) {
 132                  # @todo FIXME: We need to be able to handle virtual
 133                  # URLs better, at least when we know they refer to the
 134                  # same repo.
 135                  $results[$k] = false;
 136                  unset( $files[$k] );
 137              } elseif ( FileBackend::isStoragePath( $f ) ) {
 138                  $results[$k] = false;
 139                  unset( $files[$k] );
 140                  wfWarn( "Got mwstore:// path '$f'." );
 141              }
 142          }
 143  
 144          $data = $this->fetchImageQuery( array(
 145              'titles' => implode( $files, '|' ),
 146              'prop' => 'imageinfo' )
 147          );
 148  
 149          if ( isset( $data['query']['pages'] ) ) {
 150              # First, get results from the query. Note we only care whether the image exists,
 151              # not whether it has a description page.
 152              foreach ( $data['query']['pages'] as $p ) {
 153                  $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' );
 154              }
 155              # Second, copy the results to any redirects that were queried
 156              if ( isset( $data['query']['redirects'] ) ) {
 157                  foreach ( $data['query']['redirects'] as $r ) {
 158                      $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']];
 159                  }
 160              }
 161              # Third, copy the results to any non-normalized titles that were queried
 162              if ( isset( $data['query']['normalized'] ) ) {
 163                  foreach ( $data['query']['normalized'] as $n ) {
 164                      $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']];
 165                  }
 166              }
 167              # Finally, copy the results to the output
 168              foreach ( $files as $key => $file ) {
 169                  $results[$key] = $this->mFileExists[$file];
 170              }
 171          }
 172  
 173          return $results;
 174      }
 175  
 176      /**
 177       * @param string $virtualUrl
 178       * @return bool
 179       */
 180  	function getFileProps( $virtualUrl ) {
 181          return false;
 182      }
 183  
 184      /**
 185       * @param array $query
 186       * @return string
 187       */
 188  	function fetchImageQuery( $query ) {
 189          global $wgLanguageCode;
 190  
 191          $query = array_merge( $query,
 192              array(
 193                  'format' => 'json',
 194                  'action' => 'query',
 195                  'redirects' => 'true'
 196              ) );
 197  
 198          if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
 199              $query['uselang'] = $wgLanguageCode;
 200          }
 201  
 202          $data = $this->httpGetCached( 'Metadata', $query );
 203  
 204          if ( $data ) {
 205              return FormatJson::decode( $data, true );
 206          } else {
 207              return null;
 208          }
 209      }
 210  
 211      /**
 212       * @param array $data
 213       * @return bool|array
 214       */
 215  	function getImageInfo( $data ) {
 216          if ( $data && isset( $data['query']['pages'] ) ) {
 217              foreach ( $data['query']['pages'] as $info ) {
 218                  if ( isset( $info['imageinfo'][0] ) ) {
 219                      return $info['imageinfo'][0];
 220                  }
 221              }
 222          }
 223  
 224          return false;
 225      }
 226  
 227      /**
 228       * @param string $hash
 229       * @return array
 230       */
 231  	function findBySha1( $hash ) {
 232          $results = $this->fetchImageQuery( array(
 233              'aisha1base36' => $hash,
 234              'aiprop' => ForeignAPIFile::getProps(),
 235              'list' => 'allimages',
 236          ) );
 237          $ret = array();
 238          if ( isset( $results['query']['allimages'] ) ) {
 239              foreach ( $results['query']['allimages'] as $img ) {
 240                  // 1.14 was broken, doesn't return name attribute
 241                  if ( !isset( $img['name'] ) ) {
 242                      continue;
 243                  }
 244                  $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img );
 245              }
 246          }
 247  
 248          return $ret;
 249      }
 250  
 251      /**
 252       * @param string $name
 253       * @param int $width
 254       * @param int $height
 255       * @param array $result Out parameter that will be changed by the function.
 256       * @param string $otherParams
 257       *
 258       * @return bool
 259       */
 260  	function getThumbUrl( $name, $width = -1, $height = -1, &$result = null, $otherParams = '' ) {
 261          $data = $this->fetchImageQuery( array(
 262              'titles' => 'File:' . $name,
 263              'iiprop' => self::getIIProps(),
 264              'iiurlwidth' => $width,
 265              'iiurlheight' => $height,
 266              'iiurlparam' => $otherParams,
 267              'prop' => 'imageinfo' ) );
 268          $info = $this->getImageInfo( $data );
 269  
 270          if ( $data && $info && isset( $info['thumburl'] ) ) {
 271              wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] . "\n" );
 272              $result = $info;
 273  
 274              return $info['thumburl'];
 275          } else {
 276              return false;
 277          }
 278      }
 279  
 280      /**
 281       * @param string $name
 282       * @param int $width
 283       * @param int $height
 284       * @param string $otherParams
 285       * @param string $lang Language code for language of error
 286       * @return bool|MediaTransformError
 287       * @since 1.22
 288       */
 289  	function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) {
 290          $data = $this->fetchImageQuery( array(
 291              'titles' => 'File:' . $name,
 292              'iiprop' => self::getIIProps(),
 293              'iiurlwidth' => $width,
 294              'iiurlheight' => $height,
 295              'iiurlparam' => $otherParams,
 296              'prop' => 'imageinfo',
 297              'uselang' => $lang,
 298          ) );
 299          $info = $this->getImageInfo( $data );
 300  
 301          if ( $data && $info && isset( $info['thumberror'] ) ) {
 302              wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] . "\n" );
 303  
 304              return new MediaTransformError(
 305                  'thumbnail_error_remote',
 306                  $width,
 307                  $height,
 308                  $this->getDisplayName(),
 309                  $info['thumberror'] // already parsed message from foreign repo
 310              );
 311          } else {
 312              return false;
 313          }
 314      }
 315  
 316      /**
 317       * Return the imageurl from cache if possible
 318       *
 319       * If the url has been requested today, get it from cache
 320       * Otherwise retrieve remote thumb url, check for local file.
 321       *
 322       * @param string $name Is a dbkey form of a title
 323       * @param int $width
 324       * @param int $height
 325       * @param string $params Other rendering parameters (page number, etc)
 326       *   from handler's makeParamString.
 327       * @return bool|string
 328       */
 329  	function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
 330          global $wgMemc;
 331          // We can't check the local cache using FileRepo functions because
 332          // we override fileExistsBatch(). We have to use the FileBackend directly.
 333          $backend = $this->getBackend(); // convenience
 334  
 335          if ( !$this->canCacheThumbs() ) {
 336              $result = null; // can't pass "null" by reference, but it's ok as default value
 337              return $this->getThumbUrl( $name, $width, $height, $result, $params );
 338          }
 339          $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
 340          $sizekey = "$width:$height:$params";
 341  
 342          /* Get the array of urls that we already know */
 343          $knownThumbUrls = $wgMemc->get( $key );
 344          if ( !$knownThumbUrls ) {
 345              /* No knownThumbUrls for this file */
 346              $knownThumbUrls = array();
 347          } else {
 348              if ( isset( $knownThumbUrls[$sizekey] ) ) {
 349                  wfDebug( __METHOD__ . ': Got thumburl from local cache: ' .
 350                      "{$knownThumbUrls[$sizekey]} \n" );
 351  
 352                  return $knownThumbUrls[$sizekey];
 353              }
 354              /* This size is not yet known */
 355          }
 356  
 357          $metadata = null;
 358          $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
 359  
 360          if ( !$foreignUrl ) {
 361              wfDebug( __METHOD__ . " Could not find thumburl\n" );
 362  
 363              return false;
 364          }
 365  
 366          // We need the same filename as the remote one :)
 367          $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) );
 368          if ( !$this->validateFilename( $fileName ) ) {
 369              wfDebug( __METHOD__ . " The deduced filename $fileName is not safe\n" );
 370  
 371              return false;
 372          }
 373          $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
 374          $localFilename = $localPath . "/" . $fileName;
 375          $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
 376              rawurlencode( $name ) . "/" . rawurlencode( $fileName );
 377  
 378          if ( $backend->fileExists( array( 'src' => $localFilename ) )
 379              && isset( $metadata['timestamp'] )
 380          ) {
 381              wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" );
 382              $modified = $backend->getFileTimestamp( array( 'src' => $localFilename ) );
 383              $remoteModified = strtotime( $metadata['timestamp'] );
 384              $current = time();
 385              $diff = abs( $modified - $current );
 386              if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) {
 387                  /* Use our current and already downloaded thumbnail */
 388                  $knownThumbUrls[$sizekey] = $localUrl;
 389                  $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
 390  
 391                  return $localUrl;
 392              }
 393              /* There is a new Commons file, or existing thumbnail older than a month */
 394          }
 395          $thumb = self::httpGet( $foreignUrl );
 396          if ( !$thumb ) {
 397              wfDebug( __METHOD__ . " Could not download thumb\n" );
 398  
 399              return false;
 400          }
 401  
 402          # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
 403          $backend->prepare( array( 'dir' => dirname( $localFilename ) ) );
 404          $params = array( 'dst' => $localFilename, 'content' => $thumb );
 405          if ( !$backend->quickCreate( $params )->isOK() ) {
 406              wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'\n" );
 407  
 408              return $foreignUrl;
 409          }
 410          $knownThumbUrls[$sizekey] = $localUrl;
 411          $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
 412          wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache \n" );
 413  
 414          return $localUrl;
 415      }
 416  
 417      /**
 418       * @see FileRepo::getZoneUrl()
 419       * @param string $zone
 420       * @param string|null $ext Optional file extension
 421       * @return string
 422       */
 423  	function getZoneUrl( $zone, $ext = null ) {
 424          switch ( $zone ) {
 425              case 'public':
 426                  return $this->url;
 427              case 'thumb':
 428                  return $this->thumbUrl;
 429              default:
 430                  return parent::getZoneUrl( $zone, $ext );
 431          }
 432      }
 433  
 434      /**
 435       * Get the local directory corresponding to one of the basic zones
 436       * @param string $zone
 437       * @return bool|null|string
 438       */
 439  	function getZonePath( $zone ) {
 440          $supported = array( 'public', 'thumb' );
 441          if ( in_array( $zone, $supported ) ) {
 442              return parent::getZonePath( $zone );
 443          }
 444  
 445          return false;
 446      }
 447  
 448      /**
 449       * Are we locally caching the thumbnails?
 450       * @return bool
 451       */
 452  	public function canCacheThumbs() {
 453          return ( $this->apiThumbCacheExpiry > 0 );
 454      }
 455  
 456      /**
 457       * The user agent the ForeignAPIRepo will use.
 458       * @return string
 459       */
 460  	public static function getUserAgent() {
 461          return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION;
 462      }
 463  
 464      /**
 465       * Get information about the repo - overrides/extends the parent
 466       * class's information.
 467       * @return array
 468       * @since 1.22
 469       */
 470  	function getInfo() {
 471          $info = parent::getInfo();
 472          $info['apiurl'] = $this->getApiUrl();
 473  
 474          $query = array(
 475              'format' => 'json',
 476              'action' => 'query',
 477              'meta' => 'siteinfo',
 478              'siprop' => 'general',
 479          );
 480  
 481          $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
 482  
 483          if ( $data ) {
 484              $siteInfo = FormatJson::decode( $data, true );
 485              $general = $siteInfo['query']['general'];
 486  
 487              $info['articlepath'] = $general['articlepath'];
 488              $info['server'] = $general['server'];
 489  
 490              if ( isset( $general['favicon'] ) ) {
 491                  $info['favicon'] = $general['favicon'];
 492              }
 493          }
 494  
 495          return $info;
 496      }
 497  
 498      /**
 499       * Like a Http:get request, but with custom User-Agent.
 500       * @see Http:get
 501       * @param string $url
 502       * @param string $timeout
 503       * @param array $options
 504       * @return bool|string
 505       */
 506  	public static function httpGet( $url, $timeout = 'default', $options = array() ) {
 507          $options['timeout'] = $timeout;
 508          /* Http::get */
 509          $url = wfExpandUrl( $url, PROTO_HTTP );
 510          wfDebug( "ForeignAPIRepo: HTTP GET: $url\n" );
 511          $options['method'] = "GET";
 512  
 513          if ( !isset( $options['timeout'] ) ) {
 514              $options['timeout'] = 'default';
 515          }
 516  
 517          $req = MWHttpRequest::factory( $url, $options );
 518          $req->setUserAgent( ForeignAPIRepo::getUserAgent() );
 519          $status = $req->execute();
 520  
 521          if ( $status->isOK() ) {
 522              return $req->getContent();
 523          } else {
 524              return false;
 525          }
 526      }
 527  
 528      /**
 529       * @return string
 530       * @since 1.23
 531       */
 532  	protected static function getIIProps() {
 533          return join( '|', self::$imageInfoProps );
 534      }
 535  
 536      /**
 537       * HTTP GET request to a mediawiki API (with caching)
 538       * @param string $target Used in cache key creation, mostly
 539       * @param array $query The query parameters for the API request
 540       * @param int $cacheTTL Time to live for the memcached caching
 541       * @return null
 542       */
 543  	public function httpGetCached( $target, $query, $cacheTTL = 3600 ) {
 544          if ( $this->mApiBase ) {
 545              $url = wfAppendQuery( $this->mApiBase, $query );
 546          } else {
 547              $url = $this->makeUrl( $query, 'api' );
 548          }
 549  
 550          if ( !isset( $this->mQueryCache[$url] ) ) {
 551              global $wgMemc;
 552  
 553              $key = $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) );
 554              $data = $wgMemc->get( $key );
 555  
 556              if ( !$data ) {
 557                  $data = self::httpGet( $url );
 558  
 559                  if ( !$data ) {
 560                      return null;
 561                  }
 562  
 563                  $wgMemc->set( $key, $data, $cacheTTL );
 564              }
 565  
 566              if ( count( $this->mQueryCache ) > 100 ) {
 567                  // Keep the cache from growing infinitely
 568                  $this->mQueryCache = array();
 569              }
 570  
 571              $this->mQueryCache[$url] = $data;
 572          }
 573  
 574          return $this->mQueryCache[$url];
 575      }
 576  
 577      /**
 578       * @param callable $callback
 579       * @throws MWException
 580       */
 581  	function enumFiles( $callback ) {
 582          throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) );
 583      }
 584  
 585      /**
 586       * @throws MWException
 587       */
 588  	protected function assertWritableRepo() {
 589          throw new MWException( get_class( $this ) . ': write operations are not supported.' );
 590      }
 591  }