MediaWiki
REL1_22
|
00001 <?php 00039 class ForeignAPIRepo extends FileRepo { 00040 /* This version string is used in the user agent for requests and will help 00041 * server maintainers in identify ForeignAPI usage. 00042 * Update the version every time you make breaking or significant changes. */ 00043 const VERSION = "2.1"; 00044 00045 var $fileFactory = array( 'ForeignAPIFile', 'newFromTitle' ); 00046 /* Check back with Commons after a day */ 00047 var $apiThumbCacheExpiry = 86400; /* 24*60*60 */ 00048 /* Redownload thumbnail files after a month */ 00049 var $fileCacheExpiry = 2592000; /* 86400*30 */ 00050 00051 protected $mQueryCache = array(); 00052 protected $mFileExists = array(); 00053 00057 function __construct( $info ) { 00058 global $wgLocalFileRepo; 00059 parent::__construct( $info ); 00060 00061 // http://commons.wikimedia.org/w/api.php 00062 $this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null; 00063 00064 if ( isset( $info['apiThumbCacheExpiry'] ) ) { 00065 $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry']; 00066 } 00067 if ( isset( $info['fileCacheExpiry'] ) ) { 00068 $this->fileCacheExpiry = $info['fileCacheExpiry']; 00069 } 00070 if ( !$this->scriptDirUrl ) { 00071 // hack for description fetches 00072 $this->scriptDirUrl = dirname( $this->mApiBase ); 00073 } 00074 // If we can cache thumbs we can guess sane defaults for these 00075 if ( $this->canCacheThumbs() && !$this->url ) { 00076 $this->url = $wgLocalFileRepo['url']; 00077 } 00078 if ( $this->canCacheThumbs() && !$this->thumbUrl ) { 00079 $this->thumbUrl = $this->url . '/thumb'; 00080 } 00081 } 00082 00087 function getApiUrl() { 00088 return $this->mApiBase; 00089 } 00090 00099 function newFile( $title, $time = false ) { 00100 if ( $time ) { 00101 return false; 00102 } 00103 return parent::newFile( $title, $time ); 00104 } 00105 00110 function fileExistsBatch( array $files ) { 00111 $results = array(); 00112 foreach ( $files as $k => $f ) { 00113 if ( isset( $this->mFileExists[$f] ) ) { 00114 $results[$k] = $this->mFileExists[$f]; 00115 unset( $files[$k] ); 00116 } elseif ( self::isVirtualUrl( $f ) ) { 00117 # @todo FIXME: We need to be able to handle virtual 00118 # URLs better, at least when we know they refer to the 00119 # same repo. 00120 $results[$k] = false; 00121 unset( $files[$k] ); 00122 } elseif ( FileBackend::isStoragePath( $f ) ) { 00123 $results[$k] = false; 00124 unset( $files[$k] ); 00125 wfWarn( "Got mwstore:// path '$f'." ); 00126 } 00127 } 00128 00129 $data = $this->fetchImageQuery( array( 'titles' => implode( $files, '|' ), 00130 'prop' => 'imageinfo' ) ); 00131 if ( isset( $data['query']['pages'] ) ) { 00132 # First, get results from the query. Note we only care whether the image exists, 00133 # not whether it has a description page. 00134 foreach ( $data['query']['pages'] as $p ) { 00135 $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' ); 00136 } 00137 # Second, copy the results to any redirects that were queried 00138 if ( isset( $data['query']['redirects'] ) ) { 00139 foreach ( $data['query']['redirects'] as $r ) { 00140 $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']]; 00141 } 00142 } 00143 # Third, copy the results to any non-normalized titles that were queried 00144 if ( isset( $data['query']['normalized'] ) ) { 00145 foreach ( $data['query']['normalized'] as $n ) { 00146 $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']]; 00147 } 00148 } 00149 # Finally, copy the results to the output 00150 foreach ( $files as $key => $file ) { 00151 $results[$key] = $this->mFileExists[$file]; 00152 } 00153 } 00154 return $results; 00155 } 00156 00161 function getFileProps( $virtualUrl ) { 00162 return false; 00163 } 00164 00169 function fetchImageQuery( $query ) { 00170 global $wgMemc, $wgLanguageCode; 00171 00172 $query = array_merge( $query, 00173 array( 00174 'format' => 'json', 00175 'action' => 'query', 00176 'redirects' => 'true' 00177 ) ); 00178 00179 if ( !isset( $query['uselang'] ) ) { // uselang is unset or null 00180 $query['uselang'] = $wgLanguageCode; 00181 } 00182 00183 $data = $this->httpGetCached( 'Metadata', $query ); 00184 00185 if ( $data ) { 00186 return FormatJson::decode( $data, true ); 00187 } else { 00188 return null; 00189 } 00190 } 00191 00196 function getImageInfo( $data ) { 00197 if ( $data && isset( $data['query']['pages'] ) ) { 00198 foreach ( $data['query']['pages'] as $info ) { 00199 if ( isset( $info['imageinfo'][0] ) ) { 00200 return $info['imageinfo'][0]; 00201 } 00202 } 00203 } 00204 return false; 00205 } 00206 00211 function findBySha1( $hash ) { 00212 $results = $this->fetchImageQuery( array( 00213 'aisha1base36' => $hash, 00214 'aiprop' => ForeignAPIFile::getProps(), 00215 'list' => 'allimages', 00216 ) ); 00217 $ret = array(); 00218 if ( isset( $results['query']['allimages'] ) ) { 00219 foreach ( $results['query']['allimages'] as $img ) { 00220 // 1.14 was broken, doesn't return name attribute 00221 if ( !isset( $img['name'] ) ) { 00222 continue; 00223 } 00224 $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img ); 00225 } 00226 } 00227 return $ret; 00228 } 00229 00238 function getThumbUrl( $name, $width = -1, $height = -1, &$result = null, $otherParams = '' ) { 00239 $data = $this->fetchImageQuery( array( 00240 'titles' => 'File:' . $name, 00241 'iiprop' => 'url|timestamp', 00242 'iiurlwidth' => $width, 00243 'iiurlheight' => $height, 00244 'iiurlparam' => $otherParams, 00245 'prop' => 'imageinfo' ) ); 00246 $info = $this->getImageInfo( $data ); 00247 00248 if ( $data && $info && isset( $info['thumburl'] ) ) { 00249 wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] . "\n" ); 00250 $result = $info; 00251 return $info['thumburl']; 00252 } else { 00253 return false; 00254 } 00255 } 00256 00265 function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) { 00266 $data = $this->fetchImageQuery( array( 00267 'titles' => 'File:' . $name, 00268 'iiprop' => 'url|timestamp', 00269 'iiurlwidth' => $width, 00270 'iiurlheight' => $height, 00271 'iiurlparam' => $otherParams, 00272 'prop' => 'imageinfo', 00273 'uselang' => $lang, 00274 ) ); 00275 $info = $this->getImageInfo( $data ); 00276 00277 if ( $data && $info && isset( $info['thumberror'] ) ) { 00278 wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] . "\n" ); 00279 return new MediaTransformError( 00280 'thumbnail_error_remote', 00281 $width, 00282 $height, 00283 $this->getDisplayName(), 00284 $info['thumberror'] // already parsed message from foreign repo 00285 ); 00286 } else { 00287 return false; 00288 } 00289 } 00290 00303 function getThumbUrlFromCache( $name, $width, $height, $params = "" ) { 00304 global $wgMemc; 00305 // We can't check the local cache using FileRepo functions because 00306 // we override fileExistsBatch(). We have to use the FileBackend directly. 00307 $backend = $this->getBackend(); // convenience 00308 00309 if ( !$this->canCacheThumbs() ) { 00310 $result = null; // can't pass "null" by reference, but it's ok as default value 00311 return $this->getThumbUrl( $name, $width, $height, $result, $params ); 00312 } 00313 $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name ); 00314 $sizekey = "$width:$height:$params"; 00315 00316 /* Get the array of urls that we already know */ 00317 $knownThumbUrls = $wgMemc->get( $key ); 00318 if ( !$knownThumbUrls ) { 00319 /* No knownThumbUrls for this file */ 00320 $knownThumbUrls = array(); 00321 } else { 00322 if ( isset( $knownThumbUrls[$sizekey] ) ) { 00323 wfDebug( __METHOD__ . ': Got thumburl from local cache: ' . 00324 "{$knownThumbUrls[$sizekey]} \n" ); 00325 return $knownThumbUrls[$sizekey]; 00326 } 00327 /* This size is not yet known */ 00328 } 00329 00330 $metadata = null; 00331 $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params ); 00332 00333 if ( !$foreignUrl ) { 00334 wfDebug( __METHOD__ . " Could not find thumburl\n" ); 00335 return false; 00336 } 00337 00338 // We need the same filename as the remote one :) 00339 $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) ); 00340 if ( !$this->validateFilename( $fileName ) ) { 00341 wfDebug( __METHOD__ . " The deduced filename $fileName is not safe\n" ); 00342 return false; 00343 } 00344 $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name; 00345 $localFilename = $localPath . "/" . $fileName; 00346 $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) . rawurlencode( $name ) . "/" . rawurlencode( $fileName ); 00347 00348 if ( $backend->fileExists( array( 'src' => $localFilename ) ) 00349 && isset( $metadata['timestamp'] ) ) { 00350 wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" ); 00351 $modified = $backend->getFileTimestamp( array( 'src' => $localFilename ) ); 00352 $remoteModified = strtotime( $metadata['timestamp'] ); 00353 $current = time(); 00354 $diff = abs( $modified - $current ); 00355 if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) { 00356 /* Use our current and already downloaded thumbnail */ 00357 $knownThumbUrls[$sizekey] = $localUrl; 00358 $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry ); 00359 return $localUrl; 00360 } 00361 /* There is a new Commons file, or existing thumbnail older than a month */ 00362 } 00363 $thumb = self::httpGet( $foreignUrl ); 00364 if ( !$thumb ) { 00365 wfDebug( __METHOD__ . " Could not download thumb\n" ); 00366 return false; 00367 } 00368 00369 # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script? 00370 $backend->prepare( array( 'dir' => dirname( $localFilename ) ) ); 00371 $params = array( 'dst' => $localFilename, 'content' => $thumb ); 00372 if ( !$backend->quickCreate( $params )->isOK() ) { 00373 wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'\n" ); 00374 return $foreignUrl; 00375 } 00376 $knownThumbUrls[$sizekey] = $localUrl; 00377 $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry ); 00378 wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache \n" ); 00379 return $localUrl; 00380 } 00381 00388 function getZoneUrl( $zone, $ext = null ) { 00389 switch ( $zone ) { 00390 case 'public': 00391 return $this->url; 00392 case 'thumb': 00393 return $this->thumbUrl; 00394 default: 00395 return parent::getZoneUrl( $zone, $ext ); 00396 } 00397 } 00398 00404 function getZonePath( $zone ) { 00405 $supported = array( 'public', 'thumb' ); 00406 if ( in_array( $zone, $supported ) ) { 00407 return parent::getZonePath( $zone ); 00408 } 00409 return false; 00410 } 00411 00416 public function canCacheThumbs() { 00417 return ( $this->apiThumbCacheExpiry > 0 ); 00418 } 00419 00424 public static function getUserAgent() { 00425 return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION; 00426 } 00427 00434 function getInfo() { 00435 $info = parent::getInfo(); 00436 $info['apiurl'] = $this->getApiUrl(); 00437 00438 $query = array( 00439 'format' => 'json', 00440 'action' => 'query', 00441 'meta' => 'siteinfo', 00442 'siprop' => 'general', 00443 ); 00444 00445 $data = $this->httpGetCached( 'SiteInfo', $query, 7200 ); 00446 00447 if ( $data ) { 00448 $siteInfo = FormatJson::decode( $data, true ); 00449 $general = $siteInfo['query']['general']; 00450 00451 $info['articlepath'] = $general['articlepath']; 00452 $info['server'] = $general['server']; 00453 } 00454 00455 return $info; 00456 } 00457 00466 public static function httpGet( $url, $timeout = 'default', $options = array() ) { 00467 $options['timeout'] = $timeout; 00468 /* Http::get */ 00469 $url = wfExpandUrl( $url, PROTO_HTTP ); 00470 wfDebug( "ForeignAPIRepo: HTTP GET: $url\n" ); 00471 $options['method'] = "GET"; 00472 00473 if ( !isset( $options['timeout'] ) ) { 00474 $options['timeout'] = 'default'; 00475 } 00476 00477 $req = MWHttpRequest::factory( $url, $options ); 00478 $req->setUserAgent( ForeignAPIRepo::getUserAgent() ); 00479 $status = $req->execute(); 00480 00481 if ( $status->isOK() ) { 00482 return $req->getContent(); 00483 } else { 00484 return false; 00485 } 00486 } 00487 00494 public function httpGetCached( $target, $query, $cacheTTL = 3600 ) { 00495 if ( $this->mApiBase ) { 00496 $url = wfAppendQuery( $this->mApiBase, $query ); 00497 } else { 00498 $url = $this->makeUrl( $query, 'api' ); 00499 } 00500 00501 if ( !isset( $this->mQueryCache[$url] ) ) { 00502 global $wgMemc; 00503 00504 $key = $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) ); 00505 $data = $wgMemc->get( $key ); 00506 00507 if ( !$data ) { 00508 $data = self::httpGet( $url ); 00509 00510 if ( !$data ) { 00511 return null; 00512 } 00513 00514 $wgMemc->set( $key, $data, $cacheTTL ); 00515 } 00516 00517 if ( count( $this->mQueryCache ) > 100 ) { 00518 // Keep the cache from growing infinitely 00519 $this->mQueryCache = array(); 00520 } 00521 00522 $this->mQueryCache[$url] = $data; 00523 } 00524 00525 return $this->mQueryCache[$url]; 00526 } 00527 00532 function enumFiles( $callback ) { 00533 throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) ); 00534 } 00535 00539 protected function assertWritableRepo() { 00540 throw new MWException( get_class( $this ) . ': write operations are not supported.' ); 00541 } 00542 }