- Generated on Wed Jun 25 2014 21:01:54 for MediaWiki by
1.7.6.1
|
MediaWiki
REL1_21
|
00001 <?php 00039 class SwiftFileBackend extends FileBackendStore { 00041 protected $auth; // Swift authentication handler 00042 protected $authTTL; // integer seconds 00043 protected $swiftTempUrlKey; // string; shared secret value for making temp urls 00044 protected $swiftAnonUser; // string; username to handle unauthenticated requests 00045 protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled 00046 protected $swiftCDNExpiry; // integer; how long to cache things in the CDN 00047 protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled 00048 00049 // Rados Gateway specific options 00050 protected $rgwS3AccessKey; // string; S3 access key 00051 protected $rgwS3SecretKey; // string; S3 authentication key 00052 00054 protected $conn; // Swift connection handle 00055 protected $sessionStarted = 0; // integer UNIX timestamp 00056 00058 protected $connException; 00059 protected $connErrorTime = 0; // UNIX timestamp 00060 00062 protected $srvCache; 00063 00065 protected $connContainerCache; // container object cache 00066 00105 public function __construct( array $config ) { 00106 parent::__construct( $config ); 00107 if ( !MWInit::classExists( 'CF_Constants' ) ) { 00108 throw new MWException( 'SwiftCloudFiles extension not installed.' ); 00109 } 00110 // Required settings 00111 $this->auth = new CF_Authentication( 00112 $config['swiftUser'], 00113 $config['swiftKey'], 00114 null, // account; unused 00115 $config['swiftAuthUrl'] 00116 ); 00117 // Optional settings 00118 $this->authTTL = isset( $config['swiftAuthTTL'] ) 00119 ? $config['swiftAuthTTL'] 00120 : 5 * 60; // some sane number 00121 $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) 00122 ? $config['swiftAnonUser'] 00123 : ''; 00124 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) 00125 ? $config['swiftTempUrlKey'] 00126 : ''; 00127 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) 00128 ? $config['shardViaHashLevels'] 00129 : ''; 00130 $this->swiftUseCDN = isset( $config['swiftUseCDN'] ) 00131 ? $config['swiftUseCDN'] 00132 : false; 00133 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] ) 00134 ? $config['swiftCDNExpiry'] 00135 : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) 00136 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) 00137 ? $config['swiftCDNPurgable'] 00138 : true; 00139 $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) 00140 ? $config['rgwS3AccessKey'] 00141 : ''; 00142 $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) 00143 ? $config['rgwS3SecretKey'] 00144 : ''; 00145 // Cache container information to mask latency 00146 $this->memCache = wfGetMainCache(); 00147 // Process cache for container info 00148 $this->connContainerCache = new ProcessCacheLRU( 300 ); 00149 // Cache auth token information to avoid RTTs 00150 if ( !empty( $config['cacheAuthInfo'] ) ) { 00151 if ( PHP_SAPI === 'cli' ) { 00152 $this->srvCache = wfGetMainCache(); // preferrably memcached 00153 } else { 00154 try { // look for APC, XCache, WinCache, ect... 00155 $this->srvCache = ObjectCache::newAccelerator( array() ); 00156 } catch ( Exception $e ) {} 00157 } 00158 } 00159 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff(); 00160 } 00161 00166 protected function resolveContainerPath( $container, $relStoragePath ) { 00167 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF 00168 return null; // not UTF-8, makes it hard to use CF and the swift HTTP API 00169 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { 00170 return null; // too long for Swift 00171 } 00172 return $relStoragePath; 00173 } 00174 00179 public function isPathUsableInternal( $storagePath ) { 00180 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); 00181 if ( $rel === null ) { 00182 return false; // invalid 00183 } 00184 00185 try { 00186 $this->getContainer( $container ); 00187 return true; // container exists 00188 } catch ( NoSuchContainerException $e ) { 00189 } catch ( CloudFilesException $e ) { // some other exception? 00190 $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) ); 00191 } 00192 00193 return false; 00194 } 00195 00200 protected function truncDisp( $disposition ) { 00201 $res = ''; 00202 foreach ( explode( ';', $disposition ) as $part ) { 00203 $part = trim( $part ); 00204 $new = ( $res === '' ) ? $part : "{$res};{$part}"; 00205 if ( strlen( $new ) <= 255 ) { 00206 $res = $new; 00207 } else { 00208 break; // too long; sigh 00209 } 00210 } 00211 return $res; 00212 } 00213 00218 protected function doCreateInternal( array $params ) { 00219 $status = Status::newGood(); 00220 00221 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 00222 if ( $dstRel === null ) { 00223 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 00224 return $status; 00225 } 00226 00227 // (a) Check the destination container and object 00228 try { 00229 $dContObj = $this->getContainer( $dstCont ); 00230 } catch ( NoSuchContainerException $e ) { 00231 $status->fatal( 'backend-fail-create', $params['dst'] ); 00232 return $status; 00233 } catch ( CloudFilesException $e ) { // some other exception? 00234 $this->handleException( $e, $status, __METHOD__, $params ); 00235 return $status; 00236 } 00237 00238 // (b) Get a SHA-1 hash of the object 00239 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 ); 00240 00241 // (c) Actually create the object 00242 try { 00243 // Create a fresh CF_Object with no fields preloaded. 00244 // We don't want to preserve headers, metadata, and such. 00245 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD 00246 $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); 00247 // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59). 00248 // The MD5 here will be checked within Swift against its own MD5. 00249 $obj->set_etag( md5( $params['content'] ) ); 00250 // Use the same content type as StreamFile for security 00251 $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); 00252 if ( !strlen( $obj->content_type ) ) { // special case 00253 $obj->content_type = 'unknown/unknown'; 00254 } 00255 // Set the Content-Disposition header if requested 00256 if ( isset( $params['disposition'] ) ) { 00257 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); 00258 } 00259 // Set any other custom headers if requested 00260 if ( isset( $params['headers'] ) ) { 00261 $obj->headers += $params['headers']; 00262 } 00263 if ( !empty( $params['async'] ) ) { // deferred 00264 $op = $obj->write_async( $params['content'] ); 00265 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op ); 00266 $status->value->affectedObjects[] = $obj; 00267 } else { // actually write the object in Swift 00268 $obj->write( $params['content'] ); 00269 $this->purgeCDNCache( array( $obj ) ); 00270 } 00271 } catch ( CDNNotEnabledException $e ) { 00272 // CDN not enabled; nothing to see here 00273 } catch ( BadContentTypeException $e ) { 00274 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 00275 } catch ( CloudFilesException $e ) { // some other exception? 00276 $this->handleException( $e, $status, __METHOD__, $params ); 00277 } 00278 00279 return $status; 00280 } 00281 00285 protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) { 00286 try { 00287 $cfOp->getLastResponse(); 00288 } catch ( BadContentTypeException $e ) { 00289 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 00290 } 00291 } 00292 00297 protected function doStoreInternal( array $params ) { 00298 $status = Status::newGood(); 00299 00300 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 00301 if ( $dstRel === null ) { 00302 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 00303 return $status; 00304 } 00305 00306 // (a) Check the destination container and object 00307 try { 00308 $dContObj = $this->getContainer( $dstCont ); 00309 } catch ( NoSuchContainerException $e ) { 00310 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00311 return $status; 00312 } catch ( CloudFilesException $e ) { // some other exception? 00313 $this->handleException( $e, $status, __METHOD__, $params ); 00314 return $status; 00315 } 00316 00317 // (b) Get a SHA-1 hash of the object 00318 wfSuppressWarnings(); 00319 $sha1Hash = sha1_file( $params['src'] ); 00320 wfRestoreWarnings(); 00321 if ( $sha1Hash === false ) { // source doesn't exist? 00322 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00323 return $status; 00324 } 00325 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 ); 00326 00327 // (c) Actually store the object 00328 try { 00329 // Create a fresh CF_Object with no fields preloaded. 00330 // We don't want to preserve headers, metadata, and such. 00331 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD 00332 $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); 00333 // The MD5 here will be checked within Swift against its own MD5. 00334 $obj->set_etag( md5_file( $params['src'] ) ); 00335 // Use the same content type as StreamFile for security 00336 $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); 00337 if ( !strlen( $obj->content_type ) ) { // special case 00338 $obj->content_type = 'unknown/unknown'; 00339 } 00340 // Set the Content-Disposition header if requested 00341 if ( isset( $params['disposition'] ) ) { 00342 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); 00343 } 00344 // Set any other custom headers if requested 00345 if ( isset( $params['headers'] ) ) { 00346 $obj->headers += $params['headers']; 00347 } 00348 if ( !empty( $params['async'] ) ) { // deferred 00349 wfSuppressWarnings(); 00350 $fp = fopen( $params['src'], 'rb' ); 00351 wfRestoreWarnings(); 00352 if ( !$fp ) { 00353 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00354 } else { 00355 $op = $obj->write_async( $fp, filesize( $params['src'] ), true ); 00356 $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op ); 00357 $status->value->resourcesToClose[] = $fp; 00358 $status->value->affectedObjects[] = $obj; 00359 } 00360 } else { // actually write the object in Swift 00361 $obj->load_from_filename( $params['src'], true ); // calls $obj->write() 00362 $this->purgeCDNCache( array( $obj ) ); 00363 } 00364 } catch ( CDNNotEnabledException $e ) { 00365 // CDN not enabled; nothing to see here 00366 } catch ( BadContentTypeException $e ) { 00367 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 00368 } catch ( IOException $e ) { 00369 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00370 } catch ( CloudFilesException $e ) { // some other exception? 00371 $this->handleException( $e, $status, __METHOD__, $params ); 00372 } 00373 00374 return $status; 00375 } 00376 00380 protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) { 00381 try { 00382 $cfOp->getLastResponse(); 00383 } catch ( BadContentTypeException $e ) { 00384 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 00385 } catch ( IOException $e ) { 00386 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00387 } 00388 } 00389 00394 protected function doCopyInternal( array $params ) { 00395 $status = Status::newGood(); 00396 00397 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 00398 if ( $srcRel === null ) { 00399 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 00400 return $status; 00401 } 00402 00403 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 00404 if ( $dstRel === null ) { 00405 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 00406 return $status; 00407 } 00408 00409 // (a) Check the source/destination containers and destination object 00410 try { 00411 $sContObj = $this->getContainer( $srcCont ); 00412 $dContObj = $this->getContainer( $dstCont ); 00413 } catch ( NoSuchContainerException $e ) { 00414 if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { 00415 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00416 } 00417 return $status; 00418 } catch ( CloudFilesException $e ) { // some other exception? 00419 $this->handleException( $e, $status, __METHOD__, $params ); 00420 return $status; 00421 } 00422 00423 // (b) Actually copy the file to the destination 00424 try { 00425 $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD 00426 $hdrs = array(); // source file headers to override with new values 00427 if ( isset( $params['disposition'] ) ) { 00428 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); 00429 } 00430 if ( !empty( $params['async'] ) ) { // deferred 00431 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); 00432 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op ); 00433 $status->value->affectedObjects[] = $dstObj; 00434 } else { // actually write the object in Swift 00435 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); 00436 $this->purgeCDNCache( array( $dstObj ) ); 00437 } 00438 } catch ( CDNNotEnabledException $e ) { 00439 // CDN not enabled; nothing to see here 00440 } catch ( NoSuchObjectException $e ) { // source object does not exist 00441 if ( empty( $params['ignoreMissingSource'] ) ) { 00442 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00443 } 00444 } catch ( CloudFilesException $e ) { // some other exception? 00445 $this->handleException( $e, $status, __METHOD__, $params ); 00446 } 00447 00448 return $status; 00449 } 00450 00454 protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) { 00455 try { 00456 $cfOp->getLastResponse(); 00457 } catch ( NoSuchObjectException $e ) { // source object does not exist 00458 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 00459 } 00460 } 00461 00466 protected function doMoveInternal( array $params ) { 00467 $status = Status::newGood(); 00468 00469 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 00470 if ( $srcRel === null ) { 00471 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 00472 return $status; 00473 } 00474 00475 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 00476 if ( $dstRel === null ) { 00477 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 00478 return $status; 00479 } 00480 00481 // (a) Check the source/destination containers and destination object 00482 try { 00483 $sContObj = $this->getContainer( $srcCont ); 00484 $dContObj = $this->getContainer( $dstCont ); 00485 } catch ( NoSuchContainerException $e ) { 00486 if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { 00487 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); 00488 } 00489 return $status; 00490 } catch ( CloudFilesException $e ) { // some other exception? 00491 $this->handleException( $e, $status, __METHOD__, $params ); 00492 return $status; 00493 } 00494 00495 // (b) Actually move the file to the destination 00496 try { 00497 $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD 00498 $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD 00499 $hdrs = array(); // source file headers to override with new values 00500 if ( isset( $params['disposition'] ) ) { 00501 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); 00502 } 00503 if ( !empty( $params['async'] ) ) { // deferred 00504 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); 00505 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op ); 00506 $status->value->affectedObjects[] = $srcObj; 00507 $status->value->affectedObjects[] = $dstObj; 00508 } else { // actually write the object in Swift 00509 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); 00510 $this->purgeCDNCache( array( $srcObj ) ); 00511 $this->purgeCDNCache( array( $dstObj ) ); 00512 } 00513 } catch ( CDNNotEnabledException $e ) { 00514 // CDN not enabled; nothing to see here 00515 } catch ( NoSuchObjectException $e ) { // source object does not exist 00516 if ( empty( $params['ignoreMissingSource'] ) ) { 00517 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); 00518 } 00519 } catch ( CloudFilesException $e ) { // some other exception? 00520 $this->handleException( $e, $status, __METHOD__, $params ); 00521 } 00522 00523 return $status; 00524 } 00525 00529 protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) { 00530 try { 00531 $cfOp->getLastResponse(); 00532 } catch ( NoSuchObjectException $e ) { // source object does not exist 00533 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); 00534 } 00535 } 00536 00541 protected function doDeleteInternal( array $params ) { 00542 $status = Status::newGood(); 00543 00544 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 00545 if ( $srcRel === null ) { 00546 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 00547 return $status; 00548 } 00549 00550 try { 00551 $sContObj = $this->getContainer( $srcCont ); 00552 $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD 00553 if ( !empty( $params['async'] ) ) { // deferred 00554 $op = $sContObj->delete_object_async( $srcRel ); 00555 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op ); 00556 $status->value->affectedObjects[] = $srcObj; 00557 } else { // actually write the object in Swift 00558 $sContObj->delete_object( $srcRel ); 00559 $this->purgeCDNCache( array( $srcObj ) ); 00560 } 00561 } catch ( CDNNotEnabledException $e ) { 00562 // CDN not enabled; nothing to see here 00563 } catch ( NoSuchContainerException $e ) { 00564 if ( empty( $params['ignoreMissingSource'] ) ) { 00565 $status->fatal( 'backend-fail-delete', $params['src'] ); 00566 } 00567 } catch ( NoSuchObjectException $e ) { 00568 if ( empty( $params['ignoreMissingSource'] ) ) { 00569 $status->fatal( 'backend-fail-delete', $params['src'] ); 00570 } 00571 } catch ( CloudFilesException $e ) { // some other exception? 00572 $this->handleException( $e, $status, __METHOD__, $params ); 00573 } 00574 00575 return $status; 00576 } 00577 00581 protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) { 00582 try { 00583 $cfOp->getLastResponse(); 00584 } catch ( NoSuchContainerException $e ) { 00585 $status->fatal( 'backend-fail-delete', $params['src'] ); 00586 } catch ( NoSuchObjectException $e ) { 00587 if ( empty( $params['ignoreMissingSource'] ) ) { 00588 $status->fatal( 'backend-fail-delete', $params['src'] ); 00589 } 00590 } 00591 } 00592 00597 protected function doDescribeInternal( array $params ) { 00598 $status = Status::newGood(); 00599 00600 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 00601 if ( $srcRel === null ) { 00602 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 00603 return $status; 00604 } 00605 00606 $hdrs = isset( $params['headers'] ) ? $params['headers'] : array(); 00607 // Set the Content-Disposition header if requested 00608 if ( isset( $params['disposition'] ) ) { 00609 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); 00610 } 00611 00612 try { 00613 $sContObj = $this->getContainer( $srcCont ); 00614 // Get the latest version of the current metadata 00615 $srcObj = $sContObj->get_object( $srcRel, 00616 $this->headersFromParams( array( 'latest' => true ) ) ); 00617 // Merge in the metadata updates... 00618 $srcObj->headers = $hdrs + $srcObj->headers; 00619 $srcObj->sync_metadata(); // save to Swift 00620 $this->purgeCDNCache( array( $srcObj ) ); 00621 } catch ( CDNNotEnabledException $e ) { 00622 // CDN not enabled; nothing to see here 00623 } catch ( NoSuchContainerException $e ) { 00624 $status->fatal( 'backend-fail-describe', $params['src'] ); 00625 } catch ( NoSuchObjectException $e ) { 00626 $status->fatal( 'backend-fail-describe', $params['src'] ); 00627 } catch ( CloudFilesException $e ) { // some other exception? 00628 $this->handleException( $e, $status, __METHOD__, $params ); 00629 } 00630 00631 return $status; 00632 } 00633 00638 protected function doPrepareInternal( $fullCont, $dir, array $params ) { 00639 $status = Status::newGood(); 00640 00641 // (a) Check if container already exists 00642 try { 00643 $this->getContainer( $fullCont ); 00644 // NoSuchContainerException not thrown: container must exist 00645 return $status; // already exists 00646 } catch ( NoSuchContainerException $e ) { 00647 // NoSuchContainerException thrown: container does not exist 00648 } catch ( CloudFilesException $e ) { // some other exception? 00649 $this->handleException( $e, $status, __METHOD__, $params ); 00650 return $status; 00651 } 00652 00653 // (b) Create container as needed 00654 try { 00655 $contObj = $this->createContainer( $fullCont ); 00656 if ( !empty( $params['noAccess'] ) ) { 00657 // Make container private to end-users... 00658 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); 00659 } else { 00660 // Make container public to end-users... 00661 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); 00662 } 00663 if ( $this->swiftUseCDN ) { // Rackspace style CDN 00664 $contObj->make_public( $this->swiftCDNExpiry ); 00665 } 00666 } catch ( CDNNotEnabledException $e ) { 00667 // CDN not enabled; nothing to see here 00668 } catch ( CloudFilesException $e ) { // some other exception? 00669 $this->handleException( $e, $status, __METHOD__, $params ); 00670 return $status; 00671 } 00672 00673 return $status; 00674 } 00675 00680 protected function doSecureInternal( $fullCont, $dir, array $params ) { 00681 $status = Status::newGood(); 00682 if ( empty( $params['noAccess'] ) ) { 00683 return $status; // nothing to do 00684 } 00685 00686 // Restrict container from end-users... 00687 try { 00688 // doPrepareInternal() should have been called, 00689 // so the Swift container should already exist... 00690 $contObj = $this->getContainer( $fullCont ); // normally a cache hit 00691 // NoSuchContainerException not thrown: container must exist 00692 00693 // Make container private to end-users... 00694 $status->merge( $this->setContainerAccess( 00695 $contObj, 00696 array( $this->auth->username ), // read 00697 array( $this->auth->username ) // write 00698 ) ); 00699 if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN 00700 $contObj->make_private(); 00701 } 00702 } catch ( CDNNotEnabledException $e ) { 00703 // CDN not enabled; nothing to see here 00704 } catch ( CloudFilesException $e ) { // some other exception? 00705 $this->handleException( $e, $status, __METHOD__, $params ); 00706 } 00707 00708 return $status; 00709 } 00710 00715 protected function doPublishInternal( $fullCont, $dir, array $params ) { 00716 $status = Status::newGood(); 00717 00718 // Unrestrict container from end-users... 00719 try { 00720 // doPrepareInternal() should have been called, 00721 // so the Swift container should already exist... 00722 $contObj = $this->getContainer( $fullCont ); // normally a cache hit 00723 // NoSuchContainerException not thrown: container must exist 00724 00725 // Make container public to end-users... 00726 if ( $this->swiftAnonUser != '' ) { 00727 $status->merge( $this->setContainerAccess( 00728 $contObj, 00729 array( $this->auth->username, $this->swiftAnonUser ), // read 00730 array( $this->auth->username, $this->swiftAnonUser ) // write 00731 ) ); 00732 } else { 00733 $status->merge( $this->setContainerAccess( 00734 $contObj, 00735 array( $this->auth->username, '.r:*' ), // read 00736 array( $this->auth->username ) // write 00737 ) ); 00738 } 00739 if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN 00740 $contObj->make_public(); 00741 } 00742 } catch ( CDNNotEnabledException $e ) { 00743 // CDN not enabled; nothing to see here 00744 } catch ( CloudFilesException $e ) { // some other exception? 00745 $this->handleException( $e, $status, __METHOD__, $params ); 00746 } 00747 00748 return $status; 00749 } 00750 00755 protected function doCleanInternal( $fullCont, $dir, array $params ) { 00756 $status = Status::newGood(); 00757 00758 // Only containers themselves can be removed, all else is virtual 00759 if ( $dir != '' ) { 00760 return $status; // nothing to do 00761 } 00762 00763 // (a) Check the container 00764 try { 00765 $contObj = $this->getContainer( $fullCont, true ); 00766 } catch ( NoSuchContainerException $e ) { 00767 return $status; // ok, nothing to do 00768 } catch ( CloudFilesException $e ) { // some other exception? 00769 $this->handleException( $e, $status, __METHOD__, $params ); 00770 return $status; 00771 } 00772 00773 // (b) Delete the container if empty 00774 if ( $contObj->object_count == 0 ) { 00775 try { 00776 $this->deleteContainer( $fullCont ); 00777 } catch ( NoSuchContainerException $e ) { 00778 return $status; // race? 00779 } catch ( NonEmptyContainerException $e ) { 00780 return $status; // race? consistency delay? 00781 } catch ( CloudFilesException $e ) { // some other exception? 00782 $this->handleException( $e, $status, __METHOD__, $params ); 00783 return $status; 00784 } 00785 } 00786 00787 return $status; 00788 } 00789 00794 protected function doGetFileStat( array $params ) { 00795 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 00796 if ( $srcRel === null ) { 00797 return false; // invalid storage path 00798 } 00799 00800 $stat = false; 00801 try { 00802 $contObj = $this->getContainer( $srcCont ); 00803 $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) ); 00804 $this->addMissingMetadata( $srcObj, $params['src'] ); 00805 $stat = array( 00806 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW 00807 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), 00808 'size' => (int)$srcObj->content_length, 00809 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) 00810 ); 00811 } catch ( NoSuchContainerException $e ) { 00812 } catch ( NoSuchObjectException $e ) { 00813 } catch ( CloudFilesException $e ) { // some other exception? 00814 $stat = null; 00815 $this->handleException( $e, null, __METHOD__, $params ); 00816 } 00817 00818 return $stat; 00819 } 00820 00829 protected function addMissingMetadata( CF_Object $obj, $path ) { 00830 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) { 00831 return true; // nothing to do 00832 } 00833 wfProfileIn( __METHOD__ ); 00834 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); 00835 $status = Status::newGood(); 00836 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); 00837 if ( $status->isOK() ) { 00838 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); 00839 if ( $tmpFile ) { 00840 $hash = $tmpFile->getSha1Base36(); 00841 if ( $hash !== false ) { 00842 $obj->setMetadataValues( array( 'Sha1base36' => $hash ) ); 00843 $obj->sync_metadata(); // save to Swift 00844 wfProfileOut( __METHOD__ ); 00845 return true; // success 00846 } 00847 } 00848 } 00849 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); 00850 $obj->setMetadataValues( array( 'Sha1base36' => false ) ); 00851 wfProfileOut( __METHOD__ ); 00852 return false; // failed 00853 } 00854 00859 protected function doGetFileContentsMulti( array $params ) { 00860 $contents = array(); 00861 00862 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging 00863 // Blindly create tmp files and stream to them, catching any exception if the file does 00864 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). 00865 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { 00866 $cfOps = array(); // (path => CF_Async_Op) 00867 00868 foreach ( $pathBatch as $path ) { // each path in this concurrent batch 00869 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 00870 if ( $srcRel === null ) { 00871 $contents[$path] = false; 00872 continue; 00873 } 00874 $data = false; 00875 try { 00876 $sContObj = $this->getContainer( $srcCont ); 00877 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD 00878 // Create a new temporary memory file... 00879 $handle = fopen( 'php://temp', 'wb' ); 00880 if ( $handle ) { 00881 $headers = $this->headersFromParams( $params ); 00882 if ( count( $pathBatch ) > 1 ) { 00883 $cfOps[$path] = $obj->stream_async( $handle, $headers ); 00884 $cfOps[$path]->_file_handle = $handle; // close this later 00885 } else { 00886 $obj->stream( $handle, $headers ); 00887 rewind( $handle ); // start from the beginning 00888 $data = stream_get_contents( $handle ); 00889 fclose( $handle ); 00890 } 00891 } else { 00892 $data = false; 00893 } 00894 } catch ( NoSuchContainerException $e ) { 00895 $data = false; 00896 } catch ( NoSuchObjectException $e ) { 00897 $data = false; 00898 } catch ( CloudFilesException $e ) { // some other exception? 00899 $data = false; 00900 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); 00901 } 00902 $contents[$path] = $data; 00903 } 00904 00905 $batch = new CF_Async_Op_Batch( $cfOps ); 00906 $cfOps = $batch->execute(); 00907 foreach ( $cfOps as $path => $cfOp ) { 00908 try { 00909 $cfOp->getLastResponse(); 00910 rewind( $cfOp->_file_handle ); // start from the beginning 00911 $contents[$path] = stream_get_contents( $cfOp->_file_handle ); 00912 } catch ( NoSuchContainerException $e ) { 00913 $contents[$path] = false; 00914 } catch ( NoSuchObjectException $e ) { 00915 $contents[$path] = false; 00916 } catch ( CloudFilesException $e ) { // some other exception? 00917 $contents[$path] = false; 00918 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); 00919 } 00920 fclose( $cfOp->_file_handle ); // close open handle 00921 } 00922 } 00923 00924 return $contents; 00925 } 00926 00931 protected function doDirectoryExists( $fullCont, $dir, array $params ) { 00932 try { 00933 $container = $this->getContainer( $fullCont ); 00934 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 00935 return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 ); 00936 } catch ( NoSuchContainerException $e ) { 00937 return false; 00938 } catch ( CloudFilesException $e ) { // some other exception? 00939 $this->handleException( $e, null, __METHOD__, 00940 array( 'cont' => $fullCont, 'dir' => $dir ) ); 00941 } 00942 00943 return null; // error 00944 } 00945 00950 public function getDirectoryListInternal( $fullCont, $dir, array $params ) { 00951 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params ); 00952 } 00953 00958 public function getFileListInternal( $fullCont, $dir, array $params ) { 00959 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params ); 00960 } 00961 00972 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { 00973 $dirs = array(); 00974 if ( $after === INF ) { 00975 return $dirs; // nothing more 00976 } 00977 wfProfileIn( __METHOD__ . '-' . $this->name ); 00978 00979 try { 00980 $container = $this->getContainer( $fullCont ); 00981 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 00982 // Non-recursive: only list dirs right under $dir 00983 if ( !empty( $params['topOnly'] ) ) { 00984 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); 00985 foreach ( $objects as $object ) { // files and dirs 00986 if ( substr( $object, -1 ) === '/' ) { 00987 $dirs[] = $object; // directories end in '/' 00988 } 00989 } 00990 // Recursive: list all dirs under $dir and its subdirs 00991 } else { 00992 // Get directory from last item of prior page 00993 $lastDir = $this->getParentDir( $after ); // must be first page 00994 $objects = $container->list_objects( $limit, $after, $prefix ); 00995 foreach ( $objects as $object ) { // files 00996 $objectDir = $this->getParentDir( $object ); // directory of object 00997 if ( $objectDir !== false && $objectDir !== $dir ) { 00998 // Swift stores paths in UTF-8, using binary sorting. 00999 // See function "create_container_table" in common/db.py. 01000 // If a directory is not "greater" than the last one, 01001 // then it was already listed by the calling iterator. 01002 if ( strcmp( $objectDir, $lastDir ) > 0 ) { 01003 $pDir = $objectDir; 01004 do { // add dir and all its parent dirs 01005 $dirs[] = "{$pDir}/"; 01006 $pDir = $this->getParentDir( $pDir ); 01007 } while ( $pDir !== false // sanity 01008 && strcmp( $pDir, $lastDir ) > 0 // not done already 01009 && strlen( $pDir ) > strlen( $dir ) // within $dir 01010 ); 01011 } 01012 $lastDir = $objectDir; 01013 } 01014 } 01015 } 01016 if ( count( $objects ) < $limit ) { 01017 $after = INF; // avoid a second RTT 01018 } else { 01019 $after = end( $objects ); // update last item 01020 } 01021 } catch ( NoSuchContainerException $e ) { 01022 } catch ( CloudFilesException $e ) { // some other exception? 01023 $this->handleException( $e, null, __METHOD__, 01024 array( 'cont' => $fullCont, 'dir' => $dir ) ); 01025 } 01026 01027 wfProfileOut( __METHOD__ . '-' . $this->name ); 01028 return $dirs; 01029 } 01030 01031 protected function getParentDir( $path ) { 01032 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; 01033 } 01034 01045 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { 01046 $files = array(); 01047 if ( $after === INF ) { 01048 return $files; // nothing more 01049 } 01050 wfProfileIn( __METHOD__ . '-' . $this->name ); 01051 01052 try { 01053 $container = $this->getContainer( $fullCont ); 01054 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 01055 // Non-recursive: only list files right under $dir 01056 if ( !empty( $params['topOnly'] ) ) { // files and dirs 01057 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); 01058 foreach ( $objects as $object ) { 01059 if ( substr( $object, -1 ) !== '/' ) { 01060 $files[] = $object; // directories end in '/' 01061 } 01062 } 01063 // Recursive: list all files under $dir and its subdirs 01064 } else { // files 01065 $objects = $container->list_objects( $limit, $after, $prefix ); 01066 $files = $objects; 01067 } 01068 if ( count( $objects ) < $limit ) { 01069 $after = INF; // avoid a second RTT 01070 } else { 01071 $after = end( $objects ); // update last item 01072 } 01073 } catch ( NoSuchContainerException $e ) { 01074 } catch ( CloudFilesException $e ) { // some other exception? 01075 $this->handleException( $e, null, __METHOD__, 01076 array( 'cont' => $fullCont, 'dir' => $dir ) ); 01077 } 01078 01079 wfProfileOut( __METHOD__ . '-' . $this->name ); 01080 return $files; 01081 } 01082 01087 protected function doGetFileSha1base36( array $params ) { 01088 $stat = $this->getFileStat( $params ); 01089 if ( $stat ) { 01090 return $stat['sha1']; 01091 } else { 01092 return false; 01093 } 01094 } 01095 01100 protected function doStreamFile( array $params ) { 01101 $status = Status::newGood(); 01102 01103 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 01104 if ( $srcRel === null ) { 01105 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 01106 } 01107 01108 try { 01109 $cont = $this->getContainer( $srcCont ); 01110 } catch ( NoSuchContainerException $e ) { 01111 $status->fatal( 'backend-fail-stream', $params['src'] ); 01112 return $status; 01113 } catch ( CloudFilesException $e ) { // some other exception? 01114 $this->handleException( $e, $status, __METHOD__, $params ); 01115 return $status; 01116 } 01117 01118 try { 01119 $output = fopen( 'php://output', 'wb' ); 01120 $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD 01121 $obj->stream( $output, $this->headersFromParams( $params ) ); 01122 } catch ( NoSuchObjectException $e ) { 01123 $status->fatal( 'backend-fail-stream', $params['src'] ); 01124 } catch ( CloudFilesException $e ) { // some other exception? 01125 $this->handleException( $e, $status, __METHOD__, $params ); 01126 } 01127 01128 return $status; 01129 } 01130 01135 protected function doGetLocalCopyMulti( array $params ) { 01136 $tmpFiles = array(); 01137 01138 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging 01139 // Blindly create tmp files and stream to them, catching any exception if the file does 01140 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). 01141 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { 01142 $cfOps = array(); // (path => CF_Async_Op) 01143 01144 foreach ( $pathBatch as $path ) { // each path in this concurrent batch 01145 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 01146 if ( $srcRel === null ) { 01147 $tmpFiles[$path] = null; 01148 continue; 01149 } 01150 $tmpFile = null; 01151 try { 01152 $sContObj = $this->getContainer( $srcCont ); 01153 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD 01154 // Get source file extension 01155 $ext = FileBackend::extensionFromPath( $path ); 01156 // Create a new temporary file... 01157 $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); 01158 if ( $tmpFile ) { 01159 $handle = fopen( $tmpFile->getPath(), 'wb' ); 01160 if ( $handle ) { 01161 $headers = $this->headersFromParams( $params ); 01162 if ( count( $pathBatch ) > 1 ) { 01163 $cfOps[$path] = $obj->stream_async( $handle, $headers ); 01164 $cfOps[$path]->_file_handle = $handle; // close this later 01165 } else { 01166 $obj->stream( $handle, $headers ); 01167 fclose( $handle ); 01168 } 01169 } else { 01170 $tmpFile = null; 01171 } 01172 } 01173 } catch ( NoSuchContainerException $e ) { 01174 $tmpFile = null; 01175 } catch ( NoSuchObjectException $e ) { 01176 $tmpFile = null; 01177 } catch ( CloudFilesException $e ) { // some other exception? 01178 $tmpFile = null; 01179 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); 01180 } 01181 $tmpFiles[$path] = $tmpFile; 01182 } 01183 01184 $batch = new CF_Async_Op_Batch( $cfOps ); 01185 $cfOps = $batch->execute(); 01186 foreach ( $cfOps as $path => $cfOp ) { 01187 try { 01188 $cfOp->getLastResponse(); 01189 } catch ( NoSuchContainerException $e ) { 01190 $tmpFiles[$path] = null; 01191 } catch ( NoSuchObjectException $e ) { 01192 $tmpFiles[$path] = null; 01193 } catch ( CloudFilesException $e ) { // some other exception? 01194 $tmpFiles[$path] = null; 01195 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); 01196 } 01197 fclose( $cfOp->_file_handle ); // close open handle 01198 } 01199 } 01200 01201 return $tmpFiles; 01202 } 01203 01208 public function getFileHttpUrl( array $params ) { 01209 if ( $this->swiftTempUrlKey != '' || 01210 ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) 01211 { 01212 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 01213 if ( $srcRel === null ) { 01214 return null; // invalid path 01215 } 01216 try { 01217 $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; 01218 $sContObj = $this->getContainer( $srcCont ); 01219 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD 01220 if ( $this->swiftTempUrlKey != '' ) { 01221 return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" ); 01222 } else { // give S3 API URL for rgw 01223 $expires = time() + $ttl; 01224 // Path for signature starts with the bucket 01225 $spath = '/' . rawurlencode( $srcCont ) . '/' . 01226 str_replace( '%2F', '/', rawurlencode( $srcRel ) ); 01227 // Calculate the hash 01228 $signature = base64_encode( hash_hmac( 01229 'sha1', 01230 "GET\n\n\n{$expires}\n{$spath}", 01231 $this->rgwS3SecretKey, 01232 true // raw 01233 ) ); 01234 // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. 01235 // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. 01236 return wfAppendQuery( 01237 str_replace( '/swift/v1', '', // S3 API is the rgw default 01238 $sContObj->cfs_http->getStorageUrl() . $spath ), 01239 array( 01240 'Signature' => $signature, 01241 'Expires' => $expires, 01242 'AWSAccessKeyId' => $this->rgwS3AccessKey ) 01243 ); 01244 } 01245 } catch ( NoSuchContainerException $e ) { 01246 } catch ( CloudFilesException $e ) { // some other exception? 01247 $this->handleException( $e, null, __METHOD__, $params ); 01248 } 01249 } 01250 return null; 01251 } 01252 01257 protected function directoriesAreVirtual() { 01258 return true; 01259 } 01260 01269 protected function headersFromParams( array $params ) { 01270 $hdrs = array(); 01271 if ( !empty( $params['latest'] ) ) { 01272 $hdrs[] = 'X-Newest: true'; 01273 } 01274 return $hdrs; 01275 } 01276 01281 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { 01282 $statuses = array(); 01283 01284 $cfOps = array(); // list of CF_Async_Op objects 01285 foreach ( $fileOpHandles as $index => $fileOpHandle ) { 01286 $cfOps[$index] = $fileOpHandle->cfOp; 01287 } 01288 $batch = new CF_Async_Op_Batch( $cfOps ); 01289 01290 $cfOps = $batch->execute(); 01291 foreach ( $cfOps as $index => $cfOp ) { 01292 $status = Status::newGood(); 01293 $function = '_getResponse' . $fileOpHandles[$index]->call; 01294 try { // catch exceptions; update status 01295 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); 01296 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects ); 01297 } catch ( CloudFilesException $e ) { // some other exception? 01298 $this->handleException( $e, $status, 01299 __CLASS__ . ":$function", $fileOpHandles[$index]->params ); 01300 } 01301 $statuses[$index] = $status; 01302 } 01303 01304 return $statuses; 01305 } 01306 01333 protected function setContainerAccess( 01334 CF_Container $contObj, array $readGrps, array $writeGrps 01335 ) { 01336 $creds = $contObj->cfs_auth->export_credentials(); 01337 01338 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name ); 01339 01340 // Note: 10 second timeout consistent with php-cloudfiles 01341 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) ); 01342 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] ); 01343 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) ); 01344 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) ); 01345 01346 return $req->execute(); // should return 204 01347 } 01348 01356 public function purgeCDNCache( array $objects ) { 01357 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) { 01358 foreach ( $objects as $object ) { 01359 try { 01360 $object->purge_from_cdn(); 01361 } catch ( CDNNotEnabledException $e ) { 01362 // CDN not enabled; nothing to see here 01363 } catch ( CloudFilesException $e ) { 01364 $this->handleException( $e, null, __METHOD__, 01365 array( 'cont' => $object->container->name, 'obj' => $object->name ) ); 01366 } 01367 } 01368 } 01369 } 01370 01378 protected function getConnection() { 01379 if ( $this->connException instanceof CloudFilesException ) { 01380 if ( ( time() - $this->connErrorTime ) < 60 ) { 01381 throw $this->connException; // failed last attempt; don't bother 01382 } else { // actually retry this time 01383 $this->connException = null; 01384 $this->connErrorTime = 0; 01385 } 01386 } 01387 // Session keys expire after a while, so we renew them periodically 01388 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL ); 01389 // Authenticate with proxy and get a session key... 01390 if ( !$this->conn || $reAuth ) { 01391 $this->sessionStarted = 0; 01392 $this->connContainerCache->clear(); 01393 $cacheKey = $this->getCredsCacheKey( $this->auth->username ); 01394 $creds = $this->srvCache->get( $cacheKey ); // credentials 01395 if ( is_array( $creds ) ) { // cache hit 01396 $this->auth->load_cached_credentials( 01397 $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] ); 01398 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case 01399 } else { // cache miss 01400 try { 01401 $this->auth->authenticate(); 01402 $creds = $this->auth->export_credentials(); 01403 $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache 01404 $this->sessionStarted = time(); 01405 } catch ( CloudFilesException $e ) { 01406 $this->connException = $e; // don't keep re-trying 01407 $this->connErrorTime = time(); 01408 throw $e; // throw it back 01409 } 01410 } 01411 if ( $this->conn ) { // re-authorizing? 01412 $this->conn->close(); // close active cURL handles in CF_Http object 01413 } 01414 $this->conn = new CF_Connection( $this->auth ); 01415 } 01416 return $this->conn; 01417 } 01418 01424 protected function closeConnection() { 01425 if ( $this->conn ) { 01426 $this->conn->close(); // close active cURL handles in CF_Http object 01427 $this->conn = null; 01428 $this->sessionStarted = 0; 01429 $this->connContainerCache->clear(); 01430 } 01431 } 01432 01439 private function getCredsCacheKey( $username ) { 01440 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username ); 01441 } 01442 01452 protected function getContainer( $container, $bypassCache = false ) { 01453 $conn = $this->getConnection(); // Swift proxy connection 01454 if ( $bypassCache ) { // purge cache 01455 $this->connContainerCache->clear( $container ); 01456 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) { 01457 $this->primeContainerCache( array( $container ) ); // check persistent cache 01458 } 01459 if ( !$this->connContainerCache->has( $container, 'obj' ) ) { 01460 $contObj = $conn->get_container( $container ); 01461 // NoSuchContainerException not thrown: container must exist 01462 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it 01463 if ( !$bypassCache ) { 01464 $this->setContainerCache( $container, // update persistent cache 01465 array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count ) 01466 ); 01467 } 01468 } 01469 return $this->connContainerCache->get( $container, 'obj' ); 01470 } 01471 01479 protected function createContainer( $container ) { 01480 $conn = $this->getConnection(); // Swift proxy connection 01481 $contObj = $conn->create_container( $container ); 01482 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache 01483 return $contObj; 01484 } 01485 01493 protected function deleteContainer( $container ) { 01494 $conn = $this->getConnection(); // Swift proxy connection 01495 $this->connContainerCache->clear( $container ); // purge 01496 $conn->delete_container( $container ); 01497 } 01498 01503 protected function doPrimeContainerCache( array $containerInfo ) { 01504 try { 01505 $conn = $this->getConnection(); // Swift proxy connection 01506 foreach ( $containerInfo as $container => $info ) { 01507 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http, 01508 $container, $info['count'], $info['bytes'] ); 01509 $this->connContainerCache->set( $container, 'obj', $contObj ); 01510 } 01511 } catch ( CloudFilesException $e ) { // some other exception? 01512 $this->handleException( $e, null, __METHOD__, array() ); 01513 } 01514 } 01515 01526 protected function handleException( Exception $e, $status, $func, array $params ) { 01527 if ( $status instanceof Status ) { 01528 if ( $e instanceof AuthenticationException ) { 01529 $status->fatal( 'backend-fail-connect', $this->name ); 01530 } else { 01531 $status->fatal( 'backend-fail-internal', $this->name ); 01532 } 01533 } 01534 if ( $e->getMessage() ) { 01535 trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING ); 01536 } 01537 if ( $e instanceof InvalidResponseException ) { // possibly a stale token 01538 $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) ); 01539 $this->closeConnection(); // force a re-connect and re-auth next time 01540 } 01541 wfDebugLog( 'SwiftBackend', 01542 get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . 01543 ( $e->getMessage() ? ": {$e->getMessage()}" : "" ) 01544 ); 01545 } 01546 } 01547 01551 class SwiftFileOpHandle extends FileBackendStoreOpHandle { 01553 public $cfOp; 01555 public $affectedObjects = array(); 01556 01557 public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { 01558 $this->backend = $backend; 01559 $this->params = $params; 01560 $this->call = $call; 01561 $this->cfOp = $cfOp; 01562 } 01563 } 01564 01572 abstract class SwiftFileBackendList implements Iterator { 01574 protected $bufferIter = array(); 01575 protected $bufferAfter = null; // string; list items *after* this path 01576 protected $pos = 0; // integer 01578 protected $params = array(); 01579 01581 protected $backend; 01582 protected $container; // string; container name 01583 protected $dir; // string; storage directory 01584 protected $suffixStart; // integer 01585 01586 const PAGE_SIZE = 9000; // file listing buffer size 01587 01594 public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { 01595 $this->backend = $backend; 01596 $this->container = $fullCont; 01597 $this->dir = $dir; 01598 if ( substr( $this->dir, -1 ) === '/' ) { 01599 $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash 01600 } 01601 if ( $this->dir == '' ) { // whole container 01602 $this->suffixStart = 0; 01603 } else { // dir within container 01604 $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/" 01605 } 01606 $this->params = $params; 01607 } 01608 01613 public function key() { 01614 return $this->pos; 01615 } 01616 01621 public function next() { 01622 // Advance to the next file in the page 01623 next( $this->bufferIter ); 01624 ++$this->pos; 01625 // Check if there are no files left in this page and 01626 // advance to the next page if this page was not empty. 01627 if ( !$this->valid() && count( $this->bufferIter ) ) { 01628 $this->bufferIter = $this->pageFromList( 01629 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params 01630 ); // updates $this->bufferAfter 01631 } 01632 } 01633 01638 public function rewind() { 01639 $this->pos = 0; 01640 $this->bufferAfter = null; 01641 $this->bufferIter = $this->pageFromList( 01642 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params 01643 ); // updates $this->bufferAfter 01644 } 01645 01650 public function valid() { 01651 if ( $this->bufferIter === null ) { 01652 return false; // some failure? 01653 } else { 01654 return ( current( $this->bufferIter ) !== false ); // no paths can have this value 01655 } 01656 } 01657 01668 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); 01669 } 01670 01674 class SwiftFileBackendDirList extends SwiftFileBackendList { 01679 public function current() { 01680 return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); 01681 } 01682 01687 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { 01688 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); 01689 } 01690 } 01691 01695 class SwiftFileBackendFileList extends SwiftFileBackendList { 01700 public function current() { 01701 return substr( current( $this->bufferIter ), $this->suffixStart ); 01702 } 01703 01708 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { 01709 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); 01710 } 01711 }
1.7.6.1