MediaWiki  REL1_22
SwiftFileBackend.php
Go to the documentation of this file.
00001 <?php
00039 class SwiftFileBackend extends FileBackendStore {
00041     protected $auth; // Swift authentication handler
00042     protected $authTTL; // integer seconds
00043     protected $swiftTempUrlKey; // string; shared secret value for making temp urls
00044     protected $swiftAnonUser; // string; username to handle unauthenticated requests
00045     protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
00046     protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
00047     protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
00048 
00049     // Rados Gateway specific options
00050     protected $rgwS3AccessKey; // string; S3 access key
00051     protected $rgwS3SecretKey; // string; S3 authentication key
00052 
00054     protected $conn; // Swift connection handle
00055     protected $sessionStarted = 0; // integer UNIX timestamp
00056 
00058     protected $connException;
00059     protected $connErrorTime = 0; // UNIX timestamp
00060 
00062     protected $srvCache;
00063 
00065     protected $connContainerCache; // container object cache
00066 
00105     public function __construct( array $config ) {
00106         parent::__construct( $config );
00107         if ( !class_exists( 'CF_Constants' ) ) {
00108             throw new MWException( 'SwiftCloudFiles extension not installed.' );
00109         }
00110         // Required settings
00111         $this->auth = new CF_Authentication(
00112             $config['swiftUser'],
00113             $config['swiftKey'],
00114             null, // account; unused
00115             $config['swiftAuthUrl']
00116         );
00117         // Optional settings
00118         $this->authTTL = isset( $config['swiftAuthTTL'] )
00119             ? $config['swiftAuthTTL']
00120             : 5 * 60; // some sane number
00121         $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
00122             ? $config['swiftAnonUser']
00123             : '';
00124         $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
00125             ? $config['swiftTempUrlKey']
00126             : '';
00127         $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
00128             ? $config['shardViaHashLevels']
00129             : '';
00130         $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
00131             ? $config['swiftUseCDN']
00132             : false;
00133         $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
00134             ? $config['swiftCDNExpiry']
00135             : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
00136         $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
00137             ? $config['swiftCDNPurgable']
00138             : true;
00139         $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
00140             ? $config['rgwS3AccessKey']
00141             : '';
00142         $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
00143             ? $config['rgwS3SecretKey']
00144             : '';
00145         // Cache container information to mask latency
00146         $this->memCache = wfGetMainCache();
00147         // Process cache for container info
00148         $this->connContainerCache = new ProcessCacheLRU( 300 );
00149         // Cache auth token information to avoid RTTs
00150         if ( !empty( $config['cacheAuthInfo'] ) ) {
00151             if ( PHP_SAPI === 'cli' ) {
00152                 $this->srvCache = wfGetMainCache(); // preferrably memcached
00153             } else {
00154                 try { // look for APC, XCache, WinCache, ect...
00155                     $this->srvCache = ObjectCache::newAccelerator( array() );
00156                 } catch ( Exception $e ) {}
00157             }
00158         }
00159         $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
00160     }
00161 
00166     protected function resolveContainerPath( $container, $relStoragePath ) {
00167         if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
00168             return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
00169         } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
00170             return null; // too long for Swift
00171         }
00172         return $relStoragePath;
00173     }
00174 
00175     public function isPathUsableInternal( $storagePath ) {
00176         list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
00177         if ( $rel === null ) {
00178             return false; // invalid
00179         }
00180 
00181         try {
00182             $this->getContainer( $container );
00183             return true; // container exists
00184         } catch ( NoSuchContainerException $e ) {
00185         } catch ( CloudFilesException $e ) { // some other exception?
00186             $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
00187         }
00188 
00189         return false;
00190     }
00191 
00196     protected function sanitizeHdrs( array $headers ) {
00197         // By default, Swift has annoyingly low maximum header value limits
00198         if ( isset( $headers['Content-Disposition'] ) ) {
00199             $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] );
00200         }
00201         return $headers;
00202     }
00203 
00208     protected function truncDisp( $disposition ) {
00209         $res = '';
00210         foreach ( explode( ';', $disposition ) as $part ) {
00211             $part = trim( $part );
00212             $new = ( $res === '' ) ? $part : "{$res};{$part}";
00213             if ( strlen( $new ) <= 255 ) {
00214                 $res = $new;
00215             } else {
00216                 break; // too long; sigh
00217             }
00218         }
00219         return $res;
00220     }
00221 
00222     protected function doCreateInternal( array $params ) {
00223         $status = Status::newGood();
00224 
00225         list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
00226         if ( $dstRel === null ) {
00227             $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
00228             return $status;
00229         }
00230 
00231         // (a) Check the destination container and object
00232         try {
00233             $dContObj = $this->getContainer( $dstCont );
00234         } catch ( NoSuchContainerException $e ) {
00235             $status->fatal( 'backend-fail-create', $params['dst'] );
00236             return $status;
00237         } catch ( CloudFilesException $e ) { // some other exception?
00238             $this->handleException( $e, $status, __METHOD__, $params );
00239             return $status;
00240         }
00241 
00242         // (b) Get a SHA-1 hash of the object
00243         $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
00244 
00245         // (c) Actually create the object
00246         try {
00247             // Create a fresh CF_Object with no fields preloaded.
00248             // We don't want to preserve headers, metadata, and such.
00249             $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
00250             $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
00251             // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
00252             // The MD5 here will be checked within Swift against its own MD5.
00253             $obj->set_etag( md5( $params['content'] ) );
00254             // Use the same content type as StreamFile for security
00255             $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null );
00256             // Set any other custom headers if requested
00257             if ( isset( $params['headers'] ) ) {
00258                 $obj->headers += $this->sanitizeHdrs( $params['headers'] );
00259             }
00260             if ( !empty( $params['async'] ) ) { // deferred
00261                 $op = $obj->write_async( $params['content'] );
00262                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
00263                 $status->value->affectedObjects[] = $obj;
00264             } else { // actually write the object in Swift
00265                 $obj->write( $params['content'] );
00266                 $this->purgeCDNCache( array( $obj ) );
00267             }
00268         } catch ( CDNNotEnabledException $e ) {
00269             // CDN not enabled; nothing to see here
00270         } catch ( BadContentTypeException $e ) {
00271             $status->fatal( 'backend-fail-contenttype', $params['dst'] );
00272         } catch ( CloudFilesException $e ) { // some other exception?
00273             $this->handleException( $e, $status, __METHOD__, $params );
00274         }
00275 
00276         return $status;
00277     }
00278 
00282     protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
00283         try {
00284             $cfOp->getLastResponse();
00285         } catch ( BadContentTypeException $e ) {
00286             $status->fatal( 'backend-fail-contenttype', $params['dst'] );
00287         }
00288     }
00289 
00290     protected function doStoreInternal( array $params ) {
00291         $status = Status::newGood();
00292 
00293         list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
00294         if ( $dstRel === null ) {
00295             $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
00296             return $status;
00297         }
00298 
00299         // (a) Check the destination container and object
00300         try {
00301             $dContObj = $this->getContainer( $dstCont );
00302         } catch ( NoSuchContainerException $e ) {
00303             $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00304             return $status;
00305         } catch ( CloudFilesException $e ) { // some other exception?
00306             $this->handleException( $e, $status, __METHOD__, $params );
00307             return $status;
00308         }
00309 
00310         // (b) Get a SHA-1 hash of the object
00311         wfSuppressWarnings();
00312         $sha1Hash = sha1_file( $params['src'] );
00313         wfRestoreWarnings();
00314         if ( $sha1Hash === false ) { // source doesn't exist?
00315             $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00316             return $status;
00317         }
00318         $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
00319 
00320         // (c) Actually store the object
00321         try {
00322             // Create a fresh CF_Object with no fields preloaded.
00323             // We don't want to preserve headers, metadata, and such.
00324             $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
00325             $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
00326             // The MD5 here will be checked within Swift against its own MD5.
00327             $obj->set_etag( md5_file( $params['src'] ) );
00328             // Use the same content type as StreamFile for security
00329             $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] );
00330             // Set any other custom headers if requested
00331             if ( isset( $params['headers'] ) ) {
00332                 $obj->headers += $this->sanitizeHdrs( $params['headers'] );
00333             }
00334             if ( !empty( $params['async'] ) ) { // deferred
00335                 wfSuppressWarnings();
00336                 $fp = fopen( $params['src'], 'rb' );
00337                 wfRestoreWarnings();
00338                 if ( !$fp ) {
00339                     $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00340                 } else {
00341                     $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
00342                     $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
00343                     $status->value->resourcesToClose[] = $fp;
00344                     $status->value->affectedObjects[] = $obj;
00345                 }
00346             } else { // actually write the object in Swift
00347                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
00348                 $this->purgeCDNCache( array( $obj ) );
00349             }
00350         } catch ( CDNNotEnabledException $e ) {
00351             // CDN not enabled; nothing to see here
00352         } catch ( BadContentTypeException $e ) {
00353             $status->fatal( 'backend-fail-contenttype', $params['dst'] );
00354         } catch ( IOException $e ) {
00355             $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00356         } catch ( CloudFilesException $e ) { // some other exception?
00357             $this->handleException( $e, $status, __METHOD__, $params );
00358         }
00359 
00360         return $status;
00361     }
00362 
00366     protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
00367         try {
00368             $cfOp->getLastResponse();
00369         } catch ( BadContentTypeException $e ) {
00370             $status->fatal( 'backend-fail-contenttype', $params['dst'] );
00371         } catch ( IOException $e ) {
00372             $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00373         }
00374     }
00375 
00376     protected function doCopyInternal( array $params ) {
00377         $status = Status::newGood();
00378 
00379         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
00380         if ( $srcRel === null ) {
00381             $status->fatal( 'backend-fail-invalidpath', $params['src'] );
00382             return $status;
00383         }
00384 
00385         list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
00386         if ( $dstRel === null ) {
00387             $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
00388             return $status;
00389         }
00390 
00391         // (a) Check the source/destination containers and destination object
00392         try {
00393             $sContObj = $this->getContainer( $srcCont );
00394             $dContObj = $this->getContainer( $dstCont );
00395         } catch ( NoSuchContainerException $e ) {
00396             if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
00397                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00398             }
00399             return $status;
00400         } catch ( CloudFilesException $e ) { // some other exception?
00401             $this->handleException( $e, $status, __METHOD__, $params );
00402             return $status;
00403         }
00404 
00405         // (b) Actually copy the file to the destination
00406         try {
00407             $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
00408             $hdrs = array(); // source file headers to override with new values
00409             // Set any other custom headers if requested
00410             if ( isset( $params['headers'] ) ) {
00411                 $hdrs += $this->sanitizeHdrs( $params['headers'] );
00412             }
00413             if ( !empty( $params['async'] ) ) { // deferred
00414                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
00415                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
00416                 $status->value->affectedObjects[] = $dstObj;
00417             } else { // actually write the object in Swift
00418                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
00419                 $this->purgeCDNCache( array( $dstObj ) );
00420             }
00421         } catch ( CDNNotEnabledException $e ) {
00422             // CDN not enabled; nothing to see here
00423         } catch ( NoSuchObjectException $e ) { // source object does not exist
00424             if ( empty( $params['ignoreMissingSource'] ) ) {
00425                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00426             }
00427         } catch ( CloudFilesException $e ) { // some other exception?
00428             $this->handleException( $e, $status, __METHOD__, $params );
00429         }
00430 
00431         return $status;
00432     }
00433 
00437     protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
00438         try {
00439             $cfOp->getLastResponse();
00440         } catch ( NoSuchObjectException $e ) { // source object does not exist
00441             $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
00442         }
00443     }
00444 
00445     protected function doMoveInternal( array $params ) {
00446         $status = Status::newGood();
00447 
00448         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
00449         if ( $srcRel === null ) {
00450             $status->fatal( 'backend-fail-invalidpath', $params['src'] );
00451             return $status;
00452         }
00453 
00454         list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
00455         if ( $dstRel === null ) {
00456             $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
00457             return $status;
00458         }
00459 
00460         // (a) Check the source/destination containers and destination object
00461         try {
00462             $sContObj = $this->getContainer( $srcCont );
00463             $dContObj = $this->getContainer( $dstCont );
00464         } catch ( NoSuchContainerException $e ) {
00465             if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
00466                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
00467             }
00468             return $status;
00469         } catch ( CloudFilesException $e ) { // some other exception?
00470             $this->handleException( $e, $status, __METHOD__, $params );
00471             return $status;
00472         }
00473 
00474         // (b) Actually move the file to the destination
00475         try {
00476             $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
00477             $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
00478             $hdrs = array(); // source file headers to override with new values
00479             // Set any other custom headers if requested
00480             if ( isset( $params['headers'] ) ) {
00481                 $hdrs += $this->sanitizeHdrs( $params['headers'] );
00482             }
00483             if ( !empty( $params['async'] ) ) { // deferred
00484                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
00485                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
00486                 $status->value->affectedObjects[] = $srcObj;
00487                 $status->value->affectedObjects[] = $dstObj;
00488             } else { // actually write the object in Swift
00489                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
00490                 $this->purgeCDNCache( array( $srcObj ) );
00491                 $this->purgeCDNCache( array( $dstObj ) );
00492             }
00493         } catch ( CDNNotEnabledException $e ) {
00494             // CDN not enabled; nothing to see here
00495         } catch ( NoSuchObjectException $e ) { // source object does not exist
00496             if ( empty( $params['ignoreMissingSource'] ) ) {
00497                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
00498             }
00499         } catch ( CloudFilesException $e ) { // some other exception?
00500             $this->handleException( $e, $status, __METHOD__, $params );
00501         }
00502 
00503         return $status;
00504     }
00505 
00509     protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
00510         try {
00511             $cfOp->getLastResponse();
00512         } catch ( NoSuchObjectException $e ) { // source object does not exist
00513             $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
00514         }
00515     }
00516 
00517     protected function doDeleteInternal( array $params ) {
00518         $status = Status::newGood();
00519 
00520         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
00521         if ( $srcRel === null ) {
00522             $status->fatal( 'backend-fail-invalidpath', $params['src'] );
00523             return $status;
00524         }
00525 
00526         try {
00527             $sContObj = $this->getContainer( $srcCont );
00528             $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
00529             if ( !empty( $params['async'] ) ) { // deferred
00530                 $op = $sContObj->delete_object_async( $srcRel );
00531                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
00532                 $status->value->affectedObjects[] = $srcObj;
00533             } else { // actually write the object in Swift
00534                 $sContObj->delete_object( $srcRel );
00535                 $this->purgeCDNCache( array( $srcObj ) );
00536             }
00537         } catch ( CDNNotEnabledException $e ) {
00538             // CDN not enabled; nothing to see here
00539         } catch ( NoSuchContainerException $e ) {
00540             if ( empty( $params['ignoreMissingSource'] ) ) {
00541                 $status->fatal( 'backend-fail-delete', $params['src'] );
00542             }
00543         } catch ( NoSuchObjectException $e ) {
00544             if ( empty( $params['ignoreMissingSource'] ) ) {
00545                 $status->fatal( 'backend-fail-delete', $params['src'] );
00546             }
00547         } catch ( CloudFilesException $e ) { // some other exception?
00548             $this->handleException( $e, $status, __METHOD__, $params );
00549         }
00550 
00551         return $status;
00552     }
00553 
00557     protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
00558         try {
00559             $cfOp->getLastResponse();
00560         } catch ( NoSuchContainerException $e ) {
00561             $status->fatal( 'backend-fail-delete', $params['src'] );
00562         } catch ( NoSuchObjectException $e ) {
00563             if ( empty( $params['ignoreMissingSource'] ) ) {
00564                 $status->fatal( 'backend-fail-delete', $params['src'] );
00565             }
00566         }
00567     }
00568 
00569     protected function doDescribeInternal( array $params ) {
00570         $status = Status::newGood();
00571 
00572         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
00573         if ( $srcRel === null ) {
00574             $status->fatal( 'backend-fail-invalidpath', $params['src'] );
00575             return $status;
00576         }
00577 
00578         try {
00579             $sContObj = $this->getContainer( $srcCont );
00580             // Get the latest version of the current metadata
00581             $srcObj = $sContObj->get_object( $srcRel,
00582                 $this->headersFromParams( array( 'latest' => true ) ) );
00583             // Merge in the metadata updates...
00584             if ( isset( $params['headers'] ) ) {
00585                 $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers;
00586             }
00587             $srcObj->sync_metadata(); // save to Swift
00588             $this->purgeCDNCache( array( $srcObj ) );
00589         } catch ( CDNNotEnabledException $e ) {
00590             // CDN not enabled; nothing to see here
00591         } catch ( NoSuchContainerException $e ) {
00592             $status->fatal( 'backend-fail-describe', $params['src'] );
00593         } catch ( NoSuchObjectException $e ) {
00594             $status->fatal( 'backend-fail-describe', $params['src'] );
00595         } catch ( CloudFilesException $e ) { // some other exception?
00596             $this->handleException( $e, $status, __METHOD__, $params );
00597         }
00598 
00599         return $status;
00600     }
00601 
00602     protected function doPrepareInternal( $fullCont, $dir, array $params ) {
00603         $status = Status::newGood();
00604 
00605         // (a) Check if container already exists
00606         try {
00607             $this->getContainer( $fullCont );
00608             // NoSuchContainerException not thrown: container must exist
00609             return $status; // already exists
00610         } catch ( NoSuchContainerException $e ) {
00611             // NoSuchContainerException thrown: container does not exist
00612         } catch ( CloudFilesException $e ) { // some other exception?
00613             $this->handleException( $e, $status, __METHOD__, $params );
00614             return $status;
00615         }
00616 
00617         // (b) Create container as needed
00618         try {
00619             $contObj = $this->createContainer( $fullCont );
00620             if ( !empty( $params['noAccess'] ) ) {
00621                 // Make container private to end-users...
00622                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
00623             } else {
00624                 // Make container public to end-users...
00625                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
00626             }
00627             if ( $this->swiftUseCDN ) { // Rackspace style CDN
00628                 $contObj->make_public( $this->swiftCDNExpiry );
00629             }
00630         } catch ( CDNNotEnabledException $e ) {
00631             // CDN not enabled; nothing to see here
00632         } catch ( CloudFilesException $e ) { // some other exception?
00633             $this->handleException( $e, $status, __METHOD__, $params );
00634             return $status;
00635         }
00636 
00637         return $status;
00638     }
00639 
00644     protected function doSecureInternal( $fullCont, $dir, array $params ) {
00645         $status = Status::newGood();
00646         if ( empty( $params['noAccess'] ) ) {
00647             return $status; // nothing to do
00648         }
00649 
00650         // Restrict container from end-users...
00651         try {
00652             // doPrepareInternal() should have been called,
00653             // so the Swift container should already exist...
00654             $contObj = $this->getContainer( $fullCont ); // normally a cache hit
00655             // NoSuchContainerException not thrown: container must exist
00656 
00657             // Make container private to end-users...
00658             $status->merge( $this->setContainerAccess(
00659                 $contObj,
00660                 array( $this->auth->username ), // read
00661                 array( $this->auth->username ) // write
00662             ) );
00663             if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
00664                 $contObj->make_private();
00665             }
00666         } catch ( CDNNotEnabledException $e ) {
00667             // CDN not enabled; nothing to see here
00668         } catch ( CloudFilesException $e ) { // some other exception?
00669             $this->handleException( $e, $status, __METHOD__, $params );
00670         }
00671 
00672         return $status;
00673     }
00674 
00679     protected function doPublishInternal( $fullCont, $dir, array $params ) {
00680         $status = Status::newGood();
00681 
00682         // Unrestrict container from end-users...
00683         try {
00684             // doPrepareInternal() should have been called,
00685             // so the Swift container should already exist...
00686             $contObj = $this->getContainer( $fullCont ); // normally a cache hit
00687             // NoSuchContainerException not thrown: container must exist
00688 
00689             // Make container public to end-users...
00690             if ( $this->swiftAnonUser != '' ) {
00691                 $status->merge( $this->setContainerAccess(
00692                     $contObj,
00693                     array( $this->auth->username, $this->swiftAnonUser ), // read
00694                     array( $this->auth->username, $this->swiftAnonUser ) // write
00695                 ) );
00696             } else {
00697                 $status->merge( $this->setContainerAccess(
00698                     $contObj,
00699                     array( $this->auth->username, '.r:*' ), // read
00700                     array( $this->auth->username ) // write
00701                 ) );
00702             }
00703             if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
00704                 $contObj->make_public();
00705             }
00706         } catch ( CDNNotEnabledException $e ) {
00707             // CDN not enabled; nothing to see here
00708         } catch ( CloudFilesException $e ) { // some other exception?
00709             $this->handleException( $e, $status, __METHOD__, $params );
00710         }
00711 
00712         return $status;
00713     }
00714 
00715     protected function doCleanInternal( $fullCont, $dir, array $params ) {
00716         $status = Status::newGood();
00717 
00718         // Only containers themselves can be removed, all else is virtual
00719         if ( $dir != '' ) {
00720             return $status; // nothing to do
00721         }
00722 
00723         // (a) Check the container
00724         try {
00725             $contObj = $this->getContainer( $fullCont, true );
00726         } catch ( NoSuchContainerException $e ) {
00727             return $status; // ok, nothing to do
00728         } catch ( CloudFilesException $e ) { // some other exception?
00729             $this->handleException( $e, $status, __METHOD__, $params );
00730             return $status;
00731         }
00732 
00733         // (b) Delete the container if empty
00734         if ( $contObj->object_count == 0 ) {
00735             try {
00736                 $this->deleteContainer( $fullCont );
00737             } catch ( NoSuchContainerException $e ) {
00738                 return $status; // race?
00739             } catch ( NonEmptyContainerException $e ) {
00740                 return $status; // race? consistency delay?
00741             } catch ( CloudFilesException $e ) { // some other exception?
00742                 $this->handleException( $e, $status, __METHOD__, $params );
00743                 return $status;
00744             }
00745         }
00746 
00747         return $status;
00748     }
00749 
00750     protected function doGetFileStat( array $params ) {
00751         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
00752         if ( $srcRel === null ) {
00753             return false; // invalid storage path
00754         }
00755 
00756         $stat = false;
00757         try {
00758             $contObj = $this->getContainer( $srcCont );
00759             $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
00760             $this->addMissingMetadata( $srcObj, $params['src'] );
00761             $stat = array(
00762                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
00763                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
00764                 'size' => (int)$srcObj->content_length,
00765                 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
00766             );
00767         } catch ( NoSuchContainerException $e ) {
00768         } catch ( NoSuchObjectException $e ) {
00769         } catch ( CloudFilesException $e ) { // some other exception?
00770             $stat = null;
00771             $this->handleException( $e, null, __METHOD__, $params );
00772         }
00773 
00774         return $stat;
00775     }
00776 
00785     protected function addMissingMetadata( CF_Object $obj, $path ) {
00786         if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
00787             return true; // nothing to do
00788         }
00789         wfProfileIn( __METHOD__ );
00790         trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
00791         $status = Status::newGood();
00792         $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
00793         if ( $status->isOK() ) {
00794             $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
00795             if ( $tmpFile ) {
00796                 $hash = $tmpFile->getSha1Base36();
00797                 if ( $hash !== false ) {
00798                     $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
00799                     $obj->sync_metadata(); // save to Swift
00800                     wfProfileOut( __METHOD__ );
00801                     return true; // success
00802                 }
00803             }
00804         }
00805         trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
00806         $obj->setMetadataValues( array( 'Sha1base36' => false ) );
00807         wfProfileOut( __METHOD__ );
00808         return false; // failed
00809     }
00810 
00811     protected function doGetFileContentsMulti( array $params ) {
00812         $contents = array();
00813 
00814         $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
00815         // Blindly create tmp files and stream to them, catching any exception if the file does
00816         // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
00817         foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
00818             $cfOps = array(); // (path => CF_Async_Op)
00819 
00820             foreach ( $pathBatch as $path ) { // each path in this concurrent batch
00821                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
00822                 if ( $srcRel === null ) {
00823                     $contents[$path] = false;
00824                     continue;
00825                 }
00826                 $data = false;
00827                 try {
00828                     $sContObj = $this->getContainer( $srcCont );
00829                     $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
00830                     // Create a new temporary memory file...
00831                     $handle = fopen( 'php://temp', 'wb' );
00832                     if ( $handle ) {
00833                         $headers = $this->headersFromParams( $params );
00834                         if ( count( $pathBatch ) > 1 ) {
00835                             $cfOps[$path] = $obj->stream_async( $handle, $headers );
00836                             $cfOps[$path]->_file_handle = $handle; // close this later
00837                         } else {
00838                             $obj->stream( $handle, $headers );
00839                             rewind( $handle ); // start from the beginning
00840                             $data = stream_get_contents( $handle );
00841                             fclose( $handle );
00842                         }
00843                     } else {
00844                         $data = false;
00845                     }
00846                 } catch ( NoSuchContainerException $e ) {
00847                     $data = false;
00848                 } catch ( NoSuchObjectException $e ) {
00849                     $data = false;
00850                 } catch ( CloudFilesException $e ) { // some other exception?
00851                     $data = false;
00852                     $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
00853                 }
00854                 $contents[$path] = $data;
00855             }
00856 
00857             $batch = new CF_Async_Op_Batch( $cfOps );
00858             $cfOps = $batch->execute();
00859             foreach ( $cfOps as $path => $cfOp ) {
00860                 try {
00861                     $cfOp->getLastResponse();
00862                     rewind( $cfOp->_file_handle ); // start from the beginning
00863                     $contents[$path] = stream_get_contents( $cfOp->_file_handle );
00864                 } catch ( NoSuchContainerException $e ) {
00865                     $contents[$path] = false;
00866                 } catch ( NoSuchObjectException $e ) {
00867                     $contents[$path] = false;
00868                 } catch ( CloudFilesException $e ) { // some other exception?
00869                     $contents[$path] = false;
00870                     $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
00871                 }
00872                 fclose( $cfOp->_file_handle ); // close open handle
00873             }
00874         }
00875 
00876         return $contents;
00877     }
00878 
00883     protected function doDirectoryExists( $fullCont, $dir, array $params ) {
00884         try {
00885             $container = $this->getContainer( $fullCont );
00886             $prefix = ( $dir == '' ) ? null : "{$dir}/";
00887             return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
00888         } catch ( NoSuchContainerException $e ) {
00889             return false;
00890         } catch ( CloudFilesException $e ) { // some other exception?
00891             $this->handleException( $e, null, __METHOD__,
00892                 array( 'cont' => $fullCont, 'dir' => $dir ) );
00893         }
00894 
00895         return null; // error
00896     }
00897 
00902     public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
00903         return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
00904     }
00905 
00910     public function getFileListInternal( $fullCont, $dir, array $params ) {
00911         return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
00912     }
00913 
00925     public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
00926         $dirs = array();
00927         if ( $after === INF ) {
00928             return $dirs; // nothing more
00929         }
00930 
00931         $section = new ProfileSection( __METHOD__ . '-' . $this->name );
00932         try {
00933             $container = $this->getContainer( $fullCont );
00934             $prefix = ( $dir == '' ) ? null : "{$dir}/";
00935             // Non-recursive: only list dirs right under $dir
00936             if ( !empty( $params['topOnly'] ) ) {
00937                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
00938                 foreach ( $objects as $object ) { // files and directories
00939                     if ( substr( $object, -1 ) === '/' ) {
00940                         $dirs[] = $object; // directories end in '/'
00941                     }
00942                 }
00943             // Recursive: list all dirs under $dir and its subdirs
00944             } else {
00945                 // Get directory from last item of prior page
00946                 $lastDir = $this->getParentDir( $after ); // must be first page
00947                 $objects = $container->list_objects( $limit, $after, $prefix );
00948                 foreach ( $objects as $object ) { // files
00949                     $objectDir = $this->getParentDir( $object ); // directory of object
00950                     if ( $objectDir !== false && $objectDir !== $dir ) {
00951                         // Swift stores paths in UTF-8, using binary sorting.
00952                         // See function "create_container_table" in common/db.py.
00953                         // If a directory is not "greater" than the last one,
00954                         // then it was already listed by the calling iterator.
00955                         if ( strcmp( $objectDir, $lastDir ) > 0 ) {
00956                             $pDir = $objectDir;
00957                             do { // add dir and all its parent dirs
00958                                 $dirs[] = "{$pDir}/";
00959                                 $pDir = $this->getParentDir( $pDir );
00960                             } while ( $pDir !== false // sanity
00961                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
00962                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
00963                             );
00964                         }
00965                         $lastDir = $objectDir;
00966                     }
00967                 }
00968             }
00969             // Page on the unfiltered directory listing (what is returned may be filtered)
00970             if ( count( $objects ) < $limit ) {
00971                 $after = INF; // avoid a second RTT
00972             } else {
00973                 $after = end( $objects ); // update last item
00974             }
00975         } catch ( NoSuchContainerException $e ) {
00976         } catch ( CloudFilesException $e ) { // some other exception?
00977             $this->handleException( $e, null, __METHOD__,
00978                 array( 'cont' => $fullCont, 'dir' => $dir ) );
00979             throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
00980         }
00981 
00982         return $dirs;
00983     }
00984 
00985     protected function getParentDir( $path ) {
00986         return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
00987     }
00988 
01000     public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
01001         $files = array();
01002         if ( $after === INF ) {
01003             return $files; // nothing more
01004         }
01005 
01006         $section = new ProfileSection( __METHOD__ . '-' . $this->name );
01007         try {
01008             $container = $this->getContainer( $fullCont );
01009             $prefix = ( $dir == '' ) ? null : "{$dir}/";
01010             // Non-recursive: only list files right under $dir
01011             if ( !empty( $params['topOnly'] ) ) { // files and dirs
01012                 if ( !empty( $params['adviseStat'] ) ) {
01013                     $limit = min( $limit, self::CACHE_CHEAP_SIZE );
01014                     // Note: get_objects() does not include directories
01015                     $objects = $this->loadObjectListing( $params, $dir,
01016                         $container->get_objects( $limit, $after, $prefix, null, '/' ) );
01017                     $files = $objects;
01018                 } else {
01019                     $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
01020                     foreach ( $objects as $object ) { // files and directories
01021                         if ( substr( $object, -1 ) !== '/' ) {
01022                             $files[] = $object; // directories end in '/'
01023                         }
01024                     }
01025                 }
01026             // Recursive: list all files under $dir and its subdirs
01027             } else { // files
01028                 if ( !empty( $params['adviseStat'] ) ) {
01029                     $limit = min( $limit, self::CACHE_CHEAP_SIZE );
01030                     $objects = $this->loadObjectListing( $params, $dir,
01031                         $container->get_objects( $limit, $after, $prefix ) );
01032                 } else {
01033                     $objects = $container->list_objects( $limit, $after, $prefix );
01034                 }
01035                 $files = $objects;
01036             }
01037             // Page on the unfiltered object listing (what is returned may be filtered)
01038             if ( count( $objects ) < $limit ) {
01039                 $after = INF; // avoid a second RTT
01040             } else {
01041                 $after = end( $objects ); // update last item
01042             }
01043         } catch ( NoSuchContainerException $e ) {
01044         } catch ( CloudFilesException $e ) { // some other exception?
01045             $this->handleException( $e, null, __METHOD__,
01046                 array( 'cont' => $fullCont, 'dir' => $dir ) );
01047             throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
01048         }
01049 
01050         return $files;
01051     }
01052 
01062     private function loadObjectListing( array $params, $dir, array $cfObjects ) {
01063         $names = array();
01064         $storageDir = rtrim( $params['dir'], '/' );
01065         $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/"
01066         // Iterate over the list *backwards* as this primes the stat cache, which is LRU.
01067         // If this fills the cache and the caller stats an uncached file before stating
01068         // the ones on the listing, there would be zero cache hits if this went forwards.
01069         for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) {
01070             $object = current( $cfObjects );
01071             $path = "{$storageDir}/" . substr( $object->name, $suffixStart );
01072             $val = array(
01073                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
01074                 'mtime'  => wfTimestamp( TS_MW, $object->last_modified ),
01075                 'size'   => (int)$object->content_length,
01076                 'latest' => false // eventually consistent
01077             );
01078             $this->cheapCache->set( $path, 'stat', $val );
01079             $names[] = $object->name;
01080         }
01081         return array_reverse( $names ); // keep the paths in original order
01082     }
01083 
01084     protected function doGetFileSha1base36( array $params ) {
01085         $stat = $this->getFileStat( $params );
01086         if ( $stat ) {
01087             if ( !isset( $stat['sha1'] ) ) {
01088                 // Stat entries filled by file listings don't include SHA1
01089                 $this->clearCache( array( $params['src'] ) );
01090                 $stat = $this->getFileStat( $params );
01091             }
01092             return $stat['sha1'];
01093         } else {
01094             return false;
01095         }
01096     }
01097 
01098     protected function doStreamFile( array $params ) {
01099         $status = Status::newGood();
01100 
01101         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
01102         if ( $srcRel === null ) {
01103             $status->fatal( 'backend-fail-invalidpath', $params['src'] );
01104         }
01105 
01106         try {
01107             $cont = $this->getContainer( $srcCont );
01108         } catch ( NoSuchContainerException $e ) {
01109             $status->fatal( 'backend-fail-stream', $params['src'] );
01110             return $status;
01111         } catch ( CloudFilesException $e ) { // some other exception?
01112             $this->handleException( $e, $status, __METHOD__, $params );
01113             return $status;
01114         }
01115 
01116         try {
01117             $output = fopen( 'php://output', 'wb' );
01118             $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
01119             $obj->stream( $output, $this->headersFromParams( $params ) );
01120         } catch ( NoSuchObjectException $e ) {
01121             $status->fatal( 'backend-fail-stream', $params['src'] );
01122         } catch ( CloudFilesException $e ) { // some other exception?
01123             $this->handleException( $e, $status, __METHOD__, $params );
01124         }
01125 
01126         return $status;
01127     }
01128 
01129     protected function doGetLocalCopyMulti( array $params ) {
01130         $tmpFiles = array();
01131 
01132         $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
01133         // Blindly create tmp files and stream to them, catching any exception if the file does
01134         // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
01135         foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
01136             $cfOps = array(); // (path => CF_Async_Op)
01137 
01138             foreach ( $pathBatch as $path ) { // each path in this concurrent batch
01139                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
01140                 if ( $srcRel === null ) {
01141                     $tmpFiles[$path] = null;
01142                     continue;
01143                 }
01144                 $tmpFile = null;
01145                 try {
01146                     $sContObj = $this->getContainer( $srcCont );
01147                     $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
01148                     // Get source file extension
01149                     $ext = FileBackend::extensionFromPath( $path );
01150                     // Create a new temporary file...
01151                     $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
01152                     if ( $tmpFile ) {
01153                         $handle = fopen( $tmpFile->getPath(), 'wb' );
01154                         if ( $handle ) {
01155                             $headers = $this->headersFromParams( $params );
01156                             if ( count( $pathBatch ) > 1 ) {
01157                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
01158                                 $cfOps[$path]->_file_handle = $handle; // close this later
01159                             } else {
01160                                 $obj->stream( $handle, $headers );
01161                                 fclose( $handle );
01162                             }
01163                         } else {
01164                             $tmpFile = null;
01165                         }
01166                     }
01167                 } catch ( NoSuchContainerException $e ) {
01168                     $tmpFile = null;
01169                 } catch ( NoSuchObjectException $e ) {
01170                     $tmpFile = null;
01171                 } catch ( CloudFilesException $e ) { // some other exception?
01172                     $tmpFile = null;
01173                     $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
01174                 }
01175                 $tmpFiles[$path] = $tmpFile;
01176             }
01177 
01178             $batch = new CF_Async_Op_Batch( $cfOps );
01179             $cfOps = $batch->execute();
01180             foreach ( $cfOps as $path => $cfOp ) {
01181                 try {
01182                     $cfOp->getLastResponse();
01183                 } catch ( NoSuchContainerException $e ) {
01184                     $tmpFiles[$path] = null;
01185                 } catch ( NoSuchObjectException $e ) {
01186                     $tmpFiles[$path] = null;
01187                 } catch ( CloudFilesException $e ) { // some other exception?
01188                     $tmpFiles[$path] = null;
01189                     $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
01190                 }
01191                 fclose( $cfOp->_file_handle ); // close open handle
01192             }
01193         }
01194 
01195         return $tmpFiles;
01196     }
01197 
01198     public function getFileHttpUrl( array $params ) {
01199         if ( $this->swiftTempUrlKey != '' ||
01200             ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
01201         {
01202             list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
01203             if ( $srcRel === null ) {
01204                 return null; // invalid path
01205             }
01206             try {
01207                 $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
01208                 $sContObj = $this->getContainer( $srcCont );
01209                 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
01210                 if ( $this->swiftTempUrlKey != '' ) {
01211                     return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" );
01212                 } else { // give S3 API URL for rgw
01213                     $expires = time() + $ttl;
01214                     // Path for signature starts with the bucket
01215                     $spath = '/' . rawurlencode( $srcCont ) . '/' .
01216                         str_replace( '%2F', '/', rawurlencode( $srcRel ) );
01217                     // Calculate the hash
01218                     $signature = base64_encode( hash_hmac(
01219                         'sha1',
01220                         "GET\n\n\n{$expires}\n{$spath}",
01221                         $this->rgwS3SecretKey,
01222                         true // raw
01223                     ) );
01224                     // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
01225                     // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
01226                     return wfAppendQuery(
01227                         str_replace( '/swift/v1', '', // S3 API is the rgw default
01228                             $sContObj->cfs_http->getStorageUrl() . $spath ),
01229                         array(
01230                             'Signature' => $signature,
01231                             'Expires' => $expires,
01232                             'AWSAccessKeyId' => $this->rgwS3AccessKey )
01233                     );
01234                 }
01235             } catch ( NoSuchContainerException $e ) {
01236             } catch ( CloudFilesException $e ) { // some other exception?
01237                 $this->handleException( $e, null, __METHOD__, $params );
01238             }
01239         }
01240         return null;
01241     }
01242 
01243     protected function directoriesAreVirtual() {
01244         return true;
01245     }
01246 
01255     protected function headersFromParams( array $params ) {
01256         $hdrs = array();
01257         if ( !empty( $params['latest'] ) ) {
01258             $hdrs[] = 'X-Newest: true';
01259         }
01260         return $hdrs;
01261     }
01262 
01263     protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
01264         $statuses = array();
01265 
01266         $cfOps = array(); // list of CF_Async_Op objects
01267         foreach ( $fileOpHandles as $index => $fileOpHandle ) {
01268             $cfOps[$index] = $fileOpHandle->cfOp;
01269         }
01270         $batch = new CF_Async_Op_Batch( $cfOps );
01271 
01272         $cfOps = $batch->execute();
01273         foreach ( $cfOps as $index => $cfOp ) {
01274             $status = Status::newGood();
01275             $function = '_getResponse' . $fileOpHandles[$index]->call;
01276             try { // catch exceptions; update status
01277                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
01278                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
01279             } catch ( CloudFilesException $e ) { // some other exception?
01280                 $this->handleException( $e, $status,
01281                     __CLASS__ . ":$function", $fileOpHandles[$index]->params );
01282             }
01283             $statuses[$index] = $status;
01284         }
01285 
01286         return $statuses;
01287     }
01288 
01315     protected function setContainerAccess(
01316         CF_Container $contObj, array $readGrps, array $writeGrps
01317     ) {
01318         $creds = $contObj->cfs_auth->export_credentials();
01319 
01320         $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
01321 
01322         // Note: 10 second timeout consistent with php-cloudfiles
01323         $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
01324         $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
01325         $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
01326         $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
01327 
01328         return $req->execute(); // should return 204
01329     }
01330 
01338     public function purgeCDNCache( array $objects ) {
01339         if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
01340             foreach ( $objects as $object ) {
01341                 try {
01342                     $object->purge_from_cdn();
01343                 } catch ( CDNNotEnabledException $e ) {
01344                     // CDN not enabled; nothing to see here
01345                 } catch ( CloudFilesException $e ) {
01346                     $this->handleException( $e, null, __METHOD__,
01347                         array( 'cont' => $object->container->name, 'obj' => $object->name ) );
01348                 }
01349             }
01350         }
01351     }
01352 
01360     protected function getConnection() {
01361         if ( $this->connException instanceof CloudFilesException ) {
01362             if ( ( time() - $this->connErrorTime ) < 60 ) {
01363                 throw $this->connException; // failed last attempt; don't bother
01364             } else { // actually retry this time
01365                 $this->connException = null;
01366                 $this->connErrorTime = 0;
01367             }
01368         }
01369         // Session keys expire after a while, so we renew them periodically
01370         $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
01371         // Authenticate with proxy and get a session key...
01372         if ( !$this->conn || $reAuth ) {
01373             $this->sessionStarted = 0;
01374             $this->connContainerCache->clear();
01375             $cacheKey = $this->getCredsCacheKey( $this->auth->username );
01376             $creds = $this->srvCache->get( $cacheKey ); // credentials
01377             if ( is_array( $creds ) ) { // cache hit
01378                 $this->auth->load_cached_credentials(
01379                     $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
01380                 $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case
01381             } else { // cache miss
01382                 try {
01383                     $this->auth->authenticate();
01384                     $creds = $this->auth->export_credentials();
01385                     $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache
01386                     $this->sessionStarted = time();
01387                 } catch ( CloudFilesException $e ) {
01388                     $this->connException = $e; // don't keep re-trying
01389                     $this->connErrorTime = time();
01390                     throw $e; // throw it back
01391                 }
01392             }
01393             if ( $this->conn ) { // re-authorizing?
01394                 $this->conn->close(); // close active cURL handles in CF_Http object
01395             }
01396             $this->conn = new CF_Connection( $this->auth );
01397         }
01398         return $this->conn;
01399     }
01400 
01406     protected function closeConnection() {
01407         if ( $this->conn ) {
01408             $this->conn->close(); // close active cURL handles in CF_Http object
01409             $this->conn = null;
01410             $this->sessionStarted = 0;
01411             $this->connContainerCache->clear();
01412         }
01413     }
01414 
01421     private function getCredsCacheKey( $username ) {
01422         return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
01423     }
01424 
01434     protected function getContainer( $container, $bypassCache = false ) {
01435         $conn = $this->getConnection(); // Swift proxy connection
01436         if ( $bypassCache ) { // purge cache
01437             $this->connContainerCache->clear( $container );
01438         } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
01439             $this->primeContainerCache( array( $container ) ); // check persistent cache
01440         }
01441         if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
01442             $contObj = $conn->get_container( $container );
01443             // NoSuchContainerException not thrown: container must exist
01444             $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
01445             if ( !$bypassCache ) {
01446                 $this->setContainerCache( $container, // update persistent cache
01447                     array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
01448                 );
01449             }
01450         }
01451         return $this->connContainerCache->get( $container, 'obj' );
01452     }
01453 
01461     protected function createContainer( $container ) {
01462         $conn = $this->getConnection(); // Swift proxy connection
01463         $contObj = $conn->create_container( $container );
01464         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
01465         return $contObj;
01466     }
01467 
01475     protected function deleteContainer( $container ) {
01476         $conn = $this->getConnection(); // Swift proxy connection
01477         $this->connContainerCache->clear( $container ); // purge
01478         $conn->delete_container( $container );
01479     }
01480 
01481     protected function doPrimeContainerCache( array $containerInfo ) {
01482         try {
01483             $conn = $this->getConnection(); // Swift proxy connection
01484             foreach ( $containerInfo as $container => $info ) {
01485                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
01486                     $container, $info['count'], $info['bytes'] );
01487                 $this->connContainerCache->set( $container, 'obj', $contObj );
01488             }
01489         } catch ( CloudFilesException $e ) { // some other exception?
01490             $this->handleException( $e, null, __METHOD__, array() );
01491         }
01492     }
01493 
01504     protected function handleException( Exception $e, $status, $func, array $params ) {
01505         if ( $status instanceof Status ) {
01506             if ( $e instanceof AuthenticationException ) {
01507                 $status->fatal( 'backend-fail-connect', $this->name );
01508             } else {
01509                 $status->fatal( 'backend-fail-internal', $this->name );
01510             }
01511         }
01512         if ( $e->getMessage() ) {
01513             trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
01514         }
01515         if ( $e instanceof InvalidResponseException ) { // possibly a stale token
01516             $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
01517             $this->closeConnection(); // force a re-connect and re-auth next time
01518         }
01519         wfDebugLog( 'SwiftBackend',
01520             get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
01521             ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
01522         );
01523     }
01524 }
01525 
01529 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
01531     public $cfOp;
01533     public $affectedObjects = array();
01534 
01541     public function __construct(
01542         SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp
01543     ) {
01544         $this->backend = $backend;
01545         $this->params = $params;
01546         $this->call = $call;
01547         $this->cfOp = $cfOp;
01548     }
01549 }
01550 
01558 abstract class SwiftFileBackendList implements Iterator {
01560     protected $bufferIter = array();
01561     protected $bufferAfter = null; // string; list items *after* this path
01562     protected $pos = 0; // integer
01564     protected $params = array();
01565 
01567     protected $backend;
01568     protected $container; // string; container name
01569     protected $dir; // string; storage directory
01570     protected $suffixStart; // integer
01571 
01572     const PAGE_SIZE = 9000; // file listing buffer size
01573 
01580     public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
01581         $this->backend = $backend;
01582         $this->container = $fullCont;
01583         $this->dir = $dir;
01584         if ( substr( $this->dir, -1 ) === '/' ) {
01585             $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
01586         }
01587         if ( $this->dir == '' ) { // whole container
01588             $this->suffixStart = 0;
01589         } else { // dir within container
01590             $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
01591         }
01592         $this->params = $params;
01593     }
01594 
01599     public function key() {
01600         return $this->pos;
01601     }
01602 
01607     public function next() {
01608         // Advance to the next file in the page
01609         next( $this->bufferIter );
01610         ++$this->pos;
01611         // Check if there are no files left in this page and
01612         // advance to the next page if this page was not empty.
01613         if ( !$this->valid() && count( $this->bufferIter ) ) {
01614             $this->bufferIter = $this->pageFromList(
01615                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
01616             ); // updates $this->bufferAfter
01617         }
01618     }
01619 
01624     public function rewind() {
01625         $this->pos = 0;
01626         $this->bufferAfter = null;
01627         $this->bufferIter = $this->pageFromList(
01628             $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
01629         ); // updates $this->bufferAfter
01630     }
01631 
01636     public function valid() {
01637         if ( $this->bufferIter === null ) {
01638             return false; // some failure?
01639         } else {
01640             return ( current( $this->bufferIter ) !== false ); // no paths can have this value
01641         }
01642     }
01643 
01654     abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
01655 }
01656 
01660 class SwiftFileBackendDirList extends SwiftFileBackendList {
01665     public function current() {
01666         return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
01667     }
01668 
01673     protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
01674         return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
01675     }
01676 }
01677 
01681 class SwiftFileBackendFileList extends SwiftFileBackendList {
01686     public function current() {
01687         return substr( current( $this->bufferIter ), $this->suffixStart );
01688     }
01689 
01694     protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
01695         return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
01696     }
01697 }