MediaWiki  REL1_22
FileBackendMultiWrite.php
Go to the documentation of this file.
00001 <?php
00042 class FileBackendMultiWrite extends FileBackend {
00044     protected $backends = array(); // array of (backend index => backends)
00045     protected $masterIndex = -1; // integer; index of master backend
00046     protected $syncChecks = 0; // integer; bitfield
00047     protected $autoResync = false; // boolean
00048 
00050     protected $noPushDirConts = array();
00051     protected $noPushQuickOps = false; // boolean
00052 
00053     /* Possible internal backend consistency checks */
00054     const CHECK_SIZE = 1;
00055     const CHECK_TIME = 2;
00056     const CHECK_SHA1 = 4;
00057 
00087     public function __construct( array $config ) {
00088         parent::__construct( $config );
00089         $this->syncChecks = isset( $config['syncChecks'] )
00090             ? $config['syncChecks']
00091             : self::CHECK_SIZE;
00092         $this->autoResync = isset( $config['autoResync'] )
00093             ? $config['autoResync']
00094             : false;
00095         $this->noPushQuickOps = isset( $config['noPushQuickOps'] )
00096             ? $config['noPushQuickOps']
00097             : false;
00098         $this->noPushDirConts = isset( $config['noPushDirConts'] )
00099             ? $config['noPushDirConts']
00100             : array();
00101         // Construct backends here rather than via registration
00102         // to keep these backends hidden from outside the proxy.
00103         $namesUsed = array();
00104         foreach ( $config['backends'] as $index => $config ) {
00105             if ( isset( $config['template'] ) ) {
00106                 // Config is just a modified version of a registered backend's.
00107                 // This should only be used when that config is used only by this backend.
00108                 $config = $config + FileBackendGroup::singleton()->config( $config['template'] );
00109             }
00110             $name = $config['name'];
00111             if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
00112                 throw new MWException( "Two or more backends defined with the name $name." );
00113             }
00114             $namesUsed[$name] = 1;
00115             // Alter certain sub-backend settings for sanity
00116             unset( $config['readOnly'] ); // use proxy backend setting
00117             unset( $config['fileJournal'] ); // use proxy backend journal
00118             $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID
00119             $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
00120             if ( !empty( $config['isMultiMaster'] ) ) {
00121                 if ( $this->masterIndex >= 0 ) {
00122                     throw new MWException( 'More than one master backend defined.' );
00123                 }
00124                 $this->masterIndex = $index; // this is the "master"
00125                 $config['fileJournal'] = $this->fileJournal; // log under proxy backend
00126             }
00127             // Create sub-backend object
00128             if ( !isset( $config['class'] ) ) {
00129                 throw new MWException( 'No class given for a backend config.' );
00130             }
00131             $class = $config['class'];
00132             $this->backends[$index] = new $class( $config );
00133         }
00134         if ( $this->masterIndex < 0 ) { // need backends and must have a master
00135             throw new MWException( 'No master backend defined.' );
00136         }
00137     }
00138 
00139     final protected function doOperationsInternal( array $ops, array $opts ) {
00140         $status = Status::newGood();
00141 
00142         $mbe = $this->backends[$this->masterIndex]; // convenience
00143 
00144         // Try to lock those files for the scope of this function...
00145         if ( empty( $opts['nonLocking'] ) ) {
00146             // Try to lock those files for the scope of this function...
00147             $scopeLock = $this->getScopedLocksForOps( $ops, $status );
00148             if ( !$status->isOK() ) {
00149                 return $status; // abort
00150             }
00151         }
00152         // Clear any cache entries (after locks acquired)
00153         $this->clearCache();
00154         $opts['preserveCache'] = true; // only locked files are cached
00155         // Get the list of paths to read/write...
00156         $relevantPaths = $this->fileStoragePathsForOps( $ops );
00157         // Check if the paths are valid and accessible on all backends...
00158         $status->merge( $this->accessibilityCheck( $relevantPaths ) );
00159         if ( !$status->isOK() ) {
00160             return $status; // abort
00161         }
00162         // Do a consistency check to see if the backends are consistent...
00163         $syncStatus = $this->consistencyCheck( $relevantPaths );
00164         if ( !$syncStatus->isOK() ) {
00165             wfDebugLog( 'FileOperation', get_class( $this ) .
00166                 " failed sync check: " . FormatJson::encode( $relevantPaths ) );
00167             // Try to resync the clone backends to the master on the spot...
00168             if ( !$this->autoResync || !$this->resyncFiles( $relevantPaths )->isOK() ) {
00169                 $status->merge( $syncStatus );
00170                 return $status; // abort
00171             }
00172         }
00173         // Actually attempt the operation batch on the master backend...
00174         $realOps = $this->substOpBatchPaths( $ops, $mbe );
00175         $masterStatus = $mbe->doOperations( $realOps, $opts );
00176         $status->merge( $masterStatus );
00177         // Propagate the operations to the clone backends if there were no unexpected errors
00178         // and if there were either no expected errors or if the 'force' option was used.
00179         // However, if nothing succeeded at all, then don't replicate any of the operations.
00180         // If $ops only had one operation, this might avoid backend sync inconsistencies.
00181         if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
00182             foreach ( $this->backends as $index => $backend ) {
00183                 if ( $index !== $this->masterIndex ) { // not done already
00184                     $realOps = $this->substOpBatchPaths( $ops, $backend );
00185                     $status->merge( $backend->doOperations( $realOps, $opts ) );
00186                 }
00187             }
00188         }
00189         // Make 'success', 'successCount', and 'failCount' fields reflect
00190         // the overall operation, rather than all the batches for each backend.
00191         // Do this by only using success values from the master backend's batch.
00192         $status->success = $masterStatus->success;
00193         $status->successCount = $masterStatus->successCount;
00194         $status->failCount = $masterStatus->failCount;
00195 
00196         return $status;
00197     }
00198 
00205     public function consistencyCheck( array $paths ) {
00206         $status = Status::newGood();
00207         if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
00208             return $status; // skip checks
00209         }
00210 
00211         $mBackend = $this->backends[$this->masterIndex];
00212         foreach ( $paths as $path ) {
00213             $params = array( 'src' => $path, 'latest' => true );
00214             $mParams = $this->substOpPaths( $params, $mBackend );
00215             // Stat the file on the 'master' backend
00216             $mStat = $mBackend->getFileStat( $mParams );
00217             if ( $this->syncChecks & self::CHECK_SHA1 ) {
00218                 $mSha1 = $mBackend->getFileSha1Base36( $mParams );
00219             } else {
00220                 $mSha1 = false;
00221             }
00222             // Check if all clone backends agree with the master...
00223             foreach ( $this->backends as $index => $cBackend ) {
00224                 if ( $index === $this->masterIndex ) {
00225                     continue; // master
00226                 }
00227                 $cParams = $this->substOpPaths( $params, $cBackend );
00228                 $cStat = $cBackend->getFileStat( $cParams );
00229                 if ( $mStat ) { // file is in master
00230                     if ( !$cStat ) { // file should exist
00231                         $status->fatal( 'backend-fail-synced', $path );
00232                         continue;
00233                     }
00234                     if ( $this->syncChecks & self::CHECK_SIZE ) {
00235                         if ( $cStat['size'] != $mStat['size'] ) { // wrong size
00236                             $status->fatal( 'backend-fail-synced', $path );
00237                             continue;
00238                         }
00239                     }
00240                     if ( $this->syncChecks & self::CHECK_TIME ) {
00241                         $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
00242                         $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
00243                         if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
00244                             $status->fatal( 'backend-fail-synced', $path );
00245                             continue;
00246                         }
00247                     }
00248                     if ( $this->syncChecks & self::CHECK_SHA1 ) {
00249                         if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1
00250                             $status->fatal( 'backend-fail-synced', $path );
00251                             continue;
00252                         }
00253                     }
00254                 } else { // file is not in master
00255                     if ( $cStat ) { // file should not exist
00256                         $status->fatal( 'backend-fail-synced', $path );
00257                     }
00258                 }
00259             }
00260         }
00261 
00262         return $status;
00263     }
00264 
00271     public function accessibilityCheck( array $paths ) {
00272         $status = Status::newGood();
00273         if ( count( $this->backends ) <= 1 ) {
00274             return $status; // skip checks
00275         }
00276 
00277         foreach ( $paths as $path ) {
00278             foreach ( $this->backends as $backend ) {
00279                 $realPath = $this->substPaths( $path, $backend );
00280                 if ( !$backend->isPathUsableInternal( $realPath ) ) {
00281                     $status->fatal( 'backend-fail-usable', $path );
00282                 }
00283             }
00284         }
00285 
00286         return $status;
00287     }
00288 
00296     public function resyncFiles( array $paths ) {
00297         $status = Status::newGood();
00298 
00299         $mBackend = $this->backends[$this->masterIndex];
00300         foreach ( $paths as $path ) {
00301             $mPath = $this->substPaths( $path, $mBackend );
00302             $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath, 'latest' => true ) );
00303             $mStat = $mBackend->getFileStat( array( 'src' => $mPath, 'latest' => true ) );
00304             if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity
00305                 $status->fatal( 'backend-fail-internal', $this->name );
00306                 continue; // file is not available on the master backend...
00307             }
00308             // Check of all clone backends agree with the master...
00309             foreach ( $this->backends as $index => $cBackend ) {
00310                 if ( $index === $this->masterIndex ) {
00311                     continue; // master
00312                 }
00313                 $cPath = $this->substPaths( $path, $cBackend );
00314                 $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath, 'latest' => true ) );
00315                 $cStat = $cBackend->getFileStat( array( 'src' => $cPath, 'latest' => true ) );
00316                 if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity
00317                     $status->fatal( 'backend-fail-internal', $cBackend->getName() );
00318                     continue; // file is not available on the clone backend...
00319                 }
00320                 if ( $mSha1 === $cSha1 ) {
00321                     // already synced; nothing to do
00322                 } elseif ( $mSha1 !== false ) { // file is in master
00323                     if ( $this->autoResync === 'conservative'
00324                         && $cStat && $cStat['mtime'] > $mStat['mtime'] )
00325                     {
00326                         $status->fatal( 'backend-fail-synced', $path );
00327                         continue; // don't rollback data
00328                     }
00329                     $fsFile = $mBackend->getLocalReference(
00330                         array( 'src' => $mPath, 'latest' => true ) );
00331                     $status->merge( $cBackend->quickStore(
00332                         array( 'src' => $fsFile->getPath(), 'dst' => $cPath )
00333                     ) );
00334                 } elseif ( $mStat === false ) { // file is not in master
00335                     if ( $this->autoResync === 'conservative' ) {
00336                         $status->fatal( 'backend-fail-synced', $path );
00337                         continue; // don't delete data
00338                     }
00339                     $status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) );
00340                 }
00341             }
00342         }
00343 
00344         return $status;
00345     }
00346 
00353     protected function fileStoragePathsForOps( array $ops ) {
00354         $paths = array();
00355         foreach ( $ops as $op ) {
00356             if ( isset( $op['src'] ) ) {
00357                 // For things like copy/move/delete with "ignoreMissingSource" and there
00358                 // is no source file, nothing should happen and there should be no errors.
00359                 if ( empty( $op['ignoreMissingSource'] )
00360                     || $this->fileExists( array( 'src' => $op['src'] ) ) )
00361                 {
00362                     $paths[] = $op['src'];
00363                 }
00364             }
00365             if ( isset( $op['srcs'] ) ) {
00366                 $paths = array_merge( $paths, $op['srcs'] );
00367             }
00368             if ( isset( $op['dst'] ) ) {
00369                 $paths[] = $op['dst'];
00370             }
00371         }
00372         return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) );
00373     }
00374 
00383     protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
00384         $newOps = array(); // operations
00385         foreach ( $ops as $op ) {
00386             $newOp = $op; // operation
00387             foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
00388                 if ( isset( $newOp[$par] ) ) { // string or array
00389                     $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
00390                 }
00391             }
00392             $newOps[] = $newOp;
00393         }
00394         return $newOps;
00395     }
00396 
00404     protected function substOpPaths( array $ops, FileBackendStore $backend ) {
00405         $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
00406         return $newOps[0];
00407     }
00408 
00416     protected function substPaths( $paths, FileBackendStore $backend ) {
00417         return preg_replace(
00418             '!^mwstore://' . preg_quote( $this->name ) . '/!',
00419             StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
00420             $paths // string or array
00421         );
00422     }
00423 
00430     protected function unsubstPaths( $paths ) {
00431         return preg_replace(
00432             '!^mwstore://([^/]+)!',
00433             StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
00434             $paths // string or array
00435         );
00436     }
00437 
00438     protected function doQuickOperationsInternal( array $ops ) {
00439         $status = Status::newGood();
00440         // Do the operations on the master backend; setting Status fields...
00441         $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
00442         $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
00443         $status->merge( $masterStatus );
00444         // Propagate the operations to the clone backends...
00445         if ( !$this->noPushQuickOps ) {
00446             foreach ( $this->backends as $index => $backend ) {
00447                 if ( $index !== $this->masterIndex ) { // not done already
00448                     $realOps = $this->substOpBatchPaths( $ops, $backend );
00449                     $status->merge( $backend->doQuickOperations( $realOps ) );
00450                 }
00451             }
00452         }
00453         // Make 'success', 'successCount', and 'failCount' fields reflect
00454         // the overall operation, rather than all the batches for each backend.
00455         // Do this by only using success values from the master backend's batch.
00456         $status->success = $masterStatus->success;
00457         $status->successCount = $masterStatus->successCount;
00458         $status->failCount = $masterStatus->failCount;
00459         return $status;
00460     }
00461 
00466     protected function replicateContainerDirChanges( $path ) {
00467         list( , $shortCont, ) = self::splitStoragePath( $path );
00468         return !in_array( $shortCont, $this->noPushDirConts );
00469     }
00470 
00471     protected function doPrepare( array $params ) {
00472         $status = Status::newGood();
00473         $replicate = $this->replicateContainerDirChanges( $params['dir'] );
00474         foreach ( $this->backends as $index => $backend ) {
00475             if ( $replicate || $index == $this->masterIndex ) {
00476                 $realParams = $this->substOpPaths( $params, $backend );
00477                 $status->merge( $backend->doPrepare( $realParams ) );
00478             }
00479         }
00480         return $status;
00481     }
00482 
00483     protected function doSecure( array $params ) {
00484         $status = Status::newGood();
00485         $replicate = $this->replicateContainerDirChanges( $params['dir'] );
00486         foreach ( $this->backends as $index => $backend ) {
00487             if ( $replicate || $index == $this->masterIndex ) {
00488                 $realParams = $this->substOpPaths( $params, $backend );
00489                 $status->merge( $backend->doSecure( $realParams ) );
00490             }
00491         }
00492         return $status;
00493     }
00494 
00495     protected function doPublish( array $params ) {
00496         $status = Status::newGood();
00497         $replicate = $this->replicateContainerDirChanges( $params['dir'] );
00498         foreach ( $this->backends as $index => $backend ) {
00499             if ( $replicate || $index == $this->masterIndex ) {
00500                 $realParams = $this->substOpPaths( $params, $backend );
00501                 $status->merge( $backend->doPublish( $realParams ) );
00502             }
00503         }
00504         return $status;
00505     }
00506 
00507     protected function doClean( array $params ) {
00508         $status = Status::newGood();
00509         $replicate = $this->replicateContainerDirChanges( $params['dir'] );
00510         foreach ( $this->backends as $index => $backend ) {
00511             if ( $replicate || $index == $this->masterIndex ) {
00512                 $realParams = $this->substOpPaths( $params, $backend );
00513                 $status->merge( $backend->doClean( $realParams ) );
00514             }
00515         }
00516         return $status;
00517     }
00518 
00519     public function concatenate( array $params ) {
00520         // We are writing to an FS file, so we don't need to do this per-backend
00521         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00522         return $this->backends[$this->masterIndex]->concatenate( $realParams );
00523     }
00524 
00525     public function fileExists( array $params ) {
00526         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00527         return $this->backends[$this->masterIndex]->fileExists( $realParams );
00528     }
00529 
00530     public function getFileTimestamp( array $params ) {
00531         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00532         return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
00533     }
00534 
00535     public function getFileSize( array $params ) {
00536         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00537         return $this->backends[$this->masterIndex]->getFileSize( $realParams );
00538     }
00539 
00540     public function getFileStat( array $params ) {
00541         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00542         return $this->backends[$this->masterIndex]->getFileStat( $realParams );
00543     }
00544 
00545     public function getFileContentsMulti( array $params ) {
00546         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00547         $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams );
00548 
00549         $contents = array(); // (path => FSFile) mapping using the proxy backend's name
00550         foreach ( $contentsM as $path => $data ) {
00551             $contents[$this->unsubstPaths( $path )] = $data;
00552         }
00553         return $contents;
00554     }
00555 
00556     public function getFileSha1Base36( array $params ) {
00557         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00558         return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
00559     }
00560 
00561     public function getFileProps( array $params ) {
00562         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00563         return $this->backends[$this->masterIndex]->getFileProps( $realParams );
00564     }
00565 
00566     public function streamFile( array $params ) {
00567         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00568         return $this->backends[$this->masterIndex]->streamFile( $realParams );
00569     }
00570 
00571     public function getLocalReferenceMulti( array $params ) {
00572         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00573         $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams );
00574 
00575         $fsFiles = array(); // (path => FSFile) mapping using the proxy backend's name
00576         foreach ( $fsFilesM as $path => $fsFile ) {
00577             $fsFiles[$this->unsubstPaths( $path )] = $fsFile;
00578         }
00579         return $fsFiles;
00580     }
00581 
00582     public function getLocalCopyMulti( array $params ) {
00583         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00584         $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams );
00585 
00586         $tempFiles = array(); // (path => TempFSFile) mapping using the proxy backend's name
00587         foreach ( $tempFilesM as $path => $tempFile ) {
00588             $tempFiles[$this->unsubstPaths( $path )] = $tempFile;
00589         }
00590         return $tempFiles;
00591     }
00592 
00593     public function getFileHttpUrl( array $params ) {
00594         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00595         return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams );
00596     }
00597 
00598     public function directoryExists( array $params ) {
00599         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00600         return $this->backends[$this->masterIndex]->directoryExists( $realParams );
00601     }
00602 
00603     public function getDirectoryList( array $params ) {
00604         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00605         return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
00606     }
00607 
00608     public function getFileList( array $params ) {
00609         $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
00610         return $this->backends[$this->masterIndex]->getFileList( $realParams );
00611     }
00612 
00613     public function clearCache( array $paths = null ) {
00614         foreach ( $this->backends as $backend ) {
00615             $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
00616             $backend->clearCache( $realPaths );
00617         }
00618     }
00619 
00620     public function getScopedLocksForOps( array $ops, Status $status ) {
00621         $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
00622         $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
00623         // Get the paths to lock from the master backend
00624         $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
00625         // Get the paths under the proxy backend's name
00626         $pbPaths = array(
00627             LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ),
00628             LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] )
00629         );
00630         // Actually acquire the locks
00631         return array( $this->getScopedFileLocks( $pbPaths, 'mixed', $status ) );
00632     }
00633 }