MediaWiki
REL1_22
|
00001 <?php 00038 abstract class FileBackendStore extends FileBackend { 00040 protected $memCache; 00042 protected $cheapCache; 00044 protected $expensiveCache; 00045 00047 protected $shardViaHashLevels = array(); 00048 00050 protected $mimeCallback; 00051 00052 protected $maxFileSize = 4294967296; // integer bytes (4GiB) 00053 00054 const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries 00055 const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache" 00056 const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" 00057 00067 public function __construct( array $config ) { 00068 parent::__construct( $config ); 00069 $this->mimeCallback = isset( $config['mimeCallback'] ) 00070 ? $config['mimeCallback'] 00071 : function( $storagePath, $content, $fsPath ) { 00072 // @TODO: handle the case of extension-less files using the contents 00073 return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown'; 00074 }; 00075 $this->memCache = new EmptyBagOStuff(); // disabled by default 00076 $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); 00077 $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); 00078 } 00079 00087 final public function maxFileSizeInternal() { 00088 return $this->maxFileSize; 00089 } 00090 00100 abstract public function isPathUsableInternal( $storagePath ); 00101 00120 final public function createInternal( array $params ) { 00121 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00122 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { 00123 $status = Status::newFatal( 'backend-fail-maxsize', 00124 $params['dst'], $this->maxFileSizeInternal() ); 00125 } else { 00126 $status = $this->doCreateInternal( $params ); 00127 $this->clearCache( array( $params['dst'] ) ); 00128 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 00129 $this->deleteFileCache( $params['dst'] ); // persistent cache 00130 } 00131 } 00132 return $status; 00133 } 00134 00139 abstract protected function doCreateInternal( array $params ); 00140 00159 final public function storeInternal( array $params ) { 00160 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00161 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { 00162 $status = Status::newFatal( 'backend-fail-maxsize', 00163 $params['dst'], $this->maxFileSizeInternal() ); 00164 } else { 00165 $status = $this->doStoreInternal( $params ); 00166 $this->clearCache( array( $params['dst'] ) ); 00167 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 00168 $this->deleteFileCache( $params['dst'] ); // persistent cache 00169 } 00170 } 00171 return $status; 00172 } 00173 00178 abstract protected function doStoreInternal( array $params ); 00179 00199 final public function copyInternal( array $params ) { 00200 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00201 $status = $this->doCopyInternal( $params ); 00202 $this->clearCache( array( $params['dst'] ) ); 00203 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 00204 $this->deleteFileCache( $params['dst'] ); // persistent cache 00205 } 00206 return $status; 00207 } 00208 00213 abstract protected function doCopyInternal( array $params ); 00214 00229 final public function deleteInternal( array $params ) { 00230 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00231 $status = $this->doDeleteInternal( $params ); 00232 $this->clearCache( array( $params['src'] ) ); 00233 $this->deleteFileCache( $params['src'] ); // persistent cache 00234 return $status; 00235 } 00236 00241 abstract protected function doDeleteInternal( array $params ); 00242 00262 final public function moveInternal( array $params ) { 00263 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00264 $status = $this->doMoveInternal( $params ); 00265 $this->clearCache( array( $params['src'], $params['dst'] ) ); 00266 $this->deleteFileCache( $params['src'] ); // persistent cache 00267 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 00268 $this->deleteFileCache( $params['dst'] ); // persistent cache 00269 } 00270 return $status; 00271 } 00272 00277 protected function doMoveInternal( array $params ) { 00278 unset( $params['async'] ); // two steps, won't work here :) 00279 $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); 00280 $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); 00281 // Copy source to dest 00282 $status = $this->copyInternal( $params ); 00283 if ( $nsrc !== $ndst && $status->isOK() ) { 00284 // Delete source (only fails due to races or network problems) 00285 $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); 00286 $status->setResult( true, $status->value ); // ignore delete() errors 00287 } 00288 return $status; 00289 } 00290 00305 final public function describeInternal( array $params ) { 00306 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00307 if ( count( $params['headers'] ) ) { 00308 $status = $this->doDescribeInternal( $params ); 00309 $this->clearCache( array( $params['src'] ) ); 00310 $this->deleteFileCache( $params['src'] ); // persistent cache 00311 } else { 00312 $status = Status::newGood(); // nothing to do 00313 } 00314 return $status; 00315 } 00316 00321 protected function doDescribeInternal( array $params ) { 00322 return Status::newGood(); 00323 } 00324 00332 final public function nullInternal( array $params ) { 00333 return Status::newGood(); 00334 } 00335 00336 final public function concatenate( array $params ) { 00337 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00338 $status = Status::newGood(); 00339 00340 // Try to lock the source files for the scope of this function 00341 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); 00342 if ( $status->isOK() ) { 00343 // Actually do the file concatenation... 00344 $start_time = microtime( true ); 00345 $status->merge( $this->doConcatenate( $params ) ); 00346 $sec = microtime( true ) - $start_time; 00347 if ( !$status->isOK() ) { 00348 wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " . 00349 count( $params['srcs'] ) . " file(s) [$sec sec]" ); 00350 } 00351 } 00352 00353 return $status; 00354 } 00355 00360 protected function doConcatenate( array $params ) { 00361 $status = Status::newGood(); 00362 $tmpPath = $params['dst']; // convenience 00363 unset( $params['latest'] ); // sanity 00364 00365 // Check that the specified temp file is valid... 00366 wfSuppressWarnings(); 00367 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 ); 00368 wfRestoreWarnings(); 00369 if ( !$ok ) { // not present or not empty 00370 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 00371 return $status; 00372 } 00373 00374 // Get local FS versions of the chunks needed for the concatenation... 00375 $fsFiles = $this->getLocalReferenceMulti( $params ); 00376 foreach ( $fsFiles as $path => &$fsFile ) { 00377 if ( !$fsFile ) { // chunk failed to download? 00378 $fsFile = $this->getLocalReference( array( 'src' => $path ) ); 00379 if ( !$fsFile ) { // retry failed? 00380 $status->fatal( 'backend-fail-read', $path ); 00381 return $status; 00382 } 00383 } 00384 } 00385 unset( $fsFile ); // unset reference so we can reuse $fsFile 00386 00387 // Get a handle for the destination temp file 00388 $tmpHandle = fopen( $tmpPath, 'ab' ); 00389 if ( $tmpHandle === false ) { 00390 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 00391 return $status; 00392 } 00393 00394 // Build up the temp file using the source chunks (in order)... 00395 foreach ( $fsFiles as $virtualSource => $fsFile ) { 00396 // Get a handle to the local FS version 00397 $sourceHandle = fopen( $fsFile->getPath(), 'rb' ); 00398 if ( $sourceHandle === false ) { 00399 fclose( $tmpHandle ); 00400 $status->fatal( 'backend-fail-read', $virtualSource ); 00401 return $status; 00402 } 00403 // Append chunk to file (pass chunk size to avoid magic quotes) 00404 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { 00405 fclose( $sourceHandle ); 00406 fclose( $tmpHandle ); 00407 $status->fatal( 'backend-fail-writetemp', $tmpPath ); 00408 return $status; 00409 } 00410 fclose( $sourceHandle ); 00411 } 00412 if ( !fclose( $tmpHandle ) ) { 00413 $status->fatal( 'backend-fail-closetemp', $tmpPath ); 00414 return $status; 00415 } 00416 00417 clearstatcache(); // temp file changed 00418 00419 return $status; 00420 } 00421 00422 final protected function doPrepare( array $params ) { 00423 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00424 $status = Status::newGood(); 00425 00426 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00427 if ( $dir === null ) { 00428 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 00429 return $status; // invalid storage path 00430 } 00431 00432 if ( $shard !== null ) { // confined to a single container/shard 00433 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); 00434 } else { // directory is on several shards 00435 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00436 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00437 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 00438 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 00439 } 00440 } 00441 00442 return $status; 00443 } 00444 00449 protected function doPrepareInternal( $container, $dir, array $params ) { 00450 return Status::newGood(); 00451 } 00452 00453 final protected function doSecure( array $params ) { 00454 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00455 $status = Status::newGood(); 00456 00457 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00458 if ( $dir === null ) { 00459 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 00460 return $status; // invalid storage path 00461 } 00462 00463 if ( $shard !== null ) { // confined to a single container/shard 00464 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); 00465 } else { // directory is on several shards 00466 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00467 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00468 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 00469 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 00470 } 00471 } 00472 00473 return $status; 00474 } 00475 00480 protected function doSecureInternal( $container, $dir, array $params ) { 00481 return Status::newGood(); 00482 } 00483 00484 final protected function doPublish( array $params ) { 00485 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00486 $status = Status::newGood(); 00487 00488 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00489 if ( $dir === null ) { 00490 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 00491 return $status; // invalid storage path 00492 } 00493 00494 if ( $shard !== null ) { // confined to a single container/shard 00495 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); 00496 } else { // directory is on several shards 00497 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00498 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00499 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 00500 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 00501 } 00502 } 00503 00504 return $status; 00505 } 00506 00511 protected function doPublishInternal( $container, $dir, array $params ) { 00512 return Status::newGood(); 00513 } 00514 00515 final protected function doClean( array $params ) { 00516 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00517 $status = Status::newGood(); 00518 00519 // Recursive: first delete all empty subdirs recursively 00520 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) { 00521 $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) ); 00522 if ( $subDirsRel !== null ) { // no errors 00523 foreach ( $subDirsRel as $subDirRel ) { 00524 $subDir = $params['dir'] . "/{$subDirRel}"; // full path 00525 $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) ); 00526 } 00527 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends) 00528 } 00529 } 00530 00531 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00532 if ( $dir === null ) { 00533 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 00534 return $status; // invalid storage path 00535 } 00536 00537 // Attempt to lock this directory... 00538 $filesLockEx = array( $params['dir'] ); 00539 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); 00540 if ( !$status->isOK() ) { 00541 return $status; // abort 00542 } 00543 00544 if ( $shard !== null ) { // confined to a single container/shard 00545 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); 00546 $this->deleteContainerCache( $fullCont ); // purge cache 00547 } else { // directory is on several shards 00548 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00549 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00550 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 00551 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 00552 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache 00553 } 00554 } 00555 00556 return $status; 00557 } 00558 00563 protected function doCleanInternal( $container, $dir, array $params ) { 00564 return Status::newGood(); 00565 } 00566 00567 final public function fileExists( array $params ) { 00568 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00569 $stat = $this->getFileStat( $params ); 00570 return ( $stat === null ) ? null : (bool)$stat; // null => failure 00571 } 00572 00573 final public function getFileTimestamp( array $params ) { 00574 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00575 $stat = $this->getFileStat( $params ); 00576 return $stat ? $stat['mtime'] : false; 00577 } 00578 00579 final public function getFileSize( array $params ) { 00580 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00581 $stat = $this->getFileStat( $params ); 00582 return $stat ? $stat['size'] : false; 00583 } 00584 00585 final public function getFileStat( array $params ) { 00586 $path = self::normalizeStoragePath( $params['src'] ); 00587 if ( $path === null ) { 00588 return false; // invalid storage path 00589 } 00590 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00591 $latest = !empty( $params['latest'] ); // use latest data? 00592 if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { 00593 $this->primeFileCache( array( $path ) ); // check persistent cache 00594 } 00595 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { 00596 $stat = $this->cheapCache->get( $path, 'stat' ); 00597 // If we want the latest data, check that this cached 00598 // value was in fact fetched with the latest available data. 00599 if ( is_array( $stat ) ) { 00600 if ( !$latest || $stat['latest'] ) { 00601 return $stat; 00602 } 00603 } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) { 00604 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { 00605 return false; 00606 } 00607 } 00608 } 00609 wfProfileIn( __METHOD__ . '-miss' ); 00610 wfProfileIn( __METHOD__ . '-miss-' . $this->name ); 00611 $stat = $this->doGetFileStat( $params ); 00612 wfProfileOut( __METHOD__ . '-miss-' . $this->name ); 00613 wfProfileOut( __METHOD__ . '-miss' ); 00614 if ( is_array( $stat ) ) { // file exists 00615 $stat['latest'] = $latest; 00616 $this->cheapCache->set( $path, 'stat', $stat ); 00617 $this->setFileCache( $path, $stat ); // update persistent cache 00618 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata 00619 $this->cheapCache->set( $path, 'sha1', 00620 array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); 00621 } 00622 } elseif ( $stat === false ) { // file does not exist 00623 $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); 00624 $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too 00625 array( 'hash' => false, 'latest' => $latest ) ); 00626 wfDebug( __METHOD__ . ": File $path does not exist.\n" ); 00627 } else { // an error occurred 00628 wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); 00629 } 00630 return $stat; 00631 } 00632 00636 abstract protected function doGetFileStat( array $params ); 00637 00638 public function getFileContentsMulti( array $params ) { 00639 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00640 00641 $params = $this->setConcurrencyFlags( $params ); 00642 $contents = $this->doGetFileContentsMulti( $params ); 00643 00644 return $contents; 00645 } 00646 00651 protected function doGetFileContentsMulti( array $params ) { 00652 $contents = array(); 00653 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 00654 wfSuppressWarnings(); 00655 $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false; 00656 wfRestoreWarnings(); 00657 } 00658 return $contents; 00659 } 00660 00661 final public function getFileSha1Base36( array $params ) { 00662 $path = self::normalizeStoragePath( $params['src'] ); 00663 if ( $path === null ) { 00664 return false; // invalid storage path 00665 } 00666 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00667 $latest = !empty( $params['latest'] ); // use latest data? 00668 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { 00669 $stat = $this->cheapCache->get( $path, 'sha1' ); 00670 // If we want the latest data, check that this cached 00671 // value was in fact fetched with the latest available data. 00672 if ( !$latest || $stat['latest'] ) { 00673 return $stat['hash']; 00674 } 00675 } 00676 wfProfileIn( __METHOD__ . '-miss' ); 00677 wfProfileIn( __METHOD__ . '-miss-' . $this->name ); 00678 $hash = $this->doGetFileSha1Base36( $params ); 00679 wfProfileOut( __METHOD__ . '-miss-' . $this->name ); 00680 wfProfileOut( __METHOD__ . '-miss' ); 00681 $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); 00682 return $hash; 00683 } 00684 00689 protected function doGetFileSha1Base36( array $params ) { 00690 $fsFile = $this->getLocalReference( $params ); 00691 if ( !$fsFile ) { 00692 return false; 00693 } else { 00694 return $fsFile->getSha1Base36(); 00695 } 00696 } 00697 00698 final public function getFileProps( array $params ) { 00699 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00700 $fsFile = $this->getLocalReference( $params ); 00701 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); 00702 return $props; 00703 } 00704 00705 final public function getLocalReferenceMulti( array $params ) { 00706 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00707 00708 $params = $this->setConcurrencyFlags( $params ); 00709 00710 $fsFiles = array(); // (path => FSFile) 00711 $latest = !empty( $params['latest'] ); // use latest data? 00712 // Reuse any files already in process cache... 00713 foreach ( $params['srcs'] as $src ) { 00714 $path = self::normalizeStoragePath( $src ); 00715 if ( $path === null ) { 00716 $fsFiles[$src] = null; // invalid storage path 00717 } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) { 00718 $val = $this->expensiveCache->get( $path, 'localRef' ); 00719 // If we want the latest data, check that this cached 00720 // value was in fact fetched with the latest available data. 00721 if ( !$latest || $val['latest'] ) { 00722 $fsFiles[$src] = $val['object']; 00723 } 00724 } 00725 } 00726 // Fetch local references of any remaning files... 00727 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) ); 00728 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 00729 $fsFiles[$path] = $fsFile; 00730 if ( $fsFile ) { // update the process cache... 00731 $this->expensiveCache->set( $path, 'localRef', 00732 array( 'object' => $fsFile, 'latest' => $latest ) ); 00733 } 00734 } 00735 00736 return $fsFiles; 00737 } 00738 00743 protected function doGetLocalReferenceMulti( array $params ) { 00744 return $this->doGetLocalCopyMulti( $params ); 00745 } 00746 00747 final public function getLocalCopyMulti( array $params ) { 00748 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00749 00750 $params = $this->setConcurrencyFlags( $params ); 00751 $tmpFiles = $this->doGetLocalCopyMulti( $params ); 00752 00753 return $tmpFiles; 00754 } 00755 00760 abstract protected function doGetLocalCopyMulti( array $params ); 00761 00766 public function getFileHttpUrl( array $params ) { 00767 return null; // not supported 00768 } 00769 00770 final public function streamFile( array $params ) { 00771 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00772 $status = Status::newGood(); 00773 00774 $info = $this->getFileStat( $params ); 00775 if ( !$info ) { // let StreamFile handle the 404 00776 $status->fatal( 'backend-fail-notexists', $params['src'] ); 00777 } 00778 00779 // Set output buffer and HTTP headers for stream 00780 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array(); 00781 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders ); 00782 if ( $res == StreamFile::NOT_MODIFIED ) { 00783 // do nothing; client cache is up to date 00784 } elseif ( $res == StreamFile::READY_STREAM ) { 00785 wfProfileIn( __METHOD__ . '-send' ); 00786 wfProfileIn( __METHOD__ . '-send-' . $this->name ); 00787 $status = $this->doStreamFile( $params ); 00788 wfProfileOut( __METHOD__ . '-send-' . $this->name ); 00789 wfProfileOut( __METHOD__ . '-send' ); 00790 if ( !$status->isOK() ) { 00791 // Per bug 41113, nasty things can happen if bad cache entries get 00792 // stuck in cache. It's also possible that this error can come up 00793 // with simple race conditions. Clear out the stat cache to be safe. 00794 $this->clearCache( array( $params['src'] ) ); 00795 $this->deleteFileCache( $params['src'] ); 00796 trigger_error( "Bad stat cache or race condition for file {$params['src']}." ); 00797 } 00798 } else { 00799 $status->fatal( 'backend-fail-stream', $params['src'] ); 00800 } 00801 00802 return $status; 00803 } 00804 00809 protected function doStreamFile( array $params ) { 00810 $status = Status::newGood(); 00811 00812 $fsFile = $this->getLocalReference( $params ); 00813 if ( !$fsFile ) { 00814 $status->fatal( 'backend-fail-stream', $params['src'] ); 00815 } elseif ( !readfile( $fsFile->getPath() ) ) { 00816 $status->fatal( 'backend-fail-stream', $params['src'] ); 00817 } 00818 00819 return $status; 00820 } 00821 00822 final public function directoryExists( array $params ) { 00823 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00824 if ( $dir === null ) { 00825 return false; // invalid storage path 00826 } 00827 if ( $shard !== null ) { // confined to a single container/shard 00828 return $this->doDirectoryExists( $fullCont, $dir, $params ); 00829 } else { // directory is on several shards 00830 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00831 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00832 $res = false; // response 00833 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 00834 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params ); 00835 if ( $exists ) { 00836 $res = true; 00837 break; // found one! 00838 } elseif ( $exists === null ) { // error? 00839 $res = null; // if we don't find anything, it is indeterminate 00840 } 00841 } 00842 return $res; 00843 } 00844 } 00845 00854 abstract protected function doDirectoryExists( $container, $dir, array $params ); 00855 00856 final public function getDirectoryList( array $params ) { 00857 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00858 if ( $dir === null ) { // invalid storage path 00859 return null; 00860 } 00861 if ( $shard !== null ) { 00862 // File listing is confined to a single container/shard 00863 return $this->getDirectoryListInternal( $fullCont, $dir, $params ); 00864 } else { 00865 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00866 // File listing spans multiple containers/shards 00867 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00868 return new FileBackendStoreShardDirIterator( $this, 00869 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 00870 } 00871 } 00872 00883 abstract public function getDirectoryListInternal( $container, $dir, array $params ); 00884 00885 final public function getFileList( array $params ) { 00886 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 00887 if ( $dir === null ) { // invalid storage path 00888 return null; 00889 } 00890 if ( $shard !== null ) { 00891 // File listing is confined to a single container/shard 00892 return $this->getFileListInternal( $fullCont, $dir, $params ); 00893 } else { 00894 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 00895 // File listing spans multiple containers/shards 00896 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 00897 return new FileBackendStoreShardFileIterator( $this, 00898 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 00899 } 00900 } 00901 00912 abstract public function getFileListInternal( $container, $dir, array $params ); 00913 00925 final public function getOperationsInternal( array $ops ) { 00926 $supportedOps = array( 00927 'store' => 'StoreFileOp', 00928 'copy' => 'CopyFileOp', 00929 'move' => 'MoveFileOp', 00930 'delete' => 'DeleteFileOp', 00931 'create' => 'CreateFileOp', 00932 'describe' => 'DescribeFileOp', 00933 'null' => 'NullFileOp' 00934 ); 00935 00936 $performOps = array(); // array of FileOp objects 00937 // Build up ordered array of FileOps... 00938 foreach ( $ops as $operation ) { 00939 $opName = $operation['op']; 00940 if ( isset( $supportedOps[$opName] ) ) { 00941 $class = $supportedOps[$opName]; 00942 // Get params for this operation 00943 $params = $operation; 00944 // Append the FileOp class 00945 $performOps[] = new $class( $this, $params ); 00946 } else { 00947 throw new MWException( "Operation '$opName' is not supported." ); 00948 } 00949 } 00950 00951 return $performOps; 00952 } 00953 00964 final public function getPathsToLockForOpsInternal( array $performOps ) { 00965 // Build up a list of files to lock... 00966 $paths = array( 'sh' => array(), 'ex' => array() ); 00967 foreach ( $performOps as $fileOp ) { 00968 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() ); 00969 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() ); 00970 } 00971 // Optimization: if doing an EX lock anyway, don't also set an SH one 00972 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] ); 00973 // Get a shared lock on the parent directory of each path changed 00974 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); 00975 00976 return array( 00977 LockManager::LOCK_UW => $paths['sh'], 00978 LockManager::LOCK_EX => $paths['ex'] 00979 ); 00980 } 00981 00982 public function getScopedLocksForOps( array $ops, Status $status ) { 00983 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); 00984 return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) ); 00985 } 00986 00987 final protected function doOperationsInternal( array $ops, array $opts ) { 00988 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 00989 $status = Status::newGood(); 00990 00991 // Fix up custom header name/value pairs... 00992 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); 00993 00994 // Build up a list of FileOps... 00995 $performOps = $this->getOperationsInternal( $ops ); 00996 00997 // Acquire any locks as needed... 00998 if ( empty( $opts['nonLocking'] ) ) { 00999 // Build up a list of files to lock... 01000 $paths = $this->getPathsToLockForOpsInternal( $performOps ); 01001 // Try to lock those files for the scope of this function... 01002 $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status ); 01003 if ( !$status->isOK() ) { 01004 return $status; // abort 01005 } 01006 } 01007 01008 // Clear any file cache entries (after locks acquired) 01009 if ( empty( $opts['preserveCache'] ) ) { 01010 $this->clearCache(); 01011 } 01012 01013 // Load from the persistent file and container caches 01014 $this->primeFileCache( $performOps ); 01015 $this->primeContainerCache( $performOps ); 01016 01017 // Actually attempt the operation batch... 01018 $opts = $this->setConcurrencyFlags( $opts ); 01019 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); 01020 01021 // Merge errors into status fields 01022 $status->merge( $subStatus ); 01023 $status->success = $subStatus->success; // not done in merge() 01024 01025 return $status; 01026 } 01027 01028 final protected function doQuickOperationsInternal( array $ops ) { 01029 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 01030 $status = Status::newGood(); 01031 01032 // Fix up custom header name/value pairs... 01033 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); 01034 01035 // Clear any file cache entries 01036 $this->clearCache(); 01037 01038 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); 01039 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); 01040 $maxConcurrency = $this->concurrency; // throttle 01041 01042 $statuses = array(); // array of (index => Status) 01043 $fileOpHandles = array(); // list of (index => handle) arrays 01044 $curFileOpHandles = array(); // current handle batch 01045 // Perform the sync-only ops and build up op handles for the async ops... 01046 foreach ( $ops as $index => $params ) { 01047 if ( !in_array( $params['op'], $supportedOps ) ) { 01048 throw new MWException( "Operation '{$params['op']}' is not supported." ); 01049 } 01050 $method = $params['op'] . 'Internal'; // e.g. "storeInternal" 01051 $subStatus = $this->$method( array( 'async' => $async ) + $params ); 01052 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async 01053 if ( count( $curFileOpHandles ) >= $maxConcurrency ) { 01054 $fileOpHandles[] = $curFileOpHandles; // push this batch 01055 $curFileOpHandles = array(); 01056 } 01057 $curFileOpHandles[$index] = $subStatus->value; // keep index 01058 } else { // error or completed 01059 $statuses[$index] = $subStatus; // keep index 01060 } 01061 } 01062 if ( count( $curFileOpHandles ) ) { 01063 $fileOpHandles[] = $curFileOpHandles; // last batch 01064 } 01065 // Do all the async ops that can be done concurrently... 01066 foreach ( $fileOpHandles as $fileHandleBatch ) { 01067 $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch ); 01068 } 01069 // Marshall and merge all the responses... 01070 foreach ( $statuses as $index => $subStatus ) { 01071 $status->merge( $subStatus ); 01072 if ( $subStatus->isOK() ) { 01073 $status->success[$index] = true; 01074 ++$status->successCount; 01075 } else { 01076 $status->success[$index] = false; 01077 ++$status->failCount; 01078 } 01079 } 01080 01081 return $status; 01082 } 01083 01093 final public function executeOpHandlesInternal( array $fileOpHandles ) { 01094 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 01095 foreach ( $fileOpHandles as $fileOpHandle ) { 01096 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { 01097 throw new MWException( "Given a non-FileBackendStoreOpHandle object." ); 01098 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { 01099 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." ); 01100 } 01101 } 01102 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles ); 01103 foreach ( $fileOpHandles as $fileOpHandle ) { 01104 $fileOpHandle->closeResources(); 01105 } 01106 return $res; 01107 } 01108 01115 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { 01116 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty 01117 throw new MWException( "This backend supports no asynchronous operations." ); 01118 } 01119 return array(); 01120 } 01121 01131 protected function stripInvalidHeadersFromOp( array $op ) { 01132 static $longs = array( 'Content-Disposition' ); 01133 if ( isset( $op['headers'] ) ) { // op sets HTTP headers 01134 foreach ( $op['headers'] as $name => $value ) { 01135 $maxHVLen = in_array( $name, $longs ) ? INF : 255; 01136 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { 01137 trigger_error( "Header '$name: $value' is too long." ); 01138 unset( $op['headers'][$name] ); 01139 } elseif ( !strlen( $value ) ) { 01140 $op['headers'][$name] = ''; // null/false => "" 01141 } 01142 } 01143 } 01144 return $op; 01145 } 01146 01147 final public function preloadCache( array $paths ) { 01148 $fullConts = array(); // full container names 01149 foreach ( $paths as $path ) { 01150 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 01151 $fullConts[] = $fullCont; 01152 } 01153 // Load from the persistent file and container caches 01154 $this->primeContainerCache( $fullConts ); 01155 $this->primeFileCache( $paths ); 01156 } 01157 01158 final public function clearCache( array $paths = null ) { 01159 if ( is_array( $paths ) ) { 01160 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); 01161 $paths = array_filter( $paths, 'strlen' ); // remove nulls 01162 } 01163 if ( $paths === null ) { 01164 $this->cheapCache->clear(); 01165 $this->expensiveCache->clear(); 01166 } else { 01167 foreach ( $paths as $path ) { 01168 $this->cheapCache->clear( $path ); 01169 $this->expensiveCache->clear( $path ); 01170 } 01171 } 01172 $this->doClearCache( $paths ); 01173 } 01174 01183 protected function doClearCache( array $paths = null ) {} 01184 01192 abstract protected function directoriesAreVirtual(); 01193 01201 final protected static function isValidContainerName( $container ) { 01202 // This accounts for Swift and S3 restrictions while leaving room 01203 // for things like '.xxx' (hex shard chars) or '.seg' (segments). 01204 // This disallows directory separators or traversal characters. 01205 // Note that matching strings URL encode to the same string; 01206 // in Swift, the length restriction is *after* URL encoding. 01207 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container ); 01208 } 01209 01223 final protected function resolveStoragePath( $storagePath ) { 01224 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); 01225 if ( $backend === $this->name ) { // must be for this backend 01226 $relPath = self::normalizeContainerPath( $relPath ); 01227 if ( $relPath !== null ) { 01228 // Get shard for the normalized path if this container is sharded 01229 $cShard = $this->getContainerShard( $container, $relPath ); 01230 // Validate and sanitize the relative path (backend-specific) 01231 $relPath = $this->resolveContainerPath( $container, $relPath ); 01232 if ( $relPath !== null ) { 01233 // Prepend any wiki ID prefix to the container name 01234 $container = $this->fullContainerName( $container ); 01235 if ( self::isValidContainerName( $container ) ) { 01236 // Validate and sanitize the container name (backend-specific) 01237 $container = $this->resolveContainerName( "{$container}{$cShard}" ); 01238 if ( $container !== null ) { 01239 return array( $container, $relPath, $cShard ); 01240 } 01241 } 01242 } 01243 } 01244 } 01245 return array( null, null, null ); 01246 } 01247 01263 final protected function resolveStoragePathReal( $storagePath ) { 01264 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); 01265 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { 01266 return array( $container, $relPath ); 01267 } 01268 return array( null, null ); 01269 } 01270 01279 final protected function getContainerShard( $container, $relPath ) { 01280 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); 01281 if ( $levels == 1 || $levels == 2 ) { 01282 // Hash characters are either base 16 or 36 01283 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; 01284 // Get a regex that represents the shard portion of paths. 01285 // The concatenation of the captures gives us the shard. 01286 if ( $levels === 1 ) { // 16 or 36 shards per container 01287 $hashDirRegex = '(' . $char . ')'; 01288 } else { // 256 or 1296 shards per container 01289 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") 01290 $hashDirRegex = $char . '/(' . $char . '{2})'; 01291 } else { // short hash dir format (e.g. "a/b/c") 01292 $hashDirRegex = '(' . $char . ')/(' . $char . ')'; 01293 } 01294 } 01295 // Allow certain directories to be above the hash dirs so as 01296 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). 01297 // They must be 2+ chars to avoid any hash directory ambiguity. 01298 $m = array(); 01299 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { 01300 return '.' . implode( '', array_slice( $m, 1 ) ); 01301 } 01302 return null; // failed to match 01303 } 01304 return ''; // no sharding 01305 } 01306 01315 final public function isSingleShardPathInternal( $storagePath ) { 01316 list( , , $shard ) = $this->resolveStoragePath( $storagePath ); 01317 return ( $shard !== null ); 01318 } 01319 01328 final protected function getContainerHashLevels( $container ) { 01329 if ( isset( $this->shardViaHashLevels[$container] ) ) { 01330 $config = $this->shardViaHashLevels[$container]; 01331 $hashLevels = (int)$config['levels']; 01332 if ( $hashLevels == 1 || $hashLevels == 2 ) { 01333 $hashBase = (int)$config['base']; 01334 if ( $hashBase == 16 || $hashBase == 36 ) { 01335 return array( $hashLevels, $hashBase, $config['repeat'] ); 01336 } 01337 } 01338 } 01339 return array( 0, 0, false ); // no sharding 01340 } 01341 01348 final protected function getContainerSuffixes( $container ) { 01349 $shards = array(); 01350 list( $digits, $base ) = $this->getContainerHashLevels( $container ); 01351 if ( $digits > 0 ) { 01352 $numShards = pow( $base, $digits ); 01353 for ( $index = 0; $index < $numShards; $index++ ) { 01354 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits ); 01355 } 01356 } 01357 return $shards; 01358 } 01359 01366 final protected function fullContainerName( $container ) { 01367 if ( $this->wikiId != '' ) { 01368 return "{$this->wikiId}-$container"; 01369 } else { 01370 return $container; 01371 } 01372 } 01373 01382 protected function resolveContainerName( $container ) { 01383 return $container; 01384 } 01385 01396 protected function resolveContainerPath( $container, $relStoragePath ) { 01397 return $relStoragePath; 01398 } 01399 01406 private function containerCacheKey( $container ) { 01407 return wfMemcKey( 'backend', $this->getName(), 'container', $container ); 01408 } 01409 01417 final protected function setContainerCache( $container, array $val ) { 01418 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 ); 01419 } 01420 01428 final protected function deleteContainerCache( $container ) { 01429 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { 01430 trigger_error( "Unable to delete stat cache for container $container." ); 01431 } 01432 } 01433 01442 final protected function primeContainerCache( array $items ) { 01443 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 01444 01445 $paths = array(); // list of storage paths 01446 $contNames = array(); // (cache key => resolved container name) 01447 // Get all the paths/containers from the items... 01448 foreach ( $items as $item ) { 01449 if ( $item instanceof FileOp ) { 01450 $paths = array_merge( $paths, $item->storagePathsRead() ); 01451 $paths = array_merge( $paths, $item->storagePathsChanged() ); 01452 } elseif ( self::isStoragePath( $item ) ) { 01453 $paths[] = $item; 01454 } elseif ( is_string( $item ) ) { // full container name 01455 $contNames[$this->containerCacheKey( $item )] = $item; 01456 } 01457 } 01458 // Get all the corresponding cache keys for paths... 01459 foreach ( $paths as $path ) { 01460 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 01461 if ( $fullCont !== null ) { // valid path for this backend 01462 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; 01463 } 01464 } 01465 01466 $contInfo = array(); // (resolved container name => cache value) 01467 // Get all cache entries for these container cache keys... 01468 $values = $this->memCache->getMulti( array_keys( $contNames ) ); 01469 foreach ( $values as $cacheKey => $val ) { 01470 $contInfo[$contNames[$cacheKey]] = $val; 01471 } 01472 01473 // Populate the container process cache for the backend... 01474 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); 01475 } 01476 01485 protected function doPrimeContainerCache( array $containerInfo ) {} 01486 01493 private function fileCacheKey( $path ) { 01494 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) ); 01495 } 01496 01506 final protected function setFileCache( $path, array $val ) { 01507 $path = FileBackend::normalizeStoragePath( $path ); 01508 if ( $path === null ) { 01509 return; // invalid storage path 01510 } 01511 $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); 01512 $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); 01513 $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl ); 01514 } 01515 01525 final protected function deleteFileCache( $path ) { 01526 $path = FileBackend::normalizeStoragePath( $path ); 01527 if ( $path === null ) { 01528 return; // invalid storage path 01529 } 01530 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) { 01531 trigger_error( "Unable to delete stat cache for file $path." ); 01532 } 01533 } 01534 01543 final protected function primeFileCache( array $items ) { 01544 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 01545 01546 $paths = array(); // list of storage paths 01547 $pathNames = array(); // (cache key => storage path) 01548 // Get all the paths/containers from the items... 01549 foreach ( $items as $item ) { 01550 if ( $item instanceof FileOp ) { 01551 $paths = array_merge( $paths, $item->storagePathsRead() ); 01552 $paths = array_merge( $paths, $item->storagePathsChanged() ); 01553 } elseif ( self::isStoragePath( $item ) ) { 01554 $paths[] = FileBackend::normalizeStoragePath( $item ); 01555 } 01556 } 01557 // Get rid of any paths that failed normalization... 01558 $paths = array_filter( $paths, 'strlen' ); // remove nulls 01559 // Get all the corresponding cache keys for paths... 01560 foreach ( $paths as $path ) { 01561 list( , $rel, ) = $this->resolveStoragePath( $path ); 01562 if ( $rel !== null ) { // valid path for this backend 01563 $pathNames[$this->fileCacheKey( $path )] = $path; 01564 } 01565 } 01566 // Get all cache entries for these container cache keys... 01567 $values = $this->memCache->getMulti( array_keys( $pathNames ) ); 01568 foreach ( $values as $cacheKey => $val ) { 01569 if ( is_array( $val ) ) { 01570 $path = $pathNames[$cacheKey]; 01571 $this->cheapCache->set( $path, 'stat', $val ); 01572 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata 01573 $this->cheapCache->set( $path, 'sha1', 01574 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) ); 01575 } 01576 } 01577 } 01578 } 01579 01586 final protected function setConcurrencyFlags( array $opts ) { 01587 $opts['concurrency'] = 1; // off 01588 if ( $this->parallelize === 'implicit' ) { 01589 if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) { 01590 $opts['concurrency'] = $this->concurrency; 01591 } 01592 } elseif ( $this->parallelize === 'explicit' ) { 01593 if ( !empty( $opts['parallelize'] ) ) { 01594 $opts['concurrency'] = $this->concurrency; 01595 } 01596 } 01597 return $opts; 01598 } 01599 01608 protected function getContentType( $storagePath, $content, $fsPath ) { 01609 return call_user_func_array( $this->mimeCallback, func_get_args() ); 01610 } 01611 } 01612 01621 abstract class FileBackendStoreOpHandle { 01623 public $params = array(); // params to caller functions 01625 public $backend; 01627 public $resourcesToClose = array(); 01628 01629 public $call; // string; name that identifies the function called 01630 01636 public function closeResources() { 01637 array_map( 'fclose', $this->resourcesToClose ); 01638 } 01639 } 01640 01647 abstract class FileBackendStoreShardListIterator extends FilterIterator { 01649 protected $backend; 01651 protected $params; 01652 01653 protected $container; // string; full container name 01654 protected $directory; // string; resolved relative path 01655 01657 protected $multiShardPaths = array(); // (rel path => 1) 01658 01666 public function __construct( 01667 FileBackendStore $backend, $container, $dir, array $suffixes, array $params 01668 ) { 01669 $this->backend = $backend; 01670 $this->container = $container; 01671 $this->directory = $dir; 01672 $this->params = $params; 01673 01674 $iter = new AppendIterator(); 01675 foreach ( $suffixes as $suffix ) { 01676 $iter->append( $this->listFromShard( $this->container . $suffix ) ); 01677 } 01678 01679 parent::__construct( $iter ); 01680 } 01681 01682 public function accept() { 01683 $rel = $this->getInnerIterator()->current(); // path relative to given directory 01684 $path = $this->params['dir'] . "/{$rel}"; // full storage path 01685 if ( $this->backend->isSingleShardPathInternal( $path ) ) { 01686 return true; // path is only on one shard; no issue with duplicates 01687 } elseif ( isset( $this->multiShardPaths[$rel] ) ) { 01688 // Don't keep listing paths that are on multiple shards 01689 return false; 01690 } else { 01691 $this->multiShardPaths[$rel] = 1; 01692 return true; 01693 } 01694 } 01695 01696 public function rewind() { 01697 parent::rewind(); 01698 $this->multiShardPaths = array(); 01699 } 01700 01707 abstract protected function listFromShard( $container ); 01708 } 01709 01713 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { 01714 protected function listFromShard( $container ) { 01715 $list = $this->backend->getDirectoryListInternal( 01716 $container, $this->directory, $this->params ); 01717 if ( $list === null ) { 01718 return new ArrayIterator( array() ); 01719 } else { 01720 return is_array( $list ) ? new ArrayIterator( $list ) : $list; 01721 } 01722 } 01723 } 01724 01728 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { 01729 protected function listFromShard( $container ) { 01730 $list = $this->backend->getFileListInternal( 01731 $container, $this->directory, $this->params ); 01732 if ( $list === null ) { 01733 return new ArrayIterator( array() ); 01734 } else { 01735 return is_array( $list ) ? new ArrayIterator( $list ) : $list; 01736 } 01737 } 01738 }