| [ Index ] |
PHP Cross Reference of MediaWiki-1.24.0 |
[Summary view] [Print] [Text view]
1 <?php 2 /** 3 * Base class for all backends using particular storage medium. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program; if not, write to the Free Software Foundation, Inc., 17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 * http://www.gnu.org/copyleft/gpl.html 19 * 20 * @file 21 * @ingroup FileBackend 22 * @author Aaron Schulz 23 */ 24 25 /** 26 * @brief Base class for all backends using particular storage medium. 27 * 28 * This class defines the methods as abstract that subclasses must implement. 29 * Outside callers should *not* use functions with "Internal" in the name. 30 * 31 * The FileBackend operations are implemented using basic functions 32 * such as storeInternal(), copyInternal(), deleteInternal() and the like. 33 * This class is also responsible for path resolution and sanitization. 34 * 35 * @ingroup FileBackend 36 * @since 1.19 37 */ 38 abstract class FileBackendStore extends FileBackend { 39 /** @var BagOStuff */ 40 protected $memCache; 41 /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */ 42 protected $cheapCache; 43 /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */ 44 protected $expensiveCache; 45 46 /** @var array Map of container names to sharding config */ 47 protected $shardViaHashLevels = array(); 48 49 /** @var callable Method to get the MIME type of files */ 50 protected $mimeCallback; 51 52 protected $maxFileSize = 4294967296; // integer bytes (4GiB) 53 54 const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries 55 const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache" 56 const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" 57 58 /** 59 * @see FileBackend::__construct() 60 * Additional $config params include: 61 * - mimeCallback : Callback that takes (storage path, content, file system path) and 62 * returns the MIME type of the file or 'unknown/unknown'. The file 63 * system path parameter should be used if the content one is null. 64 * 65 * @param array $config 66 */ 67 public function __construct( array $config ) { 68 parent::__construct( $config ); 69 $this->mimeCallback = isset( $config['mimeCallback'] ) 70 ? $config['mimeCallback'] 71 : function ( $storagePath, $content, $fsPath ) { 72 // @todo handle the case of extension-less files using the contents 73 return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown'; 74 }; 75 $this->memCache = new EmptyBagOStuff(); // disabled by default 76 $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); 77 $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); 78 } 79 80 /** 81 * Get the maximum allowable file size given backend 82 * medium restrictions and basic performance constraints. 83 * Do not call this function from places outside FileBackend and FileOp. 84 * 85 * @return int Bytes 86 */ 87 final public function maxFileSizeInternal() { 88 return $this->maxFileSize; 89 } 90 91 /** 92 * Check if a file can be created or changed at a given storage path. 93 * FS backends should check if the parent directory exists, files can be 94 * written under it, and that any file already there is writable. 95 * Backends using key/value stores should check if the container exists. 96 * 97 * @param string $storagePath 98 * @return bool 99 */ 100 abstract public function isPathUsableInternal( $storagePath ); 101 102 /** 103 * Create a file in the backend with the given contents. 104 * This will overwrite any file that exists at the destination. 105 * Do not call this function from places outside FileBackend and FileOp. 106 * 107 * $params include: 108 * - content : the raw file contents 109 * - dst : destination storage path 110 * - headers : HTTP header name/value map 111 * - async : Status will be returned immediately if supported. 112 * If the status is OK, then its value field will be 113 * set to a FileBackendStoreOpHandle object. 114 * - dstExists : Whether a file exists at the destination (optimization). 115 * Callers can use "false" if no existing file is being changed. 116 * 117 * @param array $params 118 * @return Status 119 */ 120 final public function createInternal( array $params ) { 121 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 122 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { 123 $status = Status::newFatal( 'backend-fail-maxsize', 124 $params['dst'], $this->maxFileSizeInternal() ); 125 } else { 126 $status = $this->doCreateInternal( $params ); 127 $this->clearCache( array( $params['dst'] ) ); 128 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 129 $this->deleteFileCache( $params['dst'] ); // persistent cache 130 } 131 } 132 133 return $status; 134 } 135 136 /** 137 * @see FileBackendStore::createInternal() 138 * @param array $params 139 * @return Status 140 */ 141 abstract protected function doCreateInternal( array $params ); 142 143 /** 144 * Store a file into the backend from a file on disk. 145 * This will overwrite any file that exists at the destination. 146 * Do not call this function from places outside FileBackend and FileOp. 147 * 148 * $params include: 149 * - src : source path on disk 150 * - dst : destination storage path 151 * - headers : HTTP header name/value map 152 * - async : Status will be returned immediately if supported. 153 * If the status is OK, then its value field will be 154 * set to a FileBackendStoreOpHandle object. 155 * - dstExists : Whether a file exists at the destination (optimization). 156 * Callers can use "false" if no existing file is being changed. 157 * 158 * @param array $params 159 * @return Status 160 */ 161 final public function storeInternal( array $params ) { 162 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 163 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { 164 $status = Status::newFatal( 'backend-fail-maxsize', 165 $params['dst'], $this->maxFileSizeInternal() ); 166 } else { 167 $status = $this->doStoreInternal( $params ); 168 $this->clearCache( array( $params['dst'] ) ); 169 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 170 $this->deleteFileCache( $params['dst'] ); // persistent cache 171 } 172 } 173 174 return $status; 175 } 176 177 /** 178 * @see FileBackendStore::storeInternal() 179 * @param array $params 180 * @return Status 181 */ 182 abstract protected function doStoreInternal( array $params ); 183 184 /** 185 * Copy a file from one storage path to another in the backend. 186 * This will overwrite any file that exists at the destination. 187 * Do not call this function from places outside FileBackend and FileOp. 188 * 189 * $params include: 190 * - src : source storage path 191 * - dst : destination storage path 192 * - ignoreMissingSource : do nothing if the source file does not exist 193 * - headers : HTTP header name/value map 194 * - async : Status will be returned immediately if supported. 195 * If the status is OK, then its value field will be 196 * set to a FileBackendStoreOpHandle object. 197 * - dstExists : Whether a file exists at the destination (optimization). 198 * Callers can use "false" if no existing file is being changed. 199 * 200 * @param array $params 201 * @return Status 202 */ 203 final public function copyInternal( array $params ) { 204 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 205 $status = $this->doCopyInternal( $params ); 206 $this->clearCache( array( $params['dst'] ) ); 207 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 208 $this->deleteFileCache( $params['dst'] ); // persistent cache 209 } 210 211 return $status; 212 } 213 214 /** 215 * @see FileBackendStore::copyInternal() 216 * @param array $params 217 * @return Status 218 */ 219 abstract protected function doCopyInternal( array $params ); 220 221 /** 222 * Delete a file at the storage path. 223 * Do not call this function from places outside FileBackend and FileOp. 224 * 225 * $params include: 226 * - src : source storage path 227 * - ignoreMissingSource : do nothing if the source file does not exist 228 * - async : Status will be returned immediately if supported. 229 * If the status is OK, then its value field will be 230 * set to a FileBackendStoreOpHandle object. 231 * 232 * @param array $params 233 * @return Status 234 */ 235 final public function deleteInternal( array $params ) { 236 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 237 $status = $this->doDeleteInternal( $params ); 238 $this->clearCache( array( $params['src'] ) ); 239 $this->deleteFileCache( $params['src'] ); // persistent cache 240 return $status; 241 } 242 243 /** 244 * @see FileBackendStore::deleteInternal() 245 * @param array $params 246 * @return Status 247 */ 248 abstract protected function doDeleteInternal( array $params ); 249 250 /** 251 * Move a file from one storage path to another in the backend. 252 * This will overwrite any file that exists at the destination. 253 * Do not call this function from places outside FileBackend and FileOp. 254 * 255 * $params include: 256 * - src : source storage path 257 * - dst : destination storage path 258 * - ignoreMissingSource : do nothing if the source file does not exist 259 * - headers : HTTP header name/value map 260 * - async : Status will be returned immediately if supported. 261 * If the status is OK, then its value field will be 262 * set to a FileBackendStoreOpHandle object. 263 * - dstExists : Whether a file exists at the destination (optimization). 264 * Callers can use "false" if no existing file is being changed. 265 * 266 * @param array $params 267 * @return Status 268 */ 269 final public function moveInternal( array $params ) { 270 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 271 $status = $this->doMoveInternal( $params ); 272 $this->clearCache( array( $params['src'], $params['dst'] ) ); 273 $this->deleteFileCache( $params['src'] ); // persistent cache 274 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { 275 $this->deleteFileCache( $params['dst'] ); // persistent cache 276 } 277 278 return $status; 279 } 280 281 /** 282 * @see FileBackendStore::moveInternal() 283 * @param array $params 284 * @return Status 285 */ 286 protected function doMoveInternal( array $params ) { 287 unset( $params['async'] ); // two steps, won't work here :) 288 $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); 289 $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); 290 // Copy source to dest 291 $status = $this->copyInternal( $params ); 292 if ( $nsrc !== $ndst && $status->isOK() ) { 293 // Delete source (only fails due to races or network problems) 294 $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); 295 $status->setResult( true, $status->value ); // ignore delete() errors 296 } 297 298 return $status; 299 } 300 301 /** 302 * Alter metadata for a file at the storage path. 303 * Do not call this function from places outside FileBackend and FileOp. 304 * 305 * $params include: 306 * - src : source storage path 307 * - headers : HTTP header name/value map 308 * - async : Status will be returned immediately if supported. 309 * If the status is OK, then its value field will be 310 * set to a FileBackendStoreOpHandle object. 311 * 312 * @param array $params 313 * @return Status 314 */ 315 final public function describeInternal( array $params ) { 316 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 317 if ( count( $params['headers'] ) ) { 318 $status = $this->doDescribeInternal( $params ); 319 $this->clearCache( array( $params['src'] ) ); 320 $this->deleteFileCache( $params['src'] ); // persistent cache 321 } else { 322 $status = Status::newGood(); // nothing to do 323 } 324 325 return $status; 326 } 327 328 /** 329 * @see FileBackendStore::describeInternal() 330 * @param array $params 331 * @return Status 332 */ 333 protected function doDescribeInternal( array $params ) { 334 return Status::newGood(); 335 } 336 337 /** 338 * No-op file operation that does nothing. 339 * Do not call this function from places outside FileBackend and FileOp. 340 * 341 * @param array $params 342 * @return Status 343 */ 344 final public function nullInternal( array $params ) { 345 return Status::newGood(); 346 } 347 348 final public function concatenate( array $params ) { 349 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 350 $status = Status::newGood(); 351 352 // Try to lock the source files for the scope of this function 353 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); 354 if ( $status->isOK() ) { 355 // Actually do the file concatenation... 356 $start_time = microtime( true ); 357 $status->merge( $this->doConcatenate( $params ) ); 358 $sec = microtime( true ) - $start_time; 359 if ( !$status->isOK() ) { 360 wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name}" . 361 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" ); 362 } 363 } 364 365 return $status; 366 } 367 368 /** 369 * @see FileBackendStore::concatenate() 370 * @param array $params 371 * @return Status 372 */ 373 protected function doConcatenate( array $params ) { 374 $status = Status::newGood(); 375 $tmpPath = $params['dst']; // convenience 376 unset( $params['latest'] ); // sanity 377 378 // Check that the specified temp file is valid... 379 wfSuppressWarnings(); 380 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 ); 381 wfRestoreWarnings(); 382 if ( !$ok ) { // not present or not empty 383 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 384 385 return $status; 386 } 387 388 // Get local FS versions of the chunks needed for the concatenation... 389 $fsFiles = $this->getLocalReferenceMulti( $params ); 390 foreach ( $fsFiles as $path => &$fsFile ) { 391 if ( !$fsFile ) { // chunk failed to download? 392 $fsFile = $this->getLocalReference( array( 'src' => $path ) ); 393 if ( !$fsFile ) { // retry failed? 394 $status->fatal( 'backend-fail-read', $path ); 395 396 return $status; 397 } 398 } 399 } 400 unset( $fsFile ); // unset reference so we can reuse $fsFile 401 402 // Get a handle for the destination temp file 403 $tmpHandle = fopen( $tmpPath, 'ab' ); 404 if ( $tmpHandle === false ) { 405 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 406 407 return $status; 408 } 409 410 // Build up the temp file using the source chunks (in order)... 411 foreach ( $fsFiles as $virtualSource => $fsFile ) { 412 // Get a handle to the local FS version 413 $sourceHandle = fopen( $fsFile->getPath(), 'rb' ); 414 if ( $sourceHandle === false ) { 415 fclose( $tmpHandle ); 416 $status->fatal( 'backend-fail-read', $virtualSource ); 417 418 return $status; 419 } 420 // Append chunk to file (pass chunk size to avoid magic quotes) 421 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { 422 fclose( $sourceHandle ); 423 fclose( $tmpHandle ); 424 $status->fatal( 'backend-fail-writetemp', $tmpPath ); 425 426 return $status; 427 } 428 fclose( $sourceHandle ); 429 } 430 if ( !fclose( $tmpHandle ) ) { 431 $status->fatal( 'backend-fail-closetemp', $tmpPath ); 432 433 return $status; 434 } 435 436 clearstatcache(); // temp file changed 437 438 return $status; 439 } 440 441 final protected function doPrepare( array $params ) { 442 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 443 $status = Status::newGood(); 444 445 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 446 if ( $dir === null ) { 447 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 448 449 return $status; // invalid storage path 450 } 451 452 if ( $shard !== null ) { // confined to a single container/shard 453 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); 454 } else { // directory is on several shards 455 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 456 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 457 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 458 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 459 } 460 } 461 462 return $status; 463 } 464 465 /** 466 * @see FileBackendStore::doPrepare() 467 * @param string $container 468 * @param string $dir 469 * @param array $params 470 * @return Status 471 */ 472 protected function doPrepareInternal( $container, $dir, array $params ) { 473 return Status::newGood(); 474 } 475 476 final protected function doSecure( array $params ) { 477 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 478 $status = Status::newGood(); 479 480 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 481 if ( $dir === null ) { 482 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 483 484 return $status; // invalid storage path 485 } 486 487 if ( $shard !== null ) { // confined to a single container/shard 488 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); 489 } else { // directory is on several shards 490 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 491 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 492 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 493 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 494 } 495 } 496 497 return $status; 498 } 499 500 /** 501 * @see FileBackendStore::doSecure() 502 * @param string $container 503 * @param string $dir 504 * @param array $params 505 * @return Status 506 */ 507 protected function doSecureInternal( $container, $dir, array $params ) { 508 return Status::newGood(); 509 } 510 511 final protected function doPublish( array $params ) { 512 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 513 $status = Status::newGood(); 514 515 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 516 if ( $dir === null ) { 517 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 518 519 return $status; // invalid storage path 520 } 521 522 if ( $shard !== null ) { // confined to a single container/shard 523 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); 524 } else { // directory is on several shards 525 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 526 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 527 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 528 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 529 } 530 } 531 532 return $status; 533 } 534 535 /** 536 * @see FileBackendStore::doPublish() 537 * @param string $container 538 * @param string $dir 539 * @param array $params 540 * @return Status 541 */ 542 protected function doPublishInternal( $container, $dir, array $params ) { 543 return Status::newGood(); 544 } 545 546 final protected function doClean( array $params ) { 547 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 548 $status = Status::newGood(); 549 550 // Recursive: first delete all empty subdirs recursively 551 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) { 552 $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) ); 553 if ( $subDirsRel !== null ) { // no errors 554 foreach ( $subDirsRel as $subDirRel ) { 555 $subDir = $params['dir'] . "/{$subDirRel}"; // full path 556 $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) ); 557 } 558 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends) 559 } 560 } 561 562 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 563 if ( $dir === null ) { 564 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 565 566 return $status; // invalid storage path 567 } 568 569 // Attempt to lock this directory... 570 $filesLockEx = array( $params['dir'] ); 571 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); 572 if ( !$status->isOK() ) { 573 return $status; // abort 574 } 575 576 if ( $shard !== null ) { // confined to a single container/shard 577 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); 578 $this->deleteContainerCache( $fullCont ); // purge cache 579 } else { // directory is on several shards 580 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 581 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 582 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 583 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 584 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache 585 } 586 } 587 588 return $status; 589 } 590 591 /** 592 * @see FileBackendStore::doClean() 593 * @param string $container 594 * @param string $dir 595 * @param array $params 596 * @return Status 597 */ 598 protected function doCleanInternal( $container, $dir, array $params ) { 599 return Status::newGood(); 600 } 601 602 final public function fileExists( array $params ) { 603 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 604 $stat = $this->getFileStat( $params ); 605 606 return ( $stat === null ) ? null : (bool)$stat; // null => failure 607 } 608 609 final public function getFileTimestamp( array $params ) { 610 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 611 $stat = $this->getFileStat( $params ); 612 613 return $stat ? $stat['mtime'] : false; 614 } 615 616 final public function getFileSize( array $params ) { 617 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 618 $stat = $this->getFileStat( $params ); 619 620 return $stat ? $stat['size'] : false; 621 } 622 623 final public function getFileStat( array $params ) { 624 $path = self::normalizeStoragePath( $params['src'] ); 625 if ( $path === null ) { 626 return false; // invalid storage path 627 } 628 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 629 $latest = !empty( $params['latest'] ); // use latest data? 630 if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { 631 $this->primeFileCache( array( $path ) ); // check persistent cache 632 } 633 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { 634 $stat = $this->cheapCache->get( $path, 'stat' ); 635 // If we want the latest data, check that this cached 636 // value was in fact fetched with the latest available data. 637 if ( is_array( $stat ) ) { 638 if ( !$latest || $stat['latest'] ) { 639 return $stat; 640 } 641 } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) { 642 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { 643 return false; 644 } 645 } 646 } 647 wfProfileIn( __METHOD__ . '-miss-' . $this->name ); 648 $stat = $this->doGetFileStat( $params ); 649 wfProfileOut( __METHOD__ . '-miss-' . $this->name ); 650 if ( is_array( $stat ) ) { // file exists 651 // Strongly consistent backends can automatically set "latest" 652 $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; 653 $this->cheapCache->set( $path, 'stat', $stat ); 654 $this->setFileCache( $path, $stat ); // update persistent cache 655 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata 656 $this->cheapCache->set( $path, 'sha1', 657 array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); 658 } 659 if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata 660 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); 661 $this->cheapCache->set( $path, 'xattr', 662 array( 'map' => $stat['xattr'], 'latest' => $latest ) ); 663 } 664 } elseif ( $stat === false ) { // file does not exist 665 $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); 666 $this->cheapCache->set( $path, 'xattr', array( 'map' => false, 'latest' => $latest ) ); 667 $this->cheapCache->set( $path, 'sha1', array( 'hash' => false, 'latest' => $latest ) ); 668 wfDebug( __METHOD__ . ": File $path does not exist.\n" ); 669 } else { // an error occurred 670 wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); 671 } 672 673 return $stat; 674 } 675 676 /** 677 * @see FileBackendStore::getFileStat() 678 */ 679 abstract protected function doGetFileStat( array $params ); 680 681 public function getFileContentsMulti( array $params ) { 682 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 683 684 $params = $this->setConcurrencyFlags( $params ); 685 $contents = $this->doGetFileContentsMulti( $params ); 686 687 return $contents; 688 } 689 690 /** 691 * @see FileBackendStore::getFileContentsMulti() 692 * @param array $params 693 * @return array 694 */ 695 protected function doGetFileContentsMulti( array $params ) { 696 $contents = array(); 697 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 698 wfSuppressWarnings(); 699 $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false; 700 wfRestoreWarnings(); 701 } 702 703 return $contents; 704 } 705 706 final public function getFileXAttributes( array $params ) { 707 $path = self::normalizeStoragePath( $params['src'] ); 708 if ( $path === null ) { 709 return false; // invalid storage path 710 } 711 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 712 $latest = !empty( $params['latest'] ); // use latest data? 713 if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) { 714 $stat = $this->cheapCache->get( $path, 'xattr' ); 715 // If we want the latest data, check that this cached 716 // value was in fact fetched with the latest available data. 717 if ( !$latest || $stat['latest'] ) { 718 return $stat['map']; 719 } 720 } 721 wfProfileIn( __METHOD__ . '-miss' ); 722 wfProfileIn( __METHOD__ . '-miss-' . $this->name ); 723 $fields = $this->doGetFileXAttributes( $params ); 724 $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false; 725 wfProfileOut( __METHOD__ . '-miss-' . $this->name ); 726 wfProfileOut( __METHOD__ . '-miss' ); 727 $this->cheapCache->set( $path, 'xattr', array( 'map' => $fields, 'latest' => $latest ) ); 728 729 return $fields; 730 } 731 732 /** 733 * @see FileBackendStore::getFileXAttributes() 734 * @return bool|string 735 */ 736 protected function doGetFileXAttributes( array $params ) { 737 return array( 'headers' => array(), 'metadata' => array() ); // not supported 738 } 739 740 final public function getFileSha1Base36( array $params ) { 741 $path = self::normalizeStoragePath( $params['src'] ); 742 if ( $path === null ) { 743 return false; // invalid storage path 744 } 745 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 746 $latest = !empty( $params['latest'] ); // use latest data? 747 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { 748 $stat = $this->cheapCache->get( $path, 'sha1' ); 749 // If we want the latest data, check that this cached 750 // value was in fact fetched with the latest available data. 751 if ( !$latest || $stat['latest'] ) { 752 return $stat['hash']; 753 } 754 } 755 wfProfileIn( __METHOD__ . '-miss-' . $this->name ); 756 $hash = $this->doGetFileSha1Base36( $params ); 757 wfProfileOut( __METHOD__ . '-miss-' . $this->name ); 758 $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); 759 760 return $hash; 761 } 762 763 /** 764 * @see FileBackendStore::getFileSha1Base36() 765 * @param array $params 766 * @return bool|string 767 */ 768 protected function doGetFileSha1Base36( array $params ) { 769 $fsFile = $this->getLocalReference( $params ); 770 if ( !$fsFile ) { 771 return false; 772 } else { 773 return $fsFile->getSha1Base36(); 774 } 775 } 776 777 final public function getFileProps( array $params ) { 778 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 779 $fsFile = $this->getLocalReference( $params ); 780 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); 781 782 return $props; 783 } 784 785 final public function getLocalReferenceMulti( array $params ) { 786 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 787 788 $params = $this->setConcurrencyFlags( $params ); 789 790 $fsFiles = array(); // (path => FSFile) 791 $latest = !empty( $params['latest'] ); // use latest data? 792 // Reuse any files already in process cache... 793 foreach ( $params['srcs'] as $src ) { 794 $path = self::normalizeStoragePath( $src ); 795 if ( $path === null ) { 796 $fsFiles[$src] = null; // invalid storage path 797 } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) { 798 $val = $this->expensiveCache->get( $path, 'localRef' ); 799 // If we want the latest data, check that this cached 800 // value was in fact fetched with the latest available data. 801 if ( !$latest || $val['latest'] ) { 802 $fsFiles[$src] = $val['object']; 803 } 804 } 805 } 806 // Fetch local references of any remaning files... 807 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) ); 808 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 809 $fsFiles[$path] = $fsFile; 810 if ( $fsFile ) { // update the process cache... 811 $this->expensiveCache->set( $path, 'localRef', 812 array( 'object' => $fsFile, 'latest' => $latest ) ); 813 } 814 } 815 816 return $fsFiles; 817 } 818 819 /** 820 * @see FileBackendStore::getLocalReferenceMulti() 821 * @param array $params 822 * @return array 823 */ 824 protected function doGetLocalReferenceMulti( array $params ) { 825 return $this->doGetLocalCopyMulti( $params ); 826 } 827 828 final public function getLocalCopyMulti( array $params ) { 829 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 830 831 $params = $this->setConcurrencyFlags( $params ); 832 $tmpFiles = $this->doGetLocalCopyMulti( $params ); 833 834 return $tmpFiles; 835 } 836 837 /** 838 * @see FileBackendStore::getLocalCopyMulti() 839 * @param array $params 840 * @return array 841 */ 842 abstract protected function doGetLocalCopyMulti( array $params ); 843 844 /** 845 * @see FileBackend::getFileHttpUrl() 846 * @param array $params 847 * @return string|null 848 */ 849 public function getFileHttpUrl( array $params ) { 850 return null; // not supported 851 } 852 853 final public function streamFile( array $params ) { 854 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 855 $status = Status::newGood(); 856 857 $info = $this->getFileStat( $params ); 858 if ( !$info ) { // let StreamFile handle the 404 859 $status->fatal( 'backend-fail-notexists', $params['src'] ); 860 } 861 862 // Set output buffer and HTTP headers for stream 863 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array(); 864 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders ); 865 if ( $res == StreamFile::NOT_MODIFIED ) { 866 // do nothing; client cache is up to date 867 } elseif ( $res == StreamFile::READY_STREAM ) { 868 wfProfileIn( __METHOD__ . '-send-' . $this->name ); 869 $status = $this->doStreamFile( $params ); 870 wfProfileOut( __METHOD__ . '-send-' . $this->name ); 871 if ( !$status->isOK() ) { 872 // Per bug 41113, nasty things can happen if bad cache entries get 873 // stuck in cache. It's also possible that this error can come up 874 // with simple race conditions. Clear out the stat cache to be safe. 875 $this->clearCache( array( $params['src'] ) ); 876 $this->deleteFileCache( $params['src'] ); 877 trigger_error( "Bad stat cache or race condition for file {$params['src']}." ); 878 } 879 } else { 880 $status->fatal( 'backend-fail-stream', $params['src'] ); 881 } 882 883 return $status; 884 } 885 886 /** 887 * @see FileBackendStore::streamFile() 888 * @param array $params 889 * @return Status 890 */ 891 protected function doStreamFile( array $params ) { 892 $status = Status::newGood(); 893 894 $fsFile = $this->getLocalReference( $params ); 895 if ( !$fsFile ) { 896 $status->fatal( 'backend-fail-stream', $params['src'] ); 897 } elseif ( !readfile( $fsFile->getPath() ) ) { 898 $status->fatal( 'backend-fail-stream', $params['src'] ); 899 } 900 901 return $status; 902 } 903 904 final public function directoryExists( array $params ) { 905 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 906 if ( $dir === null ) { 907 return false; // invalid storage path 908 } 909 if ( $shard !== null ) { // confined to a single container/shard 910 return $this->doDirectoryExists( $fullCont, $dir, $params ); 911 } else { // directory is on several shards 912 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 913 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 914 $res = false; // response 915 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 916 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params ); 917 if ( $exists ) { 918 $res = true; 919 break; // found one! 920 } elseif ( $exists === null ) { // error? 921 $res = null; // if we don't find anything, it is indeterminate 922 } 923 } 924 925 return $res; 926 } 927 } 928 929 /** 930 * @see FileBackendStore::directoryExists() 931 * 932 * @param string $container Resolved container name 933 * @param string $dir Resolved path relative to container 934 * @param array $params 935 * @return bool|null 936 */ 937 abstract protected function doDirectoryExists( $container, $dir, array $params ); 938 939 final public function getDirectoryList( array $params ) { 940 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 941 if ( $dir === null ) { // invalid storage path 942 return null; 943 } 944 if ( $shard !== null ) { 945 // File listing is confined to a single container/shard 946 return $this->getDirectoryListInternal( $fullCont, $dir, $params ); 947 } else { 948 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 949 // File listing spans multiple containers/shards 950 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 951 952 return new FileBackendStoreShardDirIterator( $this, 953 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 954 } 955 } 956 957 /** 958 * Do not call this function from places outside FileBackend 959 * 960 * @see FileBackendStore::getDirectoryList() 961 * 962 * @param string $container Resolved container name 963 * @param string $dir Resolved path relative to container 964 * @param array $params 965 * @return Traversable|array|null Returns null on failure 966 */ 967 abstract public function getDirectoryListInternal( $container, $dir, array $params ); 968 969 final public function getFileList( array $params ) { 970 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 971 if ( $dir === null ) { // invalid storage path 972 return null; 973 } 974 if ( $shard !== null ) { 975 // File listing is confined to a single container/shard 976 return $this->getFileListInternal( $fullCont, $dir, $params ); 977 } else { 978 wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); 979 // File listing spans multiple containers/shards 980 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 981 982 return new FileBackendStoreShardFileIterator( $this, 983 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 984 } 985 } 986 987 /** 988 * Do not call this function from places outside FileBackend 989 * 990 * @see FileBackendStore::getFileList() 991 * 992 * @param string $container Resolved container name 993 * @param string $dir Resolved path relative to container 994 * @param array $params 995 * @return Traversable|array|null Returns null on failure 996 */ 997 abstract public function getFileListInternal( $container, $dir, array $params ); 998 999 /** 1000 * Return a list of FileOp objects from a list of operations. 1001 * Do not call this function from places outside FileBackend. 1002 * 1003 * The result must have the same number of items as the input. 1004 * An exception is thrown if an unsupported operation is requested. 1005 * 1006 * @param array $ops Same format as doOperations() 1007 * @return array List of FileOp objects 1008 * @throws FileBackendError 1009 */ 1010 final public function getOperationsInternal( array $ops ) { 1011 $supportedOps = array( 1012 'store' => 'StoreFileOp', 1013 'copy' => 'CopyFileOp', 1014 'move' => 'MoveFileOp', 1015 'delete' => 'DeleteFileOp', 1016 'create' => 'CreateFileOp', 1017 'describe' => 'DescribeFileOp', 1018 'null' => 'NullFileOp' 1019 ); 1020 1021 $performOps = array(); // array of FileOp objects 1022 // Build up ordered array of FileOps... 1023 foreach ( $ops as $operation ) { 1024 $opName = $operation['op']; 1025 if ( isset( $supportedOps[$opName] ) ) { 1026 $class = $supportedOps[$opName]; 1027 // Get params for this operation 1028 $params = $operation; 1029 // Append the FileOp class 1030 $performOps[] = new $class( $this, $params ); 1031 } else { 1032 throw new FileBackendError( "Operation '$opName' is not supported." ); 1033 } 1034 } 1035 1036 return $performOps; 1037 } 1038 1039 /** 1040 * Get a list of storage paths to lock for a list of operations 1041 * Returns an array with LockManager::LOCK_UW (shared locks) and 1042 * LockManager::LOCK_EX (exclusive locks) keys, each corresponding 1043 * to a list of storage paths to be locked. All returned paths are 1044 * normalized. 1045 * 1046 * @param array $performOps List of FileOp objects 1047 * @return array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) 1048 */ 1049 final public function getPathsToLockForOpsInternal( array $performOps ) { 1050 // Build up a list of files to lock... 1051 $paths = array( 'sh' => array(), 'ex' => array() ); 1052 foreach ( $performOps as $fileOp ) { 1053 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() ); 1054 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() ); 1055 } 1056 // Optimization: if doing an EX lock anyway, don't also set an SH one 1057 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] ); 1058 // Get a shared lock on the parent directory of each path changed 1059 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); 1060 1061 return array( 1062 LockManager::LOCK_UW => $paths['sh'], 1063 LockManager::LOCK_EX => $paths['ex'] 1064 ); 1065 } 1066 1067 public function getScopedLocksForOps( array $ops, Status $status ) { 1068 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); 1069 1070 return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) ); 1071 } 1072 1073 final protected function doOperationsInternal( array $ops, array $opts ) { 1074 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1075 $status = Status::newGood(); 1076 1077 // Fix up custom header name/value pairs... 1078 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); 1079 1080 // Build up a list of FileOps... 1081 $performOps = $this->getOperationsInternal( $ops ); 1082 1083 // Acquire any locks as needed... 1084 if ( empty( $opts['nonLocking'] ) ) { 1085 // Build up a list of files to lock... 1086 $paths = $this->getPathsToLockForOpsInternal( $performOps ); 1087 // Try to lock those files for the scope of this function... 1088 $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status ); 1089 if ( !$status->isOK() ) { 1090 return $status; // abort 1091 } 1092 } 1093 1094 // Clear any file cache entries (after locks acquired) 1095 if ( empty( $opts['preserveCache'] ) ) { 1096 $this->clearCache(); 1097 } 1098 1099 // Build the list of paths involved 1100 $paths = array(); 1101 foreach ( $performOps as $op ) { 1102 $paths = array_merge( $paths, $op->storagePathsRead() ); 1103 $paths = array_merge( $paths, $op->storagePathsChanged() ); 1104 } 1105 1106 // Enlarge the cache to fit the stat entries of these files 1107 $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) ); 1108 1109 // Load from the persistent container caches 1110 $this->primeContainerCache( $paths ); 1111 // Get the latest stat info for all the files (having locked them) 1112 $ok = $this->preloadFileStat( array( 'srcs' => $paths, 'latest' => true ) ); 1113 1114 if ( $ok ) { 1115 // Actually attempt the operation batch... 1116 $opts = $this->setConcurrencyFlags( $opts ); 1117 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); 1118 } else { 1119 // If we could not even stat some files, then bail out... 1120 $subStatus = Status::newFatal( 'backend-fail-internal', $this->name ); 1121 foreach ( $ops as $i => $op ) { // mark each op as failed 1122 $subStatus->success[$i] = false; 1123 ++$subStatus->failCount; 1124 } 1125 wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name} " . 1126 " stat failure; aborted operations: " . FormatJson::encode( $ops ) ); 1127 } 1128 1129 // Merge errors into status fields 1130 $status->merge( $subStatus ); 1131 $status->success = $subStatus->success; // not done in merge() 1132 1133 // Shrink the stat cache back to normal size 1134 $this->cheapCache->resize( self::CACHE_CHEAP_SIZE ); 1135 1136 return $status; 1137 } 1138 1139 final protected function doQuickOperationsInternal( array $ops ) { 1140 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1141 $status = Status::newGood(); 1142 1143 // Fix up custom header name/value pairs... 1144 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); 1145 1146 // Clear any file cache entries 1147 $this->clearCache(); 1148 1149 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ); 1150 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen()) 1151 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); 1152 $maxConcurrency = $this->concurrency; // throttle 1153 1154 $statuses = array(); // array of (index => Status) 1155 $fileOpHandles = array(); // list of (index => handle) arrays 1156 $curFileOpHandles = array(); // current handle batch 1157 // Perform the sync-only ops and build up op handles for the async ops... 1158 foreach ( $ops as $index => $params ) { 1159 if ( !in_array( $params['op'], $supportedOps ) ) { 1160 throw new FileBackendError( "Operation '{$params['op']}' is not supported." ); 1161 } 1162 $method = $params['op'] . 'Internal'; // e.g. "storeInternal" 1163 $subStatus = $this->$method( array( 'async' => $async ) + $params ); 1164 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async 1165 if ( count( $curFileOpHandles ) >= $maxConcurrency ) { 1166 $fileOpHandles[] = $curFileOpHandles; // push this batch 1167 $curFileOpHandles = array(); 1168 } 1169 $curFileOpHandles[$index] = $subStatus->value; // keep index 1170 } else { // error or completed 1171 $statuses[$index] = $subStatus; // keep index 1172 } 1173 } 1174 if ( count( $curFileOpHandles ) ) { 1175 $fileOpHandles[] = $curFileOpHandles; // last batch 1176 } 1177 // Do all the async ops that can be done concurrently... 1178 foreach ( $fileOpHandles as $fileHandleBatch ) { 1179 $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch ); 1180 } 1181 // Marshall and merge all the responses... 1182 foreach ( $statuses as $index => $subStatus ) { 1183 $status->merge( $subStatus ); 1184 if ( $subStatus->isOK() ) { 1185 $status->success[$index] = true; 1186 ++$status->successCount; 1187 } else { 1188 $status->success[$index] = false; 1189 ++$status->failCount; 1190 } 1191 } 1192 1193 return $status; 1194 } 1195 1196 /** 1197 * Execute a list of FileBackendStoreOpHandle handles in parallel. 1198 * The resulting Status object fields will correspond 1199 * to the order in which the handles where given. 1200 * 1201 * @param array $fileOpHandles 1202 * @throws FileBackendError 1203 * @internal param array $handles List of FileBackendStoreOpHandle objects 1204 * @return array Map of Status objects 1205 */ 1206 final public function executeOpHandlesInternal( array $fileOpHandles ) { 1207 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1208 1209 foreach ( $fileOpHandles as $fileOpHandle ) { 1210 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { 1211 throw new FileBackendError( "Given a non-FileBackendStoreOpHandle object." ); 1212 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { 1213 throw new FileBackendError( "Given a FileBackendStoreOpHandle for the wrong backend." ); 1214 } 1215 } 1216 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles ); 1217 foreach ( $fileOpHandles as $fileOpHandle ) { 1218 $fileOpHandle->closeResources(); 1219 } 1220 1221 return $res; 1222 } 1223 1224 /** 1225 * @see FileBackendStore::executeOpHandlesInternal() 1226 * @param array $fileOpHandles 1227 * @throws FileBackendError 1228 * @return array List of corresponding Status objects 1229 */ 1230 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { 1231 if ( count( $fileOpHandles ) ) { 1232 throw new FileBackendError( "This backend supports no asynchronous operations." ); 1233 } 1234 1235 return array(); 1236 } 1237 1238 /** 1239 * Strip long HTTP headers from a file operation. 1240 * Most headers are just numbers, but some are allowed to be long. 1241 * This function is useful for cleaning up headers and avoiding backend 1242 * specific errors, especially in the middle of batch file operations. 1243 * 1244 * @param array $op Same format as doOperation() 1245 * @return array 1246 */ 1247 protected function stripInvalidHeadersFromOp( array $op ) { 1248 static $longs = array( 'Content-Disposition' ); 1249 if ( isset( $op['headers'] ) ) { // op sets HTTP headers 1250 foreach ( $op['headers'] as $name => $value ) { 1251 $maxHVLen = in_array( $name, $longs ) ? INF : 255; 1252 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { 1253 trigger_error( "Header '$name: $value' is too long." ); 1254 unset( $op['headers'][$name] ); 1255 } elseif ( !strlen( $value ) ) { 1256 $op['headers'][$name] = ''; // null/false => "" 1257 } 1258 } 1259 } 1260 1261 return $op; 1262 } 1263 1264 final public function preloadCache( array $paths ) { 1265 $fullConts = array(); // full container names 1266 foreach ( $paths as $path ) { 1267 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 1268 $fullConts[] = $fullCont; 1269 } 1270 // Load from the persistent file and container caches 1271 $this->primeContainerCache( $fullConts ); 1272 $this->primeFileCache( $paths ); 1273 } 1274 1275 final public function clearCache( array $paths = null ) { 1276 if ( is_array( $paths ) ) { 1277 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); 1278 $paths = array_filter( $paths, 'strlen' ); // remove nulls 1279 } 1280 if ( $paths === null ) { 1281 $this->cheapCache->clear(); 1282 $this->expensiveCache->clear(); 1283 } else { 1284 foreach ( $paths as $path ) { 1285 $this->cheapCache->clear( $path ); 1286 $this->expensiveCache->clear( $path ); 1287 } 1288 } 1289 $this->doClearCache( $paths ); 1290 } 1291 1292 /** 1293 * Clears any additional stat caches for storage paths 1294 * 1295 * @see FileBackend::clearCache() 1296 * 1297 * @param array $paths Storage paths (optional) 1298 */ 1299 protected function doClearCache( array $paths = null ) { 1300 } 1301 1302 final public function preloadFileStat( array $params ) { 1303 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1304 $success = true; // no network errors 1305 1306 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1; 1307 $stats = $this->doGetFileStatMulti( $params ); 1308 if ( $stats === null ) { 1309 return true; // not supported 1310 } 1311 1312 $latest = !empty( $params['latest'] ); // use latest data? 1313 foreach ( $stats as $path => $stat ) { 1314 $path = FileBackend::normalizeStoragePath( $path ); 1315 if ( $path === null ) { 1316 continue; // this shouldn't happen 1317 } 1318 if ( is_array( $stat ) ) { // file exists 1319 // Strongly consistent backends can automatically set "latest" 1320 $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; 1321 $this->cheapCache->set( $path, 'stat', $stat ); 1322 $this->setFileCache( $path, $stat ); // update persistent cache 1323 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata 1324 $this->cheapCache->set( $path, 'sha1', 1325 array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); 1326 } 1327 if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata 1328 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); 1329 $this->cheapCache->set( $path, 'xattr', 1330 array( 'map' => $stat['xattr'], 'latest' => $latest ) ); 1331 } 1332 } elseif ( $stat === false ) { // file does not exist 1333 $this->cheapCache->set( $path, 'stat', 1334 $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); 1335 $this->cheapCache->set( $path, 'xattr', 1336 array( 'map' => false, 'latest' => $latest ) ); 1337 $this->cheapCache->set( $path, 'sha1', 1338 array( 'hash' => false, 'latest' => $latest ) ); 1339 wfDebug( __METHOD__ . ": File $path does not exist.\n" ); 1340 } else { // an error occurred 1341 $success = false; 1342 wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); 1343 } 1344 } 1345 1346 return $success; 1347 } 1348 1349 /** 1350 * Get file stat information (concurrently if possible) for several files 1351 * 1352 * @see FileBackend::getFileStat() 1353 * 1354 * @param array $params Parameters include: 1355 * - srcs : list of source storage paths 1356 * - latest : use the latest available data 1357 * @return array|null Map of storage paths to array|bool|null (returns null if not supported) 1358 * @since 1.23 1359 */ 1360 protected function doGetFileStatMulti( array $params ) { 1361 return null; // not supported 1362 } 1363 1364 /** 1365 * Is this a key/value store where directories are just virtual? 1366 * Virtual directories exists in so much as files exists that are 1367 * prefixed with the directory path followed by a forward slash. 1368 * 1369 * @return bool 1370 */ 1371 abstract protected function directoriesAreVirtual(); 1372 1373 /** 1374 * Check if a container name is valid. 1375 * This checks for for length and illegal characters. 1376 * 1377 * @param string $container 1378 * @return bool 1379 */ 1380 final protected static function isValidContainerName( $container ) { 1381 // This accounts for Swift and S3 restrictions while leaving room 1382 // for things like '.xxx' (hex shard chars) or '.seg' (segments). 1383 // This disallows directory separators or traversal characters. 1384 // Note that matching strings URL encode to the same string; 1385 // in Swift, the length restriction is *after* URL encoding. 1386 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container ); 1387 } 1388 1389 /** 1390 * Splits a storage path into an internal container name, 1391 * an internal relative file name, and a container shard suffix. 1392 * Any shard suffix is already appended to the internal container name. 1393 * This also checks that the storage path is valid and within this backend. 1394 * 1395 * If the container is sharded but a suffix could not be determined, 1396 * this means that the path can only refer to a directory and can only 1397 * be scanned by looking in all the container shards. 1398 * 1399 * @param string $storagePath 1400 * @return array (container, path, container suffix) or (null, null, null) if invalid 1401 */ 1402 final protected function resolveStoragePath( $storagePath ) { 1403 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); 1404 if ( $backend === $this->name ) { // must be for this backend 1405 $relPath = self::normalizeContainerPath( $relPath ); 1406 if ( $relPath !== null ) { 1407 // Get shard for the normalized path if this container is sharded 1408 $cShard = $this->getContainerShard( $container, $relPath ); 1409 // Validate and sanitize the relative path (backend-specific) 1410 $relPath = $this->resolveContainerPath( $container, $relPath ); 1411 if ( $relPath !== null ) { 1412 // Prepend any wiki ID prefix to the container name 1413 $container = $this->fullContainerName( $container ); 1414 if ( self::isValidContainerName( $container ) ) { 1415 // Validate and sanitize the container name (backend-specific) 1416 $container = $this->resolveContainerName( "{$container}{$cShard}" ); 1417 if ( $container !== null ) { 1418 return array( $container, $relPath, $cShard ); 1419 } 1420 } 1421 } 1422 } 1423 } 1424 1425 return array( null, null, null ); 1426 } 1427 1428 /** 1429 * Like resolveStoragePath() except null values are returned if 1430 * the container is sharded and the shard could not be determined 1431 * or if the path ends with '/'. The later case is illegal for FS 1432 * backends and can confuse listings for object store backends. 1433 * 1434 * This function is used when resolving paths that must be valid 1435 * locations for files. Directory and listing functions should 1436 * generally just use resolveStoragePath() instead. 1437 * 1438 * @see FileBackendStore::resolveStoragePath() 1439 * 1440 * @param string $storagePath 1441 * @return array (container, path) or (null, null) if invalid 1442 */ 1443 final protected function resolveStoragePathReal( $storagePath ) { 1444 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); 1445 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { 1446 return array( $container, $relPath ); 1447 } 1448 1449 return array( null, null ); 1450 } 1451 1452 /** 1453 * Get the container name shard suffix for a given path. 1454 * Any empty suffix means the container is not sharded. 1455 * 1456 * @param string $container Container name 1457 * @param string $relPath Storage path relative to the container 1458 * @return string|null Returns null if shard could not be determined 1459 */ 1460 final protected function getContainerShard( $container, $relPath ) { 1461 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); 1462 if ( $levels == 1 || $levels == 2 ) { 1463 // Hash characters are either base 16 or 36 1464 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; 1465 // Get a regex that represents the shard portion of paths. 1466 // The concatenation of the captures gives us the shard. 1467 if ( $levels === 1 ) { // 16 or 36 shards per container 1468 $hashDirRegex = '(' . $char . ')'; 1469 } else { // 256 or 1296 shards per container 1470 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") 1471 $hashDirRegex = $char . '/(' . $char . '{2})'; 1472 } else { // short hash dir format (e.g. "a/b/c") 1473 $hashDirRegex = '(' . $char . ')/(' . $char . ')'; 1474 } 1475 } 1476 // Allow certain directories to be above the hash dirs so as 1477 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). 1478 // They must be 2+ chars to avoid any hash directory ambiguity. 1479 $m = array(); 1480 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { 1481 return '.' . implode( '', array_slice( $m, 1 ) ); 1482 } 1483 1484 return null; // failed to match 1485 } 1486 1487 return ''; // no sharding 1488 } 1489 1490 /** 1491 * Check if a storage path maps to a single shard. 1492 * Container dirs like "a", where the container shards on "x/xy", 1493 * can reside on several shards. Such paths are tricky to handle. 1494 * 1495 * @param string $storagePath Storage path 1496 * @return bool 1497 */ 1498 final public function isSingleShardPathInternal( $storagePath ) { 1499 list( , , $shard ) = $this->resolveStoragePath( $storagePath ); 1500 1501 return ( $shard !== null ); 1502 } 1503 1504 /** 1505 * Get the sharding config for a container. 1506 * If greater than 0, then all file storage paths within 1507 * the container are required to be hashed accordingly. 1508 * 1509 * @param string $container 1510 * @return array (integer levels, integer base, repeat flag) or (0, 0, false) 1511 */ 1512 final protected function getContainerHashLevels( $container ) { 1513 if ( isset( $this->shardViaHashLevels[$container] ) ) { 1514 $config = $this->shardViaHashLevels[$container]; 1515 $hashLevels = (int)$config['levels']; 1516 if ( $hashLevels == 1 || $hashLevels == 2 ) { 1517 $hashBase = (int)$config['base']; 1518 if ( $hashBase == 16 || $hashBase == 36 ) { 1519 return array( $hashLevels, $hashBase, $config['repeat'] ); 1520 } 1521 } 1522 } 1523 1524 return array( 0, 0, false ); // no sharding 1525 } 1526 1527 /** 1528 * Get a list of full container shard suffixes for a container 1529 * 1530 * @param string $container 1531 * @return array 1532 */ 1533 final protected function getContainerSuffixes( $container ) { 1534 $shards = array(); 1535 list( $digits, $base ) = $this->getContainerHashLevels( $container ); 1536 if ( $digits > 0 ) { 1537 $numShards = pow( $base, $digits ); 1538 for ( $index = 0; $index < $numShards; $index++ ) { 1539 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits ); 1540 } 1541 } 1542 1543 return $shards; 1544 } 1545 1546 /** 1547 * Get the full container name, including the wiki ID prefix 1548 * 1549 * @param string $container 1550 * @return string 1551 */ 1552 final protected function fullContainerName( $container ) { 1553 if ( $this->wikiId != '' ) { 1554 return "{$this->wikiId}-$container"; 1555 } else { 1556 return $container; 1557 } 1558 } 1559 1560 /** 1561 * Resolve a container name, checking if it's allowed by the backend. 1562 * This is intended for internal use, such as encoding illegal chars. 1563 * Subclasses can override this to be more restrictive. 1564 * 1565 * @param string $container 1566 * @return string|null 1567 */ 1568 protected function resolveContainerName( $container ) { 1569 return $container; 1570 } 1571 1572 /** 1573 * Resolve a relative storage path, checking if it's allowed by the backend. 1574 * This is intended for internal use, such as encoding illegal chars or perhaps 1575 * getting absolute paths (e.g. FS based backends). Note that the relative path 1576 * may be the empty string (e.g. the path is simply to the container). 1577 * 1578 * @param string $container Container name 1579 * @param string $relStoragePath Storage path relative to the container 1580 * @return string|null Path or null if not valid 1581 */ 1582 protected function resolveContainerPath( $container, $relStoragePath ) { 1583 return $relStoragePath; 1584 } 1585 1586 /** 1587 * Get the cache key for a container 1588 * 1589 * @param string $container Resolved container name 1590 * @return string 1591 */ 1592 private function containerCacheKey( $container ) { 1593 return "filebackend:{$this->name}:{$this->wikiId}:container:{$container}"; 1594 } 1595 1596 /** 1597 * Set the cached info for a container 1598 * 1599 * @param string $container Resolved container name 1600 * @param array $val Information to cache 1601 */ 1602 final protected function setContainerCache( $container, array $val ) { 1603 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 ); 1604 } 1605 1606 /** 1607 * Delete the cached info for a container. 1608 * The cache key is salted for a while to prevent race conditions. 1609 * 1610 * @param string $container Resolved container name 1611 */ 1612 final protected function deleteContainerCache( $container ) { 1613 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { 1614 trigger_error( "Unable to delete stat cache for container $container." ); 1615 } 1616 } 1617 1618 /** 1619 * Do a batch lookup from cache for container stats for all containers 1620 * used in a list of container names or storage paths objects. 1621 * This loads the persistent cache values into the process cache. 1622 * 1623 * @param array $items 1624 */ 1625 final protected function primeContainerCache( array $items ) { 1626 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1627 1628 $paths = array(); // list of storage paths 1629 $contNames = array(); // (cache key => resolved container name) 1630 // Get all the paths/containers from the items... 1631 foreach ( $items as $item ) { 1632 if ( self::isStoragePath( $item ) ) { 1633 $paths[] = $item; 1634 } elseif ( is_string( $item ) ) { // full container name 1635 $contNames[$this->containerCacheKey( $item )] = $item; 1636 } 1637 } 1638 // Get all the corresponding cache keys for paths... 1639 foreach ( $paths as $path ) { 1640 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 1641 if ( $fullCont !== null ) { // valid path for this backend 1642 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; 1643 } 1644 } 1645 1646 $contInfo = array(); // (resolved container name => cache value) 1647 // Get all cache entries for these container cache keys... 1648 $values = $this->memCache->getMulti( array_keys( $contNames ) ); 1649 foreach ( $values as $cacheKey => $val ) { 1650 $contInfo[$contNames[$cacheKey]] = $val; 1651 } 1652 1653 // Populate the container process cache for the backend... 1654 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); 1655 } 1656 1657 /** 1658 * Fill the backend-specific process cache given an array of 1659 * resolved container names and their corresponding cached info. 1660 * Only containers that actually exist should appear in the map. 1661 * 1662 * @param array $containerInfo Map of resolved container names to cached info 1663 */ 1664 protected function doPrimeContainerCache( array $containerInfo ) { 1665 } 1666 1667 /** 1668 * Get the cache key for a file path 1669 * 1670 * @param string $path Normalized storage path 1671 * @return string 1672 */ 1673 private function fileCacheKey( $path ) { 1674 return "filebackend:{$this->name}:{$this->wikiId}:file:" . sha1( $path ); 1675 } 1676 1677 /** 1678 * Set the cached stat info for a file path. 1679 * Negatives (404s) are not cached. By not caching negatives, we can skip cache 1680 * salting for the case when a file is created at a path were there was none before. 1681 * 1682 * @param string $path Storage path 1683 * @param array $val Stat information to cache 1684 */ 1685 final protected function setFileCache( $path, array $val ) { 1686 $path = FileBackend::normalizeStoragePath( $path ); 1687 if ( $path === null ) { 1688 return; // invalid storage path 1689 } 1690 $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); 1691 $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); 1692 $key = $this->fileCacheKey( $path ); 1693 // Set the cache unless it is currently salted with the value "PURGED". 1694 // Using add() handles this except it also is a no-op in that case where 1695 // the current value is not "latest" but $val is, so use CAS in that case. 1696 if ( !$this->memCache->add( $key, $val, $ttl ) && !empty( $val['latest'] ) ) { 1697 $this->memCache->merge( 1698 $key, 1699 function ( BagOStuff $cache, $key, $cValue ) use ( $val ) { 1700 return ( is_array( $cValue ) && empty( $cValue['latest'] ) ) 1701 ? $val // update the stat cache with the lastest info 1702 : false; // do nothing (cache is salted or some error happened) 1703 }, 1704 $ttl, 1705 1 1706 ); 1707 } 1708 } 1709 1710 /** 1711 * Delete the cached stat info for a file path. 1712 * The cache key is salted for a while to prevent race conditions. 1713 * Since negatives (404s) are not cached, this does not need to be called when 1714 * a file is created at a path were there was none before. 1715 * 1716 * @param string $path Storage path 1717 */ 1718 final protected function deleteFileCache( $path ) { 1719 $path = FileBackend::normalizeStoragePath( $path ); 1720 if ( $path === null ) { 1721 return; // invalid storage path 1722 } 1723 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) { 1724 trigger_error( "Unable to delete stat cache for file $path." ); 1725 } 1726 } 1727 1728 /** 1729 * Do a batch lookup from cache for file stats for all paths 1730 * used in a list of storage paths or FileOp objects. 1731 * This loads the persistent cache values into the process cache. 1732 * 1733 * @param array $items List of storage paths 1734 */ 1735 final protected function primeFileCache( array $items ) { 1736 $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); 1737 1738 $paths = array(); // list of storage paths 1739 $pathNames = array(); // (cache key => storage path) 1740 // Get all the paths/containers from the items... 1741 foreach ( $items as $item ) { 1742 if ( self::isStoragePath( $item ) ) { 1743 $paths[] = FileBackend::normalizeStoragePath( $item ); 1744 } 1745 } 1746 // Get rid of any paths that failed normalization... 1747 $paths = array_filter( $paths, 'strlen' ); // remove nulls 1748 // Get all the corresponding cache keys for paths... 1749 foreach ( $paths as $path ) { 1750 list( , $rel, ) = $this->resolveStoragePath( $path ); 1751 if ( $rel !== null ) { // valid path for this backend 1752 $pathNames[$this->fileCacheKey( $path )] = $path; 1753 } 1754 } 1755 // Get all cache entries for these container cache keys... 1756 $values = $this->memCache->getMulti( array_keys( $pathNames ) ); 1757 foreach ( $values as $cacheKey => $val ) { 1758 if ( is_array( $val ) ) { 1759 $path = $pathNames[$cacheKey]; 1760 $this->cheapCache->set( $path, 'stat', $val ); 1761 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata 1762 $this->cheapCache->set( $path, 'sha1', 1763 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) ); 1764 } 1765 if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata 1766 $val['xattr'] = self::normalizeXAttributes( $val['xattr'] ); 1767 $this->cheapCache->set( $path, 'xattr', 1768 array( 'map' => $val['xattr'], 'latest' => $val['latest'] ) ); 1769 } 1770 } 1771 } 1772 } 1773 1774 /** 1775 * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format 1776 * 1777 * @param array $xattr 1778 * @return array 1779 * @since 1.22 1780 */ 1781 final protected static function normalizeXAttributes( array $xattr ) { 1782 $newXAttr = array( 'headers' => array(), 'metadata' => array() ); 1783 1784 foreach ( $xattr['headers'] as $name => $value ) { 1785 $newXAttr['headers'][strtolower( $name )] = $value; 1786 } 1787 1788 foreach ( $xattr['metadata'] as $name => $value ) { 1789 $newXAttr['metadata'][strtolower( $name )] = $value; 1790 } 1791 1792 return $newXAttr; 1793 } 1794 1795 /** 1796 * Set the 'concurrency' option from a list of operation options 1797 * 1798 * @param array $opts Map of operation options 1799 * @return array 1800 */ 1801 final protected function setConcurrencyFlags( array $opts ) { 1802 $opts['concurrency'] = 1; // off 1803 if ( $this->parallelize === 'implicit' ) { 1804 if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) { 1805 $opts['concurrency'] = $this->concurrency; 1806 } 1807 } elseif ( $this->parallelize === 'explicit' ) { 1808 if ( !empty( $opts['parallelize'] ) ) { 1809 $opts['concurrency'] = $this->concurrency; 1810 } 1811 } 1812 1813 return $opts; 1814 } 1815 1816 /** 1817 * Get the content type to use in HEAD/GET requests for a file 1818 * 1819 * @param string $storagePath 1820 * @param string|null $content File data 1821 * @param string|null $fsPath File system path 1822 * @return string MIME type 1823 */ 1824 protected function getContentType( $storagePath, $content, $fsPath ) { 1825 return call_user_func_array( $this->mimeCallback, func_get_args() ); 1826 } 1827 } 1828 1829 /** 1830 * FileBackendStore helper class for performing asynchronous file operations. 1831 * 1832 * For example, calling FileBackendStore::createInternal() with the "async" 1833 * param flag may result in a Status that contains this object as a value. 1834 * This class is largely backend-specific and is mostly just "magic" to be 1835 * passed to FileBackendStore::executeOpHandlesInternal(). 1836 */ 1837 abstract class FileBackendStoreOpHandle { 1838 /** @var array */ 1839 public $params = array(); // params to caller functions 1840 /** @var FileBackendStore */ 1841 public $backend; 1842 /** @var array */ 1843 public $resourcesToClose = array(); 1844 1845 public $call; // string; name that identifies the function called 1846 1847 /** 1848 * Close all open file handles 1849 */ 1850 public function closeResources() { 1851 array_map( 'fclose', $this->resourcesToClose ); 1852 } 1853 } 1854 1855 /** 1856 * FileBackendStore helper function to handle listings that span container shards. 1857 * Do not use this class from places outside of FileBackendStore. 1858 * 1859 * @ingroup FileBackend 1860 */ 1861 abstract class FileBackendStoreShardListIterator extends FilterIterator { 1862 /** @var FileBackendStore */ 1863 protected $backend; 1864 1865 /** @var array */ 1866 protected $params; 1867 1868 /** @var string Full container name */ 1869 protected $container; 1870 1871 /** @var string Resolved relative path */ 1872 protected $directory; 1873 1874 /** @var array */ 1875 protected $multiShardPaths = array(); // (rel path => 1) 1876 1877 /** 1878 * @param FileBackendStore $backend 1879 * @param string $container Full storage container name 1880 * @param string $dir Storage directory relative to container 1881 * @param array $suffixes List of container shard suffixes 1882 * @param array $params 1883 */ 1884 public function __construct( 1885 FileBackendStore $backend, $container, $dir, array $suffixes, array $params 1886 ) { 1887 $this->backend = $backend; 1888 $this->container = $container; 1889 $this->directory = $dir; 1890 $this->params = $params; 1891 1892 $iter = new AppendIterator(); 1893 foreach ( $suffixes as $suffix ) { 1894 $iter->append( $this->listFromShard( $this->container . $suffix ) ); 1895 } 1896 1897 parent::__construct( $iter ); 1898 } 1899 1900 public function accept() { 1901 $rel = $this->getInnerIterator()->current(); // path relative to given directory 1902 $path = $this->params['dir'] . "/{$rel}"; // full storage path 1903 if ( $this->backend->isSingleShardPathInternal( $path ) ) { 1904 return true; // path is only on one shard; no issue with duplicates 1905 } elseif ( isset( $this->multiShardPaths[$rel] ) ) { 1906 // Don't keep listing paths that are on multiple shards 1907 return false; 1908 } else { 1909 $this->multiShardPaths[$rel] = 1; 1910 1911 return true; 1912 } 1913 } 1914 1915 public function rewind() { 1916 parent::rewind(); 1917 $this->multiShardPaths = array(); 1918 } 1919 1920 /** 1921 * Get the list for a given container shard 1922 * 1923 * @param string $container Resolved container name 1924 * @return Iterator 1925 */ 1926 abstract protected function listFromShard( $container ); 1927 } 1928 1929 /** 1930 * Iterator for listing directories 1931 */ 1932 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { 1933 protected function listFromShard( $container ) { 1934 $list = $this->backend->getDirectoryListInternal( 1935 $container, $this->directory, $this->params ); 1936 if ( $list === null ) { 1937 return new ArrayIterator( array() ); 1938 } else { 1939 return is_array( $list ) ? new ArrayIterator( $list ) : $list; 1940 } 1941 } 1942 } 1943 1944 /** 1945 * Iterator for listing regular files 1946 */ 1947 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { 1948 protected function listFromShard( $container ) { 1949 $list = $this->backend->getFileListInternal( 1950 $container, $this->directory, $this->params ); 1951 if ( $list === null ) { 1952 return new ArrayIterator( array() ); 1953 } else { 1954 return is_array( $list ) ? new ArrayIterator( $list ) : $list; 1955 } 1956 } 1957 }
title
Description
Body
title
Description
Body
title
Description
Body
title
Body
| Generated: Fri Nov 28 14:03:12 2014 | Cross-referenced by PHPXref 0.7.1 |