| [ Index ] |
PHP Cross Reference of MediaWiki-1.24.0 |
[Summary view] [Print] [Text view]
1 <?php 2 /** 3 * OpenStack Swift based file backend. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program; if not, write to the Free Software Foundation, Inc., 17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 * http://www.gnu.org/copyleft/gpl.html 19 * 20 * @file 21 * @ingroup FileBackend 22 * @author Russ Nelson 23 * @author Aaron Schulz 24 */ 25 26 /** 27 * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. 28 * 29 * Status messages should avoid mentioning the Swift account name. 30 * Likewise, error suppression should be used to avoid path disclosure. 31 * 32 * @ingroup FileBackend 33 * @since 1.19 34 */ 35 class SwiftFileBackend extends FileBackendStore { 36 /** @var MultiHttpClient */ 37 protected $http; 38 39 /** @var int TTL in seconds */ 40 protected $authTTL; 41 42 /** @var string Authentication base URL (without version) */ 43 protected $swiftAuthUrl; 44 45 /** @var string Swift user (account:user) to authenticate as */ 46 protected $swiftUser; 47 48 /** @var string Secret key for user */ 49 protected $swiftKey; 50 51 /** @var string Shared secret value for making temp URLs */ 52 protected $swiftTempUrlKey; 53 54 /** @var string S3 access key (RADOS Gateway) */ 55 protected $rgwS3AccessKey; 56 57 /** @var string S3 authentication key (RADOS Gateway) */ 58 protected $rgwS3SecretKey; 59 60 /** @var BagOStuff */ 61 protected $srvCache; 62 63 /** @var ProcessCacheLRU Container stat cache */ 64 protected $containerStatCache; 65 66 /** @var array */ 67 protected $authCreds; 68 69 /** @var int UNIX timestamp */ 70 protected $authSessionTimestamp = 0; 71 72 /** @var int UNIX timestamp */ 73 protected $authErrorTimestamp = null; 74 75 /** @var bool Whether the server is an Ceph RGW */ 76 protected $isRGW = false; 77 78 /** 79 * @see FileBackendStore::__construct() 80 * Additional $config params include: 81 * - swiftAuthUrl : Swift authentication server URL 82 * - swiftUser : Swift user used by MediaWiki (account:username) 83 * - swiftKey : Swift authentication key for the above user 84 * - swiftAuthTTL : Swift authentication TTL (seconds) 85 * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account. 86 * Do not set this until it has been set in the backend. 87 * - shardViaHashLevels : Map of container names to sharding config with: 88 * - base : base of hash characters, 16 or 36 89 * - levels : the number of hash levels (and digits) 90 * - repeat : hash subdirectories are prefixed with all the 91 * parent hash directory names (e.g. "a/ab/abc") 92 * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect. 93 * If those are not available, then the main cache will be used. 94 * This is probably insecure in shared hosting environments. 95 * - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account. 96 * Do not set this until it has been set in the backend. 97 * This is used for generating expiring pre-authenticated URLs. 98 * Only use this when using rgw and to work around 99 * http://tracker.newdream.net/issues/3454. 100 * - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account. 101 * Do not set this until it has been set in the backend. 102 * This is used for generating expiring pre-authenticated URLs. 103 * Only use this when using rgw and to work around 104 * http://tracker.newdream.net/issues/3454. 105 */ 106 public function __construct( array $config ) { 107 parent::__construct( $config ); 108 // Required settings 109 $this->swiftAuthUrl = $config['swiftAuthUrl']; 110 $this->swiftUser = $config['swiftUser']; 111 $this->swiftKey = $config['swiftKey']; 112 // Optional settings 113 $this->authTTL = isset( $config['swiftAuthTTL'] ) 114 ? $config['swiftAuthTTL'] 115 : 5 * 60; // some sane number 116 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) 117 ? $config['swiftTempUrlKey'] 118 : ''; 119 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) 120 ? $config['shardViaHashLevels'] 121 : ''; 122 $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) 123 ? $config['rgwS3AccessKey'] 124 : ''; 125 $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) 126 ? $config['rgwS3SecretKey'] 127 : ''; 128 // HTTP helper client 129 $this->http = new MultiHttpClient( array() ); 130 // Cache container information to mask latency 131 $this->memCache = wfGetMainCache(); 132 // Process cache for container info 133 $this->containerStatCache = new ProcessCacheLRU( 300 ); 134 // Cache auth token information to avoid RTTs 135 if ( !empty( $config['cacheAuthInfo'] ) ) { 136 if ( PHP_SAPI === 'cli' ) { 137 $this->srvCache = wfGetMainCache(); // preferrably memcached 138 } else { 139 try { // look for APC, XCache, WinCache, ect... 140 $this->srvCache = ObjectCache::newAccelerator( array() ); 141 } catch ( Exception $e ) { 142 } 143 } 144 } 145 $this->srvCache = $this->srvCache ?: new EmptyBagOStuff(); 146 } 147 148 public function getFeatures() { 149 return ( FileBackend::ATTR_UNICODE_PATHS | 150 FileBackend::ATTR_HEADERS | FileBackend::ATTR_METADATA ); 151 } 152 153 protected function resolveContainerPath( $container, $relStoragePath ) { 154 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF 155 return null; // not UTF-8, makes it hard to use CF and the swift HTTP API 156 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { 157 return null; // too long for Swift 158 } 159 160 return $relStoragePath; 161 } 162 163 public function isPathUsableInternal( $storagePath ) { 164 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); 165 if ( $rel === null ) { 166 return false; // invalid 167 } 168 169 return is_array( $this->getContainerStat( $container ) ); 170 } 171 172 /** 173 * Sanitize and filter the custom headers from a $params array. 174 * We only allow certain Content- and X-Content- headers. 175 * 176 * @param array $params 177 * @return array Sanitized value of 'headers' field in $params 178 */ 179 protected function sanitizeHdrs( array $params ) { 180 $headers = array(); 181 182 // Normalize casing, and strip out illegal headers 183 if ( isset( $params['headers'] ) ) { 184 foreach ( $params['headers'] as $name => $value ) { 185 $name = strtolower( $name ); 186 if ( preg_match( '/^content-(type|length)$/', $name ) ) { 187 continue; // blacklisted 188 } elseif ( preg_match( '/^(x-)?content-/', $name ) ) { 189 $headers[$name] = $value; // allowed 190 } elseif ( preg_match( '/^content-(disposition)/', $name ) ) { 191 $headers[$name] = $value; // allowed 192 } 193 } 194 } 195 // By default, Swift has annoyingly low maximum header value limits 196 if ( isset( $headers['content-disposition'] ) ) { 197 $disposition = ''; 198 foreach ( explode( ';', $headers['content-disposition'] ) as $part ) { 199 $part = trim( $part ); 200 $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}"; 201 if ( strlen( $new ) <= 255 ) { 202 $disposition = $new; 203 } else { 204 break; // too long; sigh 205 } 206 } 207 $headers['content-disposition'] = $disposition; 208 } 209 210 return $headers; 211 } 212 213 protected function doCreateInternal( array $params ) { 214 $status = Status::newGood(); 215 216 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 217 if ( $dstRel === null ) { 218 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 219 220 return $status; 221 } 222 223 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 ); 224 $contentType = $this->getContentType( $params['dst'], $params['content'], null ); 225 226 $reqs = array( array( 227 'method' => 'PUT', 228 'url' => array( $dstCont, $dstRel ), 229 'headers' => array( 230 'content-length' => strlen( $params['content'] ), 231 'etag' => md5( $params['content'] ), 232 'content-type' => $contentType, 233 'x-object-meta-sha1base36' => $sha1Hash 234 ) + $this->sanitizeHdrs( $params ), 235 'body' => $params['content'] 236 ) ); 237 238 $be = $this; 239 $method = __METHOD__; 240 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 241 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 242 if ( $rcode === 201 ) { 243 // good 244 } elseif ( $rcode === 412 ) { 245 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 246 } else { 247 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 248 } 249 }; 250 251 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 252 if ( !empty( $params['async'] ) ) { // deferred 253 $status->value = $opHandle; 254 } else { // actually write the object in Swift 255 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 256 } 257 258 return $status; 259 } 260 261 protected function doStoreInternal( array $params ) { 262 $status = Status::newGood(); 263 264 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 265 if ( $dstRel === null ) { 266 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 267 268 return $status; 269 } 270 271 wfSuppressWarnings(); 272 $sha1Hash = sha1_file( $params['src'] ); 273 wfRestoreWarnings(); 274 if ( $sha1Hash === false ) { // source doesn't exist? 275 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); 276 277 return $status; 278 } 279 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 ); 280 $contentType = $this->getContentType( $params['dst'], null, $params['src'] ); 281 282 $handle = fopen( $params['src'], 'rb' ); 283 if ( $handle === false ) { // source doesn't exist? 284 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); 285 286 return $status; 287 } 288 289 $reqs = array( array( 290 'method' => 'PUT', 291 'url' => array( $dstCont, $dstRel ), 292 'headers' => array( 293 'content-length' => filesize( $params['src'] ), 294 'etag' => md5_file( $params['src'] ), 295 'content-type' => $contentType, 296 'x-object-meta-sha1base36' => $sha1Hash 297 ) + $this->sanitizeHdrs( $params ), 298 'body' => $handle // resource 299 ) ); 300 301 $be = $this; 302 $method = __METHOD__; 303 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 304 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 305 if ( $rcode === 201 ) { 306 // good 307 } elseif ( $rcode === 412 ) { 308 $status->fatal( 'backend-fail-contenttype', $params['dst'] ); 309 } else { 310 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 311 } 312 }; 313 314 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 315 if ( !empty( $params['async'] ) ) { // deferred 316 $status->value = $opHandle; 317 } else { // actually write the object in Swift 318 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 319 } 320 321 return $status; 322 } 323 324 protected function doCopyInternal( array $params ) { 325 $status = Status::newGood(); 326 327 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 328 if ( $srcRel === null ) { 329 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 330 331 return $status; 332 } 333 334 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 335 if ( $dstRel === null ) { 336 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 337 338 return $status; 339 } 340 341 $reqs = array( array( 342 'method' => 'PUT', 343 'url' => array( $dstCont, $dstRel ), 344 'headers' => array( 345 'x-copy-from' => '/' . rawurlencode( $srcCont ) . 346 '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) 347 ) + $this->sanitizeHdrs( $params ), // extra headers merged into object 348 ) ); 349 350 $be = $this; 351 $method = __METHOD__; 352 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 353 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 354 if ( $rcode === 201 ) { 355 // good 356 } elseif ( $rcode === 404 ) { 357 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); 358 } else { 359 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 360 } 361 }; 362 363 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 364 if ( !empty( $params['async'] ) ) { // deferred 365 $status->value = $opHandle; 366 } else { // actually write the object in Swift 367 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 368 } 369 370 return $status; 371 } 372 373 protected function doMoveInternal( array $params ) { 374 $status = Status::newGood(); 375 376 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 377 if ( $srcRel === null ) { 378 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 379 380 return $status; 381 } 382 383 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); 384 if ( $dstRel === null ) { 385 $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); 386 387 return $status; 388 } 389 390 $reqs = array( 391 array( 392 'method' => 'PUT', 393 'url' => array( $dstCont, $dstRel ), 394 'headers' => array( 395 'x-copy-from' => '/' . rawurlencode( $srcCont ) . 396 '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) 397 ) + $this->sanitizeHdrs( $params ) // extra headers merged into object 398 ) 399 ); 400 if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) { 401 $reqs[] = array( 402 'method' => 'DELETE', 403 'url' => array( $srcCont, $srcRel ), 404 'headers' => array() 405 ); 406 } 407 408 $be = $this; 409 $method = __METHOD__; 410 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 411 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 412 if ( $request['method'] === 'PUT' && $rcode === 201 ) { 413 // good 414 } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) { 415 // good 416 } elseif ( $rcode === 404 ) { 417 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); 418 } else { 419 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 420 } 421 }; 422 423 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 424 if ( !empty( $params['async'] ) ) { // deferred 425 $status->value = $opHandle; 426 } else { // actually move the object in Swift 427 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 428 } 429 430 return $status; 431 } 432 433 protected function doDeleteInternal( array $params ) { 434 $status = Status::newGood(); 435 436 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 437 if ( $srcRel === null ) { 438 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 439 440 return $status; 441 } 442 443 $reqs = array( array( 444 'method' => 'DELETE', 445 'url' => array( $srcCont, $srcRel ), 446 'headers' => array() 447 ) ); 448 449 $be = $this; 450 $method = __METHOD__; 451 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 452 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 453 if ( $rcode === 204 ) { 454 // good 455 } elseif ( $rcode === 404 ) { 456 if ( empty( $params['ignoreMissingSource'] ) ) { 457 $status->fatal( 'backend-fail-delete', $params['src'] ); 458 } 459 } else { 460 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 461 } 462 }; 463 464 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 465 if ( !empty( $params['async'] ) ) { // deferred 466 $status->value = $opHandle; 467 } else { // actually delete the object in Swift 468 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 469 } 470 471 return $status; 472 } 473 474 protected function doDescribeInternal( array $params ) { 475 $status = Status::newGood(); 476 477 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 478 if ( $srcRel === null ) { 479 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 480 481 return $status; 482 } 483 484 // Fetch the old object headers/metadata...this should be in stat cache by now 485 $stat = $this->getFileStat( array( 'src' => $params['src'], 'latest' => 1 ) ); 486 if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry 487 $stat = $this->doGetFileStat( array( 'src' => $params['src'], 'latest' => 1 ) ); 488 } 489 if ( !$stat ) { 490 $status->fatal( 'backend-fail-describe', $params['src'] ); 491 492 return $status; 493 } 494 495 // POST clears prior headers, so we need to merge the changes in to the old ones 496 $metaHdrs = array(); 497 foreach ( $stat['xattr']['metadata'] as $name => $value ) { 498 $metaHdrs["x-object-meta-$name"] = $value; 499 } 500 $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers']; 501 502 $reqs = array( array( 503 'method' => 'POST', 504 'url' => array( $srcCont, $srcRel ), 505 'headers' => $metaHdrs + $customHdrs 506 ) ); 507 508 $be = $this; 509 $method = __METHOD__; 510 $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { 511 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; 512 if ( $rcode === 202 ) { 513 // good 514 } elseif ( $rcode === 404 ) { 515 $status->fatal( 'backend-fail-describe', $params['src'] ); 516 } else { 517 $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); 518 } 519 }; 520 521 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); 522 if ( !empty( $params['async'] ) ) { // deferred 523 $status->value = $opHandle; 524 } else { // actually change the object in Swift 525 $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); 526 } 527 528 return $status; 529 } 530 531 protected function doPrepareInternal( $fullCont, $dir, array $params ) { 532 $status = Status::newGood(); 533 534 // (a) Check if container already exists 535 $stat = $this->getContainerStat( $fullCont ); 536 if ( is_array( $stat ) ) { 537 return $status; // already there 538 } elseif ( $stat === null ) { 539 $status->fatal( 'backend-fail-internal', $this->name ); 540 541 return $status; 542 } 543 544 // (b) Create container as needed with proper ACLs 545 if ( $stat === false ) { 546 $params['op'] = 'prepare'; 547 $status->merge( $this->createContainer( $fullCont, $params ) ); 548 } 549 550 return $status; 551 } 552 553 protected function doSecureInternal( $fullCont, $dir, array $params ) { 554 $status = Status::newGood(); 555 if ( empty( $params['noAccess'] ) ) { 556 return $status; // nothing to do 557 } 558 559 $stat = $this->getContainerStat( $fullCont ); 560 if ( is_array( $stat ) ) { 561 // Make container private to end-users... 562 $status->merge( $this->setContainerAccess( 563 $fullCont, 564 array( $this->swiftUser ), // read 565 array( $this->swiftUser ) // write 566 ) ); 567 } elseif ( $stat === false ) { 568 $status->fatal( 'backend-fail-usable', $params['dir'] ); 569 } else { 570 $status->fatal( 'backend-fail-internal', $this->name ); 571 } 572 573 return $status; 574 } 575 576 protected function doPublishInternal( $fullCont, $dir, array $params ) { 577 $status = Status::newGood(); 578 579 $stat = $this->getContainerStat( $fullCont ); 580 if ( is_array( $stat ) ) { 581 // Make container public to end-users... 582 $status->merge( $this->setContainerAccess( 583 $fullCont, 584 array( $this->swiftUser, '.r:*' ), // read 585 array( $this->swiftUser ) // write 586 ) ); 587 } elseif ( $stat === false ) { 588 $status->fatal( 'backend-fail-usable', $params['dir'] ); 589 } else { 590 $status->fatal( 'backend-fail-internal', $this->name ); 591 } 592 593 return $status; 594 } 595 596 protected function doCleanInternal( $fullCont, $dir, array $params ) { 597 $status = Status::newGood(); 598 599 // Only containers themselves can be removed, all else is virtual 600 if ( $dir != '' ) { 601 return $status; // nothing to do 602 } 603 604 // (a) Check the container 605 $stat = $this->getContainerStat( $fullCont, true ); 606 if ( $stat === false ) { 607 return $status; // ok, nothing to do 608 } elseif ( !is_array( $stat ) ) { 609 $status->fatal( 'backend-fail-internal', $this->name ); 610 611 return $status; 612 } 613 614 // (b) Delete the container if empty 615 if ( $stat['count'] == 0 ) { 616 $params['op'] = 'clean'; 617 $status->merge( $this->deleteContainer( $fullCont, $params ) ); 618 } 619 620 return $status; 621 } 622 623 protected function doGetFileStat( array $params ) { 624 $params = array( 'srcs' => array( $params['src'] ), 'concurrency' => 1 ) + $params; 625 unset( $params['src'] ); 626 $stats = $this->doGetFileStatMulti( $params ); 627 628 return reset( $stats ); 629 } 630 631 /** 632 * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". 633 * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings, 634 * missing the timezone suffix (though Ceph RGW does not appear to have this bug). 635 * 636 * @param string $ts 637 * @param int $format Output format (TS_* constant) 638 * @return string 639 * @throws FileBackendError 640 */ 641 protected function convertSwiftDate( $ts, $format = TS_MW ) { 642 try { 643 $timestamp = new MWTimestamp( $ts ); 644 645 return $timestamp->getTimestamp( $format ); 646 } catch ( MWException $e ) { 647 throw new FileBackendError( $e->getMessage() ); 648 } 649 } 650 651 /** 652 * Fill in any missing object metadata and save it to Swift 653 * 654 * @param array $objHdrs Object response headers 655 * @param string $path Storage path to object 656 * @return array New headers 657 */ 658 protected function addMissingMetadata( array $objHdrs, $path ) { 659 if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) { 660 return $objHdrs; // nothing to do 661 } 662 663 $section = new ProfileSection( __METHOD__ . '-' . $this->name ); 664 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); 665 666 $auth = $this->getAuthentication(); 667 if ( !$auth ) { 668 $objHdrs['x-object-meta-sha1base36'] = false; 669 670 return $objHdrs; // failed 671 } 672 673 $status = Status::newGood(); 674 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); 675 if ( $status->isOK() ) { 676 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); 677 if ( $tmpFile ) { 678 $hash = $tmpFile->getSha1Base36(); 679 if ( $hash !== false ) { 680 $objHdrs['x-object-meta-sha1base36'] = $hash; 681 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 682 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 683 'method' => 'POST', 684 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), 685 'headers' => $this->authTokenHeaders( $auth ) + $objHdrs 686 ) ); 687 if ( $rcode >= 200 && $rcode <= 299 ) { 688 return $objHdrs; // success 689 } 690 } 691 } 692 } 693 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); 694 $objHdrs['x-object-meta-sha1base36'] = false; 695 696 return $objHdrs; // failed 697 } 698 699 protected function doGetFileContentsMulti( array $params ) { 700 $contents = array(); 701 702 $auth = $this->getAuthentication(); 703 704 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging 705 // Blindly create tmp files and stream to them, catching any exception if the file does 706 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). 707 $reqs = array(); // (path => op) 708 709 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch 710 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 711 if ( $srcRel === null || !$auth ) { 712 $contents[$path] = false; 713 continue; 714 } 715 // Create a new temporary memory file... 716 $handle = fopen( 'php://temp', 'wb' ); 717 if ( $handle ) { 718 $reqs[$path] = array( 719 'method' => 'GET', 720 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), 721 'headers' => $this->authTokenHeaders( $auth ) 722 + $this->headersFromParams( $params ), 723 'stream' => $handle, 724 ); 725 } 726 $contents[$path] = false; 727 } 728 729 $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); 730 $reqs = $this->http->runMulti( $reqs, $opts ); 731 foreach ( $reqs as $path => $op ) { 732 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; 733 if ( $rcode >= 200 && $rcode <= 299 ) { 734 rewind( $op['stream'] ); // start from the beginning 735 $contents[$path] = stream_get_contents( $op['stream'] ); 736 } elseif ( $rcode === 404 ) { 737 $contents[$path] = false; 738 } else { 739 $this->onError( null, __METHOD__, 740 array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); 741 } 742 fclose( $op['stream'] ); // close open handle 743 } 744 745 return $contents; 746 } 747 748 protected function doDirectoryExists( $fullCont, $dir, array $params ) { 749 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 750 $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix ); 751 if ( $status->isOk() ) { 752 return ( count( $status->value ) ) > 0; 753 } 754 755 return null; // error 756 } 757 758 /** 759 * @see FileBackendStore::getDirectoryListInternal() 760 * @param string $fullCont 761 * @param string $dir 762 * @param array $params 763 * @return SwiftFileBackendDirList 764 */ 765 public function getDirectoryListInternal( $fullCont, $dir, array $params ) { 766 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params ); 767 } 768 769 /** 770 * @see FileBackendStore::getFileListInternal() 771 * @param string $fullCont 772 * @param string $dir 773 * @param array $params 774 * @return SwiftFileBackendFileList 775 */ 776 public function getFileListInternal( $fullCont, $dir, array $params ) { 777 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params ); 778 } 779 780 /** 781 * Do not call this function outside of SwiftFileBackendFileList 782 * 783 * @param string $fullCont Resolved container name 784 * @param string $dir Resolved storage directory with no trailing slash 785 * @param string|null $after Resolved container relative path to list items after 786 * @param int $limit Max number of items to list 787 * @param array $params Parameters for getDirectoryList() 788 * @return array List of container relative resolved paths of directories directly under $dir 789 * @throws FileBackendError 790 */ 791 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { 792 $dirs = array(); 793 if ( $after === INF ) { 794 return $dirs; // nothing more 795 } 796 797 $section = new ProfileSection( __METHOD__ . '-' . $this->name ); 798 799 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 800 // Non-recursive: only list dirs right under $dir 801 if ( !empty( $params['topOnly'] ) ) { 802 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); 803 if ( !$status->isOk() ) { 804 return $dirs; // error 805 } 806 $objects = $status->value; 807 foreach ( $objects as $object ) { // files and directories 808 if ( substr( $object, -1 ) === '/' ) { 809 $dirs[] = $object; // directories end in '/' 810 } 811 } 812 } else { 813 // Recursive: list all dirs under $dir and its subdirs 814 $getParentDir = function ( $path ) { 815 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; 816 }; 817 818 // Get directory from last item of prior page 819 $lastDir = $getParentDir( $after ); // must be first page 820 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); 821 822 if ( !$status->isOk() ) { 823 return $dirs; // error 824 } 825 826 $objects = $status->value; 827 828 foreach ( $objects as $object ) { // files 829 $objectDir = $getParentDir( $object ); // directory of object 830 831 if ( $objectDir !== false && $objectDir !== $dir ) { 832 // Swift stores paths in UTF-8, using binary sorting. 833 // See function "create_container_table" in common/db.py. 834 // If a directory is not "greater" than the last one, 835 // then it was already listed by the calling iterator. 836 if ( strcmp( $objectDir, $lastDir ) > 0 ) { 837 $pDir = $objectDir; 838 do { // add dir and all its parent dirs 839 $dirs[] = "{$pDir}/"; 840 $pDir = $getParentDir( $pDir ); 841 } while ( $pDir !== false // sanity 842 && strcmp( $pDir, $lastDir ) > 0 // not done already 843 && strlen( $pDir ) > strlen( $dir ) // within $dir 844 ); 845 } 846 $lastDir = $objectDir; 847 } 848 } 849 } 850 // Page on the unfiltered directory listing (what is returned may be filtered) 851 if ( count( $objects ) < $limit ) { 852 $after = INF; // avoid a second RTT 853 } else { 854 $after = end( $objects ); // update last item 855 } 856 857 return $dirs; 858 } 859 860 /** 861 * Do not call this function outside of SwiftFileBackendFileList 862 * 863 * @param string $fullCont Resolved container name 864 * @param string $dir Resolved storage directory with no trailing slash 865 * @param string|null $after Resolved container relative path of file to list items after 866 * @param int $limit Max number of items to list 867 * @param array $params Parameters for getDirectoryList() 868 * @return array List of resolved container relative paths of files under $dir 869 * @throws FileBackendError 870 */ 871 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { 872 $files = array(); // list of (path, stat array or null) entries 873 if ( $after === INF ) { 874 return $files; // nothing more 875 } 876 877 $section = new ProfileSection( __METHOD__ . '-' . $this->name ); 878 879 $prefix = ( $dir == '' ) ? null : "{$dir}/"; 880 // $objects will contain a list of unfiltered names or CF_Object items 881 // Non-recursive: only list files right under $dir 882 if ( !empty( $params['topOnly'] ) ) { 883 if ( !empty( $params['adviseStat'] ) ) { 884 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' ); 885 } else { 886 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); 887 } 888 } else { 889 // Recursive: list all files under $dir and its subdirs 890 if ( !empty( $params['adviseStat'] ) ) { 891 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix ); 892 } else { 893 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); 894 } 895 } 896 897 // Reformat this list into a list of (name, stat array or null) entries 898 if ( !$status->isOk() ) { 899 return $files; // error 900 } 901 902 $objects = $status->value; 903 $files = $this->buildFileObjectListing( $params, $dir, $objects ); 904 905 // Page on the unfiltered object listing (what is returned may be filtered) 906 if ( count( $objects ) < $limit ) { 907 $after = INF; // avoid a second RTT 908 } else { 909 $after = end( $objects ); // update last item 910 $after = is_object( $after ) ? $after->name : $after; 911 } 912 913 return $files; 914 } 915 916 /** 917 * Build a list of file objects, filtering out any directories 918 * and extracting any stat info if provided in $objects (for CF_Objects) 919 * 920 * @param array $params Parameters for getDirectoryList() 921 * @param string $dir Resolved container directory path 922 * @param array $objects List of CF_Object items or object names 923 * @return array List of (names,stat array or null) entries 924 */ 925 private function buildFileObjectListing( array $params, $dir, array $objects ) { 926 $names = array(); 927 foreach ( $objects as $object ) { 928 if ( is_object( $object ) ) { 929 if ( isset( $object->subdir ) || !isset( $object->name ) ) { 930 continue; // virtual directory entry; ignore 931 } 932 $stat = array( 933 // Convert various random Swift dates to TS_MW 934 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ), 935 'size' => (int)$object->bytes, 936 // Note: manifiest ETags are not an MD5 of the file 937 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null, 938 'latest' => false // eventually consistent 939 ); 940 $names[] = array( $object->name, $stat ); 941 } elseif ( substr( $object, -1 ) !== '/' ) { 942 // Omit directories, which end in '/' in listings 943 $names[] = array( $object, null ); 944 } 945 } 946 947 return $names; 948 } 949 950 /** 951 * Do not call this function outside of SwiftFileBackendFileList 952 * 953 * @param string $path Storage path 954 * @param array $val Stat value 955 */ 956 public function loadListingStatInternal( $path, array $val ) { 957 $this->cheapCache->set( $path, 'stat', $val ); 958 } 959 960 protected function doGetFileXAttributes( array $params ) { 961 $stat = $this->getFileStat( $params ); 962 if ( $stat ) { 963 if ( !isset( $stat['xattr'] ) ) { 964 // Stat entries filled by file listings don't include metadata/headers 965 $this->clearCache( array( $params['src'] ) ); 966 $stat = $this->getFileStat( $params ); 967 } 968 969 return $stat['xattr']; 970 } else { 971 return false; 972 } 973 } 974 975 protected function doGetFileSha1base36( array $params ) { 976 $stat = $this->getFileStat( $params ); 977 if ( $stat ) { 978 if ( !isset( $stat['sha1'] ) ) { 979 // Stat entries filled by file listings don't include SHA1 980 $this->clearCache( array( $params['src'] ) ); 981 $stat = $this->getFileStat( $params ); 982 } 983 984 return $stat['sha1']; 985 } else { 986 return false; 987 } 988 } 989 990 protected function doStreamFile( array $params ) { 991 $status = Status::newGood(); 992 993 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 994 if ( $srcRel === null ) { 995 $status->fatal( 'backend-fail-invalidpath', $params['src'] ); 996 } 997 998 $auth = $this->getAuthentication(); 999 if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) { 1000 $status->fatal( 'backend-fail-stream', $params['src'] ); 1001 1002 return $status; 1003 } 1004 1005 $handle = fopen( 'php://output', 'wb' ); 1006 1007 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1008 'method' => 'GET', 1009 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), 1010 'headers' => $this->authTokenHeaders( $auth ) 1011 + $this->headersFromParams( $params ), 1012 'stream' => $handle, 1013 ) ); 1014 1015 if ( $rcode >= 200 && $rcode <= 299 ) { 1016 // good 1017 } elseif ( $rcode === 404 ) { 1018 $status->fatal( 'backend-fail-stream', $params['src'] ); 1019 } else { 1020 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); 1021 } 1022 1023 return $status; 1024 } 1025 1026 protected function doGetLocalCopyMulti( array $params ) { 1027 $tmpFiles = array(); 1028 1029 $auth = $this->getAuthentication(); 1030 1031 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging 1032 // Blindly create tmp files and stream to them, catching any exception if the file does 1033 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). 1034 $reqs = array(); // (path => op) 1035 1036 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch 1037 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 1038 if ( $srcRel === null || !$auth ) { 1039 $tmpFiles[$path] = null; 1040 continue; 1041 } 1042 // Get source file extension 1043 $ext = FileBackend::extensionFromPath( $path ); 1044 // Create a new temporary file... 1045 $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); 1046 if ( $tmpFile ) { 1047 $handle = fopen( $tmpFile->getPath(), 'wb' ); 1048 if ( $handle ) { 1049 $reqs[$path] = array( 1050 'method' => 'GET', 1051 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), 1052 'headers' => $this->authTokenHeaders( $auth ) 1053 + $this->headersFromParams( $params ), 1054 'stream' => $handle, 1055 ); 1056 } else { 1057 $tmpFile = null; 1058 } 1059 } 1060 $tmpFiles[$path] = $tmpFile; 1061 } 1062 1063 $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); 1064 $reqs = $this->http->runMulti( $reqs, $opts ); 1065 foreach ( $reqs as $path => $op ) { 1066 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; 1067 fclose( $op['stream'] ); // close open handle 1068 if ( $rcode >= 200 && $rcode <= 299 ) { 1069 $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0; 1070 // Double check that the disk is not full/broken 1071 if ( $size != $rhdrs['content-length'] ) { 1072 $tmpFiles[$path] = null; 1073 $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; 1074 $this->onError( null, __METHOD__, 1075 array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); 1076 } 1077 } elseif ( $rcode === 404 ) { 1078 $tmpFiles[$path] = false; 1079 } else { 1080 $tmpFiles[$path] = null; 1081 $this->onError( null, __METHOD__, 1082 array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); 1083 } 1084 } 1085 1086 return $tmpFiles; 1087 } 1088 1089 public function getFileHttpUrl( array $params ) { 1090 if ( $this->swiftTempUrlKey != '' || 1091 ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) 1092 ) { 1093 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); 1094 if ( $srcRel === null ) { 1095 return null; // invalid path 1096 } 1097 1098 $auth = $this->getAuthentication(); 1099 if ( !$auth ) { 1100 return null; 1101 } 1102 1103 $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; 1104 $expires = time() + $ttl; 1105 1106 if ( $this->swiftTempUrlKey != '' ) { 1107 $url = $this->storageUrl( $auth, $srcCont, $srcRel ); 1108 // Swift wants the signature based on the unencoded object name 1109 $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH ); 1110 $signature = hash_hmac( 'sha1', 1111 "GET\n{$expires}\n{$contPath}/{$srcRel}", 1112 $this->swiftTempUrlKey 1113 ); 1114 1115 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}"; 1116 } else { // give S3 API URL for rgw 1117 // Path for signature starts with the bucket 1118 $spath = '/' . rawurlencode( $srcCont ) . '/' . 1119 str_replace( '%2F', '/', rawurlencode( $srcRel ) ); 1120 // Calculate the hash 1121 $signature = base64_encode( hash_hmac( 1122 'sha1', 1123 "GET\n\n\n{$expires}\n{$spath}", 1124 $this->rgwS3SecretKey, 1125 true // raw 1126 ) ); 1127 // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. 1128 // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. 1129 return wfAppendQuery( 1130 str_replace( '/swift/v1', '', // S3 API is the rgw default 1131 $this->storageUrl( $auth ) . $spath ), 1132 array( 1133 'Signature' => $signature, 1134 'Expires' => $expires, 1135 'AWSAccessKeyId' => $this->rgwS3AccessKey ) 1136 ); 1137 } 1138 } 1139 1140 return null; 1141 } 1142 1143 protected function directoriesAreVirtual() { 1144 return true; 1145 } 1146 1147 /** 1148 * Get headers to send to Swift when reading a file based 1149 * on a FileBackend params array, e.g. that of getLocalCopy(). 1150 * $params is currently only checked for a 'latest' flag. 1151 * 1152 * @param array $params 1153 * @return array 1154 */ 1155 protected function headersFromParams( array $params ) { 1156 $hdrs = array(); 1157 if ( !empty( $params['latest'] ) ) { 1158 $hdrs['x-newest'] = 'true'; 1159 } 1160 1161 return $hdrs; 1162 } 1163 1164 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { 1165 $statuses = array(); 1166 1167 $auth = $this->getAuthentication(); 1168 if ( !$auth ) { 1169 foreach ( $fileOpHandles as $index => $fileOpHandle ) { 1170 $statuses[$index] = Status::newFatal( 'backend-fail-connect', $this->name ); 1171 } 1172 1173 return $statuses; 1174 } 1175 1176 // Split the HTTP requests into stages that can be done concurrently 1177 $httpReqsByStage = array(); // map of (stage => index => HTTP request) 1178 foreach ( $fileOpHandles as $index => $fileOpHandle ) { 1179 $reqs = $fileOpHandle->httpOp; 1180 // Convert the 'url' parameter to an actual URL using $auth 1181 foreach ( $reqs as $stage => &$req ) { 1182 list( $container, $relPath ) = $req['url']; 1183 $req['url'] = $this->storageUrl( $auth, $container, $relPath ); 1184 $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : array(); 1185 $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers']; 1186 $httpReqsByStage[$stage][$index] = $req; 1187 } 1188 $statuses[$index] = Status::newGood(); 1189 } 1190 1191 // Run all requests for the first stage, then the next, and so on 1192 $reqCount = count( $httpReqsByStage ); 1193 for ( $stage = 0; $stage < $reqCount; ++$stage ) { 1194 $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] ); 1195 foreach ( $httpReqs as $index => $httpReq ) { 1196 // Run the callback for each request of this operation 1197 $callback = $fileOpHandles[$index]->callback; 1198 call_user_func_array( $callback, array( $httpReq, $statuses[$index] ) ); 1199 // On failure, abort all remaining requests for this operation 1200 // (e.g. abort the DELETE request if the COPY request fails for a move) 1201 if ( !$statuses[$index]->isOK() ) { 1202 $stages = count( $fileOpHandles[$index]->httpOp ); 1203 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) { 1204 unset( $httpReqsByStage[$s][$index] ); 1205 } 1206 } 1207 } 1208 } 1209 1210 return $statuses; 1211 } 1212 1213 /** 1214 * Set read/write permissions for a Swift container. 1215 * 1216 * @see http://swift.openstack.org/misc.html#acls 1217 * 1218 * In general, we don't allow listings to end-users. It's not useful, isn't well-defined 1219 * (lists are truncated to 10000 item with no way to page), and is just a performance risk. 1220 * 1221 * @param string $container Resolved Swift container 1222 * @param array $readGrps List of the possible criteria for a request to have 1223 * access to read a container. Each item is one of the following formats: 1224 * - account:user : Grants access if the request is by the given user 1225 * - ".r:<regex>" : Grants access if the request is from a referrer host that 1226 * matches the expression and the request is not for a listing. 1227 * Setting this to '*' effectively makes a container public. 1228 * -".rlistings:<regex>" : Grants access if the request is from a referrer host that 1229 * matches the expression and the request is for a listing. 1230 * @param array $writeGrps A list of the possible criteria for a request to have 1231 * access to write to a container. Each item is of the following format: 1232 * - account:user : Grants access if the request is by the given user 1233 * @return Status 1234 */ 1235 protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) { 1236 $status = Status::newGood(); 1237 $auth = $this->getAuthentication(); 1238 1239 if ( !$auth ) { 1240 $status->fatal( 'backend-fail-connect', $this->name ); 1241 1242 return $status; 1243 } 1244 1245 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1246 'method' => 'POST', 1247 'url' => $this->storageUrl( $auth, $container ), 1248 'headers' => $this->authTokenHeaders( $auth ) + array( 1249 'x-container-read' => implode( ',', $readGrps ), 1250 'x-container-write' => implode( ',', $writeGrps ) 1251 ) 1252 ) ); 1253 1254 if ( $rcode != 204 && $rcode !== 202 ) { 1255 $status->fatal( 'backend-fail-internal', $this->name ); 1256 } 1257 1258 return $status; 1259 } 1260 1261 /** 1262 * Get a Swift container stat array, possibly from process cache. 1263 * Use $reCache if the file count or byte count is needed. 1264 * 1265 * @param string $container Container name 1266 * @param bool $bypassCache Bypass all caches and load from Swift 1267 * @return array|bool|null False on 404, null on failure 1268 */ 1269 protected function getContainerStat( $container, $bypassCache = false ) { 1270 $section = new ProfileSection( __METHOD__ . '-' . $this->name ); 1271 1272 if ( $bypassCache ) { // purge cache 1273 $this->containerStatCache->clear( $container ); 1274 } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) { 1275 $this->primeContainerCache( array( $container ) ); // check persistent cache 1276 } 1277 if ( !$this->containerStatCache->has( $container, 'stat' ) ) { 1278 $auth = $this->getAuthentication(); 1279 if ( !$auth ) { 1280 return null; 1281 } 1282 1283 wfProfileIn( __METHOD__ . "-{$this->name}-miss" ); 1284 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1285 'method' => 'HEAD', 1286 'url' => $this->storageUrl( $auth, $container ), 1287 'headers' => $this->authTokenHeaders( $auth ) 1288 ) ); 1289 wfProfileOut( __METHOD__ . "-{$this->name}-miss" ); 1290 1291 if ( $rcode === 204 ) { 1292 $stat = array( 1293 'count' => $rhdrs['x-container-object-count'], 1294 'bytes' => $rhdrs['x-container-bytes-used'] 1295 ); 1296 if ( $bypassCache ) { 1297 return $stat; 1298 } else { 1299 $this->containerStatCache->set( $container, 'stat', $stat ); // cache it 1300 $this->setContainerCache( $container, $stat ); // update persistent cache 1301 } 1302 } elseif ( $rcode === 404 ) { 1303 return false; 1304 } else { 1305 $this->onError( null, __METHOD__, 1306 array( 'cont' => $container ), $rerr, $rcode, $rdesc ); 1307 1308 return null; 1309 } 1310 } 1311 1312 return $this->containerStatCache->get( $container, 'stat' ); 1313 } 1314 1315 /** 1316 * Create a Swift container 1317 * 1318 * @param string $container Container name 1319 * @param array $params 1320 * @return Status 1321 */ 1322 protected function createContainer( $container, array $params ) { 1323 $status = Status::newGood(); 1324 1325 $auth = $this->getAuthentication(); 1326 if ( !$auth ) { 1327 $status->fatal( 'backend-fail-connect', $this->name ); 1328 1329 return $status; 1330 } 1331 1332 // @see SwiftFileBackend::setContainerAccess() 1333 if ( empty( $params['noAccess'] ) ) { 1334 $readGrps = array( '.r:*', $this->swiftUser ); // public 1335 } else { 1336 $readGrps = array( $this->swiftUser ); // private 1337 } 1338 $writeGrps = array( $this->swiftUser ); // sanity 1339 1340 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1341 'method' => 'PUT', 1342 'url' => $this->storageUrl( $auth, $container ), 1343 'headers' => $this->authTokenHeaders( $auth ) + array( 1344 'x-container-read' => implode( ',', $readGrps ), 1345 'x-container-write' => implode( ',', $writeGrps ) 1346 ) 1347 ) ); 1348 1349 if ( $rcode === 201 ) { // new 1350 // good 1351 } elseif ( $rcode === 202 ) { // already there 1352 // this shouldn't really happen, but is OK 1353 } else { 1354 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); 1355 } 1356 1357 return $status; 1358 } 1359 1360 /** 1361 * Delete a Swift container 1362 * 1363 * @param string $container Container name 1364 * @param array $params 1365 * @return Status 1366 */ 1367 protected function deleteContainer( $container, array $params ) { 1368 $status = Status::newGood(); 1369 1370 $auth = $this->getAuthentication(); 1371 if ( !$auth ) { 1372 $status->fatal( 'backend-fail-connect', $this->name ); 1373 1374 return $status; 1375 } 1376 1377 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1378 'method' => 'DELETE', 1379 'url' => $this->storageUrl( $auth, $container ), 1380 'headers' => $this->authTokenHeaders( $auth ) 1381 ) ); 1382 1383 if ( $rcode >= 200 && $rcode <= 299 ) { // deleted 1384 $this->containerStatCache->clear( $container ); // purge 1385 } elseif ( $rcode === 404 ) { // not there 1386 // this shouldn't really happen, but is OK 1387 } elseif ( $rcode === 409 ) { // not empty 1388 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race? 1389 } else { 1390 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); 1391 } 1392 1393 return $status; 1394 } 1395 1396 /** 1397 * Get a list of objects under a container. 1398 * Either just the names or a list of stdClass objects with details can be returned. 1399 * 1400 * @param string $fullCont 1401 * @param string $type ('info' for a list of object detail maps, 'names' for names only) 1402 * @param int $limit 1403 * @param string|null $after 1404 * @param string|null $prefix 1405 * @param string|null $delim 1406 * @return Status With the list as value 1407 */ 1408 private function objectListing( 1409 $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null 1410 ) { 1411 $status = Status::newGood(); 1412 1413 $auth = $this->getAuthentication(); 1414 if ( !$auth ) { 1415 $status->fatal( 'backend-fail-connect', $this->name ); 1416 1417 return $status; 1418 } 1419 1420 $query = array( 'limit' => $limit ); 1421 if ( $type === 'info' ) { 1422 $query['format'] = 'json'; 1423 } 1424 if ( $after !== null ) { 1425 $query['marker'] = $after; 1426 } 1427 if ( $prefix !== null ) { 1428 $query['prefix'] = $prefix; 1429 } 1430 if ( $delim !== null ) { 1431 $query['delimiter'] = $delim; 1432 } 1433 1434 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1435 'method' => 'GET', 1436 'url' => $this->storageUrl( $auth, $fullCont ), 1437 'query' => $query, 1438 'headers' => $this->authTokenHeaders( $auth ) 1439 ) ); 1440 1441 $params = array( 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ); 1442 if ( $rcode === 200 ) { // good 1443 if ( $type === 'info' ) { 1444 $status->value = FormatJson::decode( trim( $rbody ) ); 1445 } else { 1446 $status->value = explode( "\n", trim( $rbody ) ); 1447 } 1448 } elseif ( $rcode === 204 ) { 1449 $status->value = array(); // empty container 1450 } elseif ( $rcode === 404 ) { 1451 $status->value = array(); // no container 1452 } else { 1453 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); 1454 } 1455 1456 return $status; 1457 } 1458 1459 protected function doPrimeContainerCache( array $containerInfo ) { 1460 foreach ( $containerInfo as $container => $info ) { 1461 $this->containerStatCache->set( $container, 'stat', $info ); 1462 } 1463 } 1464 1465 protected function doGetFileStatMulti( array $params ) { 1466 $stats = array(); 1467 1468 $auth = $this->getAuthentication(); 1469 1470 $reqs = array(); 1471 foreach ( $params['srcs'] as $path ) { 1472 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); 1473 if ( $srcRel === null ) { 1474 $stats[$path] = false; 1475 continue; // invalid storage path 1476 } elseif ( !$auth ) { 1477 $stats[$path] = null; 1478 continue; 1479 } 1480 1481 // (a) Check the container 1482 $cstat = $this->getContainerStat( $srcCont ); 1483 if ( $cstat === false ) { 1484 $stats[$path] = false; 1485 continue; // ok, nothing to do 1486 } elseif ( !is_array( $cstat ) ) { 1487 $stats[$path] = null; 1488 continue; 1489 } 1490 1491 $reqs[$path] = array( 1492 'method' => 'HEAD', 1493 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), 1494 'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params ) 1495 ); 1496 } 1497 1498 $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); 1499 $reqs = $this->http->runMulti( $reqs, $opts ); 1500 1501 foreach ( $params['srcs'] as $path ) { 1502 if ( array_key_exists( $path, $stats ) ) { 1503 continue; // some sort of failure above 1504 } 1505 // (b) Check the file 1506 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response']; 1507 if ( $rcode === 200 || $rcode === 204 ) { 1508 // Update the object if it is missing some headers 1509 $rhdrs = $this->addMissingMetadata( $rhdrs, $path ); 1510 // Fetch all of the custom metadata headers 1511 $metadata = array(); 1512 foreach ( $rhdrs as $name => $value ) { 1513 if ( strpos( $name, 'x-object-meta-' ) === 0 ) { 1514 $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value; 1515 } 1516 } 1517 // Fetch all of the custom raw HTTP headers 1518 $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) ); 1519 $stat = array( 1520 // Convert various random Swift dates to TS_MW 1521 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), 1522 // Empty objects actually return no content-length header in Ceph 1523 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, 1524 'sha1' => $rhdrs['x-object-meta-sha1base36'], 1525 // Note: manifiest ETags are not an MD5 of the file 1526 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, 1527 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers ) 1528 ); 1529 if ( $this->isRGW ) { 1530 $stat['latest'] = true; // strong consistency 1531 } 1532 } elseif ( $rcode === 404 ) { 1533 $stat = false; 1534 } else { 1535 $stat = null; 1536 $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc ); 1537 } 1538 $stats[$path] = $stat; 1539 } 1540 1541 return $stats; 1542 } 1543 1544 /** 1545 * @return array|null Credential map 1546 */ 1547 protected function getAuthentication() { 1548 if ( $this->authErrorTimestamp !== null ) { 1549 if ( ( time() - $this->authErrorTimestamp ) < 60 ) { 1550 return null; // failed last attempt; don't bother 1551 } else { // actually retry this time 1552 $this->authErrorTimestamp = null; 1553 } 1554 } 1555 // Session keys expire after a while, so we renew them periodically 1556 $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL ); 1557 // Authenticate with proxy and get a session key... 1558 if ( !$this->authCreds || $reAuth ) { 1559 $this->authSessionTimestamp = 0; 1560 $cacheKey = $this->getCredsCacheKey( $this->swiftUser ); 1561 $creds = $this->srvCache->get( $cacheKey ); // credentials 1562 // Try to use the credential cache 1563 if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) { 1564 $this->authCreds = $creds; 1565 // Skew the timestamp for worst case to avoid using stale credentials 1566 $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 ); 1567 } else { // cache miss 1568 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 1569 'method' => 'GET', 1570 'url' => "{$this->swiftAuthUrl}/v1.0", 1571 'headers' => array( 1572 'x-auth-user' => $this->swiftUser, 1573 'x-auth-key' => $this->swiftKey 1574 ) 1575 ) ); 1576 1577 if ( $rcode >= 200 && $rcode <= 299 ) { // OK 1578 $this->authCreds = array( 1579 'auth_token' => $rhdrs['x-auth-token'], 1580 'storage_url' => $rhdrs['x-storage-url'] 1581 ); 1582 $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) ); 1583 $this->authSessionTimestamp = time(); 1584 } elseif ( $rcode === 401 ) { 1585 $this->onError( null, __METHOD__, array(), "Authentication failed.", $rcode ); 1586 $this->authErrorTimestamp = time(); 1587 1588 return null; 1589 } else { 1590 $this->onError( null, __METHOD__, array(), "HTTP return code: $rcode", $rcode ); 1591 $this->authErrorTimestamp = time(); 1592 1593 return null; 1594 } 1595 } 1596 // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>") 1597 if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) { 1598 $this->isRGW = true; // take advantage of strong consistency 1599 } 1600 } 1601 1602 return $this->authCreds; 1603 } 1604 1605 /** 1606 * @param array $creds From getAuthentication() 1607 * @param string $container 1608 * @param string $object 1609 * @return array 1610 */ 1611 protected function storageUrl( array $creds, $container = null, $object = null ) { 1612 $parts = array( $creds['storage_url'] ); 1613 if ( strlen( $container ) ) { 1614 $parts[] = rawurlencode( $container ); 1615 } 1616 if ( strlen( $object ) ) { 1617 $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) ); 1618 } 1619 1620 return implode( '/', $parts ); 1621 } 1622 1623 /** 1624 * @param array $creds From getAuthentication() 1625 * @return array 1626 */ 1627 protected function authTokenHeaders( array $creds ) { 1628 return array( 'x-auth-token' => $creds['auth_token'] ); 1629 } 1630 1631 /** 1632 * Get the cache key for a container 1633 * 1634 * @param string $username 1635 * @return string 1636 */ 1637 private function getCredsCacheKey( $username ) { 1638 return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl ); 1639 } 1640 1641 /** 1642 * Log an unexpected exception for this backend. 1643 * This also sets the Status object to have a fatal error. 1644 * 1645 * @param Status|null $status 1646 * @param string $func 1647 * @param array $params 1648 * @param string $err Error string 1649 * @param int $code HTTP status 1650 * @param string $desc HTTP status description 1651 */ 1652 public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) { 1653 if ( $status instanceof Status ) { 1654 $status->fatal( 'backend-fail-internal', $this->name ); 1655 } 1656 if ( $code == 401 ) { // possibly a stale token 1657 $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) ); 1658 } 1659 wfDebugLog( 'SwiftBackend', 1660 "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . 1661 ( $err ? ": $err" : "" ) 1662 ); 1663 } 1664 } 1665 1666 /** 1667 * @see FileBackendStoreOpHandle 1668 */ 1669 class SwiftFileOpHandle extends FileBackendStoreOpHandle { 1670 /** @var array List of Requests for MultiHttpClient */ 1671 public $httpOp; 1672 /** @var Closure */ 1673 public $callback; 1674 1675 /** 1676 * @param SwiftFileBackend $backend 1677 * @param Closure $callback Function that takes (HTTP request array, status) 1678 * @param array $httpOp MultiHttpClient op 1679 */ 1680 public function __construct( SwiftFileBackend $backend, Closure $callback, array $httpOp ) { 1681 $this->backend = $backend; 1682 $this->callback = $callback; 1683 $this->httpOp = $httpOp; 1684 } 1685 } 1686 1687 /** 1688 * SwiftFileBackend helper class to page through listings. 1689 * Swift also has a listing limit of 10,000 objects for sanity. 1690 * Do not use this class from places outside SwiftFileBackend. 1691 * 1692 * @ingroup FileBackend 1693 */ 1694 abstract class SwiftFileBackendList implements Iterator { 1695 /** @var array List of path or (path,stat array) entries */ 1696 protected $bufferIter = array(); 1697 1698 /** @var string List items *after* this path */ 1699 protected $bufferAfter = null; 1700 1701 /** @var int */ 1702 protected $pos = 0; 1703 1704 /** @var array */ 1705 protected $params = array(); 1706 1707 /** @var SwiftFileBackend */ 1708 protected $backend; 1709 1710 /** @var string Container name */ 1711 protected $container; 1712 1713 /** @var string Storage directory */ 1714 protected $dir; 1715 1716 /** @var int */ 1717 protected $suffixStart; 1718 1719 const PAGE_SIZE = 9000; // file listing buffer size 1720 1721 /** 1722 * @param SwiftFileBackend $backend 1723 * @param string $fullCont Resolved container name 1724 * @param string $dir Resolved directory relative to container 1725 * @param array $params 1726 */ 1727 public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { 1728 $this->backend = $backend; 1729 $this->container = $fullCont; 1730 $this->dir = $dir; 1731 if ( substr( $this->dir, -1 ) === '/' ) { 1732 $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash 1733 } 1734 if ( $this->dir == '' ) { // whole container 1735 $this->suffixStart = 0; 1736 } else { // dir within container 1737 $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/" 1738 } 1739 $this->params = $params; 1740 } 1741 1742 /** 1743 * @see Iterator::key() 1744 * @return int 1745 */ 1746 public function key() { 1747 return $this->pos; 1748 } 1749 1750 /** 1751 * @see Iterator::next() 1752 */ 1753 public function next() { 1754 // Advance to the next file in the page 1755 next( $this->bufferIter ); 1756 ++$this->pos; 1757 // Check if there are no files left in this page and 1758 // advance to the next page if this page was not empty. 1759 if ( !$this->valid() && count( $this->bufferIter ) ) { 1760 $this->bufferIter = $this->pageFromList( 1761 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params 1762 ); // updates $this->bufferAfter 1763 } 1764 } 1765 1766 /** 1767 * @see Iterator::rewind() 1768 */ 1769 public function rewind() { 1770 $this->pos = 0; 1771 $this->bufferAfter = null; 1772 $this->bufferIter = $this->pageFromList( 1773 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params 1774 ); // updates $this->bufferAfter 1775 } 1776 1777 /** 1778 * @see Iterator::valid() 1779 * @return bool 1780 */ 1781 public function valid() { 1782 if ( $this->bufferIter === null ) { 1783 return false; // some failure? 1784 } else { 1785 return ( current( $this->bufferIter ) !== false ); // no paths can have this value 1786 } 1787 } 1788 1789 /** 1790 * Get the given list portion (page) 1791 * 1792 * @param string $container Resolved container name 1793 * @param string $dir Resolved path relative to container 1794 * @param string $after 1795 * @param int $limit 1796 * @param array $params 1797 * @return Traversable|array 1798 */ 1799 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); 1800 } 1801 1802 /** 1803 * Iterator for listing directories 1804 */ 1805 class SwiftFileBackendDirList extends SwiftFileBackendList { 1806 /** 1807 * @see Iterator::current() 1808 * @return string|bool String (relative path) or false 1809 */ 1810 public function current() { 1811 return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); 1812 } 1813 1814 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { 1815 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); 1816 } 1817 } 1818 1819 /** 1820 * Iterator for listing regular files 1821 */ 1822 class SwiftFileBackendFileList extends SwiftFileBackendList { 1823 /** 1824 * @see Iterator::current() 1825 * @return string|bool String (relative path) or false 1826 */ 1827 public function current() { 1828 list( $path, $stat ) = current( $this->bufferIter ); 1829 $relPath = substr( $path, $this->suffixStart ); 1830 if ( is_array( $stat ) ) { 1831 $storageDir = rtrim( $this->params['dir'], '/' ); 1832 $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat ); 1833 } 1834 1835 return $relPath; 1836 } 1837 1838 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { 1839 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); 1840 } 1841 }
title
Description
Body
title
Description
Body
title
Description
Body
title
Body
| Generated: Fri Nov 28 14:03:12 2014 | Cross-referenced by PHPXref 0.7.1 |