FileBackendMultiWrite.php

Go to the documentation of this file.

<?php
class FileBackendMultiWrite extends FileBackend {
        protected $backends = array(); // array of (backend index => backends)
        protected $masterIndex = -1; // integer; index of master backend
        protected $syncChecks = 0; // integer bitfield

        /* Possible internal backend consistency checks */
        const CHECK_SIZE = 1;
        const CHECK_TIME = 2;

        public function __construct( array $config ) {
                parent::__construct( $config );
                $namesUsed = array();
                // Construct backends here rather than via registration
                // to keep these backends hidden from outside the proxy.
                foreach ( $config['backends'] as $index => $config ) {
                        $name = $config['name'];
                        if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
                                throw new MWException( "Two or more backends defined with the name $name." );
                        }
                        $namesUsed[$name] = 1;
                        if ( !isset( $config['class'] ) ) {
                                throw new MWException( 'No class given for a backend config.' );
                        }
                        $class = $config['class'];
                        $this->backends[$index] = new $class( $config );
                        if ( !empty( $config['isMultiMaster'] ) ) {
                                if ( $this->masterIndex >= 0 ) {
                                        throw new MWException( 'More than one master backend defined.' );
                                }
                                $this->masterIndex = $index;
                        }
                }
                if ( $this->masterIndex < 0 ) { // need backends and must have a master
                        throw new MWException( 'No master backend defined.' );
                }
                $this->syncChecks = isset( $config['syncChecks'] )
                        ? $config['syncChecks']
                        : self::CHECK_SIZE;
        }

        final protected function doOperationsInternal( array $ops, array $opts ) {
                $status = Status::newGood();

                $performOps = array(); // list of FileOp objects
                $filesRead = $filesChanged = array(); // storage paths used
                // Build up a list of FileOps. The list will have all the ops
                // for one backend, then all the ops for the next, and so on.
                // These batches of ops are all part of a continuous array.
                // Also build up a list of files read/changed...
                foreach ( $this->backends as $index => $backend ) {
                        $backendOps = $this->substOpBatchPaths( $ops, $backend );
                        // Add on the operation batch for this backend
                        $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
                        if ( $index == 0 ) { // first batch
                                // Get the files used for these operations. Each backend has a batch of
                                // the same operations, so we only need to get them from the first batch.
                                foreach ( $performOps as $fileOp ) {
                                        $filesRead = array_merge( $filesRead, $fileOp->storagePathsRead() );
                                        $filesChanged = array_merge( $filesChanged, $fileOp->storagePathsChanged() );
                                }
                                // Get the paths under the proxy backend's name
                                $filesRead = $this->unsubstPaths( $filesRead );
                                $filesChanged = $this->unsubstPaths( $filesChanged );
                        }
                }

                // Try to lock those files for the scope of this function...
                if ( empty( $opts['nonLocking'] ) ) {
                        $filesLockSh = array_diff( $filesRead, $filesChanged ); // optimization
                        $filesLockEx = $filesChanged;
                        // Get a shared lock on the parent directory of each path changed
                        $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
                        // Try to lock those files for the scope of this function...
                        $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
                        $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
                        if ( !$status->isOK() ) {
                                return $status; // abort
                        }
                }

                // Clear any cache entries (after locks acquired)
                $this->clearCache();

                // Do a consistency check to see if the backends agree
                if ( count( $this->backends ) > 1 ) {
                        $status->merge( $this->consistencyCheck( array_merge( $filesRead, $filesChanged ) ) );
                        if ( !$status->isOK() ) {
                                return $status; // abort
                        }
                }

                // Actually attempt the operation batch...
                $subStatus = FileOp::attemptBatch( $performOps, $opts );

                $success = array();
                $failCount = $successCount = 0;
                // Make 'success', 'successCount', and 'failCount' fields reflect
                // the overall operation, rather than all the batches for each backend.
                // Do this by only using success values from the master backend's batch.
                $batchStart = $this->masterIndex * count( $ops );
                $batchEnd = $batchStart + count( $ops ) - 1;
                for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
                        if ( !isset( $subStatus->success[$i] ) ) {
                                break; // failed out before trying this op
                        } elseif ( $subStatus->success[$i] ) {
                                ++$successCount;
                        } else {
                                ++$failCount;
                        }
                        $success[] = $subStatus->success[$i];
                }
                $subStatus->success = $success;
                $subStatus->successCount = $successCount;
                $subStatus->failCount = $failCount;

                // Merge errors into status fields
                $status->merge( $subStatus );
                $status->success = $subStatus->success; // not done in merge()

                return $status;
        }

        public function consistencyCheck( array $paths ) {
                $status = Status::newGood();
                if ( $this->syncChecks == 0 ) {
                        return $status; // skip checks
                }

                $mBackend = $this->backends[$this->masterIndex];
                foreach ( array_unique( $paths ) as $path ) {
                        $params = array( 'src' => $path, 'latest' => true );
                        // Stat the file on the 'master' backend
                        $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
                        // Check of all clone backends agree with the master...
                        foreach ( $this->backends as $index => $cBackend ) {
                                if ( $index === $this->masterIndex ) {
                                        continue; // master
                                }
                                $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
                                if ( $mStat ) { // file is in master
                                        if ( !$cStat ) { // file should exist
                                                $status->fatal( 'backend-fail-synced', $path );
                                                continue;
                                        }
                                        if ( $this->syncChecks & self::CHECK_SIZE ) {
                                                if ( $cStat['size'] != $mStat['size'] ) { // wrong size
                                                        $status->fatal( 'backend-fail-synced', $path );
                                                        continue;
                                                }
                                        }
                                        if ( $this->syncChecks & self::CHECK_TIME ) {
                                                $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
                                                $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
                                                if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
                                                        $status->fatal( 'backend-fail-synced', $path );
                                                        continue;
                                                }
                                        }
                                } else { // file is not in master
                                        if ( $cStat ) { // file should not exist
                                                $status->fatal( 'backend-fail-synced', $path );
                                        }
                                }
                        }
                }

                return $status;
        }

        protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
                $newOps = array(); // operations
                foreach ( $ops as $op ) {
                        $newOp = $op; // operation
                        foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
                                if ( isset( $newOp[$par] ) ) { // string or array
                                        $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
                                }
                        }
                        $newOps[] = $newOp;
                }
                return $newOps;
        }

        protected function substOpPaths( array $ops, FileBackendStore $backend ) {
                $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
                return $newOps[0];
        }

        protected function substPaths( $paths, FileBackendStore $backend ) {
                return preg_replace(
                        '!^mwstore://' . preg_quote( $this->name ) . '/!',
                        StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
                        $paths // string or array
                );
        }

        protected function unsubstPaths( $paths ) {
                return preg_replace(
                        '!^mwstore://([^/]+)!',
                        StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
                        $paths // string or array
                );
        }

        public function doPrepare( array $params ) {
                $status = Status::newGood();
                foreach ( $this->backends as $backend ) {
                        $realParams = $this->substOpPaths( $params, $backend );
                        $status->merge( $backend->doPrepare( $realParams ) );
                }
                return $status;
        }

        public function doSecure( array $params ) {
                $status = Status::newGood();
                foreach ( $this->backends as $backend ) {
                        $realParams = $this->substOpPaths( $params, $backend );
                        $status->merge( $backend->doSecure( $realParams ) );
                }
                return $status;
        }

        public function doClean( array $params ) {
                $status = Status::newGood();
                foreach ( $this->backends as $backend ) {
                        $realParams = $this->substOpPaths( $params, $backend );
                        $status->merge( $backend->doClean( $realParams ) );
                }
                return $status;
        }

        public function concatenate( array $params ) {
                // We are writing to an FS file, so we don't need to do this per-backend
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->concatenate( $realParams );
        }

        public function fileExists( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->fileExists( $realParams );
        }

        public function getFileTimestamp( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
        }

        public function getFileSize( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileSize( $realParams );
        }

        public function getFileStat( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileStat( $realParams );
        }

        public function getFileContents( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileContents( $realParams );
        }

        public function getFileSha1Base36( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
        }

        public function getFileProps( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileProps( $realParams );
        }

        public function streamFile( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->streamFile( $realParams );
        }

        public function getLocalReference( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
        }

        public function getLocalCopy( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
        }

        public function getFileList( array $params ) {
                $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
                return $this->backends[$this->masterIndex]->getFileList( $realParams );
        }

        public function clearCache( array $paths = null ) {
                foreach ( $this->backends as $backend ) {
                        $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
                        $backend->clearCache( $realPaths );
                }
        }
}