FileOp.php

Go to the documentation of this file.

<?php
abstract class FileOp {
        protected $params = array();
        protected $backend;

        protected $state = self::STATE_NEW; // integer
        protected $failed = false; // boolean
        protected $useLatest = true; // boolean

        protected $sourceSha1; // string
        protected $destSameAsSource; // boolean

        /* Object life-cycle */
        const STATE_NEW = 1;
        const STATE_CHECKED = 2;
        const STATE_ATTEMPTED = 3;

        /* Timeout related parameters */
        const MAX_BATCH_SIZE = 1000;
        const TIME_LIMIT_SEC = 300; // 5 minutes

        final public function __construct( FileBackendStore $backend, array $params ) {
                $this->backend = $backend;
                list( $required, $optional ) = $this->allowedParams();
                foreach ( $required as $name ) {
                        if ( isset( $params[$name] ) ) {
                                $this->params[$name] = $params[$name];
                        } else {
                                throw new MWException( "File operation missing parameter '$name'." );
                        }
                }
                foreach ( $optional as $name ) {
                        if ( isset( $params[$name] ) ) {
                                $this->params[$name] = $params[$name];
                        }
                }
                $this->params = $params;
        }

        final protected function allowStaleReads() {
                $this->useLatest = false;
        }

        final public static function attemptBatch( array $performOps, array $opts ) {
                $status = Status::newGood();

                $allowStale = !empty( $opts['allowStale'] );
                $ignoreErrors = !empty( $opts['force'] );

                $n = count( $performOps );
                if ( $n > self::MAX_BATCH_SIZE ) {
                        $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
                        return $status;
                }

                $predicates = FileOp::newPredicates(); // account for previous op in prechecks
                // Do pre-checks for each operation; abort on failure...
                foreach ( $performOps as $index => $fileOp ) {
                        if ( $allowStale ) {
                                $fileOp->allowStaleReads(); // allow potentially stale reads
                        }
                        $subStatus = $fileOp->precheck( $predicates );
                        $status->merge( $subStatus );
                        if ( !$subStatus->isOK() ) { // operation failed?
                                $status->success[$index] = false;
                                ++$status->failCount;
                                if ( !$ignoreErrors ) {
                                        return $status; // abort
                                }
                        }
                }

                if ( $ignoreErrors ) {
                        # Treat all precheck() fatals as merely warnings
                        $status->setResult( true, $status->value );
                }

                // Restart PHP's execution timer and set the timeout to safe amount.
                // This handles cases where the operations take a long time or where we are
                // already running low on time left. The old timeout is restored afterwards.
                # @TODO: re-enable this for when the number of batches is high.
                #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );

                // Attempt each operation...
                foreach ( $performOps as $index => $fileOp ) {
                        if ( $fileOp->failed() ) {
                                continue; // nothing to do
                        }
                        $subStatus = $fileOp->attempt();
                        $status->merge( $subStatus );
                        if ( $subStatus->isOK() ) {
                                $status->success[$index] = true;
                                ++$status->successCount;
                        } else {
                                $status->success[$index] = false;
                                ++$status->failCount;
                                // We can't continue (even with $ignoreErrors) as $predicates is wrong.
                                // Log the remaining ops as failed for recovery...
                                for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
                                        $performOps[$i]->logFailure( 'attempt_aborted' );
                                }
                                return $status; // bail out
                        }
                }

                return $status;
        }

        final public function getParam( $name ) {
                return isset( $this->params[$name] ) ? $this->params[$name] : null;
        }

        final public function failed() {
                return $this->failed;
        }

        final public static function newPredicates() {
                return array( 'exists' => array(), 'sha1' => array() );
        }

        final public function precheck( array &$predicates ) {
                if ( $this->state !== self::STATE_NEW ) {
                        return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
                }
                $this->state = self::STATE_CHECKED;
                $status = $this->doPrecheck( $predicates );
                if ( !$status->isOK() ) {
                        $this->failed = true;
                }
                return $status;
        }

        final public function attempt() {
                if ( $this->state !== self::STATE_CHECKED ) {
                        return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
                } elseif ( $this->failed ) { // failed precheck
                        return Status::newFatal( 'fileop-fail-attempt-precheck' );
                }
                $this->state = self::STATE_ATTEMPTED;
                $status = $this->doAttempt();
                if ( !$status->isOK() ) {
                        $this->failed = true;
                        $this->logFailure( 'attempt' );
                }
                return $status;
        }

        protected function allowedParams() {
                return array( array(), array() );
        }

        public function storagePathsRead() {
                return array();
        }

        public function storagePathsChanged() {
                return array();
        }

        protected function doPrecheck( array &$predicates ) {
                return Status::newGood();
        }

        protected function doAttempt() {
                return Status::newGood();
        }

        protected function precheckDestExistence( array $predicates ) {
                $status = Status::newGood();
                // Get hash of source file/string and the destination file
                $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
                if ( $this->sourceSha1 === null ) { // file in storage?
                        $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
                }
                $this->destSameAsSource = false;
                if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
                        if ( $this->getParam( 'overwrite' ) ) {
                                return $status; // OK
                        } elseif ( $this->getParam( 'overwriteSame' ) ) {
                                $dhash = $this->fileSha1( $this->params['dst'], $predicates );
                                // Check if hashes are valid and match each other...
                                if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
                                        $status->fatal( 'backend-fail-hashes' );
                                } elseif ( $this->sourceSha1 !== $dhash ) {
                                        // Give an error if the files are not identical
                                        $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
                                } else {
                                        $this->destSameAsSource = true; // OK
                                }
                                return $status; // do nothing; either OK or bad status
                        } else {
                                $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
                                return $status;
                        }
                }
                return $status;
        }

        protected function getSourceSha1Base36() {
                return null; // N/A
        }

        final protected function fileExists( $source, array $predicates ) {
                if ( isset( $predicates['exists'][$source] ) ) {
                        return $predicates['exists'][$source]; // previous op assures this
                } else {
                        $params = array( 'src' => $source, 'latest' => $this->useLatest );
                        return $this->backend->fileExists( $params );
                }
        }

        final protected function fileSha1( $source, array $predicates ) {
                if ( isset( $predicates['sha1'][$source] ) ) {
                        return $predicates['sha1'][$source]; // previous op assures this
                } else {
                        $params = array( 'src' => $source, 'latest' => $this->useLatest );
                        return $this->backend->getFileSha1Base36( $params );
                }
        }

        final protected function logFailure( $action ) {
                $params = $this->params;
                $params['failedAction'] = $action;
                try {
                        wfDebugLog( 'FileOperation',
                                get_class( $this ) . ' failed:' . serialize( $params ) );
                } catch ( Exception $e ) {
                        // bad config? debug log error?
                }
        }
}

class FileOpScopedPHPTimeout {
        protected $startTime; // float; seconds
        protected $oldTimeout; // integer; seconds

        protected static $stackDepth = 0; // integer
        protected static $totalCalls = 0; // integer
        protected static $totalElapsed = 0; // float; seconds

        /* Prevent callers in infinite loops from running forever */
        const MAX_TOTAL_CALLS = 1000000;
        const MAX_TOTAL_TIME = 300; // seconds

        public function __construct( $seconds ) {
                if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
                        if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
                                trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
                        } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
                                trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
                        } elseif ( self::$stackDepth > 0 ) { // recursion guard
                                trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
                        } else {
                                $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
                                $this->startTime = microtime( true );
                                ++self::$stackDepth;
                                ++self::$totalCalls; // proof against < 1us scopes
                        }
                }
        }

        public function __destruct() {
                if ( $this->oldTimeout ) {
                        $elapsed = microtime( true ) - $this->startTime;
                        // Note: a limit of 0 is treated as "forever"
                        set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
                        // If each scoped timeout is for less than one second, we end up
                        // restoring the original timeout without any decrease in value.
                        // Thus web scripts in an infinite loop can run forever unless we 
                        // take some measures to prevent this. Track total time and calls.
                        self::$totalElapsed += $elapsed;
                        --self::$stackDepth;
                }
        }
}

class StoreFileOp extends FileOp {
        protected function allowedParams() {
                return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
        }

        protected function doPrecheck( array &$predicates ) {
                $status = Status::newGood();
                // Check if the source file exists on the file system
                if ( !is_file( $this->params['src'] ) ) {
                        $status->fatal( 'backend-fail-notexists', $this->params['src'] );
                        return $status;
                // Check if the source file is too big
                } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
                        $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
                        return $status;
                // Check if a file can be placed at the destination
                } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
                        $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
                        return $status;
                }
                // Check if destination file exists
                $status->merge( $this->precheckDestExistence( $predicates ) );
                if ( $status->isOK() ) {
                        // Update file existence predicates
                        $predicates['exists'][$this->params['dst']] = true;
                        $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
                }
                return $status; // safe to call attempt()
        }

        protected function doAttempt() {
                $status = Status::newGood();
                // Store the file at the destination
                if ( !$this->destSameAsSource ) {
                        $status->merge( $this->backend->storeInternal( $this->params ) );
                }
                return $status;
        }

        protected function getSourceSha1Base36() {
                wfSuppressWarnings();
                $hash = sha1_file( $this->params['src'] );
                wfRestoreWarnings();
                if ( $hash !== false ) {
                        $hash = wfBaseConvert( $hash, 16, 36, 31 );
                }
                return $hash;
        }

        public function storagePathsChanged() {
                return array( $this->params['dst'] );
        }
}

class CreateFileOp extends FileOp {
        protected function allowedParams() {
                return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
        }

        protected function doPrecheck( array &$predicates ) {
                $status = Status::newGood();
                // Check if the source data is too big
                if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
                        $status->fatal( 'backend-fail-create', $this->params['dst'] );
                        return $status;
                // Check if a file can be placed at the destination
                } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
                        $status->fatal( 'backend-fail-create', $this->params['dst'] );
                        return $status;
                }
                // Check if destination file exists
                $status->merge( $this->precheckDestExistence( $predicates ) );
                if ( $status->isOK() ) {
                        // Update file existence predicates
                        $predicates['exists'][$this->params['dst']] = true;
                        $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
                }
                return $status; // safe to call attempt()
        }

        protected function doAttempt() {
                $status = Status::newGood();
                // Create the file at the destination
                if ( !$this->destSameAsSource ) {
                        $status->merge( $this->backend->createInternal( $this->params ) );
                }
                return $status;
        }

        protected function getSourceSha1Base36() {
                return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
        }

        public function storagePathsChanged() {
                return array( $this->params['dst'] );
        }
}

class CopyFileOp extends FileOp {
        protected function allowedParams() {
                return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
        }

        protected function doPrecheck( array &$predicates ) {
                $status = Status::newGood();
                // Check if the source file exists
                if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
                        $status->fatal( 'backend-fail-notexists', $this->params['src'] );
                        return $status;
                // Check if a file can be placed at the destination
                } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
                        $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
                        return $status;
                }
                // Check if destination file exists
                $status->merge( $this->precheckDestExistence( $predicates ) );
                if ( $status->isOK() ) {
                        // Update file existence predicates
                        $predicates['exists'][$this->params['dst']] = true;
                        $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
                }
                return $status; // safe to call attempt()
        }

        protected function doAttempt() {
                $status = Status::newGood();
                // Do nothing if the src/dst paths are the same
                if ( $this->params['src'] !== $this->params['dst'] ) {
                        // Copy the file into the destination
                        if ( !$this->destSameAsSource ) {
                                $status->merge( $this->backend->copyInternal( $this->params ) );
                        }
                }
                return $status;
        }

        public function storagePathsRead() {
                return array( $this->params['src'] );
        }

        public function storagePathsChanged() {
                return array( $this->params['dst'] );
        }
}

class MoveFileOp extends FileOp {
        protected function allowedParams() {
                return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
        }

        protected function doPrecheck( array &$predicates ) {
                $status = Status::newGood();
                // Check if the source file exists
                if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
                        $status->fatal( 'backend-fail-notexists', $this->params['src'] );
                        return $status;
                // Check if a file can be placed at the destination
                } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
                        $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
                        return $status;
                }
                // Check if destination file exists
                $status->merge( $this->precheckDestExistence( $predicates ) );
                if ( $status->isOK() ) {
                        // Update file existence predicates
                        $predicates['exists'][$this->params['src']] = false;
                        $predicates['sha1'][$this->params['src']] = false;
                        $predicates['exists'][$this->params['dst']] = true;
                        $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
                }
                return $status; // safe to call attempt()
        }

        protected function doAttempt() {
                $status = Status::newGood();
                // Do nothing if the src/dst paths are the same
                if ( $this->params['src'] !== $this->params['dst'] ) {
                        if ( !$this->destSameAsSource ) {
                                // Move the file into the destination
                                $status->merge( $this->backend->moveInternal( $this->params ) );
                        } else {
                                // Just delete source as the destination needs no changes
                                $params = array( 'src' => $this->params['src'] );
                                $status->merge( $this->backend->deleteInternal( $params ) );
                        }
                }
                return $status;
        }

        public function storagePathsRead() {
                return array( $this->params['src'] );
        }

        public function storagePathsChanged() {
                return array( $this->params['dst'] );
        }
}

class DeleteFileOp extends FileOp {
        protected function allowedParams() {
                return array( array( 'src' ), array( 'ignoreMissingSource' ) );
        }

        protected $needsDelete = true;

        protected function doPrecheck( array &$predicates ) {
                $status = Status::newGood();
                // Check if the source file exists
                if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
                        if ( !$this->getParam( 'ignoreMissingSource' ) ) {
                                $status->fatal( 'backend-fail-notexists', $this->params['src'] );
                                return $status;
                        }
                        $this->needsDelete = false;
                }
                // Update file existence predicates
                $predicates['exists'][$this->params['src']] = false;
                $predicates['sha1'][$this->params['src']] = false;
                return $status; // safe to call attempt()
        }

        protected function doAttempt() {
                $status = Status::newGood();
                if ( $this->needsDelete ) {
                        // Delete the source file
                        $status->merge( $this->backend->deleteInternal( $this->params ) );
                }
                return $status;
        }

        public function storagePathsChanged() {
                return array( $this->params['src'] );
        }
}

class NullFileOp extends FileOp {}