[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * Helper class for representing batch file operations.
   4   *
   5   * This program is free software; you can redistribute it and/or modify
   6   * it under the terms of the GNU General Public License as published by
   7   * the Free Software Foundation; either version 2 of the License, or
   8   * (at your option) any later version.
   9   *
  10   * This program is distributed in the hope that it will be useful,
  11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13   * GNU General Public License for more details.
  14   *
  15   * You should have received a copy of the GNU General Public License along
  16   * with this program; if not, write to the Free Software Foundation, Inc.,
  17   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18   * http://www.gnu.org/copyleft/gpl.html
  19   *
  20   * @file
  21   * @ingroup FileBackend
  22   * @author Aaron Schulz
  23   */
  24  
  25  /**
  26   * Helper class for representing batch file operations.
  27   * Do not use this class from places outside FileBackend.
  28   *
  29   * Methods should avoid throwing exceptions at all costs.
  30   *
  31   * @ingroup FileBackend
  32   * @since 1.20
  33   */
  34  class FileOpBatch {
  35      /* Timeout related parameters */
  36      const MAX_BATCH_SIZE = 1000; // integer
  37  
  38      /**
  39       * Attempt to perform a series of file operations.
  40       * Callers are responsible for handling file locking.
  41       *
  42       * $opts is an array of options, including:
  43       *   - force        : Errors that would normally cause a rollback do not.
  44       *                    The remaining operations are still attempted if any fail.
  45       *   - nonJournaled : Don't log this operation batch in the file journal.
  46       *   - concurrency  : Try to do this many operations in parallel when possible.
  47       *
  48       * The resulting Status will be "OK" unless:
  49       *   - a) unexpected operation errors occurred (network partitions, disk full...)
  50       *   - b) significant operation errors occurred and 'force' was not set
  51       *
  52       * @param array $performOps List of FileOp operations
  53       * @param array $opts Batch operation options
  54       * @param FileJournal $journal Journal to log operations to
  55       * @return Status
  56       */
  57  	public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
  58          $section = new ProfileSection( __METHOD__ );
  59          $status = Status::newGood();
  60  
  61          $n = count( $performOps );
  62          if ( $n > self::MAX_BATCH_SIZE ) {
  63              $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
  64  
  65              return $status;
  66          }
  67  
  68          $batchId = $journal->getTimestampedUUID();
  69          $ignoreErrors = !empty( $opts['force'] );
  70          $journaled = empty( $opts['nonJournaled'] );
  71          $maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1;
  72  
  73          $entries = array(); // file journal entry list
  74          $predicates = FileOp::newPredicates(); // account for previous ops in prechecks
  75          $curBatch = array(); // concurrent FileOp sub-batch accumulation
  76          $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
  77          $pPerformOps = array(); // ordered list of concurrent FileOp sub-batches
  78          $lastBackend = null; // last op backend name
  79          // Do pre-checks for each operation; abort on failure...
  80          foreach ( $performOps as $index => $fileOp ) {
  81              $backendName = $fileOp->getBackend()->getName();
  82              $fileOp->setBatchId( $batchId ); // transaction ID
  83              // Decide if this op can be done concurrently within this sub-batch
  84              // or if a new concurrent sub-batch must be started after this one...
  85              if ( $fileOp->dependsOn( $curBatchDeps )
  86                  || count( $curBatch ) >= $maxConcurrency
  87                  || ( $backendName !== $lastBackend && count( $curBatch ) )
  88              ) {
  89                  $pPerformOps[] = $curBatch; // push this batch
  90                  $curBatch = array(); // start a new sub-batch
  91                  $curBatchDeps = FileOp::newDependencies();
  92              }
  93              $lastBackend = $backendName;
  94              $curBatch[$index] = $fileOp; // keep index
  95              // Update list of affected paths in this batch
  96              $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
  97              // Simulate performing the operation...
  98              $oldPredicates = $predicates;
  99              $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
 100              $status->merge( $subStatus );
 101              if ( $subStatus->isOK() ) {
 102                  if ( $journaled ) { // journal log entries
 103                      $entries = array_merge( $entries,
 104                          $fileOp->getJournalEntries( $oldPredicates, $predicates ) );
 105                  }
 106              } else { // operation failed?
 107                  $status->success[$index] = false;
 108                  ++$status->failCount;
 109                  if ( !$ignoreErrors ) {
 110                      return $status; // abort
 111                  }
 112              }
 113          }
 114          // Push the last sub-batch
 115          if ( count( $curBatch ) ) {
 116              $pPerformOps[] = $curBatch;
 117          }
 118  
 119          // Log the operations in the file journal...
 120          if ( count( $entries ) ) {
 121              $subStatus = $journal->logChangeBatch( $entries, $batchId );
 122              if ( !$subStatus->isOK() ) {
 123                  return $subStatus; // abort
 124              }
 125          }
 126  
 127          if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
 128              $status->setResult( true, $status->value );
 129          }
 130  
 131          // Attempt each operation (in parallel if allowed and possible)...
 132          self::runParallelBatches( $pPerformOps, $status );
 133  
 134          return $status;
 135      }
 136  
 137      /**
 138       * Attempt a list of file operations sub-batches in series.
 139       *
 140       * The operations *in* each sub-batch will be done in parallel.
 141       * The caller is responsible for making sure the operations
 142       * within any given sub-batch do not depend on each other.
 143       * This will abort remaining ops on failure.
 144       *
 145       * @param array $pPerformOps Batches of file ops (batches use original indexes)
 146       * @param Status $status
 147       */
 148  	protected static function runParallelBatches( array $pPerformOps, Status $status ) {
 149          $aborted = false; // set to true on unexpected errors
 150          foreach ( $pPerformOps as $performOpsBatch ) {
 151              if ( $aborted ) { // check batch op abort flag...
 152                  // We can't continue (even with $ignoreErrors) as $predicates is wrong.
 153                  // Log the remaining ops as failed for recovery...
 154                  foreach ( $performOpsBatch as $i => $fileOp ) {
 155                      $status->success[$i] = false;
 156                      ++$status->failCount;
 157                      $performOpsBatch[$i]->logFailure( 'attempt_aborted' );
 158                  }
 159                  continue;
 160              }
 161              $statuses = array();
 162              $opHandles = array();
 163              // Get the backend; all sub-batch ops belong to a single backend
 164              $backend = reset( $performOpsBatch )->getBackend();
 165              // Get the operation handles or actually do it if there is just one.
 166              // If attemptAsync() returns a Status, it was either due to an error
 167              // or the backend does not support async ops and did it synchronously.
 168              foreach ( $performOpsBatch as $i => $fileOp ) {
 169                  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
 170                      // Parallel ops may be disabled in config due to missing dependencies,
 171                      // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
 172                      $subStatus = ( count( $performOpsBatch ) > 1 )
 173                          ? $fileOp->attemptAsync()
 174                          : $fileOp->attempt();
 175                      if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
 176                          $opHandles[$i] = $subStatus->value; // deferred
 177                      } else {
 178                          $statuses[$i] = $subStatus; // done already
 179                      }
 180                  }
 181              }
 182              // Try to do all the operations concurrently...
 183              $statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles );
 184              // Marshall and merge all the responses (blocking)...
 185              foreach ( $performOpsBatch as $i => $fileOp ) {
 186                  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
 187                      $subStatus = $statuses[$i];
 188                      $status->merge( $subStatus );
 189                      if ( $subStatus->isOK() ) {
 190                          $status->success[$i] = true;
 191                          ++$status->successCount;
 192                      } else {
 193                          $status->success[$i] = false;
 194                          ++$status->failCount;
 195                          $aborted = true; // set abort flag; we can't continue
 196                      }
 197                  }
 198              }
 199          }
 200      }
 201  }