[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * Move revision's text to external storage
   4   *
   5   * This program is free software; you can redistribute it and/or modify
   6   * it under the terms of the GNU General Public License as published by
   7   * the Free Software Foundation; either version 2 of the License, or
   8   * (at your option) any later version.
   9   *
  10   * This program is distributed in the hope that it will be useful,
  11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13   * GNU General Public License for more details.
  14   *
  15   * You should have received a copy of the GNU General Public License along
  16   * with this program; if not, write to the Free Software Foundation, Inc.,
  17   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18   * http://www.gnu.org/copyleft/gpl.html
  19   *
  20   * @file
  21   * @ingroup Maintenance ExternalStorage
  22   */
  23  
  24  define( 'REPORTING_INTERVAL', 1 );
  25  
  26  if ( !defined( 'MEDIAWIKI' ) ) {
  27      require_once  __DIR__ . '/../commandLine.inc';
  28      require_once  __DIR__ . '/../../includes/externalstore/ExternalStoreDB.php';
  29      require_once  'resolveStubs.php';
  30  
  31      $fname = 'moveToExternal';
  32  
  33      if ( !isset( $args[0] ) ) {
  34          print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n";
  35          exit;
  36      }
  37  
  38      $cluster = $args[0];
  39      $dbw = wfGetDB( DB_MASTER );
  40  
  41      if ( isset( $options['e'] ) ) {
  42          $maxID = $options['e'];
  43      } else {
  44          $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
  45      }
  46      $minID = isset( $options['s'] ) ? $options['s'] : 1;
  47  
  48      moveToExternal( $cluster, $maxID, $minID );
  49  }
  50  
  51  function moveToExternal( $cluster, $maxID, $minID = 1 ) {
  52      $fname = 'moveToExternal';
  53      $dbw = wfGetDB( DB_MASTER );
  54      $dbr = wfGetDB( DB_SLAVE );
  55  
  56      $count = $maxID - $minID + 1;
  57      $blockSize = 1000;
  58      $numBlocks = ceil( $count / $blockSize );
  59      print "Moving text rows from $minID to $maxID to external storage\n";
  60      $ext = new ExternalStoreDB;
  61      $numMoved = 0;
  62  
  63      for ( $block = 0; $block < $numBlocks; $block++ ) {
  64          $blockStart = $block * $blockSize + $minID;
  65          $blockEnd = $blockStart + $blockSize - 1;
  66  
  67          if ( !( $block % REPORTING_INTERVAL ) ) {
  68              print "oldid=$blockStart, moved=$numMoved\n";
  69              wfWaitForSlaves();
  70          }
  71  
  72          $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
  73              array(
  74                  "old_id BETWEEN $blockStart AND $blockEnd",
  75                  'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
  76              ), $fname );
  77          foreach ( $res as $row ) {
  78              # Resolve stubs
  79              $text = $row->old_text;
  80              $id = $row->old_id;
  81              if ( $row->old_flags === '' ) {
  82                  $flags = 'external';
  83              } else {
  84                  $flags = "{$row->old_flags},external";
  85              }
  86  
  87              if ( strpos( $flags, 'object' ) !== false ) {
  88                  $obj = unserialize( $text );
  89                  $className = strtolower( get_class( $obj ) );
  90                  if ( $className == 'historyblobstub' ) {
  91                      # resolveStub( $id, $row->old_text, $row->old_flags );
  92                      # $numStubs++;
  93                      continue;
  94                  } elseif ( $className == 'historyblobcurstub' ) {
  95                      $text = gzdeflate( $obj->getText() );
  96                      $flags = 'utf-8,gzip,external';
  97                  } elseif ( $className == 'concatenatedgziphistoryblob' ) {
  98                      // Do nothing
  99                  } else {
 100                      print "Warning: unrecognised object class \"$className\"\n";
 101                      continue;
 102                  }
 103              } else {
 104                  $className = false;
 105              }
 106  
 107              if ( strlen( $text ) < 100 && $className === false ) {
 108                  // Don't move tiny revisions
 109                  continue;
 110              }
 111  
 112              # print "Storing "  . strlen( $text ) . " bytes to $url\n";
 113              # print "old_id=$id\n";
 114  
 115              $url = $ext->store( $cluster, $text );
 116              if ( !$url ) {
 117                  print "Error writing to external storage\n";
 118                  exit;
 119              }
 120              $dbw->update( 'text',
 121                  array( 'old_flags' => $flags, 'old_text' => $url ),
 122                  array( 'old_id' => $id ), $fname );
 123              $numMoved++;
 124          }
 125      }
 126  }