[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * Scan the logging table and purge affected files within a timeframe.
   4   *
   5   * @section LICENSE
   6   * This program is free software; you can redistribute it and/or modify
   7   * it under the terms of the GNU General Public License as published by
   8   * the Free Software Foundation; either version 2 of the License, or
   9   * (at your option) any later version.
  10   *
  11   * This program is distributed in the hope that it will be useful,
  12   * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14   * GNU General Public License for more details.
  15   *
  16   * You should have received a copy of the GNU General Public License along
  17   * with this program; if not, write to the Free Software Foundation, Inc.,
  18   * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19   * http://www.gnu.org/copyleft/gpl.html
  20   *
  21   * @file
  22   * @ingroup Maintenance
  23   */
  24  
  25  require_once  __DIR__ . '/Maintenance.php';
  26  
  27  /**
  28   * Maintenance script that scans the deletion log and purges affected files
  29   * within a timeframe.
  30   *
  31   * @ingroup Maintenance
  32   */
  33  class PurgeChangedFiles extends Maintenance {
  34      /**
  35       * Mapping from type option to log type and actions.
  36       * @var array
  37       */
  38      private static $typeMappings = array(
  39          'created' => array(
  40              'upload' => array( 'upload' ),
  41              'import' => array( 'upload', 'interwiki' ),
  42          ),
  43          'deleted' => array(
  44              'delete' => array( 'delete', 'revision' ),
  45              'suppress' => array( 'delete', 'revision' ),
  46          ),
  47          'modified' => array(
  48              'upload' => array( 'overwrite', 'revert' ),
  49              'move' => array( 'move', 'move_redir' ),
  50          ),
  51      );
  52  
  53      /**
  54       * @var string
  55       */
  56      private $startTimestamp;
  57  
  58      /**
  59       * @var string
  60       */
  61      private $endTimestamp;
  62  
  63  	public function __construct() {
  64          parent::__construct();
  65          $this->mDescription = "Scan the logging table and purge files and thumbnails.";
  66          $this->addOption( 'starttime', 'Starting timestamp', true, true );
  67          $this->addOption( 'endtime', 'Ending timestamp', true, true );
  68          $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
  69              implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
  70          $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
  71          $this->addOption( 'dry-run', 'Do not send purge requests' );
  72          $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
  73          $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
  74          $this->setBatchSize( 100 );
  75      }
  76  
  77  	public function execute() {
  78          global $wgHTCPRouting;
  79  
  80          if ( $this->hasOption( 'htcp-dest' ) ) {
  81              $parts = explode( ':', $this->getOption( 'htcp-dest' ) );
  82              if ( count( $parts ) < 2 ) {
  83                  // Add default htcp port
  84                  $parts[] = '4827';
  85              }
  86  
  87              // Route all HTCP messages to provided host:port
  88              $wgHTCPRouting = array(
  89                  '' => array( 'host' => $parts[0], 'port' => $parts[1] ),
  90              );
  91              $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
  92          }
  93  
  94          // Find out which actions we should be concerned with
  95          $typeOpt = $this->getOption( 'type', 'all' );
  96          $validTypes = array_keys( self::$typeMappings );
  97          if ( $typeOpt === 'all' ) {
  98              // Convert 'all' to all registered types
  99              $typeOpt = implode( ',', $validTypes );
 100          }
 101          $typeList = explode( ',', $typeOpt );
 102          foreach ( $typeList as $type ) {
 103              if ( !in_array( $type, $validTypes ) ) {
 104                  $this->error( "\nERROR: Unknown type: {$type}\n" );
 105                  $this->maybeHelp( true );
 106              }
 107          }
 108  
 109          // Validate the timestamps
 110          $dbr = $this->getDB( DB_SLAVE );
 111          $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
 112          $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
 113  
 114          if ( $this->startTimestamp > $this->endTimestamp ) {
 115              $this->error( "\nERROR: starttime after endtime\n" );
 116              $this->maybeHelp( true );
 117          }
 118  
 119          // Turn on verbose when dry-run is enabled
 120          if ( $this->hasOption( 'dry-run' ) ) {
 121              $this->mOptions['verbose'] = 1;
 122          }
 123  
 124          $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
 125          foreach ( $typeList as $type ) {
 126              $this->verbose( "Checking for {$type} files...\n" );
 127              $this->purgeFromLogType( $type );
 128              if ( !$this->hasOption( 'dry-run' ) ) {
 129                  $this->verbose( "...{$type} files purged.\n\n" );
 130              }
 131          }
 132      }
 133  
 134      /**
 135       * Purge cache and thumbnails for changes of the given type.
 136       *
 137       * @param string $type Type of change to find
 138       */
 139  	protected function purgeFromLogType( $type ) {
 140          $repo = RepoGroup::singleton()->getLocalRepo();
 141          $dbr = $this->getDB( DB_SLAVE );
 142  
 143          foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
 144              $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
 145  
 146              $res = $dbr->select(
 147                  'logging',
 148                  array( 'log_title', 'log_timestamp', 'log_params' ),
 149                  array(
 150                      'log_namespace' => NS_FILE,
 151                      'log_type' => $logType,
 152                      'log_action' => $logActions,
 153                      'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
 154                      'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
 155                  ),
 156                  __METHOD__
 157              );
 158  
 159              $bSize = 0;
 160              foreach ( $res as $row ) {
 161                  $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
 162  
 163                  if ( $this->hasOption( 'dry-run' ) ) {
 164                      $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
 165                      continue;
 166                  }
 167  
 168                  // Purge current version and any versions in oldimage table
 169                  $file->purgeCache();
 170                  $file->purgeHistory();
 171  
 172                  if ( $logType === 'delete' ) {
 173                      // If there is an orphaned storage file... delete it
 174                      if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
 175                          $dpath = $this->getDeletedPath( $repo, $file );
 176                          if ( $repo->fileExists( $dpath ) ) {
 177                              // Sanity check to avoid data loss
 178                              $repo->getBackend()->delete( array( 'src' => $file->getPath() ) );
 179                              $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
 180                          } else {
 181                              $this->error( "File was not deleted: {$file->getPath()}.\n" );
 182                          }
 183                      }
 184  
 185                      // Purge items from fileachive table (rows are likely here)
 186                      $this->purgeFromArchiveTable( $repo, $file );
 187                  } elseif ( $logType === 'move' ) {
 188                      // Purge the target file as well
 189  
 190                      $params = unserialize( $row->log_params );
 191                      if ( isset( $params['4::target'] ) ) {
 192                          $target = $params['4::target'];
 193                          $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
 194                          $targetFile->purgeCache();
 195                          $targetFile->purgeHistory();
 196                          $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
 197                      }
 198                  }
 199  
 200                  $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
 201  
 202                  if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) {
 203                      $bSize = 0;
 204                      // sleep-per-batch is milliseconds, usleep wants micro seconds.
 205                      usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
 206                  }
 207              }
 208          }
 209      }
 210  
 211  	protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
 212          $dbr = $repo->getSlaveDB();
 213          $res = $dbr->select(
 214              'filearchive',
 215              array( 'fa_archive_name' ),
 216              array( 'fa_name' => $file->getName() ),
 217              __METHOD__
 218          );
 219  
 220          foreach ( $res as $row ) {
 221              if ( $row->fa_archive_name === null ) {
 222                  // Was not an old version (current version names checked already)
 223                  continue;
 224              }
 225              $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
 226              // If there is an orphaned storage file still there...delete it
 227              if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
 228                  $dpath = $this->getDeletedPath( $repo, $ofile );
 229                  if ( $repo->fileExists( $dpath ) ) {
 230                      // Sanity check to avoid data loss
 231                      $repo->getBackend()->delete( array( 'src' => $ofile->getPath() ) );
 232                      $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
 233                  } else {
 234                      $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
 235                  }
 236              }
 237              $file->purgeOldThumbnails( $row->fa_archive_name );
 238          }
 239      }
 240  
 241  	protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
 242          $hash = $repo->getFileSha1( $file->getPath() );
 243          $key = "{$hash}.{$file->getExtension()}";
 244  
 245          return $repo->getDeletedHashPath( $key ) . $key;
 246      }
 247  
 248      /**
 249       * Send an output message iff the 'verbose' option has been provided.
 250       *
 251       * @param string $msg Message to output
 252       */
 253  	protected function verbose( $msg ) {
 254          if ( $this->hasOption( 'verbose' ) ) {
 255              $this->output( $msg );
 256          }
 257      }
 258  }
 259  
 260  $maintClass = "PurgeChangedFiles";
 261  require_once RUN_MAINTENANCE_IF_MAIN;