MediaWiki
REL1_22
|
00001 <?php 00024 require_once __DIR__ . '/Maintenance.php'; 00025 00032 class SyncFileBackend extends Maintenance { 00033 public function __construct() { 00034 parent::__construct(); 00035 $this->mDescription = "Sync one file backend with another using the journal"; 00036 $this->addOption( 'src', 'Name of backend to sync from', true, true ); 00037 $this->addOption( 'dst', 'Name of destination backend to sync', false, true ); 00038 $this->addOption( 'start', 'Starting journal ID', false, true ); 00039 $this->addOption( 'end', 'Ending journal ID', false, true ); 00040 $this->addOption( 'posdir', 'Directory to read/record journal positions', false, true ); 00041 $this->addOption( 'posdump', 'Just dump current journal position into the position dir.' ); 00042 $this->addOption( 'postime', 'For position dumps, get the ID at this time', false, true ); 00043 $this->addOption( 'backoff', 'Stop at entries younger than this age (sec).', false, true ); 00044 $this->addOption( 'verbose', 'Verbose mode', false, false, 'v' ); 00045 $this->setBatchSize( 50 ); 00046 } 00047 00048 public function execute() { 00049 $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) ); 00050 00051 $posDir = $this->getOption( 'posdir' ); 00052 $posFile = $posDir ? $posDir . '/' . wfWikiID() : false; 00053 00054 if ( $this->hasOption( 'posdump' ) ) { 00055 // Just dump the current position into the specified position dir 00056 if ( !$this->hasOption( 'posdir' ) ) { 00057 $this->error( "Param posdir required!", 1 ); 00058 } 00059 if ( $this->hasOption( 'postime' ) ) { 00060 $id = (int)$src->getJournal()->getPositionAtTime( $this->getOption( 'postime' ) ); 00061 $this->output( "Requested journal position is $id.\n" ); 00062 } else { 00063 $id = (int)$src->getJournal()->getCurrentPosition(); 00064 $this->output( "Current journal position is $id.\n" ); 00065 } 00066 if ( file_put_contents( $posFile, $id, LOCK_EX ) !== false ) { 00067 $this->output( "Saved journal position file.\n" ); 00068 } else { 00069 $this->output( "Could not save journal position file.\n" ); 00070 } 00071 if ( $this->isQuiet() ) { 00072 print $id; // give a single machine-readable number 00073 } 00074 return; 00075 } 00076 00077 if ( !$this->hasOption( 'dst' ) ) { 00078 $this->error( "Param dst required!", 1 ); 00079 } 00080 $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) ); 00081 00082 $start = $this->getOption( 'start', 0 ); 00083 if ( !$start && $posFile && is_dir( $posDir ) ) { 00084 $start = is_file( $posFile ) 00085 ? (int)trim( file_get_contents( $posFile ) ) 00086 : 0; 00087 ++$start; // we already did this ID, start with the next one 00088 $startFromPosFile = true; 00089 } else { 00090 $startFromPosFile = false; 00091 } 00092 00093 if ( $this->hasOption( 'backoff' ) ) { 00094 $time = time() - $this->getOption( 'backoff', 0 ); 00095 $end = (int)$src->getJournal()->getPositionAtTime( $time ); 00096 } else { 00097 $end = $this->getOption( 'end', INF ); 00098 } 00099 00100 $this->output( "Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n" ); 00101 $this->output( "Starting journal position is $start.\n" ); 00102 if ( is_finite( $end ) ) { 00103 $this->output( "Ending journal position is $end.\n" ); 00104 } 00105 00106 // Periodically update the position file 00107 $callback = function( $pos ) use ( $startFromPosFile, $posFile, $start ) { 00108 if ( $startFromPosFile && $pos >= $start ) { // successfully advanced 00109 file_put_contents( $posFile, $pos, LOCK_EX ); 00110 } 00111 }; 00112 00113 // Actually sync the dest backend with the reference backend 00114 $lastOKPos = $this->syncBackends( $src, $dst, $start, $end, $callback ); 00115 00116 // Update the sync position file 00117 if ( $startFromPosFile && $lastOKPos >= $start ) { // successfully advanced 00118 if ( file_put_contents( $posFile, $lastOKPos, LOCK_EX ) !== false ) { 00119 $this->output( "Updated journal position file.\n" ); 00120 } else { 00121 $this->output( "Could not update journal position file.\n" ); 00122 } 00123 } 00124 00125 if ( $lastOKPos === false ) { 00126 if ( !$start ) { 00127 $this->output( "No journal entries found.\n" ); 00128 } else { 00129 $this->output( "No new journal entries found.\n" ); 00130 } 00131 } else { 00132 $this->output( "Stopped synchronization at journal position $lastOKPos.\n" ); 00133 } 00134 00135 if ( $this->isQuiet() ) { 00136 print $lastOKPos; // give a single machine-readable number 00137 } 00138 } 00139 00151 protected function syncBackends( 00152 FileBackend $src, FileBackend $dst, $start, $end, Closure $callback 00153 ) { 00154 $lastOKPos = 0; // failed 00155 $first = true; // first batch 00156 00157 if ( $start > $end ) { // sanity 00158 $this->error( "Error: given starting ID greater than ending ID.", 1 ); 00159 } 00160 00161 do { 00162 $limit = min( $this->mBatchSize, $end - $start + 1 ); // don't go pass ending ID 00163 $this->output( "Doing id $start to " . ( $start + $limit - 1 ) . "...\n" ); 00164 00165 $entries = $src->getJournal()->getChangeEntries( $start, $limit, $next ); 00166 $start = $next; // start where we left off next time 00167 if ( $first && !count( $entries ) ) { 00168 return false; // nothing to do 00169 } 00170 $first = false; 00171 00172 $lastPosInBatch = 0; 00173 $pathsInBatch = array(); // changed paths 00174 foreach ( $entries as $entry ) { 00175 if ( $entry['op'] !== 'null' ) { // null ops are just for reference 00176 $pathsInBatch[$entry['path']] = 1; // remove duplicates 00177 } 00178 $lastPosInBatch = $entry['id']; 00179 } 00180 00181 $status = $this->syncFileBatch( array_keys( $pathsInBatch ), $src, $dst ); 00182 if ( $status->isOK() ) { 00183 $lastOKPos = max( $lastOKPos, $lastPosInBatch ); 00184 $callback( $lastOKPos ); // update position file 00185 } else { 00186 $this->error( print_r( $status->getErrorsArray(), true ) ); 00187 break; // no gaps; everything up to $lastPos must be OK 00188 } 00189 00190 if ( !$start ) { 00191 $this->output( "End of journal entries.\n" ); 00192 } 00193 } while ( $start && $start <= $end ); 00194 00195 return $lastOKPos; 00196 } 00197 00206 protected function syncFileBatch( array $paths, FileBackend $src, FileBackend $dst ) { 00207 $status = Status::newGood(); 00208 if ( !count( $paths ) ) { 00209 return $status; // nothing to do 00210 } 00211 00212 // Source: convert internal backend names (FileBackendMultiWrite) to the public one 00213 $sPaths = $this->replaceNamePaths( $paths, $src ); 00214 // Destination: get corresponding path name 00215 $dPaths = $this->replaceNamePaths( $paths, $dst ); 00216 00217 // Lock the live backend paths from modification 00218 $sLock = $src->getScopedFileLocks( $sPaths, LockManager::LOCK_UW, $status ); 00219 $eLock = $dst->getScopedFileLocks( $dPaths, LockManager::LOCK_EX, $status ); 00220 if ( !$status->isOK() ) { 00221 return $status; 00222 } 00223 00224 $ops = array(); 00225 $fsFiles = array(); 00226 foreach ( $sPaths as $i => $sPath ) { 00227 $dPath = $dPaths[$i]; // destination 00228 $sExists = $src->fileExists( array( 'src' => $sPath, 'latest' => 1 ) ); 00229 if ( $sExists === true ) { // exists in source 00230 if ( $this->filesAreSame( $src, $dst, $sPath, $dPath ) ) { 00231 continue; // avoid local copies for non-FS backends 00232 } 00233 // Note: getLocalReference() is fast for FS backends 00234 $fsFile = $src->getLocalReference( array( 'src' => $sPath, 'latest' => 1 ) ); 00235 if ( !$fsFile ) { 00236 $this->error( "Unable to sync '$dPath': could not get local copy." ); 00237 $status->fatal( 'backend-fail-internal', $src->getName() ); 00238 return $status; 00239 } 00240 $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed 00241 // Note: prepare() is usually fast for key/value backends 00242 $status->merge( $dst->prepare( array( 00243 'dir' => dirname( $dPath ), 'bypassReadOnly' => 1 ) ) ); 00244 if ( !$status->isOK() ) { 00245 return $status; 00246 } 00247 $ops[] = array( 'op' => 'store', 00248 'src' => $fsFile->getPath(), 'dst' => $dPath, 'overwrite' => 1 ); 00249 } elseif ( $sExists === false ) { // does not exist in source 00250 $ops[] = array( 'op' => 'delete', 'src' => $dPath, 'ignoreMissingSource' => 1 ); 00251 } else { // error 00252 $this->error( "Unable to sync '$dPath': could not stat file." ); 00253 $status->fatal( 'backend-fail-internal', $src->getName() ); 00254 return $status; 00255 } 00256 } 00257 00258 $t_start = microtime( true ); 00259 $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) ); 00260 if ( !$status->isOK() ) { 00261 sleep( 10 ); // wait and retry copy again 00262 $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) ); 00263 } 00264 $ellapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 ); 00265 if ( $status->isOK() && $this->getOption( 'verbose' ) ) { 00266 $this->output( "Synchronized these file(s) [{$ellapsed_ms}ms]:\n" . 00267 implode( "\n", $dPaths ) . "\n" ); 00268 } 00269 00270 return $status; 00271 } 00272 00279 protected function replaceNamePaths( $paths, FileBackend $backend ) { 00280 return preg_replace( 00281 '!^mwstore://([^/]+)!', 00282 StringUtils::escapeRegexReplacement( "mwstore://" . $backend->getName() ), 00283 $paths // string or array 00284 ); 00285 } 00286 00287 protected function filesAreSame( FileBackend $src, FileBackend $dst, $sPath, $dPath ) { 00288 return ( 00289 ( $src->getFileSize( array( 'src' => $sPath ) ) 00290 === $dst->getFileSize( array( 'src' => $dPath ) ) // short-circuit 00291 ) && ( $src->getFileSha1Base36( array( 'src' => $sPath ) ) 00292 === $dst->getFileSha1Base36( array( 'src' => $dPath ) ) 00293 ) 00294 ); 00295 } 00296 } 00297 00298 $maintClass = "SyncFileBackend"; 00299 require_once RUN_MAINTENANCE_IF_MAIN;