MediaWiki
REL1_22
|
00001 <?php 00030 class DumpDBZip2Output extends DumpPipeOutput { 00031 function __construct( $file ) { 00032 parent::__construct( "dbzip2", $file ); 00033 } 00034 } 00035 00039 class BackupDumper { 00040 var $reportingInterval = 100; 00041 var $reporting = true; 00042 var $pageCount = 0; 00043 var $revCount = 0; 00044 var $server = null; // use default 00045 var $pages = null; // all pages 00046 var $skipHeader = false; // don't output <mediawiki> and <siteinfo> 00047 var $skipFooter = false; // don't output </mediawiki> 00048 var $startId = 0; 00049 var $endId = 0; 00050 var $revStartId = 0; 00051 var $revEndId = 0; 00052 var $sink = null; // Output filters 00053 var $stubText = false; // include rev_text_id instead of text; for 2-pass dump 00054 var $dumpUploads = false; 00055 var $dumpUploadFileContents = false; 00056 var $lastTime = 0; 00057 var $pageCountLast = 0; 00058 var $revCountLast = 0; 00059 var $ID = 0; 00060 00061 var $outputTypes = array(), $filterTypes = array(); 00062 00070 protected $forcedDb = null; 00071 00075 protected $lb; 00076 00077 function __construct( $args ) { 00078 $this->stderr = fopen( "php://stderr", "wt" ); 00079 00080 // Built-in output and filter plugins 00081 $this->registerOutput( 'file', 'DumpFileOutput' ); 00082 $this->registerOutput( 'gzip', 'DumpGZipOutput' ); 00083 $this->registerOutput( 'bzip2', 'DumpBZip2Output' ); 00084 $this->registerOutput( 'dbzip2', 'DumpDBZip2Output' ); 00085 $this->registerOutput( '7zip', 'Dump7ZipOutput' ); 00086 00087 $this->registerFilter( 'latest', 'DumpLatestFilter' ); 00088 $this->registerFilter( 'notalk', 'DumpNotalkFilter' ); 00089 $this->registerFilter( 'namespace', 'DumpNamespaceFilter' ); 00090 00091 $this->sink = $this->processArgs( $args ); 00092 } 00093 00098 function registerOutput( $name, $class ) { 00099 $this->outputTypes[$name] = $class; 00100 } 00101 00106 function registerFilter( $name, $class ) { 00107 $this->filterTypes[$name] = $class; 00108 } 00109 00117 function loadPlugin( $class, $file ) { 00118 if ( $file != '' ) { 00119 require_once $file; 00120 } 00121 $register = array( $class, 'register' ); 00122 call_user_func_array( $register, array( &$this ) ); 00123 } 00124 00129 function processArgs( $args ) { 00130 $sink = null; 00131 $sinks = array(); 00132 foreach ( $args as $arg ) { 00133 $matches = array(); 00134 if ( preg_match( '/^--(.+?)(?:=(.+?)(?::(.+?))?)?$/', $arg, $matches ) ) { 00135 @list( /* $full */ , $opt, $val, $param ) = $matches; 00136 switch ( $opt ) { 00137 case "plugin": 00138 $this->loadPlugin( $val, $param ); 00139 break; 00140 case "output": 00141 if ( !is_null( $sink ) ) { 00142 $sinks[] = $sink; 00143 } 00144 if ( !isset( $this->outputTypes[$val] ) ) { 00145 $this->fatalError( "Unrecognized output sink type '$val'" ); 00146 } 00147 $type = $this->outputTypes[$val]; 00148 $sink = new $type( $param ); 00149 break; 00150 case "filter": 00151 if ( is_null( $sink ) ) { 00152 $sink = new DumpOutput(); 00153 } 00154 if ( !isset( $this->filterTypes[$val] ) ) { 00155 $this->fatalError( "Unrecognized filter type '$val'" ); 00156 } 00157 $type = $this->filterTypes[$val]; 00158 $filter = new $type( $sink, $param ); 00159 00160 // references are lame in php... 00161 unset( $sink ); 00162 $sink = $filter; 00163 00164 break; 00165 case "report": 00166 $this->reportingInterval = intval( $val ); 00167 break; 00168 case "server": 00169 $this->server = $val; 00170 break; 00171 case "force-normal": 00172 if ( !function_exists( 'utf8_normalize' ) ) { 00173 $this->fatalError( "UTF-8 normalization extension not loaded. " . 00174 "Install or remove --force-normal parameter to use slower code." ); 00175 } 00176 break; 00177 default: 00178 $this->processOption( $opt, $val, $param ); 00179 } 00180 } 00181 } 00182 00183 if ( is_null( $sink ) ) { 00184 $sink = new DumpOutput(); 00185 } 00186 $sinks[] = $sink; 00187 00188 if ( count( $sinks ) > 1 ) { 00189 return new DumpMultiWriter( $sinks ); 00190 } else { 00191 return $sink; 00192 } 00193 } 00194 00195 function processOption( $opt, $val, $param ) { 00196 // extension point for subclasses to add options 00197 } 00198 00199 function dump( $history, $text = WikiExporter::TEXT ) { 00200 # Notice messages will foul up your XML output even if they're 00201 # relatively harmless. 00202 if ( ini_get( 'display_errors' ) ) { 00203 ini_set( 'display_errors', 'stderr' ); 00204 } 00205 00206 $this->initProgress( $history ); 00207 00208 $db = $this->backupDb(); 00209 $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text ); 00210 $exporter->dumpUploads = $this->dumpUploads; 00211 $exporter->dumpUploadFileContents = $this->dumpUploadFileContents; 00212 00213 $wrapper = new ExportProgressFilter( $this->sink, $this ); 00214 $exporter->setOutputSink( $wrapper ); 00215 00216 if ( !$this->skipHeader ) { 00217 $exporter->openStream(); 00218 } 00219 # Log item dumps: all or by range 00220 if ( $history & WikiExporter::LOGS ) { 00221 if ( $this->startId || $this->endId ) { 00222 $exporter->logsByRange( $this->startId, $this->endId ); 00223 } else { 00224 $exporter->allLogs(); 00225 } 00226 # Page dumps: all or by page ID range 00227 } elseif ( is_null( $this->pages ) ) { 00228 if ( $this->startId || $this->endId ) { 00229 $exporter->pagesByRange( $this->startId, $this->endId ); 00230 } elseif ( $this->revStartId || $this->revEndId ) { 00231 $exporter->revsByRange( $this->revStartId, $this->revEndId ); 00232 } else { 00233 $exporter->allPages(); 00234 } 00235 # Dump of specific pages 00236 } else { 00237 $exporter->pagesByName( $this->pages ); 00238 } 00239 00240 if ( !$this->skipFooter ) { 00241 $exporter->closeStream(); 00242 } 00243 00244 $this->report( true ); 00245 } 00246 00253 function initProgress( $history = WikiExporter::FULL ) { 00254 $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision'; 00255 $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id'; 00256 00257 $dbr = $this->forcedDb; 00258 if ( $this->forcedDb === null ) { 00259 $dbr = wfGetDB( DB_SLAVE ); 00260 } 00261 $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ ); 00262 $this->startTime = microtime( true ); 00263 $this->lastTime = $this->startTime; 00264 $this->ID = getmypid(); 00265 } 00266 00273 function backupDb() { 00274 if ( $this->forcedDb !== null ) { 00275 return $this->forcedDb; 00276 } 00277 00278 $this->lb = wfGetLBFactory()->newMainLB(); 00279 $db = $this->lb->getConnection( DB_SLAVE, 'backup' ); 00280 00281 // Discourage the server from disconnecting us if it takes a long time 00282 // to read out the big ol' batch query. 00283 $db->setSessionOptions( array( 'connTimeout' => 3600 * 24 ) ); 00284 00285 return $db; 00286 } 00287 00296 function setDb( DatabaseBase $db = null ) { 00297 $this->forcedDb = $db; 00298 } 00299 00300 function __destruct() { 00301 if ( isset( $this->lb ) ) { 00302 $this->lb->closeAll(); 00303 } 00304 } 00305 00306 function backupServer() { 00307 global $wgDBserver; 00308 return $this->server 00309 ? $this->server 00310 : $wgDBserver; 00311 } 00312 00313 function reportPage() { 00314 $this->pageCount++; 00315 } 00316 00317 function revCount() { 00318 $this->revCount++; 00319 $this->report(); 00320 } 00321 00322 function report( $final = false ) { 00323 if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) { 00324 $this->showReport(); 00325 } 00326 } 00327 00328 function showReport() { 00329 if ( $this->reporting ) { 00330 $now = wfTimestamp( TS_DB ); 00331 $nowts = microtime( true ); 00332 $deltaAll = $nowts - $this->startTime; 00333 $deltaPart = $nowts - $this->lastTime; 00334 $this->pageCountPart = $this->pageCount - $this->pageCountLast; 00335 $this->revCountPart = $this->revCount - $this->revCountLast; 00336 00337 if ( $deltaAll ) { 00338 $portion = $this->revCount / $this->maxCount; 00339 $eta = $this->startTime + $deltaAll / $portion; 00340 $etats = wfTimestamp( TS_DB, intval( $eta ) ); 00341 $pageRate = $this->pageCount / $deltaAll; 00342 $revRate = $this->revCount / $deltaAll; 00343 } else { 00344 $pageRate = '-'; 00345 $revRate = '-'; 00346 $etats = '-'; 00347 } 00348 if ( $deltaPart ) { 00349 $pageRatePart = $this->pageCountPart / $deltaPart; 00350 $revRatePart = $this->revCountPart / $deltaPart; 00351 } else { 00352 $pageRatePart = '-'; 00353 $revRatePart = '-'; 00354 } 00355 $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]", 00356 $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats, $this->maxCount ) ); 00357 $this->lastTime = $nowts; 00358 $this->revCountLast = $this->revCount; 00359 } 00360 } 00361 00362 function progress( $string ) { 00363 fwrite( $this->stderr, $string . "\n" ); 00364 } 00365 00366 function fatalError( $msg ) { 00367 $this->progress( "$msg\n" ); 00368 die( 1 ); 00369 } 00370 } 00371 00372 class ExportProgressFilter extends DumpFilter { 00373 function __construct( &$sink, &$progress ) { 00374 parent::__construct( $sink ); 00375 $this->progress = $progress; 00376 } 00377 00378 function writeClosePage( $string ) { 00379 parent::writeClosePage( $string ); 00380 $this->progress->reportPage(); 00381 } 00382 00383 function writeRevision( $rev, $string ) { 00384 parent::writeRevision( $rev, $string ); 00385 $this->progress->revCount(); 00386 } 00387 }