- Generated on Wed Oct 1 2014 21:36:41 for MediaWiki by
1.7.6.1
|
MediaWiki
REL1_19
|
00001 <?php 00024 if ( !defined( 'MEDIAWIKI' ) ) { 00025 require_once( dirname( __FILE__ ) . '/../commandLine.inc' ); 00026 00027 $cs = new CheckStorage; 00028 $fix = isset( $options['fix'] ); 00029 if ( isset( $args[0] ) ) { 00030 $xml = $args[0]; 00031 } else { 00032 $xml = false; 00033 } 00034 $cs->check( $fix, $xml ); 00035 } 00036 00037 00038 // ---------------------------------------------------------------------------------- 00039 00043 class CheckStorage { 00044 const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"'; 00045 var $oldIdMap, $errors; 00046 var $dbStore = null; 00047 00048 var $errorDescriptions = array( 00049 'restore text' => 'Damaged text, need to be restored from a backup', 00050 'restore revision' => 'Damaged revision row, need to be restored from a backup', 00051 'unfixable' => 'Unexpected errors with no automated fixing method', 00052 'fixed' => 'Errors already fixed', 00053 'fixable' => 'Errors which would already be fixed if --fix was specified', 00054 ); 00055 00056 function check( $fix = false, $xml = '' ) { 00057 $dbr = wfGetDB( DB_SLAVE ); 00058 if ( $fix ) { 00059 print "Checking, will fix errors if possible...\n"; 00060 } else { 00061 print "Checking...\n"; 00062 } 00063 $maxRevId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, __METHOD__ ); 00064 $chunkSize = 1000; 00065 $flagStats = array(); 00066 $objectStats = array(); 00067 $knownFlags = array( 'external', 'gzip', 'object', 'utf-8' ); 00068 $this->errors = array( 00069 'restore text' => array(), 00070 'restore revision' => array(), 00071 'unfixable' => array(), 00072 'fixed' => array(), 00073 'fixable' => array(), 00074 ); 00075 00076 for ( $chunkStart = 1 ; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) { 00077 $chunkEnd = $chunkStart + $chunkSize - 1; 00078 // print "$chunkStart of $maxRevId\n"; 00079 00080 // Fetch revision rows 00081 $this->oldIdMap = array(); 00082 $dbr->ping(); 00083 $res = $dbr->select( 'revision', array( 'rev_id', 'rev_text_id' ), 00084 array( "rev_id BETWEEN $chunkStart AND $chunkEnd" ), __METHOD__ ); 00085 foreach ( $res as $row ) { 00086 $this->oldIdMap[$row->rev_id] = $row->rev_text_id; 00087 } 00088 $dbr->freeResult( $res ); 00089 00090 if ( !count( $this->oldIdMap ) ) { 00091 continue; 00092 } 00093 00094 // Fetch old_flags 00095 $missingTextRows = array_flip( $this->oldIdMap ); 00096 $externalRevs = array(); 00097 $objectRevs = array(); 00098 $res = $dbr->select( 'text', array( 'old_id', 'old_flags' ), 00099 'old_id IN (' . implode( ',', $this->oldIdMap ) . ')', __METHOD__ ); 00100 foreach ( $res as $row ) { 00104 $flags = $row->old_flags; 00105 $id = $row->old_id; 00106 00107 // Create flagStats row if it doesn't exist 00108 $flagStats = $flagStats + array( $flags => 0 ); 00109 // Increment counter 00110 $flagStats[$flags]++; 00111 00112 // Not missing 00113 unset( $missingTextRows[$row->old_id] ); 00114 00115 // Check for external or object 00116 if ( $flags == '' ) { 00117 $flagArray = array(); 00118 } else { 00119 $flagArray = explode( ',', $flags ); 00120 } 00121 if ( in_array( 'external', $flagArray ) ) { 00122 $externalRevs[] = $id; 00123 } elseif ( in_array( 'object', $flagArray ) ) { 00124 $objectRevs[] = $id; 00125 } 00126 00127 // Check for unrecognised flags 00128 if ( $flags == '0' ) { 00129 // This is a known bug from 2004 00130 // It's safe to just erase the old_flags field 00131 if ( $fix ) { 00132 $this->error( 'fixed', "Warning: old_flags set to 0", $id ); 00133 $dbw = wfGetDB( DB_MASTER ); 00134 $dbw->ping(); 00135 $dbw->update( 'text', array( 'old_flags' => '' ), 00136 array( 'old_id' => $id ), __METHOD__ ); 00137 echo "Fixed\n"; 00138 } else { 00139 $this->error( 'fixable', "Warning: old_flags set to 0", $id ); 00140 } 00141 } elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) { 00142 $this->error( 'unfixable', "Error: invalid flags field \"$flags\"", $id ); 00143 } 00144 } 00145 $dbr->freeResult( $res ); 00146 00147 // Output errors for any missing text rows 00148 foreach ( $missingTextRows as $oldId => $revId ) { 00149 $this->error( 'restore revision', "Error: missing text row", $oldId ); 00150 } 00151 00152 // Verify external revisions 00153 $externalConcatBlobs = array(); 00154 $externalNormalBlobs = array(); 00155 if ( count( $externalRevs ) ) { 00156 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ), 00157 array( 'old_id IN (' . implode( ',', $externalRevs ) . ')' ), __METHOD__ ); 00158 foreach ( $res as $row ) { 00159 $urlParts = explode( '://', $row->old_text, 2 ); 00160 if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) { 00161 $this->error( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id ); 00162 continue; 00163 } 00164 list( $proto, ) = $urlParts; 00165 if ( $proto != 'DB' ) { 00166 $this->error( 'restore text', "Error: invalid external protocol \"$proto\"", $row->old_id ); 00167 continue; 00168 } 00169 $path = explode( '/', $row->old_text ); 00170 $cluster = $path[2]; 00171 $id = $path[3]; 00172 if ( isset( $path[4] ) ) { 00173 $externalConcatBlobs[$cluster][$id][] = $row->old_id; 00174 } else { 00175 $externalNormalBlobs[$cluster][$id][] = $row->old_id; 00176 } 00177 } 00178 $dbr->freeResult( $res ); 00179 } 00180 00181 // Check external concat blobs for the right header 00182 $this->checkExternalConcatBlobs( $externalConcatBlobs ); 00183 00184 // Check external normal blobs for existence 00185 if ( count( $externalNormalBlobs ) ) { 00186 if ( is_null( $this->dbStore ) ) { 00187 $this->dbStore = new ExternalStoreDB; 00188 } 00189 foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) { 00190 $blobIds = array_keys( $xBlobIds ); 00191 $extDb =& $this->dbStore->getSlave( $cluster ); 00192 $blobsTable = $this->dbStore->getTable( $extDb ); 00193 $res = $extDb->select( $blobsTable, 00194 array( 'blob_id' ), 00195 array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), __METHOD__ ); 00196 foreach ( $res as $row ) { 00197 unset( $xBlobIds[$row->blob_id] ); 00198 } 00199 $extDb->freeResult( $res ); 00200 // Print errors for missing blobs rows 00201 foreach ( $xBlobIds as $blobId => $oldId ) { 00202 $this->error( 'restore text', "Error: missing target $blobId for one-part ES URL", $oldId ); 00203 } 00204 } 00205 } 00206 00207 // Check local objects 00208 $dbr->ping(); 00209 $concatBlobs = array(); 00210 $curIds = array(); 00211 if ( count( $objectRevs ) ) { 00212 $headerLength = 300; 00213 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ), 00214 array( 'old_id IN (' . implode( ',', $objectRevs ) . ')' ), __METHOD__ ); 00215 foreach ( $res as $row ) { 00216 $oldId = $row->old_id; 00217 $matches = array(); 00218 if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) { 00219 $this->error( 'restore text', "Error: invalid object header", $oldId ); 00220 continue; 00221 } 00222 00223 $className = strtolower( $matches[2] ); 00224 if ( strlen( $className ) != $matches[1] ) { 00225 $this->error( 'restore text', "Error: invalid object header, wrong class name length", $oldId ); 00226 continue; 00227 } 00228 00229 $objectStats = $objectStats + array( $className => 0 ); 00230 $objectStats[$className]++; 00231 00232 switch ( $className ) { 00233 case 'concatenatedgziphistoryblob': 00234 // Good 00235 break; 00236 case 'historyblobstub': 00237 case 'historyblobcurstub': 00238 if ( strlen( $row->header ) == $headerLength ) { 00239 $this->error( 'unfixable', "Error: overlong stub header", $oldId ); 00240 continue; 00241 } 00242 $stubObj = unserialize( $row->header ); 00243 if ( !is_object( $stubObj ) ) { 00244 $this->error( 'restore text', "Error: unable to unserialize stub object", $oldId ); 00245 continue; 00246 } 00247 if ( $className == 'historyblobstub' ) { 00248 $concatBlobs[$stubObj->mOldId][] = $oldId; 00249 } else { 00250 $curIds[$stubObj->mCurId][] = $oldId; 00251 } 00252 break; 00253 default: 00254 $this->error( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId ); 00255 } 00256 } 00257 $dbr->freeResult( $res ); 00258 } 00259 00260 // Check local concat blob validity 00261 $externalConcatBlobs = array(); 00262 if ( count( $concatBlobs ) ) { 00263 $headerLength = 300; 00264 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ), 00265 array( 'old_id IN (' . implode( ',', array_keys( $concatBlobs ) ) . ')' ), __METHOD__ ); 00266 foreach ( $res as $row ) { 00267 $flags = explode( ',', $row->old_flags ); 00268 if ( in_array( 'external', $flags ) ) { 00269 // Concat blob is in external storage? 00270 if ( in_array( 'object', $flags ) ) { 00271 $urlParts = explode( '/', $row->header ); 00272 if ( $urlParts[0] != 'DB:' ) { 00273 $this->error( 'unfixable', "Error: unrecognised external storage type \"{$urlParts[0]}", $row->old_id ); 00274 } else { 00275 $cluster = $urlParts[2]; 00276 $id = $urlParts[3]; 00277 if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) { 00278 $externalConcatBlobs[$cluster][$id] = array(); 00279 } 00280 $externalConcatBlobs[$cluster][$id] = array_merge( 00281 $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id] 00282 ); 00283 } 00284 } else { 00285 $this->error( 'unfixable', "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}", 00286 $concatBlobs[$row->old_id] ); 00287 } 00288 } elseif ( strcasecmp( substr( $row->header, 0, strlen( self::CONCAT_HEADER ) ), self::CONCAT_HEADER ) ) { 00289 $this->error( 'restore text', "Error: Incorrect object header for concat bulk row {$row->old_id}", 00290 $concatBlobs[$row->old_id] ); 00291 } # else good 00292 00293 unset( $concatBlobs[$row->old_id] ); 00294 } 00295 $dbr->freeResult( $res ); 00296 } 00297 00298 // Check targets of unresolved stubs 00299 $this->checkExternalConcatBlobs( $externalConcatBlobs ); 00300 00301 // next chunk 00302 } 00303 00304 print "\n\nErrors:\n"; 00305 foreach ( $this->errors as $name => $errors ) { 00306 if ( count( $errors ) ) { 00307 $description = $this->errorDescriptions[$name]; 00308 echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n"; 00309 } 00310 } 00311 00312 if ( count( $this->errors['restore text'] ) && $fix ) { 00313 if ( (string)$xml !== '' ) { 00314 $this->restoreText( array_keys( $this->errors['restore text'] ), $xml ); 00315 } else { 00316 echo "Can't fix text, no XML backup specified\n"; 00317 } 00318 } 00319 00320 print "\nFlag statistics:\n"; 00321 $total = array_sum( $flagStats ); 00322 foreach ( $flagStats as $flag => $count ) { 00323 printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 ); 00324 } 00325 print "\nLocal object statistics:\n"; 00326 $total = array_sum( $objectStats ); 00327 foreach ( $objectStats as $className => $count ) { 00328 printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 ); 00329 } 00330 } 00331 00332 00333 function error( $type, $msg, $ids ) { 00334 if ( is_array( $ids ) && count( $ids ) == 1 ) { 00335 $ids = reset( $ids ); 00336 } 00337 if ( is_array( $ids ) ) { 00338 $revIds = array(); 00339 foreach ( $ids as $id ) { 00340 $revIds = array_merge( $revIds, array_keys( $this->oldIdMap, $id ) ); 00341 } 00342 print "$msg in text rows " . implode( ', ', $ids ) . 00343 ", revisions " . implode( ', ', $revIds ) . "\n"; 00344 } else { 00345 $id = $ids; 00346 $revIds = array_keys( $this->oldIdMap, $id ); 00347 if ( count( $revIds ) == 1 ) { 00348 print "$msg in old_id $id, rev_id {$revIds[0]}\n"; 00349 } else { 00350 print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n"; 00351 } 00352 } 00353 $this->errors[$type] = $this->errors[$type] + array_flip( $revIds ); 00354 } 00355 00356 function checkExternalConcatBlobs( $externalConcatBlobs ) { 00357 if ( !count( $externalConcatBlobs ) ) { 00358 return; 00359 } 00360 00361 if ( is_null( $this->dbStore ) ) { 00362 $this->dbStore = new ExternalStoreDB; 00363 } 00364 00365 foreach ( $externalConcatBlobs as $cluster => $oldIds ) { 00366 $blobIds = array_keys( $oldIds ); 00367 $extDb =& $this->dbStore->getSlave( $cluster ); 00368 $blobsTable = $this->dbStore->getTable( $extDb ); 00369 $headerLength = strlen( self::CONCAT_HEADER ); 00370 $res = $extDb->select( $blobsTable, 00371 array( 'blob_id', "LEFT(blob_text, $headerLength) AS header" ), 00372 array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), __METHOD__ ); 00373 foreach ( $res as $row ) { 00374 if ( strcasecmp( $row->header, self::CONCAT_HEADER ) ) { 00375 $this->error( 'restore text', "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL", 00376 $oldIds[$row->blob_id] ); 00377 } 00378 unset( $oldIds[$row->blob_id] ); 00379 00380 } 00381 $extDb->freeResult( $res ); 00382 00383 // Print errors for missing blobs rows 00384 foreach ( $oldIds as $blobId => $oldIds ) { 00385 $this->error( 'restore text', "Error: missing target $cluster/$blobId for two-part ES URL", $oldIds ); 00386 } 00387 } 00388 } 00389 00390 function restoreText( $revIds, $xml ) { 00391 global $wgTmpDirectory, $wgDBname; 00392 00393 if ( !count( $revIds ) ) { 00394 return; 00395 } 00396 00397 print "Restoring text from XML backup...\n"; 00398 00399 $revFileName = "$wgTmpDirectory/broken-revlist-$wgDBname"; 00400 $filteredXmlFileName = "$wgTmpDirectory/filtered-$wgDBname.xml"; 00401 00402 // Write revision list 00403 if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) { 00404 echo "Error writing revision list, can't restore text\n"; 00405 return; 00406 } 00407 00408 // Run mwdumper 00409 echo "Filtering XML dump...\n"; 00410 $exitStatus = 0; 00411 passthru( 'mwdumper ' . 00412 wfEscapeShellArg( 00413 "--output=file:$filteredXmlFileName", 00414 "--filter=revlist:$revFileName", 00415 $xml 00416 ), $exitStatus 00417 ); 00418 00419 if ( $exitStatus ) { 00420 echo "mwdumper died with exit status $exitStatus\n"; 00421 return; 00422 } 00423 00424 $file = fopen( $filteredXmlFileName, 'r' ); 00425 if ( !$file ) { 00426 echo "Unable to open filtered XML file\n"; 00427 return; 00428 } 00429 00430 $dbr = wfGetDB( DB_SLAVE ); 00431 $dbw = wfGetDB( DB_MASTER ); 00432 $dbr->ping(); 00433 $dbw->ping(); 00434 00435 $source = new ImportStreamSource( $file ); 00436 $importer = new WikiImporter( $source ); 00437 $importer->setRevisionCallback( array( &$this, 'importRevision' ) ); 00438 $importer->doImport(); 00439 } 00440 00441 function importRevision( &$revision, &$importer ) { 00442 $id = $revision->getID(); 00443 $text = $revision->getText(); 00444 if ( $text === '' ) { 00445 // This is what happens if the revision was broken at the time the 00446 // dump was made. Unfortunately, it also happens if the revision was 00447 // legitimately blank, so there's no way to tell the difference. To 00448 // be safe, we'll skip it and leave it broken 00449 $id = $id ? $id : ''; 00450 echo "Revision $id is blank in the dump, may have been broken before export\n"; 00451 return; 00452 } 00453 00454 if ( !$id ) { 00455 // No ID, can't import 00456 echo "No id tag in revision, can't import\n"; 00457 return; 00458 } 00459 00460 // Find text row again 00461 $dbr = wfGetDB( DB_SLAVE ); 00462 $oldId = $dbr->selectField( 'revision', 'rev_text_id', array( 'rev_id' => $id ), __METHOD__ ); 00463 if ( !$oldId ) { 00464 echo "Missing revision row for rev_id $id\n"; 00465 return; 00466 } 00467 00468 // Compress the text 00469 $flags = Revision::compressRevisionText( $text ); 00470 00471 // Update the text row 00472 $dbw = wfGetDB( DB_MASTER ); 00473 $dbw->update( 'text', 00474 array( 'old_flags' => $flags, 'old_text' => $text ), 00475 array( 'old_id' => $oldId ), 00476 __METHOD__, array( 'LIMIT' => 1 ) 00477 ); 00478 00479 // Remove it from the unfixed list and add it to the fixed list 00480 unset( $this->errors['restore text'][$id] ); 00481 $this->errors['fixed'][$id] = true; 00482 } 00483 } 00484
1.7.6.1