MediaWiki
REL1_22
|
00001 <?php 00024 require_once __DIR__ . '/Maintenance.php'; 00025 00034 class ConvertLinks extends Maintenance { 00035 private $logPerformance; 00036 00037 public function __construct() { 00038 parent::__construct(); 00039 $this->mDescription = "Convert from the old links schema (string->ID) to the new schema (ID->ID). 00040 The wiki should be put into read-only mode while this script executes"; 00041 00042 $this->addArg( 'logperformance', "Log performance to perfLogFilename.", false ); 00043 $this->addArg( 'perfLogFilename', "Filename where performance is logged if --logperformance was set (defaults to 'convLinksPerf.txt').", false ); 00044 $this->addArg( 'keep-links-table', "Don't overwrite the old links table with the new one, leave the new table at links_temp.", false ); 00045 $this->addArg( 'nokeys', "Don't create keys, and so allow duplicates in the new links table.\n 00046 This gives a huge speed improvement for very large links tables which are MyISAM." /* (What about InnoDB?) */, false ); 00047 } 00048 00049 public function getDbType() { 00050 return Maintenance::DB_ADMIN; 00051 } 00052 00053 public function execute() { 00054 $dbw = wfGetDB( DB_MASTER ); 00055 00056 $type = $dbw->getType(); 00057 if ( $type != 'mysql' ) { 00058 $this->output( "Link table conversion not necessary for $type\n" ); 00059 return; 00060 } 00061 00062 global $wgContLang; 00063 00064 $numBadLinks = $curRowsRead = 0; # counters etc 00065 $totalTuplesInserted = 0; # total tuples INSERTed into links_temp 00066 00067 $reportCurReadProgress = true; # whether or not to give progress reports while reading IDs from cur table 00068 $curReadReportInterval = 1000; # number of rows between progress reports 00069 00070 $reportLinksConvProgress = true; # whether or not to give progress reports during conversion 00071 $linksConvInsertInterval = 1000; # number of rows per INSERT 00072 00073 $initialRowOffset = 0; 00074 # $finalRowOffset = 0; # not used yet; highest row number from links table to process 00075 00076 $overwriteLinksTable = !$this->hasOption( 'keep-links-table' ); 00077 $noKeys = $this->hasOption( 'noKeys' ); 00078 $this->logPerformance = $this->hasOption( 'logperformance' ); 00079 $perfLogFilename = $this->getArg( 'perfLogFilename', "convLinksPerf.txt" ); 00080 00081 # -------------------------------------------------------------------- 00082 00083 list( $cur, $links, $links_temp, $links_backup ) = $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' ); 00084 00085 if ( $dbw->tableExists( 'pagelinks' ) ) { 00086 $this->output( "...have pagelinks; skipping old links table updates\n" ); 00087 return; 00088 } 00089 00090 $res = $dbw->query( "SELECT l_from FROM $links LIMIT 1" ); 00091 if ( $dbw->fieldType( $res, 0 ) == "int" ) { 00092 $this->output( "Schema already converted\n" ); 00093 return; 00094 } 00095 00096 $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links" ); 00097 $row = $dbw->fetchObject( $res ); 00098 $numRows = $row->count; 00099 $dbw->freeResult( $res ); 00100 00101 if ( $numRows == 0 ) { 00102 $this->output( "Updating schema (no rows to convert)...\n" ); 00103 $this->createTempTable(); 00104 } else { 00105 $fh = false; 00106 if ( $this->logPerformance ) { 00107 $fh = fopen ( $perfLogFilename, "w" ); 00108 if ( !$fh ) { 00109 $this->error( "Couldn't open $perfLogFilename" ); 00110 $this->logPerformance = false; 00111 } 00112 } 00113 $baseTime = $startTime = $this->getMicroTime(); 00114 # Create a title -> cur_id map 00115 $this->output( "Loading IDs from $cur table...\n" ); 00116 $this->performanceLog ( $fh, "Reading $numRows rows from cur table...\n" ); 00117 $this->performanceLog ( $fh, "rows read vs seconds elapsed:\n" ); 00118 00119 $dbw->bufferResults( false ); 00120 $res = $dbw->query( "SELECT cur_namespace,cur_title,cur_id FROM $cur" ); 00121 $ids = array(); 00122 00123 foreach ( $res as $row ) { 00124 $title = $row->cur_title; 00125 if ( $row->cur_namespace ) { 00126 $title = $wgContLang->getNsText( $row->cur_namespace ) . ":$title"; 00127 } 00128 $ids[$title] = $row->cur_id; 00129 $curRowsRead++; 00130 if ( $reportCurReadProgress ) { 00131 if ( ( $curRowsRead % $curReadReportInterval ) == 0 ) { 00132 $this->performanceLog( $fh, $curRowsRead . " " . ( $this->getMicroTime() - $baseTime ) . "\n" ); 00133 $this->output( "\t$curRowsRead rows of $cur table read.\n" ); 00134 } 00135 } 00136 } 00137 $dbw->freeResult( $res ); 00138 $dbw->bufferResults( true ); 00139 $this->output( "Finished loading IDs.\n\n" ); 00140 $this->performanceLog( $fh, "Took " . ( $this->getMicroTime() - $baseTime ) . " seconds to load IDs.\n\n" ); 00141 00142 # -------------------------------------------------------------------- 00143 00144 # Now, step through the links table (in chunks of $linksConvInsertInterval rows), 00145 # convert, and write to the new table. 00146 $this->createTempTable(); 00147 $this->performanceLog( $fh, "Resetting timer.\n\n" ); 00148 $baseTime = $this->getMicroTime(); 00149 $this->output( "Processing $numRows rows from $links table...\n" ); 00150 $this->performanceLog( $fh, "Processing $numRows rows from $links table...\n" ); 00151 $this->performanceLog( $fh, "rows inserted vs seconds elapsed:\n" ); 00152 00153 for ( $rowOffset = $initialRowOffset; $rowOffset < $numRows; $rowOffset += $linksConvInsertInterval ) { 00154 $sqlRead = "SELECT * FROM $links "; 00155 $sqlRead = $dbw->limitResult( $sqlRead, $linksConvInsertInterval, $rowOffset ); 00156 $res = $dbw->query( $sqlRead ); 00157 if ( $noKeys ) { 00158 $sqlWrite = array( "INSERT INTO $links_temp (l_from,l_to) VALUES " ); 00159 } else { 00160 $sqlWrite = array( "INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES " ); 00161 } 00162 00163 $tuplesAdded = 0; # no tuples added to INSERT yet 00164 foreach ( $res as $row ) { 00165 $fromTitle = $row->l_from; 00166 if ( array_key_exists( $fromTitle, $ids ) ) { # valid title 00167 $from = $ids[$fromTitle]; 00168 $to = $row->l_to; 00169 if ( $tuplesAdded != 0 ) { 00170 $sqlWrite[] = ","; 00171 } 00172 $sqlWrite[] = "($from,$to)"; 00173 $tuplesAdded++; 00174 } else { # invalid title 00175 $numBadLinks++; 00176 } 00177 } 00178 $dbw->freeResult( $res ); 00179 # $this->output( "rowOffset: $rowOffset\ttuplesAdded: $tuplesAdded\tnumBadLinks: $numBadLinks\n" ); 00180 if ( $tuplesAdded != 0 ) { 00181 if ( $reportLinksConvProgress ) { 00182 $this->output( "Inserting $tuplesAdded tuples into $links_temp..." ); 00183 } 00184 $dbw->query( implode( "", $sqlWrite ) ); 00185 $totalTuplesInserted += $tuplesAdded; 00186 if ( $reportLinksConvProgress ) { 00187 $this->output( " done. Total $totalTuplesInserted tuples inserted.\n" ); 00188 $this->performanceLog( $fh, $totalTuplesInserted . " " . ( $this->getMicroTime() - $baseTime ) . "\n" ); 00189 } 00190 } 00191 } 00192 $this->output( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n\n" ); 00193 $this->performanceLog( $fh, "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n" ); 00194 $this->performanceLog( $fh, "Total execution time: " . ( $this->getMicroTime() - $startTime ) . " seconds.\n" ); 00195 if ( $this->logPerformance ) { 00196 fclose ( $fh ); 00197 } 00198 } 00199 # -------------------------------------------------------------------- 00200 00201 if ( $overwriteLinksTable ) { 00202 # Check for existing links_backup, and delete it if it exists. 00203 $this->output( "Dropping backup links table if it exists..." ); 00204 $dbw->query( "DROP TABLE IF EXISTS $links_backup", __METHOD__ ); 00205 $this->output( " done.\n" ); 00206 00207 # Swap in the new table, and move old links table to links_backup 00208 $this->output( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." ); 00209 $dbw->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", __METHOD__ ); 00210 $this->output( " done.\n\n" ); 00211 00212 $dbw->close(); 00213 $this->output( "Conversion complete. The old table remains at $links_backup;\n" ); 00214 $this->output( "delete at your leisure.\n" ); 00215 } else { 00216 $this->output( "Conversion complete. The converted table is at $links_temp;\n" ); 00217 $this->output( "the original links table is unchanged.\n" ); 00218 } 00219 } 00220 00221 private function createTempTable() { 00222 $dbConn = wfGetDB( DB_MASTER ); 00223 00224 if ( !( $dbConn->isOpen() ) ) { 00225 $this->output( "Opening connection to database failed.\n" ); 00226 return; 00227 } 00228 $links_temp = $dbConn->tableName( 'links_temp' ); 00229 00230 $this->output( "Dropping temporary links table if it exists..." ); 00231 $dbConn->query( "DROP TABLE IF EXISTS $links_temp" ); 00232 $this->output( " done.\n" ); 00233 00234 $this->output( "Creating temporary links table..." ); 00235 if ( $this->hasOption( 'noKeys' ) ) { 00236 $dbConn->query( "CREATE TABLE $links_temp ( " . 00237 "l_from int(8) unsigned NOT NULL default '0', " . 00238 "l_to int(8) unsigned NOT NULL default '0')" ); 00239 } else { 00240 $dbConn->query( "CREATE TABLE $links_temp ( " . 00241 "l_from int(8) unsigned NOT NULL default '0', " . 00242 "l_to int(8) unsigned NOT NULL default '0', " . 00243 "UNIQUE KEY l_from(l_from,l_to), " . 00244 "KEY (l_to))" ); 00245 } 00246 $this->output( " done.\n\n" ); 00247 } 00248 00249 private function performanceLog( $fh, $text ) { 00250 if ( $this->logPerformance ) { 00251 fwrite( $fh, $text ); 00252 } 00253 } 00254 00255 private function getMicroTime() { # return time in seconds, with microsecond accuracy 00256 list( $usec, $sec ) = explode( " ", microtime() ); 00257 return ( (float)$usec + (float)$sec ); 00258 } 00259 } 00260 00261 $maintClass = "ConvertLinks"; 00262 require_once RUN_MAINTENANCE_IF_MAIN;