MediaWiki
REL1_22
|
00001 <?php 00024 require_once __DIR__ . '/../Maintenance.php'; 00025 00031 class GenerateCollationData extends Maintenance { 00033 public $dataDir; 00034 00036 public $weights; 00037 00043 public $mappedChars; 00044 00045 public $debugOutFile; 00046 00050 const NORMAL_UPPERCASE = 0x08; 00051 const NORMAL_HIRAGANA = 0X0E; 00052 00053 public function __construct() { 00054 parent::__construct(); 00055 $this->addOption( 'data-dir', 'A directory on the local filesystem ' . 00056 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org', 00057 false, true ); 00058 $this->addOption( 'debug-output', 'Filename for sending debug output to', 00059 false, true ); 00060 } 00061 00062 public function execute() { 00063 $this->dataDir = $this->getOption( 'data-dir', '.' ); 00064 00065 $allkeysPresent = file_exists( "{$this->dataDir}/allkeys.txt" ); 00066 $ucdallPresent = file_exists( "{$this->dataDir}/ucd.all.grouped.xml" ); 00067 00068 // As of January 2013, these links work for all versions of Unicode 00069 // between 5.1 and 6.2, inclusive. 00070 $allkeysURL = "http://www.unicode.org/Public/UCA/<Unicode version>/allkeys.txt"; 00071 $ucdallURL = "http://www.unicode.org/Public/<Unicode version>/ucdxml/ucd.all.grouped.zip"; 00072 00073 if ( !$allkeysPresent || !$ucdallPresent ) { 00074 $icuVersion = IcuCollation::getICUVersion(); 00075 $unicodeVersion = IcuCollation::getUnicodeVersionForICU(); 00076 00077 $error = ""; 00078 00079 if ( !$allkeysPresent ) { 00080 $error .= "Unable to find allkeys.txt. " 00081 . "Download it and specify its location with --data-dir=<DIR>. " 00082 . "\n\n"; 00083 } 00084 if ( !$ucdallPresent ) { 00085 $error .= "Unable to find ucd.all.grouped.xml. " 00086 . "Download it, unzip, and specify its location with --data-dir=<DIR>. " 00087 . "\n\n"; 00088 } 00089 00090 $versionKnown = false; 00091 if ( !$icuVersion ) { 00092 // Unknown version - either very old intl, 00093 // or PHP < 5.3.7 which does not expose this information 00094 $error .= "As MediaWiki could not determine the version of ICU library used by your PHP's " 00095 . "intl extension it can't suggest which file version to download. " 00096 . "This can be caused by running a very old version of intl or PHP < 5.3.7. " 00097 . "If you are sure everything is all right, find out the ICU version " 00098 . "by running phpinfo(), check what is the Unicode version it is using " 00099 . "at http://site.icu-project.org/download, then try finding appropriate data file(s) at:"; 00100 } elseif ( version_compare( $icuVersion, "4.0", "<" ) ) { 00101 // Extra old version 00102 $error .= "You are using outdated version of ICU ($icuVersion), intended for " 00103 . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" ) 00104 . "; this file might not be avalaible for it, and it's not supported by MediaWiki. " 00105 . " You are on your own; consider upgrading PHP's intl extension or try " 00106 . "one of the files available at:"; 00107 } elseif ( version_compare( $icuVersion, "51.0", ">=" ) ) { 00108 // Extra recent version 00109 $error .= "You are using ICU $icuVersion, released after this script was last updated. " 00110 . "Check what is the Unicode version it is using at http://site.icu-project.org/download . " 00111 . "It can't be guaranteed everything will work, but appropriate file(s) should " 00112 . "be available at:"; 00113 } else { 00114 // ICU 4.0 to 50.x 00115 $versionKnown = true; 00116 $error .= "You are using ICU $icuVersion, intended for " 00117 . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" ) 00118 . ". Appropriate file(s) should be available at:"; 00119 } 00120 $error .= "\n"; 00121 00122 if ( $versionKnown && $unicodeVersion ) { 00123 $allkeysURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $allkeysURL ); 00124 $ucdallURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $ucdallURL ); 00125 } 00126 00127 if ( !$allkeysPresent ) { 00128 $error .= "* $allkeysURL\n"; 00129 } 00130 if ( !$ucdallPresent ) { 00131 $error .= "* $ucdallURL\n"; 00132 } 00133 00134 $this->error( $error ); 00135 exit( 1 ); 00136 } 00137 00138 $debugOutFileName = $this->getOption( 'debug-output' ); 00139 if ( $debugOutFileName ) { 00140 $this->debugOutFile = fopen( $debugOutFileName, 'w' ); 00141 if ( !$this->debugOutFile ) { 00142 $this->error( "Unable to open debug output file for writing" ); 00143 exit( 1 ); 00144 } 00145 } 00146 $this->loadUcd(); 00147 $this->generateFirstChars(); 00148 } 00149 00150 function loadUcd() { 00151 $uxr = new UcdXmlReader( "{$this->dataDir}/ucd.all.grouped.xml" ); 00152 $uxr->readChars( array( $this, 'charCallback' ) ); 00153 } 00154 00155 function charCallback( $data ) { 00156 // Skip non-printable characters, 00157 // but do not skip a normal space (U+0020) since 00158 // people like to use that as a fake no header symbol. 00159 $category = substr( $data['gc'], 0, 1 ); 00160 if ( strpos( 'LNPS', $category ) === false 00161 && $data['cp'] !== '0020' ) { 00162 return; 00163 } 00164 $cp = hexdec( $data['cp'] ); 00165 00166 // Skip the CJK ideograph blocks, as an optimisation measure. 00167 // UCA doesn't sort them properly anyway, without tailoring. 00168 if ( IcuCollation::isCjk( $cp ) ) { 00169 return; 00170 } 00171 00172 // Skip the composed Hangul syllables, we will use the bare Jamo 00173 // as first letters 00174 if ( $data['block'] == 'Hangul Syllables' ) { 00175 return; 00176 } 00177 00178 // Calculate implicit weight per UTS #10 v6.0.0, sec 7.1.3 00179 if ( $data['UIdeo'] === 'Y' ) { 00180 if ( $data['block'] == 'CJK Unified Ideographs' 00181 || $data['block'] == 'CJK Compatibility Ideographs' ) 00182 { 00183 $base = 0xFB40; 00184 } else { 00185 $base = 0xFB80; 00186 } 00187 } else { 00188 $base = 0xFBC0; 00189 } 00190 $a = $base + ( $cp >> 15 ); 00191 $b = ( $cp & 0x7fff ) | 0x8000; 00192 00193 $this->weights[$cp] = sprintf( ".%04X.%04X", $a, $b ); 00194 00195 if ( $data['dm'] !== '#' ) { 00196 $this->mappedChars[$cp] = true; 00197 } 00198 00199 if ( $cp % 4096 == 0 ) { 00200 print "{$data['cp']}\n"; 00201 } 00202 } 00203 00204 function generateFirstChars() { 00205 $file = fopen( "{$this->dataDir}/allkeys.txt", 'r' ); 00206 if ( !$file ) { 00207 $this->error( "Unable to open allkeys.txt" ); 00208 exit( 1 ); 00209 } 00210 global $IP; 00211 $outFile = fopen( "$IP/serialized/first-letters-root.ser", 'w' ); 00212 if ( !$outFile ) { 00213 $this->error( "Unable to open output file first-letters-root.ser" ); 00214 exit( 1 ); 00215 } 00216 00217 $goodTertiaryChars = array(); 00218 00219 // For each character with an entry in allkeys.txt, overwrite the implicit 00220 // entry in $this->weights that came from the UCD. 00221 // Also gather a list of tertiary weights, for use in selecting the group header 00222 while ( false !== ( $line = fgets( $file ) ) ) { 00223 // We're only interested in single-character weights, pick them out with a regex 00224 $line = trim( $line ); 00225 if ( !preg_match( '/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) { 00226 continue; 00227 } 00228 00229 $cp = hexdec( $m[1] ); 00230 $allWeights = trim( $m[2] ); 00231 $primary = ''; 00232 $tertiary = ''; 00233 00234 if ( !isset( $this->weights[$cp] ) ) { 00235 // Non-printable, ignore 00236 continue; 00237 } 00238 foreach ( StringUtils::explode( '[', $allWeights ) as $weightStr ) { 00239 preg_match_all( '/[*.]([0-9A-F]+)/', $weightStr, $m ); 00240 if ( !empty( $m[1] ) ) { 00241 if ( $m[1][0] !== '0000' ) { 00242 $primary .= '.' . $m[1][0]; 00243 } 00244 if ( $m[1][2] !== '0000' ) { 00245 $tertiary .= '.' . $m[1][2]; 00246 } 00247 } 00248 } 00249 $this->weights[$cp] = $primary; 00250 if ( $tertiary === '.0008' 00251 || $tertiary === '.000E' ) 00252 { 00253 $goodTertiaryChars[$cp] = true; 00254 } 00255 } 00256 fclose( $file ); 00257 00258 // Identify groups of characters with the same primary weight 00259 $this->groups = array(); 00260 asort( $this->weights, SORT_STRING ); 00261 $prevWeight = reset( $this->weights ); 00262 $group = array(); 00263 foreach ( $this->weights as $cp => $weight ) { 00264 if ( $weight !== $prevWeight ) { 00265 $this->groups[$prevWeight] = $group; 00266 $prevWeight = $weight; 00267 if ( isset( $this->groups[$weight] ) ) { 00268 $group = $this->groups[$weight]; 00269 } else { 00270 $group = array(); 00271 } 00272 } 00273 $group[] = $cp; 00274 } 00275 if ( $group ) { 00276 $this->groups[$prevWeight] = $group; 00277 } 00278 00279 // If one character has a given primary weight sequence, and a second 00280 // character has a longer primary weight sequence with an initial 00281 // portion equal to the first character, then remove the second 00282 // character. This avoids having characters like U+A732 (double A) 00283 // polluting the basic latin sort area. 00284 00285 foreach ( $this->groups as $weight => $group ) { 00286 if ( preg_match( '/(\.[0-9A-F]*)\./', $weight, $m ) ) { 00287 if ( isset( $this->groups[$m[1]] ) ) { 00288 unset( $this->groups[$weight] ); 00289 } 00290 } 00291 } 00292 00293 ksort( $this->groups, SORT_STRING ); 00294 00295 // Identify the header character in each group 00296 $headerChars = array(); 00297 $prevChar = "\000"; 00298 $tertiaryCollator = new Collator( 'root' ); 00299 $primaryCollator = new Collator( 'root' ); 00300 $primaryCollator->setStrength( Collator::PRIMARY ); 00301 $numOutOfOrder = 0; 00302 foreach ( $this->groups as $weight => $group ) { 00303 $uncomposedChars = array(); 00304 $goodChars = array(); 00305 foreach ( $group as $cp ) { 00306 if ( isset( $goodTertiaryChars[$cp] ) ) { 00307 $goodChars[] = $cp; 00308 } 00309 if ( !isset( $this->mappedChars[$cp] ) ) { 00310 $uncomposedChars[] = $cp; 00311 } 00312 } 00313 $x = array_intersect( $goodChars, $uncomposedChars ); 00314 if ( !$x ) { 00315 $x = $uncomposedChars; 00316 if ( !$x ) { 00317 $x = $group; 00318 } 00319 } 00320 00321 // Use ICU to pick the lowest sorting character in the selection 00322 $tertiaryCollator->sort( $x ); 00323 $cp = $x[0]; 00324 00325 $char = codepointToUtf8( $cp ); 00326 $headerChars[] = $char; 00327 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) { 00328 $numOutOfOrder ++; 00329 /* 00330 printf( "Out of order: U+%05X > U+%05X\n", 00331 utf8ToCodepoint( $prevChar ), 00332 utf8ToCodepoint( $char ) ); 00333 */ 00334 } 00335 $prevChar = $char; 00336 00337 if ( $this->debugOutFile ) { 00338 fwrite( $this->debugOutFile, sprintf( "%05X %s %s (%s)\n", $cp, $weight, $char, 00339 implode( ' ', array_map( 'codepointToUtf8', $group ) ) ) ); 00340 } 00341 } 00342 00343 print "Out of order: $numOutOfOrder / " . count( $headerChars ) . "\n"; 00344 00345 fwrite( $outFile, serialize( $headerChars ) ); 00346 } 00347 } 00348 00349 class UcdXmlReader { 00350 public $fileName; 00351 public $callback; 00352 public $groupAttrs; 00353 public $xml; 00354 public $blocks = array(); 00355 public $currentBlock; 00356 00357 function __construct( $fileName ) { 00358 $this->fileName = $fileName; 00359 } 00360 00361 public function readChars( $callback ) { 00362 $this->getBlocks(); 00363 $this->currentBlock = reset( $this->blocks ); 00364 $xml = $this->open(); 00365 $this->callback = $callback; 00366 00367 while ( $xml->name !== 'repertoire' && $xml->next() ); 00368 00369 while ( $xml->read() ) { 00370 if ( $xml->nodeType == XMLReader::ELEMENT ) { 00371 if ( $xml->name === 'group' ) { 00372 $this->groupAttrs = $this->readAttributes(); 00373 } elseif ( $xml->name === 'char' ) { 00374 $this->handleChar(); 00375 } 00376 } elseif ( $xml->nodeType === XMLReader::END_ELEMENT ) { 00377 if ( $xml->name === 'group' ) { 00378 $this->groupAttrs = array(); 00379 } 00380 } 00381 } 00382 $xml->close(); 00383 } 00384 00385 protected function open() { 00386 $this->xml = new XMLReader; 00387 $this->xml->open( $this->fileName ); 00388 if ( !$this->xml ) { 00389 throw new MWException( __METHOD__ . ": unable to open {$this->fileName}" ); 00390 } 00391 while ( $this->xml->name !== 'ucd' && $this->xml->read() ); 00392 $this->xml->read(); 00393 return $this->xml; 00394 } 00395 00401 protected function readAttributes() { 00402 $attrs = array(); 00403 while ( $this->xml->moveToNextAttribute() ) { 00404 $attrs[$this->xml->name] = $this->xml->value; 00405 } 00406 return $attrs; 00407 } 00408 00409 protected function handleChar() { 00410 $attrs = $this->readAttributes() + $this->groupAttrs; 00411 if ( isset( $attrs['cp'] ) ) { 00412 $first = $last = hexdec( $attrs['cp'] ); 00413 } else { 00414 $first = hexdec( $attrs['first-cp'] ); 00415 $last = hexdec( $attrs['last-cp'] ); 00416 unset( $attrs['first-cp'] ); 00417 unset( $attrs['last-cp'] ); 00418 } 00419 00420 for ( $cp = $first; $cp <= $last; $cp++ ) { 00421 $hexCp = sprintf( "%04X", $cp ); 00422 foreach ( array( 'na', 'na1' ) as $nameProp ) { 00423 if ( isset( $attrs[$nameProp] ) ) { 00424 $attrs[$nameProp] = str_replace( '#', $hexCp, $attrs[$nameProp] ); 00425 } 00426 } 00427 00428 while ( $this->currentBlock ) { 00429 if ( $cp < $this->currentBlock[0] ) { 00430 break; 00431 } elseif ( $cp <= $this->currentBlock[1] ) { 00432 $attrs['block'] = key( $this->blocks ); 00433 break; 00434 } else { 00435 $this->currentBlock = next( $this->blocks ); 00436 } 00437 } 00438 00439 $attrs['cp'] = $hexCp; 00440 call_user_func( $this->callback, $attrs ); 00441 } 00442 } 00443 00444 public function getBlocks() { 00445 if ( $this->blocks ) { 00446 return $this->blocks; 00447 } 00448 00449 $xml = $this->open(); 00450 while ( $xml->name !== 'blocks' && $xml->read() ); 00451 00452 while ( $xml->read() ) { 00453 if ( $xml->nodeType == XMLReader::ELEMENT ) { 00454 if ( $xml->name === 'block' ) { 00455 $attrs = $this->readAttributes(); 00456 $first = hexdec( $attrs['first-cp'] ); 00457 $last = hexdec( $attrs['last-cp'] ); 00458 $this->blocks[$attrs['name']] = array( $first, $last ); 00459 } 00460 } 00461 } 00462 $xml->close(); 00463 return $this->blocks; 00464 } 00465 00466 } 00467 00468 $maintClass = 'GenerateCollationData'; 00469 require_once RUN_MAINTENANCE_IF_MAIN;