MediaWiki  REL1_21
UtfNormal.php
Go to the documentation of this file.
00001 <?php
00031 define( 'NORMALIZE_ICU', function_exists( 'utf8_normalize' ) );
00032 define( 'NORMALIZE_INTL', function_exists( 'normalizer_normalize' ) );
00033 
00048 class UtfNormal {
00052         const UNORM_NONE = 1;
00053         const UNORM_NFD  = 2;
00054         const UNORM_NFKD = 3;
00055         const UNORM_NFC  = 4;
00056         const UNORM_NFKC = 5;
00057         const UNORM_FCD  = 6;
00058         const UNORM_DEFAULT = self::UNORM_NFC;
00059 
00060         static $utfCombiningClass = null;
00061         static $utfCanonicalComp = null;
00062         static $utfCanonicalDecomp = null;
00063 
00064         # Load compatibility decompositions on demand if they are needed.
00065         static $utfCompatibilityDecomp = null;
00066 
00067         static $utfCheckNFC;
00068 
00079         static function cleanUp( $string ) {
00080                 if( NORMALIZE_ICU ) {
00081                         $string = self::replaceForNativeNormalize( $string );
00082 
00083                         # UnicodeString constructor fails if the string ends with a
00084                         # head byte. Add a junk char at the end, we'll strip it off.
00085                         return rtrim( utf8_normalize( $string . "\x01", self::UNORM_NFC ), "\x01" );
00086                 } elseif( NORMALIZE_INTL ) {
00087                         $string = self::replaceForNativeNormalize( $string );
00088                         $norm = normalizer_normalize( $string, Normalizer::FORM_C );
00089                         if( $norm === null || $norm === false ) {
00090                                 # normalizer_normalize will either return false or null
00091                                 # (depending on which doc you read) if invalid utf8 string.
00092                                 # quickIsNFCVerify cleans up invalid sequences.
00093 
00094                                 if( UtfNormal::quickIsNFCVerify( $string ) ) {
00095                                         # if that's true, the string is actually already normal.
00096                                         return $string;
00097                                 } else {
00098                                         # Now we are valid but non-normal
00099                                         return normalizer_normalize( $string, Normalizer::FORM_C );
00100                                 }
00101                         } else {
00102                                 return $norm;
00103                         }
00104                 } elseif( UtfNormal::quickIsNFCVerify( $string ) ) {
00105                         # Side effect -- $string has had UTF-8 errors cleaned up.
00106                         return $string;
00107                 } else {
00108                         return UtfNormal::NFC( $string );
00109                 }
00110         }
00111 
00120         static function toNFC( $string ) {
00121                 if( NORMALIZE_INTL )
00122                         return normalizer_normalize( $string, Normalizer::FORM_C );
00123                 elseif( NORMALIZE_ICU )
00124                         return utf8_normalize( $string, self::UNORM_NFC );
00125                 elseif( UtfNormal::quickIsNFC( $string ) )
00126                         return $string;
00127                 else
00128                         return UtfNormal::NFC( $string );
00129         }
00130 
00138         static function toNFD( $string ) {
00139                 if( NORMALIZE_INTL )
00140                         return normalizer_normalize( $string, Normalizer::FORM_D );
00141                 elseif( NORMALIZE_ICU )
00142                         return utf8_normalize( $string, self::UNORM_NFD );
00143                 elseif( preg_match( '/[\x80-\xff]/', $string ) )
00144                         return UtfNormal::NFD( $string );
00145                 else
00146                         return $string;
00147         }
00148 
00157         static function toNFKC( $string ) {
00158                 if( NORMALIZE_INTL )
00159                         return normalizer_normalize( $string, Normalizer::FORM_KC );
00160                 elseif( NORMALIZE_ICU )
00161                         return utf8_normalize( $string, self::UNORM_NFKC );
00162                 elseif( preg_match( '/[\x80-\xff]/', $string ) )
00163                         return UtfNormal::NFKC( $string );
00164                 else
00165                         return $string;
00166         }
00167 
00176         static function toNFKD( $string ) {
00177                 if( NORMALIZE_INTL )
00178                         return normalizer_normalize( $string, Normalizer::FORM_KD );
00179                 elseif( NORMALIZE_ICU )
00180                         return utf8_normalize( $string, self::UNORM_NFKD );
00181                 elseif( preg_match( '/[\x80-\xff]/', $string ) )
00182                         return UtfNormal::NFKD( $string );
00183                 else
00184                         return $string;
00185         }
00186 
00191         static function loadData() {
00192                 if( !isset( self::$utfCombiningClass ) ) {
00193                         require_once( __DIR__ . '/UtfNormalData.inc' );
00194                 }
00195         }
00196 
00203         static function quickIsNFC( $string ) {
00204                 # ASCII is always valid NFC!
00205                 # If it's pure ASCII, let it through.
00206                 if( !preg_match( '/[\x80-\xff]/', $string ) ) return true;
00207 
00208                 UtfNormal::loadData();
00209                 $len = strlen( $string );
00210                 for( $i = 0; $i < $len; $i++ ) {
00211                         $c = $string[$i];
00212                         $n = ord( $c );
00213                         if( $n < 0x80 ) {
00214                                 continue;
00215                         } elseif( $n >= 0xf0 ) {
00216                                 $c = substr( $string, $i, 4 );
00217                                 $i += 3;
00218                         } elseif( $n >= 0xe0 ) {
00219                                 $c = substr( $string, $i, 3 );
00220                                 $i += 2;
00221                         } elseif( $n >= 0xc0 ) {
00222                                 $c = substr( $string, $i, 2 );
00223                                 $i++;
00224                         }
00225                         if( isset( self::$utfCheckNFC[$c] ) ) {
00226                                 # If it's NO or MAYBE, bail and do the slow check.
00227                                 return false;
00228                         }
00229                         if( isset( self::$utfCombiningClass[$c] ) ) {
00230                                 # Combining character? We might have to do sorting, at least.
00231                                 return false;
00232                         }
00233                 }
00234                 return true;
00235         }
00236 
00243         static function quickIsNFCVerify( &$string ) {
00244                 # Screen out some characters that eg won't be allowed in XML
00245                 $string = preg_replace( '/[\x00-\x08\x0b\x0c\x0e-\x1f]/', UTF8_REPLACEMENT, $string );
00246 
00247                 # ASCII is always valid NFC!
00248                 # If we're only ever given plain ASCII, we can avoid the overhead
00249                 # of initializing the decomposition tables by skipping out early.
00250                 if( !preg_match( '/[\x80-\xff]/', $string ) ) return true;
00251 
00252                 static $checkit = null, $tailBytes = null, $utfCheckOrCombining = null;
00253                 if( !isset( $checkit ) ) {
00254                         # Load/build some scary lookup tables...
00255                         UtfNormal::loadData();
00256 
00257                         $utfCheckOrCombining = array_merge( self::$utfCheckNFC, self::$utfCombiningClass );
00258 
00259                         # Head bytes for sequences which we should do further validity checks
00260                         $checkit = array_flip( array_map( 'chr',
00261                                         array( 0xc0, 0xc1, 0xe0, 0xed, 0xef,
00262                                                    0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
00263                                                    0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff ) ) );
00264 
00265                         # Each UTF-8 head byte is followed by a certain
00266                         # number of tail bytes.
00267                         $tailBytes = array();
00268                         for( $n = 0; $n < 256; $n++ ) {
00269                                 if( $n < 0xc0 ) {
00270                                         $remaining = 0;
00271                                 } elseif( $n < 0xe0 ) {
00272                                         $remaining = 1;
00273                                 } elseif( $n < 0xf0 ) {
00274                                         $remaining = 2;
00275                                 } elseif( $n < 0xf8 ) {
00276                                         $remaining = 3;
00277                                 } elseif( $n < 0xfc ) {
00278                                         $remaining = 4;
00279                                 } elseif( $n < 0xfe ) {
00280                                         $remaining = 5;
00281                                 } else {
00282                                         $remaining = 0;
00283                                 }
00284                                 $tailBytes[chr($n)] = $remaining;
00285                         }
00286                 }
00287 
00288                 # Chop the text into pure-ASCII and non-ASCII areas;
00289                 # large ASCII parts can be handled much more quickly.
00290                 # Don't chop up Unicode areas for punctuation, though,
00291                 # that wastes energy.
00292                 $matches = array();
00293                 preg_match_all(
00294                         '/([\x00-\x7f]+|[\x80-\xff][\x00-\x40\x5b-\x5f\x7b-\xff]*)/',
00295                         $string, $matches );
00296 
00297                 $looksNormal = true;
00298                 $base = 0;
00299                 $replace = array();
00300                 foreach( $matches[1] as $str ) {
00301                         $chunk = strlen( $str );
00302 
00303                         if( $str[0] < "\x80" ) {
00304                                 # ASCII chunk: guaranteed to be valid UTF-8
00305                                 # and in normal form C, so skip over it.
00306                                 $base += $chunk;
00307                                 continue;
00308                         }
00309 
00310                         # We'll have to examine the chunk byte by byte to ensure
00311                         # that it consists of valid UTF-8 sequences, and to see
00312                         # if any of them might not be normalized.
00313                         #
00314                         # Since PHP is not the fastest language on earth, some of
00315                         # this code is a little ugly with inner loop optimizations.
00316 
00317                         $head = '';
00318                         $len = $chunk + 1; # Counting down is faster. I'm *so* sorry.
00319 
00320                         for( $i = -1; --$len; ) {
00321                                 $remaining = $tailBytes[$c = $str[++$i]];
00322                                 if( $remaining ) {
00323                                         # UTF-8 head byte!
00324                                         $sequence = $head = $c;
00325                                         do {
00326                                                 # Look for the defined number of tail bytes...
00327                                                 if( --$len && ( $c = $str[++$i] ) >= "\x80" && $c < "\xc0" ) {
00328                                                         # Legal tail bytes are nice.
00329                                                         $sequence .= $c;
00330                                                 } else {
00331                                                         if( 0 == $len ) {
00332                                                                 # Premature end of string!
00333                                                                 # Drop a replacement character into output to
00334                                                                 # represent the invalid UTF-8 sequence.
00335                                                                 $replace[] = array( UTF8_REPLACEMENT,
00336                                                                                                         $base + $i + 1 - strlen( $sequence ),
00337                                                                                                         strlen( $sequence ) );
00338                                                                 break 2;
00339                                                         } else {
00340                                                                 # Illegal tail byte; abandon the sequence.
00341                                                                 $replace[] = array( UTF8_REPLACEMENT,
00342                                                                                                         $base + $i - strlen( $sequence ),
00343                                                                                                         strlen( $sequence ) );
00344                                                                 # Back up and reprocess this byte; it may itself
00345                                                                 # be a legal ASCII or UTF-8 sequence head.
00346                                                                 --$i;
00347                                                                 ++$len;
00348                                                                 continue 2;
00349                                                         }
00350                                                 }
00351                                         } while( --$remaining );
00352 
00353                                         if( isset( $checkit[$head] ) ) {
00354                                                 # Do some more detailed validity checks, for
00355                                                 # invalid characters and illegal sequences.
00356                                                 if( $head == "\xed" ) {
00357                                                         # 0xed is relatively frequent in Korean, which
00358                                                         # abuts the surrogate area, so we're doing
00359                                                         # this check separately to speed things up.
00360 
00361                                                         if( $sequence >= UTF8_SURROGATE_FIRST ) {
00362                                                                 # Surrogates are legal only in UTF-16 code.
00363                                                                 # They are totally forbidden here in UTF-8
00364                                                                 # utopia.
00365                                                                 $replace[] = array( UTF8_REPLACEMENT,
00366                                                                              $base + $i + 1 - strlen( $sequence ),
00367                                                                              strlen( $sequence ) );
00368                                                                 $head = '';
00369                                                                 continue;
00370                                                         }
00371                                                 } else {
00372                                                         # Slower, but rarer checks...
00373                                                         $n = ord( $head );
00374                                                         if(
00375                                                                 # "Overlong sequences" are those that are syntactically
00376                                                                 # correct but use more UTF-8 bytes than are necessary to
00377                                                                 # encode a character. Naïve string comparisons can be
00378                                                                 # tricked into failing to see a match for an ASCII
00379                                                                 # character, for instance, which can be a security hole
00380                                                                 # if blacklist checks are being used.
00381                                                                ($n  < 0xc2 && $sequence <= UTF8_OVERLONG_A)
00382                                                                 || ($n == 0xe0 && $sequence <= UTF8_OVERLONG_B)
00383                                                                 || ($n == 0xf0 && $sequence <= UTF8_OVERLONG_C)
00384 
00385                                                                 # U+FFFE and U+FFFF are explicitly forbidden in Unicode.
00386                                                                 || ($n == 0xef &&
00387                                                                            ($sequence == UTF8_FFFE)
00388                                                                         || ($sequence == UTF8_FFFF) )
00389 
00390                                                                 # Unicode has been limited to 21 bits; longer
00391                                                                 # sequences are not allowed.
00392                                                                 || ($n >= 0xf0 && $sequence > UTF8_MAX) ) {
00393 
00394                                                                 $replace[] = array( UTF8_REPLACEMENT,
00395                                                                                     $base + $i + 1 - strlen( $sequence ),
00396                                                                                     strlen( $sequence ) );
00397                                                                 $head = '';
00398                                                                 continue;
00399                                                         }
00400                                                 }
00401                                         }
00402 
00403                                         if( isset( $utfCheckOrCombining[$sequence] ) ) {
00404                                                 # If it's NO or MAYBE, we'll have to rip
00405                                                 # the string apart and put it back together.
00406                                                 # That's going to be mighty slow.
00407                                                 $looksNormal = false;
00408                                         }
00409 
00410                                         # The sequence is legal!
00411                                         $head = '';
00412                                 } elseif( $c < "\x80" ) {
00413                                         # ASCII byte.
00414                                         $head = '';
00415                                 } elseif( $c < "\xc0" ) {
00416                                         # Illegal tail bytes
00417                                         if( $head == '' ) {
00418                                                 # Out of the blue!
00419                                                 $replace[] = array( UTF8_REPLACEMENT, $base + $i, 1 );
00420                                         } else {
00421                                                 # Don't add if we're continuing a broken sequence;
00422                                                 # we already put a replacement character when we looked
00423                                                 # at the broken sequence.
00424                                                 $replace[] = array( '', $base + $i, 1 );
00425                                         }
00426                                 } else {
00427                                         # Miscellaneous freaks.
00428                                         $replace[] = array( UTF8_REPLACEMENT, $base + $i, 1 );
00429                                         $head = '';
00430                                 }
00431                         }
00432                         $base += $chunk;
00433                 }
00434                 if( count( $replace ) ) {
00435                         # There were illegal UTF-8 sequences we need to fix up.
00436                         $out = '';
00437                         $last = 0;
00438                         foreach( $replace as $rep ) {
00439                                 list( $replacement, $start, $length ) = $rep;
00440                                 if( $last < $start ) {
00441                                         $out .= substr( $string, $last, $start - $last );
00442                                 }
00443                                 $out .= $replacement;
00444                                 $last = $start + $length;
00445                         }
00446                         if( $last < strlen( $string ) ) {
00447                                 $out .= substr( $string, $last );
00448                         }
00449                         $string = $out;
00450                 }
00451                 return $looksNormal;
00452         }
00453 
00454         # These take a string and run the normalization on them, without
00455         # checking for validity or any optimization etc. Input must be
00456         # VALID UTF-8!
00457 
00462         static function NFC( $string ) {
00463                 return UtfNormal::fastCompose( UtfNormal::NFD( $string ) );
00464         }
00465 
00471         static function NFD( $string ) {
00472                 UtfNormal::loadData();
00473 
00474                 return UtfNormal::fastCombiningSort(
00475                         UtfNormal::fastDecompose( $string, self::$utfCanonicalDecomp ) );
00476         }
00477 
00483         static function NFKC( $string ) {
00484                 return UtfNormal::fastCompose( UtfNormal::NFKD( $string ) );
00485         }
00486 
00492         static function NFKD( $string ) {
00493                 if( !isset( self::$utfCompatibilityDecomp ) ) {
00494                         require_once( 'UtfNormalDataK.inc' );
00495                 }
00496                 return self::fastCombiningSort(
00497                         self::fastDecompose( $string, self::$utfCompatibilityDecomp ) );
00498         }
00499 
00500 
00510         static function fastDecompose( $string, $map ) {
00511                 UtfNormal::loadData();
00512                 $len = strlen( $string );
00513                 $out = '';
00514                 for( $i = 0; $i < $len; $i++ ) {
00515                         $c = $string[$i];
00516                         $n = ord( $c );
00517                         if( $n < 0x80 ) {
00518                                 # ASCII chars never decompose
00519                                 # THEY ARE IMMORTAL
00520                                 $out .= $c;
00521                                 continue;
00522                         } elseif( $n >= 0xf0 ) {
00523                                 $c = substr( $string, $i, 4 );
00524                                 $i += 3;
00525                         } elseif( $n >= 0xe0 ) {
00526                                 $c = substr( $string, $i, 3 );
00527                                 $i += 2;
00528                         } elseif( $n >= 0xc0 ) {
00529                                 $c = substr( $string, $i, 2 );
00530                                 $i++;
00531                         }
00532                         if( isset( $map[$c] ) ) {
00533                                 $out .= $map[$c];
00534                                 continue;
00535                         } else {
00536                                 if( $c >= UTF8_HANGUL_FIRST && $c <= UTF8_HANGUL_LAST ) {
00537                                         # Decompose a hangul syllable into jamo;
00538                                         # hardcoded for three-byte UTF-8 sequence.
00539                                         # A lookup table would be slightly faster,
00540                                         # but adds a lot of memory & disk needs.
00541                                         #
00542                                         $index = ( (ord( $c[0] ) & 0x0f) << 12
00543                                                  | (ord( $c[1] ) & 0x3f) <<  6
00544                                                  | (ord( $c[2] ) & 0x3f) )
00545                                                - UNICODE_HANGUL_FIRST;
00546                                         $l = intval( $index / UNICODE_HANGUL_NCOUNT );
00547                                         $v = intval( ($index % UNICODE_HANGUL_NCOUNT) / UNICODE_HANGUL_TCOUNT);
00548                                         $t = $index % UNICODE_HANGUL_TCOUNT;
00549                                         $out .= "\xe1\x84" . chr( 0x80 + $l ) . "\xe1\x85" . chr( 0xa1 + $v );
00550                                         if( $t >= 25 ) {
00551                                                 $out .= "\xe1\x87" . chr( 0x80 + $t - 25 );
00552                                         } elseif( $t ) {
00553                                                 $out .= "\xe1\x86" . chr( 0xa7 + $t );
00554                                         }
00555                                         continue;
00556                                 }
00557                         }
00558                         $out .= $c;
00559                 }
00560                 return $out;
00561         }
00562 
00570         static function fastCombiningSort( $string ) {
00571                 UtfNormal::loadData();
00572                 $len = strlen( $string );
00573                 $out = '';
00574                 $combiners = array();
00575                 $lastClass = -1;
00576                 for( $i = 0; $i < $len; $i++ ) {
00577                         $c = $string[$i];
00578                         $n = ord( $c );
00579                         if( $n >= 0x80 ) {
00580                                 if( $n >= 0xf0 ) {
00581                                         $c = substr( $string, $i, 4 );
00582                                         $i += 3;
00583                                 } elseif( $n >= 0xe0 ) {
00584                                         $c = substr( $string, $i, 3 );
00585                                         $i += 2;
00586                                 } elseif( $n >= 0xc0 ) {
00587                                         $c = substr( $string, $i, 2 );
00588                                         $i++;
00589                                 }
00590                                 if( isset( self::$utfCombiningClass[$c] ) ) {
00591                                         $lastClass = self::$utfCombiningClass[$c];
00592                                         if( isset( $combiners[$lastClass] ) ) {
00593                                                 $combiners[$lastClass] .= $c;
00594                                         } else {
00595                                                 $combiners[$lastClass] = $c;
00596                                         }
00597                                         continue;
00598                                 }
00599                         }
00600                         if( $lastClass ) {
00601                                 ksort( $combiners );
00602                                 $out .= implode( '', $combiners );
00603                                 $combiners = array();
00604                         }
00605                         $out .= $c;
00606                         $lastClass = 0;
00607                 }
00608                 if( $lastClass ) {
00609                         ksort( $combiners );
00610                         $out .= implode( '', $combiners );
00611                 }
00612                 return $out;
00613         }
00614 
00622         static function fastCompose( $string ) {
00623                 UtfNormal::loadData();
00624                 $len = strlen( $string );
00625                 $out = '';
00626                 $lastClass = -1;
00627                 $lastHangul = 0;
00628                 $startChar = '';
00629                 $combining = '';
00630                 $x1 = ord(substr(UTF8_HANGUL_VBASE, 0, 1));
00631                 $x2 = ord(substr(UTF8_HANGUL_TEND, 0, 1));
00632                 for( $i = 0; $i < $len; $i++ ) {
00633                         $c = $string[$i];
00634                         $n = ord( $c );
00635                         if( $n < 0x80 ) {
00636                                 # No combining characters here...
00637                                 $out .= $startChar;
00638                                 $out .= $combining;
00639                                 $startChar = $c;
00640                                 $combining = '';
00641                                 $lastClass = 0;
00642                                 continue;
00643                         } elseif( $n >= 0xf0 ) {
00644                                 $c = substr( $string, $i, 4 );
00645                                 $i += 3;
00646                         } elseif( $n >= 0xe0 ) {
00647                                 $c = substr( $string, $i, 3 );
00648                                 $i += 2;
00649                         } elseif( $n >= 0xc0 ) {
00650                                 $c = substr( $string, $i, 2 );
00651                                 $i++;
00652                         }
00653                         $pair = $startChar . $c;
00654                         if( $n > 0x80 ) {
00655                                 if( isset( self::$utfCombiningClass[$c] ) ) {
00656                                         # A combining char; see what we can do with it
00657                                         $class = self::$utfCombiningClass[$c];
00658                                         if( !empty( $startChar ) &&
00659                                                 $lastClass < $class &&
00660                                                 $class > 0 &&
00661                                                 isset( self::$utfCanonicalComp[$pair] ) ) {
00662                                                 $startChar = self::$utfCanonicalComp[$pair];
00663                                                 $class = 0;
00664                                         } else {
00665                                                 $combining .= $c;
00666                                         }
00667                                         $lastClass = $class;
00668                                         $lastHangul = 0;
00669                                         continue;
00670                                 }
00671                         }
00672                         # New start char
00673                         if( $lastClass == 0 ) {
00674                                 if( isset( self::$utfCanonicalComp[$pair] ) ) {
00675                                         $startChar = self::$utfCanonicalComp[$pair];
00676                                         $lastHangul = 0;
00677                                         continue;
00678                                 }
00679                                 if( $n >= $x1 && $n <= $x2 ) {
00680                                         # WARNING: Hangul code is painfully slow.
00681                                         # I apologize for this ugly, ugly code; however
00682                                         # performance is even more teh suck if we call
00683                                         # out to nice clean functions. Lookup tables are
00684                                         # marginally faster, but require a lot of space.
00685                                         #
00686                                         if( $c >= UTF8_HANGUL_VBASE &&
00687                                                 $c <= UTF8_HANGUL_VEND &&
00688                                                 $startChar >= UTF8_HANGUL_LBASE &&
00689                                                 $startChar <= UTF8_HANGUL_LEND ) {
00690                                                 #
00691                                                 #$lIndex = utf8ToCodepoint( $startChar ) - UNICODE_HANGUL_LBASE;
00692                                                 #$vIndex = utf8ToCodepoint( $c ) - UNICODE_HANGUL_VBASE;
00693                                                 $lIndex = ord( $startChar[2] ) - 0x80;
00694                                                 $vIndex = ord( $c[2]         ) - 0xa1;
00695 
00696                                                 $hangulPoint = UNICODE_HANGUL_FIRST +
00697                                                         UNICODE_HANGUL_TCOUNT *
00698                                                         (UNICODE_HANGUL_VCOUNT * $lIndex + $vIndex);
00699 
00700                                                 # Hardcode the limited-range UTF-8 conversion:
00701                                                 $startChar = chr( $hangulPoint >> 12 & 0x0f | 0xe0 ) .
00702                                                                          chr( $hangulPoint >>  6 & 0x3f | 0x80 ) .
00703                                                                          chr( $hangulPoint       & 0x3f | 0x80 );
00704                                                 $lastHangul = 0;
00705                                                 continue;
00706                                         } elseif( $c >= UTF8_HANGUL_TBASE &&
00707                                                           $c <= UTF8_HANGUL_TEND &&
00708                                                           $startChar >= UTF8_HANGUL_FIRST &&
00709                                                           $startChar <= UTF8_HANGUL_LAST &&
00710                                                           !$lastHangul ) {
00711                                                 # $tIndex = utf8ToCodepoint( $c ) - UNICODE_HANGUL_TBASE;
00712                                                 $tIndex = ord( $c[2] ) - 0xa7;
00713                                                 if( $tIndex < 0 ) $tIndex = ord( $c[2] ) - 0x80 + (0x11c0 - 0x11a7);
00714 
00715                                                 # Increment the code point by $tIndex, without
00716                                                 # the function overhead of decoding and recoding UTF-8
00717                                                 #
00718                                                 $tail = ord( $startChar[2] ) + $tIndex;
00719                                                 if( $tail > 0xbf ) {
00720                                                         $tail -= 0x40;
00721                                                         $mid = ord( $startChar[1] ) + 1;
00722                                                         if( $mid > 0xbf ) {
00723                                                                 $startChar[0] = chr( ord( $startChar[0] ) + 1 );
00724                                                                 $mid -= 0x40;
00725                                                         }
00726                                                         $startChar[1] = chr( $mid );
00727                                                 }
00728                                                 $startChar[2] = chr( $tail );
00729 
00730                                                 # If there's another jamo char after this, *don't* try to merge it.
00731                                                 $lastHangul = 1;
00732                                                 continue;
00733                                         }
00734                                 }
00735                         }
00736                         $out .= $startChar;
00737                         $out .= $combining;
00738                         $startChar = $c;
00739                         $combining = '';
00740                         $lastClass = 0;
00741                         $lastHangul = 0;
00742                 }
00743                 $out .= $startChar . $combining;
00744                 return $out;
00745         }
00746 
00753         static function placebo( $string ) {
00754                 $len = strlen( $string );
00755                 $out = '';
00756                 for( $i = 0; $i < $len; $i++ ) {
00757                         $out .= $string[$i];
00758                 }
00759                 return $out;
00760         }
00768         private static function replaceForNativeNormalize( $string ) {
00769                 $string = preg_replace(
00770                         '/[\x00-\x08\x0b\x0c\x0e-\x1f]/',
00771                         UTF8_REPLACEMENT,
00772                         $string );
00773                 $string = str_replace( UTF8_FFFE, UTF8_REPLACEMENT, $string );
00774                 $string = str_replace( UTF8_FFFF, UTF8_REPLACEMENT, $string );
00775                 return $string;
00776         }
00777 }