UtfNormalGenerate.php

Go to the documentation of this file.

<?php
if ( PHP_SAPI != 'cli' ) {
    die( "Run me from the command line please.\n" );
}

require_once 'UtfNormalDefines.php';
require_once 'UtfNormalUtil.php';

$in = fopen( "DerivedNormalizationProps.txt", "rt" );
if ( !$in ) {
    print "Can't open DerivedNormalizationProps.txt for reading.\n";
    print "If necessary, fetch this file from the internet:\n";
    print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n";
    exit( -1 );
}
print "Initializing normalization quick check tables...\n";
$checkNFC = array();
while ( false !== ( $line = fgets( $in ) ) ) {
    $matches = array();
    if ( preg_match(
        '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/',
        $line,
        $matches )
    ) {
        list( $junk, $first, $last, $prop, $value ) = $matches;
        #print "$first $last $prop $value\n";
        if ( !$last ) {
            $last = $first;
        }

        $lastInDecimal = hexdec( $last );
        for ( $i = hexdec( $first ); $i <= $lastInDecimal; $i++ ) {
            $char = codepointToUtf8( $i );
            $checkNFC[$char] = $value;
        }
    }
}
fclose( $in );

$in = fopen( "CompositionExclusions.txt", "rt" );
if ( !$in ) {
    print "Can't open CompositionExclusions.txt for reading.\n";
    print "If necessary, fetch this file from the internet:\n";
    print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
    exit( -1 );
}
$exclude = array();
while ( false !== ( $line = fgets( $in ) ) ) {
    if ( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
        $codepoint = $matches[1];
        $source = codepointToUtf8( hexdec( $codepoint ) );
        $exclude[$source] = true;
    }
}
fclose( $in );

$in = fopen( "UnicodeData.txt", "rt" );
if ( !$in ) {
    print "Can't open UnicodeData.txt for reading.\n";
    print "If necessary, fetch this file from the internet:\n";
    print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
    exit( -1 );
}

$compatibilityDecomp = array();
$canonicalDecomp = array();
$canonicalComp = array();
$combiningClass = array();
$total = 0;
$compat = 0;
$canon = 0;

print "Reading character definitions...\n";
while ( false !== ( $line = fgets( $in ) ) ) {
    $columns = explode( ';', $line );
    $codepoint = $columns[0];
    $name = $columns[1];
    $canonicalCombiningClass = $columns[3];
    $decompositionMapping = $columns[5];

    $source = codepointToUtf8( hexdec( $codepoint ) );

    if ( $canonicalCombiningClass != 0 ) {
        $combiningClass[$source] = intval( $canonicalCombiningClass );
    }

    if ( $decompositionMapping === '' ) continue;
    if ( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
        # Compatibility decomposition
        $canonical = false;
        $decompositionMapping = $matches[2];
        $compat++;
    } else {
        $canonical = true;
        $canon++;
    }
    $total++;
    $dest = hexSequenceToUtf8( $decompositionMapping );

    $compatibilityDecomp[$source] = $dest;
    if ( $canonical ) {
        $canonicalDecomp[$source] = $dest;
        if ( empty( $exclude[$source] ) ) {
            $canonicalComp[$dest] = $source;
        }
    }
    #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
}
fclose( $in );

print "Recursively expanding canonical mappings...\n";
$changed = 42;
$pass = 1;
while ( $changed > 0 ) {
    print "pass $pass\n";
    $changed = 0;
    foreach ( $canonicalDecomp as $source => $dest ) {
        $newDest = preg_replace_callback(
            '/([\xc0-\xff][\x80-\xbf]+)/',
            'callbackCanonical',
            $dest );
        if ( $newDest === $dest ) continue;
        $changed++;
        $canonicalDecomp[$source] = $newDest;
    }
    $pass++;
}

print "Recursively expanding compatibility mappings...\n";
$changed = 42;
$pass = 1;
while ( $changed > 0 ) {
    print "pass $pass\n";
    $changed = 0;
    foreach ( $compatibilityDecomp as $source => $dest ) {
        $newDest = preg_replace_callback(
            '/([\xc0-\xff][\x80-\xbf]+)/',
            'callbackCompat',
            $dest );
        if ( $newDest === $dest ) continue;
        $changed++;
        $compatibilityDecomp[$source] = $newDest;
    }
    $pass++;
}

print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";

$out = fopen( "UtfNormalData.inc", "wt" );
if ( $out ) {
    $serCombining = escapeSingleString( serialize( $combiningClass ) );
    $serComp = escapeSingleString( serialize( $canonicalComp ) );
    $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
    $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
    $outdata = "<" . "?php
// @codingStandardsIgnoreFile

UtfNormal::\$utfCombiningClass = unserialize( '$serCombining' );
UtfNormal::\$utfCanonicalComp = unserialize( '$serComp' );
UtfNormal::\$utfCanonicalDecomp = unserialize( '$serCanon' );
UtfNormal::\$utfCheckNFC = unserialize( '$serCheckNFC' );
\n";
    fputs( $out, $outdata );
    fclose( $out );
    print "Wrote out UtfNormalData.inc\n";
} else {
    print "Can't create file UtfNormalData.inc\n";
    exit( -1 );
}

$out = fopen( "UtfNormalDataK.inc", "wt" );
if ( $out ) {
    $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
    $outdata = "<" . "?php
// @codingStandardsIgnoreFile

UtfNormal::\$utfCompatibilityDecomp = unserialize( '$serCompat' );
\n";
    fputs( $out, $outdata );
    fclose( $out );
    print "Wrote out UtfNormalDataK.inc\n";
    exit( 0 );
} else {
    print "Can't create file UtfNormalDataK.inc\n";
    exit( -1 );
}

# ---------------

function callbackCanonical( $matches ) {
    // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix
    global $canonicalDecomp;
    // @codingStandardsIgnoreEnd

    if ( isset( $canonicalDecomp[$matches[1]] ) ) {
        return $canonicalDecomp[$matches[1]];
    }

    return $matches[1];
}

function callbackCompat( $matches ) {
    // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix
    global $compatibilityDecomp;
    // @codingStandardsIgnoreEnd

    if ( isset( $compatibilityDecomp[$matches[1]] ) ) {
        return $compatibilityDecomp[$matches[1]];
    }

    return $matches[1];
}