UtfNormalTest2.php

Go to the documentation of this file.

#!/usr/bin/env php
<?php
if ( PHP_SAPI != 'cli' ) {
    die( "Run me from the command line please.\n" );
}

// From http://unicode.org/Public/UNIDATA/NormalizationTest.txt
$file = "NormalizationTest.txt";

// Anything after this character is a comment
define ( 'COMMENT', '#' );

// Semicolons are used to separate the columns
define ( 'SEPARATOR', ';' );

$f = fopen( $file, "r" );

require_once './UtfNormal.php';
function normalize_form_c( $c ) {
    return UtfNormal::toNFC( $c );
}

function normalize_form_d( $c ) {
    return UtfNormal::toNFD( $c );
}

function normalize_form_kc( $c ) {
    return UtfNormal::toNFKC( $c );
}

function normalize_form_kd( $c ) {
    return UtfNormal::toNFKD( $c );
}

function normalize_form_c_php( $c ) {
    return UtfNormal::toNFC( $c, "php" );
}

function normalize_form_d_php( $c ) {
    return UtfNormal::toNFD( $c, "php" );
}

function normalize_form_kc_php( $c ) {
    return UtfNormal::toNFKC( $c, "php" );
}

function normalize_form_kd_php( $c ) {
    return UtfNormal::toNFKD( $c, "php" );
}

assert_options( ASSERT_ACTIVE, 1 );
assert_options( ASSERT_WARNING, 0 );
assert_options( ASSERT_QUIET_EVAL, 1 );
assert_options( ASSERT_CALLBACK, 'my_assert' );

function my_assert( $file, $line, $code ) {
    // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix
    global $col, $lineNo;
    // @codingStandardsIgnoreEnd

    echo "Assertion that '$code' failed on line $lineNo ($col[5])\n";
}

$count = 0;
$lineNo = 0;
if ( $f !== false ) {
    while ( ( $col = getRow( $f ) ) !== false ) {
        $lineNo++;

        if ( count( $col ) == 6 ) {
            $count++;
            if ( $count % 100 === 0 ) echo "Count: $count\n";
        } else {
            continue;
        }

        # verify that the pure PHP version is correct
        $NFCc1 = normalize_form_c( $col[0] );
        $NFCc1p = normalize_form_c_php( $col[0] );
        assert( '$NFCc1 === $NFCc1p' );
        $NFCc2 = normalize_form_c( $col[1] );
        $NFCc2p = normalize_form_c_php( $col[1] );
        assert( '$NFCc2 === $NFCc2p' );
        $NFCc3 = normalize_form_c( $col[2] );
        $NFCc3p = normalize_form_c_php( $col[2] );
        assert( '$NFCc3 === $NFCc3p' );
        $NFCc4 = normalize_form_c( $col[3] );
        $NFCc4p = normalize_form_c_php( $col[3] );
        assert( '$NFCc4 === $NFCc4p' );
        $NFCc5 = normalize_form_c( $col[4] );
        $NFCc5p = normalize_form_c_php( $col[4] );
        assert( '$NFCc5 === $NFCc5p' );

        $NFDc1 = normalize_form_d( $col[0] );
        $NFDc1p = normalize_form_d_php( $col[0] );
        assert( '$NFDc1 === $NFDc1p' );
        $NFDc2 = normalize_form_d( $col[1] );
        $NFDc2p = normalize_form_d_php( $col[1] );
        assert( '$NFDc2 === $NFDc2p' );
        $NFDc3 = normalize_form_d( $col[2] );
        $NFDc3p = normalize_form_d_php( $col[2] );
        assert( '$NFDc3 === $NFDc3p' );
        $NFDc4 = normalize_form_d( $col[3] );
        $NFDc4p = normalize_form_d_php( $col[3] );
        assert( '$NFDc4 === $NFDc4p' );
        $NFDc5 = normalize_form_d( $col[4] );
        $NFDc5p = normalize_form_d_php( $col[4] );
        assert( '$NFDc5 === $NFDc5p' );

        $NFKDc1 = normalize_form_kd( $col[0] );
        $NFKDc1p = normalize_form_kd_php( $col[0] );
        assert( '$NFKDc1 === $NFKDc1p' );
        $NFKDc2 = normalize_form_kd( $col[1] );
        $NFKDc2p = normalize_form_kd_php( $col[1] );
        assert( '$NFKDc2 === $NFKDc2p' );
        $NFKDc3 = normalize_form_kd( $col[2] );
        $NFKDc3p = normalize_form_kd_php( $col[2] );
        assert( '$NFKDc3 === $NFKDc3p' );
        $NFKDc4 = normalize_form_kd( $col[3] );
        $NFKDc4p = normalize_form_kd_php( $col[3] );
        assert( '$NFKDc4 === $NFKDc4p' );
        $NFKDc5 = normalize_form_kd( $col[4] );
        $NFKDc5p = normalize_form_kd_php( $col[4] );
        assert( '$NFKDc5 === $NFKDc5p' );

        $NFKCc1 = normalize_form_kc( $col[0] );
        $NFKCc1p = normalize_form_kc_php( $col[0] );
        assert( '$NFKCc1 === $NFKCc1p' );
        $NFKCc2 = normalize_form_kc( $col[1] );
        $NFKCc2p = normalize_form_kc_php( $col[1] );
        assert( '$NFKCc2 === $NFKCc2p' );
        $NFKCc3 = normalize_form_kc( $col[2] );
        $NFKCc3p = normalize_form_kc_php( $col[2] );
        assert( '$NFKCc3 === $NFKCc3p' );
        $NFKCc4 = normalize_form_kc( $col[3] );
        $NFKCc4p = normalize_form_kc_php( $col[3] );
        assert( '$NFKCc4 === $NFKCc4p' );
        $NFKCc5 = normalize_form_kc( $col[4] );
        $NFKCc5p = normalize_form_kc_php( $col[4] );
        assert( '$NFKCc5 === $NFKCc5p' );

        # c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
        assert( '$col[1] === $NFCc1' );
        assert( '$col[1] === $NFCc2' );
        assert( '$col[1] === $NFCc3' );

        # c4 ==  NFC(c4) ==  NFC(c5)
        assert( '$col[3] === $NFCc4' );
        assert( '$col[3] === $NFCc5' );

        # c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
        assert( '$col[2] === $NFDc1' );
        assert( '$col[2] === $NFDc2' );
        assert( '$col[2] === $NFDc3' );

        # c5 ==  NFD(c4) ==  NFD(c5)
        assert( '$col[4] === $NFDc4' );
        assert( '$col[4] === $NFDc5' );

        # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
        assert( '$col[3] === $NFKCc1' );
        assert( '$col[3] === $NFKCc2' );
        assert( '$col[3] === $NFKCc3' );
        assert( '$col[3] === $NFKCc4' );
        assert( '$col[3] === $NFKCc5' );

        # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
        assert( '$col[4] === $NFKDc1' );
        assert( '$col[4] === $NFKDc2' );
        assert( '$col[4] === $NFKDc3' );
        assert( '$col[4] === $NFKDc4' );
        assert( '$col[4] === $NFKDc5' );
    }
}
echo "done.\n";

// Compare against http://en.wikipedia.org/wiki/UTF-8#Description
function unichr( $c ) {
    if ( $c <= 0x7F ) {
        return chr( $c );
    } elseif ( $c <= 0x7FF ) {
        return chr( 0xC0 | $c >> 6 ) . chr( 0x80 | $c & 0x3F );
    } elseif ( $c <= 0xFFFF ) {
        return chr( 0xE0 | $c >> 12 ) . chr( 0x80 | $c >> 6 & 0x3F )
        . chr( 0x80 | $c & 0x3F );
    } elseif ( $c <= 0x10FFFF ) {
        return chr( 0xF0 | $c >> 18 ) . chr( 0x80 | $c >> 12 & 0x3F )
        . chr( 0x80 | $c >> 6 & 0x3F )
        . chr( 0x80 | $c & 0x3F );
    } else {
        return false;
    }
}

function unistr( $c ) {
    return implode( "", array_map( "unichr", array_map( "hexdec", explode( " ", $c ) ) ) );
}

function getRow( $f ) {
    $row = fgets( $f );
    if ( $row === false ) return false;
    $row = rtrim( $row );
    $pos = strpos( $row, COMMENT );
    $pos2 = strpos( $row, ")" );
    if ( $pos === 0 ) return array( $row );
    $c = "";

    if ( $pos ) {
        if ( $pos2 ) $c = substr( $row, $pos2 + 2 );
        else      $c = substr( $row, $pos );
        $row = substr( $row, 0, $pos );
    }

    $ret = array();
    foreach ( explode( SEPARATOR, $row ) as $ent ) {
        if ( trim( $ent ) !== "" ) {
            $ret[] = unistr( $ent );
        }
    }
    $ret[] = $c;

    return $ret;
}