LanguageConverter.php

Go to the documentation of this file.

<?php
class LanguageConverter {

    static public $languagesWithVariants = array(
        'gan',
        'iu',
        'kk',
        'ku',
        'shi',
        'sr',
        'tg',
        'uz',
        'zh',
    );

    public $mMainLanguageCode;
    public $mVariants, $mVariantFallbacks, $mVariantNames;
    public $mTablesLoaded = false;
    public $mTables;
    // 'bidirectional' 'unidirectional' 'disable' for each variant
    public $mManualLevel;

    public $mCacheKey;

    public $mLangObj;
    public $mFlags;
    public $mDescCodeSep = ':', $mDescVarSep = ';';
    public $mUcfirst = false;
    public $mConvRuleTitle = false;
    public $mURLVariant;
    public $mUserVariant;
    public $mHeaderVariant;
    public $mMaxDepth = 10;
    public $mVarSeparatorPattern;

    const CACHE_VERSION_KEY = 'VERSION 6';

    public function __construct( $langobj, $maincode, $variants = array(),
                                $variantfallbacks = array(), $flags = array(),
                                $manualLevel = array() ) {
        global $wgDisabledVariants;
        $this->mLangObj = $langobj;
        $this->mMainLanguageCode = $maincode;
        $this->mVariants = array_diff( $variants, $wgDisabledVariants );
        $this->mVariantFallbacks = $variantfallbacks;
        $this->mVariantNames = Language::fetchLanguageNames();
        $this->mCacheKey = wfMemcKey( 'conversiontables', $maincode );
        $defaultflags = array(
            // 'S' show converted text
            // '+' add rules for alltext
            // 'E' the gave flags is error
            // these flags above are reserved for program
            'A' => 'A',   // add rule for convert code (all text convert)
            'T' => 'T',   // title convert
            'R' => 'R',   // raw content
            'D' => 'D',   // convert description (subclass implement)
            '-' => '-',   // remove convert (not implement)
            'H' => 'H',   // add rule for convert code
                          // (but no display in placed code)
            'N' => 'N'    // current variant name
        );
        $this->mFlags = array_merge( $defaultflags, $flags );
        foreach ( $this->mVariants as $v ) {
            if ( array_key_exists( $v, $manualLevel ) ) {
                $this->mManualLevel[$v] = $manualLevel[$v];
            } else {
                $this->mManualLevel[$v] = 'bidirectional';
            }
            $this->mFlags[$v] = $v;
        }
    }

    public function getVariants() {
        return $this->mVariants;
    }

    public function getVariantFallbacks( $variant ) {
        if ( isset( $this->mVariantFallbacks[$variant] ) ) {
            return $this->mVariantFallbacks[$variant];
        }
        return $this->mMainLanguageCode;
    }

    public function getConvRuleTitle() {
        return $this->mConvRuleTitle;
    }

    public function getPreferredVariant() {
        global $wgDefaultLanguageVariant, $wgUser;

        $req = $this->getURLVariant();

        if ( $wgUser->isLoggedIn() && !$req ) {
            $req = $this->getUserVariant();
        } elseif ( !$req ) {
            $req = $this->getHeaderVariant();
        }

        if ( $wgDefaultLanguageVariant && !$req ) {
            $req = $this->validateVariant( $wgDefaultLanguageVariant );
        }

        // This function, unlike the other get*Variant functions, is
        // not memoized (i.e. there return value is not cached) since
        // new information might appear during processing after this
        // is first called.
        if ( $this->validateVariant( $req ) ) {
            return $req;
        }
        return $this->mMainLanguageCode;
    }

    public function getDefaultVariant() {
        global $wgDefaultLanguageVariant;

        $req = $this->getURLVariant();

        if ( !$req ) {
            $req = $this->getHeaderVariant();
        }

        if ( $wgDefaultLanguageVariant && !$req ) {
            $req = $this->validateVariant( $wgDefaultLanguageVariant );
        }

        if ( $req ) {
            return $req;
        }
        return $this->mMainLanguageCode;
    }

    public function validateVariant( $variant = null ) {
        if ( $variant !== null && in_array( $variant, $this->mVariants ) ) {
            return $variant;
        }
        return null;
    }

    public function getURLVariant() {
        global $wgRequest;

        if ( $this->mURLVariant ) {
            return $this->mURLVariant;
        }

        // see if the preference is set in the request
        $ret = $wgRequest->getText( 'variant' );

        if ( !$ret ) {
            $ret = $wgRequest->getVal( 'uselang' );
        }

        return $this->mURLVariant = $this->validateVariant( $ret );
    }

    protected function getUserVariant() {
        global $wgUser, $wgContLang;

        // memoizing this function wreaks havoc on parserTest.php
        /*
        if ( $this->mUserVariant ) {
            return $this->mUserVariant;
        }
        */

        // Get language variant preference from logged in users
        // Don't call this on stub objects because that causes infinite
        // recursion during initialisation
        if ( $wgUser->isLoggedIn() ) {
            if ( $this->mMainLanguageCode == $wgContLang->getCode() ) {
                $ret = $wgUser->getOption( 'variant' );
            } else {
                $ret = $wgUser->getOption( 'variant-' . $this->mMainLanguageCode );
            }
        } else {
            // figure out user lang without constructing wgLang to avoid
            // infinite recursion
            $ret = $wgUser->getOption( 'language' );
        }

        return $this->mUserVariant = $this->validateVariant( $ret );
    }

    protected function getHeaderVariant() {
        global $wgRequest;

        if ( $this->mHeaderVariant ) {
            return $this->mHeaderVariant;
        }

        // see if some supported language variant is set in the
        // HTTP header.
        $languages = array_keys( $wgRequest->getAcceptLang() );
        if ( empty( $languages ) ) {
            return null;
        }

        $fallbackLanguages = array();
        foreach ( $languages as $language ) {
            $this->mHeaderVariant = $this->validateVariant( $language );
            if ( $this->mHeaderVariant ) {
                break;
            }

            // To see if there are fallbacks of current language.
            // We record these fallback variants, and process
            // them later.
            $fallbacks = $this->getVariantFallbacks( $language );
            if ( is_string( $fallbacks ) && $fallbacks !== $this->mMainLanguageCode ) {
                $fallbackLanguages[] = $fallbacks;
            } elseif ( is_array( $fallbacks ) ) {
                $fallbackLanguages =
                    array_merge( $fallbackLanguages, $fallbacks );
            }
        }

        if ( !$this->mHeaderVariant ) {
            // process fallback languages now
            $fallback_languages = array_unique( $fallbackLanguages );
            foreach ( $fallback_languages as $language ) {
                $this->mHeaderVariant = $this->validateVariant( $language );
                if ( $this->mHeaderVariant ) {
                    break;
                }
            }
        }

        return $this->mHeaderVariant;
    }

    public function autoConvert( $text, $toVariant = false ) {
        wfProfileIn( __METHOD__ );

        $this->loadTables();

        if ( !$toVariant ) {
            $toVariant = $this->getPreferredVariant();
            if ( !$toVariant ) {
                wfProfileOut( __METHOD__ );
                return $text;
            }
        }

        if ( $this->guessVariant( $text, $toVariant ) ) {
            wfProfileOut( __METHOD__ );
            return $text;
        }

        /* we convert everything except:
           1. HTML markups (anything between < and >)
           2. HTML entities
           3. placeholders created by the parser
        */
        global $wgParser;
        if ( isset( $wgParser ) && $wgParser->UniqPrefix() != '' ) {
            $marker = '|' . $wgParser->UniqPrefix() . '[\-a-zA-Z0-9]+';
        } else {
            $marker = '';
        }

        // this one is needed when the text is inside an HTML markup
        $htmlfix = '|<[^>]+$|^[^<>]*>';

        // disable convert to variants between <code> tags
        $codefix = '<code>.+?<\/code>|';
        // disable conversion of <script> tags
        $scriptfix = '<script.*?>.*?<\/script>|';
        // disable conversion of <pre> tags
        $prefix = '<pre.*?>.*?<\/pre>|';

        $reg = '/' . $codefix . $scriptfix . $prefix .
            '<[^>]+>|&[a-zA-Z#][a-z0-9]+;' . $marker . $htmlfix . '/s';
        $startPos = 0;
        $sourceBlob = '';
        $literalBlob = '';

        // Guard against delimiter nulls in the input
        $text = str_replace( "\000", '', $text );

        $markupMatches = null;
        $elementMatches = null;
        while ( $startPos < strlen( $text ) ) {
            if ( preg_match( $reg, $text, $markupMatches, PREG_OFFSET_CAPTURE, $startPos ) ) {
                $elementPos = $markupMatches[0][1];
                $element = $markupMatches[0][0];
            } else {
                $elementPos = strlen( $text );
                $element = '';
            }

            // Queue the part before the markup for translation in a batch
            $sourceBlob .= substr( $text, $startPos, $elementPos - $startPos ) . "\000";

            // Advance to the next position
            $startPos = $elementPos + strlen( $element );

            // Translate any alt or title attributes inside the matched element
            if ( $element !== '' && preg_match( '/^(<[^>\s]*)\s([^>]*)(.*)$/', $element,
                $elementMatches ) )
            {
                $attrs = Sanitizer::decodeTagAttributes( $elementMatches[2] );
                $changed = false;
                foreach ( array( 'title', 'alt' ) as $attrName ) {
                    if ( !isset( $attrs[$attrName] ) ) {
                        continue;
                    }
                    $attr = $attrs[$attrName];
                    // Don't convert URLs
                    if ( !strpos( $attr, '://' ) ) {
                        $attr = $this->recursiveConvertTopLevel( $attr, $toVariant );
                    }

                    // Remove HTML tags to avoid disrupting the layout
                    $attr = preg_replace( '/<[^>]+>/', '', $attr );
                    if ( $attr !== $attrs[$attrName] ) {
                        $attrs[$attrName] = $attr;
                        $changed = true;
                    }
                }
                if ( $changed ) {
                    $element = $elementMatches[1] . Html::expandAttributes( $attrs ) .
                        $elementMatches[3];
                }
            }
            $literalBlob .= $element . "\000";
        }

        // Do the main translation batch
        $translatedBlob = $this->translate( $sourceBlob, $toVariant );

        // Put the output back together
        $translatedIter = StringUtils::explode( "\000", $translatedBlob );
        $literalIter = StringUtils::explode( "\000", $literalBlob );
        $output = '';
        while ( $translatedIter->valid() && $literalIter->valid() ) {
            $output .= $translatedIter->current();
            $output .= $literalIter->current();
            $translatedIter->next();
            $literalIter->next();
        }

        wfProfileOut( __METHOD__ );
        return $output;
    }

    public function translate( $text, $variant ) {
        wfProfileIn( __METHOD__ );
        // If $text is empty or only includes spaces, do nothing
        // Otherwise translate it
        if ( trim( $text ) ) {
            $this->loadTables();
            $text = $this->mTables[$variant]->replace( $text );
        }
        wfProfileOut( __METHOD__ );
        return $text;
    }

    public function autoConvertToAllVariants( $text ) {
        wfProfileIn( __METHOD__ );
        $this->loadTables();

        $ret = array();
        foreach ( $this->mVariants as $variant ) {
            $ret[$variant] = $this->translate( $text, $variant );
        }

        wfProfileOut( __METHOD__ );
        return $ret;
    }

    public function convertLinkToAllVariants( $text ) {
        return $this->autoConvertToAllVariants( $text );
    }

    protected function applyManualConv( $convRule ) {
        // Use syntax -{T|zh-cn:TitleCN; zh-tw:TitleTw}- to custom
        // title conversion.
        // Bug 24072: $mConvRuleTitle was overwritten by other manual
        // rule(s) not for title, this breaks the title conversion.
        $newConvRuleTitle = $convRule->getTitle();
        if ( $newConvRuleTitle ) {
            // So I add an empty check for getTitle()
            $this->mConvRuleTitle = $newConvRuleTitle;
        }

        // merge/remove manual conversion rules to/from global table
        $convTable = $convRule->getConvTable();
        $action = $convRule->getRulesAction();
        foreach ( $convTable as $variant => $pair ) {
            if ( !$this->validateVariant( $variant ) ) {
                continue;
            }

            if ( $action == 'add' ) {
                foreach ( $pair as $from => $to ) {
                    // to ensure that $from and $to not be left blank
                    // so $this->translate() could always return a string
                    if ( $from || $to ) {
                        // more efficient than array_merge(), about 2.5 times.
                        $this->mTables[$variant]->setPair( $from, $to );
                    }
                }
            } elseif ( $action == 'remove' ) {
                $this->mTables[$variant]->removeArray( $pair );
            }
        }
    }

    public function convertTitle( $title ) {
        $variant = $this->getPreferredVariant();
        $index = $title->getNamespace();
        if ( $index !== NS_MAIN ) {
            $text = $this->convertNamespace( $index, $variant ) . ':';
        } else {
            $text = '';
        }
        $text .= $this->translate( $title->getText(), $variant );
        return $text;
    }

    public function convertNamespace( $index, $variant = null ) {
        if ( $variant === null ) {
            $variant = $this->getPreferredVariant();
        }
        if ( $index === NS_MAIN ) {
            return '';
        } else {
            // First check if a message gives a converted name in the target variant.
            $nsConvMsg = wfMessage( 'conversion-ns' . $index )->inLanguage( $variant );
            if ( $nsConvMsg->exists() ) {
                return $nsConvMsg->plain();
            }
            // Then check if a message gives a converted name in content language
            // which needs extra translation to the target variant.
            $nsConvMsg = wfMessage( 'conversion-ns' . $index )->inContentLanguage();
            if ( $nsConvMsg->exists() ) {
                return $this->translate( $nsConvMsg->plain(), $variant );
            }
            // No message exists, retrieve it from the target variant's namespace names.
            $langObj = $this->mLangObj->factory( $variant );
            return $langObj->getFormattedNsText( $index );
        }
    }

    public function convert( $text ) {
        $variant = $this->getPreferredVariant();
        return $this->convertTo( $text, $variant );
    }

    public function convertTo( $text, $variant ) {
        global $wgDisableLangConversion;
        if ( $wgDisableLangConversion ) {
            return $text;
        }
        // Reset converter state for a new converter run.
        $this->mConvRuleTitle = false;
        return $this->recursiveConvertTopLevel( $text, $variant );
    }

    protected function recursiveConvertTopLevel( $text, $variant, $depth = 0 ) {
        $startPos = 0;
        $out = '';
        $length = strlen( $text );
        $shouldConvert = !$this->guessVariant( $text, $variant );

        while ( $startPos < $length ) {
            $pos = strpos( $text, '-{', $startPos );

            if ( $pos === false ) {
                // No more markup, append final segment
                $fragment = substr( $text, $startPos );
                $out .= $shouldConvert ? $this->autoConvert( $fragment, $variant ) : $fragment;
                return $out;
            }

            // Markup found
            // Append initial segment
            $fragment = substr( $text, $startPos, $pos - $startPos );
            $out .= $shouldConvert ? $this->autoConvert( $fragment, $variant ) : $fragment;

            // Advance position
            $startPos = $pos;

            // Do recursive conversion
            $out .= $this->recursiveConvertRule( $text, $variant, $startPos, $depth + 1 );
        }

        return $out;
    }

    protected function recursiveConvertRule( $text, $variant, &$startPos, $depth = 0 ) {
        // Quick sanity check (no function calls)
        if ( $text[$startPos] !== '-' || $text[$startPos + 1] !== '{' ) {
            throw new MWException( __METHOD__ . ': invalid input string' );
        }

        $startPos += 2;
        $inner = '';
        $warningDone = false;
        $length = strlen( $text );

        while ( $startPos < $length ) {
            $m = false;
            preg_match( '/-\{|\}-/', $text, $m, PREG_OFFSET_CAPTURE, $startPos );
            if ( !$m ) {
                // Unclosed rule
                break;
            }

            $token = $m[0][0];
            $pos = $m[0][1];

            // Markup found
            // Append initial segment
            $inner .= substr( $text, $startPos, $pos - $startPos );

            // Advance position
            $startPos = $pos;

            switch ( $token ) {
                case '-{':
                    // Check max depth
                    if ( $depth >= $this->mMaxDepth ) {
                        $inner .= '-{';
                        if ( !$warningDone ) {
                            $inner .= '<span class="error">' .
                                wfMessage( 'language-converter-depth-warning' )
                                    ->numParams( $this->mMaxDepth )->inContentLanguage()->text() .
                                '</span>';
                            $warningDone = true;
                        }
                        $startPos += 2;
                        continue;
                    }
                    // Recursively parse another rule
                    $inner .= $this->recursiveConvertRule( $text, $variant, $startPos, $depth + 1 );
                    break;
                case '}-':
                    // Apply the rule
                    $startPos += 2;
                    $rule = new ConverterRule( $inner, $this );
                    $rule->parse( $variant );
                    $this->applyManualConv( $rule );
                    return $rule->getDisplay();
                default:
                    throw new MWException( __METHOD__ . ': invalid regex match' );
            }
        }

        // Unclosed rule
        if ( $startPos < $length ) {
            $inner .= substr( $text, $startPos );
        }
        $startPos = $length;
        return '-{' . $this->autoConvert( $inner, $variant );
    }

    public function findVariantLink( &$link, &$nt, $ignoreOtherCond = false ) {
        # If the article has already existed, there is no need to
        # check it again, otherwise it may cause a fault.
        if ( is_object( $nt ) && $nt->exists() ) {
            return;
        }

        global $wgDisableLangConversion, $wgDisableTitleConversion, $wgRequest,
            $wgUser;
        $isredir = $wgRequest->getText( 'redirect', 'yes' );
        $action = $wgRequest->getText( 'action' );
        $linkconvert = $wgRequest->getText( 'linkconvert', 'yes' );
        $disableLinkConversion = $wgDisableLangConversion
            || $wgDisableTitleConversion;
        $linkBatch = new LinkBatch();

        $ns = NS_MAIN;

        if ( $disableLinkConversion ||
             ( !$ignoreOtherCond &&
               ( $isredir == 'no'
                 || $action == 'edit'
                 || $action == 'submit'
                 || $linkconvert == 'no'
                 || $wgUser->getOption( 'noconvertlink' ) == 1 ) ) ) {
            return;
        }

        if ( is_object( $nt ) ) {
            $ns = $nt->getNamespace();
        }

        $variants = $this->autoConvertToAllVariants( $link );
        if ( !$variants ) { // give up
            return;
        }

        $titles = array();

        foreach ( $variants as $v ) {
            if ( $v != $link ) {
                $varnt = Title::newFromText( $v, $ns );
                if ( !is_null( $varnt ) ) {
                    $linkBatch->addObj( $varnt );
                    $titles[] = $varnt;
                }
            }
        }

        // fetch all variants in single query
        $linkBatch->execute();

        foreach ( $titles as $varnt ) {
            if ( $varnt->getArticleID() > 0 ) {
                $nt = $varnt;
                $link = $varnt->getText();
                break;
            }
        }
    }

    public function getExtraHashOptions() {
        $variant = $this->getPreferredVariant();
        return '!' . $variant;
    }

    public function guessVariant( $text, $variant ) {
        return false;
    }

    function loadDefaultTables() {
        $name = get_class( $this );
        throw new MWException( "Must implement loadDefaultTables() method in class $name" );
    }

    function loadTables( $fromCache = true ) {
        global $wgLangConvMemc;

        if ( $this->mTablesLoaded ) {
            return;
        }

        wfProfileIn( __METHOD__ );
        $this->mTablesLoaded = true;
        $this->mTables = false;
        if ( $fromCache ) {
            wfProfileIn( __METHOD__ . '-cache' );
            $this->mTables = $wgLangConvMemc->get( $this->mCacheKey );
            wfProfileOut( __METHOD__ . '-cache' );
        }
        if ( !$this->mTables
             || !array_key_exists( self::CACHE_VERSION_KEY, $this->mTables ) ) {
            wfProfileIn( __METHOD__ . '-recache' );
            // not in cache, or we need a fresh reload.
            // We will first load the default tables
            // then update them using things in MediaWiki:Conversiontable/*
            $this->loadDefaultTables();
            foreach ( $this->mVariants as $var ) {
                $cached = $this->parseCachedTable( $var );
                $this->mTables[$var]->mergeArray( $cached );
            }

            $this->postLoadTables();
            $this->mTables[self::CACHE_VERSION_KEY] = true;

            $wgLangConvMemc->set( $this->mCacheKey, $this->mTables, 43200 );
            wfProfileOut( __METHOD__ . '-recache' );
        }
        wfProfileOut( __METHOD__ );
    }

    function postLoadTables() { }

    function reloadTables() {
        if ( $this->mTables ) {
            unset( $this->mTables );
        }
        $this->mTablesLoaded = false;
        $this->loadTables( false );
    }

    function parseCachedTable( $code, $subpage = '', $recursive = true ) {
        static $parsed = array();

        $key = 'Conversiontable/' . $code;
        if ( $subpage ) {
            $key .= '/' . $subpage;
        }
        if ( array_key_exists( $key, $parsed ) ) {
            return array();
        }

        $parsed[$key] = true;

        if ( $subpage === '' ) {
            $txt = MessageCache::singleton()->getMsgFromNamespace( $key, $code );
        } else {
            $txt = false;
            $title = Title::makeTitleSafe( NS_MEDIAWIKI, $key );
            if ( $title && $title->exists() ) {
                $revision = Revision::newFromTitle( $title );
                if ( $revision ) {
                    if ( $revision->getContentModel() == CONTENT_MODEL_WIKITEXT ) {
                        $txt = $revision->getContent( Revision::RAW )->getNativeData();
                    }

                    // @todo in the future, use a specialized content model, perhaps based on json!
                }
            }
        }

        # Nothing to parse if there's no text
        if ( $txt === false || $txt === null || $txt === '' ) {
            return array();
        }

        // get all subpage links of the form
        // [[MediaWiki:Conversiontable/zh-xx/...|...]]
        $linkhead = $this->mLangObj->getNsText( NS_MEDIAWIKI ) .
            ':Conversiontable';
        $subs = StringUtils::explode( '[[', $txt );
        $sublinks = array();
        foreach ( $subs as $sub ) {
            $link = explode( ']]', $sub, 2 );
            if ( count( $link ) != 2 ) {
                continue;
            }
            $b = explode( '|', $link[0], 2 );
            $b = explode( '/', trim( $b[0] ), 3 );
            if ( count( $b ) == 3 ) {
                $sublink = $b[2];
            } else {
                $sublink = '';
            }

            if ( $b[0] == $linkhead && $b[1] == $code ) {
                $sublinks[] = $sublink;
            }
        }

        // parse the mappings in this page
        $blocks = StringUtils::explode( '-{', $txt );
        $ret = array();
        $first = true;
        foreach ( $blocks as $block ) {
            if ( $first ) {
                // Skip the part before the first -{
                $first = false;
                continue;
            }
            $mappings = explode( '}-', $block, 2 );
            $stripped = str_replace( array( "'", '"', '*', '#' ), '',
                                     $mappings[0] );
            $table = StringUtils::explode( ';', $stripped );
            foreach ( $table as $t ) {
                $m = explode( '=>', $t, 3 );
                if ( count( $m ) != 2 ) {
                    continue;
                }
                // trim any trailling comments starting with '//'
                $tt = explode( '//', $m[1], 2 );
                $ret[trim( $m[0] )] = trim( $tt[0] );
            }
        }

        // recursively parse the subpages
        if ( $recursive ) {
            foreach ( $sublinks as $link ) {
                $s = $this->parseCachedTable( $code, $link, $recursive );
                $ret = array_merge( $ret, $s );
            }
        }

        if ( $this->mUcfirst ) {
            foreach ( $ret as $k => $v ) {
                $ret[$this->mLangObj->ucfirst( $k )] = $this->mLangObj->ucfirst( $v );
            }
        }
        return $ret;
    }

    public function markNoConversion( $text, $noParse = false ) {
        # don't mark if already marked
        if ( strpos( $text, '-{' ) || strpos( $text, '}-' ) ) {
            return $text;
        }

        $ret = "-{R|$text}-";
        return $ret;
    }

    function convertCategoryKey( $key ) {
        return $key;
    }

    function OnPageContentSaveComplete( $page, $user, $content, $summary, $isMinor,
            $isWatch, $section, $flags, $revision ) {
        $titleobj = $page->getTitle();
        if ( $titleobj->getNamespace() == NS_MEDIAWIKI ) {
            $title = $titleobj->getDBkey();
            $t = explode( '/', $title, 3 );
            $c = count( $t );
            if ( $c > 1 && $t[0] == 'Conversiontable' ) {
                if ( $this->validateVariant( $t[1] ) ) {
                    $this->reloadTables();
                }
            }
        }
        return true;
    }

    public function armourMath( $text ) {
        // convert '-{' and '}-' to '-&#123;' and '&#125;-' to prevent
        // any unwanted markup appearing in the math image tag.
        $text = strtr( $text, array( '-{' => '-&#123;', '}-' => '&#125;-' ) );
        return $text;
    }

    function getVarSeparatorPattern() {
        if ( is_null( $this->mVarSeparatorPattern ) ) {
            // varsep_pattern for preg_split:
            // text should be splited by ";" only if a valid variant
            // name exist after the markup, for example:
            //  -{zh-hans:<span style="font-size:120%;">xxx</span>;zh-hant:\
            //  <span style="font-size:120%;">yyy</span>;}-
            // we should split it as:
            //  array(
            //    [0] => 'zh-hans:<span style="font-size:120%;">xxx</span>'
            //    [1] => 'zh-hant:<span style="font-size:120%;">yyy</span>'
            //    [2] => ''
            //   )
            $pat = '/;\s*(?=';
            foreach ( $this->mVariants as $variant ) {
                // zh-hans:xxx;zh-hant:yyy
                $pat .= $variant . '\s*:|';
                // xxx=>zh-hans:yyy; xxx=>zh-hant:zzz
                $pat .= '[^;]*?=>\s*' . $variant . '\s*:|';
            }
            $pat .= '\s*$)/';
            $this->mVarSeparatorPattern = $pat;
        }
        return $this->mVarSeparatorPattern;
    }
}

class ConverterRule {
    public $mText; // original text in -{text}-
    public $mConverter; // LanguageConverter object
    public $mRuleDisplay = '';
    public $mRuleTitle = false;
    public $mRules = '';// string : the text of the rules
    public $mRulesAction = 'none';
    public $mFlags = array();
    public $mVariantFlags = array();
    public $mConvTable = array();
    public $mBidtable = array();// array of the translation in each variant
    public $mUnidtable = array();// array of the translation in each variant

    public function __construct( $text, $converter ) {
        $this->mText = $text;
        $this->mConverter = $converter;
    }

    public function getTextInBidtable( $variants ) {
        $variants = (array)$variants;
        if ( !$variants ) {
            return false;
        }
        foreach ( $variants as $variant ) {
            if ( isset( $this->mBidtable[$variant] ) ) {
                return $this->mBidtable[$variant];
            }
        }
        return false;
    }

    function parseFlags() {
        $text = $this->mText;
        $flags = array();
        $variantFlags = array();

        $sepPos = strpos( $text, '|' );
        if ( $sepPos !== false ) {
            $validFlags = $this->mConverter->mFlags;
            $f = StringUtils::explode( ';', substr( $text, 0, $sepPos ) );
            foreach ( $f as $ff ) {
                $ff = trim( $ff );
                if ( isset( $validFlags[$ff] ) ) {
                    $flags[$validFlags[$ff]] = true;
                }
            }
            $text = strval( substr( $text, $sepPos + 1 ) );
        }

        if ( !$flags ) {
            $flags['S'] = true;
        } elseif ( isset( $flags['R'] ) ) {
            $flags = array( 'R' => true );// remove other flags
        } elseif ( isset( $flags['N'] ) ) {
            $flags = array( 'N' => true );// remove other flags
        } elseif ( isset( $flags['-'] ) ) {
            $flags = array( '-' => true );// remove other flags
        } elseif ( count( $flags ) == 1 && isset( $flags['T'] ) ) {
            $flags['H'] = true;
        } elseif ( isset( $flags['H'] ) ) {
            // replace A flag, and remove other flags except T
            $temp = array( '+' => true, 'H' => true );
            if ( isset( $flags['T'] ) ) {
                $temp['T'] = true;
            }
            if ( isset( $flags['D'] ) ) {
                $temp['D'] = true;
            }
            $flags = $temp;
        } else {
            if ( isset( $flags['A'] ) ) {
                $flags['+'] = true;
                $flags['S'] = true;
            }
            if ( isset( $flags['D'] ) ) {
                unset( $flags['S'] );
            }
            // try to find flags like "zh-hans", "zh-hant"
            // allow syntaxes like "-{zh-hans;zh-hant|XXXX}-"
            $variantFlags = array_intersect( array_keys( $flags ), $this->mConverter->mVariants );
            if ( $variantFlags ) {
                $variantFlags = array_flip( $variantFlags );
                $flags = array();
            }
        }
        $this->mVariantFlags = $variantFlags;
        $this->mRules = $text;
        $this->mFlags = $flags;
    }

    function parseRules() {
        $rules = $this->mRules;
        $bidtable = array();
        $unidtable = array();
        $variants = $this->mConverter->mVariants;
        $varsep_pattern = $this->mConverter->getVarSeparatorPattern();

        // Split according to $varsep_pattern, but ignore semicolons from HTML entities
        $rules = preg_replace( '/(&[#a-zA-Z0-9]+);/', "$1\x01", $rules );
        $choice = preg_split( $varsep_pattern, $rules );
        $choice = str_replace( "\x01", ';', $choice );

        foreach ( $choice as $c ) {
            $v = explode( ':', $c, 2 );
            if ( count( $v ) != 2 ) {
                // syntax error, skip
                continue;
            }
            $to = trim( $v[1] );
            $v = trim( $v[0] );
            $u = explode( '=>', $v, 2 );
            // if $to is empty, strtr() could return a wrong result
            if ( count( $u ) == 1 && $to && in_array( $v, $variants ) ) {
                $bidtable[$v] = $to;
            } elseif ( count( $u ) == 2 ) {
                $from = trim( $u[0] );
                $v = trim( $u[1] );
                if ( array_key_exists( $v, $unidtable )
                     && !is_array( $unidtable[$v] )
                     && $to
                     && in_array( $v, $variants ) ) {
                    $unidtable[$v] = array( $from => $to );
                } elseif ( $to && in_array( $v, $variants ) ) {
                    $unidtable[$v][$from] = $to;
                }
            }
            // syntax error, pass
            if ( !isset( $this->mConverter->mVariantNames[$v] ) ) {
                $bidtable = array();
                $unidtable = array();
                break;
            }
        }
        $this->mBidtable = $bidtable;
        $this->mUnidtable = $unidtable;
    }

    function getRulesDesc() {
        $codesep = $this->mConverter->mDescCodeSep;
        $varsep = $this->mConverter->mDescVarSep;
        $text = '';
        foreach ( $this->mBidtable as $k => $v ) {
            $text .= $this->mConverter->mVariantNames[$k] . "$codesep$v$varsep";
        }
        foreach ( $this->mUnidtable as $k => $a ) {
            foreach ( $a as $from => $to ) {
                $text .= $from . '⇒' . $this->mConverter->mVariantNames[$k] .
                    "$codesep$to$varsep";
            }
        }
        return $text;
    }

    function getRuleConvertedStr( $variant ) {
        $bidtable = $this->mBidtable;
        $unidtable = $this->mUnidtable;

        if ( count( $bidtable ) + count( $unidtable ) == 0 ) {
            return $this->mRules;
        } else {
            // display current variant in bidirectional array
            $disp = $this->getTextInBidtable( $variant );
            // or display current variant in fallbacks
            if ( !$disp ) {
                $disp = $this->getTextInBidtable(
                        $this->mConverter->getVariantFallbacks( $variant ) );
            }
            // or display current variant in unidirectional array
            if ( !$disp && array_key_exists( $variant, $unidtable ) ) {
                $disp = array_values( $unidtable[$variant] );
                $disp = $disp[0];
            }
            // or display frist text under disable manual convert
            if ( !$disp
                 && $this->mConverter->mManualLevel[$variant] == 'disable' ) {
                if ( count( $bidtable ) > 0 ) {
                    $disp = array_values( $bidtable );
                    $disp = $disp[0];
                } else {
                    $disp = array_values( $unidtable );
                    $disp = array_values( $disp[0] );
                    $disp = $disp[0];
                }
            }
            return $disp;
        }
    }

    function getRuleConvertedTitle( $variant ) {
        if ( $variant === $this->mConverter->mMainLanguageCode ) {
            // If a string targeting exactly this variant is set,
            // use it. Otherwise, just return false, so the real
            // page name can be shown (and because variant === main,
            // there'll be no further automatic conversion).
            $disp = $this->getTextInBidtable( $variant );
            if ( $disp ) {
                return $disp;
            }
            if ( array_key_exists( $variant, $this->mUnidtable ) ) {
                $disp = array_values( $this->mUnidtable[$variant] );
                $disp = $disp[0];
            }
            // Assigned above or still false.
            return $disp;
        } else {
            return $this->getRuleConvertedStr( $variant );
        }
    }

    function generateConvTable() {
        // Special case optimisation
        if ( !$this->mBidtable && !$this->mUnidtable ) {
            $this->mConvTable = array();
            return;
        }

        $bidtable = $this->mBidtable;
        $unidtable = $this->mUnidtable;
        $manLevel = $this->mConverter->mManualLevel;

        $vmarked = array();
        foreach ( $this->mConverter->mVariants as $v ) {
            /* for bidirectional array
                fill in the missing variants, if any,
                with fallbacks */
            if ( !isset( $bidtable[$v] ) ) {
                $variantFallbacks =
                    $this->mConverter->getVariantFallbacks( $v );
                $vf = $this->getTextInBidtable( $variantFallbacks );
                if ( $vf ) {
                    $bidtable[$v] = $vf;
                }
            }

            if ( isset( $bidtable[$v] ) ) {
                foreach ( $vmarked as $vo ) {
                    // use syntax: -{A|zh:WordZh;zh-tw:WordTw}-
                    // or -{H|zh:WordZh;zh-tw:WordTw}-
                    // or -{-|zh:WordZh;zh-tw:WordTw}-
                    // to introduce a custom mapping between
                    // words WordZh and WordTw in the whole text
                    if ( $manLevel[$v] == 'bidirectional' ) {
                        $this->mConvTable[$v][$bidtable[$vo]] = $bidtable[$v];
                    }
                    if ( $manLevel[$vo] == 'bidirectional' ) {
                        $this->mConvTable[$vo][$bidtable[$v]] = $bidtable[$vo];
                    }
                }
                $vmarked[] = $v;
            }
            /* for unidirectional array fill to convert tables */
            if ( ( $manLevel[$v] == 'bidirectional' || $manLevel[$v] == 'unidirectional' )
                && isset( $unidtable[$v] ) )
            {
                if ( isset( $this->mConvTable[$v] ) ) {
                    $this->mConvTable[$v] = array_merge( $this->mConvTable[$v], $unidtable[$v] );
                } else {
                    $this->mConvTable[$v] = $unidtable[$v];
                }
            }
        }
    }

    public function parse( $variant = null ) {
        if ( !$variant ) {
            $variant = $this->mConverter->getPreferredVariant();
        }

        $this->parseFlags();
        $flags = $this->mFlags;

        // convert to specified variant
        // syntax: -{zh-hans;zh-hant[;...]|<text to convert>}-
        if ( $this->mVariantFlags ) {
            // check if current variant in flags
            if ( isset( $this->mVariantFlags[$variant] ) ) {
                // then convert <text to convert> to current language
                $this->mRules = $this->mConverter->autoConvert( $this->mRules,
                    $variant );
            } else { // if current variant no in flags,
                   // then we check its fallback variants.
                $variantFallbacks =
                    $this->mConverter->getVariantFallbacks( $variant );
                if ( is_array( $variantFallbacks ) ) {
                    foreach ( $variantFallbacks as $variantFallback ) {
                        // if current variant's fallback exist in flags
                        if ( isset( $this->mVariantFlags[$variantFallback] ) ) {
                            // then convert <text to convert> to fallback language
                            $this->mRules =
                                $this->mConverter->autoConvert( $this->mRules,
                                    $variantFallback );
                            break;
                        }
                    }
                }
            }
            $this->mFlags = $flags = array( 'R' => true );
        }

        if ( !isset( $flags['R'] ) && !isset( $flags['N'] ) ) {
            // decode => HTML entities modified by Sanitizer::removeHTMLtags
            $this->mRules = str_replace( '=&gt;', '=>', $this->mRules );
            $this->parseRules();
        }
        $rules = $this->mRules;

        if ( !$this->mBidtable && !$this->mUnidtable ) {
            if ( isset( $flags['+'] ) || isset( $flags['-'] ) ) {
                // fill all variants if text in -{A/H/-|text} without rules
                foreach ( $this->mConverter->mVariants as $v ) {
                    $this->mBidtable[$v] = $rules;
                }
            } elseif ( !isset( $flags['N'] ) && !isset( $flags['T'] ) ) {
                $this->mFlags = $flags = array( 'R' => true );
            }
        }

        $this->mRuleDisplay = false;
        foreach ( $flags as $flag => $unused ) {
            switch ( $flag ) {
                case 'R':
                    // if we don't do content convert, still strip the -{}- tags
                    $this->mRuleDisplay = $rules;
                    break;
                case 'N':
                    // process N flag: output current variant name
                    $ruleVar = trim( $rules );
                    if ( isset( $this->mConverter->mVariantNames[$ruleVar] ) ) {
                        $this->mRuleDisplay = $this->mConverter->mVariantNames[$ruleVar];
                    } else {
                        $this->mRuleDisplay = '';
                    }
                    break;
                case 'D':
                    // process D flag: output rules description
                    $this->mRuleDisplay = $this->getRulesDesc();
                    break;
                case 'H':
                    // process H,- flag or T only: output nothing
                    $this->mRuleDisplay = '';
                    break;
                case '-':
                    $this->mRulesAction = 'remove';
                    $this->mRuleDisplay = '';
                    break;
                case '+':
                    $this->mRulesAction = 'add';
                    $this->mRuleDisplay = '';
                    break;
                case 'S':
                    $this->mRuleDisplay = $this->getRuleConvertedStr( $variant );
                    break;
                case 'T':
                    $this->mRuleTitle = $this->getRuleConvertedTitle( $variant );
                    $this->mRuleDisplay = '';
                    break;
                default:
                    // ignore unknown flags (but see error case below)
            }
        }
        if ( $this->mRuleDisplay === false ) {
            $this->mRuleDisplay = '<span class="error">'
                . wfMessage( 'converter-manual-rule-error' )->inContentLanguage()->escaped()
                . '</span>';
        }

        $this->generateConvTable();
    }

    public function hasRules() {
        // TODO:
    }

    public function getDisplay() {
        return $this->mRuleDisplay;
    }

    public function getTitle() {
        return $this->mRuleTitle;
    }

    public function getRulesAction() {
        return $this->mRulesAction;
    }

    public function getConvTable() {
        return $this->mConvTable;
    }

    public function getRules() {
        return $this->mRules;
    }

    public function getFlags() {
        return $this->mFlags;
    }
}