checkLanguage.inc

Go to the documentation of this file.

<?php
class CheckLanguageCLI {
        protected $code  = null;
        protected $level = 2;
        protected $doLinks = false;
        protected $linksPrefix = '';
        protected $wikiCode = 'en';
        protected $checkAll = false;
        protected $output = 'plain';
        protected $checks = array();
        protected $L = null;

        protected $results = array();

        private $includeExif = false;

        public function __construct( Array $options ) {
                if ( isset( $options['help'] ) ) {
                        echo $this->help();
                        exit(1);
                }

                if ( isset( $options['lang'] ) ) {
                        $this->code = $options['lang'];
                } else {
                        global $wgLanguageCode;
                        $this->code = $wgLanguageCode;
                }

                if ( isset( $options['level'] ) ) {
                        $this->level = $options['level'];
                }

                $this->doLinks = isset( $options['links'] );
                $this->includeExif = !isset( $options['noexif'] );
                $this->checkAll = isset( $options['all'] );

                if ( isset( $options['prefix'] ) ) {
                        $this->linksPrefix = $options['prefix'];
                }

                if ( isset( $options['wikilang'] ) ) {
                        $this->wikiCode = $options['wikilang'];
                }

                if ( isset( $options['whitelist'] ) ) {
                        $this->checks = explode( ',', $options['whitelist'] );
                } elseif ( isset( $options['blacklist'] ) ) {
                        $this->checks = array_diff(
                                isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(),
                                explode( ',', $options['blacklist'] )
                        );
                } elseif ( isset( $options['easy'] ) ) {
                        $this->checks = $this->easyChecks();
                } else {
                        $this->checks = $this->defaultChecks();
                }

                if ( isset( $options['output'] ) ) {
                        $this->output = $options['output'];
                }

                $this->L = new languages( $this->includeExif );
        }

        protected function defaultChecks() {
                return array(
                        'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural',
                        'whitespace', 'xhtml', 'chars', 'links', 'unbalanced', 'namespace',
                        'projecttalk', 'magic', 'magic-old', 'magic-over', 'magic-case',
                        'special', 'special-old',
                );
        }

        protected function nonMessageChecks() {
                return array(
                        'namespace', 'projecttalk', 'magic', 'magic-old', 'magic-over',
                        'magic-case', 'special', 'special-old',
                );
        }

        protected function easyChecks() {
                return array(
                        'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars', 'magic-old',
                        'magic-over', 'magic-case', 'special-old',
                );
        }

        protected function getChecks() {
                return array(
                        'untranslated' => 'getUntranslatedMessages',
                        'duplicate'    => 'getDuplicateMessages',
                        'obsolete'     => 'getObsoleteMessages',
                        'variables'    => 'getMessagesWithMismatchVariables',
                        'plural'       => 'getMessagesWithoutPlural',
                        'empty'        => 'getEmptyMessages',
                        'whitespace'   => 'getMessagesWithWhitespace',
                        'xhtml'        => 'getNonXHTMLMessages',
                        'chars'        => 'getMessagesWithWrongChars',
                        'links'        => 'getMessagesWithDubiousLinks',
                        'unbalanced'   => 'getMessagesWithUnbalanced',
                        'namespace'    => 'getUntranslatedNamespaces',
                        'projecttalk'  => 'getProblematicProjectTalks',
                        'magic'        => 'getUntranslatedMagicWords',
                        'magic-old'    => 'getObsoleteMagicWords',
                        'magic-over'   => 'getOverridingMagicWords',
                        'magic-case'   => 'getCaseMismatchMagicWords',
                        'special'      => 'getUntraslatedSpecialPages',
                        'special-old'  => 'getObsoleteSpecialPages',
                );
        }

        protected function getTotalCount() {
                return array(
                        'namespace'    => array( 'getNamespaceNames', 'en' ),
                        'projecttalk'  => null,
                        'magic'        => array( 'getMagicWords', 'en' ),
                        'magic-old'    => array( 'getMagicWords', null ),
                        'magic-over'   => array( 'getMagicWords', null ),
                        'magic-case'   => array( 'getMagicWords', null ),
                        'special'      => array( 'getSpecialPageAliases', 'en' ),
                        'special-old'  => array( 'getSpecialPageAliases', null ),
                );
        }

        protected function getDescriptions() {
                return array(
                        'untranslated' => '$1 message(s) of $2 are not translated to $3, but exist in en:',
                        'duplicate'    => '$1 message(s) of $2 are translated the same in en and $3:',
                        'obsolete'     => '$1 message(s) of $2 do not exist in en or are in the ignore list, but exist in $3:',
                        'variables'    => '$1 message(s) of $2 in $3 don\'t match the variables used in en:',
                        'plural'       => '$1 message(s) of $2 in $3 don\'t use {{plural}} while en uses:',
                        'empty'        => '$1 message(s) of $2 in $3 are empty or -:',
                        'whitespace'   => '$1 message(s) of $2 in $3 have trailing whitespace:',
                        'xhtml'        => '$1 message(s) of $2 in $3 contain illegal XHTML:',
                        'chars'        => '$1 message(s) of $2 in $3 include hidden chars which should not be used in the messages:',
                        'links'        => '$1 message(s) of $2 in $3 have problematic link(s):',
                        'unbalanced'   => '$1 message(s) of $2 in $3 have unbalanced {[]}:',
                        'namespace'    => '$1 namespace name(s) of $2 are not translated to $3, but exist in en:',
                        'projecttalk'  => '$1 namespace name(s) and alias(es) in $3 are project talk namespaces without the parameter:',
                        'magic'        => '$1 magic word(s) of $2 are not translated to $3, but exist in en:',
                        'magic-old'    => '$1 magic word(s) of $2 do not exist in en, but exist in $3:',
                        'magic-over'   => '$1 magic word(s) of $2 in $3 do not contain the original en word(s):',
                        'magic-case'   => '$1 magic word(s) of $2 in $3 change the case-sensitivity of the original en word:',
                        'special'      => '$1 special page alias(es) of $2 are not translated to $3, but exist in en:',
                        'special-old'  => '$1 special page alias(es) of $2 do not exist in en, but exist in $3:',
                );
        }

        protected function help() {
                return <<<ENDS
Run this script to check a specific language file, or all of them.
Command line settings are in form --parameter[=value].
Parameters:
        --help: Show this help.
        --lang: Language code (default: the installation default language).
        --all: Check all customized languages.
        --level: Show the following display level (default: 2):
                * 0: Skip the checks (useful for checking syntax).
                * 1: Show only the stub headers and number of wrong messages, without list of messages.
                * 2: Show only the headers and the message keys, without the message values.
                * 3: Show both the headers and the complete messages, with both keys and values.
        --links: Link the message values (default off).
        --prefix: prefix to add to links.
        --wikilang: For the links, what is the content language of the wiki to display the output in (default en).
        --noexif: Do not check for EXIF messages (a bit hard and boring to translate), if you know
                that they are currently not translated and want to focus on other problems (default off).
        --whitelist: Do only the following checks (form: code,code).
        --blacklist: Do not do the following checks (form: code,code).
        --easy: Do only the easy checks, which can be treated by non-speakers of the language.

Check codes (ideally, all of them should result 0; all the checks are executed by default (except language-specific check blacklists in checkLanguage.inc):
        * untranslated: Messages which are required to translate, but are not translated.
        * duplicate: Messages which translation equal to fallback
        * obsolete: Messages which are untranslatable or do not exist, but are translated.
        * variables: Messages without variables which should be used, or with variables which should not be used.
        * empty: Empty messages and messages that contain only -.
        * whitespace: Messages which have trailing whitespace.
        * xhtml: Messages which are not well-formed XHTML (checks only few common errors).
        * chars: Messages with hidden characters.
        * links: Messages which contains broken links to pages (does not find all).
        * unbalanced: Messages which contains unequal numbers of opening {[ and closing ]}.
        * namespace: Namespace names that were not translated.
        * projecttalk: Namespace names and aliases where the project talk does not contain $1.
        * magic: Magic words that were not translated.
        * magic-old: Magic words which do not exist.
        * magic-over: Magic words that override the original English word.
        * magic-case: Magic words whose translation changes the case-sensitivity of the original English word.
        * special: Special page names that were not translated.
        * special-old: Special page names which do not exist.

ENDS;
        }

        public function execute() {
                $this->doChecks();
                if ( $this->level > 0 ) {
                        switch ( $this->output ) {
                                case 'plain':
                                        $this->outputText();
                                        break;
                                case 'wiki':
                                        $this->outputWiki();
                                        break;
                                default:
                                        throw new MWException( "Invalid output type $this->output" );
                        }
                }
        }

        protected function doChecks() {
                $ignoredCodes = array( 'en', 'enRTL' );

                $this->results = array();
                # Check the language
                if ( $this->checkAll ) {
                        foreach ( $this->L->getLanguages() as $language ) {
                                if ( !in_array( $language, $ignoredCodes ) ) {
                                        $this->results[$language] = $this->checkLanguage( $language );
                                }
                        }
                } else {
                        if ( in_array( $this->code, $ignoredCodes ) ) {
                                throw new MWException( "Cannot check code $this->code." );
                        } else {
                                $this->results[$this->code] = $this->checkLanguage( $this->code );
                        }
                }
        }

        protected function getCheckBlacklist() {
                global $checkBlacklist;
                return $checkBlacklist;
        }

        protected function checkLanguage( $code ) {
                # Syntax check only
                $results = array();
                if ( $this->level === 0 ) {
                        $this->L->getMessages( $code );
                        return $results;
                }

                $checkFunctions = $this->getChecks();
                $checkBlacklist = $this->getCheckBlacklist();
                foreach ( $this->checks as $check ) {
                        if ( isset( $checkBlacklist[$code] ) &&
                                in_array( $check, $checkBlacklist[$code] ) ) {
                                $results[$check] = array();
                                continue;
                        }

                        $callback = array( $this->L, $checkFunctions[$check] );
                        if ( !is_callable( $callback ) ) {
                                throw new MWException( "Unkown check $check." );
                        }
                        $results[$check] = call_user_func( $callback, $code );
                }

                return $results;
        }

        protected function formatKey( $key, $code ) {
                if ( $this->doLinks ) {
                        $displayKey = ucfirst( $key );
                        if ( $code == $this->wikiCode ) {
                                return "[[{$this->linksPrefix}MediaWiki:$displayKey|$key]]";
                        } else {
                                return "[[{$this->linksPrefix}MediaWiki:$displayKey/$code|$key]]";
                        }
                } else {
                        return $key;
                }
        }

        protected function outputText() {
                foreach ( $this->results as $code => $results ) {
                        $translated = $this->L->getMessages( $code );
                        $translated = count( $translated['translated'] );
                        foreach ( $results as $check => $messages ) {
                                $count = count( $messages );
                                if ( $count ) {
                                        if ( $check == 'untranslated' ) {
                                                $translatable = $this->L->getGeneralMessages();
                                                $total = count( $translatable['translatable'] );
                                        } elseif ( in_array( $check, $this->nonMessageChecks() ) ) {
                                                $totalCount = $this->getTotalCount();
                                                $totalCount = $totalCount[$check];
                                                $callback = array( $this->L, $totalCount[0] );
                                                $callCode = $totalCount[1] ? $totalCount[1] : $code;
                                                $total = count( call_user_func( $callback, $callCode ) );
                                        } else {
                                                $total = $translated;
                                        }
                                        $search = array( '$1', '$2', '$3' );
                                        $replace = array( $count, $total, $code );
                                        $descriptions = $this->getDescriptions();
                                        echo "\n" . str_replace( $search, $replace, $descriptions[$check] ) . "\n";
                                        if ( $this->level == 1 ) {
                                                echo "[messages are hidden]\n";
                                        } else {
                                                foreach ( $messages as $key => $value ) {
                                                        if( !in_array( $check, $this->nonMessageChecks() ) ) {
                                                                $key = $this->formatKey( $key, $code );
                                                        }
                                                        if ( $this->level == 2 || empty( $value ) ) {
                                                                echo "* $key\n";
                                                        } else {
                                                                echo "* $key:           '$value'\n";
                                                        }
                                                }
                                        }
                                }
                        }
                }
        }

        function outputWiki() {
                $detailText = '';
                $rows[] = '! Language !! Code !! Total !! ' . implode( ' !! ', array_diff( $this->checks, $this->nonMessageChecks() ) );
                foreach ( $this->results as $code => $results ) {
                        $detailTextForLang = "==$code==\n";
                        $numbers = array();
                        $problems = 0;
                        $detailTextForLangChecks = array();
                        foreach ( $results as $check => $messages ) {
                                if( in_array( $check, $this->nonMessageChecks() ) ) {
                                        continue;
                                }
                                $count = count( $messages );
                                if ( $count ) {
                                        $problems += $count;
                                        $messageDetails = array();
                                        foreach ( $messages as $key => $details ) {
                                                $displayKey = $this->formatKey( $key, $code );
                                                $messageDetails[] = $displayKey;
                                        }
                                        $detailTextForLangChecks[] = "=== $code-$check ===\n* " . implode( ', ', $messageDetails );
                                        $numbers[] = "'''[[#$code-$check|$count]]'''";
                                } else {
                                        $numbers[] = $count;
                                }

                        }

                        if ( count( $detailTextForLangChecks ) ) {
                                $detailText .= $detailTextForLang . implode( "\n", $detailTextForLangChecks ) . "\n";
                        }

                        if ( !$problems ) {
                                # Don't list languages without problems
                                continue;
                        }
                        $language = Language::fetchLanguageName( $code );
                        $rows[] = "| $language || $code || $problems || " . implode( ' || ', $numbers );
                }

                $tableRows = implode( "\n|-\n", $rows );

                $version = SpecialVersion::getVersion( 'nodb' );
                echo <<<EOL
'''Check results are for:''' <code>$version</code>


{| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear: both;"
$tableRows
|}

$detailText

EOL;
        }

        protected function isEmpty() {
                foreach( $this->results as $results ) {
                        foreach( $results as $messages ) {
                                if( !empty( $messages ) ) {
                                        return false;
                                }
                        }
                }
                return true;
        }
}

class CheckExtensionsCLI extends CheckLanguageCLI {
        private $extensions;

        public function __construct( Array $options, $extension ) {
                if ( isset( $options['help'] ) ) {
                        echo $this->help();
                        exit(1);
                }

                if ( isset( $options['lang'] ) ) {
                        $this->code = $options['lang'];
                } else {
                        global $wgLanguageCode;
                        $this->code = $wgLanguageCode;
                }

                if ( isset( $options['level'] ) ) {
                        $this->level = $options['level'];
                }

                $this->doLinks = isset( $options['links'] );

                if ( isset( $options['wikilang'] ) ) {
                        $this->wikiCode = $options['wikilang'];
                }

                if ( isset( $options['whitelist'] ) ) {
                        $this->checks = explode( ',', $options['whitelist'] );
                } elseif ( isset( $options['blacklist'] ) ) {
                        $this->checks = array_diff(
                                isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(),
                                explode( ',', $options['blacklist'] )
                        );
                } elseif ( isset( $options['easy'] ) ) {
                        $this->checks = $this->easyChecks();
                } else {
                        $this->checks = $this->defaultChecks();
                }

                if ( isset( $options['output'] ) ) {
                        $this->output = $options['output'];
                }

                # Some additional checks not enabled by default
                if ( isset( $options['duplicate'] ) ) {
                        $this->checks[] = 'duplicate';
                }

                $this->extensions = array();
                $extensions = new PremadeMediawikiExtensionGroups();
                $extensions->addAll();
                if ( $extension == 'all' ) {
                        foreach ( MessageGroups::singleton()->getGroups() as $group ) {
                                if ( strpos( $group->getId(), 'ext-' ) === 0 && !$group->isMeta() ) {
                                        $this->extensions[] = new extensionLanguages( $group );
                                }
                        }
                } elseif ( $extension == 'wikimedia' ) {
                        $wikimedia = MessageGroups::getGroup( 'ext-0-wikimedia' );
                        foreach ( $wikimedia->wmfextensions() as $extension ) {
                                $group = MessageGroups::getGroup( $extension );
                                $this->extensions[] = new extensionLanguages( $group );
                        }
                } elseif ( $extension == 'flaggedrevs' ) {
                        foreach ( MessageGroups::singleton()->getGroups() as $group ) {
                                if ( strpos( $group->getId(), 'ext-flaggedrevs-' ) === 0 && !$group->isMeta() ) {
                                        $this->extensions[] = new extensionLanguages( $group );
                                }
                        }
                } else {
                        $extensions = explode( ',', $extension );
                        foreach ( $extensions as $extension ) {
                                $group = MessageGroups::getGroup( 'ext-' . $extension );
                                if ( $group ) {
                                        $extension = new extensionLanguages( $group );
                                        $this->extensions[] = $extension;
                                } else {
                                        print "No such extension $extension.\n";
                                }
                        }
                }
        }

        protected function defaultChecks() {
                return array(
                        'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural',
                        'whitespace', 'xhtml', 'chars', 'links', 'unbalanced',
                );
        }

        protected function nonMessageChecks() {
                return array();
        }

        protected function easyChecks() {
                return array(
                        'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars',
                );
        }

        protected function help() {
                return <<<ENDS
Run this script to check the status of a specific language in extensions, or all of them.
Command line settings are in form --parameter[=value], except for the first one.
Parameters:
        * First parameter (mandatory): Extension name, multiple extension names (separated by commas), "all" for all the extensions, "wikimedia" for extensions used by Wikimedia or "flaggedrevs" for all FLaggedRevs extension messages.
        * lang: Language code (default: the installation default language).
        * help: Show this help.
        * level: Show the following display level (default: 2).
        * links: Link the message values (default off).
        * wikilang: For the links, what is the content language of the wiki to display the output in (default en).
        * whitelist: Do only the following checks (form: code,code).
        * blacklist: Do not perform the following checks (form: code,code).
        * easy: Do only the easy checks, which can be treated by non-speakers of the language.
Check codes (ideally, all of them should result 0; all the checks are executed by default (except language-specific check blacklists in checkLanguage.inc):
        * untranslated: Messages which are required to translate, but are not translated.
        * duplicate: Messages which translation equal to fallback
        * obsolete: Messages which are untranslatable, but translated.
        * variables: Messages without variables which should be used, or with variables which should not be used.
        * empty: Empty messages.
        * whitespace: Messages which have trailing whitespace.
        * xhtml: Messages which are not well-formed XHTML (checks only few common errors).
        * chars: Messages with hidden characters.
        * links: Messages which contains broken links to pages (does not find all).
        * unbalanced: Messages which contains unequal numbers of opening {[ and closing ]}.
Display levels (default: 2):
        * 0: Skip the checks (useful for checking syntax).
        * 1: Show only the stub headers and number of wrong messages, without list of messages.
        * 2: Show only the headers and the message keys, without the message values.
        * 3: Show both the headers and the complete messages, with both keys and values.

ENDS;
        }

        public function execute() {
                $this->doChecks();
        }

        protected function checkLanguage( $code ) {
                foreach( $this->extensions as $extension ) {
                        $this->L = $extension;
                        $this->results = array();
                        $this->results[$code] = parent::checkLanguage( $code );

                        if( !$this->isEmpty() ) {
                                echo $extension->name() . ":\n";

                                if( $this->level > 0 ) {
                                        switch( $this->output ) {
                                                case 'plain':
                                                        $this->outputText();
                                                        break;
                                                case 'wiki':
                                                        $this->outputWiki();
                                                        break;
                                                default:
                                                        throw new MWException( "Invalid output type $this->output" );
                                        }
                                }

                                echo "\n";
                        }
                }
        }
}

# Blacklist some checks for some languages
$checkBlacklist = array(
#'code'        => array( 'check1', 'check2' ... )
'az'           => array( 'plural' ),
'bo'           => array( 'plural' ),
'dz'           => array( 'plural' ),
'id'           => array( 'plural' ),
'fa'           => array( 'plural' ),
'gan'          => array( 'plural' ),
'gan-hans'     => array( 'plural' ),
'gan-hant'     => array( 'plural' ),
'gn'           => array( 'plural' ),
'hak'          => array( 'plural' ),
'hu'           => array( 'plural' ),
'ja'           => array( 'plural' ), // Does not use plural
'jv'           => array( 'plural' ),
'ka'           => array( 'plural' ),
'kk-arab'      => array( 'plural' ),
'kk-cyrl'      => array( 'plural' ),
'kk-latn'      => array( 'plural' ),
'km'           => array( 'plural' ),
'kn'           => array( 'plural' ),
'ko'           => array( 'plural' ),
'lzh'          => array( 'plural' ),
'mn'           => array( 'plural' ),
'ms'           => array( 'plural' ),
'my'           => array( 'plural', 'chars' ),  // Uses a lot zwnj
'sah'          => array( 'plural' ),
'sq'           => array( 'plural' ),
'tet'          => array( 'plural' ),
'th'           => array( 'plural' ),
'to'           => array( 'plural' ),
'tr'           => array( 'plural' ),
'vi'           => array( 'plural' ),
'wuu'          => array( 'plural' ),
'xmf'          => array( 'plural' ),
'yo'           => array( 'plural' ),
'yue'          => array( 'plural' ),
'zh'           => array( 'plural' ),
'zh-classical' => array( 'plural' ),
'zh-cn'        => array( 'plural' ),
'zh-hans'      => array( 'plural' ),
'zh-hant'      => array( 'plural' ),
'zh-hk'        => array( 'plural' ),
'zh-sg'        => array( 'plural' ),
'zh-tw'        => array( 'plural' ),
'zh-yue'       => array( 'plural' ),
);