ApiPageSet.php

Go to the documentation of this file.

<?php
class ApiPageSet extends ApiBase {

    const DISABLE_GENERATORS = 1;

    private $mDbSource;
    private $mParams;
    private $mResolveRedirects;
    private $mConvertTitles;
    private $mAllowGenerator;

    private $mAllPages = array(); // [ns][dbkey] => page_id or negative when missing
    private $mTitles = array();
    private $mGoodTitles = array();
    private $mMissingTitles = array();
    private $mInvalidTitles = array();
    private $mMissingPageIDs = array();
    private $mRedirectTitles = array();
    private $mSpecialTitles = array();
    private $mNormalizedTitles = array();
    private $mInterwikiTitles = array();
    private $mPendingRedirectIDs = array();
    private $mConvertedTitles = array();
    private $mGoodRevIDs = array();
    private $mMissingRevIDs = array();
    private $mFakePageId = -1;
    private $mCacheMode = 'public';
    private $mRequestedPageFields = array();
    private $mDefaultNamespace = NS_MAIN;

    public function __construct( ApiBase $dbSource, $flags = 0, $defaultNamespace = NS_MAIN ) {
        parent::__construct( $dbSource->getMain(), $dbSource->getModuleName() );
        $this->mDbSource = $dbSource;
        $this->mAllowGenerator = ( $flags & ApiPageSet::DISABLE_GENERATORS ) == 0;
        $this->mDefaultNamespace = $defaultNamespace;

        $this->profileIn();
        $this->mParams = $this->extractRequestParams();
        $this->mResolveRedirects = $this->mParams['redirects'];
        $this->mConvertTitles = $this->mParams['converttitles'];
        $this->profileOut();
    }

    public function executeDryRun() {
        $this->executeInternal( true );
    }

    public function execute() {
        $this->executeInternal( false );
    }

    private function executeInternal( $isDryRun ) {
        $this->profileIn();

        $generatorName = $this->mAllowGenerator ? $this->mParams['generator'] : null;
        if ( isset( $generatorName ) ) {
            $dbSource = $this->mDbSource;
            $isQuery = $dbSource instanceof ApiQuery;
            if ( !$isQuery ) {
                // If the parent container of this pageset is not ApiQuery, we must create it to run generator
                $dbSource = $this->getMain()->getModuleManager()->getModule( 'query' );
                // Enable profiling for query module because it will be used for db sql profiling
                $dbSource->profileIn();
            }
            $generator = $dbSource->getModuleManager()->getModule( $generatorName, null, true );
            if ( $generator === null ) {
                $this->dieUsage( 'Unknown generator=' . $generatorName, 'badgenerator' );
            }
            if ( !$generator instanceof ApiQueryGeneratorBase ) {
                $this->dieUsage( "Module $generatorName cannot be used as a generator", 'badgenerator' );
            }
            // Create a temporary pageset to store generator's output,
            // add any additional fields generator may need, and execute pageset to populate titles/pageids
            $tmpPageSet = new ApiPageSet( $dbSource, ApiPageSet::DISABLE_GENERATORS );
            $generator->setGeneratorMode( $tmpPageSet );
            $this->mCacheMode = $generator->getCacheMode( $generator->extractRequestParams() );

            if ( !$isDryRun ) {
                $generator->requestExtraData( $tmpPageSet );
            }
            $tmpPageSet->executeInternal( $isDryRun );

            // populate this pageset with the generator output
            $this->profileOut();
            $generator->profileIn();

            if ( !$isDryRun ) {
                $generator->executeGenerator( $this );
                wfRunHooks( 'APIQueryGeneratorAfterExecute', array( &$generator, &$this ) );
            } else {
                // Prevent warnings from being reported on these parameters
                $main = $this->getMain();
                foreach ( $generator->extractRequestParams() as $paramName => $param ) {
                    $main->getVal( $generator->encodeParamName( $paramName ) );
                }
            }
            $generator->profileOut();
            $this->profileIn();

            if ( !$isDryRun ) {
                $this->resolvePendingRedirects();
            }

            if ( !$isQuery ) {
                // If this pageset is not part of the query, we called profileIn() above
                $dbSource->profileOut();
            }
        } else {
            // Only one of the titles/pageids/revids is allowed at the same time
            $dataSource = null;
            if ( isset( $this->mParams['titles'] ) ) {
                $dataSource = 'titles';
            }
            if ( isset( $this->mParams['pageids'] ) ) {
                if ( isset( $dataSource ) ) {
                    $this->dieUsage( "Cannot use 'pageids' at the same time as '$dataSource'", 'multisource' );
                }
                $dataSource = 'pageids';
            }
            if ( isset( $this->mParams['revids'] ) ) {
                if ( isset( $dataSource ) ) {
                    $this->dieUsage( "Cannot use 'revids' at the same time as '$dataSource'", 'multisource' );
                }
                $dataSource = 'revids';
            }

            if ( !$isDryRun ) {
                // Populate page information with the original user input
                switch ( $dataSource ) {
                    case 'titles':
                        $this->initFromTitles( $this->mParams['titles'] );
                        break;
                    case 'pageids':
                        $this->initFromPageIds( $this->mParams['pageids'] );
                        break;
                    case 'revids':
                        if ( $this->mResolveRedirects ) {
                            $this->setWarning( 'Redirect resolution cannot be used together with the revids= parameter. ' .
                                'Any redirects the revids= point to have not been resolved.' );
                        }
                        $this->mResolveRedirects = false;
                        $this->initFromRevIDs( $this->mParams['revids'] );
                        break;
                    default:
                        // Do nothing - some queries do not need any of the data sources.
                        break;
                }
            }
        }
        $this->profileOut();
    }

    public function isResolvingRedirects() {
        return $this->mResolveRedirects;
    }

    public function getDataSource() {
        if ( $this->mAllowGenerator && isset( $this->mParams['generator'] ) ) {
            return 'generator';
        }
        if ( isset( $this->mParams['titles'] ) ) {
            return 'titles';
        }
        if ( isset( $this->mParams['pageids'] ) ) {
            return 'pageids';
        }
        if ( isset( $this->mParams['revids'] ) ) {
            return 'revids';
        }
        return null;
    }

    public function requestField( $fieldName ) {
        $this->mRequestedPageFields[$fieldName] = null;
    }

    public function getCustomField( $fieldName ) {
        return $this->mRequestedPageFields[$fieldName];
    }

    public function getPageTableFields() {
        // Ensure we get minimum required fields
        // DON'T change this order
        $pageFlds = array(
            'page_namespace' => null,
            'page_title' => null,
            'page_id' => null,
        );

        if ( $this->mResolveRedirects ) {
            $pageFlds['page_is_redirect'] = null;
        }

        // only store non-default fields
        $this->mRequestedPageFields = array_diff_key( $this->mRequestedPageFields, $pageFlds );

        $pageFlds = array_merge( $pageFlds, $this->mRequestedPageFields );
        return array_keys( $pageFlds );
    }

    public function getAllTitlesByNamespace() {
        return $this->mAllPages;
    }

    public function getTitles() {
        return $this->mTitles;
    }

    public function getTitleCount() {
        return count( $this->mTitles );
    }

    public function getGoodTitles() {
        return $this->mGoodTitles;
    }

    public function getGoodTitleCount() {
        return count( $this->mGoodTitles );
    }

    public function getMissingTitles() {
        return $this->mMissingTitles;
    }

    public function getInvalidTitles() {
        return $this->mInvalidTitles;
    }

    public function getMissingPageIDs() {
        return $this->mMissingPageIDs;
    }

    public function getRedirectTitles() {
        return $this->mRedirectTitles;
    }

    public function getRedirectTitlesAsResult( $result = null ) {
        $values = array();
        foreach ( $this->getRedirectTitles() as $titleStrFrom => $titleTo ) {
            $r = array(
                'from' => strval( $titleStrFrom ),
                'to' => $titleTo->getPrefixedText(),
            );
            if ( $titleTo->getFragment() !== '' ) {
                $r['tofragment'] = $titleTo->getFragment();
            }
            $values[] = $r;
        }
        if ( !empty( $values ) && $result ) {
            $result->setIndexedTagName( $values, 'r' );
        }
        return $values;
    }

    public function getNormalizedTitles() {
        return $this->mNormalizedTitles;
    }

    public function getNormalizedTitlesAsResult( $result = null ) {
        $values = array();
        foreach ( $this->getNormalizedTitles() as $rawTitleStr => $titleStr ) {
            $values[] = array(
                'from' => $rawTitleStr,
                'to' => $titleStr
            );
        }
        if ( !empty( $values ) && $result ) {
            $result->setIndexedTagName( $values, 'n' );
        }
        return $values;
    }

    public function getConvertedTitles() {
        return $this->mConvertedTitles;
    }

    public function getConvertedTitlesAsResult( $result = null ) {
        $values = array();
        foreach ( $this->getConvertedTitles() as $rawTitleStr => $titleStr ) {
            $values[] = array(
                'from' => $rawTitleStr,
                'to' => $titleStr
            );
        }
        if ( !empty( $values ) && $result ) {
            $result->setIndexedTagName( $values, 'c' );
        }
        return $values;
    }

    public function getInterwikiTitles() {
        return $this->mInterwikiTitles;
    }

    public function getInterwikiTitlesAsResult( $result = null, $iwUrl = false ) {
        $values = array();
        foreach ( $this->getInterwikiTitles() as $rawTitleStr => $interwikiStr ) {
            $item = array(
                'title' => $rawTitleStr,
                'iw' => $interwikiStr,
            );
            if ( $iwUrl ) {
                $title = Title::newFromText( $rawTitleStr );
                $item['url'] = $title->getFullURL( '', false, PROTO_CURRENT );
            }
            $values[] = $item;
        }
        if ( !empty( $values ) && $result ) {
            $result->setIndexedTagName( $values, 'i' );
        }
        return $values;
    }

    public function getRevisionIDs() {
        return $this->mGoodRevIDs;
    }

    public function getMissingRevisionIDs() {
        return $this->mMissingRevIDs;
    }

    public function getMissingRevisionIDsAsResult( $result = null ) {
        $values = array();
        foreach ( $this->getMissingRevisionIDs() as $revid ) {
            $values[$revid] = array(
                'revid' => $revid
            );
        }
        if ( !empty( $values ) && $result ) {
            $result->setIndexedTagName( $values, 'rev' );
        }
        return $values;
    }

    public function getSpecialTitles() {
        return $this->mSpecialTitles;
    }

    public function getRevisionCount() {
        return count( $this->getRevisionIDs() );
    }

    public function populateFromTitles( $titles ) {
        $this->profileIn();
        $this->initFromTitles( $titles );
        $this->profileOut();
    }

    public function populateFromPageIDs( $pageIDs ) {
        $this->profileIn();
        $this->initFromPageIds( $pageIDs );
        $this->profileOut();
    }

    public function populateFromQueryResult( $db, $queryResult ) {
        $this->profileIn();
        $this->initFromQueryResult( $queryResult );
        $this->profileOut();
    }

    public function populateFromRevisionIDs( $revIDs ) {
        $this->profileIn();
        $this->initFromRevIDs( $revIDs );
        $this->profileOut();
    }

    public function processDbRow( $row ) {
        // Store Title object in various data structures
        $title = Title::newFromRow( $row );

        $pageId = intval( $row->page_id );
        $this->mAllPages[$row->page_namespace][$row->page_title] = $pageId;
        $this->mTitles[] = $title;

        if ( $this->mResolveRedirects && $row->page_is_redirect == '1' ) {
            $this->mPendingRedirectIDs[$pageId] = $title;
        } else {
            $this->mGoodTitles[$pageId] = $title;
        }

        foreach ( $this->mRequestedPageFields as $fieldName => &$fieldValues ) {
            $fieldValues[$pageId] = $row->$fieldName;
        }
    }

    public function finishPageSetGeneration() {
        wfDeprecated( __METHOD__, '1.21' );
    }

    private function initFromTitles( $titles ) {
        // Get validated and normalized title objects
        $linkBatch = $this->processTitlesArray( $titles );
        if ( $linkBatch->isEmpty() ) {
            return;
        }

        $db = $this->getDB();
        $set = $linkBatch->constructSet( 'page', $db );

        // Get pageIDs data from the `page` table
        $this->profileDBIn();
        $res = $db->select( 'page', $this->getPageTableFields(), $set,
                    __METHOD__ );
        $this->profileDBOut();

        // Hack: get the ns:titles stored in array(ns => array(titles)) format
        $this->initFromQueryResult( $res, $linkBatch->data, true ); // process Titles

        // Resolve any found redirects
        $this->resolvePendingRedirects();
    }

    private function initFromPageIds( $pageids ) {
        if ( !$pageids ) {
            return;
        }

        $pageids = array_map( 'intval', $pageids ); // paranoia
        $remaining = array_flip( $pageids );

        $pageids = self::getPositiveIntegers( $pageids );

        $res = null;
        if ( !empty( $pageids ) ) {
            $set = array(
                'page_id' => $pageids
            );
            $db = $this->getDB();

            // Get pageIDs data from the `page` table
            $this->profileDBIn();
            $res = $db->select( 'page', $this->getPageTableFields(), $set,
                        __METHOD__ );
            $this->profileDBOut();
        }

        $this->initFromQueryResult( $res, $remaining, false ); // process PageIDs

        // Resolve any found redirects
        $this->resolvePendingRedirects();
    }

    private function initFromQueryResult( $res, &$remaining = null, $processTitles = null ) {
        if ( !is_null( $remaining ) && is_null( $processTitles ) ) {
            ApiBase::dieDebug( __METHOD__, 'Missing $processTitles parameter when $remaining is provided' );
        }

        $usernames = array();
        if ( $res ) {
            foreach ( $res as $row ) {
                $pageId = intval( $row->page_id );

                // Remove found page from the list of remaining items
                if ( isset( $remaining ) ) {
                    if ( $processTitles ) {
                        unset( $remaining[$row->page_namespace][$row->page_title] );
                    } else {
                        unset( $remaining[$pageId] );
                    }
                }

                // Store any extra fields requested by modules
                $this->processDbRow( $row );

                // Need gender information
                if ( MWNamespace::hasGenderDistinction( $row->page_namespace ) ) {
                    $usernames[] = $row->page_title;
                }
            }
        }

        if ( isset( $remaining ) ) {
            // Any items left in the $remaining list are added as missing
            if ( $processTitles ) {
                // The remaining titles in $remaining are non-existent pages
                foreach ( $remaining as $ns => $dbkeys ) {
                    foreach ( array_keys( $dbkeys ) as $dbkey ) {
                        $title = Title::makeTitle( $ns, $dbkey );
                        $this->mAllPages[$ns][$dbkey] = $this->mFakePageId;
                        $this->mMissingTitles[$this->mFakePageId] = $title;
                        $this->mFakePageId--;
                        $this->mTitles[] = $title;

                        // need gender information
                        if ( MWNamespace::hasGenderDistinction( $ns ) ) {
                            $usernames[] = $dbkey;
                        }
                    }
                }
            } else {
                // The remaining pageids do not exist
                if ( !$this->mMissingPageIDs ) {
                    $this->mMissingPageIDs = array_keys( $remaining );
                } else {
                    $this->mMissingPageIDs = array_merge( $this->mMissingPageIDs, array_keys( $remaining ) );
                }
            }
        }

        // Get gender information
        $genderCache = GenderCache::singleton();
        $genderCache->doQuery( $usernames, __METHOD__ );
    }

    private function initFromRevIDs( $revids ) {
        if ( !$revids ) {
            return;
        }

        $revids = array_map( 'intval', $revids ); // paranoia
        $db = $this->getDB();
        $pageids = array();
        $remaining = array_flip( $revids );

        $revids = self::getPositiveIntegers( $revids );

        if ( !empty( $revids ) ) {
            $tables = array( 'revision', 'page' );
            $fields = array( 'rev_id', 'rev_page' );
            $where = array( 'rev_id' => $revids, 'rev_page = page_id' );

            // Get pageIDs data from the `page` table
            $this->profileDBIn();
            $res = $db->select( $tables, $fields, $where, __METHOD__ );
            foreach ( $res as $row ) {
                $revid = intval( $row->rev_id );
                $pageid = intval( $row->rev_page );
                $this->mGoodRevIDs[$revid] = $pageid;
                $pageids[$pageid] = '';
                unset( $remaining[$revid] );
            }
            $this->profileDBOut();
        }

        $this->mMissingRevIDs = array_keys( $remaining );

        // Populate all the page information
        $this->initFromPageIds( array_keys( $pageids ) );
    }

    private function resolvePendingRedirects() {
        if ( $this->mResolveRedirects ) {
            $db = $this->getDB();
            $pageFlds = $this->getPageTableFields();

            // Repeat until all redirects have been resolved
            // The infinite loop is prevented by keeping all known pages in $this->mAllPages
            while ( $this->mPendingRedirectIDs ) {
                // Resolve redirects by querying the pagelinks table, and repeat the process
                // Create a new linkBatch object for the next pass
                $linkBatch = $this->getRedirectTargets();

                if ( $linkBatch->isEmpty() ) {
                    break;
                }

                $set = $linkBatch->constructSet( 'page', $db );
                if ( $set === false ) {
                    break;
                }

                // Get pageIDs data from the `page` table
                $this->profileDBIn();
                $res = $db->select( 'page', $pageFlds, $set, __METHOD__ );
                $this->profileDBOut();

                // Hack: get the ns:titles stored in array(ns => array(titles)) format
                $this->initFromQueryResult( $res, $linkBatch->data, true );
            }
        }
    }

    private function getRedirectTargets() {
        $lb = new LinkBatch();
        $db = $this->getDB();

        $this->profileDBIn();
        $res = $db->select(
            'redirect',
            array(
                'rd_from',
                'rd_namespace',
                'rd_fragment',
                'rd_interwiki',
                'rd_title'
            ), array( 'rd_from' => array_keys( $this->mPendingRedirectIDs ) ),
            __METHOD__
        );
        $this->profileDBOut();
        foreach ( $res as $row ) {
            $rdfrom = intval( $row->rd_from );
            $from = $this->mPendingRedirectIDs[$rdfrom]->getPrefixedText();
            $to = Title::makeTitle( $row->rd_namespace, $row->rd_title, $row->rd_fragment, $row->rd_interwiki );
            unset( $this->mPendingRedirectIDs[$rdfrom] );
            if ( !$to->isExternal() && !isset( $this->mAllPages[$row->rd_namespace][$row->rd_title] ) ) {
                $lb->add( $row->rd_namespace, $row->rd_title );
            }
            $this->mRedirectTitles[$from] = $to;
        }

        if ( $this->mPendingRedirectIDs ) {
            // We found pages that aren't in the redirect table
            // Add them
            foreach ( $this->mPendingRedirectIDs as $id => $title ) {
                $page = WikiPage::factory( $title );
                $rt = $page->insertRedirect();
                if ( !$rt ) {
                    // What the hell. Let's just ignore this
                    continue;
                }
                $lb->addObj( $rt );
                $this->mRedirectTitles[$title->getPrefixedText()] = $rt;
                unset( $this->mPendingRedirectIDs[$id] );
            }
        }
        return $lb;
    }

    public function getCacheMode( $params = null ) {
        return $this->mCacheMode;
    }

    private function processTitlesArray( $titles ) {
        $usernames = array();
        $linkBatch = new LinkBatch();

        foreach ( $titles as $title ) {
            if ( is_string( $title ) ) {
                $titleObj = Title::newFromText( $title, $this->mDefaultNamespace );
            } else {
                $titleObj = $title;
            }
            if ( !$titleObj ) {
                // Handle invalid titles gracefully
                $this->mAllPages[0][$title] = $this->mFakePageId;
                $this->mInvalidTitles[$this->mFakePageId] = $title;
                $this->mFakePageId--;
                continue; // There's nothing else we can do
            }
            $unconvertedTitle = $titleObj->getPrefixedText();
            $titleWasConverted = false;
            if ( $titleObj->isExternal() ) {
                // This title is an interwiki link.
                $this->mInterwikiTitles[$unconvertedTitle] = $titleObj->getInterwiki();
            } else {
                // Variants checking
                global $wgContLang;
                if ( $this->mConvertTitles &&
                        count( $wgContLang->getVariants() ) > 1 &&
                        !$titleObj->exists() ) {
                    // Language::findVariantLink will modify titleText and titleObj into
                    // the canonical variant if possible
                    $titleText = is_string( $title ) ? $title : $titleObj->getPrefixedText();
                    $wgContLang->findVariantLink( $titleText, $titleObj );
                    $titleWasConverted = $unconvertedTitle !== $titleObj->getPrefixedText();
                }

                if ( $titleObj->getNamespace() < 0 ) {
                    // Handle Special and Media pages
                    $titleObj = $titleObj->fixSpecialName();
                    $this->mSpecialTitles[$this->mFakePageId] = $titleObj;
                    $this->mFakePageId--;
                } else {
                    // Regular page
                    $linkBatch->addObj( $titleObj );
                }
            }

            // Make sure we remember the original title that was
            // given to us. This way the caller can correlate new
            // titles with the originally requested when e.g. the
            // namespace is localized or the capitalization is
            // different
            if ( $titleWasConverted ) {
                $this->mConvertedTitles[$unconvertedTitle] = $titleObj->getPrefixedText();
                // In this case the page can't be Special.
                if ( is_string( $title ) && $title !== $unconvertedTitle ) {
                    $this->mNormalizedTitles[$title] = $unconvertedTitle;
                }
            } elseif ( is_string( $title ) && $title !== $titleObj->getPrefixedText() ) {
                $this->mNormalizedTitles[$title] = $titleObj->getPrefixedText();
            }

            // Need gender information
            if ( MWNamespace::hasGenderDistinction( $titleObj->getNamespace() ) ) {
                $usernames[] = $titleObj->getText();
            }
        }
        // Get gender information
        $genderCache = GenderCache::singleton();
        $genderCache->doQuery( $usernames, __METHOD__ );

        return $linkBatch;
    }

    protected function getDB() {
        return $this->mDbSource->getDB();
    }

    private static function getPositiveIntegers( $array ) {
        // bug 25734 API: possible issue with revids validation
        // It seems with a load of revision rows, MySQL gets upset
        // Remove any < 0 integers, as they can't be valid
        foreach ( $array as $i => $int ) {
            if ( $int < 0 ) {
                unset( $array[$i] );
            }
        }

        return $array;
    }

    public function getAllowedParams( $flags = 0 ) {
        $result = array(
            'titles' => array(
                ApiBase::PARAM_ISMULTI => true
            ),
            'pageids' => array(
                ApiBase::PARAM_TYPE => 'integer',
                ApiBase::PARAM_ISMULTI => true
            ),
            'revids' => array(
                ApiBase::PARAM_TYPE => 'integer',
                ApiBase::PARAM_ISMULTI => true
            ),
            'redirects' => false,
            'converttitles' => false,
        );
        if ( $this->mAllowGenerator ) {
            if ( $flags & ApiBase::GET_VALUES_FOR_HELP ) {
                $result['generator'] = array(
                    ApiBase::PARAM_TYPE => $this->getGenerators()
                );
            } else {
                $result['generator'] = null;
            }
        }
        return $result;
    }

    private static $generators = null;

    private function getGenerators() {
        if ( self::$generators === null ) {
            $query = $this->mDbSource;
            if ( !( $query instanceof ApiQuery ) ) {
                // If the parent container of this pageset is not ApiQuery,
                // we must create it to get module manager
                $query = $this->getMain()->getModuleManager()->getModule( 'query' );
            }
            $gens = array();
            $mgr = $query->getModuleManager();
            foreach ( $mgr->getNamesWithClasses() as $name => $class ) {
                if ( is_subclass_of( $class, 'ApiQueryGeneratorBase' ) ) {
                    $gens[] = $name;
                }
            }
            sort( $gens );
            self::$generators = $gens;
        }
        return self::$generators;
    }

    public function getParamDescription() {
        return array(
            'titles' => 'A list of titles to work on',
            'pageids' => 'A list of page IDs to work on',
            'revids' => 'A list of revision IDs to work on',
            'generator' => array( 'Get the list of pages to work on by executing the specified query module.',
                'NOTE: generator parameter names must be prefixed with a \'g\', see examples' ),
            'redirects' => 'Automatically resolve redirects',
            'converttitles' => array( 'Convert titles to other variants if necessary. Only works if the wiki\'s content language supports variant conversion.',
                'Languages that support variant conversion include ' . implode( ', ', LanguageConverter::$languagesWithVariants ) ),
        );
    }

    public function getPossibleErrors() {
        return array_merge( parent::getPossibleErrors(), array(
            array( 'code' => 'multisource', 'info' => "Cannot use 'pageids' at the same time as 'dataSource'" ),
            array( 'code' => 'multisource', 'info' => "Cannot use 'revids' at the same time as 'dataSource'" ),
            array( 'code' => 'badgenerator', 'info' => 'Module $generatorName cannot be used as a generator' ),
        ) );
    }
}