25 use Psr\Log\LoggerAwareInterface;
26 use Psr\Log\LoggerInterface;
67 $logger = LoggerFactory::getInstance(
'runJobs' );
99 $response = [
'jobs' => [],
'reached' =>
'none-ready' ];
101 $type = isset( $options[
'type'] ) ? $options[
'type'] :
false;
102 $maxJobs = isset( $options[
'maxJobs'] ) ? $options[
'maxJobs'] :
false;
103 $maxTime = isset( $options[
'maxTime'] ) ? $options[
'maxTime'] :
false;
104 $noThrottle = isset( $options[
'throttle'] ) && !$options[
'throttle'];
107 if (
$type !==
false && !isset( $wgJobClasses[
$type] ) ) {
119 if ( $maxLag >= self::MAX_ALLOWED_LAG ) {
120 $response[
'reached'] =
'slave-lag-limit';
129 $trxProfiler->setLogger( LoggerFactory::getInstance(
'DBPerformance' ) );
130 $trxProfiler->setExpectations( $wgTrxProfilerLimits[
'JobRunner'], __METHOD__ );
141 $startTime = microtime(
true );
146 $blacklist = $noThrottle ? [] : array_keys( $backoffs );
149 if ( $type ===
false ) {
155 } elseif ( in_array( $type, $blacklist ) ) {
158 $job = $group->pop( $type );
164 $jType =
$job->getType();
173 $backoffDeltas[$jType] = isset( $backoffDeltas[$jType] )
174 ? $backoffDeltas[$jType] + $ttw
180 if ( $info[
'status'] !==
false || !
$job->allowRetries() ) {
185 if ( $info[
'status'] ===
false && mt_rand( 0, 49 ) == 0 ) {
186 $ttw = max( $ttw, self::ERROR_BACKOFF_TTL );
187 $backoffDeltas[$jType] = isset( $backoffDeltas[$jType] )
188 ? $backoffDeltas[$jType] + $ttw
194 'status' => ( $info[
'status'] ===
false ) ?
'failed' :
'ok',
195 'error' => $info[
'error'],
196 'time' => $info[
'timeMs']
198 $timeMsTotal += $info[
'timeMs'];
201 if ( $maxJobs && $jobsPopped >= $maxJobs ) {
204 } elseif ( $maxTime && ( microtime(
true ) - $startTime ) > $maxTime ) {
212 $timePassed = microtime(
true ) - $lastCheckTime;
213 if ( $timePassed >= self::LAG_CHECK_PERIOD || $timePassed < 0 ) {
216 'ifWritesSince' => $lastCheckTime,
217 'timeout' => self::MAX_ALLOWED_LAG
220 $response[
'reached'] =
'slave-lag-limit';
223 $lastCheckTime = microtime(
true );
226 if ( $jobsPopped > 0 && ( $jobsPopped % 100 ) == 0 ) {
227 $group->waitForBackups();
239 if ( $backoffDeltas ) {
257 $msg = $job->
toString() .
" STARTING";
258 $this->logger->debug( $msg );
263 $jobStartTime = microtime(
true );
271 }
catch ( Exception
$e ) {
274 $error = get_class( $e ) .
': ' . $e->getMessage();
280 }
catch ( Exception $e ) {
290 $timeMs = intval( ( microtime(
true ) - $jobStartTime ) * 1000 );
296 $pickupDelay = max( 0, $popTime - $readyTs );
297 $stats->timing(
'jobqueue.pickup_delay.all', 1000 * $pickupDelay );
298 $stats->timing(
"jobqueue.pickup_delay.$jType", 1000 * $pickupDelay );
302 if ( $rootTimestamp ) {
304 $stats->timing(
"jobqueue.pickup_root_age.$jType", 1000 * $age );
307 $stats->timing(
"jobqueue.run.$jType", $timeMs );
309 if ( $rssStart && $rssEnd ) {
310 $stats->increment(
"jobqueue.rss_delta.$jType", $rssEnd - $rssStart );
314 $msg = $job->
toString() .
" t=$timeMs error={$error}";
315 $this->logger->error( $msg );
318 $msg = $job->
toString() .
" t=$timeMs good";
319 $this->logger->info( $msg );
323 return [
'status' =>
$status,
'error' => $error,
'timeMs' => $timeMs ];
332 return isset( $info[
'ru_maxrss'] ) ? (int)$info[
'ru_maxrss'] : null;
343 if ( !isset( $wgJobBackoffThrottling[$job->
getType()] ) ||
349 $itemsPerSecond = $wgJobBackoffThrottling[$job->
getType()];
350 if ( $itemsPerSecond <= 0 ) {
358 $seconds = floor( $exactSeconds );
359 $remainder = $exactSeconds - $seconds;
360 $seconds += ( mt_rand() / mt_getrandmax() < $remainder ) ? 1 : 0;
363 return (
int)$seconds;
375 $file =
wfTempDir() .
'/mw-runJobs-backoffs.json';
376 if ( is_file( $file ) ) {
377 $noblock = ( $mode ===
'nowait' ) ? LOCK_NB : 0;
378 $handle = fopen( $file,
'rb' );
379 if ( !flock( $handle, LOCK_SH | $noblock ) ) {
383 $content = stream_get_contents( $handle );
384 flock( $handle, LOCK_UN );
386 $ctime = microtime(
true );
387 $cBackoffs = json_decode(
$content,
true ) ?: [];
390 unset( $cBackoffs[
$type] );
416 $noblock = ( $mode ===
'nowait' ) ? LOCK_NB : 0;
417 $file =
wfTempDir() .
'/mw-runJobs-backoffs.json';
418 $handle = fopen( $file,
'wb+' );
419 if ( !flock( $handle, LOCK_EX | $noblock ) ) {
423 $ctime = microtime(
true );
424 $content = stream_get_contents( $handle );
425 $cBackoffs = json_decode(
$content,
true ) ?: [];
426 foreach ( $deltas
as $type => $seconds ) {
427 $cBackoffs[
$type] = isset( $cBackoffs[
$type] ) && $cBackoffs[
$type] >= $ctime
428 ? $cBackoffs[
$type] + $seconds
433 unset( $cBackoffs[
$type] );
436 ftruncate( $handle, 0 );
437 fwrite( $handle, json_encode( $cBackoffs ) );
438 flock( $handle, LOCK_UN );
452 static $maxBytes = null;
453 if ( $maxBytes === null ) {
455 if ( preg_match(
'!^(\d+)(k|m|g|)$!i', ini_get(
'memory_limit' ), $m ) ) {
456 list( , $num, $unit ) = $m;
457 $conv = [
'g' => 1073741824,
'm' => 1048576,
'k' => 1024,
'' => 1 ];
458 $maxBytes = $num * $conv[strtolower( $unit )];
463 $usedBytes = memory_get_usage();
464 if ( $maxBytes && $usedBytes >= 0.95 * $maxBytes ) {
465 $msg =
"Detected excessive memory usage ($usedBytes/$maxBytes).";
467 $this->logger->error( $msg );
480 if ( $this->debug ) {
498 if ( $wgJobSerialCommitThreshold !==
false && $lb->getServerCount() > 1 ) {
500 $dbwSerial = $lb->getAnyOpenConnection( $lb->getWriterIndex() );
506 || !$dbwSerial->namedLocksEnqueue()
515 $ms = intval( 1000 * $dbwSerial->pendingWriteQueryDuration() );
516 $msg = $job->
toString() .
" COMMIT ENQUEUED [{$ms}ms of writes]";
517 $this->logger->info( $msg );
521 if ( !$dbwSerial->lock(
'jobrunner-serial-commit', __METHOD__, 30 ) ) {
523 throw new DBError( $dbwSerial,
"Timed out waiting on commit queue." );
526 $pos = $lb->getMasterPos();
528 $lb->waitForOne( $pos );
546 $dbwSerial->unlock(
'jobrunner-serial-commit', __METHOD__ );
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
static doUpdates($mode= 'run', $type=self::ALL)
Do any deferred updates and clear the list.
Database error base class.
the array() calling protocol came about after MediaWiki 1.4rc1.
processing should stop and the error should be shown to the user * false
callable null $debug
Debug output handler.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
setLogger(LoggerInterface $logger)
static instance()
Singleton.
teardown()
Do any final cleanup after run(), deferred updates, and all DB commits happen.
query($sql, $fname=__METHOD__, $tempIgnore=false)
Run an SQL query and return the result.
wfGetRusage()
Get system resource usage of current request context.
Class to both describe a background job and handle jobs.
this hook is for auditing only $response
when a variable name is used in a it is silently declared as a new local masking the global
$wgJobClasses
Maps jobs to their handling classes; extensions can add to this to provide custom jobs...
No-op job that does nothing.
forEachOpenConnection($callback, array $params=[])
Call a function with each open connection object.
loadBackoffs(array $backoffs, $mode= 'wait')
Get the previous backoff expiries from persistent storage On I/O or lock acquisition failure this ret...
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Database load balancing object.
wfGetLB($wiki=false)
Get a load balancer object.
Exception class for replica DB wait timeouts.
wfReadOnly()
Check whether the wiki is in read-only mode.
wfTempDir()
Tries to get the system directory for temporary files.
$wgTrxProfilerLimits
Performance expectations for DB usage.
static getMain()
Static methods.
run(array $options)
Run jobs of the specified number/type for the specified time.
static singleton()
Get an instance of this class.
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context $options
writesOrCallbacksPending()
Returns true if there is a transaction open with possible write queries or transaction pre-commit/idl...
static overrideRequestId($id)
Override the unique request ID.
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
const TS_DB
MySQL DATETIME (YYYY-MM-DD HH:MM:SS)
static rollbackMasterChangesAndLog($e)
If there are any open database transactions, roll them back and log the stack trace of the exception ...
static singleton($wiki=false)
float[] $wgJobBackoffThrottling
Map of job types to how many job "work items" should be run per second on each job runner process...
commitMasterChanges(Job $job)
Issue a commit on all masters who are currently in a transaction and have made changes to the databas...
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
wfGetLBFactory()
Get the load balancer factory object.
if(!defined( 'MEDIAWIKI')) $fname
This file is not a valid entry point, perform no further processing unless MEDIAWIKI is defined...
Job queue runner utility methods.
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content $content
getBackoffTimeToWait(Job $job)
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
MediaWiki Logger LoggerFactory implements a PSR[0] compatible message logging system Named Psr Log LoggerInterface instances can be obtained from the MediaWiki Logger LoggerFactory::getInstance() static method.MediaWiki\Logger\LoggerFactory expects a class implementing the MediaWiki\Logger\Spi interface to act as a factory for new Psr\Log\LoggerInterface instances.The"Spi"in MediaWiki\Logger\Spi stands for"service provider interface".An SPI is an API intended to be implemented or extended by a third party.This software design pattern is intended to enable framework extension and replaceable components.It is specifically used in the MediaWiki\Logger\LoggerFactory service to allow alternate PSR-3 logging implementations to be easily integrated with MediaWiki.The service provider interface allows the backend logging library to be implemented in multiple ways.The $wgMWLoggerDefaultSpi global provides the classname of the default MediaWiki\Logger\Spi implementation to be loaded at runtime.This can either be the name of a class implementing the MediaWiki\Logger\Spi with a zero argument const ructor or a callable that will return an MediaWiki\Logger\Spi instance.Alternately the MediaWiki\Logger\LoggerFactory MediaWiki Logger LoggerFactory
checkMemoryOK()
Make sure that this script is not too close to the memory usage limit.
const TS_UNIX
Unix time - the number of seconds since 1970-01-01 00:00:00 UTC.
static logException($e)
Log an exception to the exception log (if enabled).
debugCallback($msg)
Log the job message.
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk my contributions etc etc otherwise the built in rate limiting checks are if enabled allows for interception of redirect as a string mapping parameter names to values & $type
float bool $wgJobSerialCommitThreshold
Make job runners commit changes for slave-lag prone jobs one job at a time.
Basic database interface for live and lazy-loaded DB handles.
syncBackoffDeltas(array $backoffs, array &$deltas, $mode= 'wait')
Merge the current backoff expiries from persistent storage.
executeJob(Job $job, $stats, $popTime)
__construct(LoggerInterface $logger=null)