[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  abstract class PhabricatorDaemonManagementWorkflow
   4    extends PhabricatorManagementWorkflow {
   5  
   6    protected final function loadAvailableDaemonClasses() {
   7      $loader = new PhutilSymbolLoader();
   8      return $loader
   9        ->setAncestorClass('PhutilDaemon')
  10        ->setConcreteOnly(true)
  11        ->selectSymbolsWithoutLoading();
  12    }
  13  
  14    protected final function getPIDDirectory() {
  15      $path = PhabricatorEnv::getEnvConfig('phd.pid-directory');
  16      return $this->getControlDirectory($path);
  17    }
  18  
  19    protected final function getLogDirectory() {
  20      $path = PhabricatorEnv::getEnvConfig('phd.log-directory');
  21      return $this->getControlDirectory($path);
  22    }
  23  
  24    private function getControlDirectory($path) {
  25      if (!Filesystem::pathExists($path)) {
  26        list($err) = exec_manual('mkdir -p %s', $path);
  27        if ($err) {
  28          throw new Exception(
  29            "phd requires the directory '{$path}' to exist, but it does not ".
  30            "exist and could not be created. Create this directory or update ".
  31            "'phd.pid-directory' / 'phd.log-directory' in your configuration ".
  32            "to point to an existing directory.");
  33        }
  34      }
  35      return $path;
  36    }
  37  
  38    protected final function loadRunningDaemons() {
  39      $daemons = array();
  40  
  41      $pid_dir = $this->getPIDDirectory();
  42      $pid_files = Filesystem::listDirectory($pid_dir);
  43  
  44      foreach ($pid_files as $pid_file) {
  45        $daemons[] = PhabricatorDaemonReference::newFromFile(
  46          $pid_dir.'/'.$pid_file);
  47      }
  48  
  49      return $daemons;
  50    }
  51  
  52    protected final function loadAllRunningDaemons() {
  53      $local_daemons = $this->loadRunningDaemons();
  54  
  55      $local_ids = array();
  56      foreach ($local_daemons as $daemon) {
  57        $daemon_log = $daemon->getDaemonLog();
  58  
  59        if ($daemon_log) {
  60          $local_ids[] = $daemon_log->getID();
  61        }
  62      }
  63  
  64      $remote_daemons = id(new PhabricatorDaemonLogQuery())
  65        ->setViewer(PhabricatorUser::getOmnipotentUser())
  66        ->withoutIDs($local_ids)
  67        ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)
  68        ->execute();
  69  
  70      return array_merge($local_daemons, $remote_daemons);
  71    }
  72  
  73    private function findDaemonClass($substring) {
  74      $symbols = $this->loadAvailableDaemonClasses();
  75  
  76      $symbols = ipull($symbols, 'name');
  77      $match = array();
  78      foreach ($symbols as $symbol) {
  79        if (stripos($symbol, $substring) !== false) {
  80          if (strtolower($symbol) == strtolower($substring)) {
  81            $match = array($symbol);
  82            break;
  83          } else {
  84            $match[] = $symbol;
  85          }
  86        }
  87      }
  88  
  89      if (count($match) == 0) {
  90        throw new PhutilArgumentUsageException(
  91          pht(
  92            "No daemons match '%s'! Use 'phd list' for a list of available ".
  93            "daemons.",
  94            $substring));
  95      } else if (count($match) > 1) {
  96        throw new PhutilArgumentUsageException(
  97          pht(
  98            "Specify a daemon unambiguously. Multiple daemons match '%s': %s.",
  99            $substring,
 100            implode(', ', $match)));
 101      }
 102  
 103      return head($match);
 104    }
 105  
 106    protected final function launchDaemon($class, array $argv, $debug) {
 107      $daemon = $this->findDaemonClass($class);
 108      $console = PhutilConsole::getConsole();
 109  
 110      if ($debug) {
 111        if ($argv) {
 112          $console->writeOut(
 113            pht(
 114              "Launching daemon \"%s\" in debug mode (not daemonized) ".
 115              "with arguments %s.\n",
 116              $daemon,
 117              csprintf('%LR', $argv)));
 118        } else {
 119          $console->writeOut(
 120            pht(
 121              "Launching daemon \"%s\" in debug mode (not daemonized).\n",
 122              $daemon));
 123        }
 124      } else {
 125        if ($argv) {
 126          $console->writeOut(
 127            pht(
 128              "Launching daemon \"%s\" with arguments %s.\n",
 129              $daemon,
 130              csprintf('%LR', $argv)));
 131        } else {
 132          $console->writeOut(
 133            pht(
 134              "Launching daemon \"%s\".\n",
 135              $daemon));
 136        }
 137      }
 138  
 139      foreach ($argv as $key => $arg) {
 140        $argv[$key] = escapeshellarg($arg);
 141      }
 142  
 143      $flags = array();
 144      if ($debug || PhabricatorEnv::getEnvConfig('phd.trace')) {
 145        $flags[] = '--trace';
 146      }
 147  
 148      if ($debug || PhabricatorEnv::getEnvConfig('phd.verbose')) {
 149        $flags[] = '--verbose';
 150      }
 151  
 152      if (!$debug) {
 153        $flags[] = '--daemonize';
 154      }
 155  
 156      if (!$debug) {
 157        $log_file = $this->getLogDirectory().'/daemons.log';
 158        $flags[] = csprintf('--log=%s', $log_file);
 159      }
 160  
 161      $pid_dir = $this->getPIDDirectory();
 162  
 163      // TODO: This should be a much better user experience.
 164      Filesystem::assertExists($pid_dir);
 165      Filesystem::assertIsDirectory($pid_dir);
 166      Filesystem::assertWritable($pid_dir);
 167  
 168      $flags[] = csprintf('--phd=%s', $pid_dir);
 169  
 170      $command = csprintf(
 171        './phd-daemon %s %C %C',
 172        $daemon,
 173        implode(' ', $flags),
 174        implode(' ', $argv));
 175  
 176      $phabricator_root = dirname(phutil_get_library_root('phabricator'));
 177      $daemon_script_dir = $phabricator_root.'/scripts/daemon/';
 178  
 179      if ($debug) {
 180        // Don't terminate when the user sends ^C; it will be sent to the
 181        // subprocess which will terminate normally.
 182        pcntl_signal(
 183          SIGINT,
 184          array(__CLASS__, 'ignoreSignal'));
 185  
 186        echo "\n    phabricator/scripts/daemon/ \$ {$command}\n\n";
 187  
 188        phutil_passthru('(cd %s && exec %C)', $daemon_script_dir, $command);
 189      } else {
 190        $future = new ExecFuture('exec %C', $command);
 191        // Play games to keep 'ps' looking reasonable.
 192        $future->setCWD($daemon_script_dir);
 193        $future->resolvex();
 194      }
 195    }
 196  
 197    public static function ignoreSignal($signo) {
 198      return;
 199    }
 200  
 201    public static function requireExtensions() {
 202      self::mustHaveExtension('pcntl');
 203      self::mustHaveExtension('posix');
 204    }
 205  
 206    private static function mustHaveExtension($ext) {
 207      if (!extension_loaded($ext)) {
 208        echo "ERROR: The PHP extension '{$ext}' is not installed. You must ".
 209             "install it to run daemons on this machine.\n";
 210        exit(1);
 211      }
 212  
 213      $extension = new ReflectionExtension($ext);
 214      foreach ($extension->getFunctions() as $function) {
 215        $function = $function->name;
 216        if (!function_exists($function)) {
 217          echo "ERROR: The PHP function {$function}() is disabled. You must ".
 218               "enable it to run daemons on this machine.\n";
 219          exit(1);
 220        }
 221      }
 222    }
 223  
 224    protected final function willLaunchDaemons() {
 225      $console = PhutilConsole::getConsole();
 226      $console->writeErr(pht('Preparing to launch daemons.')."\n");
 227  
 228      $log_dir = $this->getLogDirectory().'/daemons.log';
 229      $console->writeErr(pht("NOTE: Logs will appear in '%s'.", $log_dir)."\n\n");
 230    }
 231  
 232  
 233  /* -(  Commands  )----------------------------------------------------------- */
 234  
 235  
 236    protected final function executeStartCommand($keep_leases = false) {
 237      $console = PhutilConsole::getConsole();
 238  
 239      $running = $this->loadRunningDaemons();
 240  
 241      // This may include daemons which were launched but which are no longer
 242      // running; check that we actually have active daemons before failing.
 243      foreach ($running as $daemon) {
 244        if ($daemon->isRunning()) {
 245          $message = pht(
 246            "phd start: Unable to start daemons because daemons are already ".
 247            "running.\n".
 248            "You can view running daemons with 'phd status'.\n".
 249            "You can stop running daemons with 'phd stop'.\n".
 250            "You can use 'phd restart' to stop all daemons before starting new ".
 251            "daemons.");
 252  
 253          $console->writeErr("%s\n", $message);
 254          exit(1);
 255        }
 256      }
 257  
 258      if ($keep_leases) {
 259        $console->writeErr("%s\n", pht('Not touching active task queue leases.'));
 260      } else {
 261        $console->writeErr("%s\n", pht('Freeing active task leases...'));
 262        $count = $this->freeActiveLeases();
 263        $console->writeErr(
 264          "%s\n",
 265          pht('Freed %s task lease(s).', new PhutilNumber($count)));
 266      }
 267  
 268      $daemons = array(
 269        array('PhabricatorRepositoryPullLocalDaemon', array()),
 270        array('PhabricatorGarbageCollectorDaemon', array()),
 271      );
 272  
 273      $taskmasters = PhabricatorEnv::getEnvConfig('phd.start-taskmasters');
 274      for ($ii = 0; $ii < $taskmasters; $ii++) {
 275        $daemons[] = array('PhabricatorTaskmasterDaemon', array());
 276      }
 277  
 278      $this->willLaunchDaemons();
 279  
 280      foreach ($daemons as $spec) {
 281        list($name, $argv) = $spec;
 282        $this->launchDaemon($name, $argv, $is_debug = false);
 283      }
 284  
 285      $console->writeErr(pht('Done.')."\n");
 286      return 0;
 287    }
 288  
 289    protected final function executeStopCommand(
 290      array $pids,
 291      $grace_period,
 292      $force) {
 293  
 294      $console = PhutilConsole::getConsole();
 295  
 296      $daemons = $this->loadRunningDaemons();
 297      if (!$daemons) {
 298        $survivors = array();
 299        if (!$pids) {
 300          $survivors = $this->processRogueDaemons(
 301            $grace_period,
 302            $warn = true,
 303            $force);
 304        }
 305        if (!$survivors) {
 306          $console->writeErr(pht(
 307            'There are no running Phabricator daemons.')."\n");
 308        }
 309        return 0;
 310      }
 311  
 312      $daemons = mpull($daemons, null, 'getPID');
 313  
 314      $running = array();
 315      if (!$pids) {
 316        $running = $daemons;
 317      } else {
 318        // We were given a PID or set of PIDs to kill.
 319        foreach ($pids as $key => $pid) {
 320          if (!preg_match('/^\d+$/', $pid)) {
 321            $console->writeErr(pht("PID '%s' is not a valid PID.", $pid)."\n");
 322            continue;
 323          } else if (empty($daemons[$pid])) {
 324            $console->writeErr(
 325              pht(
 326                "PID '%s' is not a Phabricator daemon PID. It will not ".
 327                "be killed.",
 328                $pid)."\n");
 329            continue;
 330          } else {
 331            $running[] = $daemons[$pid];
 332          }
 333        }
 334      }
 335  
 336      if (empty($running)) {
 337        $console->writeErr(pht('No daemons to kill.')."\n");
 338        return 0;
 339      }
 340  
 341      $all_daemons = $running;
 342      // don't specify force here as that's about rogue daemons
 343      $this->sendStopSignals($running, $grace_period);
 344  
 345      foreach ($all_daemons as $daemon) {
 346        if ($daemon->getPIDFile()) {
 347          Filesystem::remove($daemon->getPIDFile());
 348        }
 349      }
 350  
 351      $this->processRogueDaemons($grace_period, !$pids, $force);
 352  
 353      return 0;
 354    }
 355  
 356    private function processRogueDaemons($grace_period, $warn, $force_stop) {
 357      $console = PhutilConsole::getConsole();
 358  
 359      $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons();
 360      if ($rogue_daemons) {
 361        if ($force_stop) {
 362          $stop_rogue_daemons = $this->buildRogueDaemons($rogue_daemons);
 363          $survivors = $this->sendStopSignals(
 364            $stop_rogue_daemons,
 365            $grace_period,
 366            $force_stop);
 367          if ($survivors) {
 368            $console->writeErr(pht(
 369              'Unable to stop processes running without pid files. Try running '.
 370              'this command again with sudo.'."\n"));
 371          }
 372        } else if ($warn) {
 373          $console->writeErr($this->getForceStopHint($rogue_daemons)."\n");
 374        }
 375      }
 376      return $rogue_daemons;
 377    }
 378  
 379    private function getForceStopHint($rogue_daemons) {
 380      $debug_output = '';
 381      foreach ($rogue_daemons as $rogue) {
 382        $debug_output .= $rogue['pid'].' '.$rogue['command']."\n";
 383      }
 384      return pht(
 385        'There are processes running that look like Phabricator daemons but '.
 386        'have no corresponding PID files:'."\n\n".'%s'."\n\n".
 387        'Stop these processes by re-running this command with the --force '.
 388        'parameter.',
 389        $debug_output);
 390    }
 391  
 392    private function buildRogueDaemons(array $daemons) {
 393      $rogue_daemons = array();
 394      foreach ($daemons as $pid => $data) {
 395        $rogue_daemons[] =
 396          PhabricatorDaemonReference::newFromRogueDictionary($data);
 397      }
 398      return $rogue_daemons;
 399    }
 400  
 401    private function sendStopSignals($daemons, $grace_period, $force = false) {
 402      // If we're doing a graceful shutdown, try SIGINT first.
 403      if ($grace_period) {
 404        $daemons = $this->sendSignal($daemons, SIGINT, $grace_period, $force);
 405      }
 406  
 407      // If we still have daemons, SIGTERM them.
 408      if ($daemons) {
 409        $daemons = $this->sendSignal($daemons, SIGTERM, 15, $force);
 410      }
 411  
 412      // If the overseer is still alive, SIGKILL it.
 413      if ($daemons) {
 414        $daemons = $this->sendSignal($daemons, SIGKILL, 0, $force);
 415      }
 416      return $daemons;
 417    }
 418  
 419    private function sendSignal(array $daemons, $signo, $wait, $force = false) {
 420      $console = PhutilConsole::getConsole();
 421  
 422      foreach ($daemons as $key => $daemon) {
 423        $pid = $daemon->getPID();
 424        $name = $daemon->getName();
 425  
 426        if (!$pid && !$force) {
 427          $console->writeOut("%s\n", pht("Daemon '%s' has no PID!", $name));
 428          unset($daemons[$key]);
 429          continue;
 430        }
 431  
 432        switch ($signo) {
 433          case SIGINT:
 434            $message = pht("Interrupting daemon '%s' (%s)...", $name, $pid);
 435            break;
 436          case SIGTERM:
 437            $message = pht("Terminating daemon '%s' (%s)...", $name, $pid);
 438            break;
 439          case SIGKILL:
 440            $message = pht("Killing daemon '%s' (%s)...", $name, $pid);
 441            break;
 442        }
 443  
 444        $console->writeOut("%s\n", $message);
 445        posix_kill($pid, $signo);
 446      }
 447  
 448      if ($wait) {
 449        $start = PhabricatorTime::getNow();
 450        do {
 451          foreach ($daemons as $key => $daemon) {
 452            $pid = $daemon->getPID();
 453            if (!$daemon->isRunning()) {
 454              $console->writeOut(pht('Daemon %s exited.', $pid)."\n");
 455              unset($daemons[$key]);
 456            }
 457          }
 458          if (empty($daemons)) {
 459            break;
 460          }
 461          usleep(100000);
 462        } while (PhabricatorTime::getNow() < $start + $wait);
 463      }
 464  
 465      return $daemons;
 466    }
 467  
 468    private function freeActiveLeases() {
 469      $task_table = id(new PhabricatorWorkerActiveTask());
 470      $conn_w = $task_table->establishConnection('w');
 471      queryfx(
 472        $conn_w,
 473        'UPDATE %T SET leaseExpires = UNIX_TIMESTAMP()
 474          WHERE leaseExpires > UNIX_TIMESTAMP()',
 475        $task_table->getTableName());
 476      return $conn_w->getAffectedRows();
 477    }
 478  
 479  }