[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  /**
   4   * Moves a build forward by queuing build tasks, canceling or restarting the
   5   * build, or failing it in response to task failures.
   6   */
   7  final class HarbormasterBuildEngine extends Phobject {
   8  
   9    private $build;
  10    private $viewer;
  11    private $newBuildTargets = array();
  12    private $forceBuildableUpdate;
  13  
  14    public function setForceBuildableUpdate($force_buildable_update) {
  15      $this->forceBuildableUpdate = $force_buildable_update;
  16      return $this;
  17    }
  18  
  19    public function shouldForceBuildableUpdate() {
  20      return $this->forceBuildableUpdate;
  21    }
  22  
  23    public function queueNewBuildTarget(HarbormasterBuildTarget $target) {
  24      $this->newBuildTargets[] = $target;
  25      return $this;
  26    }
  27  
  28    public function getNewBuildTargets() {
  29      return $this->newBuildTargets;
  30    }
  31  
  32    public function setViewer(PhabricatorUser $viewer) {
  33      $this->viewer = $viewer;
  34      return $this;
  35    }
  36  
  37    public function getViewer() {
  38      return $this->viewer;
  39    }
  40  
  41    public function setBuild(HarbormasterBuild $build) {
  42      $this->build = $build;
  43      return $this;
  44    }
  45  
  46    public function getBuild() {
  47      return $this->build;
  48    }
  49  
  50    public function continueBuild() {
  51      $build = $this->getBuild();
  52  
  53      $lock_key = 'harbormaster.build:'.$build->getID();
  54      $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
  55  
  56      $build->reload();
  57      $old_status = $build->getBuildStatus();
  58  
  59      try {
  60        $this->updateBuild($build);
  61      } catch (Exception $ex) {
  62        // If any exception is raised, the build is marked as a failure and the
  63        // exception is re-thrown (this ensures we don't leave builds in an
  64        // inconsistent state).
  65        $build->setBuildStatus(HarbormasterBuild::STATUS_ERROR);
  66        $build->save();
  67  
  68        $lock->unlock();
  69  
  70        $this->releaseAllArtifacts($build);
  71  
  72        throw $ex;
  73      }
  74  
  75      $lock->unlock();
  76  
  77      // NOTE: We queue new targets after releasing the lock so that in-process
  78      // execution via `bin/harbormaster` does not reenter the locked region.
  79      foreach ($this->getNewBuildTargets() as $target) {
  80        $task = PhabricatorWorker::scheduleTask(
  81          'HarbormasterTargetWorker',
  82          array(
  83            'targetID' => $target->getID(),
  84          ));
  85      }
  86  
  87      // If the build changed status, we might need to update the overall status
  88      // on the buildable.
  89      $new_status = $build->getBuildStatus();
  90      if ($new_status != $old_status || $this->shouldForceBuildableUpdate()) {
  91        $this->updateBuildable($build->getBuildable());
  92      }
  93  
  94      // If we are no longer building for any reason, release all artifacts.
  95      if (!$build->isBuilding()) {
  96        $this->releaseAllArtifacts($build);
  97      }
  98    }
  99  
 100    private function updateBuild(HarbormasterBuild $build) {
 101      if (($build->getBuildStatus() == HarbormasterBuild::STATUS_PENDING) ||
 102          ($build->isRestarting())) {
 103        $this->restartBuild($build);
 104        $build->setBuildStatus(HarbormasterBuild::STATUS_BUILDING);
 105        $build->save();
 106      }
 107  
 108      if ($build->isResuming()) {
 109        $build->setBuildStatus(HarbormasterBuild::STATUS_BUILDING);
 110        $build->save();
 111      }
 112  
 113      if ($build->isStopping() && !$build->isComplete()) {
 114        $build->setBuildStatus(HarbormasterBuild::STATUS_STOPPED);
 115        $build->save();
 116      }
 117  
 118      $build->deleteUnprocessedCommands();
 119  
 120      if ($build->getBuildStatus() == HarbormasterBuild::STATUS_BUILDING) {
 121        $this->updateBuildSteps($build);
 122      }
 123    }
 124  
 125    private function restartBuild(HarbormasterBuild $build) {
 126  
 127      // We're restarting the build, so release all previous artifacts.
 128      $this->releaseAllArtifacts($build);
 129  
 130      // Increment the build generation counter on the build.
 131      $build->setBuildGeneration($build->getBuildGeneration() + 1);
 132  
 133      // Currently running targets should periodically check their build
 134      // generation (which won't have changed) against the build's generation.
 135      // If it is different, they will automatically stop what they're doing
 136      // and abort.
 137  
 138      // Previously we used to delete targets, logs and artifacts here.  Instead
 139      // leave them around so users can view previous generations of this build.
 140    }
 141  
 142    private function updateBuildSteps(HarbormasterBuild $build) {
 143      $targets = id(new HarbormasterBuildTargetQuery())
 144        ->setViewer($this->getViewer())
 145        ->withBuildPHIDs(array($build->getPHID()))
 146        ->withBuildGenerations(array($build->getBuildGeneration()))
 147        ->execute();
 148  
 149      $this->updateWaitingTargets($targets);
 150  
 151      $targets = mgroup($targets, 'getBuildStepPHID');
 152  
 153      $steps = id(new HarbormasterBuildStepQuery())
 154        ->setViewer($this->getViewer())
 155        ->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID()))
 156        ->execute();
 157  
 158      // Identify steps which are in various states.
 159  
 160      $queued = array();
 161      $underway = array();
 162      $waiting = array();
 163      $complete = array();
 164      $failed = array();
 165      foreach ($steps as $step) {
 166        $step_targets = idx($targets, $step->getPHID(), array());
 167  
 168        if ($step_targets) {
 169          $is_queued = false;
 170  
 171          $is_underway = false;
 172          foreach ($step_targets as $target) {
 173            if ($target->isUnderway()) {
 174              $is_underway = true;
 175              break;
 176            }
 177          }
 178  
 179          $is_waiting = false;
 180          foreach ($step_targets as $target) {
 181            if ($target->isWaiting()) {
 182              $is_waiting = true;
 183              break;
 184            }
 185          }
 186  
 187          $is_complete = true;
 188          foreach ($step_targets as $target) {
 189            if (!$target->isComplete()) {
 190              $is_complete = false;
 191              break;
 192            }
 193          }
 194  
 195          $is_failed = false;
 196          foreach ($step_targets as $target) {
 197            if ($target->isFailed()) {
 198              $is_failed = true;
 199              break;
 200            }
 201          }
 202        } else {
 203          $is_queued = true;
 204          $is_underway = false;
 205          $is_waiting = false;
 206          $is_complete = false;
 207          $is_failed = false;
 208        }
 209  
 210        if ($is_queued) {
 211          $queued[$step->getPHID()] = true;
 212        }
 213  
 214        if ($is_underway) {
 215          $underway[$step->getPHID()] = true;
 216        }
 217  
 218        if ($is_waiting) {
 219          $waiting[$step->getPHID()] = true;
 220        }
 221  
 222        if ($is_complete) {
 223          $complete[$step->getPHID()] = true;
 224        }
 225  
 226        if ($is_failed) {
 227          $failed[$step->getPHID()] = true;
 228        }
 229      }
 230  
 231      // If any step failed, fail the whole build, then bail.
 232      if (count($failed)) {
 233        $build->setBuildStatus(HarbormasterBuild::STATUS_FAILED);
 234        $build->save();
 235        return;
 236      }
 237  
 238      // If every step is complete, we're done with this build. Mark it passed
 239      // and bail.
 240      if (count($complete) == count($steps)) {
 241        $build->setBuildStatus(HarbormasterBuild::STATUS_PASSED);
 242        $build->save();
 243        return;
 244      }
 245  
 246      // Identify all the steps which are ready to run (because all their
 247      // dependencies are complete).
 248  
 249      $runnable = array();
 250      foreach ($steps as $step) {
 251        $dependencies = $step->getStepImplementation()->getDependencies($step);
 252  
 253        if (isset($queued[$step->getPHID()])) {
 254          $can_run = true;
 255          foreach ($dependencies as $dependency) {
 256            if (empty($complete[$dependency])) {
 257              $can_run = false;
 258              break;
 259            }
 260          }
 261  
 262          if ($can_run) {
 263            $runnable[] = $step;
 264          }
 265        }
 266      }
 267  
 268      if (!$runnable && !$waiting && !$underway) {
 269        // This means the build is deadlocked, and the user has configured
 270        // circular dependencies.
 271        $build->setBuildStatus(HarbormasterBuild::STATUS_DEADLOCKED);
 272        $build->save();
 273        return;
 274      }
 275  
 276      foreach ($runnable as $runnable_step) {
 277        $target = HarbormasterBuildTarget::initializeNewBuildTarget(
 278          $build,
 279          $runnable_step,
 280          $build->retrieveVariablesFromBuild());
 281        $target->save();
 282  
 283        $this->queueNewBuildTarget($target);
 284      }
 285    }
 286  
 287  
 288    /**
 289     * Process messages which were sent to these targets, kicking applicable
 290     * targets out of "Waiting" and into either "Passed" or "Failed".
 291     *
 292     * @param list<HarbormasterBuildTarget> List of targets to process.
 293     * @return void
 294     */
 295    private function updateWaitingTargets(array $targets) {
 296      assert_instances_of($targets, 'HarbormasterBuildTarget');
 297  
 298      // We only care about messages for targets which are actually in a waiting
 299      // state.
 300      $waiting_targets = array();
 301      foreach ($targets as $target) {
 302        if ($target->isWaiting()) {
 303          $waiting_targets[$target->getPHID()] = $target;
 304        }
 305      }
 306  
 307      if (!$waiting_targets) {
 308        return;
 309      }
 310  
 311      $messages = id(new HarbormasterBuildMessageQuery())
 312        ->setViewer($this->getViewer())
 313        ->withBuildTargetPHIDs(array_keys($waiting_targets))
 314        ->withConsumed(false)
 315        ->execute();
 316  
 317      foreach ($messages as $message) {
 318        $target = $waiting_targets[$message->getBuildTargetPHID()];
 319  
 320        $new_status = null;
 321        switch ($message->getType()) {
 322          case 'pass':
 323            $new_status = HarbormasterBuildTarget::STATUS_PASSED;
 324            break;
 325          case 'fail':
 326            $new_status = HarbormasterBuildTarget::STATUS_FAILED;
 327            break;
 328        }
 329  
 330        if ($new_status !== null) {
 331          $message->setIsConsumed(true);
 332          $message->save();
 333  
 334          $target->setTargetStatus($new_status);
 335          $target->save();
 336        }
 337      }
 338    }
 339  
 340  
 341    /**
 342     * Update the overall status of the buildable this build is attached to.
 343     *
 344     * After a build changes state (for example, passes or fails) it may affect
 345     * the overall state of the associated buildable. Compute the new aggregate
 346     * state and save it on the buildable.
 347     *
 348     * @param   HarbormasterBuild The buildable to update.
 349     * @return  void
 350     */
 351    private function updateBuildable(HarbormasterBuildable $buildable) {
 352      $viewer = $this->getViewer();
 353  
 354      $lock_key = 'harbormaster.buildable:'.$buildable->getID();
 355      $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
 356  
 357      $buildable = id(new HarbormasterBuildableQuery())
 358        ->setViewer($viewer)
 359        ->withIDs(array($buildable->getID()))
 360        ->needBuilds(true)
 361        ->executeOne();
 362  
 363      $all_pass = true;
 364      $any_fail = false;
 365      foreach ($buildable->getBuilds() as $build) {
 366        if ($build->getBuildStatus() != HarbormasterBuild::STATUS_PASSED) {
 367          $all_pass = false;
 368        }
 369        if ($build->getBuildStatus() == HarbormasterBuild::STATUS_FAILED ||
 370            $build->getBuildStatus() == HarbormasterBuild::STATUS_ERROR ||
 371            $build->getBuildStatus() == HarbormasterBuild::STATUS_DEADLOCKED) {
 372          $any_fail = true;
 373        }
 374      }
 375  
 376      if ($any_fail) {
 377        $new_status = HarbormasterBuildable::STATUS_FAILED;
 378      } else if ($all_pass) {
 379        $new_status = HarbormasterBuildable::STATUS_PASSED;
 380      } else {
 381        $new_status = HarbormasterBuildable::STATUS_BUILDING;
 382      }
 383  
 384      $old_status = $buildable->getBuildableStatus();
 385      $did_update = ($old_status != $new_status);
 386      if ($did_update) {
 387        $buildable->setBuildableStatus($new_status);
 388        $buildable->save();
 389      }
 390  
 391      $lock->unlock();
 392  
 393      // If we changed the buildable status, try to post a transaction to the
 394      // object about it. We can safely do this outside of the locked region.
 395  
 396      // NOTE: We only post transactions for automatic buildables, not for
 397      // manual ones: manual builds are test builds, whoever is doing tests
 398      // can look at the results themselves, and other users generally don't
 399      // care about the outcome.
 400  
 401      $should_publish = $did_update &&
 402                        $new_status != HarbormasterBuildable::STATUS_BUILDING &&
 403                        !$buildable->getIsManualBuildable();
 404      if ($should_publish) {
 405        $object = id(new PhabricatorObjectQuery())
 406          ->setViewer($viewer)
 407          ->withPHIDs(array($buildable->getBuildablePHID()))
 408          ->executeOne();
 409  
 410        if ($object instanceof PhabricatorApplicationTransactionInterface) {
 411          $template = $object->getApplicationTransactionTemplate();
 412          if ($template) {
 413            $template
 414              ->setTransactionType(PhabricatorTransactions::TYPE_BUILDABLE)
 415              ->setMetadataValue(
 416                'harbormaster:buildablePHID',
 417                $buildable->getPHID())
 418              ->setOldValue($old_status)
 419              ->setNewValue($new_status);
 420  
 421            $harbormaster_phid = id(new PhabricatorHarbormasterApplication())
 422              ->getPHID();
 423  
 424            $daemon_source = PhabricatorContentSource::newForSource(
 425              PhabricatorContentSource::SOURCE_DAEMON,
 426              array());
 427  
 428            $editor = $object->getApplicationTransactionEditor()
 429              ->setActor($viewer)
 430              ->setActingAsPHID($harbormaster_phid)
 431              ->setContentSource($daemon_source)
 432              ->setContinueOnNoEffect(true)
 433              ->setContinueOnMissingFields(true);
 434  
 435            $editor->applyTransactions(
 436              $object->getApplicationTransactionObject(),
 437              array($template));
 438          }
 439        }
 440      }
 441    }
 442  
 443    private function releaseAllArtifacts(HarbormasterBuild $build) {
 444      $targets = id(new HarbormasterBuildTargetQuery())
 445        ->setViewer(PhabricatorUser::getOmnipotentUser())
 446        ->withBuildPHIDs(array($build->getPHID()))
 447        ->withBuildGenerations(array($build->getBuildGeneration()))
 448        ->execute();
 449  
 450      if (count($targets) === 0) {
 451        return;
 452      }
 453  
 454      $target_phids = mpull($targets, 'getPHID');
 455  
 456      $artifacts = id(new HarbormasterBuildArtifactQuery())
 457        ->setViewer(PhabricatorUser::getOmnipotentUser())
 458        ->withBuildTargetPHIDs($target_phids)
 459        ->execute();
 460  
 461      foreach ($artifacts as $artifact) {
 462        $artifact->release();
 463      }
 464  
 465    }
 466  
 467  }