[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  final class DrydockAllocatorWorker extends PhabricatorWorker {
   4  
   5    private $lease;
   6  
   7    public function getRequiredLeaseTime() {
   8      return 3600 * 24;
   9    }
  10  
  11    public function getMaximumRetryCount() {
  12      // TODO: Allow Drydock allocations to retry. For now, every failure is
  13      // permanent and most of them are because I am bad at programming, so fail
  14      // fast rather than ending up in limbo.
  15      return 0;
  16    }
  17  
  18    private function loadLease() {
  19      if (empty($this->lease)) {
  20        $lease = id(new DrydockLeaseQuery())
  21          ->setViewer(PhabricatorUser::getOmnipotentUser())
  22          ->withIDs(array($this->getTaskData()))
  23          ->executeOne();
  24        if (!$lease) {
  25          throw new PhabricatorWorkerPermanentFailureException(
  26            pht('No such lease %d!', $this->getTaskData()));
  27        }
  28        $this->lease = $lease;
  29      }
  30      return $this->lease;
  31    }
  32  
  33    private function logToDrydock($message) {
  34      DrydockBlueprintImplementation::writeLog(
  35        null,
  36        $this->loadLease(),
  37        $message);
  38    }
  39  
  40    protected function doWork() {
  41      $lease = $this->loadLease();
  42      $this->logToDrydock('Allocating Lease');
  43  
  44      try {
  45        $this->allocateLease($lease);
  46      } catch (Exception $ex) {
  47  
  48        // TODO: We should really do this when archiving the task, if we've
  49        // suffered a permanent failure. But we don't have hooks for that yet
  50        // and always fail after the first retry right now, so this is
  51        // functionally equivalent.
  52        $lease->reload();
  53        if ($lease->getStatus() == DrydockLeaseStatus::STATUS_PENDING) {
  54          $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
  55          $lease->save();
  56        }
  57  
  58        throw $ex;
  59      }
  60    }
  61  
  62    private function loadAllBlueprints() {
  63      $viewer = PhabricatorUser::getOmnipotentUser();
  64      $instances = id(new DrydockBlueprintQuery())
  65        ->setViewer($viewer)
  66        ->execute();
  67      $blueprints = array();
  68      foreach ($instances as $instance) {
  69        $blueprints[$instance->getPHID()] = $instance;
  70      }
  71      return $blueprints;
  72    }
  73  
  74    private function allocateLease(DrydockLease $lease) {
  75      $type = $lease->getResourceType();
  76  
  77      $blueprints = $this->loadAllBlueprints();
  78  
  79      // TODO: Policy stuff.
  80      $pool = id(new DrydockResource())->loadAllWhere(
  81        'type = %s AND status = %s',
  82        $lease->getResourceType(),
  83        DrydockResourceStatus::STATUS_OPEN);
  84  
  85      $this->logToDrydock(
  86        pht('Found %d Open Resource(s)', count($pool)));
  87  
  88      $candidates = array();
  89      foreach ($pool as $key => $candidate) {
  90        if (!isset($blueprints[$candidate->getBlueprintPHID()])) {
  91          unset($pool[$key]);
  92          continue;
  93        }
  94  
  95        $blueprint = $blueprints[$candidate->getBlueprintPHID()];
  96        $implementation = $blueprint->getImplementation();
  97  
  98        if ($implementation->filterResource($candidate, $lease)) {
  99          $candidates[] = $candidate;
 100        }
 101      }
 102  
 103      $this->logToDrydock(pht('%d Open Resource(s) Remain', count($candidates)));
 104  
 105      $resource = null;
 106      if ($candidates) {
 107        shuffle($candidates);
 108        foreach ($candidates as $candidate_resource) {
 109          $blueprint = $blueprints[$candidate_resource->getBlueprintPHID()]
 110            ->getImplementation();
 111          if ($blueprint->allocateLease($candidate_resource, $lease)) {
 112            $resource = $candidate_resource;
 113            break;
 114          }
 115        }
 116      }
 117  
 118      if (!$resource) {
 119        $blueprints = DrydockBlueprintImplementation
 120          ::getAllBlueprintImplementationsForResource($type);
 121  
 122        $this->logToDrydock(
 123          pht('Found %d Blueprints', count($blueprints)));
 124  
 125        foreach ($blueprints as $key => $candidate_blueprint) {
 126          if (!$candidate_blueprint->isEnabled()) {
 127            unset($blueprints[$key]);
 128            continue;
 129          }
 130        }
 131  
 132        $this->logToDrydock(
 133          pht('%d Blueprints Enabled', count($blueprints)));
 134  
 135        foreach ($blueprints as $key => $candidate_blueprint) {
 136          if (!$candidate_blueprint->canAllocateMoreResources($pool)) {
 137            unset($blueprints[$key]);
 138            continue;
 139          }
 140        }
 141  
 142        $this->logToDrydock(
 143          pht('%d Blueprints Can Allocate', count($blueprints)));
 144  
 145        if (!$blueprints) {
 146          $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
 147          $lease->save();
 148  
 149          $this->logToDrydock(
 150            "There are no resources of type '{$type}' available, and no ".
 151            "blueprints which can allocate new ones.");
 152  
 153          return;
 154        }
 155  
 156        // TODO: Rank intelligently.
 157        shuffle($blueprints);
 158  
 159        $blueprint = head($blueprints);
 160        $resource = $blueprint->allocateResource($lease);
 161  
 162        if (!$blueprint->allocateLease($resource, $lease)) {
 163          // TODO: This "should" happen only if we lost a race with another lease,
 164          // which happened to acquire this resource immediately after we
 165          // allocated it. In this case, the right behavior is to retry
 166          // immediately. However, other things like a blueprint allocating a
 167          // resource it can't actually allocate the lease on might be happening
 168          // too, in which case we'd just allocate infinite resources. Probably
 169          // what we should do is test for an active or allocated lease and retry
 170          // if we find one (although it might have already been released by now)
 171          // and fail really hard ("your configuration is a huge broken mess")
 172          // otherwise. But just throw for now since this stuff is all edge-casey.
 173          // Alternatively we could bring resources up in a "BESPOKE" status
 174          // and then switch them to "OPEN" only after the allocating lease gets
 175          // its grubby mitts on the resource. This might make more sense but
 176          // is a bit messy.
 177          throw new Exception('Lost an allocation race?');
 178        }
 179      }
 180  
 181      $blueprint = $resource->getBlueprint();
 182      $blueprint->acquireLease($resource, $lease);
 183    }
 184  
 185  }