[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  /**
   4   * Manages markup engine selection, configuration, application, caching and
   5   * pipelining.
   6   *
   7   * @{class:PhabricatorMarkupEngine} can be used to render objects which
   8   * implement @{interface:PhabricatorMarkupInterface} in a batched, cache-aware
   9   * way. For example, if you have a list of comments written in remarkup (and
  10   * the objects implement the correct interface) you can render them by first
  11   * building an engine and adding the fields with @{method:addObject}.
  12   *
  13   *   $field  = 'field:body'; // Field you want to render. Each object exposes
  14   *                           // one or more fields of markup.
  15   *
  16   *   $engine = new PhabricatorMarkupEngine();
  17   *   foreach ($comments as $comment) {
  18   *     $engine->addObject($comment, $field);
  19   *   }
  20   *
  21   * Now, call @{method:process} to perform the actual cache/rendering
  22   * step. This is a heavyweight call which does batched data access and
  23   * transforms the markup into output.
  24   *
  25   *   $engine->process();
  26   *
  27   * Finally, do something with the results:
  28   *
  29   *   $results = array();
  30   *   foreach ($comments as $comment) {
  31   *     $results[] = $engine->getOutput($comment, $field);
  32   *   }
  33   *
  34   * If you have a single object to render, you can use the convenience method
  35   * @{method:renderOneObject}.
  36   *
  37   * @task markup Markup Pipeline
  38   * @task engine Engine Construction
  39   */
  40  final class PhabricatorMarkupEngine {
  41  
  42    private $objects = array();
  43    private $viewer;
  44    private $version = 14;
  45  
  46  
  47  /* -(  Markup Pipeline  )---------------------------------------------------- */
  48  
  49  
  50    /**
  51     * Convenience method for pushing a single object through the markup
  52     * pipeline.
  53     *
  54     * @param PhabricatorMarkupInterface  The object to render.
  55     * @param string                      The field to render.
  56     * @param PhabricatorUser             User viewing the markup.
  57     * @return string                     Marked up output.
  58     * @task markup
  59     */
  60    public static function renderOneObject(
  61      PhabricatorMarkupInterface $object,
  62      $field,
  63      PhabricatorUser $viewer) {
  64      return id(new PhabricatorMarkupEngine())
  65        ->setViewer($viewer)
  66        ->addObject($object, $field)
  67        ->process()
  68        ->getOutput($object, $field);
  69    }
  70  
  71  
  72    /**
  73     * Queue an object for markup generation when @{method:process} is
  74     * called. You can retrieve the output later with @{method:getOutput}.
  75     *
  76     * @param PhabricatorMarkupInterface  The object to render.
  77     * @param string                      The field to render.
  78     * @return this
  79     * @task markup
  80     */
  81    public function addObject(PhabricatorMarkupInterface $object, $field) {
  82      $key = $this->getMarkupFieldKey($object, $field);
  83      $this->objects[$key] = array(
  84        'object' => $object,
  85        'field'  => $field,
  86      );
  87  
  88      return $this;
  89    }
  90  
  91  
  92    /**
  93     * Process objects queued with @{method:addObject}. You can then retrieve
  94     * the output with @{method:getOutput}.
  95     *
  96     * @return this
  97     * @task markup
  98     */
  99    public function process() {
 100      $keys = array();
 101      foreach ($this->objects as $key => $info) {
 102        if (!isset($info['markup'])) {
 103          $keys[] = $key;
 104        }
 105      }
 106  
 107      if (!$keys) {
 108        return;
 109      }
 110  
 111      $objects = array_select_keys($this->objects, $keys);
 112  
 113      // Build all the markup engines. We need an engine for each field whether
 114      // we have a cache or not, since we still need to postprocess the cache.
 115      $engines = array();
 116      foreach ($objects as $key => $info) {
 117        $engines[$key] = $info['object']->newMarkupEngine($info['field']);
 118        $engines[$key]->setConfig('viewer', $this->viewer);
 119      }
 120  
 121      // Load or build the preprocessor caches.
 122      $blocks = $this->loadPreprocessorCaches($engines, $objects);
 123      $blocks = mpull($blocks, 'getCacheData');
 124  
 125      $this->engineCaches = $blocks;
 126  
 127      // Finalize the output.
 128      foreach ($objects as $key => $info) {
 129        $engine = $engines[$key];
 130        $field = $info['field'];
 131        $object = $info['object'];
 132  
 133        $output = $engine->postprocessText($blocks[$key]);
 134        $output = $object->didMarkupText($field, $output, $engine);
 135        $this->objects[$key]['output'] = $output;
 136      }
 137  
 138      return $this;
 139    }
 140  
 141  
 142    /**
 143     * Get the output of markup processing for a field queued with
 144     * @{method:addObject}. Before you can call this method, you must call
 145     * @{method:process}.
 146     *
 147     * @param PhabricatorMarkupInterface  The object to retrieve.
 148     * @param string                      The field to retrieve.
 149     * @return string                     Processed output.
 150     * @task markup
 151     */
 152    public function getOutput(PhabricatorMarkupInterface $object, $field) {
 153      $key = $this->getMarkupFieldKey($object, $field);
 154      $this->requireKeyProcessed($key);
 155  
 156      return $this->objects[$key]['output'];
 157    }
 158  
 159  
 160    /**
 161     * Retrieve engine metadata for a given field.
 162     *
 163     * @param PhabricatorMarkupInterface  The object to retrieve.
 164     * @param string                      The field to retrieve.
 165     * @param string                      The engine metadata field to retrieve.
 166     * @param wild                        Optional default value.
 167     * @task markup
 168     */
 169    public function getEngineMetadata(
 170      PhabricatorMarkupInterface $object,
 171      $field,
 172      $metadata_key,
 173      $default = null) {
 174  
 175      $key = $this->getMarkupFieldKey($object, $field);
 176      $this->requireKeyProcessed($key);
 177  
 178      return idx($this->engineCaches[$key]['metadata'], $metadata_key, $default);
 179    }
 180  
 181  
 182    /**
 183     * @task markup
 184     */
 185    private function requireKeyProcessed($key) {
 186      if (empty($this->objects[$key])) {
 187        throw new Exception(
 188          "Call addObject() before using results (key = '{$key}').");
 189      }
 190  
 191      if (!isset($this->objects[$key]['output'])) {
 192        throw new Exception(
 193          'Call process() before using results.');
 194      }
 195    }
 196  
 197  
 198    /**
 199     * @task markup
 200     */
 201    private function getMarkupFieldKey(
 202      PhabricatorMarkupInterface $object,
 203      $field) {
 204  
 205      static $custom;
 206      if ($custom === null) {
 207        $custom = array_merge(
 208          self::loadCustomInlineRules(),
 209          self::loadCustomBlockRules());
 210  
 211        $custom = mpull($custom, 'getRuleVersion', null);
 212        ksort($custom);
 213        $custom = PhabricatorHash::digestForIndex(serialize($custom));
 214      }
 215  
 216      return $object->getMarkupFieldKey($field).'@'.$this->version.'@'.$custom;
 217    }
 218  
 219  
 220    /**
 221     * @task markup
 222     */
 223    private function loadPreprocessorCaches(array $engines, array $objects) {
 224      $blocks = array();
 225  
 226      $use_cache = array();
 227      foreach ($objects as $key => $info) {
 228        if ($info['object']->shouldUseMarkupCache($info['field'])) {
 229          $use_cache[$key] = true;
 230        }
 231      }
 232  
 233      if ($use_cache) {
 234        try {
 235          $blocks = id(new PhabricatorMarkupCache())->loadAllWhere(
 236            'cacheKey IN (%Ls)',
 237            array_keys($use_cache));
 238          $blocks = mpull($blocks, null, 'getCacheKey');
 239        } catch (Exception $ex) {
 240          phlog($ex);
 241        }
 242      }
 243  
 244      foreach ($objects as $key => $info) {
 245        // False check in case MySQL doesn't support unicode characters
 246        // in the string (T1191), resulting in unserialize returning false.
 247        if (isset($blocks[$key]) && $blocks[$key]->getCacheData() !== false) {
 248          // If we already have a preprocessing cache, we don't need to rebuild
 249          // it.
 250          continue;
 251        }
 252  
 253        $text = $info['object']->getMarkupText($info['field']);
 254        $data = $engines[$key]->preprocessText($text);
 255  
 256        // NOTE: This is just debugging information to help sort out cache issues.
 257        // If one machine is misconfigured and poisoning caches you can use this
 258        // field to hunt it down.
 259  
 260        $metadata = array(
 261          'host' => php_uname('n'),
 262        );
 263  
 264        $blocks[$key] = id(new PhabricatorMarkupCache())
 265          ->setCacheKey($key)
 266          ->setCacheData($data)
 267          ->setMetadata($metadata);
 268  
 269        if (isset($use_cache[$key])) {
 270          // This is just filling a cache and always safe, even on a read pathway.
 271          $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
 272            $blocks[$key]->replace();
 273          unset($unguarded);
 274        }
 275      }
 276  
 277      return $blocks;
 278    }
 279  
 280  
 281    /**
 282     * Set the viewing user. Used to implement object permissions.
 283     *
 284     * @param PhabricatorUser The viewing user.
 285     * @return this
 286     * @task markup
 287     */
 288    public function setViewer(PhabricatorUser $viewer) {
 289      $this->viewer = $viewer;
 290      return $this;
 291    }
 292  
 293  
 294  /* -(  Engine Construction  )------------------------------------------------ */
 295  
 296  
 297  
 298    /**
 299     * @task engine
 300     */
 301    public static function newManiphestMarkupEngine() {
 302      return self::newMarkupEngine(array(
 303      ));
 304    }
 305  
 306  
 307    /**
 308     * @task engine
 309     */
 310    public static function newPhrictionMarkupEngine() {
 311      return self::newMarkupEngine(array(
 312        'header.generate-toc' => true,
 313      ));
 314    }
 315  
 316  
 317    /**
 318     * @task engine
 319     */
 320    public static function newPhameMarkupEngine() {
 321      return self::newMarkupEngine(array(
 322        'macros' => false,
 323      ));
 324    }
 325  
 326  
 327    /**
 328     * @task engine
 329     */
 330    public static function newFeedMarkupEngine() {
 331      return self::newMarkupEngine(
 332        array(
 333          'macros'      => false,
 334          'youtube'     => false,
 335  
 336        ));
 337    }
 338  
 339  
 340    /**
 341     * @task engine
 342     */
 343    public static function newDifferentialMarkupEngine(array $options = array()) {
 344      return self::newMarkupEngine(array(
 345        'differential.diff' => idx($options, 'differential.diff'),
 346      ));
 347    }
 348  
 349  
 350    /**
 351     * @task engine
 352     */
 353    public static function newDiffusionMarkupEngine(array $options = array()) {
 354      return self::newMarkupEngine(array(
 355        'header.generate-toc' => true,
 356      ));
 357    }
 358  
 359    /**
 360     * @task engine
 361     */
 362    public static function getEngine($ruleset = 'default') {
 363      static $engines = array();
 364      if (isset($engines[$ruleset])) {
 365        return $engines[$ruleset];
 366      }
 367  
 368      $engine = null;
 369      switch ($ruleset) {
 370        case 'default':
 371          $engine = self::newMarkupEngine(array());
 372          break;
 373        case 'nolinebreaks':
 374          $engine = self::newMarkupEngine(array());
 375          $engine->setConfig('preserve-linebreaks', false);
 376          break;
 377        case 'diffusion-readme':
 378          $engine = self::newMarkupEngine(array());
 379          $engine->setConfig('preserve-linebreaks', false);
 380          $engine->setConfig('header.generate-toc', true);
 381          break;
 382        case 'diviner':
 383          $engine = self::newMarkupEngine(array());
 384          $engine->setConfig('preserve-linebreaks', false);
 385    //    $engine->setConfig('diviner.renderer', new DivinerDefaultRenderer());
 386          $engine->setConfig('header.generate-toc', true);
 387          break;
 388        case 'extract':
 389          // Engine used for reference/edge extraction. Turn off anything which
 390          // is slow and doesn't change reference extraction.
 391          $engine = self::newMarkupEngine(array());
 392          $engine->setConfig('pygments.enabled', false);
 393          break;
 394        default:
 395          throw new Exception("Unknown engine ruleset: {$ruleset}!");
 396      }
 397  
 398      $engines[$ruleset] = $engine;
 399      return $engine;
 400    }
 401  
 402    /**
 403     * @task engine
 404     */
 405    private static function getMarkupEngineDefaultConfiguration() {
 406      return array(
 407        'pygments'      => PhabricatorEnv::getEnvConfig('pygments.enabled'),
 408        'youtube'       => PhabricatorEnv::getEnvConfig(
 409          'remarkup.enable-embedded-youtube'),
 410        'differential.diff' => null,
 411        'header.generate-toc' => false,
 412        'macros'        => true,
 413        'uri.allowed-protocols' => PhabricatorEnv::getEnvConfig(
 414          'uri.allowed-protocols'),
 415        'syntax-highlighter.engine' => PhabricatorEnv::getEnvConfig(
 416          'syntax-highlighter.engine'),
 417        'preserve-linebreaks' => true,
 418      );
 419    }
 420  
 421  
 422    /**
 423     * @task engine
 424     */
 425    public static function newMarkupEngine(array $options) {
 426  
 427      $options += self::getMarkupEngineDefaultConfiguration();
 428  
 429      $engine = new PhutilRemarkupEngine();
 430  
 431      $engine->setConfig('preserve-linebreaks', $options['preserve-linebreaks']);
 432      $engine->setConfig('pygments.enabled', $options['pygments']);
 433      $engine->setConfig(
 434        'uri.allowed-protocols',
 435        $options['uri.allowed-protocols']);
 436      $engine->setConfig('differential.diff', $options['differential.diff']);
 437      $engine->setConfig('header.generate-toc', $options['header.generate-toc']);
 438      $engine->setConfig(
 439        'syntax-highlighter.engine',
 440        $options['syntax-highlighter.engine']);
 441  
 442      $rules = array();
 443      $rules[] = new PhutilRemarkupEscapeRemarkupRule();
 444      $rules[] = new PhutilRemarkupMonospaceRule();
 445  
 446  
 447      $rules[] = new PhutilRemarkupDocumentLinkRule();
 448      $rules[] = new PhabricatorNavigationRemarkupRule();
 449  
 450      if ($options['youtube']) {
 451        $rules[] = new PhabricatorYoutubeRemarkupRule();
 452      }
 453  
 454      $applications = PhabricatorApplication::getAllInstalledApplications();
 455      foreach ($applications as $application) {
 456        foreach ($application->getRemarkupRules() as $rule) {
 457          $rules[] = $rule;
 458        }
 459      }
 460  
 461      $rules[] = new PhutilRemarkupHyperlinkRule();
 462  
 463      if ($options['macros']) {
 464        $rules[] = new PhabricatorImageMacroRemarkupRule();
 465        $rules[] = new PhabricatorMemeRemarkupRule();
 466      }
 467  
 468      $rules[] = new PhutilRemarkupBoldRule();
 469      $rules[] = new PhutilRemarkupItalicRule();
 470      $rules[] = new PhutilRemarkupDelRule();
 471      $rules[] = new PhutilRemarkupUnderlineRule();
 472  
 473      foreach (self::loadCustomInlineRules() as $rule) {
 474        $rules[] = $rule;
 475      }
 476  
 477      $blocks = array();
 478      $blocks[] = new PhutilRemarkupQuotesBlockRule();
 479      $blocks[] = new PhutilRemarkupReplyBlockRule();
 480      $blocks[] = new PhutilRemarkupLiteralBlockRule();
 481      $blocks[] = new PhutilRemarkupHeaderBlockRule();
 482      $blocks[] = new PhutilRemarkupHorizontalRuleBlockRule();
 483      $blocks[] = new PhutilRemarkupListBlockRule();
 484      $blocks[] = new PhutilRemarkupCodeBlockRule();
 485      $blocks[] = new PhutilRemarkupNoteBlockRule();
 486      $blocks[] = new PhutilRemarkupTableBlockRule();
 487      $blocks[] = new PhutilRemarkupSimpleTableBlockRule();
 488      $blocks[] = new PhutilRemarkupInterpreterBlockRule();
 489      $blocks[] = new PhutilRemarkupDefaultBlockRule();
 490  
 491      foreach (self::loadCustomBlockRules() as $rule) {
 492        $blocks[] = $rule;
 493      }
 494  
 495      foreach ($blocks as $block) {
 496        $block->setMarkupRules($rules);
 497      }
 498  
 499      $engine->setBlockRules($blocks);
 500  
 501      return $engine;
 502    }
 503  
 504    public static function extractPHIDsFromMentions(
 505      PhabricatorUser $viewer,
 506      array $content_blocks) {
 507  
 508      $mentions = array();
 509  
 510      $engine = self::newDifferentialMarkupEngine();
 511      $engine->setConfig('viewer', $viewer);
 512  
 513      foreach ($content_blocks as $content_block) {
 514        $engine->markupText($content_block);
 515        $phids = $engine->getTextMetadata(
 516          PhabricatorMentionRemarkupRule::KEY_MENTIONED,
 517          array());
 518        $mentions += $phids;
 519      }
 520  
 521      return $mentions;
 522    }
 523  
 524    public static function extractFilePHIDsFromEmbeddedFiles(
 525      PhabricatorUser $viewer,
 526      array $content_blocks) {
 527      $files = array();
 528  
 529      $engine = self::newDifferentialMarkupEngine();
 530      $engine->setConfig('viewer', $viewer);
 531  
 532      foreach ($content_blocks as $content_block) {
 533        $engine->markupText($content_block);
 534        $phids = $engine->getTextMetadata(
 535          PhabricatorEmbedFileRemarkupRule::KEY_EMBED_FILE_PHIDS,
 536          array());
 537        foreach ($phids as $phid) {
 538          $files[$phid] = $phid;
 539        }
 540      }
 541  
 542      return array_values($files);
 543    }
 544  
 545    /**
 546     * Produce a corpus summary, in a way that shortens the underlying text
 547     * without truncating it somewhere awkward.
 548     *
 549     * TODO: We could do a better job of this.
 550     *
 551     * @param string  Remarkup corpus to summarize.
 552     * @return string Summarized corpus.
 553     */
 554    public static function summarize($corpus) {
 555  
 556      // Major goals here are:
 557      //  - Don't split in the middle of a character (utf-8).
 558      //  - Don't split in the middle of, e.g., **bold** text, since
 559      //    we end up with hanging '**' in the summary.
 560      //  - Try not to pick an image macro, header, embedded file, etc.
 561      //  - Hopefully don't return too much text. We don't explicitly limit
 562      //    this right now.
 563  
 564      $blocks = preg_split("/\n *\n\s*/", trim($corpus));
 565  
 566      $best = null;
 567      foreach ($blocks as $block) {
 568        // This is a test for normal spaces in the block, i.e. a heuristic to
 569        // distinguish standard paragraphs from things like image macros. It may
 570        // not work well for non-latin text. We prefer to summarize with a
 571        // paragraph of normal words over an image macro, if possible.
 572        $has_space = preg_match('/\w\s\w/', $block);
 573  
 574        // This is a test to find embedded images and headers. We prefer to
 575        // summarize with a normal paragraph over a header or an embedded object,
 576        // if possible.
 577        $has_embed = preg_match('/^[{=]/', $block);
 578  
 579        if ($has_space && !$has_embed) {
 580          // This seems like a good summary, so return it.
 581          return $block;
 582        }
 583  
 584        if (!$best) {
 585          // This is the first block we found; if everything is garbage just
 586          // use the first block.
 587          $best = $block;
 588        }
 589      }
 590  
 591      return $best;
 592    }
 593  
 594    private static function loadCustomInlineRules() {
 595      return id(new PhutilSymbolLoader())
 596        ->setAncestorClass('PhabricatorRemarkupCustomInlineRule')
 597        ->loadObjects();
 598    }
 599  
 600    private static function loadCustomBlockRules() {
 601      return id(new PhutilSymbolLoader())
 602        ->setAncestorClass('PhabricatorRemarkupCustomBlockRule')
 603        ->loadObjects();
 604    }
 605  
 606  }