[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  final class PhabricatorRobotsController extends PhabricatorController {
   4  
   5    public function shouldRequireLogin() {
   6      return false;
   7    }
   8  
   9    public function processRequest() {
  10      $out = array();
  11  
  12      // Prevent indexing of '/diffusion/', since the content is not generally
  13      // useful to index, web spiders get stuck scraping the history of every
  14      // file, and much of the content is Ajaxed in anyway so spiders won't even
  15      // see it. These pages are also relatively expensive to generate.
  16  
  17      // Note that this still allows commits (at '/rPxxxxx') to be indexed.
  18      // They're probably not hugely useful, but suffer fewer of the problems
  19      // Diffusion suffers and are hard to omit with 'robots.txt'.
  20  
  21      $out[] = 'User-Agent: *';
  22      $out[] = 'Disallow: /diffusion/';
  23  
  24      // Add a small crawl delay (number of seconds between requests) for spiders
  25      // which respect it. The intent here is to prevent spiders from affecting
  26      // performance for users. The possible cost is slower indexing, but that
  27      // seems like a reasonable tradeoff, since most Phabricator installs are
  28      // probably not hugely concerned about cutting-edge SEO.
  29      $out[] = 'Crawl-delay: 1';
  30  
  31      $content = implode("\n", $out)."\n";
  32  
  33      return id(new AphrontPlainTextResponse())
  34        ->setContent($content)
  35        ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'));
  36    }
  37  }