[Summary view] [Print] [Text view]

   1  #!/usr/bin/env php
   2  <?php
   3  
   4  
   5  $root = dirname(dirname(dirname(__FILE__)));
   6  require_once $root.'/scripts/__init_script__.php';
   7  
   8  $args = new PhutilArgumentParser($argv);
   9  $args->setSynopsis(<<<EOSYNOPSIS
  10  **import_project_symbols.php** [__options__] __project_name__ < symbols
  11  
  12    Import project symbols (symbols are read from stdin).
  13  EOSYNOPSIS
  14    );
  15  $args->parseStandardArguments();
  16  $args->parse(
  17    array(
  18      array(
  19        'name'      => 'no-purge',
  20        'help'      => 'Do not clear all symbols for this project before '.
  21                       'uploading new symbols. Useful for incremental updating.',
  22      ),
  23      array(
  24        'name'      => 'ignore-errors',
  25        'help'      => 'If a line can\'t be parsed, ignore that line and '.
  26                       'continue instead of exiting.',
  27      ),
  28      array(
  29        'name'      => 'max-transaction',
  30        'param'     => 'num-syms',
  31        'default'  => '100000',
  32        'help'      => 'Maximum number of symbols that should '.
  33                       'be part of a single transaction',
  34      ),
  35      array(
  36        'name'      => 'more',
  37        'wildcard'  => true,
  38      ),
  39    ));
  40  
  41  $more = $args->getArg('more');
  42  if (count($more) !== 1) {
  43    $args->printHelpAndExit();
  44  }
  45  
  46  $project_name = head($more);
  47  $project = id(new PhabricatorRepositoryArcanistProject())->loadOneWhere(
  48    'name = %s',
  49    $project_name);
  50  
  51  if (!$project) {
  52    // TODO: Provide a less silly way to do this explicitly, or just do it right
  53    // here.
  54    echo "Project '{$project_name}' is unknown. Upload a diff to implicitly ".
  55         "create it.\n";
  56    exit(1);
  57  }
  58  
  59  echo "Parsing input from stdin...\n";
  60  $input = file_get_contents('php://stdin');
  61  $input = trim($input);
  62  $input = explode("\n", $input);
  63  
  64  
  65  function commit_symbols ($syms, $project, $no_purge) {
  66    echo "Looking up path IDs...\n";
  67    $path_map =
  68      PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
  69             ipull($syms, 'path'));
  70  
  71    $symbol = new PhabricatorRepositorySymbol();
  72    $conn_w = $symbol->establishConnection('w');
  73  
  74    echo "Preparing queries...\n";
  75    $sql = array();
  76    foreach ($syms as $dict) {
  77      $sql[] = qsprintf(
  78                        $conn_w,
  79                        '(%d, %s, %s, %s, %s, %d, %d)',
  80                        $project->getID(),
  81                        $dict['ctxt'],
  82                        $dict['name'],
  83                        $dict['type'],
  84                        $dict['lang'],
  85                        $dict['line'],
  86                        $path_map[$dict['path']]);
  87    }
  88  
  89    if (!$no_purge) {
  90      echo "Purging old syms...\n";
  91      queryfx($conn_w,
  92              'DELETE FROM %T WHERE arcanistProjectID = %d',
  93              $symbol->getTableName(),
  94              $project->getID());
  95    }
  96  
  97    echo "Loading ".number_format(count($sql))." syms...\n";
  98    foreach (array_chunk($sql, 128) as $chunk) {
  99      queryfx($conn_w,
 100              'INSERT INTO %T
 101        (arcanistProjectID, symbolContext, symbolName, symbolType,
 102          symbolLanguage, lineNumber, pathID) VALUES %Q',
 103              $symbol->getTableName(),
 104              implode(', ', $chunk));
 105    }
 106  
 107  }
 108  
 109  function check_string_value($value, $field_name, $line_no, $max_length) {
 110     if (strlen($value) > $max_length) {
 111        throw new Exception(
 112          "{$field_name} '{$value}' defined on line #{$line_no} is too long, ".
 113          "maximum {$field_name} length is {$max_length} characters.");
 114      }
 115  
 116      if (!phutil_is_utf8_with_only_bmp_characters($value)) {
 117        throw new Exception(
 118          "{$field_name} '{$value}' defined on line #{$line_no} is not a valid ".
 119          "UTF-8 string, ".
 120          "it should contain only UTF-8 characters.");
 121      }
 122  }
 123  
 124  $no_purge = $args->getArg('no-purge');
 125  $symbols = array();
 126  foreach ($input as $key => $line) {
 127    try {
 128      $line_no = $key + 1;
 129      $matches = null;
 130      $ok = preg_match(
 131        '/^((?P<context>[^ ]+)? )?(?P<name>[^ ]+) (?P<type>[^ ]+) '.
 132        '(?P<lang>[^ ]+) (?P<line>\d+) (?P<path>.*)$/',
 133        $line,
 134        $matches);
 135      if (!$ok) {
 136        throw new Exception(
 137          "Line #{$line_no} of input is invalid. Expected five or six ".
 138          "space-delimited fields: maybe symbol context, symbol name, symbol ".
 139          "type, symbol language, line number, path. ".
 140          "For example:\n\n".
 141          "idx function php 13 /path/to/some/file.php\n\n".
 142          "Actual line was:\n\n".
 143          "{$line}");
 144      }
 145      if (empty($matches['context'])) {
 146        $matches['context'] = '';
 147      }
 148      $context     = $matches['context'];
 149      $name        = $matches['name'];
 150      $type        = $matches['type'];
 151      $lang        = $matches['lang'];
 152      $line_number = $matches['line'];
 153      $path        = $matches['path'];
 154  
 155      check_string_value($context, 'Symbol context', $line_no, 128);
 156      check_string_value($name, 'Symbol name', $line_no, 128);
 157      check_string_value($type, 'Symbol type', $line_no, 12);
 158      check_string_value($lang, 'Symbol language', $line_no, 32);
 159      check_string_value($path, 'Path', $line_no, 512);
 160  
 161      if (!strlen($path) || $path[0] != '/') {
 162        throw new Exception(
 163          "Path '{$path}' defined on line #{$line_no} is invalid. Paths should ".
 164          "begin with '/' and specify a path from the root of the project, like ".
 165          "'/src/utils/utils.php'.");
 166      }
 167  
 168      $symbols[] = array(
 169        'ctxt' => $context,
 170        'name' => $name,
 171        'type' => $type,
 172        'lang' => $lang,
 173        'line' => $line_number,
 174        'path' => $path,
 175      );
 176    } catch (Exception $e) {
 177      if ($args->getArg('ignore-errors')) {
 178        continue;
 179      } else {
 180        throw $e;
 181      }
 182    }
 183  
 184    if (count ($symbols) >= $args->getArg('max-transaction')) {
 185        try {
 186          echo "Committing {$args->getArg('max-transaction')} symbols....\n";
 187          commit_symbols($symbols, $project, $no_purge);
 188          $no_purge = true;
 189          unset($symbols);
 190          $symbols = array();
 191        } catch (Exception $e) {
 192          if ($args->getArg('ignore-errors')) {
 193            continue;
 194          } else {
 195            throw $e;
 196          }
 197        }
 198    }
 199  }
 200  
 201  if (count($symbols)) {
 202    commit_symbols($symbols, $project, $no_purge);
 203  }
 204  
 205  echo "Done.\n";