[Summary view] [Print] [Text view]

   1  <?php
   2  /**
   3   * PHP_LexerGenerator, a php 5 lexer generator.
   4   * 
   5   * This lexer generator translates a file in a format similar to
   6   * re2c ({@link http://re2c.org}) and translates it into a PHP 5-based lexer
   7   *
   8   * PHP version 5
   9   *
  10   * LICENSE: This source file is subject to version 3.01 of the PHP license
  11   * that is available through the world-wide-web at the following URI:
  12   * http://www.php.net/license/3_01.txt.  If you did not receive a copy of
  13   * the PHP License and are unable to obtain it through the web, please
  14   * send a note to [email protected] so we can mail you a copy immediately.
  15   *
  16   * @category   php
  17   * @package    PHP_LexerGenerator
  18   * @author     Gregory Beaver <[email protected]>
  19   * @copyright  2006 Gregory Beaver
  20   * @license    http://www.php.net/license/3_01.txt  PHP License 3.01
  21   * @version    CVS: $Id$
  22   * @since      File available since Release 0.1.0
  23   */
  24  require_once 'PHP/LexerGenerator/Parser.php';
  25  /**
  26   * Token scanner for plex files.
  27   * 
  28   * This scanner detects comments beginning with "/*!lex2php" and
  29   * then returns their components (processing instructions, patterns, strings
  30   * action code, and regexes)
  31   * @package    PHP_LexerGenerator
  32   * @author     Gregory Beaver <[email protected]>
  33   * @copyright  2006 Gregory Beaver
  34   * @license    http://www.php.net/license/3_01.txt  PHP License 3.01
  35   * @version    0.2.0
  36   * @since      Class available since Release 0.1.0
  37   */
  38  class PHP_LexerGenerator_Lexer
  39  {
  40      private $data;
  41      private $N;
  42      private $state;
  43      /**
  44       * Current line number in input
  45       * @var int
  46       */
  47      public $line;
  48      /**
  49       * Number of scanning errors detected
  50       * @var int
  51       */
  52      public $errors = 0;
  53      /**
  54       * integer identifier of the current token
  55       * @var int
  56       */
  57      public $token;
  58      /**
  59       * string content of current token
  60       * @var string
  61       */
  62      public $value;
  63  
  64      const PHPCODE = PHP_LexerGenerator_Parser::PHPCODE;
  65      const COMMENTSTART = PHP_LexerGenerator_Parser::COMMENTSTART;
  66      const COMMENTEND = PHP_LexerGenerator_Parser::COMMENTEND;
  67      const QUOTE = PHP_LexerGenerator_Parser::QUOTE;
  68      const PATTERN = PHP_LexerGenerator_Parser::PATTERN;
  69      const CODE = PHP_LexerGenerator_Parser::CODE;
  70      const SUBPATTERN = PHP_LexerGenerator_Parser::SUBPATTERN;
  71      const PI = PHP_LexerGenerator_Parser::PI;
  72  
  73      /**
  74       * prepare scanning
  75       * @param string the input
  76       */
  77      function __construct($data)
  78      {
  79          $this->data = str_replace("\r\n", "\n", $data);
  80          $this->N = 0;
  81          $this->line = 1;
  82          $this->state = 'Start';
  83          $this->errors = 0;
  84      }
  85  
  86      /**
  87       * Output an error message
  88       * @param string
  89       */
  90      private function error($msg)
  91      {
  92          echo 'Error on line ' . $this->line . ': ' . $msg;
  93          $this->errors++;
  94      }
  95  
  96      /**
  97       * Initial scanning state lexer
  98       * @return boolean
  99       */
 100      private function lexStart()
 101      {
 102          if ($this->N >= strlen($this->data)) {
 103              return false;
 104          }
 105          $a = strpos($this->data, '/*!lex2php' . "\n", $this->N);
 106          if ($a === false) {
 107              $this->value = substr($this->data, $this->N);
 108              $this->N = strlen($this->data);
 109              $this->token = self::PHPCODE;
 110              return true;
 111          }
 112          if ($a > $this->N) {
 113              $this->value = substr($this->data, $this->N, $a - $this->N);
 114              $this->N = $a;
 115              $this->token = self::PHPCODE;
 116              return true;
 117          }
 118          $this->value = '/*!lex2php' . "\n";
 119          $this->N += 11; // strlen("/*lex2php\n")
 120          $this->token = self::COMMENTSTART;
 121          $this->state = 'Declare';
 122          return true;
 123      }
 124  
 125      /**
 126       * lexer for top-level canning state after the initial declaration comment
 127       * @return boolean
 128       */
 129      private function lexStartNonDeclare()
 130      {
 131          if ($this->N >= strlen($this->data)) {
 132              return false;
 133          }
 134          $a = strpos($this->data, '/*!lex2php' . "\n", $this->N);
 135          if ($a === false) {
 136              $this->value = substr($this->data, $this->N);
 137              $this->N = strlen($this->data);
 138              $this->token = self::PHPCODE;
 139              return true;
 140          }
 141          if ($a > $this->N) {
 142              $this->value = substr($this->data, $this->N, $a - $this->N);
 143              $this->N = $a;
 144              $this->token = self::PHPCODE;
 145              return true;
 146          }
 147          $this->value = '/*!lex2php' . "\n";
 148          $this->N += 11; // strlen("/*lex2php\n")
 149          $this->token = self::COMMENTSTART;
 150          $this->state = 'Rule';
 151          return true;
 152      }
 153  
 154      /**
 155       * lexer for declaration comment state
 156       * @return boolean
 157       */
 158      private function lexDeclare()
 159      {
 160          if ($this->data[$this->N] == '*' && $this->data[$this->N + 1] == '/') {
 161              $this->state = 'StartNonDeclare';
 162              $this->value = '*/';
 163              $this->N += 2;
 164              $this->token = self::COMMENTEND;
 165              return true;
 166          }
 167          if (preg_match('/^%([a-z]+)/', substr($this->data, $this->N), $token)) {
 168              $this->value = $token[1];
 169              $this->N += strlen($token[1]) + 1;
 170              $this->state = 'DeclarePI';
 171              $this->token = self::PI;
 172              return true;
 173          }
 174          if (preg_match('/^[a-zA-Z]+/', substr($this->data, $this->N), $token)) {
 175              $this->value = $token[0];
 176              $this->token = self::PATTERN;
 177              $this->N += strlen($token[0]);
 178              $this->state = 'DeclareEquals';
 179              return true;
 180          } else {
 181              $this->error('expecting declaration of sub-patterns');
 182              return false;
 183          }
 184      }
 185  
 186      /**
 187       * lexer for processor instructions within declaration comment
 188       * @return boolean
 189       */
 190      private function lexDeclarePI()
 191      {
 192          while ($this->N < strlen($this->data) && 
 193                  ($this->data[$this->N] == ' ' ||
 194                   $this->data[$this->N] == "\t")) {
 195              $this->N++; // skip whitespace
 196          }
 197          if ($this->data[$this->N] == "\n") {
 198              $this->N++;
 199              $this->state = 'Declare';
 200              $this->line++;
 201              return $this->lexDeclare();
 202          }
 203          if ($this->data[$this->N] == '{') {
 204              return $this->lexCode();
 205          }
 206          if (!preg_match("/[^\n]+/", substr($this->data, $this->N), $token)) {
 207              $this->error('Unexpected end of file');
 208              return false;
 209          }
 210          $this->value = $token[0];
 211          $this->N += strlen($this->value);
 212          $this->token = self::SUBPATTERN;
 213          return true;
 214      }
 215  
 216      /**
 217       * lexer for processor instructions inside rule comments
 218       * @return boolean
 219       */
 220      private function lexDeclarePIRule()
 221      {
 222          while ($this->N < strlen($this->data) && 
 223                  ($this->data[$this->N] == ' ' ||
 224                   $this->data[$this->N] == "\t")) {
 225              $this->N++; // skip whitespace
 226          }
 227          if ($this->data[$this->N] == "\n") {
 228              $this->N++;
 229              $this->state = 'Rule';
 230              $this->line++;
 231              return $this->lexRule();
 232          }
 233          if ($this->data[$this->N] == '{') {
 234              return $this->lexCode();
 235          }
 236          if (!preg_match("/[^\n]+/", substr($this->data, $this->N), $token)) {
 237              $this->error('Unexpected end of file');
 238              return false;
 239          }
 240          $this->value = $token[0];
 241          $this->N += strlen($this->value);
 242          $this->token = self::SUBPATTERN;
 243          return true;
 244      }
 245  
 246      /**
 247       * lexer for the state representing scanning between a pattern and the "=" sign
 248       * @return boolean
 249       */
 250      private function lexDeclareEquals()
 251      {
 252          while ($this->N < strlen($this->data) && 
 253                  ($this->data[$this->N] == ' ' || $this->data[$this->N] == "\t")) {
 254              $this->N++; // skip whitespace
 255          }
 256          if ($this->N >= strlen($this->data)) {
 257              $this->error('unexpected end of input, expecting "=" for sub-pattern declaration');
 258          }
 259          if ($this->data[$this->N] != '=') {
 260              $this->error('expecting "=" for sub-pattern declaration');
 261              return false;
 262          }
 263          $this->N++;
 264          $this->state = 'DeclareRightside';
 265          while ($this->N < strlen($this->data) && 
 266                  ($this->data[$this->N] == ' ' || $this->data[$this->N] == "\t")) {
 267              $this->N++; // skip whitespace
 268          }
 269          if ($this->N >= strlen($this->data)) {
 270              $this->error('unexpected end of file, expecting right side of sub-pattern declaration');
 271              return false;
 272          }
 273          return $this->lexDeclareRightside();
 274      }
 275  
 276      /**
 277       * lexer for the right side of a pattern, detects quotes or regexes
 278       * @return boolean
 279       */
 280      private function lexDeclareRightside()
 281      {
 282          if ($this->data[$this->N] == "\n") {
 283              $this->state = 'lexDeclare';
 284              $this->N++;
 285              $this->line++;
 286              return $this->lexDeclare();
 287          }
 288          if ($this->data[$this->N] == '"') {
 289              return $this->lexQuote();
 290          }
 291          // match a pattern
 292          $test = $this->data[$this->N];
 293          $token = $this->N + 1;
 294          $a = 0;
 295          do {
 296              if ($a++) {
 297                  $token++;
 298              }
 299              $token = strpos($this->data, $test, $token);
 300          } while ($token !== false && ($this->data[$token - 1] == '\\'
 301                   && $this->data[$token - 2] != '\\'));
 302          if ($token === false) {
 303              $this->error('Unterminated regex pattern (started with "' . $test . '"');
 304              return false;
 305          }
 306          if (substr_count($this->data, "\n", $this->N, $token - $this->N)) {
 307              $this->error('Regex pattern extends over multiple lines');
 308              return false;
 309          }
 310          $this->value = substr($this->data, $this->N + 1, $token - $this->N - 1);
 311          // unescape the regex marker
 312          // we will re-escape when creating the final regex
 313          $this->value = str_replace('\\' . $test, $test, $this->value);
 314          $this->N = $token + 1;
 315          $this->token = self::SUBPATTERN;
 316          return true;
 317      }
 318  
 319      /**
 320       * lexer for quoted literals
 321       * @return boolean
 322       */
 323      private function lexQuote()
 324      {
 325          $token = $this->N + 1;
 326          $a = 0;
 327          do {
 328              if ($a++) {
 329                  $token++;
 330              }
 331              $token = strpos($this->data, '"', $token);
 332          } while ($token !== false && $token < strlen($this->data) &&
 333                    ($this->data[$token - 1] == '\\' && $this->data[$token - 2] != '\\'));
 334          if ($token === false) {
 335              $this->error('unterminated quote');
 336              return false;
 337          }
 338          if (substr_count($this->data, "\n", $this->N, $token - $this->N)) {
 339              $this->error('quote extends over multiple lines');
 340              return false;
 341          }
 342          $this->value = substr($this->data, $this->N + 1, $token - $this->N - 1);
 343          $this->value = str_replace('\\"', '"', $this->value);
 344          $this->value = str_replace('\\\\', '\\', $this->value);
 345          $this->N = $token + 1;
 346          $this->token = self::QUOTE;
 347          return true;
 348      }
 349  
 350      /**
 351       * lexer for rules
 352       * @return boolean
 353       */
 354      private function lexRule()
 355      {
 356          while ($this->N < strlen($this->data) && 
 357                  ($this->data[$this->N] == ' ' ||
 358                   $this->data[$this->N] == "\t" ||
 359                   $this->data[$this->N] == "\n")) {
 360              if ($this->data[$this->N] == "\n") {
 361                  $this->line++;
 362              }
 363              $this->N++; // skip all whitespace
 364          }
 365          if ($this->N >= strlen($this->data)) {
 366              $this->error('unexpected end of input, expecting rule declaration');
 367          }
 368          if ($this->data[$this->N] == '*' && $this->data[$this->N + 1] == '/') {
 369              $this->state = 'StartNonDeclare';
 370              $this->value = '*/';
 371              $this->N += 2;
 372              $this->token = self::COMMENTEND;
 373              return true;
 374          }
 375          if (preg_match('/^%([a-z]+)/', substr($this->data, $this->N), $token)) {
 376              $this->value = $token[1];
 377              $this->N += strlen($token[1]) + 1;
 378              $this->state = 'DeclarePIRule';
 379              $this->token = self::PI;
 380              return true;
 381          }
 382          if ($this->data[$this->N] == "{") {
 383              return $this->lexCode();
 384          }
 385          if ($this->data[$this->N] == '"') {
 386              return $this->lexQuote();
 387          }
 388          if (preg_match('/^[a-zA-Z]+/', substr($this->data, $this->N), $token)) {
 389              $this->value = $token[0];
 390              $this->N += strlen($token[0]);
 391              $this->token = self::SUBPATTERN;
 392              return true;
 393          } else {
 394              $this->error('expecting token rule (quotes or sub-patterns)');
 395              return false;
 396          }
 397      }
 398  
 399      /**
 400       * lexer for php code blocks
 401       * @return boolean
 402       */
 403      private function lexCode()
 404      {
 405          $cp = $this->N + 1;
 406          for ($level = 1; $cp < strlen($this->data) && ($level > 1 || $this->data[$cp] != '}'); $cp++) {
 407              if ($this->data[$cp] == '{') {
 408                  $level++;
 409              } elseif ($this->data[$cp] == '}') {
 410                  $level--;
 411              } elseif ($this->data[$cp] == '/' && $this->data[$cp + 1] == '/') {
 412                  /* Skip C++ style comments */
 413                  $cp += 2;
 414                  $z = strpos($this->data, "\n", $cp);
 415                  if ($z === false) {
 416                      $cp = strlen($this->data);
 417                      break;
 418                  }
 419                  $cp = $z;
 420              } elseif ($this->data[$cp] == "'" || $this->data[$cp] == '"') {
 421                  /* String a character literals */
 422                  $startchar = $this->data[$cp];
 423                  $prevc = 0;
 424                  for ($cp++; $cp < strlen($this->data) && ($this->data[$cp] != $startchar || $prevc === '\\'); $cp++) {
 425                      if ($prevc === '\\') {
 426                          $prevc = 0;
 427                      } else {
 428                          $prevc = $this->data[$cp];
 429                      }
 430                  }
 431              }
 432          }
 433          if ($cp >= strlen($this->data)) {
 434              $this->error("PHP code starting on this line is not terminated before the end of the file.");
 435              $this->error++;
 436              return false;
 437          } else {
 438              $this->value = substr($this->data, $this->N + 1, $cp - $this->N - 1);
 439              $this->token = self::CODE;
 440              $this->N = $cp + 1;
 441              return true;
 442          }
 443      }
 444  
 445      /**
 446       * Primary scanner
 447       * 
 448       * In addition to lexing, this properly increments the line number of lexing.
 449       * This calls the proper sub-lexer based on the parser state
 450       * @param unknown_type $parser
 451       * @return unknown
 452       */
 453      public function advance($parser)
 454      {
 455          if ($this->N >= strlen($this->data)) {
 456              return false;
 457          }
 458          if ($this->{'lex' . $this->state}()) {
 459              $this->line += substr_count($this->value, "\n");
 460              return true;
 461          }
 462          return false;
 463      }
 464  }
 465  ?>