[Summary view] [Print] [Text view]

   1  <?php
   2  /*
   3   [The "BSD licence"]
   4   Copyright (c) 2005-2008 Terence Parr
   5   All rights reserved.
   6  
   7   Redistribution and use in source and binary forms, with or without
   8   modification, are permitted provided that the following conditions
   9   are met:
  10   1. Redistributions of source code must retain the above copyright
  11      notice, this list of conditions and the following disclaimer.
  12   2. Redistributions in binary form must reproduce the above copyright
  13      notice, this list of conditions and the following disclaimer in the
  14      documentation and/or other materials provided with the distribution.
  15   3. The name of the author may not be used to endorse or promote products
  16      derived from this software without specific prior written permission.
  17  
  18   THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19   IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20   OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21   IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22   INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23   NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27   THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29  
  30  /** The most common stream of tokens is one where every token is buffered up
  31   *  and tokens are prefiltered for a certain channel (the parser will only
  32   *  see these tokens and cannot change the filter channel number during the
  33   *  parse).
  34   *
  35   *  TODO: how to access the full token stream?  How to track all tokens matched per rule?
  36   */
  37  class CommonTokenStream implements TokenStream {
  38      protected $tokenSource;
  39  
  40      /** Record every single token pulled from the source so we can reproduce
  41       *  chunks of it later.
  42       */
  43      protected $tokens;
  44  
  45      /** Map<tokentype, channel> to override some Tokens' channel numbers */
  46      protected $channelOverrideMap;
  47  
  48      /** Set<tokentype>; discard any tokens with this type */
  49      protected $discardSet;
  50  
  51      /** Skip tokens on any channel but this one; this is how we skip whitespace... */
  52      protected $channel;
  53  
  54      /** By default, track all incoming tokens */
  55      protected $discardOffChannelTokens = false;
  56  
  57      /** Track the last mark() call result value for use in rewind(). */
  58      protected $lastMarker = 0;
  59  
  60      /** The index into the tokens list of the current token (next token
  61       *  to consume).  p==-1 indicates that the tokens list is empty
  62       */
  63      protected $p = -1;
  64  
  65  	public function __construct($tokenSource, $channel=null) {
  66          $this->channel = TokenConst::$DEFAULT_CHANNEL;
  67          $this->tokens = array();
  68          $this->tokenSource = $tokenSource;
  69          if($channel != null){
  70              $this->channel = $channel;
  71          }
  72      }
  73  
  74      /** Reset this token stream by setting its token source. */
  75  	public function setTokenSource($tokenSource) {
  76          $this->tokenSource = $tokenSource;
  77          $this->tokens = array();
  78          $this->p = -1;
  79          $this->channel = TokenConst::$DEFAULT_CHANNEL;
  80      }
  81  
  82      /** Load all tokens from the token source and put in tokens.
  83       *  This is done upon first LT request because you might want to
  84       *  set some token type / channel overrides before filling buffer.
  85       */
  86  	protected function fillBuffer() {
  87          $index = 0;
  88          $t = $this->tokenSource->nextToken();
  89          while ( $t!=null && $t->getType()!=CharStreamConst::$EOF ) {
  90              $discard = false;
  91              // is there a channel override for token type?
  92              if ( $this->channelOverrideMap!=null ) {
  93                  $channelI = $this->channelOverrideMap[$t->getType()];
  94                  if ( $channelI!=null ) {
  95                      $t->setChannel($channelI);
  96                  }
  97              }
  98              if ( $this->discardSet!=null &&
  99                   $this->discardSet->contains($t->getType()))
 100              {
 101                  $discard = true;
 102              }
 103              else if ( $this->discardOffChannelTokens && $t->getChannel()!=$this->channel ) {
 104                  $discard = true;
 105              }
 106              if ( !$discard )    {
 107                  $t->setTokenIndex($index);
 108                  $this->tokens[] = $t;
 109                  $index++;
 110              }
 111              $t = $this->tokenSource->nextToken();
 112          }
 113          // leave p pointing at first token on channel
 114          $this->p = 0;
 115          $this->p = $this->skipOffTokenChannels($this->p);
 116      }
 117  
 118      /** Move the input pointer to the next incoming token.  The stream
 119       *  must become active with LT(1) available.  consume() simply
 120       *  moves the input pointer so that LT(1) points at the next
 121       *  input symbol. Consume at least one token.
 122       *
 123       *  Walk past any token not on the channel the parser is listening to.
 124       */
 125  	public function consume() {
 126          if ( $this->p<sizeof($this->tokens)) {
 127              $this->p++;
 128              $this->p = $this->skipOffTokenChannels($this->p); // leave p on valid token
 129          }
 130      }
 131  
 132      /** Given a starting index, return the index of the first on-channel
 133       *  token.
 134       */
 135  	protected function skipOffTokenChannels($i) {
 136          $n = sizeof($this->tokens);
 137          while ( $i<$n && $this->tokens[$i]->getChannel()!=$this->channel ) {
 138              $i++;
 139          }
 140          return $i;
 141      }
 142  
 143  	protected function skipOffTokenChannelsReverse($i) {
 144          while ( $i>=0 && $this->tokens[$i]->getChannel()!=$this->channel) {
 145              $i--;
 146          }
 147          return $i;
 148      }
 149  
 150      /** A simple filter mechanism whereby you can tell this token stream
 151       *  to force all tokens of type ttype to be on channel.  For example,
 152       *  when interpreting, we cannot exec actions so we need to tell
 153       *  the stream to force all WS and NEWLINE to be a different, ignored
 154       *  channel.
 155       */
 156  	public function setTokenTypeChannel($ttype, $channel) {
 157          if ( $this->channelOverrideMap==null ) {
 158              $this->channelOverrideMap = array();
 159          }
 160          $this->channelOverrideMap[$ttype] = $channel;
 161      }
 162  
 163  	public function discardTokenType($ttype) {
 164          if ( $this->discardSet==null ) {
 165              $this->discardSet = new Set();
 166          }
 167          $this->discardSet.add($ttype);
 168      }
 169  
 170  	public function discardOffChannelTokens($discardOffChannelTokens) {
 171          $this->discardOffChannelTokens = $discardOffChannelTokens;
 172      }
 173  
 174  	public function getTokens() {
 175          if ( $this->p == -1 ) {
 176              $this->fillBuffer();
 177          }
 178          return $this->tokens;
 179      }
 180  
 181  	public function getTokensBetween($start, $stop) {
 182          return $this->getTokens($start, $stop, null);
 183      }
 184  
 185      /** Given a start and stop index, return a List of all tokens in
 186       *  the token type BitSet.  Return null if no tokens were found.  This
 187       *  method looks at both on and off channel tokens.
 188       */
 189  	public function getTokensOfTypeInSet($start, $stop, $types) {
 190          if ( $p == -1 ) {
 191              fillBuffer();
 192          }
 193          if ( $stop>=sizeof($this->tokens)) {
 194              $stop=sizeof($this->tokens) - 1;
 195          }
 196          if ( $start<0 ) {
 197              $start=0;
 198          }
 199          if ( $start>$stop ) {
 200              return null;
 201          }
 202  
 203          // list = tokens[start:stop]:{Token t, t.getType() in types}
 204          $filteredTokens = array();
 205          for ($i=$start; $i<=$stop; $i++) {
 206              $t = $this->tokens[$i];
 207              if ( $types==null || $types->member($t->getType())) {
 208                  $filteredTokens->add($t);
 209              }
 210          }
 211          if ( sizeof($filteredTokens)==0 ) {
 212              $filteredTokens = null;
 213          }
 214          return $filteredTokens;
 215      }
 216  
 217  	public function getTokensOfTypeInArray($start, $stop, $types) {
 218          return $this->getTokens($start, $stop,new Set(types));
 219      }
 220  
 221  	public function getTokensofType($start, $stop, $ttype) {
 222          return $this->getTokens($start, $stop, new Set(array(ttype)));
 223      }
 224  
 225      /** Get the ith token from the current position 1..n where k=1 is the
 226       *  first symbol of lookahead.
 227       */
 228      public function LT($k) {
 229          if ( $this->p == -1 ) {
 230              $this->fillBuffer();
 231          }
 232          if ( $k==0 ) {
 233              return null;
 234          }
 235          if ( $k<0 ) {
 236              return $this->LB(-$k);
 237          }
 238          //System.out.print("LT(p="+p+","+k+")=");
 239          if ( ($this->p+$k-1) >= sizeof($this->tokens)) {
 240              return TokenConst::$EOF_TOKEN;
 241          }
 242          //System.out.println(tokens.get(p+k-1));
 243          $i = $this->p;
 244          $n = 1;
 245          // find k good tokens
 246          while ( $n<$k ) {
 247              // skip off-channel tokens
 248              $i = $this->skipOffTokenChannels($i+1); // leave p on valid token
 249              $n++;
 250          }
 251          if ( $i>=sizeof($this->tokens)) {
 252              return TokenConst::$EOF_TOKEN;
 253          }
 254          return $this->tokens[$i];
 255      }
 256  
 257      /** Look backwards k tokens on-channel tokens */
 258      protected function LB($k) {
 259          //System.out.print("LB(p="+p+","+k+") ");
 260          if ( $this->p == -1 ) {
 261              $this->fillBuffer();
 262          }
 263          if ( $k==0 ) {
 264              return null;
 265          }
 266          if ( ($this->p-$k)<0 ) {
 267              return null;
 268          }
 269          
 270  
 271          $i = $this->p;
 272          $n = 1;
 273          // find k good tokens looking backwards
 274          while ( $n<=$k ) {
 275              // skip off-channel tokens
 276              $i = $this->skipOffTokenChannelsReverse($i-1); // leave p on valid token
 277              $n++;
 278          }
 279          if ( $i<0 ) {
 280              return null;
 281          }
 282          return $this->tokens[$i];
 283      }
 284  
 285      /** Return absolute token i; ignore which channel the tokens are on;
 286       *  that is, count all tokens not just on-channel tokens.
 287       */
 288  	public function get($i) {
 289          return $this->tokens[$i];
 290      }
 291  
 292      public function LA($i) {
 293          $lt = $this->LT($i);
 294          return $this->LT($i)->getType();
 295      }
 296  
 297      public function mark() {
 298          if ( $this->p == -1 ) {
 299              $this->fillBuffer();
 300          }
 301          $this->lastMarker = $this->index();
 302          return $this->lastMarker;
 303      }
 304  
 305  	public function release($marker) {
 306          // no resources to release
 307      }
 308  
 309  	public function size() {
 310          return sizeof($this->tokens);
 311      }
 312  
 313      public function index() {
 314          return $this->p;
 315      }
 316  
 317  	public function rewind($marker = null) {
 318          if($marker===null){
 319              $marker = $this->lastmarker;
 320          }
 321          $this->seek($marker);
 322      }
 323  
 324  
 325  	public function reset() {
 326          $this->p = 0;
 327          $this->lastMarker = 0;
 328      }
 329      
 330  	public function seek($index) {
 331          $this->p = $index;
 332      }
 333  
 334  	public function getTokenSource() {
 335          return $this->tokenSource;
 336      }
 337  
 338  	public function getSourceName() {
 339          return $this->getTokenSource()->getSourceName();
 340      }
 341  
 342  	public function toString() {
 343          if ( $this->p == -1 ) {
 344              $this->fillBuffer();
 345          }
 346          return $this->toStringBetween(0, sizeof($this->tokens)-1);
 347      }
 348  
 349  	public function toStringBetween($start, $stop) {
 350          if ( $start<0 || $stop<0 ) {
 351              return null;
 352          }
 353          if ( $this->p == -1 ) {
 354              $this->fillBuffer();
 355          }
 356          if ( $stop>=sizeof($this->tokens)) {
 357              $stop = sizeof($this->tokens)-1;
 358          }
 359          $buf = "";
 360          for ($i = $start; $i <= $stop; $i++) {
 361              $t = $this->tokens[$i];
 362              $buf.=$t->getText();
 363          }
 364          return $buf;
 365      }
 366  
 367  	public function toStringBetweenTokens($start, $stop) {
 368          if ( $start!=null && $stop!=null ) {
 369              return toString($this->start->getTokenIndex(), $this->stop->getTokenIndex());
 370          }
 371          return null;
 372      }
 373      
 374  	public function __toString(){
 375          return $this->toString();
 376      }
 377  }
 378  
 379  
 380  ?>