Abstract baseclass for the tokenizer. More...
#include <tokenizer.hpp>
Public Member Functions | |
tokenizer (std::istream &in) | |
~tokenizer () | |
const token & | next_token () |
const token & | current_token () const |
const std::string & | textdomain () const |
const std::string & | get_file () const |
int | get_start_line () const |
Private Types | |
enum | { TOK_NONE = 0, TOK_SPACE = 1, TOK_NUMERIC = 2, TOK_ALPHA = 4 } |
Private Member Functions | |
tokenizer () | |
void | next_char () |
void | next_char_fast () |
int | peek_char () |
int | char_type (unsigned c) const |
bool | is_space (int c) const |
bool | is_num (int c) const |
bool | is_alnum (int c) const |
void | skip_comment () |
bool | skip_command (char const *cmd) |
Returns true if the next characters are the one from cmd followed by a space. More... | |
Private Attributes | |
int | current_ |
int | lineno_ |
int | startlineno_ |
std::string | textdomain_ |
std::string | file_ |
token | token_ |
buffered_istream | in_ |
char | char_types_ [128] |
Abstract baseclass for the tokenizer.
Definition at line 58 of file tokenizer.hpp.
|
private |
Enumerator | |
---|---|
TOK_NONE | |
TOK_SPACE | |
TOK_NUMERIC | |
TOK_ALPHA |
Definition at line 135 of file tokenizer.hpp.
tokenizer::tokenizer | ( | std::istream & | in | ) |
Definition at line 24 of file tokenizer.cpp.
References c, char_types_, in_, next_char_fast(), buffered_istream::stream(), TOK_ALPHA, TOK_NUMERIC, and TOK_SPACE.
tokenizer::~tokenizer | ( | ) |
Definition at line 49 of file tokenizer.cpp.
References in_, and buffered_istream::stream().
|
private |
|
inlineprivate |
Definition at line 143 of file tokenizer.hpp.
References c, and char_types_.
Referenced by is_alnum(), is_num(), and is_space().
|
inline |
Definition at line 66 of file tokenizer.hpp.
References token_.
|
inline |
Definition at line 83 of file tokenizer.hpp.
References file_.
|
inline |
Definition at line 88 of file tokenizer.hpp.
References startlineno_.
|
inlineprivate |
Definition at line 158 of file tokenizer.hpp.
References char_type(), TOK_ALPHA, TOK_NONE, and TOK_NUMERIC.
Referenced by next_token().
|
inlineprivate |
Definition at line 153 of file tokenizer.hpp.
References char_type(), and TOK_NUMERIC.
Referenced by skip_comment().
|
inlineprivate |
Definition at line 148 of file tokenizer.hpp.
References char_type(), and TOK_SPACE.
Referenced by next_token(), skip_command(), and skip_comment().
|
inlineprivate |
Definition at line 99 of file tokenizer.hpp.
References lineno_, next_char_fast(), and UNLIKELY.
Referenced by next_token().
|
inlineprivate |
Definition at line 106 of file tokenizer.hpp.
References buffered_istream::get(), in_, LIKELY, and UNLIKELY.
Referenced by next_char(), next_token(), skip_command(), skip_comment(), and tokenizer().
const token & tokenizer::next_token | ( | ) |
Definition at line 55 of file tokenizer.cpp.
References current_, token::END, is_alnum(), is_space(), lineno_, token::MISC, next_char(), next_char_fast(), peek_char(), token::QSTRING, skip_comment(), startlineno_, token::STRING, token_, token::type, token::UNTERMINATED_QSTRING, and token::value.
|
inlineprivate |
Definition at line 130 of file tokenizer.hpp.
References in_, and buffered_istream::peek().
Referenced by next_token().
|
private |
Returns true if the next characters are the one from cmd followed by a space.
Skips all the matching characters.
Definition at line 166 of file tokenizer.cpp.
References current_, is_space(), and next_char_fast().
Referenced by skip_comment().
|
private |
Definition at line 178 of file tokenizer.cpp.
References current_, file_, is_num(), is_space(), lineno_, next_char_fast(), skip_command(), and textdomain_.
Referenced by next_token().
|
inline |
Definition at line 78 of file tokenizer.hpp.
References textdomain_.
|
private |
Definition at line 178 of file tokenizer.hpp.
Referenced by char_type(), and tokenizer().
|
private |
Definition at line 95 of file tokenizer.hpp.
Referenced by next_token(), skip_command(), and skip_comment().
|
private |
Definition at line 172 of file tokenizer.hpp.
Referenced by get_file(), and skip_comment().
|
private |
Definition at line 177 of file tokenizer.hpp.
Referenced by next_char_fast(), peek_char(), tokenizer(), and ~tokenizer().
|
private |
Definition at line 96 of file tokenizer.hpp.
Referenced by next_char(), next_token(), and skip_comment().
|
private |
Definition at line 97 of file tokenizer.hpp.
Referenced by get_start_line(), and next_token().
|
private |
Definition at line 171 of file tokenizer.hpp.
Referenced by skip_comment(), and textdomain().
|
private |
Definition at line 173 of file tokenizer.hpp.
Referenced by current_token(), and next_token().