tokenizer Class Reference

Abstract baseclass for the tokenizer. More...

#include <tokenizer.hpp>

Public Member Functions
	tokenizer (std::istream &in)
	~tokenizer ()
const token &	next_token ()
const token &	current_token () const
const std::string &	textdomain () const
const std::string &	get_file () const
int	get_start_line () const

Private Types
enum	{ TOK_NONE = 0, TOK_SPACE = 1, TOK_NUMERIC = 2, TOK_ALPHA = 4 }

Private Member Functions
	tokenizer ()
void	next_char ()
void	next_char_fast ()
int	peek_char ()
int	char_type (unsigned c) const
bool	is_space (int c) const
bool	is_num (int c) const
bool	is_alnum (int c) const
void	skip_comment ()
bool	skip_command (char const *cmd)
	Returns true if the next characters are the one from cmd followed by a space. More...

Private Attributes
int	current_
int	lineno_
int	startlineno_
std::string	textdomain_
std::string	file_
token	token_
buffered_istream	in_
char	char_types_ [128]

Detailed Description

Abstract baseclass for the tokenizer.

Definition at line 58 of file tokenizer.hpp.

Member Enumeration Documentation

anonymous enum

private

Enumerator
TOK_NONE
TOK_SPACE
TOK_NUMERIC
TOK_ALPHA

Definition at line 135 of file tokenizer.hpp.

Constructor & Destructor Documentation

tokenizer::tokenizer

(

std::istream &

in

)

Definition at line 24 of file tokenizer.cpp.

References c, char_types_, in_, next_char_fast(), buffered_istream::stream(), TOK_ALPHA, TOK_NUMERIC, and TOK_SPACE.

tokenizer::~tokenizer

(

)

Definition at line 49 of file tokenizer.cpp.

References in_, and buffered_istream::stream().

tokenizer::tokenizer ( )

private

Member Function Documentation

int tokenizer::char_type ( unsigned  c ) const

inlineprivate

Definition at line 143 of file tokenizer.hpp.

References c, and char_types_.

Referenced by is_alnum(), is_num(), and is_space().

const token& tokenizer::current_token ( ) const

inline

Definition at line 66 of file tokenizer.hpp.

References token_.

const std::string& tokenizer::get_file ( ) const

inline

Definition at line 83 of file tokenizer.hpp.

References file_.

int tokenizer::get_start_line ( ) const

inline

Definition at line 88 of file tokenizer.hpp.

References startlineno_.

bool tokenizer::is_alnum ( int  c ) const

inlineprivate

Definition at line 158 of file tokenizer.hpp.

References char_type(), TOK_ALPHA, TOK_NONE, and TOK_NUMERIC.

Referenced by next_token().

bool tokenizer::is_num ( int  c ) const

inlineprivate

Definition at line 153 of file tokenizer.hpp.

References char_type(), and TOK_NUMERIC.

Referenced by skip_comment().

bool tokenizer::is_space ( int  c ) const

inlineprivate

Definition at line 148 of file tokenizer.hpp.

References char_type(), and TOK_SPACE.

Referenced by next_token(), skip_command(), and skip_comment().

void tokenizer::next_char ( )

inlineprivate

Definition at line 99 of file tokenizer.hpp.

References lineno_, next_char_fast(), and UNLIKELY.

Referenced by next_token().

void tokenizer::next_char_fast ( )

inlineprivate

Definition at line 106 of file tokenizer.hpp.

References buffered_istream::get(), in_, LIKELY, and UNLIKELY.

Referenced by next_char(), next_token(), skip_command(), skip_comment(), and tokenizer().

const token & tokenizer::next_token

(

)

Definition at line 55 of file tokenizer.cpp.

References current_, token::END, is_alnum(), is_space(), lineno_, token::MISC, next_char(), next_char_fast(), peek_char(), token::QSTRING, skip_comment(), startlineno_, token::STRING, token_, token::type, token::UNTERMINATED_QSTRING, and token::value.

int tokenizer::peek_char ( )

inlineprivate

Definition at line 130 of file tokenizer.hpp.

References in_, and buffered_istream::peek().

Referenced by next_token().

bool tokenizer::skip_command ( char const *  cmd )

private

Returns true if the next characters are the one from cmd followed by a space.

Skips all the matching characters.

Definition at line 166 of file tokenizer.cpp.

References current_, is_space(), and next_char_fast().

Referenced by skip_comment().

void tokenizer::skip_comment ( )

private

Definition at line 178 of file tokenizer.cpp.

References current_, file_, is_num(), is_space(), lineno_, next_char_fast(), skip_command(), and textdomain_.

Referenced by next_token().

const std::string& tokenizer::textdomain ( ) const

inline

Definition at line 78 of file tokenizer.hpp.

References textdomain_.

Member Data Documentation

char tokenizer::char_types_[128]

private

Definition at line 178 of file tokenizer.hpp.

Referenced by char_type(), and tokenizer().

int tokenizer::current_

private

Definition at line 95 of file tokenizer.hpp.

Referenced by next_token(), skip_command(), and skip_comment().

std::string tokenizer::file_

private

Definition at line 172 of file tokenizer.hpp.

Referenced by get_file(), and skip_comment().

buffered_istream tokenizer::in_

private

Definition at line 177 of file tokenizer.hpp.

Referenced by next_char_fast(), peek_char(), tokenizer(), and ~tokenizer().

int tokenizer::lineno_

private

Definition at line 96 of file tokenizer.hpp.

Referenced by next_char(), next_token(), and skip_comment().

int tokenizer::startlineno_

private

Definition at line 97 of file tokenizer.hpp.

Referenced by get_start_line(), and next_token().

std::string tokenizer::textdomain_

private

Definition at line 171 of file tokenizer.hpp.

Referenced by skip_comment(), and textdomain().

token tokenizer::token_

private

Definition at line 173 of file tokenizer.hpp.

Referenced by current_token(), and next_token().

---

The documentation for this class was generated from the following files:

• serialization/tokenizer.hpp
• serialization/tokenizer.cpp