google::protobuf::io::Tokenizer Class Reference

#include <tokenizer.h>

Classes
struct	Token

Public Types
enum	TokenType {   TYPE_START, TYPE_END, TYPE_IDENTIFIER, TYPE_INTEGER,   TYPE_FLOAT, TYPE_STRING, TYPE_SYMBOL }
enum	CommentStyle { CPP_COMMENT_STYLE, SH_COMMENT_STYLE }

Public Member Functions
	Tokenizer (ZeroCopyInputStream *input, ErrorCollector *error_collector)
	~Tokenizer ()
const Token &	current ()
const Token &	previous ()
bool	Next ()
bool	NextWithComments (string *prev_trailing_comments, vector< string > *detached_comments, string *next_leading_comments)
void	set_allow_f_after_float (bool value)
void	set_comment_style (CommentStyle style)
void	set_require_space_after_number (bool require)
void	set_allow_multiline_strings (bool allow)

Static Public Member Functions
static double	ParseFloat (const string &text)
static void	ParseString (const string &text, string *output)
static void	ParseStringAppend (const string &text, string *output)
static bool	ParseInteger (const string &text, uint64 max_value, uint64 *output)
static bool	IsIdentifier (const string &text)

Private Types
enum	NextCommentStatus { LINE_COMMENT, BLOCK_COMMENT, SLASH_NOT_COMMENT, NO_COMMENT }

Private Member Functions
	GOOGLE_DISALLOW_EVIL_CONSTRUCTORS (Tokenizer)
void	NextChar ()
void	Refresh ()
void	RecordTo (string *target)
void	StopRecording ()
void	StartToken ()
void	EndToken ()
void	AddError (const string &message)
void	ConsumeString (char delimiter)
TokenType	ConsumeNumber (bool started_with_zero, bool started_with_dot)
void	ConsumeLineComment (string *content)
void	ConsumeBlockComment (string *content)
NextCommentStatus	TryConsumeCommentStart ()
template<typename CharacterClass >
bool	LookingAt ()
template<typename CharacterClass >
bool	TryConsumeOne ()
bool	TryConsume (char c)
template<typename CharacterClass >
void	ConsumeZeroOrMore ()
template<typename CharacterClass >
void	ConsumeOneOrMore (const char *error)

Private Attributes
Token	current_
Token	previous_
ZeroCopyInputStream *	input_
ErrorCollector *	error_collector_
char	current_char_
const char *	buffer_
int	buffer_size_
int	buffer_pos_
bool	read_error_
int	line_
int	column_
string *	record_target_
int	record_start_
bool	allow_f_after_float_
CommentStyle	comment_style_
bool	require_space_after_number_
bool	allow_multiline_strings_

Static Private Attributes
static const int	kTabWidth = 8

Member Enumeration Documentation

enum google::protobuf::io::Tokenizer::CommentStyle

Enumerator
CPP_COMMENT_STYLE
SH_COMMENT_STYLE

 {
 // Line comments begin with "//", block comments are delimited by "/*" and
 // "*/".
 CPP_COMMENT_STYLE,
 // Line comments begin with "#". No way to write block comments.
 SH_COMMENT_STYLE
 };

enum google::protobuf::io::Tokenizer::NextCommentStatus

private

Enumerator
LINE_COMMENT
BLOCK_COMMENT
SLASH_NOT_COMMENT
NO_COMMENT

 {
 // Started a line comment.
 LINE_COMMENT,

 // Started a block comment.
 BLOCK_COMMENT,

 // Consumed a slash, then realized it wasn't a comment. current_ has
 // been filled in with a slash token. The caller should return it.
 SLASH_NOT_COMMENT,

 // We do not appear to be starting a comment here.
 NO_COMMENT
 };

enum google::protobuf::io::Tokenizer::TokenType

Enumerator
TYPE_START
TYPE_END
TYPE_IDENTIFIER
TYPE_INTEGER
TYPE_FLOAT
TYPE_STRING
TYPE_SYMBOL

 {
 TYPE_START, // Next() has not yet been called.
 TYPE_END, // End of input reached. "text" is empty.

 TYPE_IDENTIFIER, // A sequence of letters, digits, and underscores, not
 // starting with a digit. It is an error for a number
 // to be followed by an identifier with no space in
 // between.
 TYPE_INTEGER, // A sequence of digits representing an integer. Normally
 // the digits are decimal, but a prefix of "0x" indicates
 // a hex number and a leading zero indicates octal, just
 // like with C numeric literals. A leading negative sign
 // is NOT included in the token; it's up to the parser to
 // interpret the unary minus operator on its own.
 TYPE_FLOAT, // A floating point literal, with a fractional part and/or
 // an exponent. Always in decimal. Again, never
 // negative.
 TYPE_STRING, // A quoted sequence of escaped characters. Either single
 // or double quotes can be used, but they must match.
 // A string literal cannot cross a line break.
 TYPE_SYMBOL, // Any other printable character, like '!' or '+'.
 // Symbols are always a single character, so "!+$%" is
 // four tokens.
 };

Constructor & Destructor Documentation

google::protobuf::io::Tokenizer::Tokenizer	(	ZeroCopyInputStream *	input,
		ErrorCollector *	error_collector
	)

google::protobuf::io::Tokenizer::~Tokenizer

(

)

Member Function Documentation

void google::protobuf::io::Tokenizer::AddError ( const string &  message )

inlineprivate

 {
 error_collector_->AddError(line_, column_, message);
 }

Here is the call graph for this function:

void google::protobuf::io::Tokenizer::ConsumeBlockComment ( string *  content )

private

void google::protobuf::io::Tokenizer::ConsumeLineComment ( string *  content )

private

TokenType google::protobuf::io::Tokenizer::ConsumeNumber ( bool  started_with_zero, bool  started_with_dot  )

private

template<typename CharacterClass >

void google::protobuf::io::Tokenizer::ConsumeOneOrMore ( const char *  error )

inlineprivate

void google::protobuf::io::Tokenizer::ConsumeString ( char  delimiter )

private

template<typename CharacterClass >

void google::protobuf::io::Tokenizer::ConsumeZeroOrMore ( )

inlineprivate

const Tokenizer::Token & Tokenizer::current ( )

inline

 {
 return current_;
}

void google::protobuf::io::Tokenizer::EndToken ( )

inlineprivate

google::protobuf::io::Tokenizer::GOOGLE_DISALLOW_EVIL_CONSTRUCTORS ( Tokenizer  )

private

static bool google::protobuf::io::Tokenizer::IsIdentifier ( const string &  text )

static

template<typename CharacterClass >

bool google::protobuf::io::Tokenizer::LookingAt ( )

inlineprivate

bool google::protobuf::io::Tokenizer::Next

(

)

void google::protobuf::io::Tokenizer::NextChar ( )

private

bool google::protobuf::io::Tokenizer::NextWithComments	(	string *	prev_trailing_comments,
		vector< string > *	detached_comments,
		string *	next_leading_comments
	)

static double google::protobuf::io::Tokenizer::ParseFloat ( const string &  text )

static

static bool google::protobuf::io::Tokenizer::ParseInteger ( const string &  text, uint64  max_value, uint64 *  output  )

static

void Tokenizer::ParseString ( const string &  text, string *  output  )

inlinestatic

 {
 output->clear();
 ParseStringAppend(text, output);
}

Here is the call graph for this function:

static void google::protobuf::io::Tokenizer::ParseStringAppend ( const string &  text, string *  output  )

static

Here is the caller graph for this function:

const Tokenizer::Token & Tokenizer::previous ( )

inline

 {
 return previous_;
}

void google::protobuf::io::Tokenizer::RecordTo ( string *  target )

inlineprivate

void google::protobuf::io::Tokenizer::Refresh ( )

private

void google::protobuf::io::Tokenizer::set_allow_f_after_float ( bool  value )

inline

{ allow_f_after_float_ = value; }

void google::protobuf::io::Tokenizer::set_allow_multiline_strings ( bool  allow )

inline

 {
 allow_multiline_strings_ = allow;
 }

void google::protobuf::io::Tokenizer::set_comment_style ( CommentStyle  style )

inline

{ comment_style_ = style; }

void google::protobuf::io::Tokenizer::set_require_space_after_number ( bool  require )

inline

 {
 require_space_after_number_ = require;
 }

void google::protobuf::io::Tokenizer::StartToken ( )

inlineprivate

void google::protobuf::io::Tokenizer::StopRecording ( )

inlineprivate

bool google::protobuf::io::Tokenizer::TryConsume ( char  c )

inlineprivate

NextCommentStatus google::protobuf::io::Tokenizer::TryConsumeCommentStart ( )

private

template<typename CharacterClass >

bool google::protobuf::io::Tokenizer::TryConsumeOne ( )

inlineprivate

Member Data Documentation

bool google::protobuf::io::Tokenizer::allow_f_after_float_

private

bool google::protobuf::io::Tokenizer::allow_multiline_strings_

private

const char* google::protobuf::io::Tokenizer::buffer_

private

int google::protobuf::io::Tokenizer::buffer_pos_

private

int google::protobuf::io::Tokenizer::buffer_size_

private

int google::protobuf::io::Tokenizer::column_

private

CommentStyle google::protobuf::io::Tokenizer::comment_style_

private

Token google::protobuf::io::Tokenizer::current_

private

char google::protobuf::io::Tokenizer::current_char_

private

ErrorCollector* google::protobuf::io::Tokenizer::error_collector_

private

ZeroCopyInputStream* google::protobuf::io::Tokenizer::input_

private

const int google::protobuf::io::Tokenizer::kTabWidth = 8

staticprivate

int google::protobuf::io::Tokenizer::line_

private

Token google::protobuf::io::Tokenizer::previous_

private

bool google::protobuf::io::Tokenizer::read_error_

private

int google::protobuf::io::Tokenizer::record_start_

private

string* google::protobuf::io::Tokenizer::record_target_

private

bool google::protobuf::io::Tokenizer::require_space_after_number_

private

---

The documentation for this class was generated from the following file:

• dep/protobuf/src/google/protobuf/io/tokenizer.h