utf8::internal Namespace Reference

Enumerations
enum	utf_error {   UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE,   OVERLONG_SEQUENCE, INVALID_CODE_POINT }

Functions
template<typename octet_type >
uint8_t	mask8 (octet_type oc)
template<typename u16_type >
uint16_t	mask16 (u16_type oc)
template<typename octet_type >
bool	is_trail (octet_type oc)
template<typename u16 >
bool	is_lead_surrogate (u16 cp)
template<typename u16 >
bool	is_trail_surrogate (u16 cp)
template<typename u16 >
bool	is_surrogate (u16 cp)
template<typename u32 >
bool	is_code_point_valid (u32 cp)
template<typename octet_iterator >
std::iterator_traits < octet_iterator > ::difference_type	sequence_length (octet_iterator lead_it)
template<typename octet_difference_type >
bool	is_overlong_sequence (uint32_t cp, octet_difference_type length)
template<typename octet_iterator >
utf_error	increase_safely (octet_iterator &it, octet_iterator end)
	Helper for get_sequence_x. More...
template<typename octet_iterator >
utf_error	get_sequence_1 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
	get_sequence_x functions decode utf-8 sequences of the length x More...
template<typename octet_iterator >
utf_error	get_sequence_2 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	get_sequence_3 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	get_sequence_4 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	validate_next (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	validate_next (octet_iterator &it, octet_iterator end)

Variables
const uint16_t	LEAD_SURROGATE_MIN = 0xd800u
const uint16_t	LEAD_SURROGATE_MAX = 0xdbffu
const uint16_t	TRAIL_SURROGATE_MIN = 0xdc00u
const uint16_t	TRAIL_SURROGATE_MAX = 0xdfffu
const uint16_t	LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10)
const uint32_t	SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
const uint32_t	CODE_POINT_MAX = 0x0010ffffu

Enumeration Type Documentation

enum utf8::internal::utf_error

Enumerator
UTF8_OK
NOT_ENOUGH_ROOM
INVALID_LEAD
INCOMPLETE_SEQUENCE
OVERLONG_SEQUENCE
INVALID_CODE_POINT

{UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};

Function Documentation

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_1	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point
	)

get_sequence_x functions decode utf-8 sequences of the length x

 {
 if (it == end)
 return NOT_ENOUGH_ROOM;

 code_point = utf8::internal::mask8(*it);

 return UTF8_OK;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_2	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point
	)

 {
 if (it == end) 
 return NOT_ENOUGH_ROOM;
 
 code_point = utf8::internal::mask8(*it);

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);

 return UTF8_OK;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_3	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point
	)

 {
 if (it == end)
 return NOT_ENOUGH_ROOM;
 
 code_point = utf8::internal::mask8(*it);

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point += (*it) & 0x3f;

 return UTF8_OK;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_4	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point
	)

 {
 if (it == end)
 return NOT_ENOUGH_ROOM;

 code_point = utf8::internal::mask8(*it);

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;

 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

 code_point += (*it) & 0x3f;

 return UTF8_OK;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::increase_safely	(	octet_iterator &	it,
		octet_iterator	end
	)

Helper for get_sequence_x.

 {
 if (++it == end)
 return NOT_ENOUGH_ROOM;

 if (!utf8::internal::is_trail(*it))
 return INCOMPLETE_SEQUENCE;
 
 return UTF8_OK;
 }

Here is the call graph for this function:

template<typename u32 >

bool utf8::internal::is_code_point_valid ( u32  cp )

inline

 {
 return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename u16 >

bool utf8::internal::is_lead_surrogate ( u16  cp )

inline

 {
 return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
 }

Here is the caller graph for this function:

template<typename octet_difference_type >

bool utf8::internal::is_overlong_sequence ( uint32_t  cp, octet_difference_type  length  )

inline

 {
 if (cp < 0x80) {
 if (length != 1) 
 return true;
 }
 else if (cp < 0x800) {
 if (length != 2) 
 return true;
 }
 else if (cp < 0x10000) {
 if (length != 3) 
 return true;
 }

 return false;
 }

Here is the caller graph for this function:

template<typename u16 >

bool utf8::internal::is_surrogate ( u16  cp )

inline

 {
 return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
 }

Here is the caller graph for this function:

template<typename octet_type >

bool utf8::internal::is_trail ( octet_type  oc )

inline

 {
 return ((utf8::internal::mask8(oc) >> 6) == 0x2);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename u16 >

bool utf8::internal::is_trail_surrogate ( u16  cp )

inline

 {
 return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
 }

Here is the caller graph for this function:

template<typename u16_type >

uint16_t utf8::internal::mask16 ( u16_type  oc )

inline

 {
 return static_cast<uint16_t>(0xffff & oc);
 }

Here is the caller graph for this function:

template<typename octet_type >

uint8_t utf8::internal::mask8 ( octet_type  oc )

inline

 {
 return static_cast<uint8_t>(0xff & oc);
 }

Here is the caller graph for this function:

template<typename octet_iterator >

std::iterator_traits<octet_iterator>::difference_type utf8::internal::sequence_length ( octet_iterator  lead_it )

inline

 {
 uint8_t lead = utf8::internal::mask8(*lead_it);
 if (lead < 0x80)
 return 1;
 else if ((lead >> 5) == 0x6)
 return 2;
 else if ((lead >> 4) == 0xe)
 return 3;
 else if ((lead >> 3) == 0x1e)
 return 4;
 else
 return 0;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::validate_next	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point
	)

 {
 // Save the original value of it so we can go back in case of failure
 // Of course, it does not make much sense with i.e. stream iterators
 octet_iterator original_it = it;

 uint32_t cp = 0;
 // Determine the sequence length based on the lead octet
 typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
 const octet_difference_type length = utf8::internal::sequence_length(it);

 // Get trail octets and calculate the code point
 utf_error err = UTF8_OK;
 switch (length) {
 case 0: 
 return INVALID_LEAD;
 case 1:
 err = utf8::internal::get_sequence_1(it, end, cp);
 break;
 case 2:
 err = utf8::internal::get_sequence_2(it, end, cp);
 break;
 case 3:
 err = utf8::internal::get_sequence_3(it, end, cp);
 break;
 case 4:
 err = utf8::internal::get_sequence_4(it, end, cp);
 break;
 }

 if (err == UTF8_OK) {
 // Decoding succeeded. Now, security checks...
 if (utf8::internal::is_code_point_valid(cp)) {
 if (!utf8::internal::is_overlong_sequence(cp, length)){
 // Passed! Return here.
 code_point = cp;
 ++it;
 return UTF8_OK;
 }
 else
 err = OVERLONG_SEQUENCE;
 }
 else 
 err = INVALID_CODE_POINT;
 }

 // Failure branch - restore the original value of the iterator
 it = original_it;
 return err;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename octet_iterator >

utf_error utf8::internal::validate_next ( octet_iterator &  it, octet_iterator  end  )

inline

 {
 uint32_t ignored;
 return utf8::internal::validate_next(it, end, ignored);
 }

Here is the call graph for this function:

Variable Documentation

const uint32_t utf8::internal::CODE_POINT_MAX = 0x0010ffffu

const uint16_t utf8::internal::LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10)

const uint16_t utf8::internal::LEAD_SURROGATE_MAX = 0xdbffu

const uint16_t utf8::internal::LEAD_SURROGATE_MIN = 0xd800u

const uint32_t utf8::internal::SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN

const uint16_t utf8::internal::TRAIL_SURROGATE_MAX = 0xdfffu

const uint16_t utf8::internal::TRAIL_SURROGATE_MIN = 0xdc00u