The definition of the CRTknzr (tokenizer) class. More...

#include "string.h"
#include "cr-tknzr.h"
#include "cr-doc-handler.h"

Data Structures
struct	_CRTknzrPriv

Macros
#define	PRIVATE(obj) ((obj)->priv)

#define	IS_NUM(a_char) (((a_char) >= '0' && (a_char) <= '9')?TRUE:FALSE)
	return TRUE if the character is a number ([0-9]), FALSE otherwise More...

#define	CHECK_PARSING_STATUS(status, is_exception)
	Checks if 'status' equals CR_OK. More...

#define	PEEK_NEXT_CHAR(a_tknzr, a_to_char)
	Peeks the next char from the input stream of the current tokenizer. More...

#define	READ_NEXT_CHAR(a_tknzr, to_char)
	Reads the next char from the input stream of the current parser. More...

#define	RECORD_INITIAL_POS(a_tknzr, a_pos)
	Gets information about the current position in the input of the parser. More...

#define	RECORD_CUR_BYTE_ADDR(a_tknzr, a_addr)
	Gets the address of the current byte inside the parser input. More...

#define	PEEK_BYTE(a_tknzr, a_offset, a_byte_ptr)
	Peeks a byte from the topmost parser input at a given offset from the current position. More...

#define	BYTE(a_input, a_n, a_eof) cr_input_peek_byte2 (a_input, a_n, a_eof)

#define	READ_NEXT_BYTE(a_tknzr, a_byte_ptr)
	Reads a byte from the topmost parser input steam. More...

#define	SKIP_BYTES(a_tknzr, a_nb_bytes)
	Skips a given number of byte in the topmost parser input. More...

#define	SKIP_CHARS(a_tknzr, a_nb_chars)
	Skip utf8 encoded characters. More...

#define	ENSURE_PARSING_COND(condition) if (! (condition)) {status = CR_PARSING_ERROR; goto error ;}
	Tests the condition and if it is false, sets status to "CR_PARSING_ERROR" and goto the 'error' label. More...

Functions

CRTknzr *	cr_tknzr_new (CRInput *a_input)

CRTknzr *	cr_tknzr_new_from_buf (guchar *a_buf, gulong a_len, enum CREncoding a_enc, gboolean a_free_at_destroy)

CRTknzr *	cr_tknzr_new_from_uri (const guchar *a_file_uri, enum CREncoding a_enc)

void	cr_tknzr_ref (CRTknzr *a_this)

gboolean	cr_tknzr_unref (CRTknzr *a_this)

enum CRStatus	cr_tknzr_set_input (CRTknzr a_this, CRInput a_input)

enum CRStatus	cr_tknzr_get_input (CRTknzr a_this, CRInput *a_input)

enum CRStatus	cr_tknzr_read_byte (CRTknzr a_this, guchar a_byte)
	Reads the next byte from the parser input stream. More...

enum CRStatus	cr_tknzr_read_char (CRTknzr a_this, guint32 a_char)
	Reads the next char from the parser input stream. More...

enum CRStatus	cr_tknzr_peek_char (CRTknzr a_this, guint32 a_char)
	Peeks a char from the parser input stream. More...

enum CRStatus	cr_tknzr_peek_byte (CRTknzr a_this, gulong a_offset, guchar a_byte)
	Peeks a byte ahead at a given postion in the parser input stream. More...

guchar	cr_tknzr_peek_byte2 (CRTknzr a_this, gulong a_offset, gboolean a_eof)
	Same as cr_tknzr_peek_byte() but this api returns the byte peeked. More...

glong	cr_tknzr_get_nb_bytes_left (CRTknzr *a_this)
	Gets the number of bytes left in the topmost input stream associated to this parser. More...

enum CRStatus	cr_tknzr_get_cur_pos (CRTknzr a_this, CRInputPos a_pos)

enum CRStatus	cr_tknzr_get_parsing_location (CRTknzr a_this, CRParsingLocation a_loc)

enum CRStatus	cr_tknzr_get_cur_byte_addr (CRTknzr a_this, guchar *a_addr)

enum CRStatus	cr_tknzr_seek_index (CRTknzr *a_this, enum CRSeekPos a_origin, gint a_pos)

enum CRStatus	cr_tknzr_consume_chars (CRTknzr a_this, guint32 a_char, glong a_nb_char)

enum CRStatus	cr_tknzr_set_cur_pos (CRTknzr a_this, CRInputPos a_pos)

enum CRStatus	cr_tknzr_unget_token (CRTknzr a_this, CRToken a_token)

enum CRStatus	cr_tknzr_get_next_token (CRTknzr a_this, CRToken *a_tk)
	Returns the next token of the input stream. More...

enum CRStatus	cr_tknzr_parse_token (CRTknzr *a_this, enum CRTokenType a_type, enum CRTokenExtraType a_et, gpointer a_res, gpointer a_extra_res)

void	cr_tknzr_destroy (CRTknzr *a_this)

Detailed Description

The definition of the CRTknzr (tokenizer) class.

Definition in file cr-tknzr.c.

Macro Definition Documentation

◆ BYTE

#define BYTE	(	a_input,
		a_n,
		a_eof
	)	cr_input_peek_byte2 (a_input, a_n, a_eof)

Definition at line 161 of file cr-tknzr.c.

◆ CHECK_PARSING_STATUS

#define CHECK_PARSING_STATUS	(	status,
		is_exception
	)

Value:

if ((status) != CR_OK) \
{ \
        if (is_exception == FALSE) \
        { \
                status = CR_PARSING_ERROR ; \
        } \
        goto error ; \
}

Checks if 'status' equals CR_OK.

If not, goto the 'error' label.

Parameters

status	the status (of type enum CRStatus) to test.
is_exception	if set to FALSE, the final status returned the current function will be CR_PARSING_ERROR. If set to TRUE, the current status will be the current value of the 'status' variable.

Definition at line 80 of file cr-tknzr.c.

◆ ENSURE_PARSING_COND

#define ENSURE_PARSING_COND ( condition ) if (! (condition)) {status = CR_PARSING_ERROR; goto error ;}

Tests the condition and if it is false, sets status to "CR_PARSING_ERROR" and goto the 'error' label.

Parameters

condition the condition to test.

Definition at line 210 of file cr-tknzr.c.

◆ IS_NUM

#define IS_NUM ( a_char ) (((a_char) >= '0' && (a_char) <= '9')?TRUE:FALSE)

return TRUE if the character is a number ([0-9]), FALSE otherwise

Parameters

a_char the char to test.

Definition at line 69 of file cr-tknzr.c.

◆ PEEK_BYTE

#define PEEK_BYTE	(	a_tknzr,
		a_offset,
		a_byte_ptr
	)

Value:

status = cr_tknzr_peek_byte (a_tknzr, \
                             a_offset, \
                             a_byte_ptr) ; \
CHECK_PARSING_STATUS (status, TRUE) ;

Peeks a byte from the topmost parser input at a given offset from the current position.

If it fails, goto the "error:" label.

Parameters

a_parser	the current instance of CRTknzr.
a_offset	the offset of the byte to peek, the current byte having the offset '0'.
a_byte_ptr	out parameter a pointer (guchar*) to where the peeked char is to be stored.

Definition at line 155 of file cr-tknzr.c.

◆ PEEK_NEXT_CHAR

#define PEEK_NEXT_CHAR	(	a_tknzr,
		a_to_char
	)

Value:

{\
status = cr_tknzr_peek_char  (a_tknzr, a_to_char) ; \
CHECK_PARSING_STATUS (status, TRUE) \
}

Peeks the next char from the input stream of the current tokenizer.

invokes CHECK_PARSING_STATUS on the status returned by cr_tknzr_input_peek_char().

Parameters

the	current instance of #CRTkzr.
to_char	a pointer to the char where to store the char peeked.

Definition at line 99 of file cr-tknzr.c.

◆ PRIVATE

#define PRIVATE ( obj ) ((obj)->priv)

Definition at line 63 of file cr-tknzr.c.

◆ READ_NEXT_BYTE

#define READ_NEXT_BYTE	(	a_tknzr,
		a_byte_ptr
	)

Value:

status = \
cr_input_read_byte (PRIVATE (a_tknzr)->input, a_byte_ptr) ;\
CHECK_PARSING_STATUS (status, TRUE) ;

Reads a byte from the topmost parser input steam.

If it fails, goto the "error" label.

Parameters

a_parser	the current instance of CRTknzr.
a_byte_ptr	the guchar * where to put the read char.

Definition at line 171 of file cr-tknzr.c.

◆ READ_NEXT_CHAR

#define READ_NEXT_CHAR	(	a_tknzr,
		to_char
	)

Value:

status = cr_tknzr_read_char (a_tknzr, to_char) ;\

CHECK_PARSING_STATUS (status, TRUE)

cr_tknzr_read_char

enum CRStatus cr_tknzr_read_char(CRTknzr *a_this, guint32 *a_char)

Reads the next char from the parser input stream.

Definition: cr-tknzr.c:1740

Reads the next char from the input stream of the current parser.

In case of error, jumps to the "error:" label located in the function where this macro is called.

Parameters

parser	the curent instance of CRTknzr
to_char	a pointer to the guint32 char where to store the character read.

Definition at line 113 of file cr-tknzr.c.

◆ RECORD_CUR_BYTE_ADDR

#define RECORD_CUR_BYTE_ADDR	(	a_tknzr,
		a_addr
	)

Value:

status = cr_input_get_cur_byte_addr \
            (PRIVATE (a_tknzr)->input, a_addr) ; \
CHECK_PARSING_STATUS (status, TRUE)

Gets the address of the current byte inside the parser input.

Parameters

parser	the current instance of CRTknzr.
addr	out parameter a pointer (guchar*) to where the address must be put.

Definition at line 139 of file cr-tknzr.c.

◆ RECORD_INITIAL_POS

#define RECORD_INITIAL_POS	(	a_tknzr,
		a_pos
	)

Value:

status = cr_input_get_cur_pos (PRIVATE  \
(a_tknzr)->input, a_pos) ; \
g_return_val_if_fail (status == CR_OK, status)

Gets information about the current position in the input of the parser.

In case of failure, this macro returns from the calling function and returns a status code of type enum CRStatus.

Parameters

parser	the current instance of CRTknzr.
pos	out parameter. A pointer to the position inside the current parser input. Must

Definition at line 127 of file cr-tknzr.c.

◆ SKIP_BYTES

#define SKIP_BYTES	(	a_tknzr,
		a_nb_bytes
	)

Value:

status = cr_input_seek_index (PRIVATE (a_tknzr)->input, \
                                     CR_SEEK_CUR, a_nb_bytes) ; \
CHECK_PARSING_STATUS (status, TRUE) ;

Skips a given number of byte in the topmost parser input.

Don't update line and column number. In case of error, jumps to the "error:" label of the surrounding function.

Parameters

a_parser	the current instance of CRTknzr.
a_nb_bytes	the number of bytes to skip.

Definition at line 184 of file cr-tknzr.c.

◆ SKIP_CHARS

#define SKIP_CHARS	(	a_tknzr,
		a_nb_chars
	)

Value:

{ \
gulong nb_chars = a_nb_chars ; \
status = cr_input_consume_chars \
     (PRIVATE (a_tknzr)->input,0, &nb_chars) ; \
CHECK_PARSING_STATUS (status, TRUE) ; \
}

Skip utf8 encoded characters.

Updates line and column numbers.

Parameters

a_parser	the current instance of CRTknzr.
a_nb_chars	the number of chars to skip. Must be of type glong.

Definition at line 196 of file cr-tknzr.c.

Function Documentation

◆ cr_tknzr_consume_chars()

enum CRStatus cr_tknzr_consume_chars	(	CRTknzr *	a_this,
		guint32	a_char,
		glong *	a_nb_char
	)

Definition at line 1834 of file cr-tknzr.c.

References CR_BAD_PARAM_ERROR, cr_input_get_nb_bytes_left(), cr_input_set_cur_pos(), cr_token_destroy(), and PRIVATE.

◆ cr_tknzr_destroy()

void cr_tknzr_destroy ( CRTknzr * a_this )

Definition at line 2740 of file cr-tknzr.c.

References cr_input_unref(), cr_token_destroy(), and PRIVATE.

Referenced by cr_tknzr_get_input().

◆ cr_tknzr_get_cur_byte_addr()

enum CRStatus cr_tknzr_get_cur_byte_addr	(	CRTknzr *	a_this,
		guchar **	a_addr
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_get_cur_pos()

enum CRStatus cr_tknzr_get_cur_pos	(	CRTknzr *	a_this,
		CRInputPos *	a_pos
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_get_input()

enum CRStatus cr_tknzr_get_input	(	CRTknzr *	a_this,
		CRInput **	a_input
	)

Definition at line 1668 of file cr-tknzr.c.

References cr_tknzr_destroy(), and PRIVATE.

◆ cr_tknzr_get_nb_bytes_left()

glong cr_tknzr_get_nb_bytes_left ( CRTknzr * a_this )

Gets the number of bytes left in the topmost input stream associated to this parser.

Parameters

a_this the current instance of CRTknzr

Returns: the number of bytes left or -1 in case of error.

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_get_next_token()

enum CRStatus cr_tknzr_get_next_token	(	CRTknzr *	a_this,
		CRToken **	a_tk
	)

Returns the next token of the input stream.

This method is really central. Each parsing method calls it.

Parameters

a_this	the current tokenizer.
a_tk	out parameter. The returned token. for the sake of mem leak avoidance, *a_tk must be NULL.
CR_OK	upon successfull completion, an error code otherwise.

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_get_parsing_location()

enum CRStatus cr_tknzr_get_parsing_location	(	CRTknzr *	a_this,
		CRParsingLocation *	a_loc
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_new()

CRTknzr* cr_tknzr_new ( CRInput * a_input )

Definition at line 1595 of file cr-tknzr.c.

References cr_tknzr_set_input(), and cr_utils_trace_info.

Referenced by cr_parser_new_from_input(), cr_tknzr_new_from_buf(), and cr_tknzr_new_from_uri().

◆ cr_tknzr_new_from_buf()

CRTknzr* cr_tknzr_new_from_buf	(	guchar *	a_buf,
		gulong	a_len,
		enum CREncoding	a_enc,
		gboolean	a_free_at_destroy
	)

Definition at line 1627 of file cr-tknzr.c.

References cr_input_new_from_buf(), and cr_tknzr_new().

◆ cr_tknzr_new_from_uri()

CRTknzr* cr_tknzr_new_from_uri	(	const guchar *	a_file_uri,
		enum CREncoding	a_enc
	)

Definition at line 1645 of file cr-tknzr.c.

References cr_input_new_from_uri(), and cr_tknzr_new().

Referenced by cr_parser_get_parsing_location().

◆ cr_tknzr_parse_token()

enum CRStatus cr_tknzr_parse_token	(	CRTknzr *	a_this,
		enum CRTokenType	a_type,
		enum CRTokenExtraType	a_et,
		gpointer	a_res,
		gpointer	a_extra_res
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_peek_byte()

enum CRStatus cr_tknzr_peek_byte	(	CRTknzr *	a_this,
		gulong	a_offset,
		guchar *	a_byte
	)

Peeks a byte ahead at a given postion in the parser input stream.

Parameters

a_this	the current instance of CRTknzr.
a_offset	the offset of the peeked byte starting from the current byte in the parser input stream.
a_byte	out parameter. The peeked byte upon successfull completion.

Returns: CR_OK upon successfull completion, an error code otherwise.

Definition at line 1668 of file cr-tknzr.c.

◆ cr_tknzr_peek_byte2()

guchar cr_tknzr_peek_byte2	(	CRTknzr *	a_this,
		gulong	a_offset,
		gboolean *	a_eof
	)

Same as cr_tknzr_peek_byte() but this api returns the byte peeked.

Parameters

a_this	the current instance of CRTknzr.
a_offset	the offset of the peeked byte starting from the current byte in the parser input stream.
a_eof	out parameter. If not NULL, is set to TRUE if we reached end of file, FALE otherwise. If the caller sets it to NULL, this parameter is just ignored.

Returns: the peeked byte.

Definition at line 1819 of file cr-tknzr.c.

References cr_input_peek_byte2(), and PRIVATE.

◆ cr_tknzr_peek_char()

enum CRStatus cr_tknzr_peek_char	(	CRTknzr *	a_this,
		guint32 *	a_char
	)

Peeks a char from the parser input stream.

To "peek a char" means reads the next char without consuming it. Subsequent calls to this function return the same char.

Parameters

a_this	the current instance of CRTknzr.
a_char	out parameter. The peeked char uppon successfull completion.

Returns: CR_OK upon successfull completion, an error code otherwise.

Definition at line 1668 of file cr-tknzr.c.

◆ cr_tknzr_read_byte()

enum CRStatus cr_tknzr_read_byte	(	CRTknzr *	a_this,
		guchar *	a_byte
	)

Reads the next byte from the parser input stream.

Parameters

a_this	the "this pointer" of the current instance of CRParser.
a_byte	out parameter the place where to store the byte read.

Returns: CR_OK upon successfull completion, an error code otherwise.

Definition at line 1668 of file cr-tknzr.c.

◆ cr_tknzr_read_char()

enum CRStatus cr_tknzr_read_char	(	CRTknzr *	a_this,
		guint32 *	a_char
	)

Reads the next char from the parser input stream.

Parameters

a_this	the current instance of CRTknzr.
a_char	out parameter. The read char.

Returns: CR_OK upon successfull completion, an error code otherwise.

Definition at line 1668 of file cr-tknzr.c.

◆ cr_tknzr_ref()

void cr_tknzr_ref ( CRTknzr * a_this )

Definition at line 1660 of file cr-tknzr.c.

References PRIVATE.

◆ cr_tknzr_seek_index()

enum CRStatus cr_tknzr_seek_index	(	CRTknzr *	a_this,
		enum CRSeekPos	a_origin,
		gint	a_pos
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_set_cur_pos()

enum CRStatus cr_tknzr_set_cur_pos	(	CRTknzr *	a_this,
		CRInputPos *	a_pos
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_set_input()

enum CRStatus cr_tknzr_set_input	(	CRTknzr *	a_this,
		CRInput *	a_input
	)

Definition at line 1668 of file cr-tknzr.c.

Referenced by cr_tknzr_new().

◆ cr_tknzr_unget_token()

enum CRStatus cr_tknzr_unget_token	(	CRTknzr *	a_this,
		CRToken *	a_token
	)

Definition at line 1834 of file cr-tknzr.c.

◆ cr_tknzr_unref()

gboolean cr_tknzr_unref ( CRTknzr * a_this )

Definition at line 1668 of file cr-tknzr.c.

Referenced by cr_parser_destroy().

Data Structures

Macros

Functions

Detailed Description

Macro Definition Documentation

◆ BYTE

◆ CHECK_PARSING_STATUS

◆ ENSURE_PARSING_COND

◆ IS_NUM

◆ PEEK_BYTE

◆ PEEK_NEXT_CHAR

◆ PRIVATE

◆ READ_NEXT_BYTE

◆ READ_NEXT_CHAR

◆ RECORD_CUR_BYTE_ADDR

◆ RECORD_INITIAL_POS

◆ SKIP_BYTES

◆ SKIP_CHARS

Function Documentation

◆ cr_tknzr_consume_chars()

◆ cr_tknzr_destroy()

◆ cr_tknzr_get_cur_byte_addr()

◆ cr_tknzr_get_cur_pos()

◆ cr_tknzr_get_input()

◆ cr_tknzr_get_nb_bytes_left()

◆ cr_tknzr_get_next_token()

◆ cr_tknzr_get_parsing_location()

◆ cr_tknzr_new()

◆ cr_tknzr_new_from_buf()

◆ cr_tknzr_new_from_uri()

◆ cr_tknzr_parse_token()

◆ cr_tknzr_peek_byte()

◆ cr_tknzr_peek_byte2()

◆ cr_tknzr_peek_char()

◆ cr_tknzr_read_byte()

◆ cr_tknzr_read_char()

◆ cr_tknzr_ref()

◆ cr_tknzr_seek_index()

◆ cr_tknzr_set_cur_pos()

◆ cr_tknzr_set_input()

◆ cr_tknzr_unget_token()

◆ cr_tknzr_unref()