4 * Copyright (c) Tuomo Valkonen 1999-2002.
6 * You may distribute and modify this library under the terms of either
7 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
10 #ifndef LIBTU_TOKENIZER_H
11 #define LIBTU_TOKENIZER_H
17 #define TOK_SET_BOOL(TOK, VAL) {(TOK)->type=TOK_BOOL; (TOK)->u.bval=VAL;}
18 #define TOK_SET_LONG(TOK, VAL) {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
19 #define TOK_SET_DOUBLE(TOK, VAL) {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
20 #define TOK_SET_CHAR(TOK, VAL) {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
21 #define TOK_SET_STRING(TOK, VAL) {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
22 #define TOK_SET_IDENT(TOK, VAL) {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
23 #define TOK_SET_COMMENT(TOK, VAL) {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
24 #define TOK_SET_OP(TOK, VAL) {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}
26 #define TOK_TYPE(TOK) ((TOK)->type)
27 #define TOK_BOOL_VAL(TOK) ((TOK)->u.bval)
28 #define TOK_LONG_VAL(TOK) ((TOK)->u.lval)
29 #define TOK_DOUBLE_VAL(TOK) ((TOK)->u.dval)
30 #define TOK_CHAR_VAL(TOK) ((TOK)->u.cval)
31 #define TOK_STRING_VAL(TOK) ((TOK)->u.sval)
32 #define TOK_IDENT_VAL(TOK) ((TOK)->u.sval)
33 #define TOK_COMMENT_VAL(TOK) ((TOK)->u.sval)
34 #define TOK_OP_VAL(TOK) ((TOK)->u.opval)
36 #define TOK_IS_INVALID(TOK) ((TOK)->type==TOK_INVALID)
37 #define TOK_IS_BOOL(TOK) ((TOK)->type==TOK_BOOL)
38 #define TOK_IS_LONG(TOK) ((TOK)->type==TOK_LONG)
39 #define TOK_IS_DOUBLE(TOK) ((TOK)->type==TOK_DOUBLE)
40 #define TOK_IS_CHAR(TOK) ((TOK)->type==TOK_CHAR)
41 #define TOK_IS_STRING(TOK) ((TOK)->type==TOK_STRING)
42 #define TOK_IS_IDENT(TOK) ((TOK)->type==TOK_IDENT)
43 #define TOK_IS_COMMENT(TOK) ((TOK)->type==TOK_COMMENT)
44 #define TOK_IS_OP(TOK) ((TOK)->type==TOK_OP)
46 #define TOK_OP_IS(TOK, OP) ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))
48 #define TOK_TAKE_STRING_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
49 #define TOK_TAKE_IDENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
50 #define TOK_TAKE_COMMENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
67 #define OP2(X,Y) ((X)|((Y)<<8))
68 #define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))
70 OP_L_PAR= '(', OP_R_PAR= ')', OP_L_BRK= '[', OP_R_BRK= ']',
71 OP_L_BRC= '{', OP_R_BRC= '}', OP_COMMA= ',', OP_SCOLON= ';',
73 OP_PLUS= '+', OP_MINUS= '-', OP_MUL= '*', OP_DIV= '/',
74 OP_MOD= '%', OP_POW= '^', OP_OR= '|', OP_AND= '&',
75 /*OP_NOT= '~',*/ OP_NOT= '!', OP_ASGN= '=', OP_LT= '<',
76 OP_GT= '>', OP_DOT= '.', OP_COLON= ':', OP_QMARK= '?',
78 OP_NEXTLINE='\n',OP_EOF= -1,
80 OP_INC= OP2('+','+'), OP_DEC= OP2('-','-'),
81 OP_LSHIFT= OP2('<','<'), OP_RSHIFT= OP2('>','>'),
82 OP_AS_INC= OP2('+','='), OP_AS_DEC= OP2('-','='),
83 OP_AS_MUL= OP2('*','='), OP_AS_DIV= OP2('/','='),
84 OP_AS_MOD= OP2('%','='), OP_AS_POW= OP2('^','='),
86 /* AS_OR= OP2('|','='), AS_AND= OP2('&','='), */
87 OP_EQ= OP2('=','='), OP_NE= OP2('!','='),
88 OP_LE= OP2('<','='), OP_GE= OP2('>','=')
90 /* L_AND= OP2('&','&'), L_OR= OP2('|','|'),
91 L_XOR= OP2('^','^'), */
93 /* AsLShift= OP3('<','<','='),
94 AsRShift= OP3('>','>','='), */
114 #define TOK_INIT {0, 0, {0}}
117 extern void tok_free(Token*tok);
118 extern void tok_init(Token*tok);
125 TOKZ_IGNORE_NEXTLINE=0x1,
126 TOKZ_READ_COMMENTS=0x2,
127 TOKZ_PARSER_INDENT_MODE=0x04,
128 TOKZ_ERROR_TOLERANT=0x8,
129 TOKZ_READ_FROM_BUFFER=0x10,
130 TOKZ_DEFAULT_OPTION=0x20
135 E_TOKZ_UNEXPECTED_EOF=1,
136 E_TOKZ_UNEXPECTED_EOL,
147 E_TOKZ_UNKNOWN_OPTION,
149 E_TOKZ_INVALID_ARGUMENT,
152 E_TOKZ_TOO_MANY_ARGS,
154 E_TOKZ_IDENTIFIER_EXPECTED,
156 E_TOKZ_LBRACE_EXPECTED
162 typedef struct _Tokenizer_FInfo{
170 typedef struct _Tokenizer_Buffer{
176 typedef struct _Tokenizer{
183 Tokenizer_Buffer buffer;
186 const struct _ConfOpt **optstack;
191 Tokenizer_FInfo *filestack;
197 extern Tokenizer *tokz_open(const char *fname);
198 extern Tokenizer *tokz_open_file(FILE *file, const char *fname);
199 extern Tokenizer *tokz_prepare_buffer(char *buffer, int len);
200 extern void tokz_close(Tokenizer *tokz);
201 extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
202 extern void tokz_unget_token(Tokenizer *tokz, Token *tok);
203 extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
204 extern void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...);
206 extern bool tokz_pushf(Tokenizer *tokz, const char *fname);
207 extern bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname);
208 extern bool tokz_popf(Tokenizer *tokz);
210 extern void tokz_set_includepaths(Tokenizer *tokz, char **paths);
212 #endif /* LIBTU_TOKENIZER_H */