struct tokenizer_state;
char *next_token(struct tokenizer_state *);
struct tokenizer_state *init_tokenizer(char *, char);
int tokenizer_error(struct tokenizer_state *);
void end_tokenizer(struct tokenizer_state *);
struct tokenizer_state;
char *next_token(struct tokenizer_state *);
struct tokenizer_state *init_tokenizer(char *, char);
int tokenizer_error(struct tokenizer_state *);
void end_tokenizer(struct tokenizer_state *);