From 87cb2d172843e114a8640de3fde61db3a2bf0a6a Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Thu, 29 Oct 2009 01:34:28 -0400 Subject: Even more tokenizer work. The tokenizer is now able to tokenize all of POV-Ray 3.6's include files. It turns out that some #includes may be dynamic (ex. with #writes beforehand), so the tokenizer can't directly perform includes - the executor must do this. --- dimension/tokenize.h | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'dimension/tokenize.h') diff --git a/dimension/tokenize.h b/dimension/tokenize.h index a90073b..7a36232 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -21,19 +21,27 @@ typedef enum { /* Punctuation */ - DMNSN_T_LBRACE, /* { */ - DMNSN_T_RBRACE, /* } */ - DMNSN_T_LPAREN, /* ( */ - DMNSN_T_RPAREN, /* ) */ - DMNSN_T_LBRACKET, /* [ */ - DMNSN_T_RBRACKET, /* ] */ - DMNSN_T_LT, /* < */ - DMNSN_T_GT, /* > */ - DMNSN_T_PLUS, /* + */ - DMNSN_T_MINUS, /* - */ - DMNSN_T_STAR, /* * */ - DMNSN_T_SLASH, /* / */ - DMNSN_T_COMMA, /* , */ + DMNSN_T_LBRACE, /* { */ + DMNSN_T_RBRACE, /* } */ + DMNSN_T_LPAREN, /* ( */ + DMNSN_T_RPAREN, /* ) */ + DMNSN_T_LBRACKET, /* [ */ + DMNSN_T_RBRACKET, /* ] */ + DMNSN_T_LT, /* < */ + DMNSN_T_GT, /* > */ + DMNSN_T_PLUS, /* + */ + DMNSN_T_MINUS, /* - */ + DMNSN_T_STAR, /* * */ + DMNSN_T_SLASH, /* / */ + DMNSN_T_COMMA, /* , */ + DMNSN_T_EQUALS, /* = */ + DMNSN_T_SEMICOLON, /* ; */ + DMNSN_T_QUESTION, /* ? */ + DMNSN_T_COLON, /* : */ + DMNSN_T_AND, /* & */ + DMNSN_T_EXCLAMATION, /* ! */ + DMNSN_T_DOT, /* . */ + DMNSN_T_PIPE, /* | */ /* Numeric values */ DMNSN_T_INT, @@ -59,7 +67,7 @@ typedef enum { DMNSN_T_IF, DMNSN_T_IFDEF, DMNSN_T_IFNDEF, - DMNSN_T_INCLUDE, /* Only used internally */ + DMNSN_T_INCLUDE, DMNSN_T_LOCAL, DMNSN_T_MACRO, DMNSN_T_RANGE, -- cgit v1.2.3