diff options
Diffstat (limited to 'dimension/tokenize.c')
| -rw-r--r-- | dimension/tokenize.c | 191 |
1 files changed, 150 insertions, 41 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 84d5f5d..bee03b6 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -58,13 +58,21 @@ static void dmnsn_delete_token_buffer(dmnsn_token_buffer *tbuffer) { if (tbuffer) { + unsigned int i; + for (i = 0; i < dmnsn_array_size(tbuffer->buffered); ++i) { + dmnsn_buffered_token buffered; + dmnsn_array_get(tbuffer->buffered, i, &buffered); + free(buffered.lval.value); + } + dmnsn_delete_array(tbuffer->buffered); free(tbuffer); } } -int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, - const char *filename, void *yyscanner); +static int +dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, void *yyscanner); int dmnsn_ld_yyparse(const char *filename, void *yyscanner, dmnsn_symbol_table *symtable); @@ -123,6 +131,7 @@ dmnsn_declaration_buffer(int token, dmnsn_token_buffer *prev, /* Fake EOF */ buffered.type = DMNSN_T_EOF; + buffered.lval.value = NULL; dmnsn_array_push(tbuffer->buffered, &buffered); return tbuffer; @@ -164,6 +173,7 @@ dmnsn_undef_buffer(int token, dmnsn_token_buffer *prev, /* Fake EOF */ buffered.type = DMNSN_T_EOF; + buffered.lval.value = NULL; dmnsn_array_push(tbuffer->buffered, &buffered); return tbuffer; @@ -221,6 +231,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, /* Fake EOF */ buffered.type = DMNSN_T_EOF; + buffered.lval.value = NULL; dmnsn_array_push(cond_buffer->buffered, &buffered); dmnsn_yyset_extra(cond_buffer, yyscanner); @@ -254,8 +265,8 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, int nesting = 1; while (1) { /* Non-recursive call */ - buffered.type = dmnsn_yylex_impl(&buffered.lval, &buffered.lloc, - filename, yyscanner); + buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, + filename, yyscanner); if (buffered.type == DMNSN_T_EOF) { dmnsn_diagnostic(filename, buffered.lloc.first_line, @@ -291,12 +302,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, } if (cond) { - if (buffered.type == DMNSN_T_LEX_ERROR) { - dmnsn_delete_token_buffer(tbuffer); - return NULL; - } else { - dmnsn_array_push(tbuffer->buffered, &buffered); - } + dmnsn_array_push(tbuffer->buffered, &buffered); } else { free(buffered.lval.value); } @@ -305,50 +311,141 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, return tbuffer; } +static dmnsn_token_buffer * +dmnsn_while_buffer(int token, dmnsn_token_buffer *prev, + dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) +{ + dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev); + + /* Pretend to be an if */ + dmnsn_buffered_token buffered = { + .type = DMNSN_T_IF, + .lval = *lvalp, + .lloc = *llocp + }; + dmnsn_array_push(tbuffer->buffered, &buffered); + + /* Grab all the tokens belonging to the #while ... #end */ + int nesting = 1; + while (1) { + /* Non-recursive call */ + buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, + filename, yyscanner); + + if (buffered.type == DMNSN_T_EOF) { + dmnsn_diagnostic(filename, buffered.lloc.first_line, + buffered.lloc.first_column, + "syntax error, unexpected end-of-file"); + dmnsn_delete_token_buffer(tbuffer); + return NULL; + } + + dmnsn_array_push(tbuffer->buffered, &buffered); + + switch (buffered.type) { + case DMNSN_T_IF: + case DMNSN_T_IFDEF: + case DMNSN_T_IFNDEF: + case DMNSN_T_MACRO: + case DMNSN_T_SWITCH: + case DMNSN_T_WHILE: + ++nesting; + break; + + case DMNSN_T_END: + --nesting; + break; + + default: + break; + } + + if (nesting == 0) { + break; + } + } + + return tbuffer; +} + +int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, void *yyscanner); + +static int +dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, void *yyscanner) +{ + dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner); + + if (tbuffer) { + while (tbuffer && tbuffer->i >= dmnsn_array_size(tbuffer->buffered)) { + if (tbuffer->type == DMNSN_T_WHILE) { + tbuffer->i = 0; + } else { + if (dmnsn_array_size(tbuffer->buffered) == 0 + && tbuffer->prev && tbuffer->prev->type == DMNSN_T_WHILE) + { + dmnsn_yyset_extra(tbuffer->prev, yyscanner); + dmnsn_delete_token_buffer(tbuffer); + tbuffer = dmnsn_yyget_extra(yyscanner); + } + + dmnsn_yyset_extra(tbuffer->prev, yyscanner); + dmnsn_delete_token_buffer(tbuffer); + tbuffer = dmnsn_yyget_extra(yyscanner); + } + } + } + + int token; + + if (tbuffer) { + /* Return buffered tokens */ + dmnsn_buffered_token buffered; + + dmnsn_array_get(tbuffer->buffered, tbuffer->i, &buffered); + token = buffered.type; + + if (buffered.lval.value) { + lvalp->value = strdup(buffered.lval.value); + } else { + lvalp->value = NULL; + } + + *llocp = buffered.lloc; + ++tbuffer->i; + } else { + token = dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner); + } + + return token; +} + int dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, const char *filename, dmnsn_symbol_table *symtable, void *yyscanner) { /* * So... this is kind of ugly. POV-Ray's language directives are not parsable - * by reasonable bison grammar, since some require skipping arbitrary amounts - * of tokens, or repeatedly parsing tokens. Instead, they are implemented - * transparently by the lexer. The lexing function calls a separate parser - * to handle language directives as they arrise, and returns only non- - * directive tokens. + * by any reasonable bison grammar, since some require skipping arbitrary + * amounts of tokens, or repeatedly parsing tokens. Instead, they are + * implemented transparently by the lexer. The lexing function calls a + * separate parser to handle language directives as they arrise, and returns + * only non-directive tokens. * * Ideally we'd use a push parser for the language directives, but bison * doesn't support GLR push parsers. Instead, we buffer all the appropriate * tokens and call a pull parser, then discard the buffer and continue. */ - dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner); - - int token; while (1) { - if (tbuffer) { - /* Return buffered tokens */ - dmnsn_buffered_token buffered; - - if (tbuffer->i < dmnsn_array_size(tbuffer->buffered)) { - dmnsn_array_get(tbuffer->buffered, tbuffer->i, &buffered); - token = buffered.type; - *lvalp = buffered.lval; - *llocp = buffered.lloc; - ++tbuffer->i; + int token = dmnsn_yylex_wrapper(lvalp, llocp, filename, yyscanner); + dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner); - if (tbuffer->type == DMNSN_T_LEX_VERBATIM && tbuffer->i == 1) { - /* Don't double-process the first token */ - return token; - } - } else { - dmnsn_yyset_extra(tbuffer->prev, yyscanner); - dmnsn_delete_token_buffer(tbuffer); - tbuffer = dmnsn_yyget_extra(yyscanner); - continue; - } - } else { - token = dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner); + if (tbuffer && tbuffer->type == DMNSN_T_LEX_VERBATIM && tbuffer->i == 1) { + return token; } switch (token) { @@ -409,7 +506,19 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, } dmnsn_yyset_extra(tb, yyscanner); - tbuffer = tb; + continue; + } + + case DMNSN_T_WHILE: + { + dmnsn_token_buffer *tb = dmnsn_while_buffer( + token, tbuffer, lvalp, llocp, filename, symtable, yyscanner + ); + if (!tb) { + return DMNSN_T_LEX_ERROR; + } + + dmnsn_yyset_extra(tb, yyscanner); continue; } |