diff options
| -rw-r--r-- | dimension/grammar.y | 62 | ||||
| -rw-r--r-- | dimension/lexer.l | 217 | ||||
| -rw-r--r-- | dimension/main.c | 53 | ||||
| -rw-r--r-- | dimension/parse.h | 16 | ||||
| -rw-r--r-- | dimension/realize.c | 14 | ||||
| -rw-r--r-- | dimension/realize.h | 2 | ||||
| -rw-r--r-- | dimension/tokenize.h | 7 |
7 files changed, 190 insertions, 181 deletions
diff --git a/dimension/grammar.y b/dimension/grammar.y index a8dc37f..7825bc0 100644 --- a/dimension/grammar.y +++ b/dimension/grammar.y @@ -24,23 +24,15 @@ #include <stdlib.h> #include <stdio.h> -typedef struct dmnsn_token_iterator { - const dmnsn_array *tokens; - unsigned int i; -} dmnsn_token_iterator; - -typedef struct dmnsn_location { - const char *first_filename, *last_filename; - int first_line, last_line; - int first_column, last_column; -} dmnsn_location; +#define YYSTYPE dmnsn_parse_item +#define YYLTYPE dmnsn_parse_location -typedef union YYSTYPE { - const char *value; - dmnsn_astnode astnode; -} YYSTYPE; +int dmnsn_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, const char *filename, + void *yyscanner); +void dmnsn_yylex_init(void **scannerp); +void dmnsn_yyset_in(FILE *file, void *scanner); +void dmnsn_yylex_destroy(void *scanner); -#define YYLTYPE dmnsn_location #define YYLLOC_DEFAULT(Current, Rhs, N) \ do { \ if (N) { \ @@ -154,29 +146,9 @@ dmnsn_delete_astnode(dmnsn_astnode astnode) free(astnode.ptr); } -static int -yylex(YYSTYPE *lvalp, YYLTYPE *llocp, dmnsn_token_iterator *iterator) -{ - if (iterator->i >= dmnsn_array_size(iterator->tokens)) { - return 0; - } else { - dmnsn_token token; - dmnsn_array_get(iterator->tokens, iterator->i, &token); - ++iterator->i; - - lvalp->value = token.value; - - llocp->first_filename = llocp->last_filename = token.filename; - llocp->first_line = llocp->last_line = token.line; - llocp->first_column = llocp->last_column = token.col; - - return token.type; - } -} - void -yyerror(YYLTYPE *locp, dmnsn_array *astree, dmnsn_token_iterator *iterator, - const char *str) +yyerror(YYLTYPE *locp, const char *filename, void *yyscanner, + dmnsn_array *astree, const char *str) { dmnsn_diagnostic(locp->first_filename, locp->first_line, locp->first_column, "%s", str); @@ -192,9 +164,11 @@ yyerror(YYLTYPE *locp, dmnsn_array *astree, dmnsn_token_iterator *iterator, %expect 0 +%parse-param {const char *filename} +%parse-param {void *yyscanner} %parse-param {dmnsn_array *astree} -%parse-param {dmnsn_token_iterator *iterator} -%lex-param {dmnsn_token_iterator *iterator} +%lex-param {const char *filename} +%lex-param {void *yyscanner} %token END 0 "end-of-file" @@ -1164,16 +1138,20 @@ COLOR_KEYWORD_ITEM: "red" FLOAT { %% dmnsn_array * -dmnsn_parse(const dmnsn_array *tokens) +dmnsn_parse(FILE *file, const char *filename) { + void *scanner; dmnsn_array *astree = dmnsn_new_array(sizeof(dmnsn_astnode)); - dmnsn_token_iterator iterator = { .tokens = tokens, .i = 0 }; - if (yyparse(astree, &iterator) != 0) { + dmnsn_yylex_init(&scanner); + dmnsn_yyset_in(file, scanner); + + if (yyparse(filename, scanner, astree) != 0) { dmnsn_delete_astree(astree); return NULL; } + dmnsn_yylex_destroy(scanner); return astree; } diff --git a/dimension/lexer.l b/dimension/lexer.l index bfe5b8a..b8bfbb0 100644 --- a/dimension/lexer.l +++ b/dimension/lexer.l @@ -20,12 +20,15 @@ %option reentrant stack yylineno noyywrap prefix="dmnsn_yy" outfile="lex.yy.c" %{ -#define YY_DECL static int yylex(const char *filename, dmnsn_array *tokens, \ - yyscan_t yyscanner) +#include "parse.h" #include "tokenize.h" #include "utility.h" #include <stdlib.h> #include <stdio.h> + +#define YY_DECL int yylex(dmnsn_parse_item *lvalp, \ + dmnsn_parse_location *llocp, \ + const char *filename, yyscan_t yyscanner) %} %x DMNSN_BLOCK_COMMENT @@ -38,34 +41,34 @@ %{ /* Some helpful macros that set fields of a token correctly, and other stuff */ -#define NEW_TOKEN(token_type) \ - do { \ - token.type = token_type; \ - token.filename = filename; \ - token.line = yylineno; \ - token.col = yycolumn; \ - token.value = NULL; \ +#define NEW_TOKEN(token_type) \ + do { \ + token = token_type; \ + lvalp->value = NULL; \ + llocp->first_filename = llocp->last_filename = filename; \ + llocp->first_line = llocp->last_line = yylineno; \ + llocp->first_column = llocp->last_column = yycolumn; \ } while (0) -#define CALCULATE_COLUMN() yycolumn += yyleng +#define CALCULATE_COLUMN() do { yycolumn += yyleng; } while (0) -#define PUSH() \ - do { \ - dmnsn_array_push(tokens, &token); \ - CALCULATE_COLUMN(); \ +#define RETURN() \ + do { \ + CALCULATE_COLUMN(); \ + return token; \ } while (0) -#define PUSH_TOKEN(token_type) \ +#define RETURN_TOKEN(token_type) \ do { \ NEW_TOKEN(token_type); \ - PUSH(); \ + RETURN(); \ } while (0) -#define PUSH_VALUE_TOKEN(token_type) \ +#define RETURN_VALUE_TOKEN(token_type) \ do { \ NEW_TOKEN(token_type); \ - token.value = strdup(yytext); \ - PUSH(); \ + lvalp->value = strdup(yytext); \ + RETURN(); \ } while (0) #define STRING_TOKEN() \ @@ -73,8 +76,8 @@ NEW_TOKEN(DMNSN_T_STRING); \ string_length = 0; \ string_extent = 8; \ - token.value = malloc(string_extent); \ - token.value[0] = '\0'; \ + lvalp->value = malloc(string_extent); \ + lvalp->value[0] = '\0'; \ CALCULATE_COLUMN(); \ } while (0) @@ -82,15 +85,15 @@ do { \ if (string_length + len + 1 >= string_length) { \ string_extent = 2*(string_length + len + 1); \ - token.value = realloc(token.value, string_extent); \ + lvalp->value = realloc(lvalp->value, string_extent); \ } \ \ - strncpy(token.value + string_length, str, len + 1); \ + strncpy(lvalp->value + string_length, str, len + 1); \ string_length += len; \ CALCULATE_COLUMN(); \ } while(0) -dmnsn_token token; +int token; size_t string_length, string_extent; unsigned long wchar; %} @@ -115,89 +118,89 @@ unsigned long wchar; <DMNSN_LINE_COMMENT>[^\n]+ CALCULATE_COLUMN(); (?# Punctuation) -"{" PUSH_TOKEN(DMNSN_T_LBRACE); -"}" PUSH_TOKEN(DMNSN_T_RBRACE); -"(" PUSH_TOKEN(DMNSN_T_LPAREN); -")" PUSH_TOKEN(DMNSN_T_RPAREN); -"[" PUSH_TOKEN(DMNSN_T_LBRACKET); -"]" PUSH_TOKEN(DMNSN_T_RBRACKET); -"+" PUSH_TOKEN(DMNSN_T_PLUS); -"-" PUSH_TOKEN(DMNSN_T_MINUS); -"*" PUSH_TOKEN(DMNSN_T_STAR); -"/" PUSH_TOKEN(DMNSN_T_SLASH); -"," PUSH_TOKEN(DMNSN_T_COMMA); -";" PUSH_TOKEN(DMNSN_T_SEMICOLON); -"?" PUSH_TOKEN(DMNSN_T_QUESTION); -":" PUSH_TOKEN(DMNSN_T_COLON); -"&" PUSH_TOKEN(DMNSN_T_AND); -"." PUSH_TOKEN(DMNSN_T_DOT); -"|" PUSH_TOKEN(DMNSN_T_PIPE); -"<" PUSH_TOKEN(DMNSN_T_LESS); -">" PUSH_TOKEN(DMNSN_T_GREATER); -"!" PUSH_TOKEN(DMNSN_T_BANG); -"=" PUSH_TOKEN(DMNSN_T_EQUALS); -"<=" PUSH_TOKEN(DMNSN_T_LESS_EQUAL); -">=" PUSH_TOKEN(DMNSN_T_GREATER_EQUAL); -"!=" PUSH_TOKEN(DMNSN_T_NOT_EQUAL); +"{" RETURN_TOKEN(DMNSN_T_LBRACE); +"}" RETURN_TOKEN(DMNSN_T_RBRACE); +"(" RETURN_TOKEN(DMNSN_T_LPAREN); +")" RETURN_TOKEN(DMNSN_T_RPAREN); +"[" RETURN_TOKEN(DMNSN_T_LBRACKET); +"]" RETURN_TOKEN(DMNSN_T_RBRACKET); +"+" RETURN_TOKEN(DMNSN_T_PLUS); +"-" RETURN_TOKEN(DMNSN_T_MINUS); +"*" RETURN_TOKEN(DMNSN_T_STAR); +"/" RETURN_TOKEN(DMNSN_T_SLASH); +"," RETURN_TOKEN(DMNSN_T_COMMA); +";" RETURN_TOKEN(DMNSN_T_SEMICOLON); +"?" RETURN_TOKEN(DMNSN_T_QUESTION); +":" RETURN_TOKEN(DMNSN_T_COLON); +"&" RETURN_TOKEN(DMNSN_T_AND); +"." RETURN_TOKEN(DMNSN_T_DOT); +"|" RETURN_TOKEN(DMNSN_T_PIPE); +"<" RETURN_TOKEN(DMNSN_T_LESS); +">" RETURN_TOKEN(DMNSN_T_GREATER); +"!" RETURN_TOKEN(DMNSN_T_BANG); +"=" RETURN_TOKEN(DMNSN_T_EQUALS); +"<=" RETURN_TOKEN(DMNSN_T_LESS_EQUAL); +">=" RETURN_TOKEN(DMNSN_T_GREATER_EQUAL); +"!=" RETURN_TOKEN(DMNSN_T_NOT_EQUAL); (?# Integers) [[:digit:]]+ | -0(x|X)[[:digit:]aAbBcCdDeEfF]+ PUSH_VALUE_TOKEN(DMNSN_T_INTEGER); +0(x|X)[[:digit:]aAbBcCdDeEfF]+ RETURN_VALUE_TOKEN(DMNSN_T_INTEGER); (?# Floats) [[:digit:]]*\.?[[:digit:]]+((e|E)(\+|-)?[[:digit:]]+)? { - PUSH_VALUE_TOKEN(DMNSN_T_FLOAT); + RETURN_VALUE_TOKEN(DMNSN_T_FLOAT); } (?# Keywords) -"angle" PUSH_TOKEN(DMNSN_T_ANGLE); -"background" PUSH_TOKEN(DMNSN_T_BACKGROUND); -"box" PUSH_TOKEN(DMNSN_T_BOX); -"blue" PUSH_TOKEN(DMNSN_T_BLUE); -"camera" PUSH_TOKEN(DMNSN_T_CAMERA); -"color" PUSH_TOKEN(DMNSN_T_COLOR); -"colour" PUSH_TOKEN(DMNSN_T_COLOR); -"direction" PUSH_TOKEN(DMNSN_T_DIRECTION); -"filter" PUSH_TOKEN(DMNSN_T_FILTER); -"gray" PUSH_TOKEN(DMNSN_T_GRAY); -"grey" PUSH_TOKEN(DMNSN_T_GRAY); -"green" PUSH_TOKEN(DMNSN_T_GREEN); -"location" PUSH_TOKEN(DMNSN_T_LOCATION); -"look_at" PUSH_TOKEN(DMNSN_T_LOOK_AT); -"light_source" PUSH_TOKEN(DMNSN_T_LIGHT_SOURCE); -"perspective" PUSH_TOKEN(DMNSN_T_PERSPECTIVE); -"pigment" PUSH_TOKEN(DMNSN_T_PIGMENT); -"red" PUSH_TOKEN(DMNSN_T_RED); -"rgb" PUSH_TOKEN(DMNSN_T_RGB); -"rgbf" PUSH_TOKEN(DMNSN_T_RGBF); -"rgbft" PUSH_TOKEN(DMNSN_T_RGBFT); -"rgbt" PUSH_TOKEN(DMNSN_T_RGBT); -"right" PUSH_TOKEN(DMNSN_T_RIGHT); -"rotate" PUSH_TOKEN(DMNSN_T_ROTATE); -"sphere" PUSH_TOKEN(DMNSN_T_SPHERE); -"sky" PUSH_TOKEN(DMNSN_T_SKY); -"t" PUSH_TOKEN(DMNSN_T_T); -"texture" PUSH_TOKEN(DMNSN_T_TEXTURE); -"transmit" PUSH_TOKEN(DMNSN_T_TRANSMIT); -"u" PUSH_TOKEN(DMNSN_T_U); -"up" PUSH_TOKEN(DMNSN_T_UP); -"v" PUSH_TOKEN(DMNSN_T_V); -"x" PUSH_TOKEN(DMNSN_T_X); -"y" PUSH_TOKEN(DMNSN_T_Y); -"z" PUSH_TOKEN(DMNSN_T_Z); +"angle" RETURN_TOKEN(DMNSN_T_ANGLE); +"background" RETURN_TOKEN(DMNSN_T_BACKGROUND); +"box" RETURN_TOKEN(DMNSN_T_BOX); +"blue" RETURN_TOKEN(DMNSN_T_BLUE); +"camera" RETURN_TOKEN(DMNSN_T_CAMERA); +"color" RETURN_TOKEN(DMNSN_T_COLOR); +"colour" RETURN_TOKEN(DMNSN_T_COLOR); +"direction" RETURN_TOKEN(DMNSN_T_DIRECTION); +"filter" RETURN_TOKEN(DMNSN_T_FILTER); +"gray" RETURN_TOKEN(DMNSN_T_GRAY); +"grey" RETURN_TOKEN(DMNSN_T_GRAY); +"green" RETURN_TOKEN(DMNSN_T_GREEN); +"location" RETURN_TOKEN(DMNSN_T_LOCATION); +"look_at" RETURN_TOKEN(DMNSN_T_LOOK_AT); +"light_source" RETURN_TOKEN(DMNSN_T_LIGHT_SOURCE); +"perspective" RETURN_TOKEN(DMNSN_T_PERSPECTIVE); +"pigment" RETURN_TOKEN(DMNSN_T_PIGMENT); +"red" RETURN_TOKEN(DMNSN_T_RED); +"rgb" RETURN_TOKEN(DMNSN_T_RGB); +"rgbf" RETURN_TOKEN(DMNSN_T_RGBF); +"rgbft" RETURN_TOKEN(DMNSN_T_RGBFT); +"rgbt" RETURN_TOKEN(DMNSN_T_RGBT); +"right" RETURN_TOKEN(DMNSN_T_RIGHT); +"rotate" RETURN_TOKEN(DMNSN_T_ROTATE); +"sphere" RETURN_TOKEN(DMNSN_T_SPHERE); +"sky" RETURN_TOKEN(DMNSN_T_SKY); +"t" RETURN_TOKEN(DMNSN_T_T); +"texture" RETURN_TOKEN(DMNSN_T_TEXTURE); +"transmit" RETURN_TOKEN(DMNSN_T_TRANSMIT); +"u" RETURN_TOKEN(DMNSN_T_U); +"up" RETURN_TOKEN(DMNSN_T_UP); +"v" RETURN_TOKEN(DMNSN_T_V); +"x" RETURN_TOKEN(DMNSN_T_X); +"y" RETURN_TOKEN(DMNSN_T_Y); +"z" RETURN_TOKEN(DMNSN_T_Z); (?# Directives) -"#include" PUSH_TOKEN(DMNSN_T_INCLUDE); -"#declare" PUSH_TOKEN(DMNSN_T_DECLARE); +"#include" RETURN_TOKEN(DMNSN_T_INCLUDE); +"#declare" RETURN_TOKEN(DMNSN_T_DECLARE); (?# Identifiers) -[[:alpha:]][[:alnum:]_]* PUSH_VALUE_TOKEN(DMNSN_T_IDENTIFIER); +[[:alpha:]][[:alnum:]_]* RETURN_VALUE_TOKEN(DMNSN_T_IDENTIFIER); (?# Strings) "\"" STRING_TOKEN(); yy_push_state(DMNSN_STRING, yyscanner); <DMNSN_STRING>[^\\\"\n]* STRCAT(yytext, yyleng); -<DMNSN_STRING>"\"" PUSH(); yy_pop_state(yyscanner); +<DMNSN_STRING>"\"" yy_pop_state(yyscanner); RETURN(); (?# String escape sequences) @@ -218,8 +221,8 @@ unsigned long wchar; <DMNSN_STRING_ESCAPE>"u"[[:digit:]aAbBcCdDeEfF]{4} { wchar = strtoul(yytext + 1, NULL, 16); STRCAT("", 2); - token.value[string_length - 2] = wchar/256; - token.value[string_length - 1] = wchar%256; + lvalp->value[string_length - 2] = wchar/256; + lvalp->value[string_length - 1] = wchar%256; yy_pop_state(yyscanner); } <DMNSN_STRING_ESCAPE>. { @@ -245,8 +248,11 @@ unsigned long wchar; %% dmnsn_array * -dmnsn_tokenize(const char *filename, FILE *file) +dmnsn_tokenize(FILE *file, const char *filename) { + dmnsn_token token; + dmnsn_parse_item item; + dmnsn_parse_location location; dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token)); yyscan_t scanner; @@ -254,24 +260,37 @@ dmnsn_tokenize(const char *filename, FILE *file) yylex_init(&scanner); yyset_in(file, scanner); - if (yylex(filename, tokens, scanner) != 0) { - dmnsn_delete_tokens(tokens); - tokens = NULL; + while ((token.type = yylex(&item, &location, filename, scanner)) != 0) { + if (token.type == 1 || token.type == 2) { + dmnsn_delete_tokens(tokens); + tokens = NULL; + } else { + token.value = item.value; + token.filename = location.first_filename; + token.line = location.first_line; + token.col = location.first_column; + dmnsn_array_push(tokens, &token); + } } yylex_destroy(scanner); - return tokens; } void +dmnsn_delete_token(dmnsn_token token) +{ + free(token.value); +} + +void dmnsn_delete_tokens(dmnsn_array *tokens) { - dmnsn_token *token; + dmnsn_token token; unsigned int i; for (i = 0; i < dmnsn_array_size(tokens); ++i) { - token = dmnsn_array_at(tokens, i); - free(token->value); + dmnsn_array_get(tokens, i, &token); + dmnsn_delete_token(token); } dmnsn_delete_array(tokens); } diff --git a/dimension/main.c b/dimension/main.c index e8b0d22..2c2aea5 100644 --- a/dimension/main.c +++ b/dimension/main.c @@ -137,58 +137,45 @@ main(int argc, char **argv) { return EXIT_FAILURE; } - /* Tokenize the input file */ - - if (!debugging) - printf("Tokenizing input ...\n"); - - dmnsn_array *tokens = dmnsn_tokenize(input, input_file); - if (!tokens) { - fclose(input_file); - fprintf(stderr, "Error tokenizing input file!\n"); - return EXIT_FAILURE; - } - fclose(input_file); - /* Debugging option - output the list of tokens as an S-expression */ if (tokenize) { + dmnsn_array *tokens = dmnsn_tokenize(input_file, input); + if (!tokens) { + fclose(input_file); + fprintf(stderr, "Error tokenizing input file!\n"); + return EXIT_FAILURE; + } dmnsn_print_token_sexpr(stdout, tokens); + rewind(input_file); + dmnsn_delete_tokens(tokens); if (!parse) { - dmnsn_delete_tokens(tokens); + fclose(input_file); return EXIT_SUCCESS; } } - /* Parse the input */ - - if (!debugging) - printf("Parsing input ...\n"); - - dmnsn_array *astree = dmnsn_parse(tokens); - if (!astree) { - dmnsn_delete_tokens(tokens); - fprintf(stderr, "Error parsing input file!\n"); - return EXIT_FAILURE; - } - dmnsn_delete_tokens(tokens); - /* Debugging option - output the abstract syntax tree as an S-expression */ if (parse) { + dmnsn_array *astree = dmnsn_parse(input_file, input); + if (!astree) { + fprintf(stderr, "Error parsing input file!\n"); + return EXIT_FAILURE; + } dmnsn_print_astree_sexpr(stdout, astree); dmnsn_delete_astree(astree); + + fclose(input_file); return EXIT_SUCCESS; } /* Realize the input */ - printf("Generating scene ...\n"); - dmnsn_scene *scene = dmnsn_realize(astree); + printf("Parsing scene ...\n"); + dmnsn_scene *scene = dmnsn_realize(input_file, input); if (!scene) { - dmnsn_delete_astree(astree); fprintf(stderr, "Error realizing input file!\n"); return EXIT_FAILURE; } - dmnsn_delete_astree(astree); /* Allocate a canvas */ scene->canvas = dmnsn_new_canvas(width, height); @@ -220,7 +207,7 @@ main(int argc, char **argv) { return EXIT_FAILURE; } - dmnsn_progressbar("Rendering scene: ", render_progress); + dmnsn_progressbar("Rendering scene ", render_progress); if (dmnsn_finish_progress(render_progress) != 0) { dmnsn_delete_scene(scene); @@ -237,7 +224,7 @@ main(int argc, char **argv) { return EXIT_FAILURE; } - dmnsn_progressbar("Writing PNG: ", output_progress); + dmnsn_progressbar("Writing PNG ", output_progress); if (dmnsn_finish_progress(output_progress) != 0) { fclose(output_file); diff --git a/dimension/parse.h b/dimension/parse.h index d6c6b0e..3ea073f 100644 --- a/dimension/parse.h +++ b/dimension/parse.h @@ -79,7 +79,7 @@ struct dmnsn_astnode { }; /* The workhorse */ -dmnsn_array *dmnsn_parse(const dmnsn_array *tokens); +dmnsn_array *dmnsn_parse(FILE *file, const char *filename); /* Free an abstract syntax tree */ void dmnsn_delete_astree(dmnsn_array *astree); @@ -94,4 +94,18 @@ void dmnsn_print_astree_sexpr(FILE *file, const dmnsn_array *astree); /* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */ const char *dmnsn_astnode_string(dmnsn_astnode_type astnode_type); +/* Parser internals */ + +typedef struct dmnsn_parse_location { + const char *first_filename, *last_filename; + int first_line, last_line; + int first_column, last_column; +} dmnsn_parse_location; + +typedef union dmnsn_parse_item { + char *value; + dmnsn_astnode astnode; +} dmnsn_parse_item; + + #endif /* PARSE_H */ diff --git a/dimension/realize.c b/dimension/realize.c index 1a42be9..7ca3ae2 100644 --- a/dimension/realize.c +++ b/dimension/realize.c @@ -536,8 +536,8 @@ dmnsn_realize_sphere(dmnsn_astnode astnode) return sphere; } -dmnsn_scene * -dmnsn_realize(const dmnsn_array *astree) +static dmnsn_scene * +dmnsn_realize_astree(const dmnsn_array *astree) { dmnsn_scene *scene = dmnsn_new_scene(); if (!scene) { @@ -607,3 +607,13 @@ dmnsn_realize(const dmnsn_array *astree) return scene; } + +dmnsn_scene * +dmnsn_realize(FILE *file, const char *filename) +{ + dmnsn_array *astree = dmnsn_parse(file, filename); + if (!astree) { + return NULL; + } + return dmnsn_realize_astree(astree); +} diff --git a/dimension/realize.h b/dimension/realize.h index d6f2539..dbd71df 100644 --- a/dimension/realize.h +++ b/dimension/realize.h @@ -22,6 +22,6 @@ #include "../libdimension/dimension.h" -dmnsn_scene *dmnsn_realize(const dmnsn_array *astree); +dmnsn_scene *dmnsn_realize(FILE *file, const char *filename); #endif /* REALIZE_H */ diff --git a/dimension/tokenize.h b/dimension/tokenize.h index 67a386c..541c604 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -36,10 +36,11 @@ struct dmnsn_token { unsigned int line, col; }; -/* The workhorse */ -dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file); +/* For debugging */ +dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename); -/* Free an array of tokens - use this rather than dmnsn_delete_array() */ +/* Token destruction */ +void dmnsn_delete_token(dmnsn_token token); void dmnsn_delete_tokens(dmnsn_array *tokens); /* Print an S-expression of a list of tokens to `file' */ |