Tokenize on-the-fly.

No more manual tokenization and parsing, giving a simpler interface.  Also,
this brings us closer to a dmnsn_realize_string() interface.

7 files changed, 190 insertions, 181 deletions

diff --git a/dimension/grammar.y b/dimension/grammar.y
index a8dc37f..7825bc0 100644
--- a/dimension/grammar.y
+++ b/dimension/grammar.y
@@ -24,23 +24,15 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-typedef struct dmnsn_token_iterator {
-  const dmnsn_array *tokens;
-  unsigned int i;
-} dmnsn_token_iterator;
-
-typedef struct dmnsn_location {
-  const char *first_filename, *last_filename;
-  int first_line, last_line;
-  int first_column, last_column;
-} dmnsn_location;
+#define YYSTYPE dmnsn_parse_item
+#define YYLTYPE dmnsn_parse_location
 
-typedef union YYSTYPE {
-  const char *value;
-  dmnsn_astnode astnode;
-} YYSTYPE;
+int dmnsn_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, const char *filename,
+                void *yyscanner);
+void dmnsn_yylex_init(void **scannerp);
+void dmnsn_yyset_in(FILE *file, void *scanner);
+void dmnsn_yylex_destroy(void *scanner);
 
-#define YYLTYPE dmnsn_location
 #define YYLLOC_DEFAULT(Current, Rhs, N)                                 \
   do {                                                                  \
     if (N) {                                                            \
@@ -154,29 +146,9 @@ dmnsn_delete_astnode(dmnsn_astnode astnode)
   free(astnode.ptr);
 }
 
-static int
-yylex(YYSTYPE *lvalp, YYLTYPE *llocp, dmnsn_token_iterator *iterator)
-{
-  if (iterator->i >= dmnsn_array_size(iterator->tokens)) {
-    return 0;
-  } else {
-    dmnsn_token token;
-    dmnsn_array_get(iterator->tokens, iterator->i, &token);
-    ++iterator->i;
-
-    lvalp->value = token.value;
-
-    llocp->first_filename = llocp->last_filename = token.filename;
-    llocp->first_line     = llocp->last_line     = token.line;
-    llocp->first_column   = llocp->last_column   = token.col;
-
-    return token.type;
-  }  
-}
-
 void
-yyerror(YYLTYPE *locp, dmnsn_array *astree, dmnsn_token_iterator *iterator,
-        const char *str)
+yyerror(YYLTYPE *locp, const char *filename, void *yyscanner,
+        dmnsn_array *astree, const char *str)
 {
   dmnsn_diagnostic(locp->first_filename, locp->first_line, locp->first_column,
                    "%s", str);
@@ -192,9 +164,11 @@ yyerror(YYLTYPE *locp, dmnsn_array *astree, dmnsn_token_iterator *iterator,
 
 %expect 0
 
+%parse-param {const char *filename}
+%parse-param {void *yyscanner}
 %parse-param {dmnsn_array *astree}
-%parse-param {dmnsn_token_iterator *iterator}
-%lex-param {dmnsn_token_iterator *iterator}
+%lex-param {const char *filename}
+%lex-param {void *yyscanner}
 
 %token END 0 "end-of-file"
 
@@ -1164,16 +1138,20 @@ COLOR_KEYWORD_ITEM: "red" FLOAT {
 %%
 
 dmnsn_array *
-dmnsn_parse(const dmnsn_array *tokens)
+dmnsn_parse(FILE *file, const char *filename)
 {
+  void *scanner;
   dmnsn_array *astree = dmnsn_new_array(sizeof(dmnsn_astnode));
-  dmnsn_token_iterator iterator = { .tokens = tokens, .i = 0 };
 
-  if (yyparse(astree, &iterator) != 0) {
+  dmnsn_yylex_init(&scanner);
+  dmnsn_yyset_in(file, scanner);
+
+  if (yyparse(filename, scanner, astree) != 0) {
     dmnsn_delete_astree(astree);
     return NULL;
   }
 
+  dmnsn_yylex_destroy(scanner);
   return astree;
 }
 
diff --git a/dimension/lexer.l b/dimension/lexer.l
index bfe5b8a..b8bfbb0 100644
--- a/dimension/lexer.l
+++ b/dimension/lexer.l
@@ -20,12 +20,15 @@
 %option reentrant stack yylineno noyywrap prefix="dmnsn_yy" outfile="lex.yy.c"
 
 %{
-#define YY_DECL static int yylex(const char *filename, dmnsn_array *tokens, \
-                                 yyscan_t yyscanner)
+#include "parse.h"
 #include "tokenize.h"
 #include "utility.h"
 #include <stdlib.h>
 #include <stdio.h>
+
+#define YY_DECL int yylex(dmnsn_parse_item *lvalp,                      \
+                          dmnsn_parse_location *llocp,                  \
+                          const char *filename, yyscan_t yyscanner)
 %}
 
 %x DMNSN_BLOCK_COMMENT
@@ -38,34 +41,34 @@
 %{
 /* Some helpful macros that set fields of a token correctly, and other stuff */
 
-#define NEW_TOKEN(token_type)                   \
-  do {                                          \
-    token.type     = token_type;                \
-    token.filename = filename;                  \
-    token.line     = yylineno;                  \
-    token.col      = yycolumn;                  \
-    token.value    = NULL;                      \
+#define NEW_TOKEN(token_type)                                   \
+  do {                                                          \
+    token = token_type;                                         \
+    lvalp->value = NULL;                                        \
+    llocp->first_filename = llocp->last_filename = filename;    \
+    llocp->first_line     = llocp->last_line     = yylineno;    \
+    llocp->first_column   = llocp->last_column   = yycolumn;    \
   } while (0)
 
-#define CALCULATE_COLUMN() yycolumn += yyleng
+#define CALCULATE_COLUMN() do { yycolumn += yyleng; } while (0)
 
-#define PUSH()                                  \
-  do {                                          \
-    dmnsn_array_push(tokens, &token);           \
-    CALCULATE_COLUMN();                         \
+#define RETURN()                               \
+  do {                                         \
+    CALCULATE_COLUMN();                        \
+    return token;                              \
   } while (0)
 
-#define PUSH_TOKEN(token_type)                  \
+#define RETURN_TOKEN(token_type)                \
   do {                                          \
     NEW_TOKEN(token_type);                      \
-    PUSH();                                     \
+    RETURN();                                   \
   } while (0)
 
-#define PUSH_VALUE_TOKEN(token_type)            \
+#define RETURN_VALUE_TOKEN(token_type)          \
   do {                                          \
     NEW_TOKEN(token_type);                      \
-    token.value    = strdup(yytext);            \
-    PUSH();                                     \
+    lvalp->value = strdup(yytext);              \
+    RETURN();                                   \
   } while (0)
 
 #define STRING_TOKEN()                          \
@@ -73,8 +76,8 @@
     NEW_TOKEN(DMNSN_T_STRING);                  \
     string_length  = 0;                         \
     string_extent  = 8;                         \
-    token.value    = malloc(string_extent);     \
-    token.value[0] = '\0';                      \
+    lvalp->value    = malloc(string_extent);    \
+    lvalp->value[0] = '\0';                     \
     CALCULATE_COLUMN();                         \
   } while (0)
 
@@ -82,15 +85,15 @@
   do {                                                                  \
     if (string_length + len + 1 >= string_length) {                     \
       string_extent = 2*(string_length + len + 1);                      \
-      token.value = realloc(token.value, string_extent);                \
+      lvalp->value = realloc(lvalp->value, string_extent);              \
     }                                                                   \
                                                                         \
-    strncpy(token.value + string_length, str, len + 1);                 \
+    strncpy(lvalp->value + string_length, str, len + 1);                \
     string_length += len;                                               \
     CALCULATE_COLUMN();                                                 \
   } while(0)
 
-dmnsn_token token;
+int token;
 size_t string_length, string_extent;
 unsigned long wchar;
 %}
@@ -115,89 +118,89 @@ unsigned long wchar;
 <DMNSN_LINE_COMMENT>[^\n]+      CALCULATE_COLUMN();
 
 (?# Punctuation)
-"{"     PUSH_TOKEN(DMNSN_T_LBRACE);
-"}"     PUSH_TOKEN(DMNSN_T_RBRACE);
-"("     PUSH_TOKEN(DMNSN_T_LPAREN);
-")"     PUSH_TOKEN(DMNSN_T_RPAREN);
-"["     PUSH_TOKEN(DMNSN_T_LBRACKET);
-"]"     PUSH_TOKEN(DMNSN_T_RBRACKET);
-"+"     PUSH_TOKEN(DMNSN_T_PLUS);
-"-"     PUSH_TOKEN(DMNSN_T_MINUS);
-"*"     PUSH_TOKEN(DMNSN_T_STAR);
-"/"     PUSH_TOKEN(DMNSN_T_SLASH);
-","     PUSH_TOKEN(DMNSN_T_COMMA);
-";"     PUSH_TOKEN(DMNSN_T_SEMICOLON);
-"?"     PUSH_TOKEN(DMNSN_T_QUESTION);
-":"     PUSH_TOKEN(DMNSN_T_COLON);
-"&"     PUSH_TOKEN(DMNSN_T_AND);
-"."     PUSH_TOKEN(DMNSN_T_DOT);
-"|"     PUSH_TOKEN(DMNSN_T_PIPE);
-"<"     PUSH_TOKEN(DMNSN_T_LESS);
-">"     PUSH_TOKEN(DMNSN_T_GREATER);
-"!"     PUSH_TOKEN(DMNSN_T_BANG);
-"="     PUSH_TOKEN(DMNSN_T_EQUALS);
-"<="    PUSH_TOKEN(DMNSN_T_LESS_EQUAL);
-">="    PUSH_TOKEN(DMNSN_T_GREATER_EQUAL);
-"!="    PUSH_TOKEN(DMNSN_T_NOT_EQUAL);
+"{"     RETURN_TOKEN(DMNSN_T_LBRACE);
+"}"     RETURN_TOKEN(DMNSN_T_RBRACE);
+"("     RETURN_TOKEN(DMNSN_T_LPAREN);
+")"     RETURN_TOKEN(DMNSN_T_RPAREN);
+"["     RETURN_TOKEN(DMNSN_T_LBRACKET);
+"]"     RETURN_TOKEN(DMNSN_T_RBRACKET);
+"+"     RETURN_TOKEN(DMNSN_T_PLUS);
+"-"     RETURN_TOKEN(DMNSN_T_MINUS);
+"*"     RETURN_TOKEN(DMNSN_T_STAR);
+"/"     RETURN_TOKEN(DMNSN_T_SLASH);
+","     RETURN_TOKEN(DMNSN_T_COMMA);
+";"     RETURN_TOKEN(DMNSN_T_SEMICOLON);
+"?"     RETURN_TOKEN(DMNSN_T_QUESTION);
+":"     RETURN_TOKEN(DMNSN_T_COLON);
+"&"     RETURN_TOKEN(DMNSN_T_AND);
+"."     RETURN_TOKEN(DMNSN_T_DOT);
+"|"     RETURN_TOKEN(DMNSN_T_PIPE);
+"<"     RETURN_TOKEN(DMNSN_T_LESS);
+">"     RETURN_TOKEN(DMNSN_T_GREATER);
+"!"     RETURN_TOKEN(DMNSN_T_BANG);
+"="     RETURN_TOKEN(DMNSN_T_EQUALS);
+"<="    RETURN_TOKEN(DMNSN_T_LESS_EQUAL);
+">="    RETURN_TOKEN(DMNSN_T_GREATER_EQUAL);
+"!="    RETURN_TOKEN(DMNSN_T_NOT_EQUAL);
 
 (?# Integers)
 [[:digit:]]+                    |
-0(x|X)[[:digit:]aAbBcCdDeEfF]+  PUSH_VALUE_TOKEN(DMNSN_T_INTEGER);
+0(x|X)[[:digit:]aAbBcCdDeEfF]+  RETURN_VALUE_TOKEN(DMNSN_T_INTEGER);
 
 (?# Floats)
 [[:digit:]]*\.?[[:digit:]]+((e|E)(\+|-)?[[:digit:]]+)?  {
-  PUSH_VALUE_TOKEN(DMNSN_T_FLOAT);
+  RETURN_VALUE_TOKEN(DMNSN_T_FLOAT);
 }
 
 (?# Keywords)
-"angle"         PUSH_TOKEN(DMNSN_T_ANGLE);
-"background"    PUSH_TOKEN(DMNSN_T_BACKGROUND);
-"box"           PUSH_TOKEN(DMNSN_T_BOX);
-"blue"          PUSH_TOKEN(DMNSN_T_BLUE);
-"camera"        PUSH_TOKEN(DMNSN_T_CAMERA);
-"color"         PUSH_TOKEN(DMNSN_T_COLOR);
-"colour"        PUSH_TOKEN(DMNSN_T_COLOR);
-"direction"     PUSH_TOKEN(DMNSN_T_DIRECTION);
-"filter"        PUSH_TOKEN(DMNSN_T_FILTER);
-"gray"          PUSH_TOKEN(DMNSN_T_GRAY);
-"grey"          PUSH_TOKEN(DMNSN_T_GRAY);
-"green"         PUSH_TOKEN(DMNSN_T_GREEN);
-"location"      PUSH_TOKEN(DMNSN_T_LOCATION);
-"look_at"       PUSH_TOKEN(DMNSN_T_LOOK_AT);
-"light_source"  PUSH_TOKEN(DMNSN_T_LIGHT_SOURCE);
-"perspective"   PUSH_TOKEN(DMNSN_T_PERSPECTIVE);
-"pigment"       PUSH_TOKEN(DMNSN_T_PIGMENT);
-"red"           PUSH_TOKEN(DMNSN_T_RED);
-"rgb"           PUSH_TOKEN(DMNSN_T_RGB);
-"rgbf"          PUSH_TOKEN(DMNSN_T_RGBF);
-"rgbft"         PUSH_TOKEN(DMNSN_T_RGBFT);
-"rgbt"          PUSH_TOKEN(DMNSN_T_RGBT);
-"right"         PUSH_TOKEN(DMNSN_T_RIGHT);
-"rotate"        PUSH_TOKEN(DMNSN_T_ROTATE);
-"sphere"        PUSH_TOKEN(DMNSN_T_SPHERE);
-"sky"           PUSH_TOKEN(DMNSN_T_SKY);
-"t"             PUSH_TOKEN(DMNSN_T_T);
-"texture"       PUSH_TOKEN(DMNSN_T_TEXTURE);
-"transmit"      PUSH_TOKEN(DMNSN_T_TRANSMIT);
-"u"             PUSH_TOKEN(DMNSN_T_U);
-"up"            PUSH_TOKEN(DMNSN_T_UP);
-"v"             PUSH_TOKEN(DMNSN_T_V);
-"x"             PUSH_TOKEN(DMNSN_T_X);
-"y"             PUSH_TOKEN(DMNSN_T_Y);
-"z"             PUSH_TOKEN(DMNSN_T_Z);
+"angle"         RETURN_TOKEN(DMNSN_T_ANGLE);
+"background"    RETURN_TOKEN(DMNSN_T_BACKGROUND);
+"box"           RETURN_TOKEN(DMNSN_T_BOX);
+"blue"          RETURN_TOKEN(DMNSN_T_BLUE);
+"camera"        RETURN_TOKEN(DMNSN_T_CAMERA);
+"color"         RETURN_TOKEN(DMNSN_T_COLOR);
+"colour"        RETURN_TOKEN(DMNSN_T_COLOR);
+"direction"     RETURN_TOKEN(DMNSN_T_DIRECTION);
+"filter"        RETURN_TOKEN(DMNSN_T_FILTER);
+"gray"          RETURN_TOKEN(DMNSN_T_GRAY);
+"grey"          RETURN_TOKEN(DMNSN_T_GRAY);
+"green"         RETURN_TOKEN(DMNSN_T_GREEN);
+"location"      RETURN_TOKEN(DMNSN_T_LOCATION);
+"look_at"       RETURN_TOKEN(DMNSN_T_LOOK_AT);
+"light_source"  RETURN_TOKEN(DMNSN_T_LIGHT_SOURCE);
+"perspective"   RETURN_TOKEN(DMNSN_T_PERSPECTIVE);
+"pigment"       RETURN_TOKEN(DMNSN_T_PIGMENT);
+"red"           RETURN_TOKEN(DMNSN_T_RED);
+"rgb"           RETURN_TOKEN(DMNSN_T_RGB);
+"rgbf"          RETURN_TOKEN(DMNSN_T_RGBF);
+"rgbft"         RETURN_TOKEN(DMNSN_T_RGBFT);
+"rgbt"          RETURN_TOKEN(DMNSN_T_RGBT);
+"right"         RETURN_TOKEN(DMNSN_T_RIGHT);
+"rotate"        RETURN_TOKEN(DMNSN_T_ROTATE);
+"sphere"        RETURN_TOKEN(DMNSN_T_SPHERE);
+"sky"           RETURN_TOKEN(DMNSN_T_SKY);
+"t"             RETURN_TOKEN(DMNSN_T_T);
+"texture"       RETURN_TOKEN(DMNSN_T_TEXTURE);
+"transmit"      RETURN_TOKEN(DMNSN_T_TRANSMIT);
+"u"             RETURN_TOKEN(DMNSN_T_U);
+"up"            RETURN_TOKEN(DMNSN_T_UP);
+"v"             RETURN_TOKEN(DMNSN_T_V);
+"x"             RETURN_TOKEN(DMNSN_T_X);
+"y"             RETURN_TOKEN(DMNSN_T_Y);
+"z"             RETURN_TOKEN(DMNSN_T_Z);
 
 (?# Directives)
-"#include"      PUSH_TOKEN(DMNSN_T_INCLUDE);
-"#declare"      PUSH_TOKEN(DMNSN_T_DECLARE);
+"#include"      RETURN_TOKEN(DMNSN_T_INCLUDE);
+"#declare"      RETURN_TOKEN(DMNSN_T_DECLARE);
 
 (?# Identifiers)
-[[:alpha:]][[:alnum:]_]*        PUSH_VALUE_TOKEN(DMNSN_T_IDENTIFIER);
+[[:alpha:]][[:alnum:]_]*        RETURN_VALUE_TOKEN(DMNSN_T_IDENTIFIER);
 
 (?# Strings)
 
 "\""    STRING_TOKEN(); yy_push_state(DMNSN_STRING, yyscanner);
 <DMNSN_STRING>[^\\\"\n]*        STRCAT(yytext, yyleng);
-<DMNSN_STRING>"\""              PUSH(); yy_pop_state(yyscanner);
+<DMNSN_STRING>"\""              yy_pop_state(yyscanner); RETURN();
 
 (?# String escape sequences)
 
@@ -218,8 +221,8 @@ unsigned long wchar;
 <DMNSN_STRING_ESCAPE>"u"[[:digit:]aAbBcCdDeEfF]{4}      {
   wchar = strtoul(yytext + 1, NULL, 16);
   STRCAT("", 2);
-  token.value[string_length - 2] = wchar/256;
-  token.value[string_length - 1] = wchar%256;
+  lvalp->value[string_length - 2] = wchar/256;
+  lvalp->value[string_length - 1] = wchar%256;
   yy_pop_state(yyscanner);
 }
 <DMNSN_STRING_ESCAPE>.          {
@@ -245,8 +248,11 @@ unsigned long wchar;
 %%
 
 dmnsn_array *
-dmnsn_tokenize(const char *filename, FILE *file)
+dmnsn_tokenize(FILE *file, const char *filename)
 {
+  dmnsn_token token;
+  dmnsn_parse_item item;
+  dmnsn_parse_location location;
   dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
 
   yyscan_t scanner;
@@ -254,24 +260,37 @@ dmnsn_tokenize(const char *filename, FILE *file)
   yylex_init(&scanner);
   yyset_in(file, scanner);
 
-  if (yylex(filename, tokens, scanner) != 0) {
-    dmnsn_delete_tokens(tokens);
-    tokens = NULL;
+  while ((token.type = yylex(&item, &location, filename, scanner)) != 0) {
+    if (token.type == 1 || token.type == 2) {
+      dmnsn_delete_tokens(tokens);
+      tokens = NULL;
+    } else {
+      token.value    = item.value;
+      token.filename = location.first_filename;
+      token.line     = location.first_line;
+      token.col      = location.first_column;
+      dmnsn_array_push(tokens, &token);
+    }
   }
 
   yylex_destroy(scanner);
-
   return tokens;
 }
 
 void
+dmnsn_delete_token(dmnsn_token token)
+{
+  free(token.value);
+}
+
+void
 dmnsn_delete_tokens(dmnsn_array *tokens)
 {
-  dmnsn_token *token;
+  dmnsn_token token;
   unsigned int i;
   for (i = 0; i < dmnsn_array_size(tokens); ++i) {
-    token = dmnsn_array_at(tokens, i);
-    free(token->value);
+    dmnsn_array_get(tokens, i, &token);
+    dmnsn_delete_token(token);
   }
   dmnsn_delete_array(tokens);
 }
diff --git a/dimension/main.c b/dimension/main.c
index e8b0d22..2c2aea5 100644
--- a/dimension/main.c
+++ b/dimension/main.c
@@ -137,58 +137,45 @@ main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
 
-  /* Tokenize the input file */
-
-  if (!debugging)
-    printf("Tokenizing input ...\n");
-
-  dmnsn_array *tokens = dmnsn_tokenize(input, input_file);
-  if (!tokens) {
-    fclose(input_file);
-    fprintf(stderr, "Error tokenizing input file!\n");
-    return EXIT_FAILURE;
-  }
-  fclose(input_file);
-
   /* Debugging option - output the list of tokens as an S-expression */
   if (tokenize) {
+    dmnsn_array *tokens = dmnsn_tokenize(input_file, input);
+    if (!tokens) {
+      fclose(input_file);
+      fprintf(stderr, "Error tokenizing input file!\n");
+      return EXIT_FAILURE;
+    }
     dmnsn_print_token_sexpr(stdout, tokens);
+    rewind(input_file);
+    dmnsn_delete_tokens(tokens);
 
     if (!parse) {
-      dmnsn_delete_tokens(tokens);
+      fclose(input_file);
       return EXIT_SUCCESS;
     }
   }
 
-  /* Parse the input */
-
-  if (!debugging)
-    printf("Parsing input    ...\n");
-
-  dmnsn_array *astree = dmnsn_parse(tokens);
-  if (!astree) {
-    dmnsn_delete_tokens(tokens);
-    fprintf(stderr, "Error parsing input file!\n");
-    return EXIT_FAILURE;
-  }
-  dmnsn_delete_tokens(tokens);
-
   /* Debugging option - output the abstract syntax tree as an S-expression */
   if (parse) {
+    dmnsn_array *astree = dmnsn_parse(input_file, input);
+    if (!astree) {
+      fprintf(stderr, "Error parsing input file!\n");
+      return EXIT_FAILURE;
+    }
     dmnsn_print_astree_sexpr(stdout, astree);
     dmnsn_delete_astree(astree);
+
+    fclose(input_file);
     return EXIT_SUCCESS;
   }
 
   /* Realize the input */
-  printf("Generating scene ...\n");
-  dmnsn_scene *scene = dmnsn_realize(astree);
+  printf("Parsing scene   ...\n");
+  dmnsn_scene *scene = dmnsn_realize(input_file, input);
   if (!scene) {
-    dmnsn_delete_astree(astree);
     fprintf(stderr, "Error realizing input file!\n");
     return EXIT_FAILURE;
   }
-  dmnsn_delete_astree(astree);
 
   /* Allocate a canvas */
   scene->canvas = dmnsn_new_canvas(width, height);
@@ -220,7 +207,7 @@ main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
 
-  dmnsn_progressbar("Rendering scene: ", render_progress);
+  dmnsn_progressbar("Rendering scene ", render_progress);
 
   if (dmnsn_finish_progress(render_progress) != 0) {
     dmnsn_delete_scene(scene);
@@ -237,7 +224,7 @@ main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
 
-  dmnsn_progressbar("Writing PNG:     ", output_progress);
+  dmnsn_progressbar("Writing PNG     ", output_progress);
 
   if (dmnsn_finish_progress(output_progress) != 0) {
     fclose(output_file);
diff --git a/dimension/parse.h b/dimension/parse.h
index d6c6b0e..3ea073f 100644
--- a/dimension/parse.h
+++ b/dimension/parse.h
@@ -79,7 +79,7 @@ struct dmnsn_astnode {
 };
 
 /* The workhorse */
-dmnsn_array *dmnsn_parse(const dmnsn_array *tokens);
+dmnsn_array *dmnsn_parse(FILE *file, const char *filename);
 
 /* Free an abstract syntax tree */
 void dmnsn_delete_astree(dmnsn_array *astree);
@@ -94,4 +94,18 @@ void dmnsn_print_astree_sexpr(FILE *file, const dmnsn_array *astree);
 /* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */
 const char *dmnsn_astnode_string(dmnsn_astnode_type astnode_type);
 
+/* Parser internals */
+
+typedef struct dmnsn_parse_location {
+  const char *first_filename, *last_filename;
+  int first_line, last_line;
+  int first_column, last_column;
+} dmnsn_parse_location;
+
+typedef union dmnsn_parse_item {
+  char *value;
+  dmnsn_astnode astnode;
+} dmnsn_parse_item;
+
+
 #endif /* PARSE_H */
diff --git a/dimension/realize.c b/dimension/realize.c
index 1a42be9..7ca3ae2 100644
--- a/dimension/realize.c
+++ b/dimension/realize.c
@@ -536,8 +536,8 @@ dmnsn_realize_sphere(dmnsn_astnode astnode)
   return sphere;
 }
 
-dmnsn_scene *
-dmnsn_realize(const dmnsn_array *astree)
+static dmnsn_scene *
+dmnsn_realize_astree(const dmnsn_array *astree)
 {
   dmnsn_scene *scene = dmnsn_new_scene();
   if (!scene) {
@@ -607,3 +607,13 @@ dmnsn_realize(const dmnsn_array *astree)
 
   return scene;
 }
+
+dmnsn_scene *
+dmnsn_realize(FILE *file, const char *filename)
+{
+  dmnsn_array *astree = dmnsn_parse(file, filename);
+  if (!astree) {
+    return NULL;
+  }
+  return dmnsn_realize_astree(astree);
+}
diff --git a/dimension/realize.h b/dimension/realize.h
index d6f2539..dbd71df 100644
--- a/dimension/realize.h
+++ b/dimension/realize.h
@@ -22,6 +22,6 @@
 
 #include "../libdimension/dimension.h"
 
-dmnsn_scene *dmnsn_realize(const dmnsn_array *astree);
+dmnsn_scene *dmnsn_realize(FILE *file, const char *filename);
 
 #endif /* REALIZE_H */
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 67a386c..541c604 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -36,10 +36,11 @@ struct dmnsn_token {
   unsigned int line, col;
 };
 
-/* The workhorse */
-dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file);
+/* For debugging */
+dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename);
 
-/* Free an array of tokens - use this rather than dmnsn_delete_array() */
+/* Token destruction */
+void dmnsn_delete_token(dmnsn_token token);
 void dmnsn_delete_tokens(dmnsn_array *tokens);
 
 /* Print an S-expression of a list of tokens to `file' */