Refactor tokenizer a bit.

5 files changed, 93 insertions, 28 deletions

diff --git a/dimension/Makefile.am b/dimension/Makefile.am
index 099a81b..da8d86d 100644
--- a/dimension/Makefile.am
+++ b/dimension/Makefile.am
@@ -23,5 +23,7 @@ bin_PROGRAMS = dimension
 
 dimension_SOURCES = main.c                                                     \
                     tokenize.c                                                 \
-                    tokenize.h
+                    tokenize.h                                                 \
+                    utility.c                                                  \
+                    utility.h
 dimension_LDADD   = $(top_builddir)/libdimension/libdimension.la
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 4d53bdf..ec8c6ee 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -18,28 +18,14 @@
  *************************************************************************/
 
 #include "tokenize.h"
+#include "utility.h"
 #include <stdlib.h>   /* For strtoul(), etc. */
 #include <string.h>
-#include <stdarg.h>
 #include <ctype.h>    /* For isalpha(), etc. */
 #include <sys/mman.h> /* For mmap()          */
 #include <libgen.h>   /* For dirname()       */
 #include <locale.h>
 
-static void
-dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col,
-                 const char *format, ...)
-{
-  va_list ap;
-  va_start(ap, format);
-
-  fprintf(stderr, "%s:%u:%u: ", filename, line, col);
-  vfprintf(stderr, format, ap);
-  fprintf(stderr, "\n");
-
-  va_end(ap);
-}
-
 static int
 dmnsn_tokenize_comment(const char *filename,
                        unsigned int *linep, unsigned int *colp,
@@ -596,31 +582,42 @@ dmnsn_delete_tokens(dmnsn_array *tokens)
   dmnsn_delete_array(tokens);
 }
 
-static const char *dmnsn_token_name(dmnsn_token_type token_type);
-
 static void
-dmnsn_print_token(FILE *file, dmnsn_token *token)
+dmnsn_print_token(FILE *file, const dmnsn_token *token)
 {
+  const char *tname;
+  if (token->type == DMNSN_T_LPAREN) {
+    tname = "\\(";
+  } else if (token->type == DMNSN_T_RPAREN) {
+    tname = "\\)";
+  } else {
+    tname = dmnsn_token_name(token->type);
+  }
+
   if (token->value) {
-    fprintf(file, "(%s \"%s\")", dmnsn_token_name(token->type), token->value);
+    fprintf(file, "(%s \"%s\")", tname, token->value);
   } else {
-    fprintf(file, "%s", dmnsn_token_name(token->type));
+    fprintf(file, "%s", tname);
   }
 }
 
 void
-dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens)
+dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens)
 {
+  dmnsn_token token;
   unsigned int i;
+
   if (dmnsn_array_size(tokens) == 0) {
     fprintf(file, "()");
   } else {
     fprintf(file, "(");
-    dmnsn_print_token(file, dmnsn_array_at(tokens, 0));
+    dmnsn_array_get(tokens, 0, &token);
+    dmnsn_print_token(file, &token);
 
     for (i = 1; i < dmnsn_array_size(tokens); ++i) {
       fprintf(file, " ");
-      dmnsn_print_token(file, dmnsn_array_at(tokens, i));
+      dmnsn_array_get(tokens, i, &token);
+      dmnsn_print_token(file, &token);
     }
 
     fprintf(file, ")");
@@ -629,7 +626,7 @@ dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens)
   fprintf(file, "\n");
 }
 
-static const char *
+const char *
 dmnsn_token_name(dmnsn_token_type token_type)
 {
   switch (token_type) {
@@ -641,8 +638,8 @@ dmnsn_token_name(dmnsn_token_type token_type)
   /* Punctuation */
   dmnsn_token_map(DMNSN_T_LBRACE,      "{");
   dmnsn_token_map(DMNSN_T_RBRACE,      "}")
-  dmnsn_token_map(DMNSN_T_LPAREN,      "\\(");
-  dmnsn_token_map(DMNSN_T_RPAREN,      "\\)");
+  dmnsn_token_map(DMNSN_T_LPAREN,      "(");
+  dmnsn_token_map(DMNSN_T_RPAREN,      ")");
   dmnsn_token_map(DMNSN_T_LBRACKET,    "[");
   dmnsn_token_map(DMNSN_T_RBRACKET,    "]");
   dmnsn_token_map(DMNSN_T_LT,          "<");
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 7a36232..4e16909 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -99,6 +99,14 @@ struct dmnsn_token {
   unsigned int line, col;
 };
 
+/* The workhorse */
 dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file);
+
+/* Free an array of tokens - use this rather than dmnsn_delete_array() */
 void dmnsn_delete_tokens(dmnsn_array *tokens);
-void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens);
+
+/* Print an S-expression of a list of tokens to `file' */
+void dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens);
+
+/* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */
+const char *dmnsn_token_name(dmnsn_token_type token_type);
diff --git a/dimension/utility.c b/dimension/utility.c
new file mode 100644
index 0000000..1961d3e
--- /dev/null
+++ b/dimension/utility.c
@@ -0,0 +1,36 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of Dimension.                                       *
+ *                                                                       *
+ * Dimension is free software; you can redistribute it and/or modify it  *
+ * under the terms of the GNU General Public License as published by the *
+ * Free Software Foundation; either version 3 of the License, or (at     *
+ * your option) any later version.                                       *
+ *                                                                       *
+ * Dimension is distributed in the hope that it will be useful, but      *
+ * WITHOUT ANY WARRANTY; without even the implied warranty of            *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU     *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+#include "utility.h"
+#include <stdarg.h>
+#include <stdio.h>
+
+void
+dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col,
+                 const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+
+  fprintf(stderr, "%s:%u:%u: ", filename, line, col);
+  vfprintf(stderr, format, ap);
+  fprintf(stderr, "\n");
+
+  va_end(ap);
+}
diff --git a/dimension/utility.h b/dimension/utility.h
new file mode 100644
index 0000000..5403b19
--- /dev/null
+++ b/dimension/utility.h
@@ -0,0 +1,22 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of Dimension.                                       *
+ *                                                                       *
+ * Dimension is free software; you can redistribute it and/or modify it  *
+ * under the terms of the GNU General Public License as published by the *
+ * Free Software Foundation; either version 3 of the License, or (at     *
+ * your option) any later version.                                       *
+ *                                                                       *
+ * Dimension is distributed in the hope that it will be useful, but      *
+ * WITHOUT ANY WARRANTY; without even the implied warranty of            *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU     *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+/* Print a parsing diagnostic to stderr */
+void dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col,
+                      const char *format, ...);