Support more symbols in tokenizer.

6 files changed, 93 insertions, 34 deletions

diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index e1c2fce..7019ad7 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -51,6 +51,9 @@ dmnsn_tokenize(FILE *file)
   dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
 
   while (next - map < size) {
+    /* Saves us some code repetition in the vast majority of cases */
+    token.value = NULL;
+
     switch (*next) {
     case ' ':
     case '\n':
@@ -59,19 +62,29 @@ dmnsn_tokenize(FILE *file)
     case '\f':
     case '\v':
       /* Skip whitespace */
-      break;
-
-    case '{':
-      token.type = DMNSN_LBRACE;
-      token.value = NULL;
-      dmnsn_array_push(tokens, &token);
-      break;
-
-    case '}':
-      token.type = DMNSN_RBRACE;
-      token.value = NULL;
-      dmnsn_array_push(tokens, &token);
-      break;
+      ++next;
+      continue;
+
+    /* Macro to make basic symbol tokens easier */
+    #define dmnsn_simple_token(c, tp)                                          \
+    case c:                                                                    \
+      token.type = tp;                                                         \
+      break
+
+    /* Some simple punctuation marks */
+    dmnsn_simple_token('{', DMNSN_LBRACE);
+    dmnsn_simple_token('}', DMNSN_RBRACE);
+    dmnsn_simple_token('(', DMNSN_LPAREN);
+    dmnsn_simple_token(')', DMNSN_RPAREN);
+    dmnsn_simple_token('[', DMNSN_LBRACKET);
+    dmnsn_simple_token(']', DMNSN_RBRACKET);
+    dmnsn_simple_token('<', DMNSN_LT);
+    dmnsn_simple_token('>', DMNSN_GT);
+    dmnsn_simple_token('+', DMNSN_PLUS);
+    dmnsn_simple_token('-', DMNSN_MINUS);
+    dmnsn_simple_token('*', DMNSN_STAR);
+    dmnsn_simple_token('/', DMNSN_SLASH);
+    dmnsn_simple_token(',', DMNSN_COMMA);
 
     default:
       /* Unrecognised character */
@@ -82,6 +95,7 @@ dmnsn_tokenize(FILE *file)
       return NULL;
     }
 
+    dmnsn_array_push(tokens, &token);
     ++next;
   }
 
@@ -138,13 +152,27 @@ static const char *
 dmnsn_token_name(dmnsn_token_type token_type)
 {
   switch (token_type) {
-  case DMNSN_LBRACE:
-    return "{";
-
-  case DMNSN_RBRACE:
-    return "}";
+  /* Macro to shorten this huge switch */
+  #define dmnsn_token_map(type, str)                                           \
+  case type:                                                                   \
+    return str;
+
+  dmnsn_token_map(DMNSN_LBRACE,   "{");
+  dmnsn_token_map(DMNSN_RBRACE,   "}")
+  dmnsn_token_map(DMNSN_LPAREN,   "\\(");
+  dmnsn_token_map(DMNSN_RPAREN,   "\\)");
+  dmnsn_token_map(DMNSN_LBRACKET, "[");
+  dmnsn_token_map(DMNSN_RBRACKET, "]");
+  dmnsn_token_map(DMNSN_LT,       "<");
+  dmnsn_token_map(DMNSN_GT,       ">");
+  dmnsn_token_map(DMNSN_PLUS,     "+");
+  dmnsn_token_map(DMNSN_MINUS,    "-");
+  dmnsn_token_map(DMNSN_STAR,     "*");
+  dmnsn_token_map(DMNSN_SLASH,    "/");
+  dmnsn_token_map(DMNSN_COMMA,    ",");
 
   default:
+    printf("Warning: unrecognised token %d.\n", (int)token_type);
     return "unrecognized-token";
   }
 }
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 3798d1e..e64b7eb 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -20,8 +20,19 @@
 #include "../libdimension/dimension.h"
 
 typedef enum {
-  DMNSN_LBRACE,
-  DMNSN_RBRACE
+  DMNSN_LBRACE,   /* { */
+  DMNSN_RBRACE,   /* } */
+  DMNSN_LPAREN,   /* ( */
+  DMNSN_RPAREN,   /* ) */
+  DMNSN_LBRACKET, /* [ */
+  DMNSN_RBRACKET, /* ] */
+  DMNSN_LT,       /* < */
+  DMNSN_GT,       /* > */
+  DMNSN_PLUS,     /* + */
+  DMNSN_MINUS,    /* - */
+  DMNSN_STAR,     /* * */
+  DMNSN_SLASH,    /* / */
+  DMNSN_COMMA,    /* , */
 } dmnsn_token_type;
 
 typedef struct dmnsn_token dmnsn_token;
diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am
index f5ec097..7bacab5 100644
--- a/tests/dimension/Makefile.am
+++ b/tests/dimension/Makefile.am
@@ -25,4 +25,4 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir)
 tokenizer.sh:
 	cp $(srcdir)/tokenizer.sh .
 
-EXTRA_DIST = tokenizer.sh braces.pov
+EXTRA_DIST = tokenizer.sh punctuation.pov
diff --git a/tests/dimension/braces.pov b/tests/dimension/braces.pov
deleted file mode 100644
index 295a69a..0000000
--- a/tests/dimension/braces.pov
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-  {
-  }
-}
-}
-  }
-  {
-{
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
new file mode 100644
index 0000000..db9911f
--- /dev/null
+++ b/tests/dimension/punctuation.pov
@@ -0,0 +1,9 @@
+{
+  (
+    [
+      <
+        + - * / ,
+      >
+    ]
+  )
+}
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index b6de05a..d965741 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -1,10 +1,29 @@
 #!/bin/sh
 
-braces=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/braces.pov)
-braces_exp='({ { } } } } { {)'
+#########################################################################
+# Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               #
+#                                                                       #
+# This file is part of The Dimension Test Suite.                        #
+#                                                                       #
+# The Dimension Test Suite is free software; you can redistribute it    #
+# and/or modify it under the terms of the GNU General Public License as #
+# published by the Free Software Foundation; either version 3 of the    #
+# License, or (at your option) any later version.                       #
+#                                                                       #
+# The Dimension Test Suite is distributed in the hope that it will be   #
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty   #
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  #
+# General Public License for more details.                              #
+#                                                                       #
+# You should have received a copy of the GNU General Public License     #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>. #
+#########################################################################
 
-if [ "$braces" != "$braces_exp" ]; then
-  echo "braces.pov tokenized as \"$braces\"" >&2
-  echo " -- expected \"$braces_exp\"" >&2
+punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
+punctuation_exp='({ \( [ < + - * / , > ] \) })'
+
+if [ "$punctuation" != "$punctuation_exp" ]; then
+  echo "punctuation.pov tokenized as \"$punctuation\"" >&2
+  echo " -- expected \"$punctuation_exp\"" >&2
   exit 1;
 fi