Begin middle-tier lexer.

5 files changed, 48 insertions, 11 deletions

diff --git a/dimension/Makefile.am b/dimension/Makefile.am
index 6f19389..1d18374 100644
--- a/dimension/Makefile.am
+++ b/dimension/Makefile.am
@@ -33,6 +33,7 @@ dimension_SOURCES = grammar.y                                                  \
                     progressbar.h                                              \
                     realize.c                                                  \
                     realize.h                                                  \
+                    tokenize.c                                                 \
                     tokenize.h                                                 \
                     utility.c                                                  \
                     utility.h                                                  \
diff --git a/dimension/grammar.y b/dimension/grammar.y
index 325efce..b41100f 100644
--- a/dimension/grammar.y
+++ b/dimension/grammar.y
@@ -27,12 +27,6 @@
 #define YYSTYPE dmnsn_parse_item
 #define YYLTYPE dmnsn_parse_location
 
-int dmnsn_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, const char *filename,
-                void *yyscanner);
-void dmnsn_yylex_init(void **scannerp);
-void dmnsn_yyset_in(FILE *file, void *scanner);
-void dmnsn_yylex_destroy(void *scanner);
-
 #define YYLLOC_DEFAULT(Current, Rhs, N)                                 \
   do {                                                                  \
     if (N) {                                                            \
diff --git a/dimension/lexer.l b/dimension/lexer.l
index 03f91dd..90c5a6b 100644
--- a/dimension/lexer.l
+++ b/dimension/lexer.l
@@ -32,9 +32,10 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-#define YY_DECL int yylex(dmnsn_parse_item *lvalp,                      \
-                          dmnsn_parse_location *llocp,                  \
-                          const char *filename, yyscan_t yyscanner)
+#define YY_DECL int dmnsn_yylex_impl(dmnsn_parse_item *lvalp,           \
+                                     dmnsn_parse_location *llocp,       \
+                                     const char *filename,              \
+                                     yyscan_t yyscanner)
 %}
 
 %x DMNSN_BLOCK_COMMENT
@@ -278,7 +279,8 @@ dmnsn_tokenize(FILE *file, const char *filename)
   yylex_init(&scanner);
   yyset_in(file, scanner);
 
-  while ((token.type = yylex(&item, &location, filename, scanner)) != 0) {
+  while ((token.type = dmnsn_yylex_impl(&item, &location, filename, scanner))
+         != 0) {
     if (token.type == 1 || token.type == 2) {
       dmnsn_delete_tokens(tokens);
       tokens = NULL;
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
new file mode 100644
index 0000000..74caace
--- /dev/null
+++ b/dimension/tokenize.c
@@ -0,0 +1,30 @@
+/*************************************************************************
+ * Copyright (C) 2010 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of Dimension.                                       *
+ *                                                                       *
+ * Dimension is free software; you can redistribute it and/or modify it  *
+ * under the terms of the GNU General Public License as published by the *
+ * Free Software Foundation; either version 3 of the License, or (at     *
+ * your option) any later version.                                       *
+ *                                                                       *
+ * Dimension is distributed in the hope that it will be useful, but      *
+ * WITHOUT ANY WARRANTY; without even the implied warranty of            *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU     *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+#include "tokenize.h"
+
+int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+                     const char *filename, void *yyscanner);
+
+int
+dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+            const char *filename, void *yyscanner)
+{
+  return dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner);
+}
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 9ea27ee..801092a 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -21,6 +21,7 @@
 #define TOKENIZE_H
 
 #include "../libdimension/dimension.h"
+#include "parse.h"
 
 #define yytokentype dmnsn_yytokentype
 #define YYSTYPE
@@ -45,7 +46,16 @@ struct dmnsn_token {
   int line, col;
 };
 
-/* For debugging */
+/* Set up the scanner */
+int  dmnsn_yylex_init(void **scannerp);
+void dmnsn_yyset_in(FILE *file, void *scanner);
+int  dmnsn_yylex_destroy(void *scanner);
+
+/* Actual lexer */
+int dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+                const char *filename, void *yyscanner);
+
+/* For debugging - returns an array of raw tokens */
 dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename);
 
 /* Token destruction */