diff options
| -rw-r--r-- | dimension/main.c | 2 | ||||
| -rw-r--r-- | dimension/tokenize.c | 118 | ||||
| -rw-r--r-- | dimension/tokenize.h | 5 |
3 files changed, 76 insertions, 49 deletions
diff --git a/dimension/main.c b/dimension/main.c index 2ccadbf..e15180f 100644 --- a/dimension/main.c +++ b/dimension/main.c @@ -98,7 +98,7 @@ main(int argc, char **argv) { } /* Tokenize the input file */ - tokens = dmnsn_tokenize(input_file); + tokens = dmnsn_tokenize(input, input_file); if (!tokens) { dmnsn_error(DMNSN_SEVERITY_HIGH, "Error tokenizing input file."); } diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 72418cc..da190be 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -20,13 +20,29 @@ #include "tokenize.h" #include <stdlib.h> #include <string.h> +#include <stdarg.h> #include <ctype.h> #include <sys/mman.h> #include <unistd.h> +static void +dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col, + const char *format, ...) +{ + va_list ap; + va_start(ap, format); + + fprintf(stderr, "%s:%u:%u: ", filename, line, col); + vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); + + va_end(ap); +} + static int -dmnsn_tokenize_comment(char *map, size_t size, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_comment(const char *filename, + unsigned int *line, unsigned int *col, + char *map, size_t size, char **next) { if (**next != '/') return 1; @@ -43,10 +59,10 @@ dmnsn_tokenize_comment(char *map, size_t size, } else if (*(*next + 1) == '*') { /* A multi-line comment block (like this one) */ do { - ++col; + ++*col; if (**next == '\n') { - ++line; - col = 0; + ++*line; + *col = 0; } ++*next; @@ -61,8 +77,9 @@ dmnsn_tokenize_comment(char *map, size_t size, } static int -dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_number(const char *filename, + unsigned int *line, unsigned int *col, + char *map, size_t size, char **next, dmnsn_token *token) { char *endf, *endi; @@ -98,8 +115,9 @@ dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token, /* Tokenize a keyword or an identifier */ static int -dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_label(const char *filename, + unsigned int *line, unsigned int *col, + char *map, size_t size, char **next, dmnsn_token *token) { unsigned int i = 0, alloc = 32; @@ -148,8 +166,10 @@ dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token, /* Tokenize a language directive (#include, #declare, etc.) */ static int -dmnsn_tokenize_directive(char *map, size_t size, dmnsn_token *token, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_directive(const char *filename, + unsigned int *line, unsigned int *col, + char *map, size_t size, char **next, + dmnsn_token *token) { unsigned int i = 0, alloc = 32; @@ -194,8 +214,9 @@ dmnsn_tokenize_directive(char *map, size_t size, dmnsn_token *token, /* Tokenize a string */ static int -dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_string(const char *filename, + unsigned int *line, unsigned int *col, + char *map, size_t size, char **next, dmnsn_token *token) { unsigned int i = 0, alloc = 32; @@ -231,10 +252,9 @@ dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token, break; default: - fprintf(stderr, - "Warning: unrecognised escape sequence '\\%c'" - " on line %u, column %u\n", - (int)**next, *line, *col); + dmnsn_diagnostic(filename, *line, *col, + "WARNING: unrecognised escape sequence '\\%c'", + (int)**next); token->value[i] = **next; break; } @@ -254,10 +274,10 @@ dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token, } dmnsn_array * -dmnsn_tokenize(FILE *file) +dmnsn_tokenize(const char *filename, FILE *file) { if (fseeko(file, 0, SEEK_END) != 0) { - fprintf(stderr, "Couldn't seek on input stream.\n"); + fprintf(stderr, "Couldn't seek on input stream\n"); return NULL; } @@ -265,14 +285,14 @@ dmnsn_tokenize(FILE *file) int fd = fileno(file); if (fd == -1) { - fprintf(stderr, "Couldn't get file descriptor to input stream.\n"); + fprintf(stderr, "Couldn't get file descriptor to input stream\n"); return NULL; } char *map = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0), *next = map; if (map == MAP_FAILED) { - fprintf(stderr, "Couldn't mmap() input stream.\n"); + fprintf(stderr, "Couldn't mmap() input stream\n"); return NULL; } @@ -328,7 +348,8 @@ dmnsn_tokenize(FILE *file) /* Possible comment */ case '/': - if (dmnsn_tokenize_comment(map, size, &next, &line, &col) == 0) { + if (dmnsn_tokenize_comment(filename, &line, &col, + map, size, &next) == 0) { continue; } else { /* Just the normal punctuation mark */ @@ -350,41 +371,43 @@ dmnsn_tokenize(FILE *file) case '7': case '8': case '9': - if (dmnsn_tokenize_number(map, size, &token, &next, &line, &col) != 0) { - fprintf(stderr, "Invalid numeric value on line %u, column %u.\n", - line, col); + if (dmnsn_tokenize_number(filename, &line, &col, + map, size, &next, &token) != 0) { + dmnsn_diagnostic(filename, line, col, "Invalid numeric value"); goto bailout; } break; case '#': /* Language directive */ - if (dmnsn_tokenize_directive(map, size, &token, - &next, &line, &col) == 0) { + if (dmnsn_tokenize_directive(filename, &line, &col, + map, size, &next, &token) == 0) { if (token.type == DMNSN_INCLUDE) { /* Skip whitespace */ while (next - map < size && isspace(*next) && *next != '\n') { ++next; } - if (dmnsn_tokenize_string(map, size, &token, - &next, &line, &col) != 0) { - fprintf(stderr, - "Expected string after #include on line %u, column %u.\n", - line, col); + if (dmnsn_tokenize_string(filename, &line, &col, + map, size, &next, &token) != 0) { + dmnsn_diagnostic(filename, line, col, + "Expected string after #include"); goto bailout; } FILE *included = fopen(token.value, "r"); if (!included) { - fprintf(stderr, "Couldn't include \"%s\" on line %u, column %u.\n", - token.value, line, col); + dmnsn_diagnostic(filename, line, col, + "Couldn't open included file \"%s\"", + token.value); goto bailout; } - dmnsn_array *included_tokens = dmnsn_tokenize(included); + dmnsn_array *included_tokens = dmnsn_tokenize(token.value, included); if (!included_tokens) { - fprintf(stderr, "Error tokenizing \"%s\"\n", token.value); + dmnsn_diagnostic(filename, line, col, + "Error tokenizing included file \"%s\"", + token.value); goto bailout; } @@ -393,36 +416,38 @@ dmnsn_tokenize(FILE *file) dmnsn_array_push(tokens, dmnsn_array_at(included_tokens, i)); } + free(token.value); dmnsn_delete_array(included_tokens); continue; } } else { - fprintf(stderr, "Invalid directive on line %u, column %u.\n", - line, col); + dmnsn_diagnostic(filename, line, col, "Invalid language directive"); goto bailout; } break; case '"': - if (dmnsn_tokenize_string(map, size, &token, &next, &line, &col) != 0) { - fprintf(stderr, "Invalid string on line %u, column %u.\n", - line, col); + if (dmnsn_tokenize_string(filename, &line, &col, + map, size, &next, &token) != 0) { + dmnsn_diagnostic(filename, line, col, "Invalid string"); goto bailout; } break; default: - if (dmnsn_tokenize_label(map, size, &token, &next, &line, &col) != 0) { + if (dmnsn_tokenize_label(filename, &line, &col, + map, size, &next, &token) != 0) { /* Unrecognised character */ - fprintf(stderr, - "Unrecognized character '%c' (0x%X) in input at line %u," - " column %u.\n", - (int)*next, (unsigned int)*next, line, col); + dmnsn_diagnostic(filename, line, col, + "Unrecognized character '%c' (0x%X)", + (int)*next, (unsigned int)*next); goto bailout; } break; } + token.filename = malloc(strlen(filename) + 1); + strcpy(token.filename, filename); dmnsn_array_push(tokens, &token); } @@ -442,6 +467,7 @@ dmnsn_delete_tokens(dmnsn_array *tokens) unsigned int i; for (i = 0; i < dmnsn_array_size(tokens); ++i) { token = dmnsn_array_at(tokens, i); + free(token->filename); free(token->value); } dmnsn_delete_array(tokens); diff --git a/dimension/tokenize.h b/dimension/tokenize.h index cc8148f..35255c3 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -62,10 +62,11 @@ struct dmnsn_token { dmnsn_token_type type; char *value; - /* Line and column numbers from source code */ + /* File name, and line and column numbers from source code */ + char *filename; unsigned int line, col; }; -dmnsn_array *dmnsn_tokenize(FILE *file); +dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file); void dmnsn_delete_tokens(dmnsn_array *tokens); void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens); |