Fixed Last tokens produced by tokenizer appear to be wrong. #7

diff --git a/bh.h b/bh.h
index 12de3f4eced158eef489dac66d49c9467bbd24eb..2af692984d1a4b5d283be621cde64e143877b9c2 100644 (file)
--- a/bh.h
+++ b/bh.h
@@ -383,7 +383,7 @@ char charset_contains(const char* charset, char ch) {
 
 i64 chars_match(char* ptr1, char* ptr2) {
        i64 len = 0;
-       while (*ptr1 == *ptr2) ptr1++, ptr2++, len++;
+       while (*ptr2 != '\0' && *ptr1 == *ptr2) ptr1++, ptr2++, len++;
        return *ptr2 == '\0' ? len : 0;
 }
 

diff --git a/onyx b/onyx
index 12c20cea85d1dbf22f3d155297ce643cb8735fb1..40b5131c991064593ef5ed7553e85cfb60b7276c 100755 (executable)
Binary files a/onyx and b/onyx differ

diff --git a/onyx.c b/onyx.c
index a7925bb16de6c10a4a79adcef9d91b28b779f6b6..ebb030522930c8450c1cd1ce0ffe5d281105a9a1 100644 (file)
--- a/onyx.c
+++ b/onyx.c
@@ -25,7 +25,7 @@ int main(int argc, char *argv[]) {
        printf("There are %d tokens (Allocated space for %d tokens)\n", bh_arr_length(token_arr), bh_arr_capacity(token_arr));
 
        for (OnyxToken* it = token_arr; !bh_arr_end(token_arr, it); it++) {
-               printf("%s (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), it->line_number, it->line_column);
+               printf("%s '%c' (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), *(char *)it->token, it->line_number, it->line_column);
        }
 
        bh_hash_iterator it = bh_hash_iter_setup(u16, symbol_count);

diff --git a/onyxlex.c b/onyxlex.c
index dbbfad53ec068ad9c0c6f43aa3a602a3b79b44e1..91224a03de10b74eadb4f41c3b571586e62c6414 100644 (file)
--- a/onyxlex.c
+++ b/onyxlex.c
@@ -240,7 +240,7 @@ bh_arr(OnyxToken) onyx_parse_tokens(bh_file_contents *fc, bh_hash(u16) symcount)
        OnyxTokenizer tknizer = {
                .start                  = fc->data,
                .curr                   = fc->data,
-               .end                    = fc->data + fc->length - 1,
+               .end                    = fc->data + fc->length,
                .line_number    = 1,
                .line_start     = fc->data,
                .symbol_count   = symcount,