From: Brendan Hansen <brendan.f.hansen@gmail.com>
Date: Thu, 14 May 2020 22:18:33 +0000 (-0500)
Subject: Fixed Last tokens produced by tokenizer appear to be wrong. #7
X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=931d23c98cc22749bfc41f438dbfa71e393d2650;p=onyx.git

Fixed Last tokens produced by tokenizer appear to be wrong. #7
---

diff --git a/bh.h b/bh.h
index 12de3f4e..2af69298 100644
--- a/bh.h
+++ b/bh.h
@@ -383,7 +383,7 @@ char charset_contains(const char* charset, char ch) {
 
 i64 chars_match(char* ptr1, char* ptr2) {
 	i64 len = 0;
-	while (*ptr1 == *ptr2) ptr1++, ptr2++, len++;
+	while (*ptr2 != '\0' && *ptr1 == *ptr2) ptr1++, ptr2++, len++;
 	return *ptr2 == '\0' ? len : 0;
 }
 
diff --git a/onyx b/onyx
index 12c20cea..40b5131c 100755
Binary files a/onyx and b/onyx differ
diff --git a/onyx.c b/onyx.c
index a7925bb1..ebb03052 100644
--- a/onyx.c
+++ b/onyx.c
@@ -25,7 +25,7 @@ int main(int argc, char *argv[]) {
 	printf("There are %d tokens (Allocated space for %d tokens)\n", bh_arr_length(token_arr), bh_arr_capacity(token_arr));
 
 	for (OnyxToken* it = token_arr; !bh_arr_end(token_arr, it); it++) {
-		printf("%s (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), it->line_number, it->line_column);
+		printf("%s '%c' (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), *(char *)it->token, it->line_number, it->line_column);
 	}
 
 	bh_hash_iterator it = bh_hash_iter_setup(u16, symbol_count);
diff --git a/onyxlex.c b/onyxlex.c
index dbbfad53..91224a03 100644
--- a/onyxlex.c
+++ b/onyxlex.c
@@ -240,7 +240,7 @@ bh_arr(OnyxToken) onyx_parse_tokens(bh_file_contents *fc, bh_hash(u16) symcount)
 	OnyxTokenizer tknizer = {
 		.start 			= fc->data,
 		.curr 			= fc->data,
-		.end 			= fc->data + fc->length - 1,
+		.end 			= fc->data + fc->length,
 		.line_number 	= 1,
 		.line_start 	= fc->data,
 		.symbol_count   = symcount,