realized tokens should not be arena allocated because they need to be in linear memory

diff --git a/bin/onyx b/bin/onyx
index 82a5f9a086e1a01f43c584572e2172a770118d70..53b33bfd8278737cec253aea134b5cc1bde479ad 100755 (executable)
Binary files a/bin/onyx and b/bin/onyx differ

diff --git a/include/onyxastnodes.h b/include/onyxastnodes.h
index 3fd18a17a33a2a362570f06eace11c7ad0daca8f..cd46af0544953f6d3251e9f47fbc090c253ed8eb 100644 (file)
--- a/include/onyxastnodes.h
+++ b/include/onyxastnodes.h
@@ -969,7 +969,7 @@ struct Context {
 
     CompileOptions* options;
 
-    bh_arena     token_arena, ast_arena;
+    bh_arena                  ast_arena;
     bh_allocator token_alloc, ast_alloc;
 
     bh_arr(bh_file_contents) loaded_files;

diff --git a/src/onyx.c b/src/onyx.c
index e0442773eab5d7ec1adea2fc698190aabfd2e55f..8cdf1e48360aea12f92d6a1d7b2f946fa5722b91 100644 (file)
--- a/src/onyx.c
+++ b/src/onyx.c
@@ -136,11 +136,10 @@ static void context_init(CompileOptions* opts) {
 
     onyx_errors_init(&context.loaded_files);
 
+    context.token_alloc = global_heap_allocator;
+
     // NOTE: Create the arena where tokens and AST nodes will exist
     // Prevents nodes from being scattered across memory due to fragmentation
-    bh_arena_init(&context.token_arena, global_heap_allocator, 16 * 1024 * 1024); // 16MB
-    context.token_alloc = bh_arena_allocator(&context.token_arena);
-
     bh_arena_init(&context.ast_arena, global_heap_allocator, 16 * 1024 * 1024); // 16MB
     context.ast_alloc = bh_arena_allocator(&context.ast_arena);