From: Brendan Hansen <brendan.f.hansen@gmail.com>
Date: Thu, 21 Jan 2021 01:41:44 +0000 (-0600)
Subject: realized tokens should not be arena allocated because they need to be in linear memory
X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=5421da585139c121e77258914909ea1d48711633;p=onyx.git

realized tokens should not be arena allocated because they need to be in linear memory
---

diff --git a/bin/onyx b/bin/onyx
index 82a5f9a0..53b33bfd 100755
Binary files a/bin/onyx and b/bin/onyx differ
diff --git a/include/onyxastnodes.h b/include/onyxastnodes.h
index 3fd18a17..cd46af05 100644
--- a/include/onyxastnodes.h
+++ b/include/onyxastnodes.h
@@ -969,7 +969,7 @@ struct Context {
 
     CompileOptions* options;
 
-    bh_arena     token_arena, ast_arena;
+    bh_arena                  ast_arena;
     bh_allocator token_alloc, ast_alloc;
 
     bh_arr(bh_file_contents) loaded_files;
diff --git a/src/onyx.c b/src/onyx.c
index e0442773..8cdf1e48 100644
--- a/src/onyx.c
+++ b/src/onyx.c
@@ -136,11 +136,10 @@ static void context_init(CompileOptions* opts) {
 
     onyx_errors_init(&context.loaded_files);
 
+    context.token_alloc = global_heap_allocator;
+
     // NOTE: Create the arena where tokens and AST nodes will exist
     // Prevents nodes from being scattered across memory due to fragmentation
-    bh_arena_init(&context.token_arena, global_heap_allocator, 16 * 1024 * 1024); // 16MB
-    context.token_alloc = bh_arena_allocator(&context.token_arena);
-
     bh_arena_init(&context.ast_arena, global_heap_allocator, 16 * 1024 * 1024); // 16MB
     context.ast_alloc = bh_arena_allocator(&context.ast_arena);