From: Brendan Hansen Date: Fri, 24 Jul 2020 03:36:56 +0000 (-0500) Subject: BIG changes to the way symbols are loaded; added a proper package system X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=be9280282e1c931f477735a700011f3c47641950;p=onyx.git BIG changes to the way symbols are loaded; added a proper package system --- diff --git a/Makefile b/Makefile index 3dbbbe5e..ca7a3d7f 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ OBJ_FILES=\ - build/onyxlex.o \ - build/onyxparser.o \ + build/onyxlex.o \ + build/onyxparser.o \ build/onyxtypes.o \ - build/onyxsempass.o \ - build/onyxsymres.o \ - build/onyxchecker.o \ - build/onyxmsgs.o \ - build/onyxutils.o \ - build/onyxwasm.o \ - build/onyx.o + build/onyxsempass.o \ + build/onyxsymres.o \ + build/onyxchecker.o \ + build/onyxmsgs.o \ + build/onyxutils.o \ + build/onyxwasm.o \ + build/onyx.o CC=gcc INCLUDES=-I./include @@ -25,6 +25,10 @@ $(TARGET): $(OBJ_FILES) install: $(TARGET) cp $(TARGET) /usr/bin/ +install_syntax: misc/onyx.vim misc/onyx.sublime-syntax + cp ./misc/onyx.vim /usr/share/vim/vim82/syntax/onyx.vim + cp ./misc/onyx.sublime-syntax /home/brendan/.config/sublime-text-3/Packages/User/onyx.sublime-syntax + clean: rm -f $(OBJ_FILES) 2>&1 >/dev/null diff --git a/include/onyxastnodes.h b/include/onyxastnodes.h index 270cd530..66985c1d 100644 --- a/include/onyxastnodes.h +++ b/include/onyxastnodes.h @@ -40,11 +40,14 @@ typedef struct AstStructType AstStructType; typedef struct AstStructMember AstStructMember; typedef struct AstBinding AstBinding; -typedef struct AstUse AstUse; +typedef struct AstIncludeFile AstIncludeFile; +typedef struct AstUsePackage AstUsePackage; typedef struct AstGlobal AstGlobal; typedef struct AstFunction AstFunction; typedef struct AstOverloadedFunction AstOverloadedFunction; +typedef struct AstPackage AstPackage; +typedef struct Package Package; typedef struct Scope { struct Scope *parent; @@ -57,7 +60,9 @@ extern Scope* scope_create(bh_allocator a, Scope* parent); typedef enum AstKind { Ast_Kind_Error, Ast_Kind_Program, - Ast_Kind_Use, + Ast_Kind_Package, + Ast_Kind_Include_File, + Ast_Kind_Use_Package, Ast_Kind_Binding, Ast_Kind_Function, @@ -302,8 +307,15 @@ struct AstStructMember { AstTyped_base; u64 offset; }; // Top level nodes struct AstBinding { AstTyped_base; AstNode* node; }; -struct AstForeign { AstNode_base; OnyxToken *mod_token, *name_token; AstNode *import; }; -struct AstUse { AstNode_base; OnyxToken *filename; }; +struct AstForeign { AstNode_base; OnyxToken *mod_token, *name_token; AstNode *import; }; +struct AstIncludeFile { AstNode_base; OnyxToken *filename; }; +struct AstUsePackage { + AstNode_base; + + AstPackage *package; + OnyxToken *alias; + bh_arr(OnyxToken *) only; +}; struct AstGlobal { AstTyped_base; @@ -343,12 +355,18 @@ struct AstOverloadedFunction { bh_arr(AstTyped *) overloads; }; +struct AstPackage { + AstNode_base; + + Package* package; +}; // NOTE: An Entity represents something will need to be // processed later down the pipeline. typedef enum EntityType { Entity_Type_Unknown, + Entity_Type_Use_Package, Entity_Type_String_Literal, Entity_Type_Struct, Entity_Type_Function_Header, @@ -361,8 +379,10 @@ typedef enum EntityType { typedef struct Entity { EntityType type; + Scope *scope; union { + AstUsePackage *use_package; AstFunction *function; AstOverloadedFunction *overloaded_function; AstGlobal *global; @@ -372,10 +392,16 @@ typedef struct Entity { }; } Entity; +struct Package { + char *name; + Scope *scope; +}; // NOTE: Simple data structure for storing what comes out of the parser typedef struct ProgramInfo { - bh_arr(AstBinding *) bindings; + Scope *global_scope; + + bh_table(Package *) packages; bh_arr(Entity) entities; u32 foreign_func_count; diff --git a/include/onyxlex.h b/include/onyxlex.h index cc27be0f..c81ab399 100644 --- a/include/onyxlex.h +++ b/include/onyxlex.h @@ -10,55 +10,55 @@ typedef enum TokenType { Token_Type_Comment = 258, - Token_Type_Keyword_Struct = 259, - Token_Type_Keyword_Enum = 260, - Token_Type_Keyword_Use = 261, - Token_Type_Keyword_Export = 262, - Token_Type_Keyword_If = 263, - Token_Type_Keyword_Else = 264, - Token_Type_Keyword_Elseif = 265, - Token_Type_Keyword_Return = 266, - Token_Type_Keyword_Global = 267, - Token_Type_Keyword_Proc = 268, - Token_Type_Keyword_Cast = 269, - Token_Type_Keyword_While = 270, - Token_Type_Keyword_For = 271, - Token_Type_Keyword_Break = 272, - Token_Type_Keyword_Continue = 273, - Token_Type_Keyword_Sizeof = 274, - - Token_Type_Right_Arrow = 275, - Token_Type_Left_Arrow = 276, - Token_Type_Empty_Block = 277, - - Token_Type_Greater_Equal = 278, - Token_Type_Less_Equal = 279, - Token_Type_Equal_Equal = 280, - Token_Type_Not_Equal = 281, - Token_Type_Plus_Equal = 282, - Token_Type_Minus_Equal = 283, - Token_Type_Star_Equal = 284, - Token_Type_Fslash_Equal = 285, - Token_Type_Percent_Equal = 286, - Token_Type_And_Equal = 287, - Token_Type_Or_Equal = 288, - Token_Type_Xor_Equal = 289, - Token_Type_And_And = 290, - Token_Type_Or_Or = 291, - Token_Type_Shift_Left = 292, - Token_Type_Shift_Right = 293, - Token_Type_Shift_Arith_Right = 294, - Token_Type_Shl_Equal = 295, - Token_Type_Shr_Equal = 296, - Token_Type_Sar_Equal = 297, - - Token_Type_Symbol = 298, - Token_Type_Literal_String = 299, - Token_Type_Literal_Numeric = 300, - Token_Type_Literal_True = 301, - Token_Type_Literal_False = 302, - - Token_Type_Count = 303, + Token_Type_Keyword_Package, + Token_Type_Keyword_Struct, + Token_Type_Keyword_Enum, + Token_Type_Keyword_Use, + Token_Type_Keyword_If, + Token_Type_Keyword_Else, + Token_Type_Keyword_Elseif, + Token_Type_Keyword_Return, + Token_Type_Keyword_Global, + Token_Type_Keyword_Proc, + Token_Type_Keyword_Cast, + Token_Type_Keyword_While, + Token_Type_Keyword_For, + Token_Type_Keyword_Break, + Token_Type_Keyword_Continue, + Token_Type_Keyword_Sizeof, + + Token_Type_Right_Arrow, + Token_Type_Left_Arrow, + Token_Type_Empty_Block, + + Token_Type_Greater_Equal, + Token_Type_Less_Equal, + Token_Type_Equal_Equal, + Token_Type_Not_Equal, + Token_Type_Plus_Equal, + Token_Type_Minus_Equal, + Token_Type_Star_Equal, + Token_Type_Fslash_Equal, + Token_Type_Percent_Equal, + Token_Type_And_Equal, + Token_Type_Or_Equal, + Token_Type_Xor_Equal, + Token_Type_And_And, + Token_Type_Or_Or, + Token_Type_Shift_Left, + Token_Type_Shift_Right, + Token_Type_Shift_Arith_Right, + Token_Type_Shl_Equal, + Token_Type_Shr_Equal, + Token_Type_Sar_Equal, + + Token_Type_Symbol, + Token_Type_Literal_String, + Token_Type_Literal_Numeric, + Token_Type_Literal_True, + Token_Type_Literal_False, + + Token_Type_Count, } TokenType; typedef struct OnyxFilePos { diff --git a/include/onyxparser.h b/include/onyxparser.h index 339c2549..7eeac8b0 100644 --- a/include/onyxparser.h +++ b/include/onyxparser.h @@ -7,20 +7,27 @@ #include "onyxmsgs.h" #include "onyxastnodes.h" +typedef struct NodeToProcess { + Scope *scope; + AstNode *node; +} NodeToProcess; + typedef struct ParseResults { // NOTE: The allocator used to make the arrays below bh_allocator allocator; - bh_arr(AstUse *) uses; - bh_arr(AstBinding *) bindings; + bh_arr(AstIncludeFile *) files; // NOTE: Contains all the nodes that will need some processing (symbol resolution, type checking) - bh_arr(AstNode *) nodes_to_process; + bh_arr(NodeToProcess) nodes_to_process; } ParseResults; typedef struct OnyxParser { bh_allocator allocator; + ProgramInfo *program; + Scope *package_scope; + // NOTE: not used since all tokens are lexed before parsing starts OnyxTokenizer *tokenizer; OnyxToken *prev; @@ -31,7 +38,7 @@ typedef struct OnyxParser { const char* onyx_ast_node_kind_string(AstKind kind); void* onyx_ast_node_new(bh_allocator alloc, i32 size, AstKind kind); -OnyxParser onyx_parser_create(bh_allocator alloc, OnyxTokenizer *tokenizer); +OnyxParser onyx_parser_create(bh_allocator alloc, OnyxTokenizer *tokenizer, ProgramInfo *program); void onyx_parser_free(OnyxParser* parser); ParseResults onyx_parse(OnyxParser *parser); diff --git a/include/onyxutils.h b/include/onyxutils.h index 5154227c..70bdb32d 100644 --- a/include/onyxutils.h +++ b/include/onyxutils.h @@ -10,4 +10,15 @@ extern bh_allocator global_heap_allocator; const char* onyx_ast_node_kind_string(AstKind kind); +void program_info_init(ProgramInfo* prog, bh_allocator alloc); +Package* program_info_package_lookup(ProgramInfo* prog, char* package_name); +Package* program_info_package_lookup_or_create(ProgramInfo* prog, char* package_name, Scope* parent_scope, bh_allocator alloc); + +Scope* scope_create(bh_allocator a, Scope* parent); +void scope_include(Scope* target, Scope* source); +b32 symbol_introduce(Scope* scope, OnyxToken* tkn, AstNode* symbol); +b32 symbol_raw_introduce(Scope* scope, char* tkn, OnyxFilePos pos, AstNode* symbol); +void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node); +AstNode* symbol_resolve(Scope* start_scope, OnyxToken* tkn); + void onyx_ast_print(AstNode* program, i32 indent); diff --git a/misc/onyx.sublime-syntax b/misc/onyx.sublime-syntax index 24416dce..acc867e7 100644 --- a/misc/onyx.sublime-syntax +++ b/misc/onyx.sublime-syntax @@ -23,10 +23,10 @@ contexts: # strings in YAML. When using single quoted strings, only single quotes # need to be escaped: this is done by using two single quotes next to each # other. - - match: '\b(struct|proc|use|global|enum|if|elseif|else|for|while|do|break|continue|return|as|sizeof)\b' + - match: '\b(package|struct|proc|use|global|enum|if|elseif|else|for|while|do|break|continue|return|as|sizeof)\b' scope: keyword.control.onyx - - match: '\b(unknown|bool|void|i8|u8|i16|u16|i32|u32|i64|u64|f32|f64|rawptr)\b' + - match: '\b(bool|void|i8|u8|i16|u16|i32|u32|i64|u64|f32|f64|rawptr)\b' scope: constant.type.onyx - match: '\b(true|false)\b' diff --git a/misc/onyx.vim b/misc/onyx.vim index a5465765..95e53595 100644 --- a/misc/onyx.vim +++ b/misc/onyx.vim @@ -10,13 +10,13 @@ endif let s:cpo_save = &cpo set cpo&vim -syn keyword onyxKeyword struct proc use global +syn keyword onyxKeyword package struct proc use global syn keyword onyxKeyword if elseif else syn keyword onyxKeyword for while do syn keyword onyxKeyword break continue return syn keyword onyxKeyword as sizeof -syn keyword onyxType unknown bool void +syn keyword onyxType bool void syn keyword onyxType i8 u8 syn keyword onyxType i16 u16 syn keyword onyxType i32 u32 diff --git a/onyx b/onyx index 650cbc1f..84af5265 100755 Binary files a/onyx and b/onyx differ diff --git a/progs/arrays.onyx b/progs/arrays.onyx index 669e310c..982b12bb 100644 --- a/progs/arrays.onyx +++ b/progs/arrays.onyx @@ -1,6 +1,8 @@ use "progs/intrinsics" use "progs/print_funcs" +use package printing + global_arr :: global []i32; min_i32 :: proc (a: i32, b: i32) -> i32 { @@ -82,7 +84,6 @@ proc #export "main2" { - alloc :: proc (size: u32) -> rawptr { heap_u32 :: __heap_start as ^u32; diff --git a/progs/ez.onyx b/progs/ez.onyx new file mode 100644 index 00000000..69479551 --- /dev/null +++ b/progs/ez.onyx @@ -0,0 +1,24 @@ +use "progs/print_funcs" + +use package printing + +Foo :: struct { + x : i32; + y : i32; +} + +foo_sum :: proc (foo: ^Foo) -> i32 { + return foo.x + foo.y; +} + +proc #export "main" { + foo := __heap_start as ^Foo; + foo.x = 123; + foo.y = 321; + print(foo.foo_sum()); + + + print(1234); + + print_f32(123.0f); +} \ No newline at end of file diff --git a/progs/other.onyx b/progs/other.onyx deleted file mode 100644 index 70b9a52e..00000000 --- a/progs/other.onyx +++ /dev/null @@ -1,39 +0,0 @@ -use "progs/test" - -other_value :: proc (n: i32) -> i32 { - return 8675309 + something_else(n) + global_value; -} - -fib :: proc #export (n: i32) -> i32 { - if n == 0 { return 1; } - if n == 1 { return 1; } - - a := 0; - b := 1; - - count := n; - - while count >= 0 { - count -= 1; - - c :: a + b; - b = a; - a = c; - } - - return a; -} - -factorial :: proc #export (n: i32) -> i32 { - if n <= 1 { return 1; } - - f := 1; - i := 2; - - while i <= n { - f *= i; - i += 1; - } - - return f; -} diff --git a/progs/print_funcs.onyx b/progs/print_funcs.onyx index fb7c0582..45d7e965 100644 --- a/progs/print_funcs.onyx +++ b/progs/print_funcs.onyx @@ -1,3 +1,5 @@ +package printing + print_bool :: proc #foreign "host" "print" (value: bool) --- print_i32 :: proc #foreign "host" "print" (value: i32) --- print_f32 :: proc #foreign "host" "print" (value: f32) --- diff --git a/progs/structs.onyx b/progs/structs.onyx index 687a16ec..b05fa267 100644 --- a/progs/structs.onyx +++ b/progs/structs.onyx @@ -1,6 +1,8 @@ use "progs/print_funcs" use "progs/intrinsics" +use package printing + N :: 5 Foo :: struct { diff --git a/progs/ufc.onyx b/progs/ufc.onyx index 5ab2af79..cb363677 100644 --- a/progs/ufc.onyx +++ b/progs/ufc.onyx @@ -1,6 +1,8 @@ use "progs/intrinsics" use "progs/print_funcs" +use package printing as printing + alloc :: proc (size: u32) -> rawptr { heap_u32 :: __heap_start as ^u32; @@ -71,15 +73,15 @@ proc #export "main" { vec.x = 5.0f; vec.y = 12.0f; - print(vec.x); - print(vec.y); + printing.print(vec.x); + printing.print(vec.y); mag :: vec.magnitude(); - print(mag); + printing.print(mag); - print((*vec).dot(1.0f, 1.0f)); - print(dot(2.0f, 4.0f, -6.0f, 3.0f)); + printing.print((*vec).dot(1.0f, 1.0f)); + printing.print(dot(2.0f, 4.0f, -6.0f, 3.0f)); v3 := alloc(sizeof Vec3) as ^Vec3; @@ -87,12 +89,12 @@ proc #export "main" { v3.y = 12.0f; v3.z = 13.0f; - print(v3.magnitude()); + printing.print(v3.magnitude()); foo := 5; foo <<= 2; foo &= 6; foo |= 5; foo >>= 1; - print(foo); + printing.print(foo); } \ No newline at end of file diff --git a/src/onyx.c b/src/onyx.c index a19f67aa..9a1046cc 100644 --- a/src/onyx.c +++ b/src/onyx.c @@ -112,8 +112,7 @@ typedef struct CompilerState { static void compiler_state_init(CompilerState* compiler_state, OnyxCompileOptions* opts) { compiler_state->options = opts; - bh_arr_new(global_heap_allocator, compiler_state->prog_info.bindings, 4); - bh_arr_new(global_heap_allocator, compiler_state->prog_info.entities, 4); + program_info_init(&compiler_state->prog_info, global_heap_allocator); bh_arena_init(&compiler_state->msg_arena, opts->allocator, 4096); compiler_state->msg_alloc = bh_arena_allocator(&compiler_state->msg_arena); @@ -162,7 +161,7 @@ static ParseResults parse_source_file(CompilerState* compiler_state, bh_file_con if (compiler_state->options->verbose_output) bh_printf("[Parsing] %s\n", file_contents->filename); - OnyxParser parser = onyx_parser_create(compiler_state->ast_alloc, &tokenizer); + OnyxParser parser = onyx_parser_create(compiler_state->ast_alloc, &tokenizer, &compiler_state->prog_info); return onyx_parse(&parser); } @@ -171,68 +170,76 @@ static i32 sort_entities(const void* e1, const void* e2) { } static void merge_parse_results(CompilerState* compiler_state, ParseResults* results) { - bh_arr_each(AstUse *, use_node, results->uses) { + bh_arr_each(AstIncludeFile *, include, results->files) { char* formatted_name = bh_aprintf( global_heap_allocator, "%b.onyx", - (*use_node)->filename->text, (*use_node)->filename->length); + (*include)->filename->text, (*include)->filename->length); bh_arr_push(compiler_state->queued_files, formatted_name); } - bh_arr_each(AstBinding *, binding_node, results->bindings) - bh_arr_push(compiler_state->prog_info.bindings, *binding_node); - Entity ent; - bh_arr_each(AstNode *, node, results->nodes_to_process) { - AstKind nkind = (*node)->kind; + bh_arr_each(NodeToProcess, n, results->nodes_to_process) { + AstNode* node = n->node; + AstKind nkind = node->kind; + + ent.scope = n->scope; + switch (nkind) { case Ast_Kind_Function: { ent.type = Entity_Type_Function_Header; - ent.function = (AstFunction *) *node; + ent.function = (AstFunction *) node; bh_arr_push(compiler_state->prog_info.entities, ent); ent.type = Entity_Type_Function; - ent.function = (AstFunction *) *node; + ent.function = (AstFunction *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } case Ast_Kind_Overloaded_Function: { ent.type = Entity_Type_Overloaded_Function; - ent.overloaded_function = (AstOverloadedFunction *) *node; + ent.overloaded_function = (AstOverloadedFunction *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } case Ast_Kind_Global: { ent.type = Entity_Type_Global_Header; - ent.global = (AstGlobal *) *node; + ent.global = (AstGlobal *) node; bh_arr_push(compiler_state->prog_info.entities, ent); ent.type = Entity_Type_Global; - ent.global = (AstGlobal *) *node; + ent.global = (AstGlobal *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } case Ast_Kind_StrLit: { ent.type = Entity_Type_String_Literal; - ent.strlit = (AstStrLit *) *node; + ent.strlit = (AstStrLit *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } case Ast_Kind_Struct_Type: { ent.type = Entity_Type_Struct; - ent.struct_type = (AstStructType *) *node; + ent.struct_type = (AstStructType *) node; + bh_arr_push(compiler_state->prog_info.entities, ent); + break; + } + + case Ast_Kind_Use_Package: { + ent.type = Entity_Type_Use_Package; + ent.use_package = (AstUsePackage *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } default: { ent.type = Entity_Type_Expression; - ent.expr = (AstTyped *) *node; + ent.expr = (AstTyped *) node; bh_arr_push(compiler_state->prog_info.entities, ent); break; } diff --git a/src/onyxchecker.c b/src/onyxchecker.c index f28ba783..174d9ce2 100644 --- a/src/onyxchecker.c +++ b/src/onyxchecker.c @@ -906,7 +906,7 @@ void onyx_type_check() { bh_arr_each(Entity, entity, semstate.program->entities) { switch (entity->type) { case Entity_Type_Function_Header: - if (entity->function->flags & Ast_Kind_Foreign) + if (entity->function->flags & Ast_Flag_Foreign) semstate.program->foreign_func_count++; if (check_function_header(entity->function)) return; @@ -921,7 +921,7 @@ void onyx_type_check() { break; case Entity_Type_Global: - if (entity->global->flags & Ast_Kind_Foreign) + if (entity->global->flags & Ast_Flag_Foreign) semstate.program->foreign_global_count++; if (check_global(entity->global)) return; @@ -939,6 +939,8 @@ void onyx_type_check() { case Entity_Type_Global_Header: break; + case Entity_Type_Use_Package: break; + default: DEBUG_HERE; break; } } diff --git a/src/onyxlex.c b/src/onyxlex.c index 43a4dab6..01db3fee 100644 --- a/src/onyxlex.c +++ b/src/onyxlex.c @@ -8,9 +8,10 @@ static const char* token_type_names[] = { "TOKEN_TYPE_COMMENT", + "package", "struct", + "enum", "use", - "export", "if", "else", "elseif", @@ -42,13 +43,12 @@ static const char* token_type_names[] = { "^=", "&&", "||", - "^^", "<<", ">>", ">>>", "<<=", ">>=", - ">>>=" + ">>>=", "TOKEN_TYPE_SYMBOL", "TOKEN_TYPE_LITERAL_STRING", @@ -142,6 +142,7 @@ OnyxToken* onyx_get_token(OnyxTokenizer* tokenizer) { goto token_parsed; } + LITERAL_TOKEN("package", 1, Token_Type_Keyword_Package); LITERAL_TOKEN("struct", 1, Token_Type_Keyword_Struct); LITERAL_TOKEN("enum" , 1, Token_Type_Keyword_Enum); LITERAL_TOKEN("use", 1, Token_Type_Keyword_Use); diff --git a/src/onyxparser.c b/src/onyxparser.c index f27fb0a9..bc1bf2b2 100644 --- a/src/onyxparser.c +++ b/src/onyxparser.c @@ -73,6 +73,15 @@ static OnyxToken* expect_token(OnyxParser* parser, TokenType token_type) { return token; } +static void add_node_to_process(OnyxParser* parser, AstNode* node) { + bh_arr_push(parser->results.nodes_to_process, ((NodeToProcess) { + .scope = parser->package_scope, + .node = node, + })); +} + + + static AstNumLit* parse_numeric_literal(OnyxParser* parser) { AstNumLit* lit_node = make_node(AstNumLit, Ast_Kind_NumLit); lit_node->token = expect_token(parser, Token_Type_Literal_Numeric); @@ -210,7 +219,7 @@ static AstTyped* parse_factor(OnyxParser* parser) { str_node->type_node = (AstType *) str_type; str_node->addr = 0; - bh_arr_push(parser->results.nodes_to_process, (AstNode *) str_node); + add_node_to_process(parser, (AstNode *) str_node); retval = (AstTyped *) str_node; break; @@ -1034,7 +1043,7 @@ static AstTyped* parse_global_declaration(OnyxParser* parser) { global_node->type_node = parse_type(parser); - bh_arr_push(parser->results.nodes_to_process, (AstNode *) global_node); + add_node_to_process(parser, (AstNode *) global_node); return (AstTyped *) global_node; } @@ -1046,7 +1055,7 @@ static AstTyped* parse_top_level_expression(OnyxParser* parser) { if (parser->curr->type == Token_Type_Keyword_Proc) { AstFunction* func_node = parse_function_definition(parser); - bh_arr_push(parser->results.nodes_to_process, (AstNode *) func_node); + add_node_to_process(parser, (AstNode *) func_node); return (AstTyped *) func_node; } @@ -1070,11 +1079,51 @@ static AstTyped* parse_top_level_expression(OnyxParser* parser) { static AstNode* parse_top_level_statement(OnyxParser* parser) { switch (parser->curr->type) { case Token_Type_Keyword_Use: { - AstUse* use_node = make_node(AstUse, Ast_Kind_Use); - use_node->token = expect_token(parser, Token_Type_Keyword_Use); - use_node->filename = expect_token(parser, Token_Type_Literal_String); + OnyxToken* use_token = expect_token(parser, Token_Type_Keyword_Use); + + if (parser->curr->type == Token_Type_Keyword_Package) { + consume_token(parser); + + AstUsePackage* upack = make_node(AstUsePackage, Ast_Kind_Use_Package); + upack->token = use_token; + + AstNode* pack_symbol = make_node(AstNode, Ast_Kind_Symbol); + pack_symbol->token = expect_token(parser, Token_Type_Symbol); + upack->package = (AstPackage *) pack_symbol; + + // followed by 'as' + if (parser->curr->type == Token_Type_Keyword_Cast) { + consume_token(parser); + upack->alias = expect_token(parser, Token_Type_Symbol); + } + + if (parser->curr->type == '{') { + consume_token(parser); + + bh_arr_new(global_heap_allocator, upack->only, 4); + + while (parser->curr->type != '}') { + OnyxToken* only_token = expect_token(parser, Token_Type_Symbol); + + bh_arr_push(upack->only, only_token); + + if (parser->curr->type != '}') + expect_token(parser, ','); + } - return (AstNode *) use_node; + consume_token(parser); + } + + add_node_to_process(parser, (AstNode *) upack); + return NULL; + + } else { + AstIncludeFile* include = make_node(AstIncludeFile, Ast_Kind_Include_File); + include->token = use_token; + include->filename = expect_token(parser, Token_Type_Literal_String); + + return (AstNode *) include; + } } case Token_Type_Keyword_Proc: @@ -1111,7 +1160,7 @@ static AstNode* parse_top_level_statement(OnyxParser* parser) { } // HACK - bh_arr_push(parser->results.nodes_to_process, (AstNode *) node); + add_node_to_process(parser, (AstNode *) node); } AstBinding* binding = make_node(AstBinding, Ast_Kind_Binding); @@ -1142,24 +1191,23 @@ void* onyx_ast_node_new(bh_allocator alloc, i32 size, AstKind kind) { return node; } -OnyxParser onyx_parser_create(bh_allocator alloc, OnyxTokenizer *tokenizer) { +OnyxParser onyx_parser_create(bh_allocator alloc, OnyxTokenizer *tokenizer, ProgramInfo* program) { OnyxParser parser; parser.allocator = alloc; parser.tokenizer = tokenizer; parser.curr = tokenizer->tokens; parser.prev = NULL; + parser.program = program; parser.results = (ParseResults) { .allocator = global_heap_allocator, - .uses = NULL, - .bindings = NULL, + .files = NULL, .nodes_to_process = NULL, }; - bh_arr_new(parser.results.allocator, parser.results.uses, 4); - bh_arr_new(parser.results.allocator, parser.results.bindings, 4); + bh_arr_new(parser.results.allocator, parser.results.files, 4); bh_arr_new(parser.results.allocator, parser.results.nodes_to_process, 4); return parser; @@ -1169,6 +1217,31 @@ void onyx_parser_free(OnyxParser* parser) { } ParseResults onyx_parse(OnyxParser *parser) { + if (parser->curr->type == Token_Type_Keyword_Package) { + consume_token(parser); + + OnyxToken* symbol = expect_token(parser, Token_Type_Symbol); + + token_toggle_end(symbol); + Package *package = program_info_package_lookup_or_create( + parser->program, + symbol->text, + parser->program->global_scope, + parser->allocator); + token_toggle_end(symbol); + + parser->package_scope = package->scope; + + } else { + Package *package = program_info_package_lookup_or_create( + parser->program, + "main", + parser->program->global_scope, + parser->allocator); + + parser->package_scope = package->scope; + } + while (parser->curr->type != Token_Type_End_Stream) { AstNode* curr_stmt = parse_top_level_statement(parser); @@ -1176,8 +1249,14 @@ ParseResults onyx_parse(OnyxParser *parser) { while (curr_stmt != NULL) { switch (curr_stmt->kind) { - case Ast_Kind_Use: bh_arr_push(parser->results.uses, (AstUse *) curr_stmt); break; - case Ast_Kind_Binding: bh_arr_push(parser->results.bindings, (AstBinding *) curr_stmt); break; + case Ast_Kind_Include_File: bh_arr_push(parser->results.files, (AstIncludeFile *) curr_stmt); break; + case Ast_Kind_Binding: { + symbol_introduce(parser->package_scope, + ((AstBinding *) curr_stmt)->token, + ((AstBinding *) curr_stmt)->node); + break; + } + default: assert(("Invalid top level node", 0)); } diff --git a/src/onyxsymres.c b/src/onyxsymres.c index 7964578b..18e5df55 100644 --- a/src/onyxsymres.c +++ b/src/onyxsymres.c @@ -1,6 +1,7 @@ #define BH_DEBUG #include "onyxsempass.h" #include "onyxparser.h" +#include "onyxutils.h" AstBasicType basic_type_void = { { Ast_Kind_Basic_Type, 0, NULL, "void" }, &basic_types[Basic_Kind_Void] }; AstBasicType basic_type_bool = { { Ast_Kind_Basic_Type, 0, NULL, "bool" }, &basic_types[Basic_Kind_Bool] }; @@ -39,9 +40,6 @@ const BuiltinSymbol builtin_symbols[] = { { NULL, NULL }, }; -static b32 symbol_introduce(OnyxToken* tkn, AstNode* symbol); -static void symbol_builtin_introduce(char* sym, AstNode *node); -static AstNode* symbol_resolve(OnyxToken* tkn); static void scope_enter(Scope* new_scope); static void scope_leave(); @@ -49,6 +47,7 @@ static AstType* symres_type(AstType* type); static void symres_local(AstLocal** local); static void symres_call(AstCall* call); static void symres_size_of(AstSizeOf* so); +static void symres_field_access(AstFieldAccess** fa); static void symres_expression(AstTyped** expr); static void symres_return(AstReturn* ret); static void symres_if(AstIf* ifnode); @@ -60,62 +59,7 @@ static void symres_block(AstBlock* block); static void symres_function(AstFunction* func); static void symres_global(AstGlobal* global); static void symres_overloaded_function(AstOverloadedFunction* ofunc); - -static b32 symbol_introduce(OnyxToken* tkn, AstNode* symbol) { - token_toggle_end(tkn); - - if (bh_table_has(AstNode *, semstate.curr_scope->symbols, tkn->text)) { - onyx_message_add(Msg_Type_Redeclare_Symbol, - tkn->pos, - tkn->text); - token_toggle_end(tkn); - return 0; - } - - bh_table_put(AstNode *, semstate.curr_scope->symbols, tkn->text, symbol); - - if (symbol->kind == Ast_Kind_Local) - bh_arr_push(semstate.curr_function->locals, (AstLocal *) symbol); - - token_toggle_end(tkn); - return 1; -} - -static void symbol_builtin_introduce(char* sym, AstNode *node) { - bh_table_put(AstNode *, semstate.curr_scope->symbols, sym, node); -} - -static AstNode* symbol_resolve(OnyxToken* tkn) { - token_toggle_end(tkn); - - AstNode* res = NULL; - Scope* scope = semstate.curr_scope; - - while (res == NULL && scope != NULL) { - if (bh_table_has(AstNode *, scope->symbols, tkn->text)) { - res = bh_table_get(AstNode *, scope->symbols, tkn->text); - } else { - scope = scope->parent; - } - } - - if (res == NULL ) { - onyx_message_add(Msg_Type_Unknown_Symbol, - tkn->pos, - tkn->text); - - token_toggle_end(tkn); - return NULL; - } - - if (res->kind == Ast_Kind_Symbol) { - token_toggle_end(tkn); - return symbol_resolve(res->token); - } - - token_toggle_end(tkn); - return res; -} +static void symres_use_package(AstUsePackage* package); static void scope_enter(Scope* new_scope) { new_scope->parent = semstate.curr_scope; @@ -130,7 +74,7 @@ static AstType* symres_type(AstType* type) { if (type == NULL) return NULL; if (type->kind == Ast_Kind_Symbol) { - return (AstType *) symbol_resolve(((AstNode *) type)->token); + return (AstType *) symbol_resolve(semstate.curr_scope, ((AstNode *) type)->token); } // NOTE: Already resolved @@ -183,38 +127,29 @@ static AstType* symres_type(AstType* type) { static void symres_local(AstLocal** local) { (*local)->type_node = symres_type((*local)->type_node); - symbol_introduce((*local)->token, (AstNode *) *local); + bh_arr_push(semstate.curr_function->locals, *local); + + symbol_introduce(semstate.curr_scope, (*local)->token, (AstNode *) *local); } static void symres_call(AstCall* call) { + symres_expression((AstTyped **) &call->callee); + if (call->callee == NULL) return; + if (call->callee->kind == Ast_Kind_Field_Access) { AstFieldAccess* fa = (AstFieldAccess *) call->callee; - - AstTyped* symbol = onyx_ast_node_new(semstate.node_allocator, sizeof(AstTyped), Ast_Kind_Symbol); - symbol->token = fa->token; + if (fa->expr == NULL) return; AstArgument* implicit_arg = onyx_ast_node_new(semstate.node_allocator, sizeof(AstArgument), Ast_Kind_Argument); implicit_arg->value = fa->expr; implicit_arg->token = fa->expr->token; implicit_arg->next = (AstNode *) call->arguments; - call->callee = (AstNode *) symbol; + call->callee = symbol_resolve(semstate.curr_scope, fa->token); call->arguments = implicit_arg; call->arg_count++; } - AstNode* callee = symbol_resolve(call->callee->token); - if (callee) - call->callee = callee; - else { - token_toggle_end(call->callee->token); - onyx_message_add(Msg_Type_Unknown_Symbol, - call->callee->token->pos, - call->callee->token->text); - token_toggle_end(call->callee->token); - return; - } - symres_statement_chain((AstNode *) call->arguments, (AstNode **) &call->arguments); } @@ -223,6 +158,20 @@ static void symres_size_of(AstSizeOf* so) { so->so_type = symres_type(so->so_type); } +static void symres_field_access(AstFieldAccess** fa) { + symres_expression(&(*fa)->expr); + if ((*fa)->expr == NULL) return; + + if ((*fa)->expr->kind == Ast_Kind_Package) { + AstPackage* package = (AstPackage *) (*fa)->expr; + AstNode* n = symbol_resolve(package->package->scope, (*fa)->token); + if (n) { + // NOTE: not field access + *fa = (AstFieldAccess *) n; + } + } +} + static void symres_unaryop(AstUnaryOp** unaryop) { if ((*unaryop)->operation == Unary_Op_Cast) { (*unaryop)->type_node = symres_type((*unaryop)->type_node); @@ -243,7 +192,7 @@ static void symres_expression(AstTyped** expr) { case Ast_Kind_Block: symres_block((AstBlock *) *expr); break; case Ast_Kind_Symbol: - *expr = (AstTyped *) symbol_resolve(((AstNode *) *expr)->token); + *expr = (AstTyped *) symbol_resolve(semstate.curr_scope, ((AstNode *) *expr)->token); break; // NOTE: This is a good case, since it means the symbol is already resolved @@ -257,7 +206,7 @@ static void symres_expression(AstTyped** expr) { case Ast_Kind_Address_Of: symres_expression(&((AstAddressOf *)(*expr))->expr); break; case Ast_Kind_Dereference: symres_expression(&((AstDereference *)(*expr))->expr); break; - case Ast_Kind_Field_Access: symres_expression(&((AstFieldAccess *)(*expr))->expr); break; + case Ast_Kind_Field_Access: symres_field_access((AstFieldAccess **) expr); break; case Ast_Kind_Size_Of: symres_size_of((AstSizeOf *)*expr); break; case Ast_Kind_Array_Access: @@ -298,7 +247,8 @@ static void symres_for(AstFor* fornode) { fornode->scope = scope_create(semstate.node_allocator, semstate.curr_scope); scope_enter(fornode->scope); - symbol_introduce(fornode->var->token, (AstNode *) fornode->var); + bh_arr_push(semstate.curr_function->locals, fornode->var); + symbol_introduce(semstate.curr_scope, fornode->var->token, (AstNode *) fornode->var); symres_expression(&fornode->start); symres_expression(&fornode->end); @@ -364,7 +314,7 @@ static void symres_function(AstFunction* func) { for (AstLocal *param = func->params; param != NULL; param = (AstLocal *) param->next) { param->type_node = symres_type(param->type_node); - symbol_introduce(param->token, (AstNode *) param); + symbol_introduce(semstate.curr_scope, param->token, (AstNode *) param); } if (func->type_node != NULL) { @@ -384,28 +334,66 @@ static void symres_global(AstGlobal* global) { static void symres_overloaded_function(AstOverloadedFunction* ofunc) { bh_arr_each(AstTyped *, node, ofunc->overloads) { if ((*node)->kind == Ast_Kind_Symbol) { - *node = (AstTyped *) symbol_resolve((*node)->token); + *node = (AstTyped *) symbol_resolve(semstate.curr_scope, (*node)->token); } } } +static void symres_use_package(AstUsePackage* package) { + token_toggle_end(package->package->token); + Package* p = program_info_package_lookup(semstate.program, package->package->token->text); + token_toggle_end(package->package->token); + + if (p == NULL) { + onyx_message_add(Msg_Type_Literal, + package->token->pos, + "package not found in included source files"); + return; + } + + if (p->scope == semstate.curr_scope) return; + + if (package->alias != NULL) { + AstPackage *pac_node = onyx_ast_node_new(semstate.node_allocator, sizeof(AstPackage), Ast_Kind_Package); + pac_node->package = p; + + symbol_introduce(semstate.curr_scope, package->alias, (AstNode *) pac_node); + } + + if (package->only != NULL) { + bh_arr_each(OnyxToken *, tkn, package->only) { + + AstNode* thing = symbol_resolve(p->scope, *tkn); + if (thing == NULL) { + onyx_message_add(Msg_Type_Literal, + (*tkn)->pos, + "not found in package"); + return; + } + symbol_introduce(semstate.curr_scope, *tkn, thing); + } + } + + if (package->alias == NULL && package->only == NULL) + scope_include(semstate.curr_scope, p->scope); +} + void onyx_resolve_symbols() { - semstate.global_scope = scope_create(semstate.node_allocator, NULL); - scope_enter(semstate.global_scope); + semstate.curr_scope = semstate.program->global_scope; // NOTE: Add types to global scope BuiltinSymbol* bsym = (BuiltinSymbol *) &builtin_symbols[0]; while (bsym->sym != NULL) { - symbol_builtin_introduce(bsym->sym, bsym->node); + symbol_builtin_introduce(semstate.curr_scope, bsym->sym, bsym->node); bsym++; } - bh_arr_each(AstBinding *, binding, semstate.program->bindings) - if (!symbol_introduce((*binding)->token, (*binding)->node)) return; - bh_arr_each(Entity, entity, semstate.program->entities) { + scope_enter(entity->scope); + switch (entity->type) { + case Entity_Type_Use_Package: symres_use_package(entity->use_package); break; case Entity_Type_Function: symres_function(entity->function); break; case Entity_Type_Overloaded_Function: symres_overloaded_function(entity->overloaded_function); break; case Entity_Type_Global: symres_global(entity->global); break; @@ -414,5 +402,7 @@ void onyx_resolve_symbols() { default: break; } + + scope_leave(); } } diff --git a/src/onyxutils.c b/src/onyxutils.c index 0151a73e..7031ee4c 100644 --- a/src/onyxutils.c +++ b/src/onyxutils.c @@ -1,6 +1,7 @@ #include "onyxutils.h" #include "onyxlex.h" #include "onyxastnodes.h" +#include "onyxmsgs.h" bh_scratch global_scratch; bh_allocator global_scratch_allocator; @@ -10,6 +11,7 @@ bh_allocator global_heap_allocator; static const char* ast_node_names[] = { "ERROR", + "PACKAGE", "PROGRAM", "USE", @@ -62,7 +64,41 @@ const char* onyx_ast_node_kind_string(AstKind kind) { } +void program_info_init(ProgramInfo* prog, bh_allocator alloc) { + prog->global_scope = scope_create(alloc, NULL); + bh_table_init(alloc, prog->packages, 16); + + prog->entities = NULL; + bh_arr_new(alloc, prog->entities, 4); +} + +Package* program_info_package_lookup(ProgramInfo* prog, char* package_name) { + if (bh_table_has(Package *, prog->packages, package_name)) { + return bh_table_get(Package *, prog->packages, package_name); + } else { + return NULL; + } +} + +Package* program_info_package_lookup_or_create(ProgramInfo* prog, char* package_name, Scope* parent_scope, bh_allocator alloc) { + if (bh_table_has(Package *, prog->packages, package_name)) { + return bh_table_get(Package *, prog->packages, package_name); + + } else { + Package* package = bh_alloc_item(alloc, Package); + + char* pac_name = bh_alloc_array(alloc, char, strlen(package_name) + 1); + memcpy(pac_name, package_name, strlen(package_name) + 1); + + package->name = pac_name; + package->scope = scope_create(alloc, parent_scope); + + bh_table_put(Package *, prog->packages, pac_name, package); + + return package; + } +} Scope* scope_create(bh_allocator a, Scope* parent) { Scope* scope = bh_alloc_item(a, Scope); @@ -74,6 +110,69 @@ Scope* scope_create(bh_allocator a, Scope* parent) { return scope; } +void scope_include(Scope* target, Scope* source) { + bh_table_each_start(AstNode *, source->symbols); + symbol_raw_introduce(target, (char *) key, value->token->pos, value); + bh_table_each_end; +} + +b32 symbol_introduce(Scope* scope, OnyxToken* tkn, AstNode* symbol) { + token_toggle_end(tkn); + + b32 ret = symbol_raw_introduce(scope, tkn->text, tkn->pos, symbol); + + token_toggle_end(tkn); + return ret; +} + +b32 symbol_raw_introduce(Scope* scope, char* name, OnyxFilePos pos, AstNode* symbol) { + + if (bh_table_has(AstNode *, scope->symbols, name)) { + onyx_message_add(Msg_Type_Redeclare_Symbol, pos, name); + return 0; + } + + bh_table_put(AstNode *, scope->symbols, name, symbol); + return 1; +} + +void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node) { + bh_table_put(AstNode *, scope->symbols, sym, node); +} + +AstNode* symbol_resolve(Scope* start_scope, OnyxToken* tkn) { + token_toggle_end(tkn); + + AstNode* res = NULL; + Scope* scope = start_scope; + + while (res == NULL && scope != NULL) { + if (bh_table_has(AstNode *, scope->symbols, tkn->text)) { + res = bh_table_get(AstNode *, scope->symbols, tkn->text); + } else { + scope = scope->parent; + } + } + + if (res == NULL ) { + onyx_message_add(Msg_Type_Unknown_Symbol, + tkn->pos, + tkn->text); + + token_toggle_end(tkn); + return NULL; + } + + if (res->kind == Ast_Kind_Symbol) { + token_toggle_end(tkn); + return symbol_resolve(start_scope, res->token); + } + + token_toggle_end(tkn); + return res; +} + + void onyx_ast_print(AstNode* node, i32 indent) { assert(0); }