From da81dcd2ab5d9d9abbee7406ce5dff0b034fbd60 Mon Sep 17 00:00:00 2001 From: Brendan Hansen Date: Wed, 23 Mar 2022 19:47:24 -0500 Subject: [PATCH] bugfixing and flushing out core libraries --- core/alloc.onyx | 6 +++ core/alloc/arena.onyx | 37 ++++++++++++++ core/alloc/auto_heap.onyx | 73 +++++++++++++++++++++++++++ core/container/iter.onyx | 2 +- core/container/set.onyx | 18 ++++--- core/io/reader.onyx | 12 +++-- core/net/tcp.onyx | 9 ++-- modules/http/README.md | 9 +++- modules/http/headers.onyx | 20 ++------ modules/http/http.onyx | 102 +++++++++++++++++++++++++++++++++++--- modules/http/module.onyx | 3 +- modules/http/utils.onyx | 41 +++++++++++++++ src/lex.c | 13 +++++ src/parser.c | 30 ++++++++--- 14 files changed, 325 insertions(+), 50 deletions(-) create mode 100644 core/alloc/auto_heap.onyx create mode 100644 modules/http/utils.onyx diff --git a/core/alloc.onyx b/core/alloc.onyx index 72a202f5..ef9a79b3 100644 --- a/core/alloc.onyx +++ b/core/alloc.onyx @@ -6,6 +6,7 @@ package core.alloc #load "./alloc/ring" #load "./alloc/pool" #load "./alloc/logging" +#load "./alloc/auto_heap" as_allocator :: #match { macro (a: Allocator) => a @@ -19,6 +20,11 @@ from_stack :: macro (size: u32) -> rawptr { return __stack_top; } +array_from_stack :: macro ($T: type_expr, size: u32) -> [] T { + defer __stack_top = ~~(cast(^u8) __stack_top + size * sizeof T); + return (cast(^T) __stack_top)[0 .. size]; +} + TEMPORARY_ALLOCATOR_SIZE :: 1 << 12; // 4Kb // The global heap allocator, set up upon program intialization. diff --git a/core/alloc/arena.onyx b/core/alloc/arena.onyx index aad48fb7..5de268c0 100644 --- a/core/alloc/arena.onyx +++ b/core/alloc/arena.onyx @@ -56,6 +56,22 @@ arena_alloc_proc :: (data: rawptr, aa: AllocationAction, size: u32, align: u32, return retval; } + if aa == .Resize { + // An allocation of this size does not fit into a single arena. + if size > alloc_arena.arena_size - sizeof rawptr { + return null; + } + + newptr := arena_alloc_proc(data, .Alloc, size, align, oldptr); + + // This is incorrect, but because there is not an "old size", + // this is the best possible. + memory :: package core.memory + memory.copy(newptr, oldptr, size); + + return newptr; + } + return null; } @@ -97,3 +113,24 @@ free :: (arena: ^ArenaState) { arena.current_arena = null; arena.size = 0; } + +auto :: #match { + macro (size := 32 * 1024) { + alloc :: package core.alloc + + arena := alloc.arena.make(alloc.heap_allocator, size); + context.allocator = alloc.arena.make_allocator(^arena); + defer alloc.arena.free(^arena); + }, + + macro (body: Code, size := 32 * 1024) -> i32 { + auto :: auto + + #context_scope { + auto(size); + #insert body; + } + + return 0; + } +} diff --git a/core/alloc/auto_heap.onyx b/core/alloc/auto_heap.onyx new file mode 100644 index 00000000..84e54f87 --- /dev/null +++ b/core/alloc/auto_heap.onyx @@ -0,0 +1,73 @@ +package core.alloc.heap + +#local _Z :: (package core.intrinsics.onyx).__zero_value + +AutoHeapState :: struct { + backing_allocator: Allocator; + set := _Z(Set(rawptr)); +} + +#local auto_heap_alloc_proc :: (data: ^AutoHeapState, aa: AllocationAction, size: u32, align: u32, oldptr: rawptr) -> rawptr { + newptr := data.backing_allocator.func(data.backing_allocator.data, aa, size, align, oldptr); + + switch aa { + case .Alloc { + data.set->insert(newptr); + } + + case .Resize { + data.set->remove(oldptr); + data.set->insert(newptr); + } + + case .Free { + data.set->remove(oldptr); + } + } + + return newptr; +} + +auto_heap_make :: (backing := context.allocator) -> AutoHeapState { + hs: AutoHeapState; + hs.backing_allocator = backing; + hs.set->init(allocator = backing); + return hs; +} + +auto_heap_free :: (hs: ^AutoHeapState) { + for^ hs.set.entries { + raw_free(hs.backing_allocator, it.value); + } + + hs.set->free(); +} + +#match (package core.alloc).as_allocator auto_heap_make_allocator +auto_heap_make_allocator :: (hs: ^AutoHeapState) -> Allocator { + return Allocator.{ + func = auto_heap_alloc_proc, + data = hs + }; +} + +auto :: #match { + macro () { + alloc :: package core.alloc + + auto_heap := alloc.heap.auto_heap_make(); + context.allocator = alloc.heap.auto_heap_make_allocator(^auto_heap); + defer alloc.heap.auto_heap_free(^auto_heap); + }, + + macro (body: Code) -> i32 { + auto :: auto + + #context_scope { + auto(); + #insert body; + } + + return 0; + } +} diff --git a/core/container/iter.onyx b/core/container/iter.onyx index 54fc23e0..6fec7997 100644 --- a/core/container/iter.onyx +++ b/core/container/iter.onyx @@ -600,7 +600,7 @@ to_array :: (it: Iterator($T), allocator := context.allocator) -> [..] T { data = thread_data, }; - threads := (cast(^thread.Thread) alloc.from_stack(thread_count * sizeof thread.Thread))[0 .. (thread_count - 1)]; + threads := alloc.array_from_stack(thread.Thread, thread_count - 1); for^ threads do thread.spawn(it, ^t_data, #solidify thread_function {body=body}); thread_function(^t_data, body); diff --git a/core/container/set.onyx b/core/container/set.onyx index f4374f42..76b3ec77 100644 --- a/core/container/set.onyx +++ b/core/container/set.onyx @@ -26,6 +26,8 @@ Set :: struct (Elem_Type: type_expr) where SetValue(Elem_Type) { value : T; } + init :: init + free :: free has :: has get :: get insert :: insert @@ -35,20 +37,20 @@ Set :: struct (Elem_Type: type_expr) where SetValue(Elem_Type) { iterator :: iterator } -make :: ($T: type_expr, default := __zero_value(T)) -> Set(T) { +make :: ($T: type_expr, default := __zero_value(T), allocator := context.allocator) -> Set(T) { set : Set(T); - init(^set, default=default); + init(^set, default=default, allocator=allocator); return set; } -init :: (use set: ^Set($T), default := __zero_value(T)) { - allocator = context.allocator; - default_value = default; +init :: (set: ^Set($T), default := __zero_value(T), allocator := context.allocator) { + set.allocator = allocator; + set.default_value = default; - memory.alloc_slice(^hashes, 8, allocator=allocator); - memory.fill_slice(hashes, -1); + memory.alloc_slice(^set.hashes, 8, allocator=allocator); + memory.fill_slice(set.hashes, -1); - array.init(^entries, 4, allocator=allocator); + array.init(^set.entries, 4, allocator=allocator); } free :: (use set: ^Set) { diff --git a/core/io/reader.onyx b/core/io/reader.onyx index 83f51feb..fcd3abec 100644 --- a/core/io/reader.onyx +++ b/core/io/reader.onyx @@ -309,7 +309,9 @@ read_line :: (use reader: ^Reader, consume_newline := true, allocator := context start = count; } - while reader_read_next_chunk(reader) == .ReadPending --- + if !done { + while reader_read_next_chunk(reader) == .ReadPending --- + } if reader_empty(reader) do done = true; } @@ -377,7 +379,9 @@ read_word :: (use reader: ^Reader, numeric_allowed := false, allocator := contex start = count; } - while reader_read_next_chunk(reader) == .ReadPending --- + if !done { + while reader_read_next_chunk(reader) == .ReadPending --- + } if reader_empty(reader) do done = true; } @@ -442,7 +446,9 @@ read_until :: (use reader: ^Reader, until: u8, skip: u32 = 0, allocator := conte start = count; } - while reader_read_next_chunk(reader) == .ReadPending --- + if !done { + while reader_read_next_chunk(reader) == .ReadPending --- + } if reader_empty(reader) do done = true; } diff --git a/core/net/tcp.onyx b/core/net/tcp.onyx index 92741bf7..3944ae56 100644 --- a/core/net/tcp.onyx +++ b/core/net/tcp.onyx @@ -280,8 +280,9 @@ TCP_Client :: struct { } wait_to_get_client_messages :: (use server: ^TCP_Server) -> [] ^TCP_Server.Client { - active_client_memory := alloc.from_stack(client_count * sizeof ^TCP_Server.Client); - active_clients: [] ^TCP_Server.Client = .{ ~~active_client_memory, 0 }; + active_clients := alloc.array_from_stack(^TCP_Server.Client, client_count); + active_clients.count = 0; + for clients { if it == null do continue; @@ -291,8 +292,8 @@ TCP_Client :: struct { } } - changed_buffer := cast(^i32) alloc.from_stack(client_count * sizeof i32); - changed := socket_poll_all(cast([] ^Socket) active_clients, pulse_time_ms, changed_buffer[0 .. client_count]); + changed_buffer := alloc.array_from_stack(i32, client_count); + changed := socket_poll_all(cast([] ^Socket) active_clients, pulse_time_ms, changed_buffer); recv_clients: [..] ^TCP_Server.Client; for changed { diff --git a/modules/http/README.md b/modules/http/README.md index 559a6743..c8e9da0a 100644 --- a/modules/http/README.md +++ b/modules/http/README.md @@ -1,3 +1,10 @@ # HTTP library -Minimal implementation of HTTP/1.0 and HTTP/1.1 protocols. \ No newline at end of file +Minimal implementation of HTTP/1.0 and HTTP/1.1 protocols. + +## What this module is and isn't +For now, this module is solely meant for making HTTP requests, +NOT for making an HTTP server. An HTTP server implementation may +be coming, but for now I think HTTP requests are more universally +useful. They enable a programmer to interact with a whole host of +online services, making Onyx more powerful for application development. diff --git a/modules/http/headers.onyx b/modules/http/headers.onyx index 25980591..5d7de533 100644 --- a/modules/http/headers.onyx +++ b/modules/http/headers.onyx @@ -1,20 +1,5 @@ package http -// -// These are named so that the string version of these values -// will be valid request methods. -Method :: enum { - GET; - HEAD; - POST; - PUT; - DELETE; - CONNECT; - OPTIONS; - TRACE; - PATCH; -} - HTTP_Header :: enum { Content_Length; Content_Type; @@ -74,6 +59,7 @@ media_type_from_str :: (s: str) -> (MediaType, success: bool) { HTTP_Headers :: struct { content_length: u32; + content_type: MediaType; expect: bool; chunked: bool; accept: MediaType; @@ -101,7 +87,7 @@ HTTP_Headers :: struct { } case .Content_Type { - _, success := media_type_from_str(content); + content_type', success := media_type_from_str(content); return success; } @@ -143,4 +129,4 @@ HTTP_Headers :: struct { return false; } -} \ No newline at end of file +} diff --git a/modules/http/http.onyx b/modules/http/http.onyx index 9ce1ca93..1b73de3f 100644 --- a/modules/http/http.onyx +++ b/modules/http/http.onyx @@ -1,8 +1,18 @@ package http -#local { - HTTP_VERSION_STRING :: "HTTP/1.1" - USER_AGENT :: "onyx/0.1.0" +// +// These are named so that the string version of these values +// will be valid request methods. +Method :: enum { + GET; + HEAD; + POST; + PUT; + DELETE; + CONNECT; + OPTIONS; + TRACE; + PATCH; } Request :: struct { @@ -22,7 +32,7 @@ Request :: struct { // // "Factory" that constructs a GET request - get :: (resource: str, params: [] struct {key, value: str;}) -> Request { + get :: (resource: str, params: [] Key_Value_Pair, headers: [] Key_Value_Pair = .[]) -> Request { req := init(Request); req.method = .GET; @@ -38,10 +48,57 @@ Request :: struct { res.count -= 1; } + for headers { + req.headers[it.key] = it.value; + } + req.resource = res; req.body = null_str; return req; } + + post :: #match { + (resource: str, data: [] Key_Value_Pair, encoding := Post_Data_Encoding.Url_Encoded, headers: [] Key_Value_Pair = .[]) -> Request { + req := init(Request); + req.method = .POST; + req.resource = resource; + + for headers { + req.headers[it.key] = it.value; + } + + body: [..] u8; + switch encoding { + case .Json { + string.concat(^body, "{"); + for data { + string.concat(^body, "\"", it.key, "\":\"", it.value, "\","); + } + + body.count -= 1; + string.concat(^body, "}"); + } + + case .Url_Encoded { + for data { + key := urlencode(it.key); + value := urlencode(it.value); + defer { + string.free(key); + string.free(value); + } + + string.concat(^body, key, "=", value, "&"); + } + + body.count -= 1; + } + } + + req.body = body; + return req; + } + } } request_write :: (use req: ^Request, writer: ^io.Writer) { @@ -57,6 +114,7 @@ request_write :: (use req: ^Request, writer: ^io.Writer) { io.write(writer, "\r\n"); if body.count > 0 { io.write(writer, body); + io.write(writer, "\r\n"); } } @@ -65,6 +123,11 @@ Response :: struct { status: i32; headers: HTTP_Headers; body: [] u8; + + json :: (use this: ^Response) => { + json :: package json + return json.decode(body); + } } @@ -75,6 +138,11 @@ Connection :: struct { socket: ^net.Socket; r: io.Reader; w: io.Writer; + + get :: (this: ^Connection, resource: str, params: [] Key_Value_Pair, headers: [] Key_Value_Pair = .[]) -> Response { + req := Request.get(resource, params, headers); + return send_request(this, ^req); + } } Connection_Error :: enum { @@ -126,7 +194,7 @@ connect :: (url_: str, port: u16 = 80) -> (Connection, Connection_Error) { return conn, .None; } -request :: (connection: ^Connection, req: ^Request) -> Response { +send_request :: (connection: ^Connection, req: ^Request) -> Response { req.host = connection.url; req->send(^connection.w); @@ -148,10 +216,14 @@ request :: (connection: ^Connection, req: ^Request) -> Response { data: [..] u8; if res.headers.chunked { read_chunked_body(); - } else { + } elseif res.headers.content_length > 0 { read_simple_body(); } + @HACK @HACK @HACK + connection.r.start = 0; + connection.r.end = 0; + res.body = data; return res; @@ -181,4 +253,20 @@ request :: (connection: ^Connection, req: ^Request) -> Response { data.count += bytes_read; } } -} \ No newline at end of file +} + + + +#local { + HTTP_VERSION_STRING :: "HTTP/1.1" + USER_AGENT :: "onyx/0.1.0" + + Key_Value_Pair :: struct { key, value: str; } + + Post_Data_Encoding :: enum { + Json; + Url_Encoded; + Form_Data; + } +} + diff --git a/modules/http/module.onyx b/modules/http/module.onyx index 47ded7f3..93c791db 100644 --- a/modules/http/module.onyx +++ b/modules/http/module.onyx @@ -12,4 +12,5 @@ package http __zero_value :: (package core.intrinsics.onyx).__zero_value } -#load_all "./." \ No newline at end of file +#load_all "./." +#load "./../json/module.onyx" diff --git a/modules/http/utils.onyx b/modules/http/utils.onyx new file mode 100644 index 00000000..6b8d0345 --- /dev/null +++ b/modules/http/utils.onyx @@ -0,0 +1,41 @@ +package http + +urlencode :: (s: str, allocator := context.allocator) -> str { + out := array.make(u8, s.count, allocator=allocator); + + for ch: s do switch ch { + // + // This list was taken from: + // https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding + match(#char ":", "%3A"); + match(#char "/", "%2F"); + match(#char "?", "%3F"); + match(#char "#", "%23"); + match(#char "[", "%5B"); + match(#char "]", "%5D"); + match(#char "@", "%40"); + match(#char "!", "%21"); + match(#char "$", "%24"); + match(#char "&", "%26"); + match(#char "'", "%27"); + match(#char "(", "%28"); + match(#char ")", "%29"); + match(#char "*", "%2A"); + match(#char "+", "%2B"); + match(#char ",", "%2C"); + match(#char ";", "%3B"); + match(#char "=", "%3D"); + match(#char "%", "%25"); + match(#char " ", "%20"); + + case #default do out << ch; + + match :: macro (from: u8, to: str) { + // + // How's this for a one-liner + case from do for to do out << it; + } + } + + return out; +} diff --git a/src/lex.c b/src/lex.c index 7a458b88..e65ff964 100644 --- a/src/lex.c +++ b/src/lex.c @@ -168,6 +168,19 @@ whitespace_skipped: goto token_parsed; } + // She-bang + if (tokenizer->curr == tokenizer->start) { + if (*tokenizer->curr == '#' && *(tokenizer->curr + 1) == '!') { + tk.type = Token_Type_Comment; + + tokenizer->curr += 2; + while (*tokenizer->curr++ != '\n' && tokenizer->curr != tokenizer->end); + + tk.length = tokenizer->curr - tk.text; + goto token_parsed; + } + } + // Comments if (*tokenizer->curr == '/' && *(tokenizer->curr + 1) == '/') { tokenizer->curr += 2; diff --git a/src/parser.c b/src/parser.c index 583492ee..5c40a124 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1493,29 +1493,43 @@ static AstNode* parse_statement(OnyxParser* parser) { case '#': { if (parse_possible_directive(parser, "context_scope")) { - AstLocal* context_tmp = make_local(parser->allocator, NULL, builtin_context_variable->type_node); + // :LinearTokenDependent + OnyxToken* directive_token = parser->curr - 2; + + OnyxToken* sym_token = bh_alloc_item(parser->allocator, OnyxToken); + sym_token->type = Token_Type_Symbol; + sym_token->length = 15; + sym_token->text = bh_strdup(parser->allocator, "__saved_context "); + sym_token->pos = ((OnyxFilePos) {0}); + + AstNode *sym_node = make_symbol(parser->allocator, sym_token); + + AstLocal* context_tmp = make_local(parser->allocator, sym_token, NULL); + retval = (AstNode *) context_tmp; AstBinaryOp* assignment = make_node(AstBinaryOp, Ast_Kind_Binary_Op); + assignment->token = directive_token; assignment->operation = Binary_Op_Assign; - assignment->left = (AstTyped *) context_tmp; + assignment->left = (AstTyped *) sym_node; assignment->right = builtin_context_variable; context_tmp->next = (AstNode *) assignment; AstBinaryOp* assignment2 = make_node(AstBinaryOp, Ast_Kind_Binary_Op); + assignment2->token = directive_token + 1; assignment2->operation = Binary_Op_Assign; assignment2->left = builtin_context_variable; - assignment2->right = (AstTyped *) context_tmp; - - AstBlock* context_block = parse_block(parser, 1, NULL); - assignment->next = (AstNode *) context_block; + assignment2->right = (AstTyped *) sym_node; AstDefer* defer_node = make_node(AstDefer, Ast_Kind_Defer); + defer_node->token = directive_token; defer_node->stmt = (AstNode *) assignment2; + assignment->next = (AstNode *) defer_node; + + AstBlock* context_block = parse_block(parser, 1, NULL); defer_node->next = context_block->body; - context_block->body = (AstNode *) defer_node; + context_block->body = (AstNode *) context_tmp; needs_semicolon = 0; - retval = (AstNode *) context_tmp; break; } -- 2.25.1