From: Brendan Hansen Date: Thu, 24 Nov 2022 23:01:24 +0000 (-0600) Subject: added string_pool; io.writer is buffered; bugfixes X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=aba4573c37e959a46f4a93399b1b835ac5a4e5d4;p=onyx.git added string_pool; io.writer is buffered; bugfixes --- diff --git a/compiler/src/types.c b/compiler/src/types.c index ec3b3334..38339ebb 100644 --- a/compiler/src/types.c +++ b/compiler/src/types.c @@ -761,11 +761,18 @@ Type* type_build_implicit_type_of_struct_literal(bh_allocator alloc, AstStructLi smem->idx = idx; smem->name = bh_strdup(alloc, nv->token->text); smem->token = nv->token; - smem->initial_value = &nv->value; smem->meta_tags = NULL; smem->included_through_use = 0; smem->used = 0; smem->use_through_pointer_index = -1; + + // Having this present caused more issues than its + // worth. I don't think this is necessary, and it allows + // you to access variables outside of where they are + // defined. + // smem->initial_value = &nv->value; + smem->initial_value = NULL; + shput(type->Struct.members, nv->token->text, smem); bh_arr_push(type->Struct.memarr, smem); token_toggle_end(nv->token); diff --git a/compiler/src/utils.c b/compiler/src/utils.c index b8e12832..810c5cf2 100644 --- a/compiler/src/utils.c +++ b/compiler/src/utils.c @@ -719,7 +719,10 @@ static AstNode* lookup_default_value_by_idx(AstNode* provider, i32 idx) { bh_arr(StructMember *) memarr = sl->type->Struct.memarr; if (idx >= bh_arr_length(memarr)) return NULL; - return (AstNode *) *memarr[idx]->initial_value; + if (memarr[idx]->initial_value) + return (AstNode *) *memarr[idx]->initial_value; + + return NULL; } return NULL; diff --git a/core/alloc/alloc.onyx b/core/alloc/alloc.onyx index 13550f78..245e4ae1 100644 --- a/core/alloc/alloc.onyx +++ b/core/alloc/alloc.onyx @@ -25,6 +25,13 @@ array_from_stack :: macro ($T: type_expr, size: u32) -> [] T { return (cast(^T) __stack_top)[0 .. size]; } +on_heap :: macro (v: $V) -> ^V { + out := cast(^V) raw_alloc(context.allocator, sizeof V); + core.memory.set(out, 0, sizeof V); + *out = v; + return out; +} + TEMPORARY_ALLOCATOR_SIZE :: 1 << 16; // 16Kb // The global heap allocator, set up upon program intialization. diff --git a/core/container/list.onyx b/core/container/list.onyx index 59db247b..0838fda9 100644 --- a/core/container/list.onyx +++ b/core/container/list.onyx @@ -11,8 +11,6 @@ List :: struct (Elem_Type: type_expr) { first: ^ListElem(Elem_Type) = null; last: ^ListElem(Elem_Type) = null; - - // "Method" like things } #inject List { @@ -131,43 +129,35 @@ fold :: (list: ^List($T), init: $R, f: (T, R) -> R) -> R { } map :: #match #local {} -#match map (list: ^List($T), f: (^T) -> void) { +#match map (list: ^List($T), f: (T) -> $R) -> List(R) { + new_list := make(R, allocator=list.allocator); elem := list.first; while elem != null { - f(^elem.data); + push_end(^new_list, f(elem.data)); elem = elem.next; } + + return new_list; } -#match map (list: ^List($T), f: (T) -> $R) -> List(R) { - new_list := make(R, allocator=list.allocator); +#match map (list: ^List($T), f: (^T) -> void) { elem := list.first; while elem != null { - push_end(new_list, f(elem.data)); + f(^elem.data); elem = elem.next; } } -#match core.iter.as_iterator as_iter -as_iter :: (list: ^List($T)) -> Iterator(T) { - iterator_next :: (list_iter: ^ListIterator($T)) -> (T, bool) { - if list_iter.current == null do return .{}, false; - defer list_iter.current = list_iter.current.next; - return list_iter.current.data, true; - } - - ListIterator :: struct (T: type_expr) { - current: ^ListElem(T); - } - - list_iterator := new_temp(ListIterator(T)); - list_iterator.current = list.first; - - return .{ - data = list_iterator, - next = #solidify iterator_next { T = T }, - }; -} +#match core.iter.as_iterator as_iter +as_iter :: (list: ^List) => + core.iter.generator(^.{current = list.first}, (ctx) => { + if ctx.current != null { + defer ctx.current = ctx.current.next; + return ctx.current.data, true; + } + + return .{}, false; + }); #local allocate_elem :: macro (list: ^List($T)) => new(ListElem(T), allocator=list.allocator); diff --git a/core/conv/conv.onyx b/core/conv/conv.onyx index 2d475517..a587266a 100644 --- a/core/conv/conv.onyx +++ b/core/conv/conv.onyx @@ -2,7 +2,7 @@ package core.conv Enable_Custom_Formatters :: true -use core {map, string, array, math} +use core {string, math} str_to_i64 :: #match #local {} @@ -133,6 +133,7 @@ str_to_f64 :: (s: ^str) -> f64 { } } + i64_to_str :: (n: i64, base: u64, buf: [] u8, min_length := 0, prefix := false) -> str { is_neg := false; if n < 0 && base == 10 { @@ -293,3 +294,14 @@ f64_to_str :: (f: f64, buf: [] u8, digits_after_decimal := 4) -> str { } +// I like the way that parse_int reads better than 'str_to_i64'. +// For a soft transistion, I am allowing the programmer to use either. +// At some point, it might be worth deprecating 'str_to_i64', but no +// in leaving it here for now. Same thing applied to all other functions +// below. +parse_int :: str_to_i64 +parse_float :: str_to_f64 + +format_int :: i64_to_str +format_uint :: u64_to_str +format_float :: f64_to_str diff --git a/core/conv/format.onyx b/core/conv/format.onyx index 07002c93..ddae9ea8 100644 --- a/core/conv/format.onyx +++ b/core/conv/format.onyx @@ -417,8 +417,9 @@ format_any :: (output: ^Format_Output, formatting: ^Format, v: any) { // is through a io.Writer. That should maybe be changed in the future? Also, I think // 256 bytes is enough for the name of a type but I'm not entirely sure... stream := io.buffer_stream_make(~~buf, fixed=true); - writer := io.writer_make(^stream); + writer := io.writer_make(^stream, 0); write_type_name(^writer, value); + io.writer_flush(^writer); output->write(io.buffer_stream_to_str(^stream)); } diff --git a/core/io/stdio.onyx b/core/io/stdio.onyx index b79f5737..43fa58a3 100644 --- a/core/io/stdio.onyx +++ b/core/io/stdio.onyx @@ -121,7 +121,7 @@ byte_dump :: (ptr: rawptr, byte_count: u32, bytes_per_line := 8) { __stdio_init :: () { stdio.print_stream = io.buffer_stream_make(2048, context.allocator); - stdio.print_writer = io.writer_make(^stdio.print_stream); + stdio.print_writer = io.writer_make(^stdio.print_stream, 0); // This shouldn't need to be here, but because ^stdin_vtable is not a compile-time // known value (even through it should be). diff --git a/core/io/stream.onyx b/core/io/stream.onyx index 690bbe1a..1916843d 100644 --- a/core/io/stream.onyx +++ b/core/io/stream.onyx @@ -311,5 +311,13 @@ buffer_stream_vtable := Stream_Vtable.{ return .None; }, + + close = (use dss: ^BufferStream) -> Error { + if write_enabled && !fixed { + delete(^data); + } + + return .None; + } } diff --git a/core/io/writer.onyx b/core/io/writer.onyx index 715c846c..222b78bb 100644 --- a/core/io/writer.onyx +++ b/core/io/writer.onyx @@ -1,34 +1,82 @@ package core.io -use core {conv, string} +use core {conv, string, memory} + +// io.Writer is a buffered-writer. The important thing to not forget +// when using io.Writer is that it has to be flushed when you are done +// using it! Flushing automatically happens when you free the writer, +// so simply call writer_free(w). If you pass 0 as the second parameter +// of writer_make, all buffering will be disabled. Writer :: struct { stream : ^Stream; + + buffer: [] u8; + buffer_filled: u32; } -writer_make :: (s: ^Stream) -> Writer { +writer_make :: (s: ^Stream, buffer_size := 4096) -> Writer { assert(s.vtable != null, "Stream vtable was not setup correctly."); - return Writer.{ s }; + w := Writer.{s}; + + if buffer_size > 0 { + w.buffer = make([] u8, buffer_size, context.allocator); + } + + return w; } // // Future-proofing the API -writer_free :: (w: ^Writer) {} +writer_free :: (w: ^Writer) { + writer_flush(w); + delete(^w.buffer); +} + +writer_flush :: (w: ^Writer) { + if w.buffer_filled == 0 do return; + + stream_write(w.stream, w.buffer[0 .. w.buffer_filled]); + w.buffer_filled = 0; +} + +writer_remaining_capacity :: (w: ^Writer) -> u32 { + return w.buffer.count - w.buffer_filled; +} string_builder :: (allocator := context.allocator) -> (Writer, ^BufferStream) { new_stream := new(BufferStream, allocator=allocator); *new_stream = buffer_stream_make(); - return writer_make(new_stream), new_stream; + return writer_make(new_stream, 0), new_stream; } write_byte :: (use writer: ^Writer, byte: u8) { - stream_write_byte(stream, byte); + if buffer.count == 0 { + stream_write_byte(stream, byte); + } else { + if writer_remaining_capacity(writer) == 0 { + writer_flush(writer); + } + + buffer[buffer_filled] = byte; + buffer_filled += 1; + } } write_str :: (use writer: ^Writer, s: str) { - stream_write(stream, s); + if buffer.count == 0 { + stream_write(stream, s); + + } elseif writer_remaining_capacity(writer) > s.count { + memory.copy(^buffer[buffer_filled], s.data, s.count); + buffer_filled += s.count; + + } else { + writer_flush(writer); + stream_write(stream, s); + } } write_cstr :: (use writer: ^Writer, cs: cstr) { diff --git a/core/onyx/cbindgen.onyx b/core/onyx/cbindgen.onyx index 76c67282..7b0dc749 100644 --- a/core/onyx/cbindgen.onyx +++ b/core/onyx/cbindgen.onyx @@ -76,6 +76,8 @@ generate_c_binding :: (use binding_config: Binding_Config) -> bool { wrote := false; for file: os.with_file(output_file, .Write) { writer := io.writer_make(file); + defer io.writer_free(^writer); + fb := runtime.info.get_foreign_block(foreign_block); write_file_introduction(^writer, preamble, fb.module_name); @@ -231,9 +233,9 @@ compile_c_file :: ( print_body :: (writer, method_name, method_info, cast_map) => { use runtime.info; - call := io.buffer_stream_make(); - defer io.buffer_stream_free(^call); - callw := io.writer_make(^call); + callw, call := io.string_builder(); + defer io.buffer_stream_free(call); + defer delete(cast(rawptr) call); param_num := 0; for method_info.parameter_types { @@ -261,7 +263,7 @@ compile_c_file :: ( param_num += 1; } - call_str := io.buffer_stream_to_str(^call); + call_str := io.buffer_stream_to_str(call); wasm_return_type := type_to_wasm_type(method_info.return_type, for_return=true); switch wasm_return_type { case "" do io.write_format(writer, " {}({});\n", method_name, call_str[0..call_str.count-2]); diff --git a/core/os/file.onyx b/core/os/file.onyx index 0bba2aa2..9e71615c 100644 --- a/core/os/file.onyx +++ b/core/os/file.onyx @@ -155,7 +155,7 @@ file_logger_close :: (logger := context.logger) { #local file_logger_proc :: (data: ^File, msg: str) { - writer := io.writer_make(data); + writer := io.writer_make(data, 0); io.write(^writer, msg); io.write(^writer, "\n"); } diff --git a/core/std.onyx b/core/std.onyx index cced1f57..7d79a1ab 100644 --- a/core/std.onyx +++ b/core/std.onyx @@ -25,6 +25,7 @@ package core #load "./string/reader" #load "./string/buffer" #load "./string/char_utils" +#load "./string/string_pool" #load "./intrinsics/onyx" #load "./intrinsics/wasm" diff --git a/core/string/char_utils.onyx b/core/string/char_utils.onyx index e66669ef..29c1207e 100644 --- a/core/string/char_utils.onyx +++ b/core/string/char_utils.onyx @@ -10,6 +10,14 @@ package core.string return (c >= #char "0" && c <= #char "9"); } + is_lower :: (c: u8) -> bool { + return (c >= #char "a" && c <= #char "b"); + } + + is_upper :: (c: u8) -> bool { + return (c >= #char "A" && c <= #char "Z"); + } + is_alphanum :: (c: u8) -> bool { return c->is_alpha() || c->is_num(); } diff --git a/core/string/string_pool.onyx b/core/string/string_pool.onyx new file mode 100644 index 00000000..9a9514ca --- /dev/null +++ b/core/string/string_pool.onyx @@ -0,0 +1,41 @@ +package core.string + +use core { alloc, memory } +use core.alloc { arena } + +StringPool :: struct { + arena: arena.Arena; +} + +#inject StringPool { + add :: pool_add; + flush :: pool_flush; + free :: pool_free; +} + +pool_make :: (maximum_string_length := 16384, allocator := context.allocator) + => StringPool.{ + arena.make(allocator, maximum_string_length * 2) + } + +pool_add :: (sp: ^StringPool, s: str) -> str { + if s.count > sp.arena.arena_size do return ""; + + allocator := alloc.as_allocator(^sp.arena); + + new_str := make(str, s.count, allocator); + memory.copy(new_str.data, s.data, s.count); + return new_str; +} + +pool_flush :: (sp: ^StringPool) { + arena.clear(^sp.arena); +} + +pool_free :: (sp: ^StringPool) { + arena.free(^sp.arena); +} + +#overload +builtin.delete :: pool_free + diff --git a/misc/onyx.vim b/misc/onyx.vim index 2ec599ca..4d90cba9 100644 --- a/misc/onyx.vim +++ b/misc/onyx.vim @@ -46,9 +46,10 @@ syn match onyxCallGroup "\<[a-zA-Z_][a-zA-Z0-9_\.]*\> *(" contains=onyxC syn match onyxCall "\<[a-zA-Z_][a-zA-Z0-9_\.]*\>" contained syn match onyxDirective "\#[a-zA-Z_]\+" -syn match onyxTag "@.\+$" +syn match onyxTag "@[a-zA-Z0-9_]\+" -syn region onyxString display start=+"+ skip=+\\\\\|\\"+ end=+"+ keepend +syn region onyxString start=+"+ skip=+\\\\\|\\"+ end=+"\|$+ extend contains=@Spell +syn region onyxMultiString start=+"""+ end=+"""+ extend contains=@Spell hi def link onyxKeyword Statement hi def link onyxType Type @@ -57,6 +58,7 @@ hi def link onyxCommentStart Todo hi def link onyxConstant Constant hi def link onyxDirective Constant hi def link onyxString String +hi def link onyxMultiString String hi def link onyxNumber Number hi def link onyxDefinition Identifier hi def link onyxCall Function diff --git a/scripts/onyx-pkg.onyx b/scripts/onyx-pkg.onyx index a580cdb7..fb816806 100644 --- a/scripts/onyx-pkg.onyx +++ b/scripts/onyx-pkg.onyx @@ -932,6 +932,8 @@ load_config_file :: () -> bool { store_config_file :: () -> bool { for os.with_file(global_arguments.config_file, .Write) { writer := io.writer_make(it); + defer io.writer_free(^writer); + return write_ini_file(^writer, config); } } diff --git a/tests/aoc-2020/day22.onyx b/tests/aoc-2020/day22.onyx index 0289afc7..077477b8 100644 --- a/tests/aoc-2020/day22.onyx +++ b/tests/aoc-2020/day22.onyx @@ -44,7 +44,7 @@ combat :: (player1: ^[..] u32, player2: ^[..] u32) -> u32 { // Larger numbers are encoded in base 64. encode_hands :: (alloc: Allocator, p1: ^[..] u32, p2: ^[..] u32) -> str { stream := io.buffer_stream_make(256, alloc); - writer := io.writer_make(^stream); + writer := io.writer_make(^stream, 0); for n: *p1 { io.write_i64(^writer, ~~n, 64); diff --git a/tests/aoc-2021/day04.onyx b/tests/aoc-2021/day04.onyx index ed12c761..1fd397a5 100644 --- a/tests/aoc-2021/day04.onyx +++ b/tests/aoc-2021/day04.onyx @@ -25,7 +25,7 @@ main :: (args) => { numbers_line := io.read_line(^reader, inplace=true, consume_newline=true); numbers_str := string.split(numbers_line, #char ","); numbers := memory.make_slice(Cell, numbers_str.count); - for numbers_str.count do numbers[it] = ~~ conv.str_to_i64(numbers_str[it]); + for numbers_str.count do numbers[it] = ~~ conv.parse_int(numbers_str[it]); boards: [..] Board; while !io.reader_empty(^reader) { @@ -79,4 +79,4 @@ main :: (args) => { printf("Part 1: {}\n", winning_board_score); printf("Part 2: {}\n", worst_board_score); } -} \ No newline at end of file +} diff --git a/tests/linked_lists b/tests/linked_lists new file mode 100644 index 00000000..c2534692 --- /dev/null +++ b/tests/linked_lists @@ -0,0 +1,5 @@ +4 +5.0000 +1.0000 +2.0000 +3.0000 diff --git a/tests/linked_lists.onyx b/tests/linked_lists.onyx new file mode 100644 index 00000000..89435a5e --- /dev/null +++ b/tests/linked_lists.onyx @@ -0,0 +1,22 @@ +use core + +main :: () { + l := list.make(i32); + + l->push_end(1); + l->push_end(2); + l->push_end(3); + l->push_end(4); + l->push_begin(5); + l->push_begin(6); + l->pop_end(); + l->pop_begin(); + + println(l->count()); + + float_list := l->map(x => cast(f32) x); + + for float_list->as_iter() { + println(it); + } +}