From: Brendan Hansen Date: Sun, 12 Dec 2021 04:02:37 +0000 (-0600) Subject: added directory support to onyx fs X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=4cb307a50b7c3a8bc6d6c8d2fc9e0ff31d7b595c;p=onyx.git added directory support to onyx fs --- diff --git a/core/os/dir.onyx b/core/os/dir.onyx new file mode 100644 index 00000000..7728b980 --- /dev/null +++ b/core/os/dir.onyx @@ -0,0 +1,41 @@ +package core.os + +#local fs :: package runtime.fs + +Directory :: fs.DirectoryData; + +DirectoryEntry :: struct { + // Most of these types were stollen directly from + // man readdir(3), and the most reasonable types + // were used for Windows. + Type :: enum { + Unknown :: 0x00; + Block :: 0x01; + Char :: 0x02; + Directory :: 0x03; + RegularFile :: 0x04; + SymLink :: 0x05; + Other :: 0x06; + } + + type : Type; + identifier : u32; + name_length : u32; + name_data : [256] u8; + + name :: (use dir: ^DirectoryEntry) => str.{ ~~name_data, name_length }; +} + +dir_open :: (path: str) -> (Directory, bool) { + dir: Directory; + success := fs.__dir_open(path, ^dir); + return dir, success; +} + +dir_close :: (dir: Directory) { + fs.__dir_close(dir); +} + +dir_read :: (dir: Directory, out_entry: ^DirectoryEntry) -> bool { + return fs.__dir_read(dir, out_entry); +} diff --git a/core/os/onyx_fs.onyx b/core/os/onyx_fs.onyx index f176a528..c177e761 100644 --- a/core/os/onyx_fs.onyx +++ b/core/os/onyx_fs.onyx @@ -8,6 +8,8 @@ FileData :: struct { handle: Handle = -1; } +DirectoryData :: #distinct u64 + __file_open :: (path: str, mode := os.OpenMode.Read) -> (FileData, os.FileError) { handle: FileData.Handle; error := __file_open_impl(path, mode, ^handle); @@ -27,6 +29,10 @@ __file_open :: (path: str, mode := os.OpenMode.Read) -> (FileData, os.FileError) __file_read :: (handle: FileData.Handle, output_buffer: [] u8, bytes_read: ^u64) -> io.Error --- __file_write :: (handle: FileData.Handle, input_buffer: [] u8, bytes_wrote: ^u64) -> io.Error --- __file_flush :: (handle: FileData.Handle) -> io.Error --- + + __dir_open :: (path: str, dir: ^DirectoryData) -> bool --- + __dir_close :: (dir: DirectoryData) -> void --- + __dir_read :: (dir: DirectoryData, out_entry: ^os.DirectoryEntry) -> bool --- } __file_stream_vtable := io.Stream_Vtable.{ diff --git a/core/os/os.onyx b/core/os/os.onyx index c26a6d59..5aac907a 100644 --- a/core/os/os.onyx +++ b/core/os/os.onyx @@ -7,68 +7,34 @@ package core.os #local { runtime :: package runtime - wasi :: package wasi } -Directory_Entry :: struct { - dirent : wasi.DirEnt; - name : str; -} - -list_directory :: (path: str) -> Iterator(Directory_Entry) { +list_directory :: (path: str) -> Iterator(DirectoryEntry) { Context :: struct { - dir_fd: wasi.FileDescriptor; + dir: Directory; opened := false; - reached_end := false; - - entry_walker: ^u8; - - buffer_used: u32 = 0; - buffer: [1024] u8; - - last_cookie: wasi.DirCookie = 0; } - next :: (use c: ^Context) -> (Directory_Entry, bool) { + next :: (use c: ^Context) -> (DirectoryEntry, bool) { use package core.intrinsics.onyx {__zero_value} - if !opened do return __zero_value(Directory_Entry), false; - - read_more :: macro () { - if !reached_end { - err := wasi.fd_readdir(dir_fd, ~~buffer, sizeof typeof buffer, last_cookie, ^buffer_used); - reached_end = buffer_used != sizeof typeof buffer; + if !opened do return __zero_value(DirectoryEntry), false; - entry_walker = ~~buffer; - } else { - return __zero_value(Directory_Entry), false; - } + entry: DirectoryEntry; + if !dir_read(dir, ^entry) { + return __zero_value(DirectoryEntry), false; } - if buffer_used < sizeof(wasi.DirEnt) do read_more(); - - dirent := cast(^wasi.DirEnt) entry_walker; - if buffer_used < sizeof(wasi.DirEnt) + dirent.d_namlen { - read_more(); - dirent = ~~entry_walker; - } - - name := str.{~~(dirent + 1), dirent.d_namlen}; - - entry_size := sizeof(wasi.DirEnt) + dirent.d_namlen; - entry_walker += entry_size; - buffer_used -= entry_size; - last_cookie = dirent.d_next; - - return .{*dirent, name}, true; + return entry, true; } close :: (use c: ^Context) { - wasi.fd_close(dir_fd); + dir_close(dir); cfree(c); } c := new(Context); - if err := wasi.path_open(4, 0, path, .Directory, ~~0xffffffff, ~~0xffffffff, .Sync, ^c.dir_fd); err == .Success { + if dir, success := dir_open(path); success { + c.dir = dir; c.opened = true; } diff --git a/core/os/process.onyx b/core/os/process.onyx index f22aee04..5f66ab4a 100644 --- a/core/os/process.onyx +++ b/core/os/process.onyx @@ -1,13 +1,5 @@ package core.os -// Some thoughts about processes and the API. -// -// Should processes ever directly dump to standard output? -// Or should their output always be buffered through a pipe to the user? -// -// - - #if runtime.Runtime != runtime.Runtime_Onyx { #error "This file can only be included in the 'onyx' runtime, because Wasi has not defined how to spawn and manage processes."; } @@ -25,7 +17,7 @@ Process :: struct { } process_spawn :: (path: str, args: [] str, non_blocking_io := false) -> Process { - handle := runtime.__process_spawn(path, args, non_blocking_io); + handle := __process_spawn(path, args, non_blocking_io); return .{ .{ ^process_stream_vtable }, @@ -34,15 +26,15 @@ process_spawn :: (path: str, args: [] str, non_blocking_io := false) -> Process } process_kill :: (use p: ^Process) -> bool { - return runtime.__process_kill(process_handle); + return __process_kill(process_handle); } process_wait :: (use p: ^Process) => { - return runtime.__process_wait(process_handle); + return __process_wait(process_handle); } process_destroy :: (use p: ^Process) => { - runtime.__process_destroy(process_handle); + __process_destroy(process_handle); } #local process_stream_vtable := io.Stream_Vtable.{ @@ -50,7 +42,7 @@ process_destroy :: (use p: ^Process) => { // Read from the process stdout if cast(i64) process_handle == 0 do return .BadFile, 0; - bytes_read := runtime.__process_read(process_handle, buffer); + bytes_read := __process_read(process_handle, buffer); return .None, bytes_read; }, @@ -58,7 +50,7 @@ process_destroy :: (use p: ^Process) => { // Write to the process stdin if cast(i64) process_handle == 0 do return .BadFile, 0; - bytes_written := runtime.__process_write(process_handle, buffer); + bytes_written := __process_write(process_handle, buffer); return .None, bytes_written; }, @@ -67,3 +59,20 @@ process_destroy :: (use p: ^Process) => { return .None; } } + +#local ProcessResult :: enum { + Success :: 0x00; + FailedToRun :: 0x01; + Error :: 0x02; + InternalErr :: 0x03; +} + +#foreign "onyx_runtime" { + __process_spawn :: (path: str, args: [] str, non_blocking_io: bool) -> Process.Handle --- + __process_read :: (handle: Process.Handle, buffer: [] u8) -> i32 --- + __process_write :: (handle: Process.Handle, buffer: [] u8) -> i32 --- + __process_kill :: (handle: Process.Handle) -> bool --- + __process_wait :: (handle: Process.Handle) -> ProcessResult --- + __process_destroy :: (handle: Process.Handle) -> void --- +} + diff --git a/core/runtime/onyx_run.onyx b/core/runtime/onyx_run.onyx index c2f6497d..43530fb3 100644 --- a/core/runtime/onyx_run.onyx +++ b/core/runtime/onyx_run.onyx @@ -1,13 +1,54 @@ package runtime -#load "core/wasi/wasi" #load "core/runtime/common" use package core -use package wasi -// The "onyx_run" runtime extends the WASI runtime. -#load "core/runtime/wasi" +#local { + __stdout: os.File; + __stdin: os.File; +} + +__output_string :: (s: str) -> u32 { + err, wrote := io.stream_write(^__stdout, s); + return wrote; +} + +__exit :: (status: i32) --- + +__read_from_input :: (buffer: [] u8) -> i32 { + err, read := io.stream_read(^__stdin, buffer); + if err != .None do return -1; + return read; +} + +#library "onyx_runtime" + +#local __file_get_standard :: (fd: i32, out: ^fs.FileData.Handle) -> bool #foreign "onyx_runtime" "__file_get_standard" --- + +#export "_start" () { + fd: fs.FileData.Handle; + __file_get_standard(1, ^fd); + __stdout = .{ + .{ ^fs.__file_stream_vtable }, + .{ fd }, + }; + + __file_get_standard(0, ^fd); + __stdin = .{ + .{ ^fs.__file_stream_vtable }, + .{ fd }, + }; + + __runtime_initialize(); + context.thread_id = 0; + + args : [] cstr; + + (package main).main(args); + + __flush_stdio(); +} #if Multi_Threading_Enabled { __spawn_thread :: (id: i32, tls_base: rawptr, func: (data: rawptr) -> void, data: rawptr) -> bool #foreign "onyx_runtime" "__spawn_thread" --- @@ -16,21 +57,3 @@ use package wasi #export "_thread_start" _thread_start #export "_thread_exit" _thread_exit } - -#local ProcessResult :: enum { - Success :: 0x00; - FailedToRun :: 0x01; - Error :: 0x02; - InternalErr :: 0x03; -} - -#library "onyx_runtime" - -#foreign "onyx_runtime" { - __process_spawn :: (path: str, args: [] str, non_blocking_io: bool) -> os.Process.Handle --- - __process_read :: (handle: os.Process.Handle, buffer: [] u8) -> i32 --- - __process_write :: (handle: os.Process.Handle, buffer: [] u8) -> i32 --- - __process_kill :: (handle: os.Process.Handle) -> bool --- - __process_wait :: (handle: os.Process.Handle) -> ProcessResult --- - __process_destroy :: (handle: os.Process.Handle) -> void --- -} \ No newline at end of file diff --git a/core/std.onyx b/core/std.onyx index 04ece571..8c3bb028 100644 --- a/core/std.onyx +++ b/core/std.onyx @@ -38,11 +38,9 @@ package core #local runtime :: package runtime #if runtime.Runtime == runtime.Runtime_Wasi || runtime.Runtime == runtime.Runtime_Onyx { - #load "./wasi/wasi" - #load "./wasi/env" - #load "./wasi/clock" #load "./os/file" #load "./os/os" + #load "./os/dir" } #if runtime.Runtime == runtime.Runtime_Onyx { @@ -51,7 +49,10 @@ package core #load "./os/onyx_fs" } #if runtime.Runtime == runtime.Runtime_Wasi { + #load "./wasi/wasi" #load "./runtime/wasi" + #load "./wasi/clock" + #load "./wasi/env" #load "./wasi/wasi_fs" } #if runtime.Runtime == runtime.Runtime_Js { #load "./runtime/js" } diff --git a/core/wasi/wasi_fs.onyx b/core/wasi/wasi_fs.onyx index d46ca72a..d4bb4e1b 100644 --- a/core/wasi/wasi_fs.onyx +++ b/core/wasi/wasi_fs.onyx @@ -218,3 +218,5 @@ __file_stream_vtable := io.Stream_Vtable.{ }, } + + diff --git a/include/small_windows.h b/include/small_windows.h index 276ea408..a71f238d 100644 --- a/include/small_windows.h +++ b/include/small_windows.h @@ -244,6 +244,18 @@ typedef struct _WIN32_FIND_DATAW { wchar_t cFileName[MAX_PATH]; wchar_t cAlternateFileName[14]; } WIN32_FIND_DATAW; +typedef struct _WIN32_FIND_DATAA { + DWORD dwFileAttributes; + FILETIME ftCreationTime; + FILETIME ftLastAccessTime; + FILETIME ftLastWriteTime; + DWORD nFileSizeHigh; + DWORD nFileSizeLow; + DWORD dwReserved0; + DWORD dwReserved1; + char cFileName[MAX_PATH]; + char cAlternateFileName[14]; +} WIN32_FIND_DATAA; typedef struct _WIN32_FILE_ATTRIBUTE_DATA { DWORD dwFileAttributes; FILETIME ftCreationTime; @@ -374,6 +386,7 @@ GB_DLL_IMPORT void WINAPI GetSystemInfo(SYSTEM_INFO *system_info); #define FILE_ATTRIBUTE_READONLY 0x00000001 #define FILE_ATTRIBUTE_NORMAL 0x00000080 #define FILE_ATTRIBUTE_TEMPORARY 0x00000100 +#define FILE_ATTRIBUTE_DIRECTORY 0x00000010 #define ERROR_FILE_NOT_FOUND 2l #define ERROR_ACCESS_DENIED 5L #define ERROR_NO_MORE_FILES 18l @@ -399,6 +412,8 @@ GB_DLL_IMPORT HANDLE WINAPI GetStdHandle (DWORD std_handle); GB_DLL_IMPORT BOOL WINAPI GetFileSizeEx (HANDLE file, LARGE_INTEGER *size); GB_DLL_IMPORT BOOL WINAPI SetEndOfFile (HANDLE file); GB_DLL_IMPORT HANDLE WINAPI FindFirstFileW (wchar_t const *path, WIN32_FIND_DATAW *data); +GB_DLL_IMPORT HANDLE WINAPI FindFirstFileA (char const *path, WIN32_FIND_DATAA *data); +GB_DLL_IMPORT BOOL WINAPI FindNextFileA (HANDLE find_find, WIN32_FIND_DATAA *data); GB_DLL_IMPORT BOOL WINAPI FindClose (HANDLE find_file); GB_DLL_IMPORT BOOL WINAPI GetFileAttributesExW(wchar_t const *path, GET_FILEEX_INFO_LEVELS info_level_id, WIN32_FILE_ATTRIBUTE_DATA *data); GB_DLL_IMPORT BOOL WINAPI CopyFileW(wchar_t const *old_f, wchar_t const *new_f, BOOL fail_if_exists); diff --git a/modules/onyx_runtime/onyx_runtime.c b/modules/onyx_runtime/onyx_runtime.c index 1c26f0f7..f2ce9e61 100644 --- a/modules/onyx_runtime/onyx_runtime.c +++ b/modules/onyx_runtime/onyx_runtime.c @@ -10,7 +10,9 @@ #include #include #include + #include #include + #include #endif #include "types.h" // For POINTER_SIZE @@ -146,6 +148,153 @@ ONYX_DEF(__file_flush, (WASM_I64), (WASM_I32)) { return NULL; } +ONYX_DEF(__file_get_standard, (WASM_I32, WASM_I32), (WASM_I32)) { + bh_file_standard standard = (bh_file_standard) params->data[0].of.i32; + + bh_file file; + bh_file_error error = bh_file_get_standard(&file, standard); + if (error == BH_FILE_ERROR_NONE) { + *(u64 *) ONYX_PTR(params->data[1].of.i32) = (u64) file.fd; + } + + results->data[0] = WASM_I32_VAL(error == BH_FILE_ERROR_NONE); + return NULL; +} + +// +// Directories +// +#ifdef _BH_WINDOWS +typedef struct Windows_Directory_Opened { + HANDLE hndl; + WIN32_FIND_DATAA found_file; +} Windows_Directory_Opened; +#endif + +ONYX_DEF(__dir_open, (WASM_I32, WASM_I32, WASM_I32), (WASM_I32)) { + char *path_ptr = ONYX_PTR(params->data[0].of.i32); + int path_len = params->data[1].of.i32; + + char path[512] = {0}; + strncpy(path, path_ptr, path_len); + path[path_len] = 0; + +#ifdef _BH_WINDOWS + for (int i=0; ihndl = FindFirstFileA(path, &dir->found_file); + if (dir->hndl == INVALID_HANDLE_VALUE) { + results->data[0] = WASM_I32_VAL(0); + return NULL; + } + + *(u64 *) ONYX_PTR(params->data[2].of.i32) = (u64) dir; + + results->data[0] = WASM_I32_VAL(1); + return NULL; +#endif + +#ifdef _BH_LINUX + DIR* dir = opendir(path); + *(u64 *) ONYX_PTR(params->data[2].of.i32) = (u64) dir; + results->data[0] = WASM_I32_VAL(dir != NULL); + return NULL; +#endif +} + +// (DIR*, PTR) -> BOOL +ONYX_DEF(__dir_read, (WASM_I64, WASM_I32), (WASM_I32)) { +#ifdef _BH_WINDOWS + Windows_Directory_Opened* dir = (Windows_Directory_Opened *) params->data[0].of.i64; + if (dir == NULL) { + results->data[0] = WASM_I32_VAL(0); + return NULL; + } + + do { + BOOL success = FindNextFileA(dir->hndl, &dir->found_file); + if (!success) { + results->data[0] = WASM_I32_VAL(0); + return NULL; + } + } while (!strcmp(dir->found_file.cFileName, ".") || !strcmp(dir->found_file.cFileName, "..")); + + u32 out = params->data[1].of.i32; + assert(out != 0); + + *(u32 *) ONYX_PTR(out + 0) = (dir->found_file.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? 3 : 4; + *(u32 *) ONYX_PTR(out + 4) = 0; + *(u32 *) ONYX_PTR(out + 8) = strlen(dir->found_file.cFileName); + strncpy(ONYX_PTR(out + 12), dir->found_file.cFileName, 256); + + results->data[0] = WASM_I32_VAL(1); + return NULL; +#endif + +#ifdef _BH_LINUX + DIR* dir = (DIR *) params->data[0].of.i64; + if (dir == NULL) { + results->data[0] = WASM_I32_VAL(0); + return NULL; + } + + struct dirent *ent; + while (1) { + ent = readdir(dir); + if (ent == NULL) { + results->data[0] = WASM_I32_VAL(0); + return NULL; + } + + // Skip the current directory and parent directory + if (strcmp(ent->d_name, ".") && strcmp(ent->d_name, "..")) break; + } + + u32 type = 0; + switch (ent->d_type) { + case DT_UNKNOWN: break; + case DT_BLK: type = 1; break; + case DT_CHR: type = 2; break; + case DT_DIR: type = 3; break; + case DT_LNK: type = 5; break; + case DT_REG: type = 4; break; + default: type = 6; break; + } + + u32 out = params->data[1].of.i32; + assert(out != 0); + + *(u32 *) ONYX_PTR(out + 0) = type; + *(u32 *) ONYX_PTR(out + 4) = (u32) ent->d_ino; + *(u32 *) ONYX_PTR(out + 8) = strlen(ent->d_name); + strncpy(ONYX_PTR(out + 12), ent->d_name, 256); + + results->data[0] = WASM_I32_VAL(1); + return NULL; +#endif +} + +ONYX_DEF(__dir_close, (WASM_I64), ()) { +#ifdef _BH_WINDOWS + Windows_Directory_Opened* dir = (Windows_Directory_Opened *) params->data[0].of.i64; + + FindClose(dir->hndl); + free(dir); + + return NULL; +#endif + +#ifdef _BH_LINUX + DIR* dir = (DIR *) params->data[0].of.i64; + if (dir == NULL) return NULL; + + closedir(dir); +#endif + return NULL; +} + // // THREADS @@ -196,7 +345,7 @@ static i32 onyx_run_thread(void *data) { { // Call the _thread_start procedure wasm_val_t args[] = { WASM_I32_VAL(thread_id), WASM_I32_VAL(thread->tls_base), WASM_I32_VAL(thread->funcidx), WASM_I32_VAL(thread->dataptr) }; - wasm_val_vec_t results; + wasm_val_vec_t results = { 0, 0 }; wasm_val_vec_t args_array = WASM_ARRAY_VEC(args); trap = runtime->wasm_func_call(start_func, &args_array, &results); @@ -607,6 +756,11 @@ ONYX_LIBRARY { ONYX_FUNC(__file_read) ONYX_FUNC(__file_write) ONYX_FUNC(__file_flush) + ONYX_FUNC(__file_get_standard) + + ONYX_FUNC(__dir_open) + ONYX_FUNC(__dir_read) + ONYX_FUNC(__dir_close) ONYX_FUNC(__spawn_thread) ONYX_FUNC(__kill_thread) diff --git a/scripts/run_tests.onyx b/scripts/run_tests.onyx index b17e81da..01569939 100644 --- a/scripts/run_tests.onyx +++ b/scripts/run_tests.onyx @@ -97,8 +97,8 @@ Test_Case :: struct { find_onyx_files :: (root: str, cases: ^[..] Test_Case) { for os.list_directory(root) { path_buffer: [512] u8; - if string.ends_with(it.name, ".onyx") { - test_case := string.concat(path_buffer, root, "/", it.name) |> string.alloc_copy(); + if string.ends_with(it->name(), ".onyx") { + test_case := string.concat(path_buffer, root, "/", it->name()) |> string.alloc_copy(); expected_file := test_case[0 .. (test_case.count - 5)]; if !os.file_exists(expected_file) { @@ -109,8 +109,8 @@ find_onyx_files :: (root: str, cases: ^[..] Test_Case) { array.push(cases, .{ test_case, expected_file }); } - if it.dirent.d_type == .Directory { - find_onyx_files(string.concat(path_buffer, root, "/", it.name), cases); + if it.type == .Directory { + find_onyx_files(string.concat(path_buffer, root, "/", it->name()), cases); } } @@ -260,7 +260,7 @@ main :: (args) => { if at_least_one_test_failed { print_color(.Red, "FAILED\n"); - (package wasi).proc_exit(1); + // (package wasi).proc_exit(1); } else { print_color(.Green, "SUCCESS\n"); diff --git a/src/wasm_runtime.c b/src/wasm_runtime.c index ed54afe4..61e703c5 100644 --- a/src/wasm_runtime.c +++ b/src/wasm_runtime.c @@ -17,8 +17,6 @@ #endif static wasm_config_t* wasm_config; -static wasi_config_t* wasi_config; -static wasi_env_t* wasi_env; static wasm_engine_t* wasm_engine; static wasm_store_t* wasm_store; static wasm_extern_vec_t wasm_imports; @@ -191,18 +189,6 @@ b32 onyx_run_wasm(bh_buffer wasm_bytes, int argc, char *argv[]) { wasmer_features_bulk_memory(features, 1); wasm_config_set_features(wasm_config, features); - wasi_config = wasi_config_new("onyx"); - if (argc > 0) { - fori (i, 0, argc) { - wasi_config_arg(wasi_config, argv[i]); - } - } - - wasi_config_preopen_dir(wasi_config, "./"); - - wasi_env = wasi_env_new(wasi_config); - if (!wasi_env) goto error_handling; - wasm_engine = wasm_engine_new_with_config(wasm_config); if (!wasm_engine) goto error_handling; @@ -216,9 +202,6 @@ b32 onyx_run_wasm(bh_buffer wasm_bytes, int argc, char *argv[]) { wasm_module = wasm_module_new(wasm_store, &wasm_data); if (!wasm_module) goto error_handling; - wasmer_named_extern_vec_t wasi_imports; - wasi_get_unordered_imports(wasm_store, wasm_module, wasi_env, &wasi_imports); - wasm_importtype_vec_t module_imports; // @Free wasm_module_imports(wasm_module, &module_imports); @@ -231,16 +214,6 @@ b32 onyx_run_wasm(bh_buffer wasm_bytes, int argc, char *argv[]) { wasm_extern_t* import = NULL; - // First try WASI - fori (j, 0, (i32) wasi_imports.size) { - const wasm_name_t* wasi_module_name = wasmer_named_extern_module(wasi_imports.data[j]); - const wasm_name_t* wasi_import_name = wasmer_named_extern_name(wasi_imports.data[j]); - if (wasm_name_equals(module_name, wasi_module_name) && wasm_name_equals(import_name, wasi_import_name)) { - import = (wasm_extern_t *) wasmer_named_extern_unwrap(wasi_imports.data[j]); - goto import_found; - } - } - if (wasm_name_equals_string(module_name, "onyx")) { if (wasm_name_equals_string(import_name, "memory")) { if (wasm_memory == NULL) {