From: Brendan Hansen Date: Sat, 11 Dec 2021 19:27:22 +0000 (-0600) Subject: started the process of ditching WASI for Onyx runtime X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=683fe6aa19b8de465df715ed13a3a41b165136a7;p=onyx.git started the process of ditching WASI for Onyx runtime --- diff --git a/core/os/file.onyx b/core/os/file.onyx index 5518e2a5..9795e64e 100644 --- a/core/os/file.onyx +++ b/core/os/file.onyx @@ -1,20 +1,14 @@ package core.os -#local runtime :: package runtime -#if runtime.Runtime != runtime.Runtime_Wasi - && runtime.Runtime != runtime.Runtime_Onyx { - #error "The file system library is currently only available on the WASI runtime, and should only be included if that is the chosen runtime." -} - use package core +#local fs :: package runtime.fs -#local wasi :: package wasi -use package wasi { - FileDescriptor, - FDFlags, OFlags, Rights, - LookupFlags, Errno, - IOVec, IOVecArray, Size, - FileStat, Whence +FileError :: enum { + None; + NotFound; + Exists; + Permission; + BadFile; } OpenMode :: enum { @@ -25,185 +19,76 @@ OpenMode :: enum { } File :: struct { - fd : FileDescriptor; - - mode : OpenMode = .Invalid; - rights : Rights = ~~ 0; - flags : FDFlags = ~~ 0; + use stream : io.Stream; + use data : fs.FileData; } -file_open :: (path: str, mode := OpenMode.Read, flags := FDFlags.Sync) -> (File, bool) { - // Requesting all of the rights because why not. - rights := - Rights.DataSync - | Rights.Read - | Rights.Seek - | Rights.FdStatSetFlags - | Rights.Sync - | Rights.Tell - | Rights.Write - | Rights.Advise - | Rights.Allocate - | Rights.PathCreateDirectory - | Rights.PathCreateFile - | Rights.PathLinkSource - | Rights.PathLinkTarget - | Rights.PathOpen - | Rights.ReadDir - | Rights.PathReadlink - | Rights.PathRenameSource - | Rights.PathRenameTarget - | Rights.PathFilestatGet - | Rights.PathFilestateSetSize - | Rights.PathFilestateSetTimes - | Rights.FilestatGet - | Rights.FilestatSetSize - | Rights.FilestatSetTimes - | Rights.PathSymlink - | Rights.PathRemoveDirectory - | Rights.PathUnlinkFile - | Rights.PollFDReadWrite; - - open_flags := cast(OFlags) 0; - fd_flags := flags; - - switch mode { - case .Write { - open_flags |= OFlags.Creat | OFlags.Trunc; - rights |= Rights.Write | Rights.Seek | Rights.Tell; - } - - case .Append { - open_flags |= OFlags.Creat; - rights |= Rights.Write | Rights.Seek | Rights.Tell; - fd_flags |= FDFlags.Append; - } - - case .Read { - rights |= Rights.Read | Rights.Seek | Rights.Tell; - } - } - - file := File.{ fd = -1 }; - file.mode = mode; - file.rights = rights; - file.flags = fd_flags; - - // Currently the directory's file descriptor appears to always be 3 - // However, this is not necessarily correct, so also try a preopened directory - for DIR_FD: .[ 3, 4 ] { - if err := wasi.path_open( - DIR_FD, - .SymLinkFollow, - path, - open_flags, - rights, - rights, - fd_flags, - ^file.fd); - err == .Success { - return file, true; - } +// Are these even necessary?? +#local { + file_open :: (path: str, mode := OpenMode.Read) -> (fs.FileData, FileError) { + return fs.__file_open(path, mode); } - return file, false; -} - -file_close :: (file: File) -> bool { - if wasi.fd_close(file.fd) != .Success { - return false; + file_close :: (file: fs.FileData) -> FileError { + return fs.__file_close(file); } - - return true; -} - -file_write :: (file: File, data: str) { - vec := IOVec.{ buf = cast(u32) data.data, len = data.count }; - tmp : Size; - wasi.fd_write(file.fd, .{ cast(u32) ^vec, 1 }, ^tmp); - wasi.fd_datasync(file.fd); } -file_get_size :: (file: File) -> u64 { - fs: FileStat; - if wasi.fd_filestat_get(file.fd, ^fs) != Errno.Success do return 0; - - return fs.size; +file_exists :: (path: str) -> bool { + return fs.__file_exists(path); } -get_contents_from_file :: (file: File) -> str { - size := cast(u32) file_get_size(file); +get_contents_from_file :: (file: ^File) -> str { + size := cast(u32) io.stream_size(file); data := cast(^u8) raw_alloc(context.allocator, size); - prev_loc: i64; - wasi.fd_tell(file.fd, ^prev_loc); + _, prev_loc := io.stream_tell(file); + io.stream_seek(file, 0, .Start); - dummy: i64; - wasi.fd_seek(file.fd, 0, .Set, ^dummy); + io.stream_read(file, .{ data, size }); - dummy2: u32; - buf := IOVec.{ cast(u32) data, size }; - wasi.fd_pread(file.fd, .{ cast(u32) ^buf, 1 }, 0, ^dummy2); - - wasi.fd_seek(file.fd, prev_loc, .Set, ^dummy); + io.stream_seek(file, prev_loc, .Start); return data[0 .. size]; } -get_contents :: #match { - get_contents_from_file, - - (path: str) -> str { - tmp_file, success := file_open(path, .Read); - if !success do return .{ null, 0 }; - defer file_close(tmp_file); - - return get_contents(tmp_file); - } -} - -file_exists :: (path: str) -> bool { - fs: wasi.FileStat; +open :: (path: str, mode := OpenMode.Read) -> (os.FileError, File) { + file := File.{ + stream = .{ vtable = null }, + data = .{}, + }; - exists := false; - for .[3, 4] { // Trying both preopened directories - err := wasi.path_filestat_get(it, .SymLinkFollow, path, ^fs); - if err == .Success do exists = true; - } + file_data, error := file_open(path, mode); + if error != .None do return error, file; - return exists; + file.data = file_data; + file.vtable = ^fs.__file_stream_vtable; + return .None, file; } -FileStream :: struct { - use stream : io.Stream; - use file : File; +close :: (file: ^File) { + file_close(file.data); + file.stream.vtable = null; } -open :: (path: str, mode := OpenMode.Read) -> (io.Error, FileStream) { - fs := FileStream.{ - stream = .{ vtable = null }, - file = .{ fd = -1 }, - }; - - file, success := file_open(path, mode); - if !success do return .NotFound, fs; +get_contents :: #match { + get_contents_from_file, - fs.file = file; - fs.vtable = ^wasi_file_stream_vtable; - return .None, fs; -} + (path: str) -> str { + error, file := open(path, .Read); + if error != .None do return .{ null, 0 }; + defer close(^file); -close :: (file: ^FileStream) { - file_close(file.file); - file.stream.vtable = null; + return get_contents(^file); + } } -with_file :: (path: str, mode := OpenMode.Read) -> Iterator(^FileStream) { +with_file :: (path: str, mode := OpenMode.Read) -> Iterator(^File) { Context :: struct { valid_file_stream := false; - file_stream: FileStream; + file_stream: File; } c := new(Context); @@ -212,7 +97,7 @@ with_file :: (path: str, mode := OpenMode.Read) -> Iterator(^FileStream) { c.valid_file_stream = true; } - next :: (use c: ^Context) -> (^FileStream, bool) { + next :: (use c: ^Context) -> (^File, bool) { if !valid_file_stream do return null, false; defer valid_file_stream = false; @@ -227,118 +112,21 @@ with_file :: (path: str, mode := OpenMode.Read) -> Iterator(^FileStream) { return .{ c, next, close_context }; } -#package -wasi_file_stream_vtable := io.Stream_Vtable.{ - seek = (use fs: ^FileStream, to: i32, whence: io.SeekFrom) -> io.Error { - // Currently, the new offset is just ignored. - newoffset : wasi.Filesize; - error := wasi.fd_seek(file.fd, ~~ to, ~~ whence, ^newoffset); - if error != .Success do return .BadFile; - - return .None; - }, - - tell = (use fs: ^FileStream) -> (io.Error, u32) { - location : wasi.Filesize; - error := wasi.fd_tell(file.fd, ^location); - if error != .Success do return .BadFile, 0; - - return .None, ~~location; - }, - - read = (use fs: ^FileStream, buffer: [] u8) -> (io.Error, u32) { - bytes_read : wasi.Size; - vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; - error := wasi.fd_read(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_read); - if error != .Success do return .BadFile, 0; - - return .None, bytes_read; - }, - - read_at = (use fs: ^FileStream, at: u32, buffer: [] u8) -> (io.Error, u32) { - bytes_read : wasi.Size; - vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; - error := wasi.fd_pread(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ~~at, ^bytes_read); - - // FIX: Maybe report io.Error.OutOfBounds if the 'at' was out of bounds? - if error != .Success do return .BadFile, 0; - - return .None, bytes_read; - }, - - read_byte = (use fs: ^FileStream) -> (io.Error, u8) { - bytes_read : wasi.Size; - byte : u8; - vec := IOVec.{ buf = cast(u32) ^byte, len = 1}; - error := wasi.fd_read(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_read); - if error != .Success do return .BadFile, 0; - - return .None, byte; - }, - - write = (use fs: ^FileStream, buffer: [] u8) -> (io.Error, u32) { - bytes_written : wasi.Size; - vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; - error := wasi.fd_write(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_written); - if error != .Success do return .BadFile, 0; - - return .None, bytes_written; - }, - - write_at = (use fs: ^FileStream, at: u32, buffer: [] u8) -> (io.Error, u32) { - bytes_written : wasi.Size; - vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; - error := wasi.fd_pwrite(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ~~at, ^bytes_written); - - // FIX: Maybe report io.Error.OutOfBounds if the 'at' was out of bounds? - if error != .Success do return .BadFile, 0; - - return .None, bytes_written; - }, - - write_byte = (use fs: ^FileStream, byte: u8) -> io.Error { - bytes_written : wasi.Size; - byte_to_write := byte; - vec := IOVec.{ buf = cast(u32) ^byte_to_write, len = 1 }; - error := wasi.fd_write(file.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_written); - if error != .Success do return .BadFile; - - return .None; - }, - - close = (use fs: ^FileStream) -> io.Error { - file_close(file); - return .None; - }, - - flush = (use fs: ^FileStream) -> io.Error { - wasi.fd_datasync(file.fd); - return .None; - }, - - size = (use fs: ^FileStream) -> i32 { - file_stat: FileStat; - if wasi.fd_filestat_get(file.fd, ^file_stat) != .Success do return 0; - - return ~~ file_stat.size; - }, -} - file_logger_open :: (filename: str, allocator := context.allocator) -> Logger { file := new(File, allocator); - success := false; - *file, success = file_open(filename, mode=.Append); - assert(success, "Unable to open file for logging."); + error := os.FileError.None; + error, *file = open(filename, mode=.Append); + assert(error != .None, "Unable to open file for logging."); return .{ file_logger_proc, file }; } file_logger_close :: (logger := context.logger) { - file_close(*cast(^File) logger.data); + close(cast(^File) logger.data); // @Robustness: this could be the wrong allocator if the context allocator wasn't used. cfree(logger.data); @@ -346,6 +134,7 @@ file_logger_close :: (logger := context.logger) { #local file_logger_proc :: (data: ^File, msg: str) { - file_write(*data, msg); - file_write(*data, "\n"); + writer := io.writer_make(data); + io.write(^writer, msg); + io.write(^writer, "\n"); } diff --git a/core/os/onyx_fs.onyx b/core/os/onyx_fs.onyx new file mode 100644 index 00000000..afd229cd --- /dev/null +++ b/core/os/onyx_fs.onyx @@ -0,0 +1,95 @@ +package runtime.fs + +use package core + +FileData :: struct { + Handle :: #distinct i64 + + handle: Handle; +} + +__file_open :: (path: str, mode := os.OpenMode.Read) -> (FileData, os.FileError) { + handle: FileData.Handle; + error := __file_open_impl(path, mode, ^handle); + + fd := FileData.{ handle }; + return fd, error; +} + +#package { + #foreign "onyx_runtime" { + __file_open_impl :: (path: str, mode: os.OpenMode, out_handle: ^Handle) -> os.FileError --- + + __file_close :: (fd: FileData) -> os.FileError --- + __file_exists :: (path: str) -> bool --- + + __file_seek :: (handle: Handle, to: i32, whence: io.SeekFrom) -> io.Error --- + __file_tell :: (handle: Handle, out_position: ^u32) -> io.Error --- + __file_read :: (handle: Handle, output_buffer: [] u8, bytes_read: ^u32) -> io.Error --- + __file_write :: (handle: Handle, input_buffer: [] u8, bytes_wrote: ^u32) -> io.Error --- + __file_flush :: (handle: Handle) -> io.Error; + } +} + +__file_stream_vtable := io.Stream_Vtable.{ + seek = (use fs: ^os.File, to: i32, whence: io.SeekFrom) -> io.Error { + return __file_seek(data.handle, to, whence); + }, + + tell = (use fs: ^os.File) -> (io.Error, u32) { + position: u32; + error := __file_tell(data.handle, ^position); + return error, position; + }, + + read = (use fs: ^os.File, buffer: [] u8) -> (io.Error, u32) { + bytes_read: u32; + error := __file_read(data.handle, buffer, ^bytes_read); + return error, bytes_read; + }, + + read_at = (use fs: ^os.File, at: u32, buffer: [] u8) -> (io.Error, u32) { + __file_seek(data.handle, at, .Start); + bytes_read: u32; + error := __file_read(data.handle, buffer, ^bytes_read); + return error, bytes_read; + }, + + read_byte = (use fs: ^os.File) -> (io.Error, u8) { + byte: u8; + error := __file_read(data.handle, ~~ cast([1] u8) ^byte, null); + return byte; + }, + + write = (use fs: ^os.File, buffer: [] u8) -> (io.Error, u32) { + bytes_wrote: u32; + error := __file_write(data.handle, buffer, ^bytes_wrote); + return error, bytes_wrote; + }, + + write_at = (use fs: ^os.File, at: u32, buffer: [] u8) -> (io.Error, u32) { + __file_seek(data.handle, at, .Start); + bytes_wrote: u32; + error := __file_write(data.handle, buffer, ^bytes_wrote); + return error, bytes_wrote; + }, + + write_byte = (use fs: ^os.File, byte: u8) -> io.Error { + b := byte; + error := __file_write(data.handle, .{ ^b, 1 }, null); + return error; + }, + + close = (use fs: ^os.File) -> io.Error { + __file_close(data); + return .None; + }, + + flush = (use fs: ^os.File) -> io.Error { + return __file_flush(data.handle); + }, + + size = (use fs: ^os.File) -> i32 { + }, +}; + diff --git a/core/runtime/onyx_run.onyx b/core/runtime/onyx_run.onyx index d034f230..c2f6497d 100644 --- a/core/runtime/onyx_run.onyx +++ b/core/runtime/onyx_run.onyx @@ -17,8 +17,6 @@ use package wasi #export "_thread_exit" _thread_exit } -#load "core/os/process" - #local ProcessResult :: enum { Success :: 0x00; FailedToRun :: 0x01; diff --git a/core/std.onyx b/core/std.onyx index c076b04f..e951680e 100644 --- a/core/std.onyx +++ b/core/std.onyx @@ -45,8 +45,14 @@ package core #load "./os/os" } -#if runtime.Runtime == runtime.Runtime_Onyx { #load "./runtime/onyx_run" } -#if runtime.Runtime == runtime.Runtime_Wasi { #load "./runtime/wasi" } +#if runtime.Runtime == runtime.Runtime_Onyx { + #load "./runtime/onyx_run" + #load "core/os/process" +} +#if runtime.Runtime == runtime.Runtime_Wasi { + #load "./runtime/wasi" + #load "./wasi/wasi_fs" +} #if runtime.Runtime == runtime.Runtime_Js { #load "./runtime/js" } #if runtime.Runtime != runtime.Runtime_Custom { #load "./stdio" } diff --git a/core/wasi/wasi_fs.onyx b/core/wasi/wasi_fs.onyx new file mode 100644 index 00000000..d46ca72a --- /dev/null +++ b/core/wasi/wasi_fs.onyx @@ -0,0 +1,220 @@ +package runtime.fs + +#local runtime :: package runtime +#if runtime.Runtime != runtime.Runtime_Wasi { + #error "The file system library is currently only available on the WASI runtime, and should only be included if that is the chosen runtime." +} + +use package core + +#local wasi :: package wasi +use package wasi { + FileDescriptor, + FDFlags, OFlags, Rights, + LookupFlags, Errno, + IOVec, IOVecArray, Size, + FileStat, Whence +} + +FileData :: struct { + fd : FileDescriptor = -1; + + mode : os.OpenMode = .Invalid; + rights : Rights = ~~ 0; + flags : FDFlags = ~~ 0; +} + +__file_open :: (path: str, mode := os.OpenMode.Read) -> (FileData, os.FileError) { + // Requesting all of the rights because why not. + rights := + Rights.DataSync + | Rights.Read + | Rights.Seek + | Rights.FdStatSetFlags + | Rights.Sync + | Rights.Tell + | Rights.Write + | Rights.Advise + | Rights.Allocate + | Rights.PathCreateDirectory + | Rights.PathCreateFile + | Rights.PathLinkSource + | Rights.PathLinkTarget + | Rights.PathOpen + | Rights.ReadDir + | Rights.PathReadlink + | Rights.PathRenameSource + | Rights.PathRenameTarget + | Rights.PathFilestatGet + | Rights.PathFilestateSetSize + | Rights.PathFilestateSetTimes + | Rights.FilestatGet + | Rights.FilestatSetSize + | Rights.FilestatSetTimes + | Rights.PathSymlink + | Rights.PathRemoveDirectory + | Rights.PathUnlinkFile + | Rights.PollFDReadWrite; + + open_flags := cast(OFlags) 0; + fd_flags := FDFlags.Sync; + + switch mode { + case .Write { + open_flags |= OFlags.Creat | OFlags.Trunc; + rights |= Rights.Write | Rights.Seek | Rights.Tell; + } + + case .Append { + open_flags |= OFlags.Creat; + rights |= Rights.Write | Rights.Seek | Rights.Tell; + fd_flags |= FDFlags.Append; + } + + case .Read { + rights |= Rights.Read | Rights.Seek | Rights.Tell; + } + } + + file := FileData.{ fd = -1 }; + file.mode = mode; + file.rights = rights; + file.flags = fd_flags; + + // Currently the directory's file descriptor appears to always be 3 + // However, this is not necessarily correct, so also try a preopened directory + for DIR_FD: .[ 3, 4 ] { + if err := wasi.path_open( + DIR_FD, + .SymLinkFollow, + path, + open_flags, + rights, + rights, + fd_flags, + ^file.fd); + err == .Success { + return file, .None; + } + } + + @TODO // provide a better error code. + return file, .NotFound; +} + +__file_close :: (file: FileData) -> os.FileError { + if wasi.fd_close(file.fd) != .Success { + return .BadFile; + } + + return .None; +} + +__file_exists :: (path: str) -> bool { + fs: wasi.FileStat; + + exists := false; + for .[3, 4] { // Trying both preopened directories + err := wasi.path_filestat_get(it, .SymLinkFollow, path, ^fs); + if err == .Success do exists = true; + } + + return exists; +} + +__file_stream_vtable := io.Stream_Vtable.{ + seek = (use fs: ^os.File, to: i32, whence: io.SeekFrom) -> io.Error { + // Currently, the new offset is just ignored. + newoffset : wasi.Filesize; + error := wasi.fd_seek(data.fd, ~~ to, ~~ whence, ^newoffset); + if error != .Success do return .BadFile; + + return .None; + }, + + tell = (use fs: ^os.File) -> (io.Error, u32) { + location : wasi.Filesize; + error := wasi.fd_tell(data.fd, ^location); + if error != .Success do return .BadFile, 0; + + return .None, ~~location; + }, + + read = (use fs: ^os.File, buffer: [] u8) -> (io.Error, u32) { + bytes_read : wasi.Size; + vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; + error := wasi.fd_read(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_read); + if error != .Success do return .BadFile, 0; + + return .None, bytes_read; + }, + + read_at = (use fs: ^os.File, at: u32, buffer: [] u8) -> (io.Error, u32) { + bytes_read : wasi.Size; + vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; + error := wasi.fd_pread(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ~~at, ^bytes_read); + + // FIX: Maybe report io.Error.OutOfBounds if the 'at' was out of bounds? + if error != .Success do return .BadFile, 0; + + return .None, bytes_read; + }, + + read_byte = (use fs: ^os.File) -> (io.Error, u8) { + bytes_read : wasi.Size; + byte : u8; + vec := IOVec.{ buf = cast(u32) ^byte, len = 1}; + error := wasi.fd_read(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_read); + if error != .Success do return .BadFile, 0; + + return .None, byte; + }, + + write = (use fs: ^os.File, buffer: [] u8) -> (io.Error, u32) { + bytes_written : wasi.Size; + vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; + error := wasi.fd_write(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_written); + if error != .Success do return .BadFile, 0; + + return .None, bytes_written; + }, + + write_at = (use fs: ^os.File, at: u32, buffer: [] u8) -> (io.Error, u32) { + bytes_written : wasi.Size; + vec := IOVec.{ buf = cast(u32) buffer.data, len = buffer.count }; + error := wasi.fd_pwrite(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ~~at, ^bytes_written); + + // FIX: Maybe report io.Error.OutOfBounds if the 'at' was out of bounds? + if error != .Success do return .BadFile, 0; + + return .None, bytes_written; + }, + + write_byte = (use fs: ^os.File, byte: u8) -> io.Error { + bytes_written : wasi.Size; + byte_to_write := byte; + vec := IOVec.{ buf = cast(u32) ^byte_to_write, len = 1 }; + error := wasi.fd_write(data.fd, IOVecArray.{ cast(u32) ^vec, 1 }, ^bytes_written); + if error != .Success do return .BadFile; + + return .None; + }, + + close = (use fs: ^os.File) -> io.Error { + __file_close(data); + return .None; + }, + + flush = (use fs: ^os.File) -> io.Error { + wasi.fd_datasync(data.fd); + return .None; + }, + + size = (use fs: ^os.File) -> i32 { + file_stat: FileStat; + if wasi.fd_filestat_get(data.fd, ^file_stat) != .Success do return 0; + + return ~~ file_stat.size; + }, +} + diff --git a/docs/todo b/docs/todo index 0dd4f089..02dbdee4 100644 --- a/docs/todo +++ b/docs/todo @@ -208,9 +208,19 @@ Wishlist: [X] Complete the set of combinations of procedure declaration syntaxes - Currently (x: i32) => { ... } isn't legal. [X] #init functions (run before main, added to list at compile time, emitted with __run_init_procedures()) - [ ] Tests for new language features + [X] Tests for new language features - Do blocks - Code blocks - Quick functions - Macros - Tags + + +Revamping File System: + [ ] runtime.fs should contain runtime specific file system API calls + [ ] 'os.File' should effectily be 'os.FileStream' and FileStream should go away + os.File :: struct { + use stream: io.Stream; + use data: runtime.fs.FileData; + } + [ ] Most file functionality will be provided using the stream API.