From f43a9f316838a498522919e73b1fe98a24a2295a Mon Sep 17 00:00:00 2001 From: Brendan Hansen Date: Sat, 4 Dec 2021 23:23:49 -0600 Subject: [PATCH] started planning the pluggable modules --- bin/test | 68 -------------------------- bin/test.bat | 34 ------------- build.sh | 2 +- docs/pluggable_modules.md | 71 ++++++++++++++++++++++++++++ misc/onyx_module.h | 47 ++++++++++++++++++ modules/test_library/module.onyx | 5 ++ modules/test_library/test_library.c | 13 +++++ src/wasm_runtime.c | 14 ++++++ test_library.so | Bin 0 -> 15856 bytes 9 files changed, 151 insertions(+), 103 deletions(-) delete mode 100755 bin/test delete mode 100644 bin/test.bat create mode 100644 docs/pluggable_modules.md create mode 100644 misc/onyx_module.h create mode 100644 modules/test_library/module.onyx create mode 100644 modules/test_library/test_library.c create mode 100755 test_library.so diff --git a/bin/test b/bin/test deleted file mode 100755 index e9ca232f..00000000 --- a/bin/test +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh - -print_check() { - if [ ! -z "$TERM" ]; then - printf "%-$((($(tput cols) - 8)))s" "⏲ Checking $1.onyx" ; - else - printf "%s ... " "⏲ Checking $1.onyx" ; - fi -} - -if command -v wasmer >/dev/null; then - printf "Using $(wasmer --version)\n" -else - printf "Using node $(node -v)\n" -fi - -failed=0 -for test_file in $(find tests/ -name '*.onyx'); do - filename=$(basename -- "$test_file") - dirname="$(dirname -- "$test_file")" - name="${filename%.*}" - - print_check "$name" - - if true; then - if ! ./bin/onyx run "$test_file" > ./tmpoutput; then - printf "\n❌ Failed to compile $name.onyx.\n" - failed=1 - continue - fi - - # if ! wasmer "./tests/$name.wasm" > ./tmpoutput; then - # printf "\n❌ Failed to run $name.onyx.\n" - # failed=1 - # continue - # fi - else - if ! ./bin/onyx -r js --use-post-mvp-features "$test_file" -o "./tests/$name.wasm" >/dev/null; then - printf "\n❌ Failed to compile $name.onyx.\n" - failed=1 - continue - fi - - if ! ./bin/onyx-js "./tests/$name.wasm" > ./tmpoutput; then - printf "\n❌ Failed to run $name.onyx.\n" - failed=1 - continue - fi - fi - - if ! diff ./tmpoutput "$dirname/$name" >/dev/null; then - printf "\n❌ Test output did not match.\n" - diff ./tmpoutput "$dirname/$name" - # failed=0 - continue - fi - - echo "✔ Passed" - - rm "./tests/$name.wasm" 2>/dev/null -done -rm ./tmpoutput - -([ $failed = 0 ] && echo "✔ All tests passed.") \ - || echo "❌ Some tests failed." - -exit $failed - diff --git a/bin/test.bat b/bin/test.bat deleted file mode 100644 index 4dbd39f4..00000000 --- a/bin/test.bat +++ /dev/null @@ -1,34 +0,0 @@ -@echo off - -set ONYX_DIR=\dev\onyx - -set failed=0 -for /r tests %%v in (*.onyx) do ( - echo Checking %%~pv%%~nv.onyx - - set ERRORLEVEL=0 - %ONYX_DIR%\onyx.exe run "%%v" > %ONYX_DIR%\tmpoutput - if %ERRORLEVEL% GEQ 1 ( - echo Failed to compile %%~nv.onyx. - set failed=1 - goto continue - ) - - set ERRORLEVEL=0 - fc %ONYX_DIR%\tmpoutput "%%~pv%%~nv" > nul - if %ERRORLEVEL% GEQ 1 ( - echo Output did not match for %%~nv.onyx. - set failed=1 - ) - -::continue:: - del %ONYX_DIR%\tmpoutput -) - -del %ONYX_DIR%\tmpoutput - -if %failed% NEQ 0 ( - echo Some of the test cases failed. -) else ( - echo All test cases passed. -) \ No newline at end of file diff --git a/build.sh b/build.sh index 90e7e8a0..88388a8e 100755 --- a/build.sh +++ b/build.sh @@ -45,7 +45,7 @@ fi C_FILES="$C_FILES wasm_runtime" FLAGS="$FLAGS -DENABLE_RUN_WITH_WASMER" -LIBS="-L$CORE_DIR/lib -lwasmer -Wl,-rpath=$CORE_DIR/lib -lpthread" +LIBS="-L$CORE_DIR/lib -lwasmer -Wl,-rpath=$CORE_DIR/lib -lpthread -ldl" INCLUDES="-I$WASMER_INCLUDE_DIR" mkdir -p "$BUILD_DIR" diff --git a/docs/pluggable_modules.md b/docs/pluggable_modules.md new file mode 100644 index 00000000..3151fdac --- /dev/null +++ b/docs/pluggable_modules.md @@ -0,0 +1,71 @@ +Pluggable Modules +----------------- + + +After using the 'onyx run' functionality for a while, I've come to the realization +that maybe pushing that feature farther may be the future of Onyx. Onyx can still +target WebAssembly and be used for performance based web-applications. But using +libwasmer and providing a way to do "pluggable modules" gives me the ability to use +Onyx more natively. I'm basing this functionality on Lua's approach to C interop, +which does seem a little cumbersome, but also is easy enough to understand and +program for. + + +The canonical name for this with be a "library", where "module" is reserved for a +collection of "packages". A library is included in Onyx using the following syntax: + + // Includes "some_useful_code.so / .dll" in this execution. + // Currently, this will only be allowed with the "onyx" runtime. + #library "some_useful_code" + + // #library_path can be used to add to the places that libraries are + // searched for. + #if runtime.OS == runtime.OS_Linux { + #library_path "./libs/linux" + } else { + #library_path ".\libs\windows" + } + +Libraries will be shared objects files or dlls, depending on the OS. Hopefully, that +is the only change that has to be added to the Onyx language itself. Everything else +will be handled behind the scenes from the programmer. + +The "behinds the scenes" stuff is as follows: + - Locate and load the library using dlopen or LoadLibrary. + - Lookup the predefined function that will return a pointer to the table describing + the provided functions. + - During the module linking phase (aka building the import table), these functions + will be considered as options. + + + + +A typical Onyx file for a library will look like: + + package some_useful_code + + #library "some_useful_code" + + the_code :: (a, b: i32) -> i32 #foreign "some_useful_code" "the_code" --- + + +The corresponding C file that will be compiled to a so/dll looks like: + + #include "onyx_module.h" + + #define ONYX_LIBRARY_NAME some_useful_code + + ONYX_DEF(the_code, (I32, I32), (I32)) { + i32 a = params->data[0].of.i32; + i32 b = params->data[1].of.i32; + results->data[0] = WASM_I32_VAL(a + b); + return NULL; + } + + ONYX_LIBRARY { + ONYX_FUNC(the_code) + } + +Compiling the C file with: + + gcc -o some_useful_code.so -shared -fPIC some_useful_code.c -I ... \ No newline at end of file diff --git a/misc/onyx_module.h b/misc/onyx_module.h new file mode 100644 index 00000000..3ad2e353 --- /dev/null +++ b/misc/onyx_module.h @@ -0,0 +1,47 @@ + +#include "wasm.h" + +#define NUM_VALS(...) (sizeof((wasm_valkind_t []){ 0, __VA_ARGS__ }) / sizeof(wasm_valkind_t)) +#define _VALS(...) { NUM_VALS(__VA_ARGS__) - 1, __VA_ARGS__ } + +typedef struct WasmValkindBuffer { + unsigned int count; + wasm_valkind_t types[20]; +} WasmValkindBuffer; + +typedef struct WasmFuncDefinition { + char* module_name; + char* import_name; + wasm_func_callback_t func; + + WasmValkindBuffer *params; + WasmValkindBuffer *results; +} WasmFuncDefinition; + +#define STRINGIFY1(a) #a +#define CONCAT2(a, b) a ## _ ## b +#define CONCAT3(a, b, c) a ## _ ## b ## _ ## c +#define ONYX_MODULE_NAME_GEN(m) CONCAT2(__onyx_module, m) +#define ONYX_FUNC_NAME(m, n) CONCAT3(__onyx_internal, m, n) +#define ONYX_DEF_NAME(m, n) CONCAT3(__onyx_internal_def, m, n) +#define ONYX_PARAM_NAME(m, n) CONCAT3(__onyx_internal_param_buffer, m, n) +#define ONYX_RESULT_NAME(m, n) CONCAT3(__onyx_internal_result_buffer, m, n) +#define ONYX_IMPORT_NAME(m, n) STRINGIFY1(m) "_" #n + +#define ONYX_DEF(name, params_types, result_types) \ + static wasm_trap_t* ONYX_FUNC_NAME(ONYX_MODULE_NAME, name)(const wasm_val_vec_t* params, wasm_val_vec_t* results); \ + static struct WasmValkindBuffer ONYX_PARAM_NAME(ONYX_MODULE_NAME, name) = _VALS params_types; \ + static struct WasmValkindBuffer ONYX_RESULT_NAME(ONYX_MODULE_NAME, name) = _VALS result_types; \ + static struct WasmFuncDefinition ONYX_DEF_NAME(ONYX_MODULE_NAME, name) = { STRINGIFY1(ONYX_MODULE_NAME), #name, ONYX_FUNC_NAME(ONYX_MODULE_NAME, name), & ONYX_PARAM_NAME(ONYX_MODULE_NAME, name), & ONYX_RESULT_NAME(ONYX_MODULE_NAME, name) }; \ + \ + static wasm_trap_t* ONYX_FUNC_NAME(ONYX_MODULE_NAME, name)(const wasm_val_vec_t* params, wasm_val_vec_t* results) + +#define ONYX_FUNC(name) & ONYX_DEF_NAME(ONYX_MODULE_NAME, name), +#define ONYX_MODULE struct WasmFuncDefinition *ONYX_MODULE_NAME_GEN(ONYX_MODULE_NAME) [] = + +// Shorter names +#define I32 WASM_I32 +#define I64 WASM_I64 +#define F32 WASM_F32 +#define F64 WASM_F64 + diff --git a/modules/test_library/module.onyx b/modules/test_library/module.onyx new file mode 100644 index 00000000..7b505cb6 --- /dev/null +++ b/modules/test_library/module.onyx @@ -0,0 +1,5 @@ +package test_library + +#library "test_library" + +foo :: () -> void #foreign "test_library" "foo" --- \ No newline at end of file diff --git a/modules/test_library/test_library.c b/modules/test_library/test_library.c new file mode 100644 index 00000000..0a8fd25a --- /dev/null +++ b/modules/test_library/test_library.c @@ -0,0 +1,13 @@ +#include "onyx_module.h" +#include + +#define ONYX_MODULE_NAME test_library + +ONYX_DEF(foo, (), ()) { + printf("This worked!\n"); + return NULL; +} + +ONYX_MODULE { + ONYX_FUNC(foo) +}; \ No newline at end of file diff --git a/src/wasm_runtime.c b/src/wasm_runtime.c index 6a5d3dfb..ef74408c 100644 --- a/src/wasm_runtime.c +++ b/src/wasm_runtime.c @@ -498,8 +498,22 @@ WASM_INTEROP(onyx_process_destroy_impl) { return NULL; } +#include + // Returns 1 if successful b32 onyx_run_wasm(bh_buffer wasm_bytes) { + + // NOCHECKIN + void* handle = dlopen("./test_library.so", RTLD_LAZY); + printf("HANDLE: %p\n", handle); + if (handle == NULL) { + printf("ERROR: %s\n", dlerror()); + } + void *wasm_library = dlsym(handle, "__onyx_module_test_library"); + printf("LOADED: %p %s\n", wasm_library, wasm_library); + dlclose(handle); + + wasm_instance_t* instance = NULL; wasmer_features_t* features = NULL; wasm_trap_t* run_trap = NULL; diff --git a/test_library.so b/test_library.so new file mode 100755 index 0000000000000000000000000000000000000000..cba999b442acfd045204c2291bfd61180a7dcf54 GIT binary patch literal 15856 zcmeHOU2Ggz6~4QP8;AUO6Pnf}1y4eXlB!JePoZ)%j=lB-{D)&pOVCzNmC=kPilrk`A+XfFY6Bhb zP&*>qwacm-YKv`}0xF5iZA2=O!2`RcOF-~UN9`Fpu8QHyLIa!qZXsR9)CQURSK=dS zEx#lPKNi`~5I;lwfET5oz;+!b>Ld3P@#45o?0|8<-%3B?xLf>yQ7(u(@phQ|DD91k z-yuT>j^iED9uVz)nfy2}H(PL-$E3a0uEUb#6DVSoPlFocF*7hcLE|oUk^2IDT4bW4 z!n#}OvxYuw-}SFgzp&2xtN-YCp8WQP=kEOR&pvqPAvVPI+6dhUGr?bAV~J_v-14m_ZaqGx6dxwufKED ze*M}SJN1VBo9p4N6ktLEwAUA>JG%Oo{kgyM-LH^Z)ecVBr@Ieu)js=nxXC`(JxoG< zn?c9RQ`EUP+P+2A)K?a!&5o|8xEpF=db!T$!||@nknIG;phRTsvtPevpRFy?8MSqA zEFC-qO*`OSsMKFU`kL##+jl7S=xNd}S(BpFCDkYpgqK$3wZ14#yw3?v!&|IPrt z+SYXs^}kXh!zUkbhDQ1(2D8p^&rnuP`@R~_7plf7KlrMb+eyZ$9c%9EB;@~@uD)BZ zA0oW8RIfiw$p5)=Nj*NMQnQ)Vj*aWupP|sZiR1r}FOa=Ks5WQ%H+O!qW5cQTd3ALA zeRmz)y^CqKq1S!&HH!BcCVDn!&ZK)c-QrRpmXnfXAjv?Ifg}S-29gXU8AvjaWFW~v zl7S=x|D6oTx<{<9#9BpOv&m(nhQx!ZO+x5C1 znaE;DtZ{rxWL`hXWsk@o)1o>jxIvKLPq~PhQdT53)nwp>X*_m_-EE? z1^RifB!(8RMg{S%*LD9ZVo$Hv(fJ^c^jaIq4~zbNkr7A2{m%gF*y9ZziLI}V>mg%Z z?M_28ST{RwY57G#OK_mK_Y1}zT9Lii*uUT0Z_=WH4=tRxwd&9E8^y9IkHa*LMoan$ z&1T^%F@C4;7}v1>992?u)hm`}aGiQ1eyhSb)mo()_aEbU<>*smjfw3|~ByCK$UeEtY;#=9@y$at? z{H_-9)l?8x!y2tlnNrKk^_{TfO!0z!UVfkRotdIP=@y+_=m%BDt<9<_zf>uDp_ijC zQcDY7#9wgSAaLg#uN=~1{pr9hc}}iYD$S8gljM+PxWcq~kw5k8lv^#)n*UNEtU4uk zmbMj4yR~AdhO$E=V<(-_o`I}$;`oDEY!xV!X~zL=A#f@#wdqXK*?GZ=odXX4x+oh9 z8rFeVrKmS;PQCcgF`Lgs8tosWpEsTCa3AlI;4ExHH~!jD9&)Q7&<2t+q^j*uD(jZeca=q-eFL z<0kfPtcan^>Rc&wCkewqV;;4#AA06Yxn@=Zzv2bqoH85KWUWxleYTKOkwMo6lvV+trG+s;|Xvz$|Svp3UHv>QChORQbyo|_vj(j0$ z4?Q%s2kI2qEfuDyRi9$f!HgQv!6vI!WzsjKL<7IoSMH;5JfCUaW3k%4F2u(D8!~Sr zYxZAle;4fWT!7p|mA})Rfo>zZV1gsKTWI*j+y4^jd5#2s%%gxWi9b(&=vJuUkLSt> z;zG{0r~mO1^tH|Fi$$3V?o9|3uv%2m3aR(tJEo~ zaBTQvUI)ay4(-R!|4H#bCIK-|1R_uDPfw#8^2hn%oM0Z?crMBF3G-z#i3<7w`3zN> z{SEPJAPfcZ*aJQr^S49+e0LRp(9gyE@tgg`=PmliwqmW1;_;2Uf)2j@f5o@;^xC_QkxRh5yzD zqbqF7r5Q2yv_>DFMnLHpzvuD?=Z*UUzvrRv+EX8&KB&=+1xLHE-9nXC{#Oo*Ma&^4 Gs(%CVw7aSR literal 0 HcmV?d00001 -- 2.25.1