From: Brendan Hansen Date: Sat, 18 Feb 2023 04:40:33 +0000 (-0600) Subject: starting work on documentation file generation X-Git-Url: https://git.brendanfh.com/?a=commitdiff_plain;h=fe351e0ccfc0f53d8a6de7862315ee483b756f2e;p=onyx.git starting work on documentation file generation --- diff --git a/compiler/include/astnodes.h b/compiler/include/astnodes.h index bb0422f2..413f1a92 100644 --- a/compiler/include/astnodes.h +++ b/compiler/include/astnodes.h @@ -1570,6 +1570,9 @@ struct Package { u32 id; + // NOTE: id's of the sub-packages + bh_arr(u32) sub_packages; + // NOTE: This tracks all of the 'use package' statements of this package throughout // the code base. This is used when a static if clears and new symbols are introduced. // 'use package' statements have to be reevaluated to pull in the new symbols. diff --git a/compiler/include/doc.h b/compiler/include/doc.h index 3dd647a0..93a8348d 100644 --- a/compiler/include/doc.h +++ b/compiler/include/doc.h @@ -4,36 +4,9 @@ #include "bh.h" #include "astnodes.h" -typedef enum DocFormat { - Doc_Format_Human, - Doc_Format_Tags, - Doc_Format_Html, -} DocFormat; +// Onyx Documentation generation -typedef struct DocEntry { - OnyxFilePos pos; - char* sym; // Unused by doc generator - char* def; - char* additional; -} DocEntry; - -typedef struct DocPackage { - const char* name; - - bh_arr(DocEntry) public_entries; - bh_arr(DocEntry) private_entries; -} DocPackage; - -typedef struct OnyxDocumentation { - bh_arena doc_arena; - - DocFormat format; - - bh_arr(DocPackage) package_docs; -} OnyxDocumentation; - -OnyxDocumentation onyx_docs_generate(); -void onyx_docs_emit(OnyxDocumentation* doc, const char* filename); +void onyx_docs_emit_odoc(const char *dest); // Tag generation diff --git a/compiler/include/utils.h b/compiler/include/utils.h index 2b0083c2..c35cbf06 100644 --- a/compiler/include/utils.h +++ b/compiler/include/utils.h @@ -20,7 +20,7 @@ void scope_include(Scope* target, Scope* source, OnyxFilePos pos); b32 symbol_introduce(Scope* scope, OnyxToken* tkn, AstNode* symbol); b32 symbol_raw_introduce(Scope* scope, char* tkn, OnyxFilePos pos, AstNode* symbol); void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node); -void symbol_subpackage_introduce(Scope* scope, char* sym, AstPackage *node); +void symbol_subpackage_introduce(Package *parent, char* sym, AstPackage *node); AstNode* symbol_raw_resolve(Scope* start_scope, char* sym); AstNode* symbol_resolve(Scope* start_scope, OnyxToken* tkn); AstNode* try_symbol_raw_resolve_from_node(AstNode* node, char* symbol); diff --git a/compiler/src/doc.c b/compiler/src/doc.c index 1808946f..b02d28f1 100644 --- a/compiler/src/doc.c +++ b/compiler/src/doc.c @@ -136,245 +136,102 @@ void onyx_docs_emit_symbol_info(const char *dest) { bh_imap_free(&syminfo->node_to_id); } -// static i32 cmp_doc_entry(const void * a, const void * b) { -// DocEntry* d1 = (DocEntry *) a; -// DocEntry* d2 = (DocEntry *) b; -// -// return strncmp(d1->def, d2->def, 1024); -// } -// -// static i32 cmp_doc_package(const void * a, const void * b) { -// DocPackage* d1 = (DocPackage *) a; -// DocPackage* d2 = (DocPackage *) b; -// -// return strncmp(d1->name, d2->name, 1024); -// } -// -// static char* node_to_doc_def(const char* sym, AstNode *node, bh_allocator a) { -// static char buf[1024]; -// memset(buf, 0, 1023); -// -// strncat(buf, sym, 1023); -// strncat(buf, " :: ", 1023); -// -// switch (node->kind) { -// case Ast_Kind_Function: { -// strncat(buf, "(", 1023); -// -// AstFunction *func = (AstFunction *) node; -// bh_arr_each(AstParam, param, func->params) { -// if (param != func->params) -// strncat(buf, ", ", 1023); -// -// token_toggle_end(param->local->token); -// strncat(buf, param->local->token->text, 1023); -// token_toggle_end(param->local->token); -// -// strncat(buf, ": ", 1023); -// -// strncat(buf, type_get_name(param->local->type), 1023); -// -// if (param->default_value != NULL) { -// strncat(buf, " = ", 1023); -// } -// } -// -// strncat(buf, ") -> ", 1023); -// strncat(buf, type_get_name(func->type->Function.return_type), 1023); -// -// break; -// } -// -// case Ast_Kind_Struct_Type: { -// strncat(buf, "struct { ", 1023); -// -// AstStructType* st = (AstStructType *) node; -// bh_arr_each(AstStructMember *, smem, st->members) { -// -// token_toggle_end((*smem)->token); -// strncat(buf, (*smem)->token->text, 1023); -// token_toggle_end((*smem)->token); -// -// strncat(buf, ": ", 1023); -// -// strncat(buf, type_get_name((*smem)->type), 1023); -// -// strncat(buf, "; ", 1023); -// } -// -// strncat(buf, "}", 1023); -// break; -// } -// -// case Ast_Kind_Basic_Type: -// case Ast_Kind_Array_Type: -// case Ast_Kind_Type_Alias: -// case Ast_Kind_Slice_Type: -// case Ast_Kind_DynArr_Type: { -// strncat(buf, type_get_name(type_build_from_ast(global_heap_allocator, (AstType *) node)), 1023); -// break; -// } -// -// default: { -// strncat(buf, "", 1023); -// } -// } -// -// return bh_strdup(a, buf); -// } -// -// static DocPackage doc_package_create(Package* p, bh_allocator a) { -// DocPackage dp; -// dp.name = p->name; -// dp.public_entries = NULL; -// dp.private_entries = NULL; -// -// bh_arr_new(global_heap_allocator, dp.public_entries, 16); -// bh_arr_new(global_heap_allocator, dp.private_entries, 16); -// -// fori (i, 0, shlen(p->scope->symbols)) { -// char *key = p->scope->symbols[i].key; -// AstNode *value = p->scope->symbols[i].value; -// if (!key || !value) continue; -// -// DocEntry de; -// if (value->token) de.pos = value->token->pos; -// de.def = node_to_doc_def(key, value, a); -// de.sym = (char *) key; -// de.additional = NULL; -// -// bh_arr_push(dp.public_entries, de); -// } -// -// fori (i, 0, shlen(p->private_scope->symbols)) { -// char *key = p->scope->symbols[i].key; -// AstNode *value = p->scope->symbols[i].value; -// if (!key || !value) continue; -// -// DocEntry de; -// if (value->token) de.pos = value->token->pos; -// de.def = node_to_doc_def(key, value, a); -// de.sym = (char *) key; -// de.additional = NULL; -// -// bh_arr_push(dp.private_entries, de); -// } -// -// qsort(dp.public_entries, bh_arr_length(dp.public_entries), sizeof(DocEntry), cmp_doc_entry); -// qsort(dp.private_entries, bh_arr_length(dp.private_entries), sizeof(DocEntry), cmp_doc_entry); -// -// return dp; -// } -// -// OnyxDocumentation onyx_docs_generate() { -// OnyxDocumentation doc; -// -// bh_arena_init(&doc.doc_arena, global_heap_allocator, 16 * 1024); -// bh_allocator a = bh_arena_allocator(&doc.doc_arena); -// -// doc.package_docs = NULL; -// bh_arr_new(global_heap_allocator, doc.package_docs, 16); -// -// fori (i, 0, shlen(context.packages)) { -// DocPackage dp = doc_package_create(context.packages[i].value, a); -// bh_arr_push(doc.package_docs, dp); -// } -// -// qsort(doc.package_docs, bh_arr_length(doc.package_docs), sizeof(DocPackage), cmp_doc_package); -// -// return doc; -// } -// -// static void onyx_docs_emit_human(OnyxDocumentation* doc, bh_file* file) { -// // NOTE: Disabling fancy line printing until I can make it better -// #if 0 -// bh_arr_each(DocPackage, dp, doc->package_docs) { -// bh_printf("Package '%s'\n", dp->name); -// -// if (bh_arr_length(dp->public_entries) > 0) { -// bh_printf("\xe2\x94\x9c\xe2\x94\x80 Public symbols\n"); -// bh_arr_each(DocEntry, de, dp->public_entries) { -// bh_printf("\xe2\x94\x82 \xe2\x94\x9c\xe2\x94\x80 %s\n", de->def); -// -// if (de->pos.filename != NULL) -// bh_printf("\xe2\x94\x82 \xe2\x94\x82 at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column); -// else -// bh_printf("\xe2\x94\x82 \xe2\x94\x82 compiler built-in\n"); -// -// bh_printf("\xe2\x94\x82 \xe2\x94\x82\n"); -// } -// } -// -// if (bh_arr_length(dp->private_entries) > 0) { -// bh_printf("\xe2\x94\x9c\xe2\x94\x80 Private symbols\n"); -// bh_arr_each(DocEntry, de, dp->private_entries) { -// bh_printf("\xe2\x94\x82 \xe2\x94\x9c\xe2\x94\x80 %s\n", de->def); -// if (de->pos.filename != NULL) -// bh_printf("\xe2\x94\x82 \xe2\x94\x82 at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column); -// else -// bh_printf("\xe2\x94\x82 \xe2\x94\x82 compiler built-in\n"); -// -// bh_printf("\xe2\x94\x82 \xe2\x94\x82\n"); -// } -// -// bh_printf("\n"); -// } -// } -// #else -// bh_arr_each(DocPackage, dp, doc->package_docs) { -// bh_fprintf(file, "Package '%s'\n", dp->name); -// -// if (bh_arr_length(dp->public_entries) > 0) { -// bh_fprintf(file, " Public symbols\n"); -// bh_arr_each(DocEntry, de, dp->public_entries) { -// bh_fprintf(file, " %s\n", de->def); -// -// if (de->pos.filename != NULL) -// bh_fprintf(file, " at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column); -// else -// bh_fprintf(file, " compiler built-in\n"); -// -// bh_fprintf(file, " \n"); -// } -// } -// -// if (bh_arr_length(dp->private_entries) > 0) { -// bh_fprintf(file, " Private symbols\n"); -// bh_arr_each(DocEntry, de, dp->private_entries) { -// bh_fprintf(file, " %s\n", de->def); -// if (de->pos.filename != NULL) -// bh_fprintf(file, " at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column); -// else -// bh_fprintf(file, " compiler built-in\n"); -// -// bh_fprintf(file, " \n"); -// } -// -// bh_fprintf(file, "\n"); -// } -// } -// #endif -// } -// -// -// -// static void onyx_docs_emit_html(OnyxDocumentation* doc, bh_file* file) { -// bh_fprintf(file, "HTML documentation output not supported yet.\n"); -// return; -// } -// -// void onyx_docs_emit(OnyxDocumentation* doc, const char* filename) { -// bh_file file; -// if (bh_file_create(&file, filename) != BH_FILE_ERROR_NONE) { -// bh_printf("ERROR: Failed to open file '%s' for writing documentation.\n", filename); -// return; -// } -// -// switch (doc->format) { -// case Doc_Format_Human: onyx_docs_emit_human(doc, &file); break; -// case Doc_Format_Html: onyx_docs_emit_html(doc, &file); break; -// } -// -// bh_file_close(&file); -// } + + +// +// Onyx Documentation Format +// + +#define Doc_Magic_Bytes "ODOC" + + +struct Doc_String { + u32 offset; + u32 length; +}; + +struct Doc_Array { + u32 offset; + u32 length; +}; + +struct Doc_Header { + u8 magic_bytes[4]; + u32 version; + + struct Doc_String program_name; + u32 build_time; +}; + +struct Doc_File { + u32 package_id; + struct Doc_String name; +}; + +struct Doc_Location { + u32 file_id; + i32 line; + i32 col; +}; + +struct Doc_Package { + struct Doc_String name; + struct Doc_String qualified_name; + + struct Doc_Array subpackages; +}; + +enum Doc_Entity_Kind { + Entity_Kind_Procedure = 1, +}; + +struct Doc_Entity { + enum Doc_Entity_Kind kind; + + u32 package_id; + struct Doc_Location location; + + struct Doc_String name; + struct Doc_String notes; +}; + +struct Doc { + struct Doc_Header header; + + struct Doc_Array packages; + struct Doc_Array entities; + struct Doc_Array files; +}; + +void onyx_docs_emit_odoc(const char *dest) { + bh_file doc_file; + if (bh_file_create(&doc_file, dest) != BH_FILE_ERROR_NONE) { + bh_printf("Cannot create '%s'.\n", dest); + return; + } + + struct Doc final_doc; + + memcpy(final_doc.header.magic_bytes, Doc_Magic_Bytes, 4); + final_doc.header.version = 1; + + final_doc.header.program_name.offset = 0; + final_doc.header.program_name.length = 0; + + final_doc.header.build_time = bh_time_curr() / 1000; + + final_doc.packages.offset = 0; + final_doc.packages.length = 0; + + final_doc.entities.offset = 0; + final_doc.entities.length = 0; + + final_doc.files.offset = 0; + final_doc.files.length = 0; + + bh_file_write(&doc_file, &final_doc, sizeof(struct Doc)); + bh_file_close(&doc_file); +} + + diff --git a/compiler/src/onyx.c b/compiler/src/onyx.c index 13f2ab5c..7362c7e3 100644 --- a/compiler/src/onyx.c +++ b/compiler/src/onyx.c @@ -59,8 +59,8 @@ static const char *build_docstring = DOCSTRING_HEADER "\t Automatically enabled for \"onyx\" runtime.\n" "\t--tag Generates a C-Tag file.\n" "\t--syminfo Generates a symbol resolution information file. Used by onyx-lsp.\n" - "\t--generate-foreign-info Generates information for #foreign blocks.\n" - // "\t--doc \n" + "\t--doc \n" + "\t--generate-foreign-info\n" "\n" "Developer options:\n" "\t--no-colors Disables colors in the error message.\n" @@ -205,9 +205,9 @@ static CompileOptions compile_opts_parse(bh_allocator alloc, int argc, char *arg options.runtime = Runtime_Onyx; } } - // else if (!strcmp(argv[i], "--doc")) { - // options.documentation_file = argv[++i]; - // } + else if (!strcmp(argv[i], "--doc")) { + options.documentation_file = argv[++i]; + } else if (!strcmp(argv[i], "--tag")) { options.generate_tag_file = 1; } @@ -781,6 +781,10 @@ static i32 onyx_compile() { onyx_docs_emit_symbol_info(context.options->symbol_info_file); } + if (context.options->documentation_file != NULL) { + onyx_docs_emit_odoc(context.options->documentation_file); + } + return ONYX_COMPILER_PROGRESS_SUCCESS; } @@ -842,12 +846,6 @@ static CompilerProgress onyx_flush_module() { bh_file_close(&output_file); - // if (context.options->documentation_file != NULL) { - // OnyxDocumentation docs = onyx_docs_generate(); - // docs.format = Doc_Format_Human; - // onyx_docs_emit(&docs, context.options->documentation_file); - // } - return ONYX_COMPILER_PROGRESS_SUCCESS; } diff --git a/compiler/src/parser.c b/compiler/src/parser.c index aa18ca87..679fc79b 100644 --- a/compiler/src/parser.c +++ b/compiler/src/parser.c @@ -3660,7 +3660,7 @@ static Package* parse_file_package(OnyxParser* parser) { pnode->flags |= Ast_Flag_Comptime; if (prevpackage != NULL) { - symbol_subpackage_introduce(prevpackage->scope, (*symbol)->text, pnode); + symbol_subpackage_introduce(prevpackage, (*symbol)->text, pnode); package_reinsert_use_packages(prevpackage); } diff --git a/compiler/src/utils.c b/compiler/src/utils.c index 2e235de9..c7d7033e 100644 --- a/compiler/src/utils.c +++ b/compiler/src/utils.c @@ -44,6 +44,7 @@ Package* package_lookup_or_create(char* package_name, Scope* parent_scope, bh_al package->name = pac_name; package->use_package_entities = NULL; package->id = next_package_id++; + bh_arr_new(global_heap_allocator, package->sub_packages, 4); if (!strcmp(pac_name, "builtin")) { package->private_scope = scope_create(alloc, context.global_scope, pos); @@ -153,15 +154,22 @@ void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node) { shput(scope->symbols, sym, node); } -void symbol_subpackage_introduce(Scope* scope, char* sym, AstPackage* package) { +void symbol_subpackage_introduce(Package* parent, char* sym, AstPackage* subpackage) { + Scope *scope = parent->scope; + i32 index = shgeti(scope->symbols, sym); if (index != -1) { AstNode* maybe_package = scope->symbols[index].value; // CLEANUP: Make this assertion an actual error message. assert(maybe_package->kind == Ast_Kind_Package); + } else { - shput(scope->symbols, sym, (AstNode *) package); + shput(scope->symbols, sym, (AstNode *) subpackage); + + // Parent: parent->id + // Child: subpackage->package->id + bh_arr_push(parent->sub_packages, subpackage->package->id); } } diff --git a/core/doc/doc.onyx b/core/doc/doc.onyx new file mode 100644 index 00000000..9b341e29 --- /dev/null +++ b/core/doc/doc.onyx @@ -0,0 +1,58 @@ +package core.doc + +Doc :: struct { + header: Doc_Header; + + packages: [] Doc_Package; + entities: [] Doc_Entity; + files: [] Doc_File; +} + + +Doc_Magic_Bytes :: "ODOC" + +Doc_Header :: struct { + magic_bytes: [4] u8; + version: u32; + + program_name: str; + build_time: u32; +} + +Doc_File :: struct { + Id :: #distinct i32 + + package_id: Doc_Package.Id; + name: str; +} + +Doc_Location :: struct { + file_id: Doc_File.Id; + line: i32; + col: i32; +} + +Doc_Package :: struct { + Id :: #distinct i32 + + name: str; // i.e. 'io' + qualified_name: str; // i.e. 'core.io' + + subpackages: [] Id; +} + + +Doc_Entity_Kind :: enum { + Procedure :: 1; +} + +Doc_Entity :: struct { + kind: Doc_Entity_Kind; + + package_id: Doc_Package.Id; + location: Doc_Location; + + name: str; + notes: str; +} + diff --git a/docs/ideas/doc_format.md b/docs/ideas/doc_format.md new file mode 100644 index 00000000..35d4b61e --- /dev/null +++ b/docs/ideas/doc_format.md @@ -0,0 +1,12 @@ +Documentation file format +========================= + +Capturing the follow: + - Package Hierarchy + - Type info + - Public symbols + - Private symbols + - Procedures + + +