starting work on documentation file generation

diff --git a/compiler/include/astnodes.h b/compiler/include/astnodes.h
index bb0422f2e9098b7d269551f49b63683dbacfce53..413f1a9246eb1ee0ea72e24a3a9a0b73e0c392e6 100644 (file)
--- a/compiler/include/astnodes.h
+++ b/compiler/include/astnodes.h
@@ -1570,6 +1570,9 @@ struct Package {
 
     u32 id;
 
+    // NOTE: id's of the sub-packages
+    bh_arr(u32) sub_packages;
+
     // NOTE: This tracks all of the 'use package' statements of this package throughout
     // the code base. This is used when a static if clears and new symbols are introduced.
     // 'use package' statements have to be reevaluated to pull in the new symbols.

diff --git a/compiler/include/doc.h b/compiler/include/doc.h
index 3dd647a0bfbbe2b0edfa87537f1c1c722f78f15e..93a8348d9ceffda815bcaefa358f4a89bf666d39 100644 (file)
--- a/compiler/include/doc.h
+++ b/compiler/include/doc.h
@@ -4,36 +4,9 @@
 #include "bh.h"
 #include "astnodes.h"
 
-typedef enum DocFormat {
-    Doc_Format_Human,
-    Doc_Format_Tags,
-    Doc_Format_Html,
-} DocFormat;
+// Onyx Documentation generation
 
-typedef struct DocEntry {
-       OnyxFilePos pos;
-    char* sym; // Unused by doc generator
-       char* def;
-       char* additional;
-} DocEntry;
-
-typedef struct DocPackage {
-       const char* name;
-
-       bh_arr(DocEntry) public_entries;
-       bh_arr(DocEntry) private_entries;
-} DocPackage;
-
-typedef struct OnyxDocumentation {
-       bh_arena doc_arena;
-
-    DocFormat format;
-
-       bh_arr(DocPackage) package_docs;
-} OnyxDocumentation;
-
-OnyxDocumentation onyx_docs_generate();
-void onyx_docs_emit(OnyxDocumentation* doc, const char* filename);
+void onyx_docs_emit_odoc(const char *dest);
 
 
 // Tag generation

diff --git a/compiler/include/utils.h b/compiler/include/utils.h
index 2b0083c2c7afccb3c013c253818d9f11968fd6d1..c35cbf06f05639393d8a08742638617d538c28c9 100644 (file)
--- a/compiler/include/utils.h
+++ b/compiler/include/utils.h
@@ -20,7 +20,7 @@ void scope_include(Scope* target, Scope* source, OnyxFilePos pos);
 b32 symbol_introduce(Scope* scope, OnyxToken* tkn, AstNode* symbol);
 b32 symbol_raw_introduce(Scope* scope, char* tkn, OnyxFilePos pos, AstNode* symbol);
 void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node);
-void symbol_subpackage_introduce(Scope* scope, char* sym, AstPackage *node);
+void symbol_subpackage_introduce(Package *parent, char* sym, AstPackage *node);
 AstNode* symbol_raw_resolve(Scope* start_scope, char* sym);
 AstNode* symbol_resolve(Scope* start_scope, OnyxToken* tkn);
 AstNode* try_symbol_raw_resolve_from_node(AstNode* node, char* symbol);

diff --git a/compiler/src/doc.c b/compiler/src/doc.c
index 1808946fdd8bccc90461f8f7db0cef0dbfaf0d83..b02d28f15de467f783fbbd80c7573eeb0266f8e4 100644 (file)
--- a/compiler/src/doc.c
+++ b/compiler/src/doc.c
@@ -136,245 +136,102 @@ void onyx_docs_emit_symbol_info(const char *dest) {
     bh_imap_free(&syminfo->node_to_id);
 }
 
-// static i32 cmp_doc_entry(const void * a, const void * b) {
-//     DocEntry* d1 = (DocEntry *) a;
-//     DocEntry* d2 = (DocEntry *) b;
-// 
-//     return strncmp(d1->def, d2->def, 1024);
-// }
-// 
-// static i32 cmp_doc_package(const void * a, const void * b) {
-//     DocPackage* d1 = (DocPackage *) a;
-//     DocPackage* d2 = (DocPackage *) b;
-// 
-//     return strncmp(d1->name, d2->name, 1024);
-// }
-// 
-// static char* node_to_doc_def(const char* sym, AstNode *node, bh_allocator a) {
-//     static char buf[1024];
-//     memset(buf, 0, 1023);
-// 
-//     strncat(buf, sym, 1023);
-//     strncat(buf, " :: ", 1023);
-// 
-//     switch (node->kind) {
-//         case Ast_Kind_Function: {
-//             strncat(buf, "(", 1023);
-// 
-//             AstFunction *func = (AstFunction *) node;
-//             bh_arr_each(AstParam, param, func->params) {
-//                 if (param != func->params)
-//                     strncat(buf, ", ", 1023);
-// 
-//                 token_toggle_end(param->local->token);
-//                 strncat(buf, param->local->token->text, 1023);
-//                 token_toggle_end(param->local->token);
-// 
-//                 strncat(buf, ": ", 1023);
-// 
-//                 strncat(buf, type_get_name(param->local->type), 1023);
-// 
-//                 if (param->default_value != NULL) {
-//                     strncat(buf, " = <default>", 1023);
-//                 }
-//             }
-// 
-//             strncat(buf, ") -> ", 1023);
-//             strncat(buf, type_get_name(func->type->Function.return_type), 1023);
-// 
-//             break;
-//         }
-// 
-//         case Ast_Kind_Struct_Type: {
-//             strncat(buf, "struct { ", 1023);
-// 
-//             AstStructType* st = (AstStructType *) node;
-//             bh_arr_each(AstStructMember *, smem, st->members) {
-// 
-//                 token_toggle_end((*smem)->token);
-//                 strncat(buf, (*smem)->token->text, 1023);
-//                 token_toggle_end((*smem)->token);
-// 
-//                 strncat(buf, ": ", 1023);
-// 
-//                 strncat(buf, type_get_name((*smem)->type), 1023);
-// 
-//                 strncat(buf, "; ", 1023);
-//             }
-// 
-//             strncat(buf, "}", 1023);
-//             break;
-//         }
-// 
-//         case Ast_Kind_Basic_Type:
-//         case Ast_Kind_Array_Type:
-//         case Ast_Kind_Type_Alias:
-//         case Ast_Kind_Slice_Type:
-//         case Ast_Kind_DynArr_Type: {
-//             strncat(buf, type_get_name(type_build_from_ast(global_heap_allocator, (AstType *) node)), 1023);
-//             break;
-//         }
-// 
-//         default: {
-//             strncat(buf, "<unimplemented printing>", 1023);
-//         }
-//     }
-// 
-//     return bh_strdup(a, buf);
-// }
-// 
-// static DocPackage doc_package_create(Package* p, bh_allocator a) {
-//     DocPackage dp;
-//     dp.name = p->name;
-//     dp.public_entries = NULL;
-//     dp.private_entries = NULL;
-// 
-//     bh_arr_new(global_heap_allocator, dp.public_entries, 16);
-//     bh_arr_new(global_heap_allocator, dp.private_entries, 16);
-// 
-//     fori (i, 0, shlen(p->scope->symbols)) {
-//         char *key = p->scope->symbols[i].key;
-//         AstNode *value = p->scope->symbols[i].value;
-//         if (!key || !value) continue;
-// 
-//         DocEntry de;
-//         if (value->token) de.pos = value->token->pos;
-//         de.def = node_to_doc_def(key, value, a);
-//         de.sym = (char *) key;
-//         de.additional = NULL;
-// 
-//         bh_arr_push(dp.public_entries, de);
-//     }
-// 
-//     fori (i, 0, shlen(p->private_scope->symbols)) {
-//         char *key = p->scope->symbols[i].key;
-//         AstNode *value = p->scope->symbols[i].value;
-//         if (!key || !value) continue;
-// 
-//         DocEntry de;
-//         if (value->token) de.pos = value->token->pos;
-//         de.def = node_to_doc_def(key, value, a);
-//         de.sym = (char *) key;
-//         de.additional = NULL;
-// 
-//         bh_arr_push(dp.private_entries, de);
-//     }
-// 
-//     qsort(dp.public_entries,  bh_arr_length(dp.public_entries),  sizeof(DocEntry), cmp_doc_entry);
-//     qsort(dp.private_entries, bh_arr_length(dp.private_entries), sizeof(DocEntry), cmp_doc_entry);
-// 
-//     return dp;
-// }
-// 
-// OnyxDocumentation onyx_docs_generate() {
-//     OnyxDocumentation doc;
-// 
-//     bh_arena_init(&doc.doc_arena, global_heap_allocator, 16 * 1024);
-//     bh_allocator a = bh_arena_allocator(&doc.doc_arena);
-// 
-//     doc.package_docs = NULL;
-//     bh_arr_new(global_heap_allocator, doc.package_docs, 16);
-// 
-//     fori (i, 0, shlen(context.packages)) {
-//         DocPackage dp = doc_package_create(context.packages[i].value, a);
-//         bh_arr_push(doc.package_docs, dp);
-//     }
-// 
-//     qsort(doc.package_docs, bh_arr_length(doc.package_docs), sizeof(DocPackage), cmp_doc_package);
-// 
-//     return doc;
-// }
-// 
-// static void onyx_docs_emit_human(OnyxDocumentation* doc, bh_file* file) {
-//     // NOTE: Disabling fancy line printing until I can make it better
-//     #if 0
-//     bh_arr_each(DocPackage, dp, doc->package_docs) {
-//         bh_printf("Package '%s'\n", dp->name);
-// 
-//         if (bh_arr_length(dp->public_entries) > 0) {
-//             bh_printf("\xe2\x94\x9c\xe2\x94\x80 Public symbols\n");
-//             bh_arr_each(DocEntry, de, dp->public_entries) {
-//                 bh_printf("\xe2\x94\x82  \xe2\x94\x9c\xe2\x94\x80 %s\n", de->def);
-// 
-//                 if (de->pos.filename != NULL)
-//                     bh_printf("\xe2\x94\x82  \xe2\x94\x82    at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column);
-//                 else
-//                     bh_printf("\xe2\x94\x82  \xe2\x94\x82    compiler built-in\n");
-// 
-//                 bh_printf("\xe2\x94\x82  \xe2\x94\x82\n");
-//             }
-//         }
-// 
-//         if (bh_arr_length(dp->private_entries) > 0) {
-//             bh_printf("\xe2\x94\x9c\xe2\x94\x80 Private symbols\n");
-//             bh_arr_each(DocEntry, de, dp->private_entries) {
-//                 bh_printf("\xe2\x94\x82  \xe2\x94\x9c\xe2\x94\x80 %s\n", de->def);
-//                 if (de->pos.filename != NULL)
-//                     bh_printf("\xe2\x94\x82  \xe2\x94\x82    at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column);
-//                 else
-//                     bh_printf("\xe2\x94\x82  \xe2\x94\x82    compiler built-in\n");
-// 
-//                 bh_printf("\xe2\x94\x82  \xe2\x94\x82\n");
-//             }
-// 
-//             bh_printf("\n");
-//         }
-//     }
-//     #else
-//     bh_arr_each(DocPackage, dp, doc->package_docs) {
-//         bh_fprintf(file, "Package '%s'\n", dp->name);
-// 
-//         if (bh_arr_length(dp->public_entries) > 0) {
-//             bh_fprintf(file, "   Public symbols\n");
-//             bh_arr_each(DocEntry, de, dp->public_entries) {
-//                 bh_fprintf(file, "     %s\n", de->def);
-// 
-//                 if (de->pos.filename != NULL)
-//                     bh_fprintf(file, "        at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column);
-//                 else
-//                     bh_fprintf(file, "        compiler built-in\n");
-// 
-//                 bh_fprintf(file, "    \n");
-//             }
-//         }
-// 
-//         if (bh_arr_length(dp->private_entries) > 0) {
-//             bh_fprintf(file, "   Private symbols\n");
-//             bh_arr_each(DocEntry, de, dp->private_entries) {
-//                 bh_fprintf(file, "      %s\n", de->def);
-//                 if (de->pos.filename != NULL)
-//                     bh_fprintf(file, "        at %s:%d,%d\n", de->pos.filename, de->pos.line, de->pos.column);
-//                 else
-//                     bh_fprintf(file, "        compiler built-in\n");
-// 
-//                 bh_fprintf(file, "    \n");
-//             }
-// 
-//             bh_fprintf(file, "\n");
-//         }
-//     }
-//     #endif
-// }
-// 
-// 
-// 
-// static void onyx_docs_emit_html(OnyxDocumentation* doc, bh_file* file) {
-//     bh_fprintf(file, "HTML documentation output not supported yet.\n");
-//     return;
-// }
-// 
-// void onyx_docs_emit(OnyxDocumentation* doc, const char* filename) {
-//     bh_file file;
-//     if (bh_file_create(&file, filename) != BH_FILE_ERROR_NONE) {
-//         bh_printf("ERROR: Failed to open file '%s' for writing documentation.\n", filename);
-//         return;
-//     }
-// 
-//     switch (doc->format) {
-//         case Doc_Format_Human: onyx_docs_emit_human(doc, &file); break;
-//         case Doc_Format_Html: onyx_docs_emit_html(doc, &file); break;
-//     }
-// 
-//     bh_file_close(&file);
-// }
+
+
+//
+// Onyx Documentation Format
+//
+
+#define Doc_Magic_Bytes "ODOC"
+
+
+struct Doc_String {
+    u32 offset;
+    u32 length;
+};
+
+struct Doc_Array {
+    u32 offset;
+    u32 length;
+};
+
+struct Doc_Header {
+    u8  magic_bytes[4];
+    u32 version;
+
+    struct Doc_String program_name;
+    u32 build_time;
+};
+
+struct Doc_File {
+    u32 package_id;
+    struct Doc_String name;
+};
+
+struct Doc_Location {
+    u32 file_id;
+    i32 line;
+    i32 col;
+};
+
+struct Doc_Package {
+    struct Doc_String name;
+    struct Doc_String qualified_name;
+    
+    struct Doc_Array subpackages;
+};
+
+enum Doc_Entity_Kind {
+    Entity_Kind_Procedure = 1,
+};
+
+struct Doc_Entity {
+    enum Doc_Entity_Kind kind;
+
+    u32 package_id;
+    struct Doc_Location location;
+
+    struct Doc_String name;
+    struct Doc_String notes;
+};
+
+struct Doc {
+    struct Doc_Header header;
+
+    struct Doc_Array packages;
+    struct Doc_Array entities;
+    struct Doc_Array files;
+};
+
+void onyx_docs_emit_odoc(const char *dest) {
+    bh_file doc_file;
+    if (bh_file_create(&doc_file, dest) != BH_FILE_ERROR_NONE) {
+        bh_printf("Cannot create '%s'.\n", dest);
+        return;
+    }
+
+    struct Doc final_doc;
+
+    memcpy(final_doc.header.magic_bytes, Doc_Magic_Bytes, 4);
+    final_doc.header.version = 1;
+
+    final_doc.header.program_name.offset = 0;
+    final_doc.header.program_name.length = 0;
+
+    final_doc.header.build_time = bh_time_curr() / 1000;
+
+    final_doc.packages.offset = 0;
+    final_doc.packages.length = 0;
+
+    final_doc.entities.offset = 0;
+    final_doc.entities.length = 0;
+
+    final_doc.files.offset = 0;
+    final_doc.files.length = 0;
+
+    bh_file_write(&doc_file, &final_doc, sizeof(struct Doc));
+    bh_file_close(&doc_file);
+}
+
+
 

diff --git a/compiler/src/onyx.c b/compiler/src/onyx.c
index 13f2ab5c361fc388b187ddd1fc4f95501e85af7d..7362c7e368e90f5c7ee1a7337041b0728942ad52 100644 (file)
--- a/compiler/src/onyx.c
+++ b/compiler/src/onyx.c
@@ -59,8 +59,8 @@ static const char *build_docstring = DOCSTRING_HEADER
     "\t                        Automatically enabled for \"onyx\" runtime.\n"
     "\t--tag                   Generates a C-Tag file.\n"
     "\t--syminfo <target_file> Generates a symbol resolution information file. Used by onyx-lsp.\n"
-    "\t--generate-foreign-info Generates information for #foreign blocks.\n"
-    // "\t--doc <doc_file>\n"
+    "\t--doc <doc_file>\n"
+    "\t--generate-foreign-info\n"
     "\n"
     "Developer options:\n"
     "\t--no-colors               Disables colors in the error message.\n"
@@ -205,9 +205,9 @@ static CompileOptions compile_opts_parse(bh_allocator alloc, int argc, char *arg
                     options.runtime = Runtime_Onyx;
                 }
             }
-            // else if (!strcmp(argv[i], "--doc")) {
-            //     options.documentation_file = argv[++i];
-            // }
+            else if (!strcmp(argv[i], "--doc")) {
+                options.documentation_file = argv[++i];
+            }
             else if (!strcmp(argv[i], "--tag")) {
                 options.generate_tag_file = 1;
             }
@@ -781,6 +781,10 @@ static i32 onyx_compile() {
         onyx_docs_emit_symbol_info(context.options->symbol_info_file);
     }
 
+    if (context.options->documentation_file != NULL) {
+        onyx_docs_emit_odoc(context.options->documentation_file);
+    }
+
     return ONYX_COMPILER_PROGRESS_SUCCESS;
 }
 
@@ -842,12 +846,6 @@ static CompilerProgress onyx_flush_module() {
 
     bh_file_close(&output_file);
 
-    // if (context.options->documentation_file != NULL) {
-    //     OnyxDocumentation docs = onyx_docs_generate();
-    //     docs.format = Doc_Format_Human;
-    //     onyx_docs_emit(&docs, context.options->documentation_file);
-    // }
-
     return ONYX_COMPILER_PROGRESS_SUCCESS;
 }
 

diff --git a/compiler/src/parser.c b/compiler/src/parser.c
index aa18ca87181829647b3809ad3797083347f382af..679fc79b5dc351fcc26e662596c2c547b4a1c247 100644 (file)
--- a/compiler/src/parser.c
+++ b/compiler/src/parser.c
@@ -3660,7 +3660,7 @@ static Package* parse_file_package(OnyxParser* parser) {
         pnode->flags |= Ast_Flag_Comptime;
 
         if (prevpackage != NULL) {
-            symbol_subpackage_introduce(prevpackage->scope, (*symbol)->text, pnode);
+            symbol_subpackage_introduce(prevpackage, (*symbol)->text, pnode);
             package_reinsert_use_packages(prevpackage);
         }
 

diff --git a/compiler/src/utils.c b/compiler/src/utils.c
index 2e235de9d501fb9a8d7fbca39a5df52c76d1768e..c7d7033e6117cc5818c68ccb4f3b900ee32031b4 100644 (file)
--- a/compiler/src/utils.c
+++ b/compiler/src/utils.c
@@ -44,6 +44,7 @@ Package* package_lookup_or_create(char* package_name, Scope* parent_scope, bh_al
         package->name = pac_name;
         package->use_package_entities = NULL;
         package->id = next_package_id++;
+        bh_arr_new(global_heap_allocator, package->sub_packages, 4);
 
         if (!strcmp(pac_name, "builtin")) {
             package->private_scope = scope_create(alloc, context.global_scope, pos);
@@ -153,15 +154,22 @@ void symbol_builtin_introduce(Scope* scope, char* sym, AstNode *node) {
     shput(scope->symbols, sym, node);
 }
 
-void symbol_subpackage_introduce(Scope* scope, char* sym, AstPackage* package) {
+void symbol_subpackage_introduce(Package* parent, char* sym, AstPackage* subpackage) {
+    Scope *scope = parent->scope;
+
     i32 index = shgeti(scope->symbols, sym);
     if (index != -1) {
         AstNode* maybe_package = scope->symbols[index].value;
         
         // CLEANUP: Make this assertion an actual error message.
         assert(maybe_package->kind == Ast_Kind_Package);
+
     } else {
-        shput(scope->symbols, sym, (AstNode *) package);
+        shput(scope->symbols, sym, (AstNode *) subpackage);
+
+        // Parent: parent->id
+        // Child:  subpackage->package->id
+        bh_arr_push(parent->sub_packages, subpackage->package->id);
     }
 }
 

diff --git a/core/doc/doc.onyx b/core/doc/doc.onyx
new file mode 100644 (file)
index 0000000..9b341e2
--- /dev/null
+++ b/core/doc/doc.onyx
@@ -0,0 +1,58 @@
+package core.doc
+
+Doc :: struct {
+    header: Doc_Header;
+
+    packages: [] Doc_Package;
+    entities: [] Doc_Entity;
+    files:    [] Doc_File;
+}
+
+
+Doc_Magic_Bytes :: "ODOC"
+
+Doc_Header :: struct {
+    magic_bytes: [4] u8;
+    version: u32;
+    
+    program_name: str;
+    build_time: u32;
+}
+
+Doc_File :: struct {
+    Id :: #distinct i32
+
+    package_id: Doc_Package.Id;
+    name: str;
+}
+
+Doc_Location :: struct {
+    file_id: Doc_File.Id;
+    line: i32;
+    col: i32;
+}
+
+Doc_Package :: struct {
+    Id :: #distinct i32
+
+    name: str;             // i.e. 'io'
+    qualified_name: str;   // i.e. 'core.io'
+
+    subpackages: [] Id;
+}
+
+
+Doc_Entity_Kind :: enum {
+    Procedure :: 1;
+}
+
+Doc_Entity :: struct {
+    kind: Doc_Entity_Kind;
+
+    package_id: Doc_Package.Id;
+    location: Doc_Location;
+
+    name: str;
+    notes: str;
+}
+

diff --git a/docs/ideas/doc_format.md b/docs/ideas/doc_format.md
new file mode 100644 (file)
index 0000000..35d4b61
--- /dev/null
+++ b/docs/ideas/doc_format.md
@@ -0,0 +1,12 @@
+Documentation file format
+=========================
+
+Capturing the follow:
+    - Package Hierarchy
+    - Type info
+    - Public symbols
+    - Private symbols
+    - Procedures
+
+
+