Removed unnecessary tokens in the lexer

diff --git a/include/onyxlex.h b/include/onyxlex.h
index 6502862d4c039a2024b92ca9646009db41dfc7f9..6b35fe6187fd090eb4875f42ab6e871f54df7aa0 100644 (file)
--- a/include/onyxlex.h
+++ b/include/onyxlex.h
@@ -4,68 +4,46 @@
 #include "bh.h"
 
 typedef enum TokenType {
-    TOKEN_TYPE_UNKNOWN,
-    TOKEN_TYPE_END_STREAM,
-
-    TOKEN_TYPE_COMMENT,
-
-    TOKEN_TYPE_KEYWORD_STRUCT,
-    TOKEN_TYPE_KEYWORD_USE,
-    TOKEN_TYPE_KEYWORD_EXPORT,
-    TOKEN_TYPE_KEYWORD_IF,
-    TOKEN_TYPE_KEYWORD_ELSE,
-    TOKEN_TYPE_KEYWORD_ELSEIF,
-    TOKEN_TYPE_KEYWORD_RETURN,
-    TOKEN_TYPE_KEYWORD_FOREIGN,
-    TOKEN_TYPE_KEYWORD_PROC,
-    TOKEN_TYPE_KEYWORD_CAST,
-    TOKEN_TYPE_KEYWORD_WHILE,
-    TOKEN_TYPE_KEYWORD_BREAK,
-    TOKEN_TYPE_KEYWORD_CONTINUE,
-
-    TOKEN_TYPE_RIGHT_ARROW,
-    TOKEN_TYPE_LEFT_ARROW,
-    TOKEN_TYPE_OPEN_PAREN,
-    TOKEN_TYPE_CLOSE_PAREN,
-    TOKEN_TYPE_OPEN_BRACE,
-    TOKEN_TYPE_CLOSE_BRACE,
-    TOKEN_TYPE_OPEN_BRACKET,
-    TOKEN_TYPE_CLOSE_BRACKET,
-
-    TOKEN_TYPE_SYM_PLUS,
-    TOKEN_TYPE_SYM_MINUS,
-    TOKEN_TYPE_SYM_STAR,
-    TOKEN_TYPE_SYM_PERCENT,
-    TOKEN_TYPE_SYM_DOT,
-    TOKEN_TYPE_SYM_FSLASH,
-    TOKEN_TYPE_SYM_BSLASH,
-    TOKEN_TYPE_SYM_COLON,
-    TOKEN_TYPE_SYM_SEMICOLON,
-    TOKEN_TYPE_SYM_COMMA,
-    TOKEN_TYPE_SYM_GREATER,
-    TOKEN_TYPE_SYM_GREATER_EQUAL,
-    TOKEN_TYPE_SYM_LESS,
-    TOKEN_TYPE_SYM_LESS_EQUAL,
-    TOKEN_TYPE_SYM_EQUAL_EQUAL,
-    TOKEN_TYPE_SYM_NOT_EQUAL,
-    TOKEN_TYPE_SYM_EQUALS,
-    TOKEN_TYPE_SYM_TILDE,
-    TOKEN_TYPE_SYM_BANG,
-    TOKEN_TYPE_SYM_CARET,
-    TOKEN_TYPE_SYM_AMPERSAND,
-    TOKEN_TYPE_SYM_PLUS_EQUAL,
-    TOKEN_TYPE_SYM_MINUS_EQUAL,
-    TOKEN_TYPE_SYM_STAR_EQUAL,
-    TOKEN_TYPE_SYM_FSLASH_EQUAL,
-    TOKEN_TYPE_SYM_PERCENT_EQUAL,
-
-    TOKEN_TYPE_SYMBOL,
-    TOKEN_TYPE_LITERAL_STRING,
-    TOKEN_TYPE_LITERAL_NUMERIC,
-    TOKEN_TYPE_LITERAL_BOOL_TRUE,
-    TOKEN_TYPE_LITERAL_BOOL_FALSE,
-
-    TOKEN_TYPE_COUNT
+    TOKEN_TYPE_ASCII_END            = 256,
+    TOKEN_TYPE_UNKNOWN              = 256,
+    TOKEN_TYPE_END_STREAM           = 257,
+
+    TOKEN_TYPE_COMMENT              = 258,
+
+    TOKEN_TYPE_KEYWORD_STRUCT       = 259,
+    TOKEN_TYPE_KEYWORD_USE          = 260,
+    TOKEN_TYPE_KEYWORD_EXPORT       = 261,
+    TOKEN_TYPE_KEYWORD_IF           = 262,
+    TOKEN_TYPE_KEYWORD_ELSE         = 263,
+    TOKEN_TYPE_KEYWORD_ELSEIF       = 264,
+    TOKEN_TYPE_KEYWORD_RETURN       = 265,
+    TOKEN_TYPE_KEYWORD_FOREIGN      = 266,
+    TOKEN_TYPE_KEYWORD_PROC         = 267,
+    TOKEN_TYPE_KEYWORD_CAST         = 268,
+    TOKEN_TYPE_KEYWORD_WHILE        = 269,
+    TOKEN_TYPE_KEYWORD_BREAK        = 270,
+    TOKEN_TYPE_KEYWORD_CONTINUE     = 271,
+
+    TOKEN_TYPE_RIGHT_ARROW          = 272,
+    TOKEN_TYPE_LEFT_ARROW           = 273,
+
+    TOKEN_TYPE_GREATER_EQUAL        = 274,
+    TOKEN_TYPE_LESS_EQUAL           = 275,
+    TOKEN_TYPE_EQUAL_EQUAL          = 276,
+    TOKEN_TYPE_NOT_EQUAL            = 277,
+    TOKEN_TYPE_PLUS_EQUAL           = 278,
+    TOKEN_TYPE_MINUS_EQUAL          = 279,
+    TOKEN_TYPE_STAR_EQUAL           = 280,
+    TOKEN_TYPE_FSLASH_EQUAL         = 281,
+    TOKEN_TYPE_PERCENT_EQUAL        = 282,
+
+    TOKEN_TYPE_SYMBOL               = 283,
+    TOKEN_TYPE_LITERAL_STRING       = 284,
+    TOKEN_TYPE_LITERAL_NUMERIC      = 285,
+    TOKEN_TYPE_LITERAL_BOOL_TRUE    = 286,
+    TOKEN_TYPE_LITERAL_BOOL_FALSE   = 287,
+
+    TOKEN_TYPE_COUNT                = 288
 } TokenType;
 
 typedef struct OnyxFilePos {

diff --git a/onyx b/onyx
index 1ca270bf0e65bfd757b6bcdca4221f22e6bbfb74..283c8b4757ba6fae35aa15d501aed108cc124665 100755 (executable)
Binary files a/onyx and b/onyx differ

diff --git a/src/onyxlex.c b/src/onyxlex.c
index a280e6a420970264748df45a7f0ce05731f80776..600d796d3db0c6f13c4f06d8fb657d37d3ad00be 100644 (file)
--- a/src/onyxlex.c
+++ b/src/onyxlex.c
@@ -1,5 +1,6 @@
 #include "bh.h"
 #include "onyxlex.h"
+#include "onyxutils.h"
 
 static const char* token_type_names[] = {
     "TOKEN_TYPE_UNKNOWN",
@@ -23,34 +24,11 @@ static const char* token_type_names[] = {
 
     "->", //"TOKEN_TYPE_RIGHT_ARROW",
     "<-", //"TOKEN_TYPE_LEFT_ARROW",
-    "(",  //"TOKEN_TYPE_OPEN_PAREN",
-    ")",  //"TOKEN_TYPE_CLOSE_PAREN",
-    "{",  //"TOKEN_TYPE_OPEN_BRACE",
-    "}",  //"TOKEN_TYPE_CLOSE_BRACE",
-    "[",  //"TOKEN_TYPE_OPEN_BRACKET",
-    "]",  //"TOKEN_TYPE_CLOSE_BRACKET",
-
-    "+",  // "TOKEN_TYPE_SYM_PLUS",
-    "-",  // "TOKEN_TYPE_SYM_MINUS",
-    "*",  // "TOKEN_TYPE_SYM_STAR",
-    "%",  // "TOKEN_TYPE_SYM_PERCENT",
-    ".",  // "TOKEN_TYPE_SYM_DOT",
-    "/",  // "TOKEN_TYPE_SYM_FSLASH",
-    "\\", // "TOKEN_TYPE_SYM_BSLASH",
-    ":",  // "TOKEN_TYPE_SYM_COLON",
-    ";",  // "TOKEN_TYPE_SYM_SEMICOLON",
-    ",",  // "TOKEN_TYPE_SYM_COMMA",
-    ">",  // "TOKEN_TYPE_SYM_GREATER",
+
     ">=", // "TOKEN_TYPE_SYM_GREATER_EQUAL",
-    "<",  // "TOKEN_TYPE_SYM_LESS",
     "<=", // "TOKEN_TYPE_SYM_LESS_EQUAL",
     "==", // "TOKEN_TYPE_SYM_EQUALS_EQUALS",
     "!=", // "TOKEN_TYPE_SYM_NOT_EQUAL",
-    "=",  // "TOKEN_TYPE_SYM_EQUALS",
-    "~",  // "TOKEN_TYPE_SYM_TILDE",
-    "!",  // "TOKEN_TYPE_SYM_BANG",
-    "^",  // "TOKEN_TYPE_SYM_CARET",
-    "&",  // "TOKEN_TYPE_SYM_AMPERSAND",
     "+=", // "TOKEN_TYPE_SYM_PLUS_EQUAL",
     "-=", // "TOKEN_TYPE_SYM_MINUS_EQUAL",
     "*=", // "TOKEN_TYPE_SYM_STAR_EQUAL",
@@ -101,7 +79,11 @@ static b32 token_lit(OnyxTokenizer* tokenizer, OnyxToken* tk, char* lit, b32 is_
 }
 
 const char* onyx_get_token_type_name(TokenType tkn_type) {
-    return token_type_names[tkn_type];
+    if (tkn_type < TOKEN_TYPE_ASCII_END) {
+        return bh_aprintf(global_scratch_allocator, "%c", (char) tkn_type);
+    } else {
+        return token_type_names[tkn_type - TOKEN_TYPE_ASCII_END];
+    }
 }
 
 void onyx_token_null_toggle(OnyxToken* tkn) {
@@ -161,38 +143,15 @@ OnyxToken* onyx_get_token(OnyxTokenizer* tokenizer) {
     LITERAL_TOKEN("false",      1, TOKEN_TYPE_LITERAL_BOOL_FALSE);
     LITERAL_TOKEN("->",         0, TOKEN_TYPE_RIGHT_ARROW);
     LITERAL_TOKEN("<-",         0, TOKEN_TYPE_RIGHT_ARROW);
-    LITERAL_TOKEN("<=",         0, TOKEN_TYPE_SYM_LESS_EQUAL);
-    LITERAL_TOKEN(">=",         0, TOKEN_TYPE_SYM_GREATER_EQUAL);
-    LITERAL_TOKEN("==",         0, TOKEN_TYPE_SYM_EQUAL_EQUAL);
-    LITERAL_TOKEN("!=",         0, TOKEN_TYPE_SYM_NOT_EQUAL);
-    LITERAL_TOKEN("+=",         0, TOKEN_TYPE_SYM_PLUS_EQUAL);
-    LITERAL_TOKEN("-=",         0, TOKEN_TYPE_SYM_MINUS_EQUAL);
-    LITERAL_TOKEN("*=",         0, TOKEN_TYPE_SYM_STAR_EQUAL);
-    LITERAL_TOKEN("/=",         0, TOKEN_TYPE_SYM_FSLASH_EQUAL);
-    LITERAL_TOKEN("%=",         0, TOKEN_TYPE_SYM_PERCENT_EQUAL);
-    LITERAL_TOKEN("(",          0, TOKEN_TYPE_OPEN_PAREN);
-    LITERAL_TOKEN(")",          0, TOKEN_TYPE_CLOSE_PAREN);
-    LITERAL_TOKEN("{",          0, TOKEN_TYPE_OPEN_BRACE);
-    LITERAL_TOKEN("}",          0, TOKEN_TYPE_CLOSE_BRACE);
-    LITERAL_TOKEN("[",          0, TOKEN_TYPE_OPEN_BRACKET);
-    LITERAL_TOKEN("]",          0, TOKEN_TYPE_CLOSE_BRACKET);
-    LITERAL_TOKEN("+",          0, TOKEN_TYPE_SYM_PLUS);
-    LITERAL_TOKEN("-",          0, TOKEN_TYPE_SYM_MINUS);
-    LITERAL_TOKEN("*",          0, TOKEN_TYPE_SYM_STAR);
-    LITERAL_TOKEN(".",          0, TOKEN_TYPE_SYM_DOT);
-    LITERAL_TOKEN("%",          0, TOKEN_TYPE_SYM_PERCENT);
-    LITERAL_TOKEN("/",          0, TOKEN_TYPE_SYM_FSLASH);
-    LITERAL_TOKEN("\\",         0, TOKEN_TYPE_SYM_BSLASH);
-    LITERAL_TOKEN(":",          0, TOKEN_TYPE_SYM_COLON);
-    LITERAL_TOKEN(";",          0, TOKEN_TYPE_SYM_SEMICOLON);
-    LITERAL_TOKEN(",",          0, TOKEN_TYPE_SYM_COMMA);
-    LITERAL_TOKEN(">",          0, TOKEN_TYPE_SYM_GREATER);
-    LITERAL_TOKEN("<",          0, TOKEN_TYPE_SYM_LESS);
-    LITERAL_TOKEN("=",          0, TOKEN_TYPE_SYM_EQUALS);
-    LITERAL_TOKEN("~",          0, TOKEN_TYPE_SYM_TILDE);
-    LITERAL_TOKEN("!",          0, TOKEN_TYPE_SYM_BANG);
-    LITERAL_TOKEN("^",          0, TOKEN_TYPE_SYM_CARET);
-    LITERAL_TOKEN("&",          0, TOKEN_TYPE_SYM_AMPERSAND);
+    LITERAL_TOKEN("<=",         0, TOKEN_TYPE_LESS_EQUAL);
+    LITERAL_TOKEN(">=",         0, TOKEN_TYPE_GREATER_EQUAL);
+    LITERAL_TOKEN("==",         0, TOKEN_TYPE_EQUAL_EQUAL);
+    LITERAL_TOKEN("!=",         0, TOKEN_TYPE_NOT_EQUAL);
+    LITERAL_TOKEN("+=",         0, TOKEN_TYPE_PLUS_EQUAL);
+    LITERAL_TOKEN("-=",         0, TOKEN_TYPE_MINUS_EQUAL);
+    LITERAL_TOKEN("*=",         0, TOKEN_TYPE_STAR_EQUAL);
+    LITERAL_TOKEN("/=",         0, TOKEN_TYPE_FSLASH_EQUAL);
+    LITERAL_TOKEN("%=",         0, TOKEN_TYPE_PERCENT_EQUAL);
 
     // Symbols
     if (char_is_alpha(*tk.token)) {
@@ -250,8 +209,12 @@ OnyxToken* onyx_get_token(OnyxTokenizer* tokenizer) {
 
         tk.type = TOKEN_TYPE_LITERAL_NUMERIC;
         tk.length = len;
+
+        INCREMENT_CURR_TOKEN(tokenizer);
+        goto token_parsed;
     }
 
+    tk.type = (TokenType) *tokenizer->curr;
     INCREMENT_CURR_TOKEN(tokenizer);
 
 token_parsed:
@@ -262,15 +225,15 @@ token_parsed:
 
 OnyxTokenizer onyx_tokenizer_create(bh_allocator allocator, bh_file_contents *fc) {
     OnyxTokenizer tknizer = {
-        .start             = fc->data,
-        .curr             = fc->data,
-        .end             = fc->data + fc->length,
+        .start          = fc->data,
+        .curr           = fc->data,
+        .end            = fc->data + fc->length,
 
-        .filename         = fc->filename,
+        .filename       = fc->filename,
 
-        .line_number     = 1,
+        .line_number    = 1,
         .line_start     = fc->data,
-        .tokens            = NULL,
+        .tokens         = NULL,
     };
 
     bh_arr_new(allocator, tknizer.tokens, 512);

diff --git a/src/onyxparser.c b/src/onyxparser.c
index 0d203444bd55d9533d69eb8f0e986eafdd02bdc9..4391ab5150832fd189691c25f2838873a1dc9bfc 100644 (file)
--- a/src/onyxparser.c
+++ b/src/onyxparser.c
@@ -65,15 +65,10 @@ static void parser_prev_token(OnyxParser* parser) {
 }
 
 static b32 is_terminating_token(TokenType token_type) {
-    switch (token_type) {
-    case TOKEN_TYPE_SYM_SEMICOLON:
-    case TOKEN_TYPE_CLOSE_BRACE:
-    case TOKEN_TYPE_OPEN_BRACE:
-    case TOKEN_TYPE_END_STREAM:
-        return 1;
-    default:
-        return 0;
-    }
+    return (token_type == ';')
+        || (token_type == '}')
+        || (token_type == '{')
+        || (token_type == TOKEN_TYPE_END_STREAM);
 }
 
 static void find_token(OnyxParser* parser, TokenType token_type) {
@@ -138,17 +133,17 @@ static AstNodeNumLit* parse_numeric_literal(OnyxParser* parser) {
 static AstNodeTyped* parse_factor(OnyxParser* parser) {
     AstNodeTyped* retval = NULL;
 
-    switch (parser->curr_token->type) {
-        case TOKEN_TYPE_OPEN_PAREN:
+    switch ((u16) parser->curr_token->type) {
+        case '(':
             {
                 parser_next_token(parser);
                 AstNodeTyped* expr = parse_expression(parser);
-                expect(parser, TOKEN_TYPE_CLOSE_PAREN);
+                expect(parser, ')');
                 retval = expr;
                 break;
             }
 
-        case TOKEN_TYPE_SYM_MINUS:
+        case '-':
             {
                 parser_next_token(parser);
                 AstNodeTyped* factor = parse_factor(parser);
@@ -171,19 +166,19 @@ static AstNodeTyped* parse_factor(OnyxParser* parser) {
                 AstNodeTyped* sym_node = make_node(AstNode, AST_NODE_KIND_SYMBOL);
                 sym_node->token = sym_token;
 
-                if (parser->curr_token->type != TOKEN_TYPE_OPEN_PAREN) {
+                if (parser->curr_token->type != '(') {
                     retval = sym_node;
                     break;
                 }
 
                 // NOTE: Function call
                 AstNodeCall* call_node = make_node(AstNodeCall, AST_NODE_KIND_CALL);
-                call_node->base.token = expect(parser, TOKEN_TYPE_OPEN_PAREN);
+                call_node->base.token = expect(parser, '(');
                 call_node->callee = (AstNode *) sym_node;
 
                 AstNodeArgument** prev = &call_node->arguments;
                 AstNodeArgument* curr = NULL;
-                while (parser->curr_token->type != TOKEN_TYPE_CLOSE_PAREN) {
+                while (parser->curr_token->type != ')') {
                     curr = make_node(AstNodeArgument, AST_NODE_KIND_ARGUMENT);
                     curr->value = parse_expression(parser);
 
@@ -192,14 +187,14 @@ static AstNodeTyped* parse_factor(OnyxParser* parser) {
                         prev = (AstNodeArgument **) &curr->base.next;
                     }
 
-                    if (parser->curr_token->type == TOKEN_TYPE_CLOSE_PAREN)
+                    if (parser->curr_token->type == ')')
                         break;
 
-                    if (parser->curr_token->type != TOKEN_TYPE_SYM_COMMA) {
+                    if (parser->curr_token->type != ',') {
                         onyx_message_add(parser->msgs,
                                 ONYX_MESSAGE_TYPE_EXPECTED_TOKEN,
                                 parser->curr_token->pos,
-                                onyx_get_token_type_name(TOKEN_TYPE_SYM_COMMA),
+                                onyx_get_token_type_name(','),
                                 onyx_get_token_type_name(parser->curr_token->type));
                         return (AstNodeTyped *) &error_node;
                     }
@@ -292,19 +287,19 @@ static AstNodeTyped* parse_expression(OnyxParser* parser) {
 
     while (1) {
         bin_op_kind = -1;
-        switch (parser->curr_token->type) {
-            case TOKEN_TYPE_SYM_EQUAL_EQUAL:    bin_op_kind = ONYX_BINARY_OP_EQUAL; break;
-            case TOKEN_TYPE_SYM_NOT_EQUAL:      bin_op_kind = ONYX_BINARY_OP_NOT_EQUAL; break;
-            case TOKEN_TYPE_SYM_LESS_EQUAL:     bin_op_kind = ONYX_BINARY_OP_LESS_EQUAL; break;
-            case TOKEN_TYPE_SYM_LESS:           bin_op_kind = ONYX_BINARY_OP_LESS; break;
-            case TOKEN_TYPE_SYM_GREATER_EQUAL:  bin_op_kind = ONYX_BINARY_OP_GREATER_EQUAL; break;
-            case TOKEN_TYPE_SYM_GREATER:        bin_op_kind = ONYX_BINARY_OP_GREATER; break;
-
-            case TOKEN_TYPE_SYM_PLUS:       bin_op_kind = ONYX_BINARY_OP_ADD; break;
-            case TOKEN_TYPE_SYM_MINUS:      bin_op_kind = ONYX_BINARY_OP_MINUS; break;
-            case TOKEN_TYPE_SYM_STAR:       bin_op_kind = ONYX_BINARY_OP_MULTIPLY; break;
-            case TOKEN_TYPE_SYM_FSLASH:     bin_op_kind = ONYX_BINARY_OP_DIVIDE; break;
-            case TOKEN_TYPE_SYM_PERCENT:    bin_op_kind = ONYX_BINARY_OP_MODULUS; break;
+        switch ((u16) parser->curr_token->type) {
+            case TOKEN_TYPE_EQUAL_EQUAL:    bin_op_kind = ONYX_BINARY_OP_EQUAL; break;
+            case TOKEN_TYPE_NOT_EQUAL:      bin_op_kind = ONYX_BINARY_OP_NOT_EQUAL; break;
+            case TOKEN_TYPE_LESS_EQUAL:     bin_op_kind = ONYX_BINARY_OP_LESS_EQUAL; break;
+            case TOKEN_TYPE_GREATER_EQUAL:  bin_op_kind = ONYX_BINARY_OP_GREATER_EQUAL; break;
+            case '<':                       bin_op_kind = ONYX_BINARY_OP_LESS; break;
+            case '>':                       bin_op_kind = ONYX_BINARY_OP_GREATER; break;
+
+            case '+':                       bin_op_kind = ONYX_BINARY_OP_ADD; break;
+            case '-':                       bin_op_kind = ONYX_BINARY_OP_MINUS; break;
+            case '*':                       bin_op_kind = ONYX_BINARY_OP_MULTIPLY; break;
+            case '/':                       bin_op_kind = ONYX_BINARY_OP_DIVIDE; break;
+            case '%':                       bin_op_kind = ONYX_BINARY_OP_MODULUS; break;
             default: goto expression_done;
         }
 
@@ -403,9 +398,9 @@ static b32 parse_symbol_statement(OnyxParser* parser, AstNode** ret) {
     if (parser->curr_token->type != TOKEN_TYPE_SYMBOL) return 0;
     OnyxToken* symbol = expect(parser, TOKEN_TYPE_SYMBOL);
 
-    switch (parser->curr_token->type) {
+    switch ((u16) parser->curr_token->type) {
         // NOTE: Declaration
-        case TOKEN_TYPE_SYM_COLON:
+        case ':':
             {
                 parser_next_token(parser);
                 TypeInfo* type = &builtin_types[TYPE_INFO_KIND_UNKNOWN];
@@ -421,8 +416,8 @@ static b32 parse_symbol_statement(OnyxParser* parser, AstNode** ret) {
                 local->base.flags |= ONYX_AST_FLAG_LVAL; // NOTE: DELETE
                 *ret = (AstNode *) local;
 
-                if (parser->curr_token->type == TOKEN_TYPE_SYM_EQUALS || parser->curr_token->type == TOKEN_TYPE_SYM_COLON) {
-                    if (parser->curr_token->type == TOKEN_TYPE_SYM_COLON) {
+                if (parser->curr_token->type == '=' || parser->curr_token->type == ':') {
+                    if (parser->curr_token->type == ':') {
                         local->base.flags |= ONYX_AST_FLAG_CONST;
                     }
 
@@ -451,7 +446,7 @@ static b32 parse_symbol_statement(OnyxParser* parser, AstNode** ret) {
             }
 
             // NOTE: Assignment
-        case TOKEN_TYPE_SYM_EQUALS:
+        case '=':
             {
                 AstNodeAssign* assignment = make_node(AstNodeAssign, AST_NODE_KIND_ASSIGNMENT);
                 assignment->base.token = parser->curr_token;
@@ -467,18 +462,18 @@ static b32 parse_symbol_statement(OnyxParser* parser, AstNode** ret) {
                 return 1;
             }
 
-        case TOKEN_TYPE_SYM_PLUS_EQUAL:
-        case TOKEN_TYPE_SYM_MINUS_EQUAL:
-        case TOKEN_TYPE_SYM_STAR_EQUAL:
-        case TOKEN_TYPE_SYM_FSLASH_EQUAL:
-        case TOKEN_TYPE_SYM_PERCENT_EQUAL:
+        case TOKEN_TYPE_PLUS_EQUAL:
+        case TOKEN_TYPE_MINUS_EQUAL:
+        case TOKEN_TYPE_STAR_EQUAL:
+        case TOKEN_TYPE_FSLASH_EQUAL:
+        case TOKEN_TYPE_PERCENT_EQUAL:
             {
                 OnyxBinaryOp bin_op;
-                if      (parser->curr_token->type == TOKEN_TYPE_SYM_PLUS_EQUAL)    bin_op = ONYX_BINARY_OP_ADD;
-                else if (parser->curr_token->type == TOKEN_TYPE_SYM_MINUS_EQUAL)   bin_op = ONYX_BINARY_OP_MINUS;
-                else if (parser->curr_token->type == TOKEN_TYPE_SYM_STAR_EQUAL)    bin_op = ONYX_BINARY_OP_MULTIPLY;
-                else if (parser->curr_token->type == TOKEN_TYPE_SYM_FSLASH_EQUAL)  bin_op = ONYX_BINARY_OP_DIVIDE;
-                else if (parser->curr_token->type == TOKEN_TYPE_SYM_PERCENT_EQUAL) bin_op = ONYX_BINARY_OP_MODULUS;
+                if      (parser->curr_token->type == TOKEN_TYPE_PLUS_EQUAL)    bin_op = ONYX_BINARY_OP_ADD;
+                else if (parser->curr_token->type == TOKEN_TYPE_MINUS_EQUAL)   bin_op = ONYX_BINARY_OP_MINUS;
+                else if (parser->curr_token->type == TOKEN_TYPE_STAR_EQUAL)    bin_op = ONYX_BINARY_OP_MULTIPLY;
+                else if (parser->curr_token->type == TOKEN_TYPE_FSLASH_EQUAL)  bin_op = ONYX_BINARY_OP_DIVIDE;
+                else if (parser->curr_token->type == TOKEN_TYPE_PERCENT_EQUAL) bin_op = ONYX_BINARY_OP_MODULUS;
 
                 parser_next_token(parser);
 
@@ -518,7 +513,7 @@ static AstNodeReturn* parse_return_statement(OnyxParser* parser) {
 
     AstNodeTyped* expr = NULL;
 
-    if (parser->curr_token->type != TOKEN_TYPE_SYM_SEMICOLON) {
+    if (parser->curr_token->type != ';') {
         expr = parse_expression(parser);
 
         if (expr == NULL || expr == (AstNodeTyped *) &error_node) {
@@ -535,12 +530,12 @@ static AstNode* parse_statement(OnyxParser* parser) {
     b32 needs_semicolon = 1;
     AstNode* retval = NULL;
 
-    switch (parser->curr_token->type) {
+    switch ((u16) parser->curr_token->type) {
         case TOKEN_TYPE_KEYWORD_RETURN:
             retval = (AstNode *) parse_return_statement(parser);
             break;
 
-        case TOKEN_TYPE_OPEN_BRACE:
+        case '{':
             needs_semicolon = 0;
             retval = (AstNode *) parse_block(parser);
             break;
@@ -549,10 +544,10 @@ static AstNode* parse_statement(OnyxParser* parser) {
             if (parse_symbol_statement(parser, &retval)) break;
             // fallthrough
 
-        case TOKEN_TYPE_OPEN_PAREN:
-        case TOKEN_TYPE_SYM_PLUS:
-        case TOKEN_TYPE_SYM_MINUS:
-        case TOKEN_TYPE_SYM_BANG:
+        case '(':
+        case '+':
+        case '-':
+        case '!':
         case TOKEN_TYPE_LITERAL_NUMERIC:
         case TOKEN_TYPE_LITERAL_STRING:
             retval = (AstNode *) parse_expression(parser);
@@ -583,14 +578,14 @@ static AstNode* parse_statement(OnyxParser* parser) {
     }
 
     if (needs_semicolon) {
-        if (parser->curr_token->type != TOKEN_TYPE_SYM_SEMICOLON) {
+        if (parser->curr_token->type != ';') {
             onyx_message_add(parser->msgs,
                 ONYX_MESSAGE_TYPE_EXPECTED_TOKEN,
                 parser->curr_token->pos,
-                onyx_get_token_type_name(TOKEN_TYPE_SYM_SEMICOLON),
+                onyx_get_token_type_name(';'),
                 onyx_get_token_type_name(parser->curr_token->type));
 
-            find_token(parser, TOKEN_TYPE_SYM_SEMICOLON);
+            find_token(parser, ';');
         }
         parser_next_token(parser);
     }
@@ -604,18 +599,18 @@ static AstNodeBlock* parse_block(OnyxParser* parser) {
     block->scope = scope;
 
     // --- is for an empty block
-    if (parser->curr_token->type == TOKEN_TYPE_SYM_MINUS) {
-        expect(parser, TOKEN_TYPE_SYM_MINUS);
-        expect(parser, TOKEN_TYPE_SYM_MINUS);
-        expect(parser, TOKEN_TYPE_SYM_MINUS);
+    if (parser->curr_token->type == '-') {
+        expect(parser, '-');
+        expect(parser, '-');
+        expect(parser, '-');
         return block;
     }
 
-    expect(parser, TOKEN_TYPE_OPEN_BRACE);
+    expect(parser, '{');
 
     AstNode** next = &block->body;
     AstNode* stmt = NULL;
-    while (parser->curr_token->type != TOKEN_TYPE_CLOSE_BRACE) {
+    while (parser->curr_token->type != '}') {
         stmt = parse_statement(parser);
 
         if (stmt != NULL && stmt->kind != AST_NODE_KIND_ERROR) {
@@ -626,7 +621,7 @@ static AstNodeBlock* parse_block(OnyxParser* parser) {
         }
     }
 
-    expect(parser, TOKEN_TYPE_CLOSE_BRACE);
+    expect(parser, '}');
 
     return block;
 }
@@ -654,12 +649,12 @@ static TypeInfo* parse_type(OnyxParser* parser) {
 }
 
 static AstNodeLocal* parse_function_params(OnyxParser* parser) {
-    if (parser->curr_token->type != TOKEN_TYPE_OPEN_PAREN)
+    if (parser->curr_token->type != '(')
         return NULL;
 
-    expect(parser, TOKEN_TYPE_OPEN_PAREN);
+    expect(parser, '(');
 
-    if (parser->curr_token->type == TOKEN_TYPE_CLOSE_PAREN) {
+    if (parser->curr_token->type == ')') {
         parser_next_token(parser);
         return NULL;
     }
@@ -669,11 +664,11 @@ static AstNodeLocal* parse_function_params(OnyxParser* parser) {
     AstNodeLocal* trailer = NULL;
 
     OnyxToken* symbol;
-    while (parser->curr_token->type != TOKEN_TYPE_CLOSE_PAREN) {
-        if (parser->curr_token->type == TOKEN_TYPE_SYM_COMMA) parser_next_token(parser);
+    while (parser->curr_token->type != ')') {
+        if (parser->curr_token->type == ',') parser_next_token(parser);
 
         symbol = expect(parser, TOKEN_TYPE_SYMBOL);
-        expect(parser, TOKEN_TYPE_SYM_COLON);
+        expect(parser, ':');
 
         curr_param = make_node(AstNodeLocal, AST_NODE_KIND_PARAM);
         curr_param->base.token = symbol;
@@ -794,7 +789,7 @@ static AstNode* parse_top_level_statement(OnyxParser* parser) {
                 OnyxToken* symbol = parser->curr_token;
                 parser_next_token(parser);
 
-                expect(parser, TOKEN_TYPE_SYM_COLON);
+                expect(parser, ':');
 
                 TypeInfo* type = &builtin_types[TYPE_INFO_KIND_UNKNOWN];
 
@@ -802,7 +797,7 @@ static AstNode* parse_top_level_statement(OnyxParser* parser) {
                     type = parse_type(parser);
                 }
 
-                if (parser->curr_token->type == TOKEN_TYPE_SYM_COLON) {
+                if (parser->curr_token->type == ':') {
                     parser_next_token(parser);
 
                     AstNode* node = parse_top_level_constant_symbol(parser);
@@ -820,7 +815,7 @@ static AstNode* parse_top_level_statement(OnyxParser* parser) {
 
                     return node;
 
-                } else if (parser->curr_token->type == TOKEN_TYPE_SYM_EQUALS) {
+                } else if (parser->curr_token->type == '=') {
                     parser_next_token(parser);
 
                     AstNodeGlobal* global = make_node(AstNodeGlobal, AST_NODE_KIND_GLOBAL);