From 7f3294f2a23f62f727543083e9c79b13aed35c9b Mon Sep 17 00:00:00 2001 From: Brendan Hansen Date: Wed, 8 Mar 2023 06:24:45 -0600 Subject: [PATCH] added: single quote character literals changed: no re-declaration specifier is now `~` --- compiler/include/lex.h | 1 + compiler/src/lex.c | 10 ++++++---- compiler/src/parser.c | 25 ++++++++++++++++++++++--- core/conv/format.onyx | 2 +- core/string/string.onyx | 4 ++++ core/time/date.onyx | 4 ++-- examples/50_misc.onyx | 4 ++-- misc/onyx-mode.el | 2 ++ misc/onyx.sublime-syntax | 11 +++++++++++ misc/onyx.vim | 1 + misc/vscode/onyx-0.1.5.vsix | Bin 1531262 -> 1531262 bytes tests/aoc-2020/day11.onyx | 2 +- tests/aoc-2020/day14.onyx | 2 +- tests/aoc-2020/day17.onyx | 2 +- tests/aoc-2020/day19.onyx | 2 +- tests/aoc-2020/day20.onyx | 2 +- tests/char_literals | 4 ++++ tests/char_literals.onyx | 20 ++++++++++++++++++++ 18 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 tests/char_literals create mode 100644 tests/char_literals.onyx diff --git a/compiler/include/lex.h b/compiler/include/lex.h index cd437ece..23818d58 100644 --- a/compiler/include/lex.h +++ b/compiler/include/lex.h @@ -74,6 +74,7 @@ typedef enum TokenType { Token_Type_Symbol, Token_Type_Literal_String, + Token_Type_Literal_Char, Token_Type_Literal_Integer, Token_Type_Literal_Float, Token_Type_Literal_True, diff --git a/compiler/src/lex.c b/compiler/src/lex.c index 46c466b9..3e4a91c7 100644 --- a/compiler/src/lex.c +++ b/compiler/src/lex.c @@ -71,6 +71,7 @@ static const char* token_type_names[] = { "TOKEN_TYPE_SYMBOL", "TOKEN_TYPE_LITERAL_STRING", + "TOKEN_TYPE_LITERAL_CHAR", "TOKEN_TYPE_LITERAL_INTEGER", "TOKEN_TYPE_LITERAL_FLOAT", "true", @@ -243,14 +244,15 @@ whitespace_skipped: goto token_parsed; } - // String literal - if (*tk.text == '"') { + // String/Character literal + if (*tk.text == '"' || *tk.text == '\'') { u64 len = 0; u64 slash_count = 0; + char ch = *tk.text; INCREMENT_CURR_TOKEN(tokenizer); - while (!(*tokenizer->curr == '"' && slash_count == 0)) { + while (!(*tokenizer->curr == ch && slash_count == 0)) { len++; // if (*tokenizer->curr == '\n') { @@ -270,7 +272,7 @@ whitespace_skipped: INCREMENT_CURR_TOKEN(tokenizer); tk.text++; - tk.type = Token_Type_Literal_String; + tk.type = ch == '"' ? Token_Type_Literal_String : Token_Type_Literal_Char; tk.length = len; goto token_parsed; } diff --git a/compiler/src/parser.c b/compiler/src/parser.c index 465ae793..c6cbccce 100644 --- a/compiler/src/parser.c +++ b/compiler/src/parser.c @@ -605,6 +605,25 @@ static AstTyped* parse_factor(OnyxParser* parser) { break; } + case Token_Type_Literal_Char: { + AstNumLit* char_lit = make_node(AstNumLit, Ast_Kind_NumLit); + char_lit->flags |= Ast_Flag_Comptime; + char_lit->type_node = (AstType *) &basic_type_int_unsized; + char_lit->token = expect_token(parser, Token_Type_Literal_Char); + char_lit->was_char_literal = 1; + + i8 dest = '\0'; + i32 length = string_process_escape_seqs((char *) &dest, char_lit->token->text, 1); + char_lit->value.i = (u32) dest; + + if (length != 1) { + onyx_report_error(char_lit->token->pos, Error_Critical, "Expected only a single character in character literal."); + } + + retval = (AstTyped *) char_lit; + break; + } + case '#': { if (parse_possible_directive(parser, "file_contents")) { AstFileContents* fc = make_node(AstFileContents, Ast_Kind_File_Contents); @@ -1322,7 +1341,7 @@ static i32 parse_possible_compound_symbol_declaration(OnyxParser* parser, AstNod while (peek_token(token_offset)->type == Token_Type_Symbol) { token_offset += 1; - if (peek_token(token_offset)->type == '\'') token_offset += 1; + if (peek_token(token_offset)->type == '~') token_offset += 1; if (peek_token(token_offset)->type != ',') break; token_offset += 1; @@ -1344,7 +1363,7 @@ static i32 parse_possible_compound_symbol_declaration(OnyxParser* parser, AstNod AstNode* sym_node = make_symbol(parser->allocator, local_sym); bh_arr_push(local_compound->exprs, (AstTyped *) sym_node); - if (!consume_token_if_next(parser, '\'')) { + if (!consume_token_if_next(parser, '~')) { AstLocal* new_local = make_local(parser->allocator, local_sym, NULL); if (prev_local == NULL) { first_local = new_local; @@ -1394,7 +1413,7 @@ static i32 parse_possible_symbol_declaration(OnyxParser* parser, AstNode** ret) // If the token after the symbol is a comma, assume this is a compound declaration. if (peek_token(1)->type == ',' || - (peek_token(1)->type == '\'' && peek_token(2)->type == ',')) { + (peek_token(1)->type == '~' && peek_token(2)->type == ',')) { return parse_possible_compound_symbol_declaration(parser, ret); } diff --git a/core/conv/format.onyx b/core/conv/format.onyx index e6e3a564..3c0dc145 100644 --- a/core/conv/format.onyx +++ b/core/conv/format.onyx @@ -443,7 +443,7 @@ format_any :: (output: &Format_Output, formatting: &Format, v: any) { case u8 { value := *(cast(&u8) v.data); - if value > 31 { + if formatting.interpret_numbers { output->write(value); } else { diff --git a/core/string/string.onyx b/core/string/string.onyx index ba6e4c66..07fa5d97 100644 --- a/core/string/string.onyx +++ b/core/string/string.onyx @@ -651,6 +651,10 @@ bisect :: (s: str, substr: str) -> (str, str) { // Used by dyn_str // +to_dyn_str :: (x: str, allocator := context.allocator) -> dyn_str { + return (package core.array).make(x, allocator); +} + delete :: macro (x: &dyn_str, idx: u32) -> u8 { return (package core.array).delete(x, idx); } diff --git a/core/time/date.onyx b/core/time/date.onyx index ec865414..9e2e99cd 100644 --- a/core/time/date.onyx +++ b/core/time/date.onyx @@ -81,8 +81,8 @@ Date :: struct { @conv.Custom_Parse_Proc.{ Date } (d: &Date, text: str, _: Allocator) -> bool { year, t := string.bisect(text, #char "-"); - month, t' := string.bisect(t, #char "-"); - day, t' := string.bisect(t, #char "-"); + month, t~ := string.bisect(t, #char "-"); + day, t~ := string.bisect(t, #char "-"); d.year = ~~ conv.str_to_i64(year); d.month = ~~ (conv.str_to_i64(month) - 1); diff --git a/examples/50_misc.onyx b/examples/50_misc.onyx index feb89ef5..4b0e4b75 100644 --- a/examples/50_misc.onyx +++ b/examples/50_misc.onyx @@ -47,10 +47,10 @@ multiple_declaration_improvements :: () { // You're trying to declare a new variable k, while using the old variable // s in the same declaration. To get around this issue, there is a special // syntax you can use that tell the compiler that s is a not a new - // variable, but the same one as above. You simple place a ' after the + // variable, but the same one as above. You simple place a ~ after the // variable that you want to reuse. - k, s' := multiple_returns(); + k, s~ := multiple_returns(); printf("k: {}\ns: {}\n\n", k, s); // diff --git a/misc/onyx-mode.el b/misc/onyx-mode.el index 5f8e5ab6..338e7c4d 100644 --- a/misc/onyx-mode.el +++ b/misc/onyx-mode.el @@ -97,6 +97,8 @@ ;; Strings ("\\\".*\\\"" . font-lock-string-face) + ("\\\'.*\\\'" . font-lock-string-face) + ;; Numbers (,(onyx-wrap-word-rx onyx-number-rx) . font-lock-constant-face) diff --git a/misc/onyx.sublime-syntax b/misc/onyx.sublime-syntax index 1bbf0fea..3f917e28 100644 --- a/misc/onyx.sublime-syntax +++ b/misc/onyx.sublime-syntax @@ -15,6 +15,9 @@ contexts: - match: '"' scope: punctuation.definition.string.begin.onyx push: double_quoted_string + - match: "'" + scope: punctuation.definition.string.begin.onyx + push: single_quoted_string # Comments begin with a '//' and finish at the end of the line - match: '//' @@ -92,6 +95,14 @@ contexts: # 1: variable # 2: variable + single_quoted_string: + - meta_scope: string.quoted.single.onyx + - match: '\\.' + scope: constant.character.escape.onyx + - match: "'" + scope: punctuation.definition.string.end.onyx + pop: true + double_quoted_string: - meta_scope: string.quoted.double.onyx - match: '\\.' diff --git a/misc/onyx.vim b/misc/onyx.vim index 53eccd91..0eb3647a 100644 --- a/misc/onyx.vim +++ b/misc/onyx.vim @@ -49,6 +49,7 @@ syn match onyxDirective "\#[a-zA-Z_]\+" syn match onyxTag "@[a-zA-Z0-9_]\+" syn region onyxString start=+"+ skip=+\\\\\|\\"+ end=+"\|$+ extend contains=@Spell +syn region onyxString start=+'+ skip=+\\\\\|\\'+ end=+'\|$+ extend contains=@Spell syn region onyxMultiString start=+"""+ end=+"""+ extend contains=@Spell hi def link onyxKeyword Statement diff --git a/misc/vscode/onyx-0.1.5.vsix b/misc/vscode/onyx-0.1.5.vsix index 82722466d1202c47bded8a01025f07baa107567a..e0dbf32bf12798099c1d710523401f74f33ea430 100644 GIT binary patch delta 198 zcmeyjEB4>6Sl$3{W)=|!1`ZB}cZL}oc|S2R1L?&~%o5CR3^O*fO1m?I8UAwL!Hi~4 zrFKsxMj&PaVrC#_0b*7lW&>h&Am#vKP9Ww2Vs0Sj0b*Vt<^y7WAQk{(K_C_aVqqW_ z0b C(LG84 delta 198 zcmeyjEB4>6Sl$3{W)=|!1`ZB}S)J({c|S2R1L?&~%o5DgI@33^O1m?I8UAwL!Hi~4 zrFKsxMj&PaVrC#_0b*7lW&>h&Am#vKP9Ww2Vs0Sj0b*Vt<^y7WAQk{(K_C_aVqqW_ z0b