From 931d23c98cc22749bfc41f438dbfa71e393d2650 Mon Sep 17 00:00:00 2001 From: Brendan Hansen Date: Thu, 14 May 2020 17:18:33 -0500 Subject: [PATCH] Fixed Last tokens produced by tokenizer appear to be wrong. #7 --- bh.h | 2 +- onyx | Bin 42848 -> 42848 bytes onyx.c | 2 +- onyxlex.c | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bh.h b/bh.h index 12de3f4e..2af69298 100644 --- a/bh.h +++ b/bh.h @@ -383,7 +383,7 @@ char charset_contains(const char* charset, char ch) { i64 chars_match(char* ptr1, char* ptr2) { i64 len = 0; - while (*ptr1 == *ptr2) ptr1++, ptr2++, len++; + while (*ptr2 != '\0' && *ptr1 == *ptr2) ptr1++, ptr2++, len++; return *ptr2 == '\0' ? len : 0; } diff --git a/onyx b/onyx index 12c20cea85d1dbf22f3d155297ce643cb8735fb1..40b5131c991064593ef5ed7553e85cfb60b7276c 100755 GIT binary patch delta 5084 zcmZu#4OmoF8ouYwFbs$Sf)1lF12Z5$B7=)4DhO!6pcqJMN?M9S{$QeF`BMyrnBzc! zUmBaWY8Hi3X_HoL7FtP^mAIv7Zp!*M2(8+(t+lpy-#hnScXyth=b7(4=Y7BPch7gu ze0MH=&0qSOuRrC>P2r@^l!o&k1-49(D)*3@+4sTk=Cv9=KK1&iw(LOt#J8#~Yc~vU zb~f@>^}@(x$v>tb++EO0Rc@TjCwlPHWj@7&Um)`t9{dK%^w#McWVOIUJwU6V9+TAt z9_l&T>#eihlGSn#RWnwa>N*cTT;{7h_=z&V)q{UP=IcH96%tS5w1N7~vdd~WyFB=bGJnZ~FQf<`oxW67Z+WO2 zX$I8iWR;r{?jBUx>n7R+^+Q<=@K8_FAs_65tcH82UMbT2V?20+%qM#ANiv_}!RN|+ zh6n$!#8ZxMpni?)Q{dtA46X9j>0gr71s>{Q+6%Q^R?9uqZtC>avAfP5U%Sfx#St85 zts7`|L)skYra?iBH92?q$1!%(*%okuXVvsfkeQXzfgm%V8%3vrOnht<^#im~6cTJ= zJv1@c%)X*WVBe%_*m`;$b|rlYTR=C1&3sZM1rIUtgCi+rh>6{%hlZHhH)Ml-llH+n zs2z4G^$s!fIc5qNYT~2JGzOqD)109u)=e9Sn%T#+8}?=T1a>1`hn-6UbY^~>i4t`t zeu#-?>P)Pkp3s@uY1##QkUoZemM+7Vk#~rhpBh2&AtpX1f(ihd2&xD%u`BdKNH`qc z4|$yBkVQXClZ1$Iz3a@^H!`-IT*Dq`{mw^*S24OAIyrc{ulU%q>aBIN+N}07R(re4 zm&S)>vFT(B3kkasz;XWXai#XGEmp^@Myn&s^%0M#5&ibjyJ6GVn(c;z}VJ>4I z(pScphh6eN!fkG=7%V#b9|-`6Ov>?!Z)tsaLF@-1V)Lv!z0vJi{Y>nu&g+1}1%Y$- zU+47)`YHUtpu0MbbEtXiqEm3YNe3ckvJu-&4z{0aP0zAvG~8@pCuxE?feoak=8=ri z4)YdPM4^#IYQZH4bj%z@yCakNy)FuiG^6NG+NfxjO^HzvY_YR2DuuC!$T6aYO`??O zFm{P%M;EbVIuu>U-lO!G{mc(Glr*u~e9vV{kJnL2tWAIWvLJ{Hv^v(_S!_)j6BUD% zRCu}8({HggjHo*96gy5cM~+0iRgFB#?$D(84Ax7w_&KbSy5iee6YUvgzz`pgisEZ- z(Mt(YG$3IL^QQcSX^5y@3EBFV0UXy_tr8zm@+J31rTqYKwa~8#CN`eJEc4k-+GugG zeDY45z{WTyCpI$Mtjw7YSU2@GVn+6(^8`%Pez*Xz*y>Qb=ZVMPHM-5}s1aFfb(_}e z2yh316aXV`3WB|+3|jS3(Q*|@9=;B*amH|QRxf#ro7w(8hT>dnz88e|8E50T9zSfN z)MLBryT6Gu$l7DI&w4}Q4*vwMuBKcZAEETa9iU@}yILoDx&!fSkEqtH6P+9q)U;I~ zprmrOOY105_jwib7SxE1ZLerdDZ^GCI_Uv#~qd$A zK&Jh-T>H;?u~*$s{n&3RSB|rC%o+9QX*keB@ipQrmSbcT%gT2&$`j=J6sj4&Tpb;? zD4K4LpFwF8%(l&^1)&eO&Z~bCgkU`4cYzT?x`qxw><0D$PXPVY z9JlQ{IshI9<^jF2hB9CWuoh^*VQ2w92|NiD4@Vy``3Bm1aa=tx9{3qB52)@%d*Bpc zE$}5^ix=Xh(Fp7Vo&fqG5YGLG4uG5giVlDu0Ly^$aU0YEe+9MxKfi_cz@*z~ z58My*!?`lzn8yRfLYR~RU->)Q1Bd>M_Q5D5B4ADe`+pIHe&AxdJu#ZU)k(ur z6YztZn;Olw(yCM)+eY?OBa5VFSQB-m8Y8A$6olL2Z${>zfGzx#ff`jG^Yhknp`b3{ z8%#cFM*j6~ic2%_N4jY$;Dc^j1~}bK+W=>}X+PjvH+_+2i1-(Fw+QX-^~cW=yOa9E zI4R^a$%tR+Sl9!UJITnu^A}n+$-ril9n?Z2L8i z<;6pJ(NJD2loyE&a@w!FIvkMe%FDtzxvsn(D6a!b`mdz>N_sD+^PHT@+qf5%tX;-- z8I^p!U#=?|x{{GA8F-c%Hi1<&v{i;C`=L6Z z9CwJSp~@CkvU)Yg`-xYRs^LOqSm`2;AN-A5tha^E15gJIu~Cb)obz#t$XK3StF$`PGMit+QqdvhJ4Ao zp>K|oI_F~ki50SJRUE^Nwyk#UV&u1}IwLF7;LB~*DOMzvpnUgk`J&!9xzv3x2mR?wKKZd!s z;$h^lS&wC~Fxv81KGt^Tv4?#og-h!xqO_GcOhYv*)7csNY~?ie4h5E5*m}x@olUD? zqiHW}g|oA~-W!uCUYEjxY1g{jSaaR_7YC{|2^_CxS77)Sr)2}LvbA30%|Q)(Y|vh9 z+R>}p4@F(mDmwfGqn-nUl!{kt#70`Ns?}%{0p1;=;pYYdm_`&-TCsPvR`k@&(VW!o zwp6>tr?qD^T9fwlFMqsxO(f=OuJ8dIV>9!oIp2QDf1vZzZ8O=_VEG!5ch3zfE`SUF zzkDF4{4esq{)4%KRA+JRY+k?CEGe^Ohl@al{u-(CSskv^o@XuWF6p0pg0)b?a~3`z znT|h~fQ`Hjs*#q~TlA}=rM5xhj|@H|a9s}O260Vvs9vY{iIdouCE&7UHJZAi9#53$ zVRGA2S=~f|4LP_wsi+|vyScZ)f*9$9Eg`mDrw@yenkC4c{!4C_N~5>y5c#>=E&4M? zNt8$6(h%U{v@TM^_8e^Wx$TGn@*xW%Zan3TR-Pb^ulw_>?0+w^WL^UYv7Z<^L^}PR zF%n%Ot8ub=KXsE1@%sy2C%Z;SVwvnyCHu^#%;)oQA2mFmj>znM-lD&E{a`)f=c?@4 zP5K=f@lEl7qPd6bZDmzXQyN5BW#*8_Js!rhz(w`_$pzD9cq1LY@qV& o6x>E^pT$Rsy%rk3Z=7#Isx$^7Z^J(PvrtT#P2$xsucVyJ;3KA|*CR5VR8b3l@u25iRV zgR-ttefF}4%@=t!3(caw>-B=Aso7PsbPO!*iDmcP|C~8z)VcS3-&z0O>tB2AefC~^ zpR>-{BmCJTyr*6JN~vZR zcB#zH31MB-9HDdWklCsbwnf%ghv?g7eNBkIOV+Os(W`C`jNB8VPYUR1q$XCMCEM%> zu^CV08l8TQ%)TAMt|TwmMwx95VPB#H8r`_}Wwt$pJtpfWvyQp=k zqQ?*7I7efj&j*T-JK0wo701|XzRl68jQ#9ujroLU!|BO53+qR($65HR$MU$6ZG)UcA41+i=O7a)BEiD{VWji~Gk?NJlK?G7TApBL zn`mo-kyX*2gh$u}vg-S*&lxz*zsFakZ(``y-~SQT&wz-2kDKSR*Z&XKL~kce zVDsr((lVy?Eg3M4q4JsG`DlN1OANf6jc(zx4=z7LHO5I8XNNJaZ$*OW*51<<*Jl4k z>Z38dgsvK2kMij_&K<@(?r(QAhVP_zOvP*{MVQ@eH<5XRamx3CfQFa5lHmA+n{&7) zwqro84)=;rG0e}&X31djRAotH8hXxB&zE1Muahlw$r4RdlLzzai{wn6!Cs-$$z@oK zkpoR^p>Nv2Y{n{R|pnwl|zKlLMB8aj|Z%`obx_vX00bt-YWMr@T< zEq61(?;~b4BUqEIGua=s*6LH84>;Um z(m*kT{$ZH2b)wc$*Q#;2V-OEJ)yatIwC&tJJ$rEYYxm$U`$PJ zdwX3EzubVsY&&|`YZjlbal{0a(Wwaj?@vmPZ~|+UCvC%ZtKd5Nz0D%PPg7ukqk{u{ z=A;t@x)rTlyi`W9tlX_$BZ#Q2ibJ0EYl0v9BtC>A-qm zA+QBl3H$-r0L(p)_A1WF9Yo3!#S=Km=5gMh4#P_U?p%9umN}pxGUVragTSy0TdU3-9X(XIKYCZ z{s0HSW57b77Q43+_$IIcIN>rJ0QFbU9{2>X8~7(M65F>5(U1-t^b6Vpe+O0qe@0|9 z03ZAf?c-6XMR*(rUb`v?zW^&~T+U$Lag>(jWH3K9=M2`L?GOazLq!avMBFoQe?y%) z2DX}5u7Ry0Q?9|Z?TjG&DxMX#sF-^G&OU0@WfmEsky@OFx{Atk4NetS(TZBo>LJl4 z;5QMiZvk()k)L&)&jH^T{HPoG<=6Qc;4{D{-^kCt&aY-%1$P?^GH*0kc-`PRILHT| zeIx(yb^d+uW#Ef$pvY9X&t^Bo403U8J}H?{aM*kT3(_hssY z?K{NA8rZjFf~+SSnO*RRx^hv&e)h{F>GZB$~XD$>aH`XzIYn zeO_1?otYe|tsWq*G|s`klDhX46N~Z`w+F#M170`)_ z3HbU?zJD9e*Vg+V!>3`+1KI2lt$3h;6%jwTy5G*B0q0!GpLjxcx2g%wPLOBrK%65j zb2IU!dI9Af)Mwrxww0+o(K%7qBLrQdUgB4k*{^^yhn7mJ0`PM8t_;dy)?Fv+0THvwj7Y+`zm5z552jQr^ zg^S|0({7JW&!+}h=_p6>ZkbJ^PO$r=w<$i4#8(5Zc{00-V!Z`8I;qTiHx{$mYek53 zK+Ynz5&ud|0rQLiaQ~5;ji6y0bqM@%8?E|NNPERg?t#lie2c?6LEeo8Sm~1+t%&hn zWJSP@qJm+{3}W|6*B`QdFtB938U~^t6YVCQev&-zb7eMFW(%m3@Na^yA_eal6d)^Q zn;O}sjBJ~Va2$Czjm4(y*ksiQPaiywczq{Zo+AC`0_?W(%{pwJM>kvbhve-mz0Y7d z83&0tDk~j;jg3yi#y?PU!b$IN6t~GXxn$a6#YUP8*+R8jbowDf0+#YChU<;vMf6ri zySM1Dbw1lNo89Fbvh}{+NQ`S=uxgbs#0u?w;r3W%?i5^0>_uxYCH9JG)QcmvJ!1lW WAo5ndh`%v2$hLDNX6VtK7yb*AM%MWN diff --git a/onyx.c b/onyx.c index a7925bb1..ebb03052 100644 --- a/onyx.c +++ b/onyx.c @@ -25,7 +25,7 @@ int main(int argc, char *argv[]) { printf("There are %d tokens (Allocated space for %d tokens)\n", bh_arr_length(token_arr), bh_arr_capacity(token_arr)); for (OnyxToken* it = token_arr; !bh_arr_end(token_arr, it); it++) { - printf("%s (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), it->line_number, it->line_column); + printf("%s '%c' (Line %ld, Col %ld)\n", onyx_get_token_type_name(*it), *(char *)it->token, it->line_number, it->line_column); } bh_hash_iterator it = bh_hash_iter_setup(u16, symbol_count); diff --git a/onyxlex.c b/onyxlex.c index dbbfad53..91224a03 100644 --- a/onyxlex.c +++ b/onyxlex.c @@ -240,7 +240,7 @@ bh_arr(OnyxToken) onyx_parse_tokens(bh_file_contents *fc, bh_hash(u16) symcount) OnyxTokenizer tknizer = { .start = fc->data, .curr = fc->data, - .end = fc->data + fc->length - 1, + .end = fc->data + fc->length, .line_number = 1, .line_start = fc->data, .symbol_count = symcount, -- 2.25.1