corey/lr
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix lexer offset bug

Browse Source
This commit is contained in:
corey 2023-11-10 15:27:39 -06:00
parent 0a035e5901
commit 8355dcab12
2 changed files with 21 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lex.c
View File

@ -36,7 +36,7 @@ void lex_free(Lexer*l)
}
// Read string and store tokens
void lex_string(Lexer*l,char*s)
void lex_string(Lexer*lex,char*s)
{
//Reg regex=reg_new();
Str tmpstr=str_new();
@ -44,7 +44,7 @@ void lex_string(Lexer*l,char*s)
size_t current_line=1;
size_t strl;
if(!l||!s)return;
if(!lex||!s)return;
strl=strlen(s);
@ -60,17 +60,15 @@ void lex_string(Lexer*l,char*s)
* mode uint32_t change lexer mode to this
* keepch bool will we retain this character in the token string?
*****/
#define initmatch(chset,lmode,keepch) if(s[i]&&memchr((chset),s[i],strlen((chset)))){l->mode=(lmode);if(keepch)--i;\
Tok _tmptok={.str=str_new(),.type=l->mode,.line=current_line};vec_push(&l->tokens,&_tmptok);\
str_clear(&tmpstr);}
#define initmatch(chset,lmode,keepch) if(s[i]&&memchr((chset),s[i],strlen((chset)))){lex->mode=(lmode);if(keepch)--i;Tok _tmptok={.str=str_new(),.type=lex->mode,.line=current_line};vec_push(&lex->tokens,&_tmptok);str_clear(&tmpstr);}
/*****
* modeterminate
* - Finalize current token lexing and set state to LNONE
* keepch bool will we retain this character in the token string?
*****/
//&((Tok*)l->tokens.buffer)[l->tokens.size-1].str
#define modeterminate(keepch) do{l->mode=LNONE;if(keepch)--i;str_assign(&(vec_at(&l->tokens,l->tokens.size-1,Tok*)->str),tmpstr.buffer);vec_at(&l->tokens,l->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
//&((Tok*)lex->tokens.buffer)[lex->tokens.size-1].str
#define modeterminate(keepch) do{lex->mode=LNONE;if(keepch)--i;str_assign(&(vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->str),tmpstr.buffer);vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
/*****
* modematch(chset,logic,keepch)
@ -82,7 +80,7 @@ void lex_string(Lexer*l,char*s)
*****/
#define modematch(chset,logic,keepch) do{if(!s[i]||(logic==(!!memchr((chset),s[i],strlen(chset)))) ){modeterminate(keepch);}ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
switch(l->mode)
switch(lex->mode)
{
// New lexeme boundary
@ -97,10 +95,10 @@ void lex_string(Lexer*l,char*s)
case LNONE:
// Extra step: turn identifiers into keywords
if(l->tokens.size)
if(lex->tokens.size>1)
{
//Tok*lasttok=&((Tok*)l->tokens.buffer)[l->tokens.size-2];
Tok*lasttok=vec_at(&l->tokens,l->tokens.size-2,Tok*);
//Tok*lasttok=&((Tok*)lex->tokens.buffer)[lex->tokens.size-2];
Tok*lasttok=vec_at(&lex->tokens,lex->tokens.size-2,Tok*);
if(lasttok->type==LIDENTIFIER)
{
for(size_t j=0;j<sizeof(keywords)/sizeof(char*);++j)
@ -133,16 +131,16 @@ void lex_string(Lexer*l,char*s)
case LINTEGER:modematch("0123456789.",false,true);
if(s[i]=='.')
{
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LFLOAT;
l->mode=LFLOAT;
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LFLOAT;
lex->mode=LFLOAT;
}
break;
case LSTRING:modematch("\"",true,false);break;
case LOPERATOR:modematch(operator_chars,false,true);
switch(s[i])
{
#define opmatch(lstype,keepch) do{vec_at(&l->tokens,l->tokens.size-1,Tok*)->subtype=lstype;modeterminate(keepch);}while(0)
#define opmatch_nodup(lstype,keepch) do{l->mode=LNONE;vec_at(&l->tokens,l->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
#define opmatch(lstype,keepch) do{vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->subtype=lstype;modeterminate(keepch);}while(0)
#define opmatch_nodup(lstype,keepch) do{lex->mode=LNONE;vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
case ';':opmatch(LENDSTATEMENT,false);break;
case '(':opmatch(LLPAREN,false);break;
case ')':opmatch(LRPAREN,false);break;
@ -151,7 +149,7 @@ void lex_string(Lexer*l,char*s)
case '*':opmatch(0,false);break;
case '=':
{
char*ss=vec_at(&l->tokens,l->tokens.size-1,Tok*)->str.buffer;
char*ss=vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->str.buffer;
if(!ss)break;
if(strcmp("==",ss)==0)opmatch_nodup(0,false);
else if(strcmp("+=",ss)==0)opmatch_nodup(LASSIGN,false);
@ -168,15 +166,15 @@ void lex_string(Lexer*l,char*s)
case LMINUS://modematch("0123456789=",true,true);
if(s[i]=='-')
{
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LOPERATOR;
vec_at(&l->tokens,l->tokens.size-1,Tok*)->subtype=LSMINUS;
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LOPERATOR;
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->subtype=LSMINUS;
ch[0]=s[i];str_append(&tmpstr,ch);
modeterminate(false);
}
else// if(s[i]=='=')
{
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LOPERATOR;
l->mode=LOPERATOR;
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LOPERATOR;
lex->mode=LOPERATOR;
modeterminate(true);
}
break;

3
vec.c
View File

@ -59,7 +59,10 @@ void vec_grow(Vec*v)
if(!v){if(VECVERBOSE)fprintf(stderr,"vec_grow: NULL Vec*\n");return;}
if(!v->unitsize){if(VECVERBOSE)fprintf(stderr,"vec_grow: unitsize is zero\n");return;}
if(!v->buffer)
{
if(VECVERBOSE)if(v->capacity>0)fprintf(stderr,"vec_grow: NULL buffer with non-zero capacity\n");
v->buffer=malloc(VECDEFSIZE*v->unitsize);
}
else
v->buffer=realloc(v->buffer,(v->capacity+VECDEFSIZE)*v->unitsize);