fix lexer offset bug
This commit is contained in:
parent
0a035e5901
commit
8355dcab12
38
lex.c
38
lex.c
@ -36,7 +36,7 @@ void lex_free(Lexer*l)
|
||||
}
|
||||
|
||||
// Read string and store tokens
|
||||
void lex_string(Lexer*l,char*s)
|
||||
void lex_string(Lexer*lex,char*s)
|
||||
{
|
||||
//Reg regex=reg_new();
|
||||
Str tmpstr=str_new();
|
||||
@ -44,7 +44,7 @@ void lex_string(Lexer*l,char*s)
|
||||
size_t current_line=1;
|
||||
size_t strl;
|
||||
|
||||
if(!l||!s)return;
|
||||
if(!lex||!s)return;
|
||||
|
||||
strl=strlen(s);
|
||||
|
||||
@ -60,17 +60,15 @@ void lex_string(Lexer*l,char*s)
|
||||
* mode uint32_t change lexer mode to this
|
||||
* keepch bool will we retain this character in the token string?
|
||||
*****/
|
||||
#define initmatch(chset,lmode,keepch) if(s[i]&&memchr((chset),s[i],strlen((chset)))){l->mode=(lmode);if(keepch)--i;\
|
||||
Tok _tmptok={.str=str_new(),.type=l->mode,.line=current_line};vec_push(&l->tokens,&_tmptok);\
|
||||
str_clear(&tmpstr);}
|
||||
#define initmatch(chset,lmode,keepch) if(s[i]&&memchr((chset),s[i],strlen((chset)))){lex->mode=(lmode);if(keepch)--i;Tok _tmptok={.str=str_new(),.type=lex->mode,.line=current_line};vec_push(&lex->tokens,&_tmptok);str_clear(&tmpstr);}
|
||||
|
||||
/*****
|
||||
* modeterminate
|
||||
* - Finalize current token lexing and set state to LNONE
|
||||
* keepch bool will we retain this character in the token string?
|
||||
*****/
|
||||
//&((Tok*)l->tokens.buffer)[l->tokens.size-1].str
|
||||
#define modeterminate(keepch) do{l->mode=LNONE;if(keepch)--i;str_assign(&(vec_at(&l->tokens,l->tokens.size-1,Tok*)->str),tmpstr.buffer);vec_at(&l->tokens,l->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
|
||||
//&((Tok*)lex->tokens.buffer)[lex->tokens.size-1].str
|
||||
#define modeterminate(keepch) do{lex->mode=LNONE;if(keepch)--i;str_assign(&(vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->str),tmpstr.buffer);vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
|
||||
|
||||
/*****
|
||||
* modematch(chset,logic,keepch)
|
||||
@ -82,7 +80,7 @@ void lex_string(Lexer*l,char*s)
|
||||
*****/
|
||||
#define modematch(chset,logic,keepch) do{if(!s[i]||(logic==(!!memchr((chset),s[i],strlen(chset)))) ){modeterminate(keepch);}ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
|
||||
|
||||
switch(l->mode)
|
||||
switch(lex->mode)
|
||||
{
|
||||
|
||||
// New lexeme boundary
|
||||
@ -97,10 +95,10 @@ void lex_string(Lexer*l,char*s)
|
||||
|
||||
case LNONE:
|
||||
// Extra step: turn identifiers into keywords
|
||||
if(l->tokens.size)
|
||||
if(lex->tokens.size>1)
|
||||
{
|
||||
//Tok*lasttok=&((Tok*)l->tokens.buffer)[l->tokens.size-2];
|
||||
Tok*lasttok=vec_at(&l->tokens,l->tokens.size-2,Tok*);
|
||||
//Tok*lasttok=&((Tok*)lex->tokens.buffer)[lex->tokens.size-2];
|
||||
Tok*lasttok=vec_at(&lex->tokens,lex->tokens.size-2,Tok*);
|
||||
if(lasttok->type==LIDENTIFIER)
|
||||
{
|
||||
for(size_t j=0;j<sizeof(keywords)/sizeof(char*);++j)
|
||||
@ -133,16 +131,16 @@ void lex_string(Lexer*l,char*s)
|
||||
case LINTEGER:modematch("0123456789.",false,true);
|
||||
if(s[i]=='.')
|
||||
{
|
||||
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LFLOAT;
|
||||
l->mode=LFLOAT;
|
||||
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LFLOAT;
|
||||
lex->mode=LFLOAT;
|
||||
}
|
||||
break;
|
||||
case LSTRING:modematch("\"",true,false);break;
|
||||
case LOPERATOR:modematch(operator_chars,false,true);
|
||||
switch(s[i])
|
||||
{
|
||||
#define opmatch(lstype,keepch) do{vec_at(&l->tokens,l->tokens.size-1,Tok*)->subtype=lstype;modeterminate(keepch);}while(0)
|
||||
#define opmatch_nodup(lstype,keepch) do{l->mode=LNONE;vec_at(&l->tokens,l->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
|
||||
#define opmatch(lstype,keepch) do{vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->subtype=lstype;modeterminate(keepch);}while(0)
|
||||
#define opmatch_nodup(lstype,keepch) do{lex->mode=LNONE;vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->line=current_line;ch[0]=s[i];str_append(&tmpstr,ch);}while(0)
|
||||
case ';':opmatch(LENDSTATEMENT,false);break;
|
||||
case '(':opmatch(LLPAREN,false);break;
|
||||
case ')':opmatch(LRPAREN,false);break;
|
||||
@ -151,7 +149,7 @@ void lex_string(Lexer*l,char*s)
|
||||
case '*':opmatch(0,false);break;
|
||||
case '=':
|
||||
{
|
||||
char*ss=vec_at(&l->tokens,l->tokens.size-1,Tok*)->str.buffer;
|
||||
char*ss=vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->str.buffer;
|
||||
if(!ss)break;
|
||||
if(strcmp("==",ss)==0)opmatch_nodup(0,false);
|
||||
else if(strcmp("+=",ss)==0)opmatch_nodup(LASSIGN,false);
|
||||
@ -168,15 +166,15 @@ void lex_string(Lexer*l,char*s)
|
||||
case LMINUS://modematch("0123456789=",true,true);
|
||||
if(s[i]=='-')
|
||||
{
|
||||
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LOPERATOR;
|
||||
vec_at(&l->tokens,l->tokens.size-1,Tok*)->subtype=LSMINUS;
|
||||
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LOPERATOR;
|
||||
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->subtype=LSMINUS;
|
||||
ch[0]=s[i];str_append(&tmpstr,ch);
|
||||
modeterminate(false);
|
||||
}
|
||||
else// if(s[i]=='=')
|
||||
{
|
||||
vec_at(&l->tokens,l->tokens.size-1,Tok*)->type=LOPERATOR;
|
||||
l->mode=LOPERATOR;
|
||||
vec_at(&lex->tokens,lex->tokens.size-1,Tok*)->type=LOPERATOR;
|
||||
lex->mode=LOPERATOR;
|
||||
modeterminate(true);
|
||||
}
|
||||
break;
|
||||
|
3
vec.c
3
vec.c
@ -59,7 +59,10 @@ void vec_grow(Vec*v)
|
||||
if(!v){if(VECVERBOSE)fprintf(stderr,"vec_grow: NULL Vec*\n");return;}
|
||||
if(!v->unitsize){if(VECVERBOSE)fprintf(stderr,"vec_grow: unitsize is zero\n");return;}
|
||||
if(!v->buffer)
|
||||
{
|
||||
if(VECVERBOSE)if(v->capacity>0)fprintf(stderr,"vec_grow: NULL buffer with non-zero capacity\n");
|
||||
v->buffer=malloc(VECDEFSIZE*v->unitsize);
|
||||
}
|
||||
else
|
||||
v->buffer=realloc(v->buffer,(v->capacity+VECDEFSIZE)*v->unitsize);
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user