Timerix/cobek
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lexer changes

Browse Source
This commit is contained in:
timerix 2023-02-10 13:00:29 +06:00
parent 69449891e1
commit f5b4d6c7ee
1 changed files with 66 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
src/lexer/lexer.c
View File

@ -30,22 +30,10 @@ Maybe _throw_wrongchar(SharedLexerData* sld){
}
#define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(sld); }
#define addTok(id) Autoarr_add(tokens, default_tokens[id])
void _addTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
if(*(++source)==next){
addTok(yes);
}
else {
source--;
addTok(no);
}
}
#define addTok_ifnext(nextChar, yesTok, noTok) _addTok_ifnext(nextChar, yesTok, noTok, sld)
// adds <label> to <tokens> as tok_label or tok_number
void _tryAddLabel(SharedLexerData* sld){
if(label.length==0) return;
Unitype uni=ST_pullString(keywordsSearchTree, label);
if(uni.VoidPtr!=NULL) // built-in keyword
Autoarr_add(tokens, *(Token*)uni.VoidPtr);
@ -68,6 +56,17 @@ void _tryAddLabel(SharedLexerData* sld){
};
#define tryAddLabel() _tryAddLabel(sld)
#define addDefTok(id) tryAddLabel(); Autoarr_add(tokens, default_tokens[id])
void _addDefTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
if(*(source+1)==next){
source++;
addDefTok(yes);
}
else addDefTok(no);
}
#define addDefTok_ifnext(nextChar, yesTok, noTok) _addDefTok_ifnext(nextChar, yesTok, noTok, sld)
// returns text in quotes or error
Maybe _readString(char quotChar, SharedLexerData* sld){
char c;
@ -94,35 +93,24 @@ Maybe _readString(char quotChar, SharedLexerData* sld){
Maybe _lexan(SharedLexerData* sld){
char c;
source--;
while ((c=*++source)) switch(c){
while ((c=*++source)) {
// tryAddLabel() resets label.length, but label.ptr still pointing to prev token beginning
if(label.length==0)
label.ptr=source;
switch(c){
case ' ': case '\t':
case '\r': case '\n':
tryAddLabel();
break;
case '(':
tryAddLabel();
addTok(tok_lbracket);
break;
case '{':
tryAddLabel();
addTok(tok_lbracket_fi);
break;
case '[':
tryAddLabel();
addTok(tok_lbracket_sq);
break;
case ')':
tryAddLabel();
addTok(tok_rbracket);
break;
case '}':
tryAddLabel();
addTok(tok_rbracket_fi);
break;
case ']':
tryAddLabel();
addTok(tok_rbracket_sq);
break;
case '(': addDefTok(tok_lbracket); break;
case '{': addDefTok(tok_lbracket_fi); break;
case '[': addDefTok(tok_lbracket_sq); break;
case ')': addDefTok(tok_rbracket); break;
case '}': addDefTok(tok_rbracket_fi); break;
case ']': addDefTok(tok_rbracket_sq); break;
case '\'':
tryAddLabel();
try(readString('\''), maybeC, ;){
@ -143,37 +131,26 @@ Maybe _lexan(SharedLexerData* sld){
Autoarr_add(tokens, stok);
}
break;
case '<':
tryAddLabel();
addTok(tok_less);
break;
case '>':
tryAddLabel();
addTok(tok_more);
break;
case '+':
tryAddLabel();
addTok(tok_plus);
break;
case '-':
tryAddLabel();
addTok(tok_minus);
break;
case '*':
tryAddLabel();
addTok(tok_asterisk);
break;
case '<': addDefTok(tok_less); break;
case '>': addDefTok(tok_more); break;
case '+': addDefTok(tok_plus); break;
case '-': addDefTok(tok_minus); break;
case '*': addDefTok(tok_asterisk); break;
case '/':
tryAddLabel();
string commentStr={source,0};
string commentStr={source, 1};
c=*++source;
if(c=='/') { // single-line comment
commentStr.length++;
while((c=*++source)){
if(c=='\n' || c=='\r') break;
else commentStr.length++;
}
}
else if(c=='*') { // multi-line comment
commentStr.length++;
while((c=*++source)){
commentStr.length++;
if(c=='*' && *(++source)=='/') break;
@ -181,7 +158,7 @@ Maybe _lexan(SharedLexerData* sld){
}
else { // not comment
source--;
addTok(tok_slash);
addDefTok(tok_slash);
break;
}
Token comTok={
@ -190,71 +167,28 @@ Maybe _lexan(SharedLexerData* sld){
};
Autoarr_add(tokens, comTok);
break;
case '=':
tryAddLabel();
addTok_ifnext('=',tok_equals,tok_assign);
break;
case '!':
tryAddLabel();
addTok_ifnext('=',tok_not_equals,tok_not);
break;
case '&':
tryAddLabel();
addTok_ifnext('&',tok_and_d,tok_and);
break;
case '|':
tryAddLabel();
addTok_ifnext('|',tok_or_d,tok_or);
break;
case '?':
tryAddLabel();
addTok_ifnext('?',tok_question_d,tok_question);
break;
case ':':
tryAddLabel();
addTok(tok_colon);
break;
case ';':
tryAddLabel();
addTok(tok_semicolon);
break;
case '.':
tryAddLabel();
addTok(tok_point);
break;
case ',':
tryAddLabel();
addTok(tok_comma);
break;
case '~':
tryAddLabel();
addTok(tok_tilda);
break;
case '\\':
tryAddLabel();
addTok(tok_backslash);
break;
case '%':
tryAddLabel();
addTok(tok_percent);
break;
case '^':
tryAddLabel();
addTok(tok_xor);
break;
case '$':
tryAddLabel();
addTok(tok_dollar);
break;
case '@':
tryAddLabel();
addTok(tok_at);
break;
case '=': addDefTok_ifnext('=', tok_equals, tok_assign); break;
case '!': addDefTok_ifnext('=', tok_not_equals, tok_not); break;
case '&': addDefTok_ifnext('&', tok_and_d, tok_and); break;
case '|': addDefTok_ifnext('|', tok_or_d, tok_or); break;
case '?': addDefTok_ifnext('?', tok_question_d, tok_question);break;
case ':': addDefTok(tok_colon); break;
case ';': addDefTok(tok_semicolon); break;
case '.': addDefTok(tok_point); break;
case ',': addDefTok(tok_comma); break;
case '~': addDefTok(tok_tilda); break;
case '\\': addDefTok(tok_backslash); break;
case '%': addDefTok(tok_percent); break;
case '^': addDefTok(tok_xor); break;
case '$': addDefTok(tok_dollar); break;
case '@': addDefTok(tok_at); break;
default:
label.length++;
break;
}
}
return SUCCESS(UniHeapPtr(Autoarr(Token), tokens));
}