Timerix/cobek
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lexer changes

Browse Source
This commit is contained in:
timerix 2023-02-10 13:00:29 +06:00
parent 69449891e1
commit f5b4d6c7ee
1 changed files with 66 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
src/lexer/lexer.c
View File

@ -30,22 +30,10 @@ Maybe _throw_wrongchar(SharedLexerData* sld){
} }
#define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(sld); } #define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(sld); }
#define addTok(id) Autoarr_add(tokens, default_tokens[id])
void _addTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
if(*(++source)==next){
addTok(yes);
}
else {
source--;
addTok(no);
}
}
#define addTok_ifnext(nextChar, yesTok, noTok) _addTok_ifnext(nextChar, yesTok, noTok, sld)
// adds <label> to <tokens> as tok_label or tok_number // adds <label> to <tokens> as tok_label or tok_number
void _tryAddLabel(SharedLexerData* sld){ void _tryAddLabel(SharedLexerData* sld){
if(label.length==0) return; if(label.length==0) return;
Unitype uni=ST_pullString(keywordsSearchTree, label); Unitype uni=ST_pullString(keywordsSearchTree, label);
if(uni.VoidPtr!=NULL) // built-in keyword if(uni.VoidPtr!=NULL) // built-in keyword
Autoarr_add(tokens, *(Token*)uni.VoidPtr); Autoarr_add(tokens, *(Token*)uni.VoidPtr);
@ -68,6 +56,17 @@ void _tryAddLabel(SharedLexerData* sld){
}; };
#define tryAddLabel() _tryAddLabel(sld) #define tryAddLabel() _tryAddLabel(sld)
#define addDefTok(id) tryAddLabel(); Autoarr_add(tokens, default_tokens[id])
void _addDefTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
if(*(source+1)==next){
source++;
addDefTok(yes);
}
else addDefTok(no);
}
#define addDefTok_ifnext(nextChar, yesTok, noTok) _addDefTok_ifnext(nextChar, yesTok, noTok, sld)
// returns text in quotes or error // returns text in quotes or error
Maybe _readString(char quotChar, SharedLexerData* sld){ Maybe _readString(char quotChar, SharedLexerData* sld){
char c; char c;
@ -94,35 +93,24 @@ Maybe _readString(char quotChar, SharedLexerData* sld){
Maybe _lexan(SharedLexerData* sld){ Maybe _lexan(SharedLexerData* sld){
char c; char c;
source--; source--;
while ((c=*++source)) switch(c){ while ((c=*++source)) {
// tryAddLabel() resets label.length, but label.ptr still pointing to prev token beginning
if(label.length==0)
label.ptr=source;
switch(c){
case ' ': case '\t': case ' ': case '\t':
case '\r': case '\n': case '\r': case '\n':
tryAddLabel(); tryAddLabel();
break; break;
case '(':
tryAddLabel(); case '(': addDefTok(tok_lbracket); break;
addTok(tok_lbracket); case '{': addDefTok(tok_lbracket_fi); break;
break; case '[': addDefTok(tok_lbracket_sq); break;
case '{': case ')': addDefTok(tok_rbracket); break;
tryAddLabel(); case '}': addDefTok(tok_rbracket_fi); break;
addTok(tok_lbracket_fi); case ']': addDefTok(tok_rbracket_sq); break;
break;
case '[':
tryAddLabel();
addTok(tok_lbracket_sq);
break;
case ')':
tryAddLabel();
addTok(tok_rbracket);
break;
case '}':
tryAddLabel();
addTok(tok_rbracket_fi);
break;
case ']':
tryAddLabel();
addTok(tok_rbracket_sq);
break;
case '\'': case '\'':
tryAddLabel(); tryAddLabel();
try(readString('\''), maybeC, ;){ try(readString('\''), maybeC, ;){
@ -143,37 +131,26 @@ Maybe _lexan(SharedLexerData* sld){
Autoarr_add(tokens, stok); Autoarr_add(tokens, stok);
} }
break; break;
case '<':
tryAddLabel(); case '<': addDefTok(tok_less); break;
addTok(tok_less); case '>': addDefTok(tok_more); break;
break; case '+': addDefTok(tok_plus); break;
case '>': case '-': addDefTok(tok_minus); break;
tryAddLabel(); case '*': addDefTok(tok_asterisk); break;
addTok(tok_more);
break;
case '+':
tryAddLabel();
addTok(tok_plus);
break;
case '-':
tryAddLabel();
addTok(tok_minus);
break;
case '*':
tryAddLabel();
addTok(tok_asterisk);
break;
case '/': case '/':
tryAddLabel(); tryAddLabel();
string commentStr={source,0}; string commentStr={source, 1};
c=*++source; c=*++source;
if(c=='/') { // single-line comment if(c=='/') { // single-line comment
commentStr.length++;
while((c=*++source)){ while((c=*++source)){
if(c=='\n' || c=='\r') break; if(c=='\n' || c=='\r') break;
else commentStr.length++; else commentStr.length++;
} }
} }
else if(c=='*') { // multi-line comment else if(c=='*') { // multi-line comment
commentStr.length++;
while((c=*++source)){ while((c=*++source)){
commentStr.length++; commentStr.length++;
if(c=='*' && *(++source)=='/') break; if(c=='*' && *(++source)=='/') break;
@ -181,7 +158,7 @@ Maybe _lexan(SharedLexerData* sld){
} }
else { // not comment else { // not comment
source--; source--;
addTok(tok_slash); addDefTok(tok_slash);
break; break;
} }
Token comTok={ Token comTok={
@ -190,71 +167,28 @@ Maybe _lexan(SharedLexerData* sld){
}; };
Autoarr_add(tokens, comTok); Autoarr_add(tokens, comTok);
break; break;
case '=':
tryAddLabel(); case '=': addDefTok_ifnext('=', tok_equals, tok_assign); break;
addTok_ifnext('=',tok_equals,tok_assign); case '!': addDefTok_ifnext('=', tok_not_equals, tok_not); break;
break; case '&': addDefTok_ifnext('&', tok_and_d, tok_and); break;
case '!': case '|': addDefTok_ifnext('|', tok_or_d, tok_or); break;
tryAddLabel(); case '?': addDefTok_ifnext('?', tok_question_d, tok_question);break;
addTok_ifnext('=',tok_not_equals,tok_not); case ':': addDefTok(tok_colon); break;
break; case ';': addDefTok(tok_semicolon); break;
case '&': case '.': addDefTok(tok_point); break;
tryAddLabel(); case ',': addDefTok(tok_comma); break;
addTok_ifnext('&',tok_and_d,tok_and); case '~': addDefTok(tok_tilda); break;
break; case '\\': addDefTok(tok_backslash); break;
case '|': case '%': addDefTok(tok_percent); break;
tryAddLabel(); case '^': addDefTok(tok_xor); break;
addTok_ifnext('|',tok_or_d,tok_or); case '$': addDefTok(tok_dollar); break;
break; case '@': addDefTok(tok_at); break;
case '?':
tryAddLabel();
addTok_ifnext('?',tok_question_d,tok_question);
break;
case ':':
tryAddLabel();
addTok(tok_colon);
break;
case ';':
tryAddLabel();
addTok(tok_semicolon);
break;
case '.':
tryAddLabel();
addTok(tok_point);
break;
case ',':
tryAddLabel();
addTok(tok_comma);
break;
case '~':
tryAddLabel();
addTok(tok_tilda);
break;
case '\\':
tryAddLabel();
addTok(tok_backslash);
break;
case '%':
tryAddLabel();
addTok(tok_percent);
break;
case '^':
tryAddLabel();
addTok(tok_xor);
break;
case '$':
tryAddLabel();
addTok(tok_dollar);
break;
case '@':
tryAddLabel();
addTok(tok_at);
break;
default: default:
label.length++; label.length++;
break; break;
} }
}
return SUCCESS(UniHeapPtr(Autoarr(Token), tokens)); return SUCCESS(UniHeapPtr(Autoarr(Token), tokens));
} }