From 60988206444fe51796a68d31fe88811b69fc2930 Mon Sep 17 00:00:00 2001
From: Timerix22 <timerix11@gmail.com>
Date: Thu, 21 Jul 2022 01:18:55 +0300
Subject: [PATCH] 7

---
 src/copler/main.c       |   5 +-
 src/lexer/lexer.c       | 491 ++++++++++++++++++++--------------------
 src/lexer/lexer.h       |   7 +-
 src/lexer/my_type_ext.h |   7 +
 src/lexer/tokens.c      |  12 +
 src/lexer/tokens.h      |  29 ++-
 6 files changed, 289 insertions(+), 262 deletions(-)
 create mode 100644 src/lexer/my_type_ext.h
 create mode 100644 src/lexer/tokens.c

diff --git a/src/copler/main.c b/src/copler/main.c
index 53bd933..f2d9a40 100644
--- a/src/copler/main.c
+++ b/src/copler/main.c
@@ -1,6 +1,9 @@
 #include "copler.h"
+#include "../lexer/lexer.h"
 
 int main(){
-    
+    init_keywordsSearchTree();
+
+    STNode_free(keywordsSearchTree);
     return 0;
 }
\ No newline at end of file
diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c
index eb18510..aa52791 100644
--- a/src/lexer/lexer.c
+++ b/src/lexer/lexer.c
@@ -1,8 +1,7 @@
 #include "lexer.h"
+#include "../../../kerep/src/String/string.h"
 
-Autoarr_define(Token);
-
-typedef struct SourceFileInfo{
+typedef struct SharedLexerData{
     char* _source;
     char* _filename;
     Autoarr(Token)* _tokens;
@@ -11,42 +10,257 @@ typedef struct SourceFileInfo{
     string _label;
     uint32 _linenum;
     uint32 _charnum;
-} SourceFileInfo;
+} SharedLexerData;
 
-#define source sfi->_source
-#define filename sfi->_filename
-#define tokens sfi->_tokens
-#define context sfi->_context
-#define line sfi->_line
-#define label sfi->_label
-#define linenum sfi->_linenum
-#define charnum sfi->_charnum
+#define source sld->_source
+#define filename sld->_filename
+#define tokens sld->_tokens
+#define context sld->_context
+#define line sld->_line
+#define label sld->_label
+#define linenum sld->_linenum
+#define charnum sld->_charnum
 
-Maybe _throw_wrongchar(SourceFileInfo* sfi){
+Maybe _throw_wrongchar(SharedLexerData* sld){
     char* errline=string_extract(line);
+    char* _context=string_extract(context);
     printf("\n\e[91mwrong char <%c> at [%s:%u:%u %s]\n >>> %s\n",
                 source[charnum], filename,linenum,charnum,context, errline);
     exit(96);
 }
-#define throw_wrongchar(freeMem) { freeMem; _throw_wrongchar(sfi); }
-
-char _charFromEscapeStr(string estr, SourceFileInfo* sfi){
-    if(*estr.ptr!='\\') throw("first char is not \\");
-    switch(*(++estr.ptr)){
-        case 'n': return '\n';
-        case 'r': return '\r';
-        case 't': return '\t';
-        case 'e': return '\e';
-        case '\\': return '\\';
-        case '"': return '"';
-        case '\'': return '\'';
-        default: throw_wrongchar(;);
+#define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(sld); }
+    
+#define addTok(id) Autoarr_add(tokens, default_tokens[id])
+    
+void _addTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
+    if(*(++source)==next){
+        addTok(yes);
+    }
+    else {
+        source--;
+        addTok(no);
     }
 }
-#define charFromEscapeStr(estr) _charFromEscapeStr(estr, sfi)
+#define addTok_ifnext(nextChar, yesTok, noTok) _addTok_ifnext(nextChar, yesTok, noTok, sld)
 
-Autoarr(Token)* lexan(char* _source, char* _filename){
-    SourceFileInfo sfi={
+// adds <label> to <tokens> as tok_label or tok_number
+void _tryAddLabel(SharedLexerData* sld){
+    if(label.length==0) return;
+    Unitype fake_uni=ST_pullString(keywordsSearchTree,label);
+    if(fake_uni.VoidPtr!=NULL) // built-in keyword
+        Autoarr_add(tokens, *(Token*)(void*)&fake_uni);
+    else {          // user-defined label
+        Token ut;
+        ut.value=string_extract(label);
+        switch(*label.ptr){
+            case '0': case '1': case '2': case '3': case '4':
+            case '5': case '6': case '7': case '8': case '9': 
+                ut.id=tok_number;
+                break;
+            default:
+                ut.id=tok_label;
+                break;
+        }
+        Autoarr_add(tokens, ut);
+    }
+    
+    label=(string){source,0};
+};
+#define tryAddLabel() _tryAddLabel(sld)
+
+// returns text in quotes or error
+Maybe _readString(char quotChar, SharedLexerData* sld){
+    char c;
+    bool prevIsBackslash=false;
+    char* srcFirst=source;
+    while ((c=*++source)){
+        if(c==quotChar) {
+            if(prevIsBackslash) // \"
+                prevIsBackslash=false;
+            else { // "
+                string str={srcFirst, source-srcFirst+1};
+                char* extracted=string_extract(str);
+                return SUCCESS(UniPtr(CharPtr, extracted));
+            }
+        } 
+        else prevIsBackslash= c=='\\' && !prevIsBackslash;
+    }
+    source=srcFirst;
+    throw_wrongchar(;);
+}
+#define readString(quotChar) _readString(quotChar, sld)
+
+
+Maybe _lexan(SharedLexerData* sld){
+    char c;
+    source--;
+    while ((c=*++source)) switch(c){
+        case ' ': case '\t':
+        case '\r': case '\n':
+            tryAddLabel();
+            break;
+        case '(':
+            tryAddLabel();
+            addTok(tok_lbracket);
+            break;
+        case '{':
+            tryAddLabel();
+            addTok(tok_lbracket_fi);
+            break;
+        case '[':
+            tryAddLabel();
+            addTok(tok_lbracket_sq);
+            break;
+        case ')':
+            tryAddLabel();
+            addTok(tok_rbracket);
+            break;
+        case '}':
+            tryAddLabel();
+            addTok(tok_rbracket_fi);
+            break;
+        case ']':
+            tryAddLabel();
+            addTok(tok_rbracket_sq);
+            break;
+        case '\'':
+            tryAddLabel();
+            try(readString('\''), maybeC, ;){
+                Token ctok={ 
+                    .id=tok_character,
+                    .value=(char*)maybeC.value.VoidPtr
+                };
+                Autoarr_add(tokens, ctok);
+            }
+            break;
+        case '"':
+            tryAddLabel();
+            try(readString('"'), maybeS, ;){
+                Token stok={ 
+                    .id=tok_string,
+                    .value=(char*)maybeS.value.VoidPtr
+                };
+                Autoarr_add(tokens, stok);
+            }
+            break;
+        case '<':
+            tryAddLabel();
+            addTok(tok_less);
+            break;
+        case '>':
+            tryAddLabel();
+            addTok(tok_more);
+            break;
+        case '+':
+            tryAddLabel();
+            addTok(tok_plus);
+            break;
+        case '-':
+            tryAddLabel();
+            addTok(tok_minus);
+            break;
+        case '*':
+            tryAddLabel();
+            addTok(tok_asterisk);
+            break;
+        case '/':
+            tryAddLabel();
+            string commentStr={source,0};
+            c=*++source;
+            if(c=='/') { // single-line comment
+                while((c=*++source)){
+                    if(c=='\n' || c=='\r') break;
+                    else commentStr.length++;
+                }
+            }
+            else if(c=='*') { // multi-line comment
+                while((c=*++source)){
+                    commentStr.length++;
+                    if(c=='*' && *(++source)=='/') break;
+                }
+            }
+            else { // not comment
+                source--;
+                addTok(tok_slash);
+                break;
+            }
+            Token comTok={
+                .value=string_extract(commentStr),
+                .id=tok_comment
+            };
+            Autoarr_add(tokens,comTok);
+            break;
+        case '=':
+            tryAddLabel();
+            addTok_ifnext('=',tok_equals,tok_assign);
+            break;
+        case '!':
+            tryAddLabel();
+            addTok_ifnext('=',tok_not_equals,tok_not);
+            break;
+        case '&':
+            tryAddLabel();
+            addTok_ifnext('&',tok_and_d,tok_and);
+            break;
+        case '|':
+            tryAddLabel();
+            addTok_ifnext('|',tok_or_d,tok_or);
+            break;
+        case '?':
+            tryAddLabel();
+            addTok_ifnext('?',tok_question_d,tok_question);
+            break;
+        case ':':
+            tryAddLabel();
+            addTok(tok_colon);
+            break;
+        case ';':
+            tryAddLabel();
+            addTok(tok_semicolon);
+            break;
+        case '.':
+            tryAddLabel();
+            addTok(tok_point);
+            break;
+        case ',':
+            tryAddLabel();
+            addTok(tok_comma);
+            break;
+        case '~':
+            tryAddLabel();
+            addTok(tok_tilda);
+            break;
+        case '\\':
+            tryAddLabel();
+            addTok(tok_backslash);
+            break;
+        case '%':
+            tryAddLabel();
+            addTok(tok_percent);
+            break;
+        case '^':
+            tryAddLabel();
+            addTok(tok_xor);
+            break;
+        case '$':
+            tryAddLabel();
+            addTok(tok_dollar);
+            break;
+        case '@':
+            tryAddLabel();
+            addTok(tok_at);
+            break;
+        default:
+            label.length++;
+            break;
+    }
+    
+    return SUCCESS(UniPtr(AutoarrTokenPtr,tokens));
+}
+
+
+Maybe lexan(char* _source, char* _filename){
+    SharedLexerData sld={
         ._source=_source,
         ._filename=_filename,
         ._tokens=Autoarr_create(Token,64,1024),
@@ -55,220 +269,5 @@ Autoarr(Token)* lexan(char* _source, char* _filename){
         ._linenum=0,
         ._charnum=0
     };
-    return _lexan(&sfi);
-}
-
-Autoarr(Token)* _lexan(SourceFileInfo* sfi){
-    
-    void addlabel(){
-        if(label.length==0) return;
-        Unitype token_ptr_u=ST_pull_str(label);
-        // user-defined label
-        if(token_ptr_u.type==Null){
-            Token ut={
-                .id=tok_label,
-                .value=string_cpToCharPtr(label)
-            };
-            switch(*label.ptr){
-                case '0': case '1': case '2': case '3': case '4':
-                case '5': case '6': case '7': case '8': case '9': 
-                    ut.id=tok_number;
-                    break;
-                default:
-                    break;
-            }
-            Autoarr_add(tokens, ut);
-        }
-        // built-in keyword
-        else Autoarr_add(tokens, (*token_ptr_u.VoidPtr));
-        
-        label={source,0};
-    };
-    
-    
-    addtok(TokenId id){
-       Autoarr_add(tokens, default_tokens[id]);
-    }
-    
-    
-    addtok_ifnext(char next, TokenId yes, TokenId no){
-        if(*(source+1)==next){
-             addtok(yes);
-             source++;
-        }
-        else addtok(no);
-    }
-    
-    
-    while ((c=*source++)) switch(c){
-        case ' ': case '\t':
-        case '\r': case '\n':
-                addlabel();
-                break;
-        case '(':
-            addlabel();
-            addtok(tok_lbracket);
-            break;
-        case '{':
-            addlabel();
-            addtok(tok_lbracket_fi);
-            break;
-        case '[':
-            addlabel();
-            addtok(tok_lbracket_sq);
-            break;
-        case ')':
-            addlabel();
-            addtok(tok_rbracket);
-            break;
-        case '}':
-            addlabel();
-            addtok(tok_rbracket_fi);
-            break;
-        case ']':
-            addlabel();
-            addtok(tok_rbracket_sq);
-            break;
-        case '\'':
-            addlabel();
-            Token ctok={ 
-                .id=tok_char ,
-                .value={++source,1}
-            };
-            if(*source=='\\'){
-                ctok.value=malloc(1);
-                *ctok.value=charFromEscapeStr(source++,2);
-            }
-            if(*(++source)!='\'')
-                throw_wrongchar();
-            Autoarr_add(tokens, ctok);
-            break;
-        case '"':
-            addlabel();
-            
-            
-            break;
-        case '<':
-            addlabel();
-            addtok(tok_less);
-            break;
-        case '>':
-            addlabel();
-            addtok(tok_more);
-            break;
-        case '+':
-            addlabel();
-            addtok(tok_plus);
-            break;
-        case '-':
-            addlabel();
-            addtok(tok_minus);
-            break;
-        case '*':
-            addlabel();
-            addtok(tok_asterisk);
-            break;
-        case '/':
-            addlabel();
-            if(*(++source)=='/'){
-                label={++source,0};
-                    while((c=*(source++))
-                        if(c=='\n'){
-                            // removes \r from the end of comment line
-                            if(*(source-2)=='\r')
-                                label.length--;
-                            goto add_comment;
-                        }
-                        else label.length++;
-                    add_comment:
-                    Token comt={
-                        .id=tok_comment,
-                        .value=label
-                    }
-                    Autoarr_add(tokens,comt);
-            }
-            else if(*source=='*'){
-                label={++source,0};
-                    while((c=*(source++))
-                        if(c=='*'&&*(source++)=='/')
-                            goto add_comment;
-                        else label.length++;
-                    add_comment:
-                    Token comt={
-                        .id=tok_comment,
-                        .value=label
-                    }
-                    Autoarr_add(tokens,comt);
-            }
-            else{
-                source--;
-                addtok(tok_slash);
-            }
-            break;
-        case '=':
-            addlabel();
-            addtok_ifnext('=',tok_equals,tok_assign);
-            break;
-        case '!':
-            addlabel();
-addtok_ifnext('=',tok_not_equals,tok_not);
-            break;
-        case '&':
-            addlabel();
-            addtok_ifnext('&',tok_,tok_);
-            break;
-        case '|':
-            addlabel();
-            addtok_ifnext('|',tok_or,tok_or_d);
-            break;
-        case '?':
-            addlabel();
-            addtok_ifnext('?',tok_question,tok_question_d);
-            break;
-        case ':':
-            addlabel();
-            addtok(tok_colon);
-            break;
-        case ';':
-            addlabel();
-            addtok(tok_semicolon);
-            break;
-        case '.':
-            addlabel();
-            addtok(tok_point);
-            break;
-        case ',':
-            addlabel();
-            addtok(tok_comma);
-            break;
-        case '~':
-            addlabel();
-            addtok(tok_tilda);
-            break;
-        case '\\':
-            addlabel();
-            addtok(tok_backslash);
-            break;
-        case '%':
-            addlabel();
-            addtok(tok_percent);
-            break;
-        case '^':
-            addlabel();
-            addtok(tok_xor);
-            break;
-        case '$':
-            addlabel();
-            addtok(tok_dollar);
-            break;
-        case '@':
-            addlabel();
-            addtok(tok_at);
-            break;
-        default:
-            label.length++;
-            break;
-    }
-    
-    return tokens;
-}
+    return _lexan(&sld);
+}
\ No newline at end of file
diff --git a/src/lexer/lexer.h b/src/lexer/lexer.h
index ea9807b..3db5159 100644
--- a/src/lexer/lexer.h
+++ b/src/lexer/lexer.h
@@ -1,9 +1,6 @@
 #pragma once
 
-#include "../../../kerep/src/Autoarr/Autoarr.h"
-#include "../../../kerep/src/String/string.h"
 #include "tokens.h"
 
-Autoarr_declare(Token)
-
-Autoarr(Token)* lexan(char* source, char* filename);
+//Autoarr(Token)*
+Maybe lexan(char* source, char* filename);
diff --git a/src/lexer/my_type_ext.h b/src/lexer/my_type_ext.h
new file mode 100644
index 0000000..748d859
--- /dev/null
+++ b/src/lexer/my_type_ext.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include "../../../kerep/src/base/base.h"
+
+typedef enum lexer_type{
+    AutoarrTokenPtr=my_type_last+1
+} lexer_type;
\ No newline at end of file
diff --git a/src/lexer/tokens.c b/src/lexer/tokens.c
new file mode 100644
index 0000000..4c0b2d0
--- /dev/null
+++ b/src/lexer/tokens.c
@@ -0,0 +1,12 @@
+#include "tokens.h"
+
+Autoarr_define(Token)
+
+void init_keywordsSearchTree(){
+    keywordsSearchTree=STNode_create();
+    for(TokenId keywordId=0; keywordId<=tok_typeof; keywordId++){
+        Token keyword=default_tokens[keywordId];
+        Unitype fake_uni=*(Unitype*)(void*)&keyword;
+        ST_push(keywordsSearchTree, keyword.value, fake_uni);
+    }
+}
diff --git a/src/lexer/tokens.h b/src/lexer/tokens.h
index c5e5879..7d01fc1 100644
--- a/src/lexer/tokens.h
+++ b/src/lexer/tokens.h
@@ -1,6 +1,8 @@
 #pragma once
 
-#include "../../../kerep/src/base/base.h"
+#include "../../../kerep/src/Autoarr/Autoarr.h"
+#include "../../../kerep/src/SearchTree/SearchTree.h"
+#include "my_type_ext.h"
 
 typedef enum TokenId{
     // base types
@@ -35,6 +37,7 @@ typedef enum TokenId{
     tok_return,
     tok_goto,
     // declaration keywords
+    tok_class,
     tok_struct,
     tok_enum,
     tok_union,
@@ -55,12 +58,6 @@ typedef enum TokenId{
     tok_new, // allocates struct in heap
     tok_sizeof, // size of variable value
     tok_typeof, // type of variable
-    // user-defined
-    tok_label,
-    tok_number,
-    tok_character,
-    tok_string,
-    tok_comment,
     // special characters
     tok_lbracket,    // (
     tok_lbracket_fi, // {
@@ -68,8 +65,6 @@ typedef enum TokenId{
     tok_rbracket,    // )
     tok_rbracket_fi, // }
     tok_rbracket_sq, // ]
-    //tok_quot,        // '
-    //tok_quot_d,      // "
     tok_less,        // <
     tok_more,        // >
     tok_plus,        // +
@@ -96,7 +91,13 @@ typedef enum TokenId{
     tok_xor,         // ^
     tok_lattice,     // #
     tok_dollar,      // $
-    tok_at           // @
+    tok_at,           // @
+    // user-defined
+    tok_label,
+    tok_number,
+    tok_character,
+    tok_string,
+    tok_comment
 } __attribute__((__packed__)) TokenId;
 
 typedef struct Token{
@@ -104,6 +105,11 @@ typedef struct Token{
     TokenId id;
 } Token;
 
+static STNode* keywordsSearchTree;
+// dont forget to free it
+void init_keywordsSearchTree();
+
+
 static const Token default_tokens[]={
     // base types
     {"void", tok_void_t},
@@ -137,6 +143,7 @@ static const Token default_tokens[]={
     {"return", tok_return},
     {"goto", tok_goto},
     // declaration keywords
+    {"class", tok_class},
     {"struct", tok_struct},
     {"enum", tok_enum},
     {"union", tok_union},
@@ -192,3 +199,5 @@ static const Token default_tokens[]={
     {"$", tok_dollar},
     {"@", tok_at}
 };
+
+Autoarr_declare(Token)