submodules updated

.gitignore

@@ -1,12 +1,16 @@
-# Build results
+# build results
 bin/
 obj/
+*.log
+*.tmp
 
-# user files
-.old*/
+# IDE files
 .vs/
 .vshistory/
 .editorconfig
 *.user
 *.vcxproj.filters
+
+# other files
+.old*/
 current.config

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "cbuild"]
 	path = cbuild
 	url = https://github.com/Timerix22/cbuild.git
+[submodule "GraphC"]
+	path = GraphC
+	url = https://github.com/Timerix22/GraphC.git

.vscode/launch.json

@@ -1,52 +1,50 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "(gdb) Debug",
+            "name": "gdb_debug_unix",
             "type": "cppdbg",
             "request": "launch",
-            "preLaunchTask": "build_cb2c_dbg",
-            "program": "bin/cb2c.com",
-            "args": [],
+            "program": "${workspaceFolder}/bin/cobek",
+            "preLaunchTask": "build_exec_dbg",
             "stopAtEntry": false,
-            "cwd": "${workspaceFolder}",
-            "environment": [],
+            "cwd": "${workspaceFolder}/bin",
             "externalConsole": false,
-            "internalConsoleOptions": "openOnSessionStart",
-            "pipeTransport": {
-                "debuggerPath": "gdb",
-                "pipeProgram": "bash",
-                "pipeArgs": ["-c"],
-                "pipeCwd": ""
-            },
+            "MIMode": "gdb",
+            "miDebuggerPath": "gdb",
             "setupCommands": [
                 {
-                    "description": "Enable pretty-printing for gdb",
                     "text": "-enable-pretty-printing",
                     "ignoreFailures": true
                 },
                 {
-                    "description":  "Set Disassembly Flavor to Intel",
                     "text": "-gdb-set disassembly-flavor intel",
                     "ignoreFailures": true
                 }
             ]
         },
-        
+        // the only difference is .exe
         {
-            "name": "(Windows) Debug",
-            "type": "cppvsdbg",
+            "name": "gdb_debug_win",
+            "type": "cppdbg",
             "request": "launch",
-            "preLaunchTask": "build_cb2c_dbg",
-            "cwd": "${workspaceRoot}\\bin",
-            "program": "${workspaceRoot}\\bin\\cb2c.com",
-            "args": [],
-            "environment": [],
+            "program": "${workspaceFolder}/bin/cobek.exe",
+            "preLaunchTask": "build_exec_dbg",
             "stopAtEntry": false,
-            "console": "integratedTerminal"
+            "cwd": "${workspaceFolder}/bin",
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "miDebuggerPath": "gdb",
+            "setupCommands": [
+                {
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "text": "-gdb-set disassembly-flavor intel",
+                    "ignoreFailures": true
+                }
+            ]
         }
     ]
 }

.vscode/tasks.json

@@ -1,41 +1,13 @@
-
 {
     
     "version": "2.0.0",
     "tasks": [
       {
-        "label": "build",
-        "detail": "build project",
-        "type": "cppbuild",
-        "command": "make",
-        "args": [
-            "build"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}"
-        },
-        "problemMatcher": ["$gcc"],
-        "group": {
-          "kind": "build",
-          "isDefault": true
-        },
-        "presentation": {
-          "echo": true,
-          "reveal": "always",
-          "focus": true,
-          "panel": "shared",
-          "showReuseMessage": false,
-          "clear": true
-        }
-      },
-
-      {
-        "label": "build_cb2c_dbg",
+        "label": "build_exec_dbg",
         "detail": "build project with debug symbols",
         "type": "cppbuild",
         "command": "make",
         "args": [
-            "rebuild_kerep",
             "build_exec_dbg"
         ],
         "options": {

GraphC

@@ -0,0 +1 @@
+Subproject commit 497bd5144e17eca61b8c73fb73b7354e7955e842

Makefile

@@ -1,28 +1,65 @@
-######      Build cbuild/default_tasks     #######
+######################################
+######       Build tasks       #######
+######################################
 
-all: build_exec
+all: build_exec_dbg
 
-rebuild_kerep:
-	@cbuild/call_task.sh rebuild_kerep
+# creates executable using profiling info generated by profile
+build_exec: profile
+	@cbuild/call_task.sh build_exec  2>&1 | tee -a make_raw.log
 
-build_exec: 
-	@cbuild/call_task.sh build_exec
-build_exec_dbg: 
-	@cbuild/call_task.sh build_exec_dbg
+# creates executable with debug info and no optimizations
+build_exec_dbg:
+	@cbuild/call_task.sh build_exec_dbg 2>&1 | tee make_raw.log
 
-build_shared_lib: 
-	@cbuild/call_task.sh build_shared_lib
-build_shared_lib_dbg: 
-	@cbuild/call_task.sh build_shared_lib_dbg
+######################################
+######       Launch tasks      #######
+######################################
 
-build_static_lib: 
-	@cbuild/call_task.sh build_static_lib
-build_static_lib_dbg: 
-	@cbuild/call_task.sh build_static_lib_dbg
-
-######      Launch cbuild/default_tasks      #######
+# executes $EXEC_FILE
 exec: build_exec
-	@cbuild/call_task.sh exec
+	@cbuild/call_task.sh exec 2>&1 | tee -a make_raw.log
 
+# executes $EXEC_FILE
+exec_dbg: build_exec_dbg
+	@cbuild/call_task.sh exec 2>&1 | tee -a make_raw.log
+
+# executes $EXEC_FILE with valgrind memory checker
 valgrind: build_exec_dbg
-	@cbuild/call_task.sh 
+	@cbuild/call_task.sh valgrind 2>&1 | tee -a make_raw.log
+
+# generates profiling info
+profile:
+	@cbuild/call_task.sh profile 2>&1 | tee make_raw.log
+
+# compiles program with -pg and runs it with gprof
+# uses gprof2dot python script to generate function call tree  
+gprof:
+	@cbuild/call_task.sh gprof 2>&1 | tee make_raw.log
+	
+# compiles executable with sanitizers and executes it to find errors and warnings
+sanitize:
+	@cbuild/call_task.sh sanitize 2>&1 | tee make_raw.log
+
+######################################
+######       Other tasks       #######
+######################################
+
+# deletes generated files
+clean:
+	@cbuild/call_task.sh clean 2>&1 | tee make_raw.log
+
+# removes all unreadable characters copied from stdio
+fix_log:
+	sed 's/[^[:blank:][:print:]]//g' make_raw.log \
+		| sed 's/\[0;[0-9][0-9]m//g' \
+		| sed 's/\[0;[0-9]m//g' \
+		| sed 's/\[[0-9][0-9]m//g' \
+		| sed 's/\[[0-9]m//g' \
+		| sed 's/  H  //g' \
+		| sed 's/\[3gH  //g' \
+		> make_fixed.log
+
+# recompile kerep.a in the next build task
+rebuild_kerep: 
+	@cbuild/call_task.sh rebuild_kerep

cbuild

@@ -1 +1 @@
-Subproject commit 15a9661c467facf65b2155eab3fad1ecf5f0c945
+Subproject commit d72aa0c8098fe2a427e50355250f473d70886502

default.config

@@ -1,54 +1,123 @@
 #!/bin/bash
-CBUILD_VERSION=4
-CONFIG_VERSION=5
+CBUILD_VERSION=6
+CONFIG_VERSION=7
 
 PROJECT=cobek
 CMP_C=gcc
 CMP_CPP=g++
 STD_C=c11
-STD_CPP=c++17
-WARN_C="-Wall -Wno-discarded-qualifiers"
-WARN_CPP="-Wall"
+STD_CPP=c++11
+WARN_C="-Wall -Wno-discarded-qualifiers -Wextra -Wno-unused-parameter"
+WARN_CPP="-Wall -Wextra -Wno-unused-parameter"
 SRC_C="$(    find src -name '*.c')"
 SRC_CPP="$(  find src -name '*.cpp')"
 #TESTS_C="$(  find tests -name '*.c')"
 #TESTS_CPP="$(find tests -name '*.cpp')"
 
-OUTDIR=bin
-OBJDIR=obj
-EXEC_FILE=cb2c.com
+# OBJDIR structure:
+#  ├── objects  - dir where compiled *.o files are stored. cleans every call of build task
+#  ├── profile  - dir where gcc *.gcda profiling info files stored
+#  ├── libs     - there you can put static libs and linker will find them
+#  └── out      - output files are created here and then copied to OUTDIR
+OBJDIR="obj"
+OUTDIR="bin"
 
-case $TASK in
-    rebuild_kerep)
-        TASK_SCRIPT=tasks/rebuild_kerep.sh 
+# OS-specific options
+case "$OS" in
+    WINDOWS)
+        EXEC_FILE="$PROJECT.exe"
         ;;
+    LINUX)
+        EXEC_FILE="$PROJECT"
+        ;;
+    *)
+        error "operating system $OS has no configuration variants"
+        ;;
+esac
+
+# TASKS
+case "$TASK" in
+    # recompile kerep.a in the next build task
+    rebuild_kerep)
+        TASK_SCRIPT=tasks/rebuild_kerep.sh
+        ;;
+    # creates executable using profiling info if it exists
     build_exec)
-        C_ARGS="-O2"
+        # -flto applies more optimizations across object files
+        # -flto=auto is needed to multithreaded copilation
+        # -fuse-linker-plugin is required to use static libs with lto
+        # -fprofile-use enables compiler to use profiling info files to optimize executable
+        # -fprofile-prefix-path sets path where profiling info about objects are be saved
+        C_ARGS="-O2 -flto=auto -fuse-linker-plugin -fprofile-use -fprofile-prefix-path=$(realpath $OBJDIR)/objects"
         CPP_ARGS="$C_ARGS"
-        LINKER_ARGS=""
-        TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
+        LINKER_ARGS="$CPP_ARGS"
         PRE_TASK_SCRIPT=tasks/pre_build.sh
-        POST_TASK_SCRIPT=""
+        TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
         KEREP_BUILD_TASK=build_static_lib
         ;;
+    # creates executable with debug info and no optimizations
     build_exec_dbg)
-        C_ARGS="-O0 -g"
+        C_ARGS="-O0 -g3"
         CPP_ARGS="$C_ARGS"
-        LINKER_ARGS=""
-        TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
+        LINKER_ARGS="$CPP_ARGS"
         PRE_TASK_SCRIPT=tasks/pre_build.sh
-        POST_TASK_SCRIPT=""
+        TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
         KEREP_BUILD_TASK=build_static_lib_dbg
         ;;
+    # executes $EXEC_FILE
     exec)
         TASK_SCRIPT=cbuild/default_tasks/exec.sh
-        PRE_TASK_SCRIPT=""
-        POST_TASK_SCRIPT=""
         ;;
+    # executes $EXEC_FILE with valgrind memory checker
     valgrind)  
         VALGRIND_ARGS="-s --log-file=valgrind.log --read-var-info=yes --track-origins=yes --fullpath-after=$PROJECT/ --leak-check=full --show-leak-kinds=all" 
         TASK_SCRIPT=cbuild/default_tasks/valgrind.sh
-        PRE_TASK_SCRIPT=""
-        POST_TASK_SCRIPT=""
         ;;
+    # generates profiling info
+    profile)
+        OUTDIR="$OUTDIR/profile"
+        # -flto applies more optimizations across object files
+        # -flto=auto is needed to multithreaded copilation
+        # -fuse-linker-plugin is required to use static libs with lto
+        # -pg adds code to executable, that generates file containing function call info (gmon.out)
+        # -fprofile-generate generates executable with profiling code
+        # -fprofile-prefix-path sets path where profiling info about objects will be saved
+        C_ARGS="-O2 -flto=auto -fuse-linker-plugin -fprofile-generate -fprofile-prefix-path=$(realpath $OBJDIR)/objects"
+        CPP_ARGS="$C_ARGS"
+        LINKER_ARGS="$CPP_ARGS"
+        PRE_TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
+        TASK_SCRIPT=cbuild/default_tasks/profile.sh
+        POST_TASK_SCRIPT=
+        ;;
+    # compiles program with -pg and runs it with gprof
+    # uses gprof2dot python script to generate function call tree (pip install gprof2dot)
+    # requires graphviz (https://www.graphviz.org/download/source/)
+    gprof)
+        OUTDIR="$OUTDIR/gprof"
+        # -pg adds code to executable, that generates file containing function call info (gmon.out)
+        C_ARGS="-O2 -flto=auto -fuse-linker-plugin -pg"
+        CPP_ARGS="$C_ARGS"
+        LINKER_ARGS="$CPP_ARGS"
+        PRE_TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
+        TASK_SCRIPT=cbuild/default_tasks/gprof.sh
+        POST_TASK_SCRIPT=
+        ;;
+    # compiles executable with sanitizers and executes it to find errors and warnings
+    sanitize)
+        OUTDIR="$OUTDIR/sanitize"
+        C_ARGS="-O0 -g3 -fsanitize=undefined,address"
+        CPP_ARGS="$C_ARGS"
+        LINKER_ARGS="$CPP_ARGS"
+        PRE_TASK_SCRIPT=cbuild/default_tasks/build_exec.sh
+        TASK_SCRIPT=cbuild/default_tasks/exec.sh
+        POST_TASK_SCRIPT=
+        ;;
+    # deletes generated files
+    clean)
+        TASK_SCRIPT=cbuild/default_tasks/clean.sh
+        ;;
+    # unknown task
+    *)
+        error "task <$TASK> not found"
+    ;;
 esac

kerep

@@ -1 +1 @@
-Subproject commit 40d1828f2cfa68e03cccaa2b3658eba07c146b02
+Subproject commit 20537813ecbd08041318e832d645a982c09ac7a1

src/cb2c/cb2c.c

@@ -1,25 +1,80 @@
 #include "cb2c.h"
 #include "../../kerep/src/String/StringBuilder.h"
 
-void namespaceToC(NamespaceContext* context, StringBuilder* b){
-    
-}
+#define addc(C) StringBuilder_append_char(b, C);
+#define adds(S) StringBuilder_append_cptr(b, S);
 
-void functionToC(FunctionContext* context, StringBuilder* b){
-    
-}
+void appendToken(StringBuilder* b, LLNode(Token)* tokNode, u16* _tab_count){
+    u16 tab_count=*_tab_count;
+    adds(tokNode->value.value)
 
-void classToC(ClassContext* context, StringBuilder* b){
-    
-}
-
-char* contextToC(void* context){
-    StringBuilder* b=StringBuilder_create();
-    switch(((Context*)context)->type){
-        case CT_Namespace: namespaceToC(context, b); break;
-        case CT_Function: functionToC(context, b); break;
-        case CT_Class: classToC(context, b); break;
-        default: throw(ERR_WRONGTYPE);
+    switch(tokNode->value.id){
+        case tok_lbracket_fi:
+            tab_count++;
+            goto add_new_line;
+        case tok_rbracket_fi:
+            tab_count--;
+            goto add_new_line;
+        case tok_semicolon:
+        case tok_colon:
+        case tok_comment: ;
+            add_new_line:
+            addc('\n');
+            for(u16 i=0; i<tab_count; i++)
+                addc('\t');
+            break;
+        case tok_plus: case tok_minus: case tok_asterisk: case tok_slash:
+        case tok_assign: case tok_equal: case tok_not: case tok_not_equal:
+        case tok_and: case tok_and_d: case tok_or: case tok_or_d:
+        case tok_less: case tok_more: case tok_percent:
+        case tok_for: case tok_if: case tok_while: case tok_switch:
+        case tok_lbracket: case tok_rbracket: case tok_lbracket_sq: case tok_rbracket_sq:
+        case tok_break: case tok_backslash:
+            break;
+        default:
+            addc(' ');
     }
-    return StringBuilder_build(b).ptr;
+
+    *_tab_count=tab_count;
+}
+
+Maybe appendNamespaceContext(StringBuilder* b, NamespaceContext* context){
+    adds("/* context ")
+    if(context->base.namespace!=NULL){
+        adds(context->base.namespace)
+        addc('.')
+    }
+    adds(context->base.name)
+    adds(" */\n\n")
+
+    u16 tab_count=0;
+    LinkedList_foreach(context->base.tokens, tok, appendToken(b, tok, &tab_count));
+    addc('\n');
+
+    return MaybeNull;
+}
+
+Maybe appendClassContext(StringBuilder* b, ClassContext* context){
+    safethrow(ERR_NOTIMPLEMENTED,;);
+}
+
+Maybe appendFunctionContext(StringBuilder* b, FunctionContext* context){
+    safethrow(ERR_NOTIMPLEMENTED,;);
+}
+
+Maybe translateToC(Context* context){
+    StringBuilder* b=StringBuilder_create();
+    switch(context->type){
+        case ContextType_Namespace:
+            try(appendNamespaceContext(b, (NamespaceContext*)context), _0, StringBuilder_free(b));
+            break;
+        case ContextType_Class:
+            try(appendClassContext(b, (ClassContext*)context), _2, StringBuilder_free(b));
+            break;
+        case ContextType_Function:
+            try(appendFunctionContext(b, (FunctionContext*)context), _1, StringBuilder_free(b));
+            break;
+        default: safethrow(ERR_WRONGTYPE, StringBuilder_free(b));
+    }
+    return SUCCESS(UniHeapPtr(char, StringBuilder_build(b).ptr));
 }

src/cb2c/cb2c.h

@@ -2,4 +2,7 @@
 
 #include "../lexer/lexer.h"
 
-char* contextToC(void* context);
+/// @brief generates C source code from tokens
+/// @param context contains code tokens
+/// @return Maybe<char*> C source code 
+Maybe translateToC(Context* context);

src/cb2c/main.c

@@ -1,17 +1,157 @@
-#include "cb2c.h"
 #include "../../kerep/src/kprint/kprint.h"
+#include "../../kerep/src/Filesystem/filesystem.h"
+#include "cb2c.h"
+
+char cb2c_version[]="0.0.1";
+
+i32 main(const int argc, const char* const* argv){
+    if(setlocale(LC_CTYPE, "C.UTF-8")!=0)
+        kprintf("\e[93msetlocale failed\e[37m\n");
+
+    Autoarr(Pointer)* source_files=Autoarr_create(Pointer, 64, 64);
+    bool  compile=false;
+    char* compiler=NULL;
+    char* compilation_out_dir=NULL;
+    bool  translate=false;
+    char* translation_traget=NULL; // single .c + single .h; namespace structured project
+    char* translation_out_dir=NULL;
+
+    // command arguments
+    if(argc==1){
+        kprintf("\e[91mno arguments\e[37m\n");
+        return -1;
+    }
+    for(int argi=1; argi<argc; argi++){
+        char* a=argv[argi];
+        kprintf("%s\n", a);
+
+        if(cptr_compare(a, "-v") || cptr_compare(a, "--version")){
+            kprintf("  cb2c v%s\n", cb2c_version);
+            return 0;
+        }
+
+        if(cptr_compare(a, "-c") || cptr_compare(a, "--compiler")){
+            compile=true;
+            kprintf("  compile=true\n");
+
+            if(argi+1>=argc || argv[argi+1][0]=='-')
+                throw("after argument --compiler must be compiler name");
+            
+            compiler=argv[++argi];
+            kprintf("  compiler=%s\n", compiler);
+        }
+
+        else if(cptr_compare(a, "-t") || cptr_compare(a, "--translate")){
+            translate=true;
+            kprintf("  translate=true\n");
+
+            if(argi+1>=argc || argv[argi+1][0]=='-')
+                throw("after argument --translation-out-dir must be directory path");
+            
+            translation_traget=argv[++argi];
+            kprintf("  translation_traget=%s\n",translation_traget);
+        }
+
+        else if(cptr_compare(a, "-to") || cptr_compare(a, "--tranlation-out-dir")){
+            if(argi+1>=argc || argv[argi+1][0]=='-')
+                throw("after argument --translation-out-dir must be directory path");
+            
+            translation_out_dir=argv[++argi];
+            kprintf("  translation_out_dir=%s\n", translation_out_dir);
+
+            if(!dir_exists(translation_out_dir)){
+                dir_create(translation_out_dir);
+                kprintf("  directory was created\n");
+            }
+            else kprintf("  directory exists\n");
+        }
+
+        else if(cptr_compare(a, "-o") || cptr_compare(a, "--compilation-out-dir")){
+            if(argi+1>=argc || argv[argi+1][0]=='-')
+                throw("after argument --compilation-out-dir must be directory path");
+            
+            compilation_out_dir=argv[++argi];
+            kprintf("  compilation_out_dir=%s\n", compilation_out_dir);
+
+            if(!dir_exists(compilation_out_dir)){
+                dir_create(compilation_out_dir);
+                kprintf("  directory was created\n");
+            }
+            else kprintf("  directory exists\n");
+        }
+
+        else {
+            if(a[0]=='-'){
+                kprintf("  \e[91minvalid argument: %s\e[37m\n", a);
+                return -2;
+            }
+            if(file_exists(a)){
+                Autoarr_add(source_files, a);
+                kprintf("  found source file: %s\n", a);
+            }
+            else {
+                kprintf("  \e[91mfile not found: %s\e[37m\n", a);
+                return -3;
+            }
+        }
+    }
+
+    if(!compile && !translate){
+        kprintf("\e[91moperation not specified: select --translate and/or --compile\e[37m\n");
+        return -4;
+    }
+    if(Autoarr_length(source_files)==0){
+        kprintf("\e[91mno source files specified\e[37m\n");
+        return -5;
+    }
 
-int main(){
-    if(!setlocale(LC_ALL, "C.UTF8"))
-        kprintf("\e[93msetlocale failed\n");
     // kerep type system
-    ktDescriptors_beginInit();
-    ktDescriptors_initKerepTypes();
-    ktDescriptors_initCbLexerTypes();
-    ktDescriptors_endInit();
-    // keywords search tree
-    init_keywordsSearchTree();
-    kprint(kp_s|kp_fgGray, "keywordsSearchTree: ", kp_h|kp_pre, keywordsSearchTree, kp_c, '\n');
-    STNode_free(keywordsSearchTree);
+    kt_beginInit();
+    kt_initKerepTypes();
+    kt_initLexerTypes();
+    kt_endInit();
+    kprint_setColor(kp_fgGray);
+
+    Lexer* lexer=Lexer_create();
+    
+    Autoarr_foreach(source_files, src_file_name, ({
+        tryLast(Lexer_parseFile(lexer, src_file_name), m_tokens)
+            LinkedList(Token)* tokens=m_tokens.value.VoidPtr;
+        kprintf("tokens count: %u\n", tokens->count);
+        LinkedList_foreach(tokens, tokNode, 
+            kprintf("%u %s\n", tokNode->value.id, tokNode->value.value));
+        char* basename=path_basename(src_file_name, false);
+        NamespaceContext file_context={
+            .base={
+                .name=basename,
+                .namespace=NULL,
+                .parent=NULL,
+                .type=ContextType_Namespace,
+                .tokens=tokens
+            }
+        };
+        tryLast(translateToC(&file_context.base), m_c_code)
+            char* generated_code=m_c_code.value.VoidPtr;
+        kprintf("%s", generated_code);
+
+        char* generated_file_name;
+        if(file_context.base.namespace!=NULL)
+            generated_file_name=cptr_concat(file_context.base.namespace,".",file_context.base.name,".g.c");
+        else generated_file_name=cptr_concat(file_context.base.name,".g.c");
+        tryLast(file_open(generated_file_name, FileOpenMode_Write), m_generated_file)
+            FileHandle generated_file=m_generated_file.value.VoidPtr;
+        kprintf("created file %s\n", generated_file_name);
+        tryLast(file_writeCptr(generated_file, generated_code),_m_1885);
+        tryLast(file_close(generated_file),_m_14415);
+        kprintf("source code has been written to the file\n");
+        free(basename);
+        LinkedList_free(tokens);
+        free(generated_code);
+        free(generated_file_name);
+    }));
+
+    Autoarr_free(source_files, true);
+    Lexer_destroy(lexer);
+    kt_free();
     return 0;
-}
+}

src/lexer/analize.c

@@ -0,0 +1,173 @@
+#include "lexer.h"
+
+Maybe _throw_wrongchar(Lexer* lex){
+    char* _errline=string_extract(lex->line);
+    char* _context=string_extract(lex->context);
+    printf("\n\e[91mwrong char <%c> at [%s:%u:%u %s]\n >>> %s\n",
+        lex->source[lex->charnum], lex->filename,lex->linenum,lex->charnum,_context, _errline);
+    exit(96);
+}
+#define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(lex); }
+
+// adds <lex->label> to <lex->tokens> as tok_label or tok_number
+void _tryAddLabel(Lexer* lex){
+    if(lex->label.length==0) return;
+
+    Unitype uni=ST_pullString(lex->keywordSearchTree, lex->label);
+    if(uni.VoidPtr!=NULL) // built-in keyword
+        LinkedList_addToEnd(lex->tokens, LLNode_create(Token,*(Token*)uni.VoidPtr));
+    else {          // user-defined lex->label
+        TokenId udt_id=tok_label;
+        switch(*lex->label.ptr){ // starts with number
+            case '0': case '1': case '2': case '3': case '4':
+            case '5': case '6': case '7': case '8': case '9':
+                udt_id=tok_number;
+                break;
+        }
+        Token udt=Token_createUD(udt_id, string_extract(lex->label));
+        LinkedList_addToEnd(lex->tokens, LLNode_create(Token,udt));
+    }
+  
+    lex->label=(string){lex->source, 0};
+};
+#define tryAddLabel() _tryAddLabel(lex)
+  
+#define addDefTok(id) { tryAddLabel(); LinkedList_addToEnd(lex->tokens, LLNode_create(Token,default_tokens[id])); }
+  
+void _addDefTok_ifnext(Lexer* lex, char next, TokenId yes, TokenId no){
+    if(*(lex->source+1)==next){
+        lex->source++;
+        addDefTok(yes);
+    }
+    else addDefTok(no);
+}
+#define addDefTok_ifnext(nextChar, yesTok, noTok) _addDefTok_ifnext(lex, nextChar, yesTok, noTok)
+
+// returns text in quotes or error
+Maybe _readString(Lexer* lex, char quotChar){
+    char c;
+    bool prevIsBackslash=false;
+    char* srcFirst=lex->source;
+    while ((c=*++lex->source)){
+        if(c==quotChar) {
+            if(prevIsBackslash) // \"
+                prevIsBackslash=false;
+            else { // "
+                string str={srcFirst, lex->source-srcFirst+1};
+                char* extracted=string_extract(str);
+                return SUCCESS(UniHeapPtr(char, extracted));
+            }
+        }
+        else prevIsBackslash= c=='\\' && !prevIsBackslash;
+    }
+    lex->source=srcFirst;
+    throw_wrongchar(;);
+}
+#define readString(quotChar) _readString(lex, quotChar)
+
+
+Maybe __Lexer_analize(Lexer* lex, char* _filename, char* _source){
+    lex->filename=_filename;
+    lex->source=_source;
+    if(lex->tokens!=NULL)
+        LinkedList_free(lex->tokens);
+    lex->tokens=LinkedList_create(Token);
+    lex->label=(string){_source, 0};
+    lex->line=(string){_source, 0};
+    lex->linenum=0;
+    lex->charnum=0;
+
+    char c;
+    lex->source--;
+    while ((c=*++(lex->source))) {
+        // tryAddLabel() resets lex->label.length, but lex->label.ptr still pointing to prev token beginning
+        if(lex->label.length==0)
+            lex->label.ptr=lex->source;
+
+        switch(c){
+        case ' ': case '\t':
+        case '\r': case '\n':
+            tryAddLabel();
+            break;
+		
+        case '(': addDefTok(tok_lbracket);    break;
+        case '{': addDefTok(tok_lbracket_fi); break;
+        case '[': addDefTok(tok_lbracket_sq); break;
+        case ')': addDefTok(tok_rbracket);    break;
+        case '}': addDefTok(tok_rbracket_fi); break;
+        case ']': addDefTok(tok_rbracket_sq); break;
+		
+        case '\'':
+            tryAddLabel();
+            try(readString('\''), m_ch, ;){
+                Token ctok=Token_createUD(tok_character, m_ch.value.VoidPtr);
+                LinkedList_addToEnd(lex->tokens, LLNode_create(Token,ctok));
+            }
+            break;
+        case '"':
+            tryAddLabel();
+            try(readString('"'), m_Str, ;){
+                Token stok=Token_createUD(tok_string, m_Str.value.VoidPtr);
+                LinkedList_addToEnd(lex->tokens, LLNode_create(Token,stok));
+            }
+            break;
+		
+        case '<': addDefTok(tok_less); break;
+        case '>': addDefTok(tok_more); break;
+        case '+': addDefTok(tok_plus); break;
+        case '-': addDefTok(tok_minus); break;
+        case '*': addDefTok(tok_asterisk); break;
+		
+        case '/':
+            tryAddLabel();
+            string commentStr={lex->source, 1};
+            c=*++lex->source;
+            if(c=='/') { // single-lex->line comment
+                commentStr.length++;
+                while((c=*++lex->source)){
+                    if(c=='\n' || c=='\r') break;
+                    else commentStr.length++;
+                }
+            }
+            else if(c=='*') { // multi-lex->line comment
+                commentStr.length++; // first slash
+                while((c=*++lex->source)){
+                    commentStr.length++;
+                    if(c=='*' && *(++lex->source)=='/') break;
+                }
+                commentStr.length++; // last slash
+            }
+            else { // not comment
+                lex->source--;
+                addDefTok(tok_slash);
+                break;
+            }
+            Token comTok=Token_createUD(tok_comment, string_extract(commentStr));
+            LinkedList_addToEnd(lex->tokens, LLNode_create(Token,comTok));
+            break;
+		
+        case '=': addDefTok_ifnext('=', tok_equal, tok_assign);       break;
+        case '!': addDefTok_ifnext('=', tok_not_equal, tok_not);      break;
+        case '&': addDefTok_ifnext('&', tok_and_d, tok_and);          break;
+        case '|': addDefTok_ifnext('|', tok_or_d,  tok_or);           break;
+        case '?': addDefTok_ifnext('?', tok_question_d, tok_question);break;
+        case ':': addDefTok(tok_colon);      break;
+        case ';': addDefTok(tok_semicolon);  break;
+        case '.': addDefTok(tok_point);      break;
+        case ',': addDefTok(tok_comma);      break;
+        case '~': addDefTok(tok_tilda);      break;
+        case '\\':addDefTok(tok_backslash);  break;
+        case '%': addDefTok(tok_percent);    break;
+        case '^': addDefTok(tok_xor);        break;
+        case '$': addDefTok(tok_dollar);     break;
+        case '@': addDefTok(tok_at);         break;
+		
+        default:
+            lex->label.length++;
+            break;
+        }
+    }
+    
+    tryAddLabel();
+    return SUCCESS(UniHeapPtr(LinkedList(Token), lex->tokens));
+}

src/lexer/context.h

@@ -3,35 +3,35 @@
 #include "../../kerep/src/base/base.h"
 #include "tokens.h"
 
-typedef struct ContextStruct Context;
 
-PACK_ENUM(ContextType,
-    CT_Namespace,
-    CT_Function,
-    CT_Class
+PACKED_ENUM(ContextType,
+    ContextType_Namespace,
+    ContextType_Class,
+    ContextType_Function
 )
 
-struct ContextStruct {
+STRUCT(Context,
     char* name;
-    Context* parent;
-    Autoarr(Token)* tokens;
+    char* namespace; /* nullable */
+    Context* parent; /* nullable */
+    LinkedList(Token)* tokens;
     ContextType type;
-};
+)
 
-typedef struct NamespaceContext{
+STRUCT(NamespaceContext,
     Context base;
-} NamespaceContext;
+)
 
-typedef struct FunctionContext {
+STRUCT(ClassContext,
     Context base;
-    Autoarr(Token)* arguments;
-    Autoarr(Token)* attributes;
+    LinkedList(Token)* attributes;
+    Token accessModifier;
+)
+
+STRUCT(FunctionContext,
+    Context base;
+    LinkedList(Token)* arguments;
+    LinkedList(Token)* attributes;
     Token accessModifier;
     Token returnType;
-} FunctionContext;
-
-typedef struct ClassContext {
-    Context base;
-    Autoarr(Token)* attributes;
-    Token accessModifier;
-} ClassContext;
+)

src/lexer/init.c

@@ -1,6 +1,7 @@
 #include "lexer.h"
 
-void ktDescriptors_initCbLexerTypes(){
-    kt_register(Token, NULL, NULL);
-    kt_register(Autoarr_Token, ____Autoarr_free_Token, NULL);
+void kt_initLexerTypes(){
+    kt_register(Token);
+    kt_register(LinkedList_Token);
+    kt_register(LLNode_Token);
 }

src/lexer/lexer.c

@@ -1,273 +1,45 @@
 #include "lexer.h"
 #include "../../kerep/src/String/string.h"
+#include "../../kerep/src/Filesystem/filesystem.h"
 
-typedef struct SharedLexerData{
-    char* _source;
-    char* _filename;
-    Autoarr(Token)* _tokens;
-    string _context;
-    string _line;
-    string _label;
-    uint32 _linenum;
-    uint32 _charnum;
-} SharedLexerData;
+Maybe __Lexer_analize(Lexer* lex, char* _filename, char* _source);
 
-#define source sld->_source
-#define filename sld->_filename
-#define tokens sld->_tokens
-#define context sld->_context
-#define line sld->_line
-#define label sld->_label
-#define linenum sld->_linenum
-#define charnum sld->_charnum
-
-Maybe _throw_wrongchar(SharedLexerData* sld){
-    char* errline=string_extract(line);
-    char* _context=string_extract(context);
-    printf("\n\e[91mwrong char <%c> at [%s:%u:%u %s]\n >>> %s\n",
-                source[charnum], filename,linenum,charnum,_context, errline);
-    exit(96);
-}
-#define throw_wrongchar(freeMem) { freeMem; return _throw_wrongchar(sld); }
-    
-#define addTok(id) Autoarr_add(tokens, default_tokens[id])
-    
-void _addTok_ifnext(char next, TokenId yes, TokenId no, SharedLexerData* sld){
-    if(*(++source)==next){
-        addTok(yes);
+Lexer* Lexer_create(){
+    Lexer* lex=malloc(sizeof(*lex));
+    *lex=(Lexer){0};
+    lex->analize=__Lexer_analize;
+    lex->keywordSearchTree=STNode_create();
+    for(TokenId keywordId=0; keywordId<=tok_typeof; keywordId++){
+        const Token* keywordptr=&default_tokens[keywordId];
+        Unitype uni=UniStackPtr(Token, keywordptr);
+        ST_push(lex->keywordSearchTree, keywordptr->value, uni);
     }
-    else {
-        source--;
-        addTok(no);
-    }
-}
-#define addTok_ifnext(nextChar, yesTok, noTok) _addTok_ifnext(nextChar, yesTok, noTok, sld)
-
-// adds <label> to <tokens> as tok_label or tok_number
-void _tryAddLabel(SharedLexerData* sld){
-    if(label.length==0) return;
-    Unitype uni=ST_pullString(keywordsSearchTree,label);
-    if(uni.VoidPtr!=NULL) // built-in keyword
-        Autoarr_add(tokens, *(Token*)uni.VoidPtr);
-    else {          // user-defined label
-        Token ut;
-        ut.value=string_extract(label);
-        switch(*label.ptr){
-            case '0': case '1': case '2': case '3': case '4':
-            case '5': case '6': case '7': case '8': case '9': 
-                ut.id=tok_number;
-                break;
-            default:
-                ut.id=tok_label;
-                break;
-        }
-        Autoarr_add(tokens, ut);
-    }
-    
-    label=(string){source,0};
-};
-#define tryAddLabel() _tryAddLabel(sld)
-
-// returns text in quotes or error
-Maybe _readString(char quotChar, SharedLexerData* sld){
-    char c;
-    bool prevIsBackslash=false;
-    char* srcFirst=source;
-    while ((c=*++source)){
-        if(c==quotChar) {
-            if(prevIsBackslash) // \"
-                prevIsBackslash=false;
-            else { // "
-                string str={srcFirst, source-srcFirst+1};
-                char* extracted=string_extract(str);
-                return SUCCESS(UniHeapPtr(char, extracted));
-            }
-        } 
-        else prevIsBackslash= c=='\\' && !prevIsBackslash;
-    }
-    source=srcFirst;
-    throw_wrongchar(;);
-}
-#define readString(quotChar) _readString(quotChar, sld)
-
-
-Maybe _lexan(SharedLexerData* sld){
-    char c;
-    source--;
-    while ((c=*++source)) switch(c){
-        case ' ': case '\t':
-        case '\r': case '\n':
-            tryAddLabel();
-            break;
-        case '(':
-            tryAddLabel();
-            addTok(tok_lbracket);
-            break;
-        case '{':
-            tryAddLabel();
-            addTok(tok_lbracket_fi);
-            break;
-        case '[':
-            tryAddLabel();
-            addTok(tok_lbracket_sq);
-            break;
-        case ')':
-            tryAddLabel();
-            addTok(tok_rbracket);
-            break;
-        case '}':
-            tryAddLabel();
-            addTok(tok_rbracket_fi);
-            break;
-        case ']':
-            tryAddLabel();
-            addTok(tok_rbracket_sq);
-            break;
-        case '\'':
-            tryAddLabel();
-            try(readString('\''), maybeC, ;){
-                Token ctok={ 
-                    .id=tok_character,
-                    .value=(char*)maybeC.value.VoidPtr
-                };
-                Autoarr_add(tokens, ctok);
-            }
-            break;
-        case '"':
-            tryAddLabel();
-            try(readString('"'), maybeS, ;){
-                Token stok={ 
-                    .id=tok_string,
-                    .value=(char*)maybeS.value.VoidPtr
-                };
-                Autoarr_add(tokens, stok);
-            }
-            break;
-        case '<':
-            tryAddLabel();
-            addTok(tok_less);
-            break;
-        case '>':
-            tryAddLabel();
-            addTok(tok_more);
-            break;
-        case '+':
-            tryAddLabel();
-            addTok(tok_plus);
-            break;
-        case '-':
-            tryAddLabel();
-            addTok(tok_minus);
-            break;
-        case '*':
-            tryAddLabel();
-            addTok(tok_asterisk);
-            break;
-        case '/':
-            tryAddLabel();
-            string commentStr={source,0};
-            c=*++source;
-            if(c=='/') { // single-line comment
-                while((c=*++source)){
-                    if(c=='\n' || c=='\r') break;
-                    else commentStr.length++;
-                }
-            }
-            else if(c=='*') { // multi-line comment
-                while((c=*++source)){
-                    commentStr.length++;
-                    if(c=='*' && *(++source)=='/') break;
-                }
-            }
-            else { // not comment
-                source--;
-                addTok(tok_slash);
-                break;
-            }
-            Token comTok={
-                .value=string_extract(commentStr),
-                .id=tok_comment
-            };
-            Autoarr_add(tokens,comTok);
-            break;
-        case '=':
-            tryAddLabel();
-            addTok_ifnext('=',tok_equals,tok_assign);
-            break;
-        case '!':
-            tryAddLabel();
-            addTok_ifnext('=',tok_not_equals,tok_not);
-            break;
-        case '&':
-            tryAddLabel();
-            addTok_ifnext('&',tok_and_d,tok_and);
-            break;
-        case '|':
-            tryAddLabel();
-            addTok_ifnext('|',tok_or_d,tok_or);
-            break;
-        case '?':
-            tryAddLabel();
-            addTok_ifnext('?',tok_question_d,tok_question);
-            break;
-        case ':':
-            tryAddLabel();
-            addTok(tok_colon);
-            break;
-        case ';':
-            tryAddLabel();
-            addTok(tok_semicolon);
-            break;
-        case '.':
-            tryAddLabel();
-            addTok(tok_point);
-            break;
-        case ',':
-            tryAddLabel();
-            addTok(tok_comma);
-            break;
-        case '~':
-            tryAddLabel();
-            addTok(tok_tilda);
-            break;
-        case '\\':
-            tryAddLabel();
-            addTok(tok_backslash);
-            break;
-        case '%':
-            tryAddLabel();
-            addTok(tok_percent);
-            break;
-        case '^':
-            tryAddLabel();
-            addTok(tok_xor);
-            break;
-        case '$':
-            tryAddLabel();
-            addTok(tok_dollar);
-            break;
-        case '@':
-            tryAddLabel();
-            addTok(tok_at);
-            break;
-        default:
-            label.length++;
-            break;
-    }
-    
-    return SUCCESS(UniHeapPtr(Autoarr(Token), tokens));
+    return lex;
 }
 
+void Lexer_freeMembers(void* _l){
+    Lexer* lex=(Lexer*)_l;
+    STNode_free(lex->keywordSearchTree);
+}
 
-Maybe lexan(char* _source, char* _filename){
-    SharedLexerData sld={
-        ._source=_source,
-        ._filename=_filename,
-        ._tokens=Autoarr_create(Token,64,1024),
-        ._label={_source,0},
-        ._line={_source,0},
-        ._linenum=0,
-        ._charnum=0
-    };
-    return _lexan(&sld);
-}
+kt_define(Lexer, Lexer_freeMembers, NULL)
+
+void Lexer_destroy(Lexer* lex){
+    Lexer_freeMembers(lex);
+    free(lex);
+}
+
+///@return Maybe<LinkedList(Token)*>
+Maybe Lexer_parseFile(Lexer* lex, char* src_file_name){
+    try(file_open(src_file_name, FileOpenMode_Read), m_src_file,;)
+        FileHandle src_file=m_src_file.value.VoidPtr;
+    char* src_text;
+    try(file_readAll(src_file, &src_text), m_src_len, file_close(src_file))
+        u64 src_len=m_src_len.value.UInt64;
+    try(file_close(src_file),_m_215, free(src_text));
+    kprintf("srclen: %lu\n", src_len);
+    try(Lexer_parseText(lex, src_file_name, src_text), m_tokens,;)
+        LinkedList(Token)* tokens=m_tokens.value.VoidPtr;
+    free(src_text);
+    return SUCCESS(UniHeapPtr(LinkedList(Token), tokens));
+}

src/lexer/lexer.h

@@ -3,7 +3,30 @@
 #include "tokens.h"
 #include "context.h"
 
-//Autoarr(Token)*
-Maybe lexan(char* source, char* filename);
+void kt_initLexerTypes();
 
-void ktDescriptors_initCbLexerTypes();
+STRUCT(Lexer,
+    STNode* keywordSearchTree;
+    Maybe (*analize)(Lexer* lex, char* src_file_name, char* src_file_text);
+    
+    char* source;
+    char* filename;
+    LinkedList(Token)* tokens;
+    string context;
+    string line;
+    string label;
+    u32 linenum;
+    u32 charnum;
+)
+
+Lexer* Lexer_create();
+void Lexer_destroy(Lexer* lex);
+///@return Maybe<LinkedList(Token)*>
+
+///@return Maybe<LinkedList(Token)*>
+Maybe Lexer_parseFile(Lexer* lex, char* src_file_name);
+
+///@return Maybe<LinkedList(Token)*>
+static inline Maybe Lexer_parseText(Lexer* lex, char* src_file_name, char* src_file_text){
+    return lex->analize(lex, src_file_name, src_file_text);
+}

src/lexer/tokens.c

@@ -1,15 +1,14 @@
 #include "tokens.h"
 
-ktid_define(Token);
-Autoarr_define(Token)
-
-
-STNode* keywordsSearchTree=NULL;
-void init_keywordsSearchTree(){
-    keywordsSearchTree=STNode_create();
-    for(TokenId keywordId=0; keywordId<=tok_typeof; keywordId++){
-        const Token* keywordptr=&default_tokens[keywordId];
-        Unitype uni=UniStackPtr(Token, keywordptr);
-        ST_push(keywordsSearchTree, keywordptr->value, uni);
-    }
+void Token_freeMembers(void* _t){
+    Token* t=(Token*)_t;
+    if(t->on_heap)
+        free(t->value);
 }
+
+char* Token_toString(void* _t, u32 fmt){
+    return cptr_copy(((Token*)_t)->value);
+}
+
+kt_define(Token, Token_freeMembers, Token_toString);
+LinkedList_define(Token, false)

src/lexer/tokens.h

@@ -1,41 +1,41 @@
 #pragma once
 
-#include "../../kerep/src/Autoarr/Autoarr.h"
+#include "../../kerep/src/LinkedList/LinkedList.h"
 #include "../../kerep/src/SearchTree/SearchTree.h"
 
-PACK_ENUM(TokenId,
-    // base types
+PACKED_ENUM(TokenId,
+    /* base types */
     tok_void_t,
-    tok_int8_t,
-    tok_uint8_t,
-    tok_int16_t,
-    tok_uint16_t,
-    tok_int32_t,
-    tok_uint32_t,
-    tok_int64_t,
-    tok_uint64_t,
-    tok_char_t,      //ascii
-    tok_char8_t,     //utf8
-    tok_char16_t,    //utf16
+    tok_i8_t,
+    tok_u8_t,
+    tok_i16_t,
+    tok_u16_t,
+    tok_i32_t,
+    tok_u32_t,
+    tok_i64_t,
+    tok_u64_t,
+    tok_char_t,      /* ansi */
+    tok_char8_t,     /* utf8 *? */
+    tok_char16_t,    /* utf16 */
     tok_bool_t,
-    tok_float32_t,
-    tok_float64_t,
-    // constant
+    tok_f32_t,
+    tok_f64_t,
+    /* constant */
     tok_null,
     tok_true,
     tok_false,
-    // control flow operators
+    /* control flow operators */
     tok_if,
     tok_else,
     tok_for,
     tok_while,
     tok_switch,
     tok_case,
-    tok_brake,
-    tok_braketo,
+    tok_break,
+    tok_breakto,
     tok_return,
     tok_goto,
-    // declaration keywords
+    /* declaration keywords */
     tok_class,
     tok_struct,
     tok_enum,
@@ -43,7 +43,7 @@ PACK_ENUM(TokenId,
     tok_event,
     tok_import,
     tok_alias,
-    // access modifiers
+    /* access modifiers */
     tok_static,
     tok_const,
     tok_readonly,
@@ -53,45 +53,45 @@ PACK_ENUM(TokenId,
     tok_virtual,
     tok_override,
     tok_partial,
-    // allocation keywords
-    tok_new, // allocates struct in heap
-    tok_sizeof, // size of variable value
-    tok_typeof, // type of variable
-    // special characters
-    tok_lbracket,    // (
-    tok_lbracket_fi, // {
-    tok_lbracket_sq, // [
-    tok_rbracket,    // )
-    tok_rbracket_fi, // }
-    tok_rbracket_sq, // ]
-    tok_less,        // <
-    tok_more,        // >
-    tok_plus,        // +
-    tok_minus,       // -
-    tok_asterisk,    // *
-    tok_slash,       // /
-    tok_assign,      // =
-    tok_not,         // !
-    tok_equals,      // ==
-    tok_not_equals,  // !=
-    tok_and,         // &
-    tok_and_d,       // &&
-    tok_or,          // |
-    tok_or_d,        // ||
-    tok_question,    // ?
-    tok_question_d,  // ??
-    tok_colon,       // :
-    tok_semicolon,   // ;
-    tok_point,       // .
-    tok_comma,       // ,
-    tok_tilda,       // ~
-    tok_backslash,   // \ /
-    tok_percent,     // %
-    tok_xor,         // ^
-    tok_lattice,     // #
-    tok_dollar,      // $
-    tok_at,           // @
-    // user-defined
+    /* allocation keywords */
+    tok_new, /* allocates struct in heap */
+    tok_sizeof, /* size of variable value */
+    tok_typeof, /* type of variable */
+    /* special characters */
+    tok_lbracket,    /* ( */
+    tok_lbracket_fi, /* { */
+    tok_lbracket_sq, /* [ */
+    tok_rbracket,    /* ) */
+    tok_rbracket_fi, /* } */
+    tok_rbracket_sq, /* ] */
+    tok_less,        /* < */
+    tok_more,        /* > */
+    tok_plus,        /* + */
+    tok_minus,       /* - */
+    tok_asterisk,    /* * */
+    tok_slash,       /* / */
+    tok_assign,      /* = */
+    tok_not,         /* ! */
+    tok_equal,      /* == */
+    tok_not_equal,  /* != */
+    tok_and,         /* & */
+    tok_and_d,       /* && */
+    tok_or,          /* | */
+    tok_or_d,        /* || */
+    tok_question,    /* ? */
+    tok_question_d,  /* ?? */
+    tok_colon,       /* : */
+    tok_semicolon,   /* ; */
+    tok_point,       /* . */
+    tok_comma,       /* , */
+    tok_tilda,       /* ~ */
+    tok_backslash,   /* \ / */
+    tok_percent,     /* % */
+    tok_xor,         /* ^ */
+    tok_lattice,     /* # */
+    tok_dollar,      /* $ */
+    tok_at,           /* @ */
+    /* user-defined */
     tok_label,
     tok_number,
     tok_character,
@@ -99,104 +99,101 @@ PACK_ENUM(TokenId,
     tok_comment
 )
 
-typedef struct Token{
+STRUCT(Token,
     char* value;
     TokenId id;
-} Token;
-ktid_declare(Token);
-Autoarr_declare(Token)
-
-extern STNode* keywordsSearchTree;
-// dont forget to free it
-void init_keywordsSearchTree();
-
+    bool on_heap; // allocated on stack or heap
+)
+LinkedList_declare(Token)
+/// user-defined token constructor
+#define Token_createUD(ID, VALUE) (Token){ .id=ID, .value=VALUE, .on_heap=true }
 
 static const Token default_tokens[]={
-    // base types
-    {"void", tok_void_t},
-    {"int8", tok_int8_t},
-    {"uint8", tok_uint8_t},
-    {"int16", tok_int16_t},
-    {"uint16", tok_uint16_t},
-    {"int32", tok_int32_t},
-    {"uint32", tok_uint32_t},
-    {"int64", tok_int64_t},
-    {"uint64", tok_uint64_t},
-    {"char", tok_char_t},
-    {"char8", tok_char8_t},
-    {"char16", tok_char16_t},
-    {"bool", tok_bool_t},
-    {"float32", tok_float32_t},
-    {"float64", tok_float64_t},
-    // constant
-    {"null", tok_null},
-    {"true", tok_true},
-    {"false", tok_false},
-    // control flow operators
-    {"if", tok_if},
-    {"else", tok_else},
-    {"for", tok_for},
-    {"while", tok_while},
-    {"switch", tok_switch},
-    {"case", tok_case},
-    {"brake", tok_brake},
-    {"braketo", tok_braketo},
-    {"return", tok_return},
-    {"goto", tok_goto},
-    // declaration keywords
-    {"class", tok_class},
-    {"struct", tok_struct},
-    {"enum", tok_enum},
-    {"union", tok_union},
-    {"event", tok_event},
-    {"import", tok_import},
-    {"alias", tok_alias},
-    // access modifiers
-    {"static", tok_static},
-    {"const", tok_const},
-    {"readonly", tok_readonly},
-    {"protected", tok_protected},
-    {"internal", tok_internal},
-    {"public", tok_public},
-    {"virtual", tok_virtual},
-    {"override", tok_override},
-    {"partial", tok_partial},
-    // allocation keywords
-    {"new", tok_new},
-    {"sizeof", tok_sizeof},
-    {"typeof", tok_typeof},
-    // special characters
-    {"(", tok_lbracket},
-    {"{", tok_lbracket_fi},
-    {"[", tok_lbracket_sq},
-    {")", tok_rbracket},
-    {"}", tok_rbracket_fi},
-    {"]", tok_rbracket_sq},
-    {"<", tok_less},
-    {">", tok_more},
-    {"+", tok_plus},
-    {"-", tok_minus},
-    {"*", tok_asterisk},
-    {"/", tok_slash},
-    {"=", tok_assign},
-    {"!", tok_not},
-    {"==", tok_equals},
-    {"!=", tok_not_equals},
-    {"&", tok_and},
-    {"&&", tok_and_d},
-    {"|", tok_or},
-    {"||", tok_or_d},
-    {"?", tok_question},
-    {"??", tok_question_d},
-    {":", tok_colon},
-    {";", tok_semicolon},
-    {".", tok_point},
-    {",", tok_comma},
-    {"~", tok_tilda},
-    {"\\", tok_backslash},
-    {"%", tok_percent},
-    {"^", tok_xor},
-    {"#", tok_lattice},
-    {"$", tok_dollar},
-    {"@", tok_at}
+    /* base types */
+    {"void", tok_void_t, 0},
+    {"i8", tok_i8_t, 0},
+    {"u8", tok_u8_t, 0},
+    {"i16", tok_i16_t, 0},
+    {"u16", tok_u16_t, 0},
+    {"i32", tok_i32_t, 0},
+    {"u32", tok_u32_t, 0},
+    {"i64", tok_i64_t, 0},
+    {"u64", tok_u64_t, 0},
+    {"char", tok_char_t, 0},
+    {"char8", tok_char8_t, 0},
+    {"char16", tok_char16_t, 0},
+    {"bool", tok_bool_t, 0},
+    {"f32", tok_f32_t, 0},
+    {"f64", tok_f64_t, 0},
+    /* constant */
+    {"null", tok_null, 0},
+    {"true", tok_true, 0},
+    {"false", tok_false, 0},
+    /* control flow operators */
+    {"if", tok_if, 0},
+    {"else", tok_else, 0},
+    {"for", tok_for, 0},
+    {"while", tok_while, 0},
+    {"switch", tok_switch, 0},
+    {"case", tok_case, 0},
+    {"brake", tok_break, 0},
+    {"braketo", tok_breakto, 0},
+    {"return", tok_return, 0},
+    {"goto", tok_goto, 0},
+    /* declaration keywords */
+    {"class", tok_class, 0},
+    {"struct", tok_struct, 0},
+    {"enum", tok_enum, 0},
+    {"union", tok_union, 0},
+    {"event", tok_event, 0},
+    {"import", tok_import, 0},
+    {"alias", tok_alias, 0},
+    /* access modifiers */
+    {"static", tok_static, 0},
+    {"const", tok_const, 0},
+    {"readonly", tok_readonly, 0},
+    {"protected", tok_protected, 0},
+    {"internal", tok_internal, 0},
+    {"public", tok_public, 0},
+    {"virtual", tok_virtual, 0},
+    {"override", tok_override, 0},
+    {"partial", tok_partial, 0},
+    /* allocation keywords */
+    {"new", tok_new, 0},
+    {"sizeof", tok_sizeof, 0},
+    {"typeof", tok_typeof, 0},
+    /* special characters */
+    {"(", tok_lbracket, 0},
+    {"{", tok_lbracket_fi, 0},
+    {"[", tok_lbracket_sq, 0},
+    {")", tok_rbracket, 0},
+    {"}", tok_rbracket_fi, 0},
+    {"]", tok_rbracket_sq, 0},
+    {"<", tok_less, 0},
+    {">", tok_more, 0},
+    {"+", tok_plus, 0},
+    {"-", tok_minus, 0},
+    {"*", tok_asterisk, 0},
+    {"/", tok_slash, 0},
+    {"=", tok_assign, 0},
+    {"!", tok_not, 0},
+    {"==", tok_equal, 0},
+    {"!=", tok_not_equal, 0},
+    {"&", tok_and, 0},
+    {"&&", tok_and_d, 0},
+    {"|", tok_or, 0},
+    {"||", tok_or_d, 0},
+    {"?", tok_question, 0},
+    {"??", tok_question_d, 0},
+    {":", tok_colon, 0},
+    {";", tok_semicolon, 0},
+    {".", tok_point, 0},
+    {",", tok_comma, 0},
+    {"~", tok_tilda, 0},
+    {"\\", tok_backslash, 0},
+    {"%", tok_percent, 0},
+    {"^", tok_xor, 0},
+    {"#", tok_lattice, 0},
+    {"$", tok_dollar, 0},
+    {"@", tok_at, 0}
 };

tasks/clean_additions.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+
+myprint "${WHITE}deleting .kerep_rebuild.tmp"
+rm -rf .kerep_rebuild.tmp

tasks/pre_build.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 
-if [ ! -f "kerep/bin/kerep.a" ]  || [ -f .rebuild_kerep ] 
-then
+# check if kerep static lib exists or kerep_rebuild task was executed
+if [ ! -f "$OBJDIR/libs/kerep.a" ] || [ -f .rebuild_kerep.tmp ]; then
+    [[ -z "$KEREP_BUILD_TASK" ]] && error "KEREP_BUILD_TASK is empty" 
+    myprint "${BLUE}making kerep task <$KEREP_BUILD_TASK>"
+
     cd kerep
     if ! make "$KEREP_BUILD_TASK"; then
         exit 1
     fi
     cd ..
-    rm -rf .rebuild_kerep
+
+    cp kerep/bin/kerep.a $OBJDIR/libs/
+    myprint "${GREEN}copied ${CYAN}kerep.a"
+    rm -f .rebuild_kerep.tmp
 fi
-cp kerep/bin/kerep.a obj/
-printf "${GREEN}copied ${CYAN}kerep.a\n"

tasks/rebuild_kerep.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-touch .rebuild_kerep
-printf "kerep.a will be rebuilt in the next build task"
+touch .rebuild_kerep.tmp
+myprint "${YELLOW}kerep.a will be rebuilt in the next build task"