From ca8c22009a74d2020a8a3908e505fc0af69b400e Mon Sep 17 00:00:00 2001 From: Chinay-Domitrix <55807470+Chinay-Domitrix@users.noreply.github.com> Date: Mon, 18 Oct 2021 08:56:46 -0400 Subject: [PATCH 1/4] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..1919bd4 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '38 18 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From ea33061e77aaf080750a9d329c98e5b38fcb020f Mon Sep 17 00:00:00 2001 From: Chinay-Domitrix <55807470+Chinay-Domitrix@users.noreply.github.com> Date: Mon, 18 Oct 2021 18:46:31 -0400 Subject: [PATCH 2/4] :art: Reformat codebase Signed-off-by: Chinay-Domitrix <55807470+Chinay-Domitrix@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/jet----bug-report.md | 24 +- .github/workflows/codeql-analysis.yml | 50 +- .github/workflows/linux.yml | 22 +- .github/workflows/macos.yml | 22 +- .github/workflows/windows.yml | 22 +- CMakeLists.txt | 2 +- CODE_OF_CONDUCT.md | 114 +- CONTRIBUTING.md | 64 +- LANG.md | 188 +- README.md | 55 +- import/nextgen/allocator.h | 338 +-- import/nextgen/core-util.h | 139 +- import/nextgen/io.h | 74 +- import/nextgen/jet/jet-analyzer.h | 23 +- import/nextgen/jet/jet-ast.h | 329 +-- import/nextgen/jet/jet-diagnostics.h | 157 +- import/nextgen/jet/jet-lexer.h | 280 ++- import/nextgen/jet/jet-parser.h | 728 +++--- import/nextgen/jet/jet-syntax-nodes.h | 1283 +++++----- import/nextgen/jet/jet-token.h | 1057 ++++---- import/nextgen/jet/jet-x64-assembler.h | 544 ++-- import/nextgen/os-config.h | 34 +- import/nextgen/panic.h | 199 +- import/nextgen/str.h | 359 +-- lib/allocator.cpp | 39 +- lib/io.cpp | 12 +- lib/jet-analyzer.cpp | 2 +- lib/jet-diagnostics.cpp | 813 +++--- lib/jet-lexer-v1.cpp | 2495 ++++++++++--------- lib/jet-lexer-v2.cpp | 1495 +++++------ lib/jet-parser.cpp | 608 +++-- lib/jet-x64-assembler.cpp | 547 ++-- lib/str.cpp | 12 +- main.cpp | 12 +- tests/CMakeLists.txt | 3 +- tests/lex-test.cpp | 437 ++-- tests/parse-test.cpp | 475 ++-- tests/test-main.cpp | 10 +- tests/x64-test.cpp | 154 +- 39 files changed, 6745 insertions(+), 6476 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/jet----bug-report.md b/.github/ISSUE_TEMPLATE/jet----bug-report.md index bb2d33a..60fd1b9 100644 --- a/.github/ISSUE_TEMPLATE/jet----bug-report.md +++ b/.github/ISSUE_TEMPLATE/jet----bug-report.md @@ -1,9 +1,6 @@ --- -name: Jet ✈ī¸ Bug Report -about: Help Report bugs and any other security vulnerabilities. -title: '' -labels: bug -assignees: '' +name: Jet ✈ī¸ Bug Report about: Help Report bugs and any other security vulnerabilities. title: '' +labels: bug assignees: '' --- @@ -12,21 +9,22 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Find '...' 2. Click on '....' 3. Scroll down to '....' 4. Error occurs **Expected behavior** -A clear and concise description of what was supposed to happen. Also please described how the -unit tests reacted when you tested for this behavior. +A clear and concise description of what was supposed to happen. Also, please describe how the unit tests reacted when +you tested for this behavior. **Screenshots and Videos** -If you can, please add screenshots and videos to help us understand the problem. -Doing this is a sure fire way that we will be able to reproduce the error and likely get -the problem solved faster. +If you can, please add screenshots and videos to help us understand the problem. Doing this is a sure fire way that we +will be able to reproduce the error and likely get the problem solved faster. **Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] + +- OS: [e.g. iOS] +- Browser [e.g. chrome, safari] +- Version [e.g. 22] diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1919bd4..d842af4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,34 +38,34 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - #- run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 54a9065..f5a132c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,6 +1,6 @@ name: Linux -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && ./tests diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2986190..0a62ff4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,6 @@ name: MacOS -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: macos-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && ./tests diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a86fd39..e7ce79b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,6 +1,6 @@ name: Windows -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && cd Release && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && cd Release && ./tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b5252b..0a2494f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ add_subdirectory(tests) add_executable(jet main.cpp - # Import Support Libraries. + # Import Support Libraries. lib/panic.cpp lib/allocator.cpp lib/str.cpp diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 705bb97..65c8b4d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,126 +1,100 @@ # Jet ✈ī¸ Code of Conduct - ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for +everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity +and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, +or sexual identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to a positive environment for our -community include: +Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexual language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take +appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, +issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for +moderation decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing +the community in public spaces. Examples of representing our community include using an official e-mail address, posting +via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -nextgenprojectofficial@gmail.com. -All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible +for enforcement at nextgenprojectofficial@gmail.com. All complaints will be reviewed and investigated promptly and +fairly. -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. +All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem +in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the +community. -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation +and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of actions. -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including +unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding +interactions in community spaces as well as external channels like social media. Violating these terms may lead to a +temporary or permanent ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified +period of time. No public or private interaction with the people involved, including unsolicited interaction with those +enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate +behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +**Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). +Community Impact Guidelines were inspired +by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d466ac5..c7d1879 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -👍 Thanks for considering to contribute to the project, we appreciate it. 👍 +👍 Thanks for considering contributing to the project, we appreciate it. 👍 Outline ----------------- @@ -23,16 +23,16 @@ Before starting the rest of the process, please take time read and understand th Spread the Word --------------- -Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on -some online forums to encourage a friendly discussion. The language is at its early stages, so -we want as much exposure as possible. We have currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) +Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on some online forums to +encourage a friendly discussion. The language is at its early stages, so we want as much exposure as possible. We +currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) where we discuss new changes and possible integrations with new technologies. Add your own Features --------------------- -Want to add amazing language features yourself? Submit your -own language syntax ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if -selected, the syntax/feature will be named after you! (or an appropriate name of your choosing). +Want to add amazing language features yourself? Submit your own language syntax +ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if selected, the syntax/feature will be named after +you! (or an appropriate name of your choosing). > **Note:** Existing Language Features/Syntax suggested will not be named after you. We love suggestions > of new features of the language, but please be mindful to see if your suggestion has been already been @@ -40,20 +40,23 @@ selected, the syntax/feature will be named after you! (or an appropriate name of Style Guidelines ---------------- -All programs under the NextGen Project follow the following C++ Style Guidlines. +All programs under the NextGen Project follow the following C++ Style Guidelines. * Getter and Setter functions are to be named using `camelCase` * Functions (including in namespaces and in classes) should be denoted with `MyFunction` notation -* Class and Struct member **variables** should be deonoted with `MyVariable` +* Class and Struct member **variables** should be denoted with `MyVariable` * Don't be *too* generous with your comments. Use them appropriately, especially when describing a complicated task -* Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistant with config -* Lines should be no longer than **80 characters**. We like to limit this line length for readibility +* Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistent with + config +* Lines should be no longer than **80 characters**. We like to limit this line length for readability * Mark implicit constructors with `/*implicit*/` indication -* Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format guidlines with a library -* Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in that scope, for example, - a class with that name requires all its methods and variables to be denoted as `snake_case`. +* Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format + guidelines with a library +* Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in + that scope, for example, a class with that name requires all its methods and variables to be denoted as `snake_case`. When writing nested namespace use the following setup to be consistent with other files: + ```c++ namespace A { namespace B { namespace C { @@ -62,8 +65,8 @@ namespace A { namespace B { namespace C ``` > **Note:** There are a few exceptions to the rules above: If we create an STL-like object, keep styling -> based on STL styling, and just make sure to use common sense. If something has a very long typename and -> has better readibility using trailing return type, use it, **prioritize readibility** over anything else. +> based on STL styling, and just make sure to use common sense. If something has a very long typename and +> has better readability using trailing return type, use it, **prioritize readability** over anything else. Committing Guidelines --------------------- @@ -71,25 +74,26 @@ All commits to the project's repository are follow the following Guidelines. * Please limit your commit messages to no more than **10** words * Start all of your commit messages with a capital letter after the designated icon (see below) -* **Don't** end your commiting messages with `.` +* **Don't** end your committing messages with `.` * Please use the following icons before your commit message to indicate changes made: - * :art: `:art:` when improving the format/structure of the code - * :heavy_plus_sign: `:heavy_plus_sign:` when adding new features to existing code - * :zap: `:zap:` when improving performance - * :book: `:book:` when writing docs - * :penguin: `:penguin:` when updating portability - * :bug: `:bug:` when fixing a bug - * :fire: `:fire:` when source files are moved, changed, or deleted - * :wrench: `:wrench:` when fixing build system (CMake, CI) - * :white_check_mark: `:white_check_mark:` when working on tests + * :art: `:art:` when improving the format/structure of the code + * :heavy_plus_sign: `:heavy_plus_sign:` when adding new features to existing code + * :zap: `:zap:` when improving performance + * :book: `:book:` when writing docs + * :penguin: `:penguin:` when updating portability + * :bug: `:bug:` when fixing a bug + * :fire: `:fire:` when source files are moved, changed, or deleted + * :wrench: `:wrench:` when fixing build system (CMake, CI) + * :white_check_mark: `:white_check_mark:` when working on tests Pull Requests --------------- -Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want create -a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to -merge into the branch you requested for. +Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want to +create a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to merge into +the branch you requested for. Licensing ---------- -As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise specified. +As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise +specified. diff --git a/LANG.md b/LANG.md index 2bec113..cda12ec 100644 --- a/LANG.md +++ b/LANG.md @@ -1,6 +1,7 @@ # Jet ✈ī¸ Language Documentation -The language specification is listed below, everything from function variables, and the build systems that are planned to -be implemented or are already implemented are below. + +The language specification is listed below, everything from function variables, and the build systems that are planned +to be implemented or are already implemented are below. > **Note:** To determine whether a feature currently exists check for the keywords: **Implemented**, **WIP**, and **Planned**. @@ -17,25 +18,18 @@ Outline * [Enum](#enum-declaration) * [Match Statement](#match-statement) - Primitives -------------- Current Status: **WIP** -| Core Type | Description | -| ------------- | ------------- | -| `i32` | By default, all integer types are designated to 32 bits unless specified differently. | -| `i64` | 64 bit integer is separate and is using for longer integer values. | -| `u32` | Unsigned 32 bit integer. | -| `u64` | Unsigned 64 bit integer. | -| `f32` | 32 bit decimal value. | -| `f64` | 64 bit decimal value. | -| `bool` | Boolean Value: true or false. | -| `fn(Arg Types...) -> Ret` | Function Lambda. | -| `char` | A 1 byte character. | -| `str` | Array of characters | -| `None` | Optional value for `None` | +| Core Type | Description | | ------------- | ------------- | | `i32` | By default, all integer types are +designated to 32 bits unless specified differently. | | `i64` | 64-bit integer is separate and is using for longer +integer values. | | `u32` | Unsigned 32-bit integer. | | `u64` | Unsigned 64-bit integer. | | `f32` | +32-bit decimal value. | | `f64` | 64-bit decimal value. | | `bool` | Boolean Value: true or false. | +| `fn(Arg Types...) -> Ret` | Function Lambda. | | `char` | A 1 byte character. | | `str` | Array +of characters | | `None` | Optional value for `None` | ### Type Annotations + | Annotation | Description | | ------------- | ------------- | | `box` | Designates a type as smart pointer value | @@ -46,22 +40,11 @@ Current Status: **WIP** Operators ----------- Current Status: **Completed** -| Operator | Description | -| ------------- | ------------- | -| `+` | Addition. | -| `-` | Subtraction. | -| `/` | Division. | -| `*` | Multiplication. | -| `%` | Modulus. | -| `^` | XOR. | -| `&` | Bitwise AND. | -| `<<` | Shift Left. | -| `>>` | Shift Right. | -| `~` | Bitwise Not and Concat. | -| `**` | Exponential Power. | -| `!` | Logical Not. | -======= -| `??` | Then operator | +| Operator | Description | | ------------- | ------------- | | `+` | Addition. | | `-` | +Subtraction. | | `/` | Division. | | `*` | Multiplication. | | `%` | Modulus. | | `^` +| XOR. | | `&` | Bitwise AND. | | `<<` | Shift Left. | | `>>` | Shift Right. | | `~` +| Bitwise Not and Concat. | | `**` | Exponential Power. | | `!` | Logical Not. | ======= | `??` +| Then operator | @@ -71,20 +54,24 @@ Current Status: **Completed** The programming language is gradually typed. Static typing is only required when the compiler cannot infer the type. Variables may be declared in two different ways: + ```v my_variable := 23; another_variable: bool = false; ``` + All variables are immutable by default (except for lists), and require the `mut` keyword in order to change the way you interact with that variable. + ```v cant_change_me := 2324; cant_change_me = 24354; // Compiler error! mut change_me := 3342; change_me = 23424; // Ok ``` -Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, the -specification is standard for all data types. + +Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, +the specification is standard for all data types. Arrays @@ -92,95 +79,115 @@ Arrays Current Status: **WIP** ### Initialization -We wanted arrays to be simple and easy to initalize like in Python. + +We wanted arrays to be simple and easy to initialize like in Python. + ```rust mut list := [1, 2, 3, 4, 5]; // Can now add values to the list fixed: [5,i32] = [1, 2, 3, 4, 5]; ``` ### Designated Initializers + Current Status: **Planned** -Lists are mutable by default (unlike most of the other data types) because they are most -commonly dynamic and require mutability. Since we like to acknowledge the strengths of some -languages and weaknesses of others, one may consider a similar example of initalizing an array -using designated initalizers in Jet. +Lists are mutable by default (unlike most of the other data types) because they are most commonly dynamic and require +mutability. Since we like to acknowledge the strengths of some languages and weaknesses of others, one may consider a +similar example of initializing an array using designated initializers in Jet. + ```v list: [256,i32] = [ ['>'] = 2323, ['3'] = 343, ... ] ``` -The advantages of such a method reduce excess array initalization code, and allow for better readability to -the programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. -We feel that this as a design principle is good for dealing with code that relies on jump comparisons. + +The advantages of such a method reduce excess array initialization code, and allow for better readability to the +programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. We feel +that this as a design principle is good for dealing with code that relies on jump comparisons. ### Length + Current Status: **Planned** ```python -len(list); +len(list) ``` -Length is checked in a similar global length function that can find the length of various types, including lists. +Length is checked in a similar global length function that can find the length of various types, including lists. ### Slices + Current Status: **Planned** + ``` list[ Initial : End : IndexJump ] ``` -List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the list. + +List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the +list. Functions ------------ Current Status: **WIP** -Functions like anything else are declared explicitly in structs or are declared -in the global module scope. If functions are not exported, they are only viewable to the members in -the module that it was declared in. The `export` keyword is equivalent to the `public` keyword in other -languages. +Functions like anything else are declared explicitly in structs or are declared in the global module scope. If functions +are not exported, they are only viewable to the members in the module that it was declared in. The `export` keyword is +equivalent to the `public` keyword in other languages. ### Function Declaration + Current Status: **WIP** + ```zig my_func => (a: i32, b: i32, c: str) { ... } ``` -parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would be -written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are also -supported: + +parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would +be written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are +also supported: + ```zig generics => ($a, $b) { ... } ``` -Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is guarnteed to be valid -otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and dynamic typing flexibility for types that can be -generalized for similar actions. To make it easier for anonymous trait building, it is best for the programmer to statically define their variables -to make it easier for the compiler. + +Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is +guaranteed to be valid otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and +dynamic typing flexibility for types that can be generalized for similar actions. To make it easier for anonymous trait +building, it is best for the programmer to statically define their variables to make it easier for the compiler. ### Default Values + ```zig default_values := (a: int = 0xffff, b: str) { ... } ``` -Default values allow for flexibility in programming, especially when you want to handle a case without passing in a parameter. Currently -it is not planned to have default values for parameters that don't have their type explicitely stated. + +Default values allow for flexibility in programming, especially when you want to handle a case without passing in a +parameter. Currently, it is not planned to have default values for parameters that don't have their type explicitly +stated. ### Lambdas + Current Status: **Planned** -Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally declared as a parameters -by the following method: +Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally +declared as a parameters by the following method: + ```zig func => (lambda: fn(i32) -> i32) { lambda(23); // Calling the function } ``` -You may pass in lambdas through generic parameters as well and call them as long as function definition generation works as intended. An Example -for this function call may be (in reference to the above example): + +You may pass in lambdas through generic parameters as well and call them as long as function definition generation works +as intended. An Example for this function call may be (in reference to the above example): + ```zig func(|x| => { return x * x @@ -191,52 +198,69 @@ Modules -------- Current Status: **WIP** -The way a module system of a programming language dictates its entire package functionality structure. In order to be able to give -the programmer a lot of control we have decided to make it explicit as to functions / types are exported out in the file's module. +The way a module system of a programming language dictates its entire package functionality structure. In order to be +able to give the programmer a lot of control we have decided to make it explicit as to functions / types are exported +out in the file's module. ### Imports + Current Status: **WIP** ```python import SomeModuleName ``` + This is how most modules will be imported directly. > **Note:** Sub module structure like `std.func` are still in debate as of now, and this will be updated in the future to reflect language > changes. -### Github Imports +### GitHub Imports + Current Status: **Planned** ```python import 'https://github.com/The-NextGen-Project/jet.git' as SomeModule ``` -This is especially a very interesting part of the programming language. We will have support for open source imports. These projects -will need to be Jet projects that can be recognizable and imported directly into your project. This may be a good way to consistantly -ensure that you have the latest stable version of the git repository imported into your programming language. + +This is especially a very interesting part of the programming language. We will have support for open source imports. +These projects will need to be Jet projects that can be recognizable and imported directly into your project. This may +be a good way to consistently ensure that you have the latest stable version of the git repository imported into your +programming language. ### Exporting values + Current Status: **Planned** Functions: + ```zig export my_func => (...) { ... } ``` + Types: + ```zig export MyStruct => struct { ... } ``` + Variables: + ```js -export MyGlobalVariable := 32; +export +MyGlobalVariable +: += 32; ``` -values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted by the prefix -of the `export` before the declaration. +values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted +by the prefix of the `export` before the declaration. ### Static Compilation (Generally JIT Compiled) + Examples: + ```zig // Evaluated at compile-time some_value := static { @@ -245,8 +269,9 @@ some_value := static { ``` ### Match Statement -Match statements allow for expressing complex situations in shorter and more readable code. -Examples: + +Match statements allow for expressing complex situations in shorter and more readable code. Examples: + ```rust match value { 0 -> { @@ -259,8 +284,10 @@ match value { ``` ### Struct Declaration -Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while allowing for zero cost abstractions to take place. -Examples: + +Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while +allowing for zero cost abstractions to take place. Examples: + ```zig MyStruct => struct { prop: str; @@ -269,8 +296,10 @@ MyStruct => struct { ``` ### Enum Declaration -Enum Declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as constant values may be implemented. -Regular Enum: + +Enum Declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as +constant values may be implemented. Regular Enum: + ```zig Opcodes => enum { Add, @@ -279,7 +308,9 @@ Opcodes => enum { Div } ``` + Tagged Enums: + ```zig Operation => enum { Add { @@ -296,4 +327,5 @@ Operation => enum { End ------- -You made it to the end? Wow, thank you for the interest in reading the language. Before we you lose your attention, please consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). +You made it to the end? Wow, thank you for the interest in reading the language. Before we lose your attention, please +consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). diff --git a/README.md b/README.md index 4735745..ece2584 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ ![Jet](https://github.com/The-NextGen-Project/jet/blob/main/.github/JETV4Large3.png) -[![GitHub license](https://img.shields.io/github/license/The-NextGen-Project/jet?color=%2320ba37&label=License&logo=%2320ba37&logoColor=%2320ba37&style=for-the-badge)](https://github.com/The-NextGen-Project/jet/blob/main/LICENSE) +[![GitHub license](https://img.shields.io/github/license/The-NextGen-Project/jet?color=%2320ba37&label=License&logo=%2320ba37&logoColor=%2320ba37&style=for-the-badge)](https://github.com/The-NextGen-Project/jet/blob/main/LICENSE) [![Linux](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Linux?label=Linux&style=for-the-badge&logo=github)]() [![Mac](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/MacOS?label=MacOS&style=for-the-badge&logo=github)]() [![Windows](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Windows?label=Windows&style=for-the-badge&logo=github)]() + #### Fast Language for Developing Consistent Native, Mobile App, and Web Applications. + > **NOTE:** The language is in its early development stages. Don't expect syntax to be fixed until v1.0. If you want to see the development branch, you can view it [here](https://github.com/The-NextGen-Project/jet/tree/dev). The main branch is the most stable with all tests passing on every platform. Resources @@ -17,8 +19,9 @@ Resources * [r/Programming Languages Discord Channel](https://discord.gg/tft9Nw5jm6) * [License](#license) -Download +Download --------- + ``` git clone https://github.com/The-NextGen-Project/jet.git cd jet @@ -27,15 +30,19 @@ git submodule update cmake -DCMAKE_BUILD_TYPE=Release -GYourBuildGenerator make ``` -> **Note:** For building the project on windows, it can be done by simply building with MSVC or, if you have MinGW installed it will work with that C compiler as well. For setting up a build system that is fast, I reccommend using [Ninja](https://ninja-build.org/) which works on Windows, Linux, and Mac as a CMake generator. + +> **Note:** For building the project on Windows, it can be done by simply building with MSVC or, if you have MinGW installed it will work with that C compiler as well. For setting up a build system that is fast, I recommend using [Ninja](https://ninja-build.org/) which works on Windows, Linux, and Mac as a CMake generator. A Test Flight ------------- Offering a little taste of the lang, here is some syntax. A hello world program is as follows: + ```zig print("Hello, World!") ``` + Fibonacci Implemented in the lang would be as follows: + ```zig fib => (n: i32) i32 { return match n { @@ -45,34 +52,39 @@ fib => (n: i32) i32 { }; } ``` -The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is by default, statically-typed, however, types are -usually inferred by the langugage most of the time, and generate clean and easy to read code. This way we get to appeal to both those who like static typing and those that -prefer mainly dynamic typing. + +The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is +by default, statically-typed, however, types are usually inferred by the language most of the time, and generate clean +and easy to read code. This way we get to appeal to both those who like static typing and those that prefer mainly +dynamic typing. ### Error Messages -Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why their code -is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer overflow**: + +Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why +their code is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer +overflow**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/LexErrorExample.png) -The error message points out the initial error, points out the offending line and character, gives a potential solution, and explains -why what the programmer wrote was invalid. These components identify the problem and give opportunities for a solution to help the -programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a hybrid error message type that -encapsulates all that the programmer needs to identify the problem and solve it. +The error message points out the initial error, points out the offending line and character, gives a potential solution, +and explains why what the programmer wrote was invalid. These components identify the problem and give opportunities for +a solution to help the programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a +hybrid error message type that encapsulates all that the programmer needs to identify the problem and solve it. Let's have a look at another example, for an **unexpected token in the global scope**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/ParseErrorExample1.png) -People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help provide clarity -and provide potential solutions to a problem they may not understand. By telling the programmer what is directly valid, it prevents programmers to lookup documentation -unless it is necessary to explain an error. +People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help +provide clarity and provide potential solutions to a problem they may not understand. By telling the programmer what is +directly valid, it prevents programmers to lookup documentation unless it is necessary to explain an error. Progress ---------- -- [x] Lexer -- [x] Parser + +- [x] Lexer +- [x] Parser - [x] Semantic Analysis (**WIP**) - [ ] IR-Gen - [ ] IR-Optimization @@ -83,11 +95,12 @@ Progress Contributing ------------- -We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a community effort -and we would love to have many of you who are willing to help. Before your start your wonderful contribution, have a look at our +We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a +community effort, and we would love to have many of you who are willing to help. Before your start your wonderful +contribution, have a look at our [Contributing Guidelines](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). License ---------- -We want the project to be able to integrate with other with a lot of other open source software and determined that -we will be using the **MIT License**. +We want the project to be able to integrate with other with a lot of other open source software and determined that we +will be using the **MIT License**. diff --git a/import/nextgen/allocator.h b/import/nextgen/allocator.h index bcfc428..bf67ff4 100644 --- a/import/nextgen/allocator.h +++ b/import/nextgen/allocator.h @@ -1,205 +1,213 @@ # ifndef NEXTGEN_ALLOCATOR_H # define NEXTGEN_ALLOCATOR_H + # include "core-util.h" +namespace nextgen { + namespace mem { + using namespace nextgen::core; + + namespace os { // System allocators/libc + + void *calloc(size_t, size_t); + + void *malloc(size_t); -namespace nextgen { namespace mem { using namespace nextgen::core; + void *realloc(void *, size_t); - namespace os { // System allocators/libc + void free(void *); - void *calloc(size_t, size_t); - void *malloc(size_t); - void *realloc(void *, size_t); - void free(void *); - void *memmove(void *dst, const void *src, size_t size); + void *memmove(void *dst, const void *src, size_t size); - } // namespace nextgen::mem::os + } // namespace nextgen::mem::os - namespace detail { - struct c_deleter { - void operator()(void *ptr) const { - os::free(ptr); - } - }; - } + namespace detail { + struct c_deleter { + void operator()(void *ptr) const { + os::free(ptr); + } + }; + } - template - using Box = std::unique_ptr; - template - using Rc = std::shared_ptr; + template + using Box = std::unique_ptr; + template + using Rc = std::shared_ptr; - namespace arena { + namespace arena { - static constexpr auto PAGE_ALLOCATION_SIZE = 65'536; + static constexpr auto PAGE_ALLOCATION_SIZE = 65'536; - /** @brief 100 GiB */ - static constexpr decltype(UINTPTR_MAX) - OVER_COMMIT_MEMORY_SIZE = 107374182400ULL; + /** @brief 100 GiB */ + static constexpr decltype(UINTPTR_MAX) + OVER_COMMIT_MEMORY_SIZE = 107374182400ULL; - /** @brief 10 GiB */ - static constexpr decltype(UINTPTR_MAX) - DATA_ALLOCATION_MEMORY_SIZE = 10737418240ULL; + /** @brief 10 GiB */ + static constexpr decltype(UINTPTR_MAX) + DATA_ALLOCATION_MEMORY_SIZE = 10737418240ULL; - static uint8_t *ArenaMemory = + static uint8_t *ArenaMemory = #ifdef NG_OS_WINDOWS - (uint8_t *) VirtualAlloc(nullptr,OVER_COMMIT_MEMORY_SIZE,MEM_RESERVE, PAGE_READWRITE); + (uint8_t *) VirtualAlloc(nullptr, OVER_COMMIT_MEMORY_SIZE, MEM_RESERVE, PAGE_READWRITE); #endif #ifdef NG_OS_LINUX - (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | - PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); + (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | + PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); #endif #ifdef NG_OS_APPLE - (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | - PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | + PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); #endif - NG_INLINE size_t &length() { - static size_t BytesWritten = 0; - return BytesWritten; - } + NG_INLINE size_t &length() { + static size_t BytesWritten = 0; + return BytesWritten; + } - /** @summary Give portion of memory from the arena */ - NG_INLINE uint8_t *give(decltype(UINTPTR_MAX) memory) { - uint8_t * bytes; - switch (memory) { - case 0: return nullptr; - default: - auto written_bytes = arena::length(); + /** @summary Give portion of memory from the arena */ + NG_INLINE uint8_t *give(decltype(UINTPTR_MAX) memory) { + uint8_t *bytes; + switch (memory) { + case 0: + return nullptr; + default: + auto written_bytes = arena::length(); - bytes = ArenaMemory + written_bytes; - arena::length() += memory; - return bytes; - } - } + bytes = ArenaMemory + written_bytes; + arena::length() += memory; + return bytes; + } + } - /** @summary Get the memory from the arena and ensure the OS maps it */ - // TODO: Setup Linux and macOS memory allocation - NG_INLINE uint8_t *give_and_commit(decltype(UINTPTR_MAX) memory) { - uint8_t *mem = arena::give(memory); + /** @summary Get the memory from the arena and ensure the OS maps it */ + // TODO: Setup Linux and macOS memory allocation + NG_INLINE uint8_t *give_and_commit(decltype(UINTPTR_MAX) memory) { + uint8_t *mem = arena::give(memory); #ifdef NG_OS_WINDOWS - // Windows is a weird specimen. We cannot over-commit, so we need to - // reserve an impossible amount of memory and then request it from the - // same buffer. - return (uint8_t *) VirtualAlloc(mem, memory, MEM_COMMIT, PAGE_READWRITE); + // Windows is a weird specimen. We cannot over-commit, so we need to + // reserve an impossible amount of memory and then request it from the + // same buffer. + return (uint8_t *) VirtualAlloc(mem, memory, MEM_COMMIT, PAGE_READWRITE); #else - return mem; + return mem; #endif - } + } - // TODO: Setup Linux and macOS memory allocation - NG_INLINE void commit(uint8_t *arena, decltype(UINTPTR_MAX) memory) { + // TODO: Setup Linux and macOS memory allocation + NG_INLINE void commit(uint8_t *arena, decltype(UINTPTR_MAX) memory) { #ifdef NG_OS_WINDOWS - VirtualAlloc(arena, memory, MEM_COMMIT, - PAGE_READWRITE); + VirtualAlloc(arena, memory, MEM_COMMIT, + PAGE_READWRITE); #endif - } - } - - class Allocator { - uint8_t *bytes = nullptr; - size_t bytes_written = 0; - size_t capacity = 0; - public: - explicit Allocator(size_t reserved_memory) - : bytes(arena::give_and_commit(arena::PAGE_ALLOCATION_SIZE)), - bytes_written(0), - capacity(reserved_memory) {} - - auto current() const { - return bytes + bytes_written; - } - - auto length() const { - return bytes_written; - } - - auto reserve(size_t size) { - auto curr = current(); - bytes_written += size; - return curr; - } - - auto allocate(size_t allocation_size) { - auto ret_value = bytes_written + bytes; - if (bytes_written + allocation_size > capacity) { - arena::commit(ret_value, arena::PAGE_ALLOCATION_SIZE); - } - bytes_written += allocation_size; - return ret_value; - } - - template - auto allocate(size_t n) { - return (T*) (allocate(sizeof(T) * n)); - } - }; - - extern Allocator GLOBAL_OBJECT_ALLOC; - extern Allocator GLOBAL_DATA_ALLOC; - - NG_INLINE void deinit() { + } + } + + class Allocator { + uint8_t *bytes = nullptr; + size_t bytes_written = 0; + size_t capacity = 0; + public: + explicit Allocator(size_t reserved_memory) + : bytes(arena::give_and_commit(arena::PAGE_ALLOCATION_SIZE)), + bytes_written(0), + capacity(reserved_memory) {} + + auto current() const { + return bytes + bytes_written; + } + + auto length() const { + return bytes_written; + } + + auto reserve(size_t size) { + auto curr = current(); + bytes_written += size; + return curr; + } + + auto allocate(size_t allocation_size) { + auto ret_value = bytes_written + bytes; + if (bytes_written + allocation_size > capacity) { + arena::commit(ret_value, arena::PAGE_ALLOCATION_SIZE); + } + bytes_written += allocation_size; + return ret_value; + } + + template + auto allocate(size_t n) { + return (T *) (allocate(sizeof(T) * n)); + } + }; + + extern Allocator GLOBAL_OBJECT_ALLOC; + extern Allocator GLOBAL_DATA_ALLOC; + + NG_INLINE void deinit() { #ifdef NG_OS_WINDOWS - VirtualFree(arena::ArenaMemory, arena::length(), MEM_DECOMMIT); + VirtualFree(arena::ArenaMemory, arena::length(), MEM_DECOMMIT); #else - munmap(arena::ArenaMemory, GLOBAL_DATA_ALLOC.length() - +GLOBAL_OBJECT_ALLOC.length()); + munmap(arena::ArenaMemory, GLOBAL_DATA_ALLOC.length() + +GLOBAL_OBJECT_ALLOC.length()); #endif - } - - - template - class ObjectVector { - T *objects = GLOBAL_OBJECT_ALLOC.template allocate(N); - size_t len = 0; - size_t cap = N; - public: - - T operator[](size_t index) { - return objects[index]; - } - - void operator<<(T const &elem) { - if (len >= cap) { - cap *= 2; - auto new_ptr = GLOBAL_OBJECT_ALLOC.template allocate(cap); - FOR(i, len) { - new_ptr[i] = objects[i]; - } - objects = new_ptr; - } - objects[len++] = elem; - } - }; - - template - struct ArenaVec { - /** @brief Start of vector */ - T *begin; - /** @brief End of vector */ - T *end; - - /** @brief Indexes the value of the vector. This is slightly more - * expensive than normal array access */ - NG_INLINE T *operator[](size_t index) { - return (T*) (((uint8_t *)begin) + (sizeof(T) * index)); - } - - /** @brief Iterates over the vector with any other potential factors */ - template - NG_AINLINE void iter(Lambda lambda) { - FOR(i, len()) lambda(); - } - - /** @brief Span of the vector in memory */ - auto NG_AINLINE len() { - return end-begin; - } - }; - -}} // namespace nextgen::mem + } + + + template + class ObjectVector { + T *objects = GLOBAL_OBJECT_ALLOC.template allocate(N); + size_t len = 0; + size_t cap = N; + public: + + T operator[](size_t index) { + return objects[index]; + } + + void operator<<(T const &elem) { + if (len >= cap) { + cap *= 2; + auto new_ptr = GLOBAL_OBJECT_ALLOC.template allocate(cap); + FOR(i, len) { + new_ptr[i] = objects[i]; + } + objects = new_ptr; + } + objects[len++] = elem; + } + }; + + template + struct ArenaVec { + /** @brief Start of vector */ + T *begin; + /** @brief End of vector */ + T *end; + + /** @brief Indexes the value of the vector. This is slightly more + * expensive than normal array access */ + NG_INLINE T *operator[](size_t index) { + return (T *) (((uint8_t *) begin) + (sizeof(T) * index)); + } + + /** @brief Iterates over the vector with any other potential factors */ + template + NG_AINLINE void iter(Lambda lambda) { + FOR(i, len()) lambda(); + } + + /** @brief Span of the vector in memory */ + auto NG_AINLINE len() { + return end - begin; + } + }; + + } +} // namespace nextgen::mem # endif //NEXTGEN_ALLOCATOR_H diff --git a/import/nextgen/core-util.h b/import/nextgen/core-util.h index e098259..31b5e73 100644 --- a/import/nextgen/core-util.h +++ b/import/nextgen/core-util.h @@ -1,79 +1,82 @@ #ifndef NEXTGEN_CORE_H #define NEXTGEN_CORE_H + #include "os-config.h" #include "panic.h" namespace nextgen { - enum class NoneValue { None = 1 }; - static constexpr NoneValue None = NoneValue::None; - - template - struct Array { - size_t len; - const T arr[]; - }; - - template - constexpr size_t SizeOfArray(const T(&arr)[N]) { - return N; - } - - namespace core { - - /// Optional Type Value - Emulates Rust's style of an optional value in C++11. - /// We must ensure that values are owned by the value when passed. - template - class Option { - public: - // None value by default - Option() : is(false) {}; - - // Direct Copy - Option(Option const &other) : is(other.is), Some(other.Some) {} - - Option(T const &value) : - Some(std::move(value)), is(true) {} - - /*implicit*/ Option(NoneValue) : - is(false) {} - - NG_AINLINE bool isSome() const { - return is; - } - - NG_AINLINE bool isNone() const { - return !is; - } - - template - bool contains(U x) const { - if (is) return x == Some; - return false; - } - - auto unwrap() const { - ASSERT(is, "Unwrapped on None Value"); - return std::move(Some); - } - - template - auto unwrap_or_else(Lambda f) { - if (is) return std::move(Some); - return f(); - } - - private: - T Some; - bool is; - }; - - template - static NG_AINLINE auto Some(T value) { - return Option(value); // NOTE: Make explicit - } - } + enum class NoneValue { + None = 1 + }; + static constexpr NoneValue None = NoneValue::None; + + template + struct Array { + size_t len; + const T arr[]; + }; + + template + constexpr size_t SizeOfArray(const T(&arr)[N]) { + return N; + } + + namespace core { + + /// Optional Type Value - Emulates Rust's style of an optional value in C++11. + /// We must ensure that values are owned by the value when passed. + template + class Option { + public: + // None value by default + Option() : is(false) {}; + + // Direct Copy + Option(Option const &other) : is(other.is), Some(other.Some) {} + + Option(T const &value) : + Some(std::move(value)), is(true) {} + + /*implicit*/ Option(NoneValue) : + is(false) {} + + NG_AINLINE bool isSome() const { + return is; + } + + NG_AINLINE bool isNone() const { + return !is; + } + + template + bool contains(U x) const { + if (is) return x == Some; + return false; + } + + auto unwrap() const { + ASSERT(is, "Unwrapped on None Value"); + return std::move(Some); + } + + template + auto unwrap_or_else(Lambda f) { + if (is) return std::move(Some); + return f(); + } + + private: + T Some; + bool is; + }; + + template + static NG_AINLINE auto Some(T value) { + return Option(value); // NOTE: Make explicit + } + } } // namespace nextgen::core diff --git a/import/nextgen/io.h b/import/nextgen/io.h index e726aad..2b1228e 100644 --- a/import/nextgen/io.h +++ b/import/nextgen/io.h @@ -1,53 +1,57 @@ # ifndef NEXTGEN_IO_H # define NEXTGEN_IO_H + # include "allocator.h" -namespace nextgen { namespace io { +namespace nextgen { + namespace io { + + enum FileID { + JetSourceCode, + CSourceCode, + Mach_O, + ELF, + PE + }; - enum FileID { - JetSourceCode, - CSourceCode, - Mach_O, - ELF, - PE - }; + class FileBuf { + public: + using Files = FileBuf[]; - class FileBuf { - public: - using Files = FileBuf[]; + FileBuf(char *buffer, FileID id) : buffer(buffer), id(id) {} - FileBuf(char *buffer, FileID id) : buffer(buffer), id(id) {} + FileID getFileID() const { + return id; + } - FileID getFileID() const { - return id; - } + char *getFileBuffer() const { + return buffer; + } - char *getFileBuffer() const { - return buffer; - } + // Given a list of files, output the type of file given. For example, + // given a list of source code for Jet, and set it to output a binary + // ELF, it will output an ELF binary for the given source input. + // + // Example: + // FileBuf files[] = ... + // FileBuf::Output(files, FileID::CSourceCode); + // + // NOTE: The types of files need to be able to be linked together to form + // the resulting FileID file type. + static void Output(Files files, FileID output); - // Given a list of files, output the type of file given. For example, - // given a list of source code for Jet, and set it to output a binary - // ELF, it will output an ELF binary for the given source input. - // - // Example: - // FileBuf files[] = ... - // FileBuf::Output(files, FileID::CSourceCode); - // - // NOTE: The types of files need to be able to be linked together to form - // the resulting FileID file type. - static void Output(Files files, FileID output); - private: - char *buffer; // Buffer pointer to the source text of a file - FileID id; // File Type to know how file should be read - }; + private: + char *buffer; // Buffer pointer to the source text of a file + FileID id; // File Type to know how file should be read + }; - FileBuf CreateFileBuffer(const char *FILE, FileID id); + FileBuf CreateFileBuffer(const char *FILE, FileID id); -}} // namespace nextgen::io + } +} // namespace nextgen::io diff --git a/import/nextgen/jet/jet-analyzer.h b/import/nextgen/jet/jet-analyzer.h index 57281f5..3767d2b 100644 --- a/import/nextgen/jet/jet-analyzer.h +++ b/import/nextgen/jet/jet-analyzer.h @@ -4,19 +4,22 @@ #include "jet-parser.h" #include "jet-ast.h" -namespace nextgen { namespace jet { +namespace nextgen { + namespace jet { - class Analyzer { - VariableMap global_variables; - FunctionMap functions; - public: - void analyze(const ParserOutput output_to_analyze); - void analyze_function(const SyntaxFunction *function); - void analyze_syntax_node(const SyntaxNode *node); - }; -}} + class Analyzer { + VariableMap global_variables; + FunctionMap functions; + public: + void analyze(const ParserOutput output_to_analyze); + void analyze_function(const SyntaxFunction *function); + + void analyze_syntax_node(const SyntaxNode *node); + }; + } +} #endif //JET_JET_ANALYZER_H diff --git a/import/nextgen/jet/jet-ast.h b/import/nextgen/jet/jet-ast.h index 5281a28..bd8fb2e 100644 --- a/import/nextgen/jet/jet-ast.h +++ b/import/nextgen/jet/jet-ast.h @@ -1,169 +1,172 @@ # ifndef JET_AST_H # define JET_AST_H -namespace nextgen { namespace jet { - - enum ASTKind { - BinaryExprNode, - LiteralExprNode, - IfNode, - ElseNode, - ElifNode, - WhileNode, - ForListNode, - ForRangeNode, - Structure, - FunctionNode, - MatchNode - }; - - struct AST { - ASTKind kind; - }; - - enum TypeTag { - Void, - I8, - I16, - I32, - I65, - U8, - U16, - U32, - U64, - Str, - PointerTy, - ArrayView, - SmartPointer - }; - - struct Type { - TypeTag tag; - Type *ref; - }; - - struct Pointer : public Type { - Type pointer_of; - /*implicit*/ Pointer(const Type &p) : pointer_of(p) {} - }; - - struct ArrayView : public Type { - Type array_of; - size_t size = 0; - bool has_size = false; - /*implicit*/ ArrayView(const Type &a) : array_of(a) {} - }; - - struct Variable { - const Token *initial_decl; - const Type type; - bool is_mutable = false; - - struct intern_hash { - size_t operator()(Variable const &s) const { - return s.initial_decl->name().getHashCache(); - } - }; - - struct intern_eq { - bool operator()(Variable const &LHS, Variable const &RHS) const { - auto lhs = LHS.initial_decl->name(); - auto rhs = RHS.initial_decl->name(); - return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; - } - }; - }; - - struct Scope { - Scope *parent; - std::unordered_set variables; - }; - - struct NodeLiteral { - Type type; - const Token *literal_token; - }; - - struct NodeBinary { - enum Op { - MemberGet, // struct_instance.member_that_is_struct.its_member - Add, - Sub, - Div, - Mul, - Power, - Gtr, - GtrEq, - LessThanValue, - LessEq, - LShft, - LShftEq, - RShft, - RShftEq, - l_AND, // and - l_OR, // or - _XOR, // ^ - _AND, // & - _OR // | - }; - - const AST *lhs = nullptr; - const AST *rhs = nullptr; - Op op; - }; - - struct NodeWhile { - const AST *cond; - ArenaVec statements; - }; - - struct NodeElse { - ArenaVec statements; - }; - - struct NodeElif { - ArenaVec statements; - }; - - struct NodeIf { - const AST *cond; - NodeElif *else_; - Option> elif_branches; - }; - - struct FunctionParameter { - const Token *param_name; - const Type type; - }; - - struct NodeFunction { - const Token *name = nullptr; - ArenaVec statements; - ArenaVec parameters; - - struct intern_hash { - size_t operator()(NodeFunction const &s) const { - return s.name->name().getHashCache(); - } - }; - - struct intern_eq { - bool operator()(NodeFunction const &LHS, NodeFunction const &RHS) const { - auto lhs = LHS.name->name(); - auto rhs = RHS.name->name(); - return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; - } - }; - }; - - using VariableMap = std::unordered_set ; - using FunctionMap = std::unordered_set ; - -}} - +namespace nextgen { + namespace jet { + + enum ASTKind { + BinaryExprNode, + LiteralExprNode, + IfNode, + ElseNode, + ElifNode, + WhileNode, + ForListNode, + ForRangeNode, + Structure, + FunctionNode, + MatchNode + }; + + struct AST { + ASTKind kind; + }; + + enum TypeTag { + Void, + I8, + I16, + I32, + I65, + U8, + U16, + U32, + U64, + Str, + PointerTy, + ArrayView, + SmartPointer + }; + + struct Type { + TypeTag tag; + Type *ref; + }; + + struct Pointer : public Type { + Type pointer_of; + + /*implicit*/ Pointer(const Type &p) : pointer_of(p) {} + }; + + struct ArrayView : public Type { + Type array_of; + size_t size = 0; + bool has_size = false; + + /*implicit*/ ArrayView(const Type &a) : array_of(a) {} + }; + + struct Variable { + const Token *initial_decl; + const Type type; + bool is_mutable = false; + + struct intern_hash { + size_t operator()(Variable const &s) const { + return s.initial_decl->name().getHashCache(); + } + }; + + struct intern_eq { + bool operator()(Variable const &LHS, Variable const &RHS) const { + auto lhs = LHS.initial_decl->name(); + auto rhs = RHS.initial_decl->name(); + return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; + } + }; + }; + + struct Scope { + Scope *parent; + std::unordered_set variables; + }; + + struct NodeLiteral { + Type type; + const Token *literal_token; + }; + + struct NodeBinary { + enum Op { + MemberGet, // struct_instance.member_that_is_struct.its_member + Add, + Sub, + Div, + Mul, + Power, + Gtr, + GtrEq, + LessThanValue, + LessEq, + LShft, + LShftEq, + RShft, + RShftEq, + l_AND, // and + l_OR, // or + _XOR, // ^ + _AND, // & + _OR // | + }; + + const AST *lhs = nullptr; + const AST *rhs = nullptr; + Op op; + }; + + struct NodeWhile { + const AST *cond; + ArenaVec statements; + }; + + struct NodeElse { + ArenaVec statements; + }; + + struct NodeElif { + ArenaVec statements; + }; + + struct NodeIf { + const AST *cond; + NodeElif *else_; + Option > elif_branches; + }; + + struct FunctionParameter { + const Token *param_name; + const Type type; + }; + + struct NodeFunction { + const Token *name = nullptr; + ArenaVec statements; + ArenaVec parameters; + + struct intern_hash { + size_t operator()(NodeFunction const &s) const { + return s.name->name().getHashCache(); + } + }; + + struct intern_eq { + bool operator()(NodeFunction const &LHS, NodeFunction const &RHS) const { + auto lhs = LHS.name->name(); + auto rhs = RHS.name->name(); + return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; + } + }; + }; + + using VariableMap = std::unordered_set; + using FunctionMap = std::unordered_set; + + } +} # endif //JET_AST_H \ No newline at end of file diff --git a/import/nextgen/jet/jet-diagnostics.h b/import/nextgen/jet/jet-diagnostics.h index c94b76f..e36cf62 100644 --- a/import/nextgen/jet/jet-diagnostics.h +++ b/import/nextgen/jet/jet-diagnostics.h @@ -1,86 +1,97 @@ #ifndef JET_JET_DIAGNOSTICS_H #define JET_JET_DIAGNOSTICS_H + #include "nextgen/io.h" #include "nextgen/allocator.h" #include "nextgen/str.h" #include "nextgen/panic.h" #include "jet-token.h" -namespace nextgen { namespace jet { - - // Lex Errors - struct LexError; - - // Printing Mode re-analyzes the file, while printing out tokens with - // syntax-highlighting to the console. This allows for pretty-error messages - // with colored output. - enum LexMode { - TokenMode, - PrintingMode - }; - - template - class Lexer; - - // Parsing Errors - struct ParseError; - class Parser; - - class Diagnostic { - private: - const char *file_buf; - const char *file_name; - size_t buf_len; - - // Line where error has occurred - str source_line; - - public: - Diagnostic() = default; - - Diagnostic(const char *buffer, size_t buf_len, const - char *file_name) - : file_buf(buffer), file_name(file_name), buf_len(buf_len) {} - - /// Build error Message for a Lexing error - void build(LexError error); - - /// Build error Message for a Lexing error - void build(ParseError error); - - void send_exception() { - Console::Log(Colors::RED, "error: ", Colors::RESET, "Aborting due to " - "previous errors\n"); - throw std::exception(); - } - private: - // Helpers - void ErrorLexSetup(std::string &line, const char *message, LexError &error); - - template - void ErrorParseSetup(size_t const ln, - const char *message, - Token const *reported_token, - TokenTraits::SourceLocation loc); - - // Lex Errors (We will keep these PascalCase) - void ErrorIntegerOverflow(LexError &error); - void ErrorDigitOutOfRange(LexError &error); - void ErrorInvalidChar(LexError &error); - void ErrorMalformedUTF8(LexError &error); - void ErrorMissingClosingDelim(LexError &error); - void ErrorHexEscapeOutOfRange(LexError &error); - void ErrorInvalidStringEscape(LexError &error); - - // Parse Errors (We will keep these PascalCase) - void ErrorParseExpectedToken(ParseError const &error); - void ErrorParseMissingClosingDelim(ParseError const &error); - void ErrorParseInvalidTokenAfterIdentInGlobalScope(ParseError const &error); - }; - -}} +namespace nextgen { + namespace jet { + + // Lex Errors + struct LexError; + + // Printing Mode re-analyzes the file, while printing out tokens with + // syntax-highlighting to the console. This allows for pretty-error messages + // with colored output. + enum LexMode { + TokenMode, + PrintingMode + }; + + template + class Lexer; + + // Parsing Errors + struct ParseError; + + class Parser; + + class Diagnostic { + private: + const char *file_buf; + const char *file_name; + size_t buf_len; + + // Line where error has occurred + str source_line; + + public: + Diagnostic() = default; + + Diagnostic(const char *buffer, size_t buf_len, const + char *file_name) + : file_buf(buffer), file_name(file_name), buf_len(buf_len) {} + + /// Build error Message for a Lexing error + void build(LexError error); + + /// Build error Message for a Lexing error + void build(ParseError error); + + void send_exception() { + Console::Log(Colors::RED, "error: ", Colors::RESET, "Aborting due to " + "previous errors\n"); + throw std::exception(); + } + + private: + // Helpers + void ErrorLexSetup(std::string &line, const char *message, LexError &error); + + template + void ErrorParseSetup(size_t const ln, + const char *message, + Token const *reported_token, + TokenTraits::SourceLocation loc); + + // Lex Errors (We will keep these PascalCase) + void ErrorIntegerOverflow(LexError &error); + + void ErrorDigitOutOfRange(LexError &error); + + void ErrorInvalidChar(LexError &error); + + void ErrorMalformedUTF8(LexError &error); + + void ErrorMissingClosingDelim(LexError &error); + + void ErrorHexEscapeOutOfRange(LexError &error); + + void ErrorInvalidStringEscape(LexError &error); + + // Parse Errors (We will keep these PascalCase) + void ErrorParseExpectedToken(ParseError const &error); + + void ErrorParseMissingClosingDelim(ParseError const &error); + void ErrorParseInvalidTokenAfterIdentInGlobalScope(ParseError const &error); + }; + } +} #endif //JET_JET_DIAGNOSTICS_H diff --git a/import/nextgen/jet/jet-lexer.h b/import/nextgen/jet/jet-lexer.h index a8f744b..b5171d8 100644 --- a/import/nextgen/jet/jet-lexer.h +++ b/import/nextgen/jet/jet-lexer.h @@ -4,146 +4,152 @@ # include "jet-token.h" # include "jet-diagnostics.h" -namespace nextgen { namespace jet { using namespace nextgen::core; - - enum LexErrorType { - // Other - UnexpectedEOF, - OutOfRange, - Unreachable, - - // Strings - MalformedUTF8, - InvalidChar, - InvalidStringEscape, - InvalidUnicodeEscapeClose, - HexEscapeOutOfRange, - MissingClosingDelim, - - // Numbers - DigitOutOfRange, - InvalidDot, - InvalidNumberValue, - IntegerOverflow, - FloatingPointOverflow, - }; - - struct LexError { - // The type of error - LexErrorType error; - - // Error location - TokenTraits::SourceLocation location; - - // Token value lexed until error - Token failed_token; - - // Error metadata - int metadata[5]; - }; - - // Lexical analysis has two different modes for two purposes. Depending on - // the case, the lexer will either register tokens or it will simply, print - // tokens to the terminal. - // - // TokenMode: Specifies the mode in which tokens are lexed and output is - // returned. - template - class Lexer { - using File = char; - - const File *buffer; - const size_t buf_size; - - size_t line = 1; - size_t column = 1; - size_t buf_pos = 0; - - Diagnostic diagnostics; - ArenaVec tokens; - - bool fatal = false; - - // Ensure that branches are taken away during compile-time with guaranteed - // compile-time value if statements. This is used so that both printing mode - // and normal mode do not have to be separate functions. - static constexpr bool OUTPUT_MODE = Mode == LexMode::PrintingMode; - static constexpr bool NORMAL_MODE = Mode == LexMode::TokenMode; - - public: - friend class Parser; - - Lexer(const File *buffer, const char *file_name, - const size_t buffer_size) - : buffer(buffer), buf_size(buffer_size) { - diagnostics = Diagnostic(buffer, buffer_size, file_name); - } - - void lex_int(); - template - void lex_int(); - void lex_float(int skip, int start=0); - - void lex_str(); - void lex_ident(); - - ArenaVec lex(); - - - /// peek `NChars` characters in the file buffer. - char peek(size_t NChars) { - return *(buffer + NChars); - } - - /// Get the latest character in the file buffer - char curr() { - return *buffer; - } - - /// Move the lexer forward by `n` and fill in `Error` for - /// unexpected EOF or Out of Range Error that may be encountered. - char next(size_t n) { - - if (buf_pos >= buf_size) { - if (Mode == TokenMode) { - diagnostics.build(LexError { - LexErrorType::OutOfRange, - { line, column }, - Token("\0", {line, column}, TokenKind::EOFToken) - }); - UNREACHABLE; - } else fatal = true; - } - - column += n; - buf_pos += n; - buffer += n; - return *buffer; - } - - private: - - /// Matches a new line encountered in the buffer and handles - /// them appropriately depending on \n or \r. - void skip_new_line() { - line++; - column = 1; - - next(1); +namespace nextgen { + namespace jet { + using namespace nextgen::core; + + enum LexErrorType { + // Other + UnexpectedEOF, + OutOfRange, + Unreachable, + + // Strings + MalformedUTF8, + InvalidChar, + InvalidStringEscape, + InvalidUnicodeEscapeClose, + HexEscapeOutOfRange, + MissingClosingDelim, + + // Numbers + DigitOutOfRange, + InvalidDot, + InvalidNumberValue, + IntegerOverflow, + FloatingPointOverflow, + }; + + struct LexError { + // The type of error + LexErrorType error; + + // Error location + TokenTraits::SourceLocation location; + + // Token value lexed until error + Token failed_token; + + // Error metadata + int metadata[5]; + }; + + // Lexical analysis has two different modes for two purposes. Depending on + // the case, the lexer will either register tokens or it will simply, print + // tokens to the terminal. + // + // TokenMode: Specifies the mode in which tokens are lexed and output is + // returned. + template + class Lexer { + using File = char; + + const File *buffer; + const size_t buf_size; + + size_t line = 1; + size_t column = 1; + size_t buf_pos = 0; + + Diagnostic diagnostics; + ArenaVec tokens; + + bool fatal = false; + + // Ensure that branches are taken away during compile-time with guaranteed + // compile-time value if statements. This is used so that both printing mode + // and normal mode do not have to be separate functions. + static constexpr bool OUTPUT_MODE = Mode == LexMode::PrintingMode; + static constexpr bool NORMAL_MODE = Mode == LexMode::TokenMode; + + public: + friend class Parser; + + Lexer(const File *buffer, const char *file_name, + const size_t buffer_size) + : buffer(buffer), buf_size(buffer_size) { + diagnostics = Diagnostic(buffer, buffer_size, file_name); + } + + void lex_int(); + + template + void lex_int(); + + void lex_float(int skip, int start = 0); + + void lex_str(); + + void lex_ident(); + + ArenaVec lex(); + + + /// peek `NChars` characters in the file buffer. + char peek(size_t NChars) { + return *(buffer + NChars); + } + + /// Get the latest character in the file buffer + char curr() { + return *buffer; + } + + /// Move the lexer forward by `n` and fill in `Error` for + /// unexpected EOF or Out of Range Error that may be encountered. + char next(size_t n) { + + if (buf_pos >= buf_size) { + if (Mode == TokenMode) { + diagnostics.build(LexError{ + LexErrorType::OutOfRange, + {line, column}, + Token("\0", {line, column}, TokenKind::EOFToken) + }); + UNREACHABLE; + } else fatal = true; + } - char current = *buffer; - char next_ch = peek(1); + column += n; + buf_pos += n; + buffer += n; + return *buffer; + } - if (next_ch != current && (next_ch == '\n' || next_ch == '\r')) { - if (Mode == LexMode::PrintingMode) Console::Log(next_ch); - ++line, column = 1, next(1); - } - } - - }; + private: - -}} // namespace nextgen::jet + /// Matches a new line encountered in the buffer and handles + /// them appropriately depending on \n or \r. + void skip_new_line() { + line++; + column = 1; + + next(1); + + char current = *buffer; + char next_ch = peek(1); + + if (next_ch != current && (next_ch == '\n' || next_ch == '\r')) { + if (Mode == LexMode::PrintingMode) Console::Log(next_ch); + ++line, column = 1, next(1); + } + } + + }; + + + } +} // namespace nextgen::jet diff --git a/import/nextgen/jet/jet-parser.h b/import/nextgen/jet/jet-parser.h index fef7555..6998b63 100644 --- a/import/nextgen/jet/jet-parser.h +++ b/import/nextgen/jet/jet-parser.h @@ -1,362 +1,382 @@ # ifndef JET_JET_PARSER_H # define JET_JET_PARSER_H + # include "jet-lexer.h" # include "jet-syntax-nodes.h" -namespace nextgen { namespace jet { using namespace nextgen::core; - - enum ParseErrorType { - ReservedIdentifierAsVariableName, - UnexpectedExpression, - InvalidToken, - ExpectedToken, - - MissingFunctionName, - MissingVariableName, - MissingForLoopVariable, - - ExpectedIdentifierForFunctionParameter, - ExpectedIdentifierForStructProperty, - InvalidTokenAfterIdentInGlobalScope, - - MissingClosingPair, - UnexpectedEndOfFile - }; - - enum ParseContext { - AnonymousBlock, - StatementBlock, - LoopBlock, - LambdaBlock, - GeneralScope - }; - - - struct ParserOutput { - ObjectVector functions; - ObjectVector structures; - ObjectVector global_variables; - }; - - struct ParseError { - union Metadata; - - ParseErrorType error; - TokenTraits::SourceLocation location; - std::initializer_list metadata; - - /** @brief Contains specific error information pertaining to the error - * message. Only a few parser errors have specific items, while most are - * invalid or missing tokens - * */ - union Metadata { - struct { - TokenKind expected; - Token const *got; - const char *message; - } expected_error; - TokenTraits::SourceLocation location; - TokenKind const misc_kind; - Token const *misc_tok; - const char *context_message; - - // GCC and CLANG let us get away with not using union constructors, - // but we need them for MSVC, and it's probably more clear anyway - Metadata(TokenTraits::SourceLocation loc) : location(loc) {} - Metadata(TokenKind const kind) : misc_kind(kind) {} - Metadata(Token const *tok) : misc_tok(tok) {} - Metadata(const char *msg) : context_message(msg) {} - Metadata(TokenKind const expected, Token const *got, const char - *message) { - expected_error.expected = expected; - expected_error.got = got; - expected_error.message = message; - } - }; - - ParseError(ParseErrorType error, TokenTraits::SourceLocation loc) - : error(error), location(loc) {} - - ParseError(ParseErrorType error, TokenTraits::SourceLocation loc, - const std::initializer_list &metadata) - : error(error), location(loc), metadata(metadata) {} - }; - - class Parser { - - // Parser's position - size_t position = 0; - - // Parsing context (valid statements per context scope) - // TODO: Set this up so stuff actually works with this... - ParseContext context = GeneralScope; - - // Parsing errors - Diagnostic diagnostics; - - // Encountered error that results in stopped compilation - char fatal = 0; - - - public: - ArenaVec tokens; - Parser() = default; - - // Why are we passing a pointer here? Each pass in the frontend is done - // through the main function, thus, when we go ahead and pass the lexer, - // we know that it has the lifetime of the program. - explicit Parser(Lexer *lexer) - : tokens(lexer->lex()), diagnostics(lexer->diagnostics), - fatal(0) {} - - /// Parse all statements - auto parse() -> const ParserOutput; - - // ========= Parsing Language Generalizations ========== - auto NG_AINLINE curr() { return tokens[position]; } - - // ========= Parsing Utils ========== - - /** @brief Lookahead 'n' amount of times in the list */ - auto NG_AINLINE peek(size_t n) { - return tokens[position + n]; - } - - /** @brief Returns the current token and skips 'n' tokens ahead */ - auto NG_INLINE skip(size_t n) { - auto *ret = tokens[position]; - position += n; - return ret; - } - - /** @brief Skips 'n' tokens and returns the latest token after the skip */ - auto NG_INLINE next(size_t n) { - position += n; - return tokens[position]; - } - - /** @brief Asserts next token kind is 'TK' or errors with type 'PE' */ - template - auto skip() -> const Token* { - auto next = tokens[(position++)]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError( - PE, - next->getSourceLocation() - )); - } - return next; - } - - /** @brief Expect the next token to be 'kind', if not, build an error. */ - template - auto expect(char const (&msg)[N]) -> const Token* { - const Token *next = tokens[(position++)]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError( - ParseErrorType::ExpectedToken, - next->getSourceLocation(), - { - ParseError::Metadata { TK, next, msg } - } - )); - if (curr()->getKind() == EOFToken) - this->diagnostics.send_exception(); - else - fatal++; - } - return next; - } - - /** @brief Expected a closing delim in a statement or expression */ - template - void expect_delim(const TokenTraits::SourceLocation &loc) { - Token *next = tokens[position++]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError ( - ParseErrorType::MissingClosingPair, - next->getSourceLocation(), - { - ParseError::Metadata { TK }, - ParseError::Metadata { next }, - ParseError::Metadata { loc } - } - )); - this->diagnostics.send_exception(); - } - } - - - /** @brief Unary operator precedence */ - static NG_INLINE int UnaryOperatorBinding(TokenKind kind) { - switch (kind) { - case TokenKind::Plus: - case TokenKind::Minus: - case TokenKind::ExclamationPoint: - case TokenKind::NOT: - return 17; - default: - return -1; - } - } - - /** @brief Infix operator binding for expressions */ - static NG_INLINE - std::array InfixOperatorBinding(TokenKind kind) { - switch (kind) { - case TokenKind::Dot: - return {16, 15}; // Property value operator - case TokenKind::Star: - case TokenKind::Slash: - case TokenKind::Percent: - return {13, 14}; - case TokenKind::Plus: - case TokenKind::Minus: - return {11, 12}; - case TokenKind::GreaterThan: - case TokenKind::LessThan: - return {9, 10}; - case TokenKind::EqualsEquals: - return {7, 8}; - case TokenKind::KeywordAnd: - return {5, 6}; - case TokenKind::KeywordOr: - return {3, 4}; - case TokenKind::AND: - case TokenKind::NOT: - case TokenKind::XOR: - case TokenKind::Pipe: - return {1, 2}; - default: - return {-1, -1}; - } - } - auto parse_expr(int previous_binding = -1) -> const SyntaxNode*; - auto match_expr() -> const SyntaxNode*; - - // ========= Parsing Language Constructs ========== - - auto parse_block() -> SyntaxBlock; - auto parse_type() -> SyntaxType*; - auto parse_function_param() -> ArenaVec; - auto parse_variable_assignment(const Token *name) -> const SyntaxNode*; - - auto parse_function(const Token *name) -> const SyntaxFunction*; - auto parse_struct_data_members() -> ArenaVec; - auto parse_function_call(const Token *name, const Token *delim) -> const SyntaxNode*; - - auto parse_for() -> const SyntaxNode*; - auto parse_match() -> const SyntaxNode*; - auto parse_match_pair_value() -> const SyntaxNode*; - - //========== INLINED FUNCTIONS ========== - - - template - auto parse_if() -> const SyntaxNode* { - auto cond = parse_expr(); - auto body = parse_block(); - - SyntaxElse *else_ = nullptr; SyntaxElif *elif = nullptr; - auto is_else = next(1); - if (is_else->getKind() == TokenKind::KeywordElse) { - else_ = new SyntaxElse(parse_block()); - } - else if (is_else->getKind() == TokenKind::KeywordElif) { - elif = (SyntaxElif*)((SyntaxNode*) parse_if()); - } - auto E = (SyntaxNode*) new SyntaxIf(cond, body, else_, elif); - if (ELIF) - E->kind = SyntaxKind::Elif; - return E; - } - - auto NG_INLINE - parse_while() { - auto cond = parse_expr(); - auto body = parse_block(); - return new SyntaxWhile(cond, body); - } - - auto NG_INLINE - parse_variable_value_assignment(const Token *name, SyntaxAssignmentOp op) { - auto ret = new SyntaxVariableValueAssignment(name, parse_expr(), op); - expect("Expected ';' after declaration"); - return ret; - } - - auto parse_struct(const Token *name) -> const SyntaxStruct* { - expect("Opening curly brace is required " - "before struct member declarations"); - return new SyntaxStruct(name, parse_struct_data_members()); - } - - auto parse_stmt() -> const SyntaxNode* { - switch (curr()->getKind()) { - case KeywordIf: - skip(1); - return parse_if(); - case KeywordWhile: - context = ParseContext::LoopBlock; - skip(1); - return parse_while(); - case KeywordFor: - context = ParseContext::LoopBlock; - skip(1); - return parse_for(); - case KeywordBreak: { - if (context != ParseContext::LoopBlock) { - // TODO: Add Error - } - break; - } - case KeywordContinue: { - if (context != ParseContext::LoopBlock) { - // TODO: Add Error - } - break; - } - case KeywordDefer: - break; - case KeywordNone: - break; - case KeywordReturn: - break; - case KeywordMatch: - break; - case Identifier: { - auto C1 = peek(1); - if (C1->getKind() == TokenKind::ColonEquals) { - auto name = curr(); - skip(2); - return parse_variable_assignment(name); - } - if (C1->isValueAssignmentOp()) { - auto name = curr(); - skip(2); - return parse_variable_value_assignment(name, - SyntaxVariableValueAssignment::MatchOp(C1->getKind())); - } - break; - } - case LCurlyBrace: - break; - case Then: - break; - case At: - break; - case Error: - break; - case EOFToken: - break; - default: break; - } - next(1); - return nullptr; - } - - }; - }} +namespace nextgen { + namespace jet { + using namespace nextgen::core; + + enum ParseErrorType { + ReservedIdentifierAsVariableName, + UnexpectedExpression, + InvalidToken, + ExpectedToken, + + MissingFunctionName, + MissingVariableName, + MissingForLoopVariable, + + ExpectedIdentifierForFunctionParameter, + ExpectedIdentifierForStructProperty, + InvalidTokenAfterIdentInGlobalScope, + + MissingClosingPair, + UnexpectedEndOfFile + }; + + enum ParseContext { + AnonymousBlock, + StatementBlock, + LoopBlock, + LambdaBlock, + GeneralScope + }; + + + struct ParserOutput { + ObjectVector functions; + ObjectVector structures; + ObjectVector global_variables; + }; + + struct ParseError { + union Metadata; + + ParseErrorType error; + TokenTraits::SourceLocation location; + std::initializer_list metadata; + + /** @brief Contains specific error information pertaining to the error + * message. Only a few parser errors have specific items, while most are + * invalid or missing tokens + * */ + union Metadata { + struct { + TokenKind expected; + Token const *got; + const char *message; + } expected_error; + TokenTraits::SourceLocation location; + TokenKind const misc_kind; + Token const *misc_tok; + const char *context_message; + + // GCC and CLANG let us get away with not using union constructors, + // but we need them for MSVC, and it's probably more clear anyway + Metadata(TokenTraits::SourceLocation loc) : location(loc) {} + + Metadata(TokenKind const kind) : misc_kind(kind) {} + + Metadata(Token const *tok) : misc_tok(tok) {} + + Metadata(const char *msg) : context_message(msg) {} + + Metadata(TokenKind const expected, Token const *got, const char + *message) { + expected_error.expected = expected; + expected_error.got = got; + expected_error.message = message; + } + }; + + ParseError(ParseErrorType error, TokenTraits::SourceLocation loc) + : error(error), location(loc) {} + + ParseError(ParseErrorType error, TokenTraits::SourceLocation loc, + const std::initializer_list &metadata) + : error(error), location(loc), metadata(metadata) {} + }; + + class Parser { + + // Parser's position + size_t position = 0; + + // Parsing context (valid statements per context scope) + // TODO: Set this up so stuff actually works with this... + ParseContext context = GeneralScope; + + // Parsing errors + Diagnostic diagnostics; + + // Encountered error that results in stopped compilation + char fatal = 0; + + + public: + ArenaVec tokens; + + Parser() = default; + + // Why are we passing a pointer here? Each pass in the frontend is done + // through the main function, thus, when we go ahead and pass the lexer, + // we know that it has the lifetime of the program. + explicit Parser(Lexer *lexer) + : tokens(lexer->lex()), diagnostics(lexer->diagnostics), + fatal(0) {} + + /// Parse all statements + auto parse() -> const ParserOutput; + + // ========= Parsing Language Generalizations ========== + auto NG_AINLINE curr() { return tokens[position]; } + + // ========= Parsing Utils ========== + + /** @brief Lookahead 'n' amount of times in the list */ + auto NG_AINLINE peek(size_t n) { + return tokens[position + n]; + } + + /** @brief Returns the current token and skips 'n' tokens ahead */ + auto NG_INLINE skip(size_t n) { + auto *ret = tokens[position]; + position += n; + return ret; + } + + /** @brief Skips 'n' tokens and returns the latest token after the skip */ + auto NG_INLINE next(size_t n) { + position += n; + return tokens[position]; + } + + /** @brief Asserts next token kind is 'TK' or errors with type 'PE' */ + template + auto skip() -> const Token * { + auto next = tokens[(position++)]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + PE, + next->getSourceLocation() + )); + } + return next; + } + + /** @brief Expect the next token to be 'kind', if not, build an error. */ + template + auto expect(char const (&msg)[N]) -> const Token * { + const Token *next = tokens[(position++)]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + ParseErrorType::ExpectedToken, + next->getSourceLocation(), + { + ParseError::Metadata{TK, next, msg} + } + )); + if (curr()->getKind() == EOFToken) + this->diagnostics.send_exception(); + else + fatal++; + } + return next; + } + + /** @brief Expected a closing delim in a statement or expression */ + template + void expect_delim(const TokenTraits::SourceLocation &loc) { + Token *next = tokens[position++]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + ParseErrorType::MissingClosingPair, + next->getSourceLocation(), + { + ParseError::Metadata{TK}, + ParseError::Metadata{next}, + ParseError::Metadata{loc} + } + )); + this->diagnostics.send_exception(); + } + } + + + /** @brief Unary operator precedence */ + static NG_INLINE int UnaryOperatorBinding(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + case TokenKind::Minus: + case TokenKind::ExclamationPoint: + case TokenKind::NOT: + return 17; + default: + return -1; + } + } + + /** @brief Infix operator binding for expressions */ + static NG_INLINE + std::array InfixOperatorBinding(TokenKind kind) { + switch (kind) { + case TokenKind::Dot: + return {16, 15}; // Property value operator + case TokenKind::Star: + case TokenKind::Slash: + case TokenKind::Percent: + return {13, 14}; + case TokenKind::Plus: + case TokenKind::Minus: + return {11, 12}; + case TokenKind::GreaterThan: + case TokenKind::LessThan: + return {9, 10}; + case TokenKind::EqualsEquals: + return {7, 8}; + case TokenKind::KeywordAnd: + return {5, 6}; + case TokenKind::KeywordOr: + return {3, 4}; + case TokenKind::AND: + case TokenKind::NOT: + case TokenKind::XOR: + case TokenKind::Pipe: + return {1, 2}; + default: + return {-1, -1}; + } + } + + auto parse_expr(int previous_binding = -1) -> const SyntaxNode *; + + auto match_expr() -> const SyntaxNode *; + + // ========= Parsing Language Constructs ========== + + auto parse_block() -> SyntaxBlock; + + auto parse_type() -> SyntaxType *; + + auto parse_function_param() -> ArenaVec; + + auto parse_variable_assignment(const Token *name) -> const SyntaxNode *; + + auto parse_function(const Token *name) -> const SyntaxFunction *; + + auto parse_struct_data_members() -> ArenaVec; + + auto parse_function_call(const Token *name, const Token *delim) -> const SyntaxNode *; + + auto parse_for() -> const SyntaxNode *; + + auto parse_match() -> const SyntaxNode *; + + auto parse_match_pair_value() -> const SyntaxNode *; + + //========== INLINED FUNCTIONS ========== + + + template + auto parse_if() -> const SyntaxNode * { + auto cond = parse_expr(); + auto body = parse_block(); + + SyntaxElse *else_ = nullptr; + SyntaxElif *elif = nullptr; + auto is_else = next(1); + if (is_else->getKind() == TokenKind::KeywordElse) { + else_ = new SyntaxElse(parse_block()); + } else if (is_else->getKind() == TokenKind::KeywordElif) { + elif = (SyntaxElif *) ((SyntaxNode *) parse_if()); + } + auto E = (SyntaxNode *) new SyntaxIf(cond, body, else_, elif); + if (ELIF) + E->kind = SyntaxKind::Elif; + return E; + } + + auto NG_INLINE + parse_while() { + auto cond = parse_expr(); + auto body = parse_block(); + return new SyntaxWhile(cond, body); + } + + auto NG_INLINE + parse_variable_value_assignment(const Token *name, SyntaxAssignmentOp op) { + auto ret = new SyntaxVariableValueAssignment(name, parse_expr(), op); + expect("Expected ';' after declaration"); + return ret; + } + + auto parse_struct(const Token *name) -> const SyntaxStruct * { + expect("Opening curly brace is required " + "before struct member declarations"); + return new SyntaxStruct(name, parse_struct_data_members()); + } + + auto parse_stmt() -> const SyntaxNode * { + switch (curr()->getKind()) { + case KeywordIf: + skip(1); + return parse_if(); + case KeywordWhile: + context = ParseContext::LoopBlock; + skip(1); + return parse_while(); + case KeywordFor: + context = ParseContext::LoopBlock; + skip(1); + return parse_for(); + case KeywordBreak: { + if (context != ParseContext::LoopBlock) { + // TODO: Add Error + } + break; + } + case KeywordContinue: { + if (context != ParseContext::LoopBlock) { + // TODO: Add Error + } + break; + } + case KeywordDefer: + break; + case KeywordNone: + break; + case KeywordReturn: + break; + case KeywordMatch: + break; + case Identifier: { + auto C1 = peek(1); + if (C1->getKind() == TokenKind::ColonEquals) { + auto name = curr(); + skip(2); + return parse_variable_assignment(name); + } + if (C1->isValueAssignmentOp()) { + auto name = curr(); + skip(2); + return parse_variable_value_assignment(name, + SyntaxVariableValueAssignment::MatchOp( + C1->getKind())); + } + break; + } + case LCurlyBrace: + break; + case Then: + break; + case At: + break; + case Error: + break; + case EOFToken: + break; + default: + break; + } + next(1); + return nullptr; + } + + }; + } +} # endif //JET_JET_PARSER_H \ No newline at end of file diff --git a/import/nextgen/jet/jet-syntax-nodes.h b/import/nextgen/jet/jet-syntax-nodes.h index 2e1ca56..c2b89a9 100644 --- a/import/nextgen/jet/jet-syntax-nodes.h +++ b/import/nextgen/jet/jet-syntax-nodes.h @@ -1,644 +1,649 @@ #ifndef JET_JET_SYNTAX_NODES_H #define JET_JET_SYNTAX_NODES_H + # include "jet-token.h" -namespace nextgen { namespace jet { - - enum SyntaxKind { - LiteralValue, - Binary, - Unary, - List, - If, - Else, - Elif, - For, - While, - FunctionCall, - Struct, - ForList, - ForRange, - Enum, - VariableAssignment, - VariableValueAssignment, - VariableReassignment, - ArrayInitialization, - Assert, - FunctionDefault, - FunctionStructProperty, - Match, - }; - - enum SyntaxUnaryOp { - BitwiseNot, - Positive, - Negative, - ConditionalNot, - MemoryReference, - Inc, - Dec - }; - - enum SyntaxBinaryOp { - Addition, - Subtraction, - Division, - Multiplication, - Modulus, - Greater, - Less, - GreaterEqualTo, - LessEqualTo, - BooleanOr, - BooleanAnd, - BinaryShLeft, - BinaryShRight, - BinaryXOR, - BinaryAND, - BinaryOR - }; - - enum SyntaxTypeAnnotation { - Optional, // ?int - Reference, // &int - Pointer, // *int - Box, // box int - ArrayType // [] int - }; - - enum SyntaxTypenameKind { - Integer8, - Integer16, - Integer32, - Integer64, - UInteger8, - UInteger16, - UInteger32, - UInteger64, - Boxed, - Float32, - Float64, - StringValue, - UserDefined - }; - - enum SyntaxAssignmentOp { - AddAssign, - SubAssign, - MulAssign, - DivAssign, - ModAssign, - PowAssign, - LShiftAssign, - RShiftAssign, - ORAssign, - XORAssign, - ANDAssign, - }; - - struct SyntaxTypename { - SyntaxTypenameKind kind; - - // When dealing with user-defined structs, when expanding the type - // we need to ensure that the name is recorded somewhere so that - // type-checking can proceed in the third pass of the compiler. - Option> user_defined_typename; - - SyntaxTypename() = default; - SyntaxTypename(SyntaxTypenameKind kind, - const Option> &optional) - : kind(kind), user_defined_typename(optional) - {} - - bool operator ==(const SyntaxTypename &other) const { - if ( - kind == SyntaxTypenameKind::UserDefined - && - other.kind == SyntaxTypenameKind::UserDefined - ) { - auto first = user_defined_typename.unwrap(); - auto second = other.user_defined_typename.unwrap(); - return strncmp(first.begin, second.begin, first.range()); - } - return other.kind == kind; - } - }; - - struct SyntaxNode { - SyntaxKind kind; - SyntaxNode() = default; - }; - - - struct SyntaxLiteral : public SyntaxNode { - - explicit SyntaxLiteral(const Token *l) : literal(l) {} - - - // contains the literal token value. Token's already contain the actual - // data for the literal, and therefore nothing else is needed. The token - // is required for any diagnostics that could be recorded as they contain - // source location & pretty printing options. - // - // Literal examples can be seen in the [Lexer] class. - const Token *literal; - }; - - - struct SyntaxBinary : public SyntaxNode { - - // Binary expression syntax operation. - // Ex: - // + : Addition - // - : Subtraction - // * : Multiplication - // / : Division - // ** : Exponential - // << : Left Shift - // >> : Right Shift - // ^ : XOR - // | : OR - // & : AND - const SyntaxBinaryOp operation; - const Token *op; - - // Left and right grouped expressions. - const SyntaxNode *lhs; - const SyntaxNode *rhs; - - /// Convert a `TokenKind` to `SyntaxBinaryOp` kind - static NG_INLINE const SyntaxBinaryOp MatchOp(TokenKind kind) { - switch(kind) { - case TokenKind::Plus: - return SyntaxBinaryOp::Addition; - case TokenKind::Minus: - return SyntaxBinaryOp::Subtraction; - case TokenKind::Star: - return SyntaxBinaryOp::Multiplication; - case TokenKind::Slash: - return SyntaxBinaryOp::Division; - case TokenKind::Percent: - return SyntaxBinaryOp::Modulus; - case TokenKind::AND: - return SyntaxBinaryOp::BinaryAND; - case TokenKind::Pipe: - return SyntaxBinaryOp::BinaryOR; - case TokenKind::XOR: - return SyntaxBinaryOp::BinaryXOR; - case TokenKind::LeftShift: - return SyntaxBinaryOp::BinaryShLeft; - case TokenKind::RightShift: - return SyntaxBinaryOp::BinaryShRight; - case TokenKind::GreaterThan: - return SyntaxBinaryOp::Greater; - case TokenKind::GreaterThanEquals: - return SyntaxBinaryOp::GreaterEqualTo; - case TokenKind::LessThan: - return SyntaxBinaryOp::Less; - case TokenKind::LessThanEquals: - return SyntaxBinaryOp::LessEqualTo; - case TokenKind::KeywordOr: - return SyntaxBinaryOp::BooleanOr; - case TokenKind::KeywordAnd: - return SyntaxBinaryOp::BooleanAnd; - default: - UNREACHABLE; - } - } - - SyntaxBinary(const SyntaxBinaryOp op, - const Token *oo, - const SyntaxNode *lhs, - const SyntaxNode*rhs) - : operation(op), op(oo), lhs(lhs), rhs(rhs) {} - }; - - struct SyntaxType : public SyntaxNode { - SyntaxType() = default; - SyntaxType(const Option &modifier, - const Option &ty, - const SyntaxType *type) - : modifier(modifier), ty_name(ty), type(type) {} - - - const Option modifier = None; - const Option ty_name = None; - const SyntaxType *type = nullptr; - - static NG_INLINE SyntaxTypename MatchTypename(const Token *kind) { - switch (kind->getKind()) { - case Typename_str: - return SyntaxTypename(SyntaxTypenameKind::StringValue, None); - case Typename_i8: - return SyntaxTypename(SyntaxTypenameKind::Integer8, None); - case Typename_i16: - return SyntaxTypename(SyntaxTypenameKind::Integer16, None); - case Typename_i32: - return SyntaxTypename(SyntaxTypenameKind::Integer32, None); - case Typename_i64: - return SyntaxTypename(SyntaxTypenameKind::Integer64, None); - case Typename_u8: - return SyntaxTypename(SyntaxTypenameKind::UInteger8, None); - case Typename_u16: - return SyntaxTypename(SyntaxTypenameKind::UInteger16, None); - case Typename_u32: - return SyntaxTypename(SyntaxTypenameKind::UInteger32, None); - case Typename_u64: - return SyntaxTypename(SyntaxTypenameKind::UInteger64, None); - case Typename_box: - return SyntaxTypename(SyntaxTypenameKind::Boxed, None); - case Identifier: - return SyntaxTypename(SyntaxTypenameKind::UserDefined, - (Range)kind->name()); - default: - UNREACHABLE; - } - } - - - bool has_typename() const { - return ty_name.isSome(); - } - - bool has_modifier() const { - return modifier.isSome(); - } - - bool has_type() const { - return type; - } - - bool operator ==(const SyntaxType &other) const { - auto ret = (other.has_typename() && has_typename()) - && (other.has_modifier() && has_modifier()) - && (other.has_type()); - if (ret) { - return (*type) == (*other.type) - && ty_name.contains(other.ty_name.unwrap()) - && modifier.contains(other.modifier.unwrap()); - } - return ret; - } - - const SyntaxTypename get_typename() const { - if (!has_typename()) { - type->get_typename(); - } - return ty_name.unwrap(); - } - - - // *int = SyntaxType { modifier: Pointer, ty_name: None, - // type: SyntaxType { - // modifier: None, - // ty_name: Integer32, - // type: None - // } - - }; - - struct SyntaxList : public SyntaxNode { - ObjectVector values; - }; - - struct SyntaxVariableValueAssignment : public SyntaxNode { - const Token *name = nullptr; - const SyntaxNode *expression = nullptr; - const SyntaxAssignmentOp op; - - static NG_INLINE SyntaxAssignmentOp MatchOp(TokenKind kind) { - switch(kind) { - case PlusEquals: - return SyntaxAssignmentOp::AddAssign; - case MinusEquals: - return SyntaxAssignmentOp::SubAssign; - case DivEquals: - return SyntaxAssignmentOp::DivAssign; - case PowEquals: - return SyntaxAssignmentOp::PowAssign; - case MulEquals: - return SyntaxAssignmentOp::MulAssign; - case LeftShiftEquals: - return SyntaxAssignmentOp::LShiftAssign; - case RightShiftEquals: - return SyntaxAssignmentOp::RShiftAssign; - case ANDEquals: - return SyntaxAssignmentOp::ANDAssign; - case XOREquals: - return SyntaxAssignmentOp::XORAssign; - case OREquals: - return SyntaxAssignmentOp::ORAssign; - case PercentEquals: - return SyntaxAssignmentOp::ModAssign; - default: - break; - } - } - - SyntaxVariableValueAssignment(const Token *name, - const SyntaxNode *expr, - const SyntaxAssignmentOp op) - : name(name), expression(expr), op(op) - { this->kind = SyntaxKind::VariableAssignment; } - - }; - - struct SyntaxVariableAssignment : public SyntaxNode { - - SyntaxVariableAssignment(const Token *name, - const Option &ty, - const SyntaxNode*expr) - : name(name), type(ty), expression(expr) { - this->kind = SyntaxKind::VariableAssignment; - } - - // The actual variable name; ex: var my_variable_name = ... - // ^^^^^^^^^^^^^^^^ - const Token *name = nullptr; - - // Optional variable type. It may need to be inferred so it can be set - // to 'None'. ex: (var ident: int) = 23 OR (var ident = 23) - // ^^^ ^ No type here - Option type = None; - - // Node representing the actual value that the variable contains - const SyntaxNode *expression; - }; - - - struct SyntaxVariableReassignment : public SyntaxNode { - - SyntaxVariableReassignment(const Token *v, const SyntaxNode *expr) - : variable(v), expression(expr) {} - - // Identifier representing the variable that is being reassigned. - // Ex: var some_var = 23 - // ^^^^^^^^ First declared here - // some_var = 242 - // ^^^^^^^^ Reassigned here. Note: You cannot reassign variables with - // values that are not the same as its first initialization. For example, - // the variable `some_var` is declared as an 'int' and cannot be - // reassigned with a value of 'str'. - const Token *variable = nullptr; - - // Node representing the actual value that the variable contains - const SyntaxNode *expression = nullptr; - }; - - struct SyntaxBlock : public SyntaxNode { - - // A sequential list of statements categorized as a current node. - // Ex: - // { - // x := 23 - // x = x ** x - // a_function_call(x) - // } - // There are 3 statements here, and the current will contain 3 statements. - // Note: There is an exception to parsing of this node, in statements such - // as 'while', 'if', 'elif', 'else', 'for' may contain a single statement - // after them without a set of corresponding '{}'. - ObjectVector statements; - - - // Special statement for function / lambda return check. Allows for void - // function type inference. - bool contains_return = false; - }; - - struct SyntaxElse : public SyntaxNode { - - explicit SyntaxElse(SyntaxBlock block) : body(block) {} - - - // Else statement current in an if-elif-else statement chain - // Ex: - // if cond { ... } - // else { ... } - // ^^^^ Declared here. Note that the curly braces may be omitted for one - // statement bodies. - SyntaxBlock body; - }; - - struct SyntaxElif : public SyntaxNode { - - explicit SyntaxElif(SyntaxBlock block) : body(block) {} - - - // Elif statement current in an if-elif-else statement chain - // Ex: - // if cond { ... } - // elif cond { ... } - // ^^^^ Declared here. Note that the curly braces may be omitted for one - // statement bodies. - SyntaxBlock body; - }; - - struct SyntaxWhile : public SyntaxNode { - // Boolean expression representing the while condition. - // Ex: while cond - // ^^^^ Declared here - const SyntaxNode *condition; - // Statement list of commands executed upon the condition above being true - // Ex: while cond { - // variable := "Hello" - // } - SyntaxBlock body; - - SyntaxWhile(const SyntaxNode *cond, const SyntaxBlock &body) - : condition(cond), body(body) { - this->kind = SyntaxKind::While; - } - }; - - struct SyntaxMatch : public SyntaxNode { - - struct MatchPair { - const SyntaxNode *value; - SyntaxBlock action; - }; - - const SyntaxNode *value; - ArenaVec pairs; - }; - - struct SyntaxIf : public SyntaxNode { - - // Boolean expression representing the if condition. - // Ex: if cond - // ^^^^ Declared here - const SyntaxNode *condition; - - // Statement list of commands executed upon the condition above being true - // Ex: if cond { - // flag = true - // } - SyntaxBlock body; - - // See `SyntaxElse` - const SyntaxElse *else_ = nullptr; - - // See `SyntaxElif` - const SyntaxNode *elif = nullptr; - - SyntaxIf(const SyntaxNode *cond, - const SyntaxBlock &body, - const SyntaxElse *else_, - const SyntaxNode *elif) - : condition(cond), body(body), else_(else_), elif(elif) { - this->kind = SyntaxKind::If; - } - }; - - struct SyntaxUnary : public SyntaxNode { - - // Unary expression. We denote a few unary operators: - // ++ : Increment and return latest value - // -- : Decrement and return latest value - // & : Take the memory address of the expression on the right - // * : Dereference the expression on the right - // + : Assert RHS expression is positive - // - : Negate RHS expression - // ~ : Bitwise NOT - const SyntaxUnaryOp operation; - const Token *op; - - // Expression to apply unary operation - // Ex: ++some_var, &memory_address, *pointer_value, -some_value - const SyntaxNode *expression = nullptr; - - /// Convert `TokenKind` to `SyntaxUnaryOp` operation - static NG_INLINE const SyntaxUnaryOp MatchOp(TokenKind kind) { - switch(kind) { - case TokenKind::Plus: - return SyntaxUnaryOp::Positive; - case TokenKind::NOT: - return SyntaxUnaryOp::BitwiseNot; - case TokenKind::Minus: - return SyntaxUnaryOp::Negative; - case TokenKind::PlusPlus: - break; - case TokenKind::MinusMinus: - break; - case TokenKind::ExclamationPoint: - return SyntaxUnaryOp::ConditionalNot; - default: - UNREACHABLE; - } - UNREACHABLE; - } - - SyntaxUnary(const SyntaxUnaryOp op, - const Token *oo, - const SyntaxNode *expr) - : operation(op), op(oo), expression(expr) { - this->kind = SyntaxKind::Unary; - } - }; - - struct SyntaxFunctionParameter : public SyntaxNode { - - SyntaxFunctionParameter(const Token *pn, SyntaxType *ty) - : param_name(pn), type(ty) {} - - // Function parameter name - const Token *param_name = nullptr; - - // Function Parameter type. If no type is given, then it is inferred to - // be a generic parameter. - // Ex: - // fn some_func(param_1: int, param_2: str, $param_3) - // ^^^^^^^^ Generic Parameter - const SyntaxType *type; - }; - - struct SyntaxFunction : public SyntaxNode { - - // Function name - const Token *function_name = nullptr; - - // Function return type. If 'None' then infer return type. - Option function_type; - - // Function body - SyntaxBlock body; - - // Function parameters - ArenaVec parameters{nullptr, nullptr}; - - SyntaxFunction(const Token *name, - const Option &ty, - const SyntaxBlock &body, - const ArenaVec ¶ms) - : function_name(name), body(body), function_type(ty), - parameters(params){} - }; - - struct SyntaxFunctionCall : public SyntaxNode { - - // NOTE: Function Declarations should be lowered - // before function call declarations. - // - // Ex: - // some_function(1, 2, 2, 3, 5) - const Token *function_name = nullptr; - ObjectVector parameters; - - SyntaxFunctionCall(const Token *name) - : function_name(name) - { this->kind = SyntaxKind::FunctionCall; } - }; - - struct SyntaxForList : public SyntaxNode { - SyntaxForList(const Token *ln, const Token *lv, SyntaxBlock body) - : list_name(ln), list_var(lv), body(body) {} - - // For loop iteration over a variable of type list. - // Ex: - // for num in numbers { ... } - // ^^^ Loop var ^^^^^^^ List variable - const Token *list_name = nullptr; - const Token *list_var = nullptr; - SyntaxBlock body; - }; - - struct RangeFunction { - size_t set[3] = {0, 0, 0}; - }; - - struct SyntaxForRange : public SyntaxNode { - SyntaxForRange(const Token *list_var, const RangeFunction - range, SyntaxBlock body) : - list_var(list_var), range(range), body(body) {} - - // Range based for loop. A range consists of a few factors: - // start, end, and step. - // Start - the value that initializes the loop variable. - // End - the last value the variable should loop to. - // Step - is how much a loop variable should change. A step of 2, for - // example, will increment the loop variable by 2 each time. - - // Ex: - // for i in range(1, 5) { ... } OR - // for i in range(1, 232, 3) { ... } - const Token *list_var = nullptr; - const RangeFunction range; - SyntaxBlock body; - }; - - struct SyntaxStructMember { - const Token *name = nullptr; - SyntaxType *type; - SyntaxStructMember(const Token *name, SyntaxType *ty) - : name(name), type(ty) {} - }; - - struct SyntaxStruct : public SyntaxNode { - const Token *name = nullptr; - ArenaVec members; - SyntaxStruct(const Token *name, const ArenaVec - &members) : name(name), members(members) {} - }; - -}} +namespace nextgen { + namespace jet { + + enum SyntaxKind { + LiteralValue, + Binary, + Unary, + List, + If, + Else, + Elif, + For, + While, + FunctionCall, + Struct, + ForList, + ForRange, + Enum, + VariableAssignment, + VariableValueAssignment, + VariableReassignment, + ArrayInitialization, + Assert, + FunctionDefault, + FunctionStructProperty, + Match, + }; + + enum SyntaxUnaryOp { + BitwiseNot, + Positive, + Negative, + ConditionalNot, + MemoryReference, + Inc, + Dec + }; + + enum SyntaxBinaryOp { + Addition, + Subtraction, + Division, + Multiplication, + Modulus, + Greater, + Less, + GreaterEqualTo, + LessEqualTo, + BooleanOr, + BooleanAnd, + BinaryShLeft, + BinaryShRight, + BinaryXOR, + BinaryAND, + BinaryOR + }; + + enum SyntaxTypeAnnotation { + Optional, // ?int + Reference, // &int + Pointer, // *int + Box, // box int + ArrayType // [] int + }; + + enum SyntaxTypenameKind { + Integer8, + Integer16, + Integer32, + Integer64, + UInteger8, + UInteger16, + UInteger32, + UInteger64, + Boxed, + Float32, + Float64, + StringValue, + UserDefined + }; + + enum SyntaxAssignmentOp { + AddAssign, + SubAssign, + MulAssign, + DivAssign, + ModAssign, + PowAssign, + LShiftAssign, + RShiftAssign, + ORAssign, + XORAssign, + ANDAssign, + }; + + struct SyntaxTypename { + SyntaxTypenameKind kind; + + // When dealing with user-defined structs, when expanding the type + // we need to ensure that the name is recorded somewhere so that + // type-checking can proceed in the third pass of the compiler. + Option > user_defined_typename; + + SyntaxTypename() = default; + + SyntaxTypename(SyntaxTypenameKind kind, + const Option > &optional) + : kind(kind), user_defined_typename(optional) {} + + bool operator==(const SyntaxTypename &other) const { + if ( + kind == SyntaxTypenameKind::UserDefined + && + other.kind == SyntaxTypenameKind::UserDefined + ) { + auto first = user_defined_typename.unwrap(); + auto second = other.user_defined_typename.unwrap(); + return strncmp(first.begin, second.begin, first.range()); + } + return other.kind == kind; + } + }; + + struct SyntaxNode { + SyntaxKind kind; + + SyntaxNode() = default; + }; + + + struct SyntaxLiteral : public SyntaxNode { + + explicit SyntaxLiteral(const Token *l) : literal(l) {} + + + // contains the literal token value. Token's already contain the actual + // data for the literal, and therefore nothing else is needed. The token + // is required for any diagnostics that could be recorded as they contain + // source location & pretty printing options. + // + // Literal examples can be seen in the [Lexer] class. + const Token *literal; + }; + + + struct SyntaxBinary : public SyntaxNode { + + // Binary expression syntax operation. + // Ex: + // + : Addition + // - : Subtraction + // * : Multiplication + // / : Division + // ** : Exponential + // << : Left Shift + // >> : Right Shift + // ^ : XOR + // | : OR + // & : AND + const SyntaxBinaryOp operation; + const Token *op; + + // Left and right grouped expressions. + const SyntaxNode *lhs; + const SyntaxNode *rhs; + + /// Convert a `TokenKind` to `SyntaxBinaryOp` kind + static NG_INLINE const SyntaxBinaryOp MatchOp(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + return SyntaxBinaryOp::Addition; + case TokenKind::Minus: + return SyntaxBinaryOp::Subtraction; + case TokenKind::Star: + return SyntaxBinaryOp::Multiplication; + case TokenKind::Slash: + return SyntaxBinaryOp::Division; + case TokenKind::Percent: + return SyntaxBinaryOp::Modulus; + case TokenKind::AND: + return SyntaxBinaryOp::BinaryAND; + case TokenKind::Pipe: + return SyntaxBinaryOp::BinaryOR; + case TokenKind::XOR: + return SyntaxBinaryOp::BinaryXOR; + case TokenKind::LeftShift: + return SyntaxBinaryOp::BinaryShLeft; + case TokenKind::RightShift: + return SyntaxBinaryOp::BinaryShRight; + case TokenKind::GreaterThan: + return SyntaxBinaryOp::Greater; + case TokenKind::GreaterThanEquals: + return SyntaxBinaryOp::GreaterEqualTo; + case TokenKind::LessThan: + return SyntaxBinaryOp::Less; + case TokenKind::LessThanEquals: + return SyntaxBinaryOp::LessEqualTo; + case TokenKind::KeywordOr: + return SyntaxBinaryOp::BooleanOr; + case TokenKind::KeywordAnd: + return SyntaxBinaryOp::BooleanAnd; + default: + UNREACHABLE; + } + } + + SyntaxBinary(const SyntaxBinaryOp op, + const Token *oo, + const SyntaxNode *lhs, + const SyntaxNode *rhs) + : operation(op), op(oo), lhs(lhs), rhs(rhs) {} + }; + + struct SyntaxType : public SyntaxNode { + SyntaxType() = default; + + SyntaxType(const Option &modifier, + const Option &ty, + const SyntaxType *type) + : modifier(modifier), ty_name(ty), type(type) {} + + + const Option modifier = None; + const Option ty_name = None; + const SyntaxType *type = nullptr; + + static NG_INLINE SyntaxTypename MatchTypename(const Token *kind) { + switch (kind->getKind()) { + case Typename_str: + return SyntaxTypename(SyntaxTypenameKind::StringValue, None); + case Typename_i8: + return SyntaxTypename(SyntaxTypenameKind::Integer8, None); + case Typename_i16: + return SyntaxTypename(SyntaxTypenameKind::Integer16, None); + case Typename_i32: + return SyntaxTypename(SyntaxTypenameKind::Integer32, None); + case Typename_i64: + return SyntaxTypename(SyntaxTypenameKind::Integer64, None); + case Typename_u8: + return SyntaxTypename(SyntaxTypenameKind::UInteger8, None); + case Typename_u16: + return SyntaxTypename(SyntaxTypenameKind::UInteger16, None); + case Typename_u32: + return SyntaxTypename(SyntaxTypenameKind::UInteger32, None); + case Typename_u64: + return SyntaxTypename(SyntaxTypenameKind::UInteger64, None); + case Typename_box: + return SyntaxTypename(SyntaxTypenameKind::Boxed, None); + case Identifier: + return SyntaxTypename(SyntaxTypenameKind::UserDefined, + (Range) kind->name()); + default: + UNREACHABLE; + } + } + + + bool has_typename() const { + return ty_name.isSome(); + } + + bool has_modifier() const { + return modifier.isSome(); + } + + bool has_type() const { + return type; + } + + bool operator==(const SyntaxType &other) const { + auto ret = (other.has_typename() && has_typename()) + && (other.has_modifier() && has_modifier()) + && (other.has_type()); + if (ret) { + return (*type) == (*other.type) + && ty_name.contains(other.ty_name.unwrap()) + && modifier.contains(other.modifier.unwrap()); + } + return ret; + } + + const SyntaxTypename get_typename() const { + if (!has_typename()) { + type->get_typename(); + } + return ty_name.unwrap(); + } + + + // *int = SyntaxType { modifier: Pointer, ty_name: None, + // type: SyntaxType { + // modifier: None, + // ty_name: Integer32, + // type: None + // } + + }; + + struct SyntaxList : public SyntaxNode { + ObjectVector values; + }; + + struct SyntaxVariableValueAssignment : public SyntaxNode { + const Token *name = nullptr; + const SyntaxNode *expression = nullptr; + const SyntaxAssignmentOp op; + + static NG_INLINE SyntaxAssignmentOp MatchOp(TokenKind kind) { + switch (kind) { + case PlusEquals: + return SyntaxAssignmentOp::AddAssign; + case MinusEquals: + return SyntaxAssignmentOp::SubAssign; + case DivEquals: + return SyntaxAssignmentOp::DivAssign; + case PowEquals: + return SyntaxAssignmentOp::PowAssign; + case MulEquals: + return SyntaxAssignmentOp::MulAssign; + case LeftShiftEquals: + return SyntaxAssignmentOp::LShiftAssign; + case RightShiftEquals: + return SyntaxAssignmentOp::RShiftAssign; + case ANDEquals: + return SyntaxAssignmentOp::ANDAssign; + case XOREquals: + return SyntaxAssignmentOp::XORAssign; + case OREquals: + return SyntaxAssignmentOp::ORAssign; + case PercentEquals: + return SyntaxAssignmentOp::ModAssign; + default: + break; + } + } + + SyntaxVariableValueAssignment(const Token *name, + const SyntaxNode *expr, + const SyntaxAssignmentOp op) + : name(name), expression(expr), op(op) { this->kind = SyntaxKind::VariableAssignment; } + + }; + + struct SyntaxVariableAssignment : public SyntaxNode { + + SyntaxVariableAssignment(const Token *name, + const Option &ty, + const SyntaxNode *expr) + : name(name), type(ty), expression(expr) { + this->kind = SyntaxKind::VariableAssignment; + } + + // The actual variable name; ex: var my_variable_name = ... + // ^^^^^^^^^^^^^^^^ + const Token *name = nullptr; + + // Optional variable type. It may need to be inferred so it can be set + // to 'None'. ex: (var ident: int) = 23 OR (var ident = 23) + // ^^^ ^ No type here + Option type = None; + + // Node representing the actual value that the variable contains + const SyntaxNode *expression; + }; + + + struct SyntaxVariableReassignment : public SyntaxNode { + + SyntaxVariableReassignment(const Token *v, const SyntaxNode *expr) + : variable(v), expression(expr) {} + + // Identifier representing the variable that is being reassigned. + // Ex: var some_var = 23 + // ^^^^^^^^ First declared here + // some_var = 242 + // ^^^^^^^^ Reassigned here. Note: You cannot reassign variables with + // values that are not the same as its first initialization. For example, + // the variable `some_var` is declared as an 'int' and cannot be + // reassigned with a value of 'str'. + const Token *variable = nullptr; + + // Node representing the actual value that the variable contains + const SyntaxNode *expression = nullptr; + }; + + struct SyntaxBlock : public SyntaxNode { + + // A sequential list of statements categorized as a current node. + // Ex: + // { + // x := 23 + // x = x ** x + // a_function_call(x) + // } + // There are 3 statements here, and the current will contain 3 statements. + // Note: There is an exception to parsing of this node, in statements such + // as 'while', 'if', 'elif', 'else', 'for' may contain a single statement + // after them without a set of corresponding '{}'. + ObjectVector statements; + + + // Special statement for function / lambda return check. Allows for void + // function type inference. + bool contains_return = false; + }; + + struct SyntaxElse : public SyntaxNode { + + explicit SyntaxElse(SyntaxBlock block) : body(block) {} + + + // Else statement current in an if-elif-else statement chain + // Ex: + // if cond { ... } + // else { ... } + // ^^^^ Declared here. Note that the curly braces may be omitted for one + // statement bodies. + SyntaxBlock body; + }; + + struct SyntaxElif : public SyntaxNode { + + explicit SyntaxElif(SyntaxBlock block) : body(block) {} + + + // Elif statement current in an if-elif-else statement chain + // Ex: + // if cond { ... } + // elif cond { ... } + // ^^^^ Declared here. Note that the curly braces may be omitted for one + // statement bodies. + SyntaxBlock body; + }; + + struct SyntaxWhile : public SyntaxNode { + // Boolean expression representing the while condition. + // Ex: while cond + // ^^^^ Declared here + const SyntaxNode *condition; + // Statement list of commands executed upon the condition above being true + // Ex: while cond { + // variable := "Hello" + // } + SyntaxBlock body; + + SyntaxWhile(const SyntaxNode *cond, const SyntaxBlock &body) + : condition(cond), body(body) { + this->kind = SyntaxKind::While; + } + }; + + struct SyntaxMatch : public SyntaxNode { + + struct MatchPair { + const SyntaxNode *value; + SyntaxBlock action; + }; + + const SyntaxNode *value; + ArenaVec pairs; + }; + + struct SyntaxIf : public SyntaxNode { + + // Boolean expression representing the if condition. + // Ex: if cond + // ^^^^ Declared here + const SyntaxNode *condition; + + // Statement list of commands executed upon the condition above being true + // Ex: if cond { + // flag = true + // } + SyntaxBlock body; + + // See `SyntaxElse` + const SyntaxElse *else_ = nullptr; + + // See `SyntaxElif` + const SyntaxNode *elif = nullptr; + + SyntaxIf(const SyntaxNode *cond, + const SyntaxBlock &body, + const SyntaxElse *else_, + const SyntaxNode *elif) + : condition(cond), body(body), else_(else_), elif(elif) { + this->kind = SyntaxKind::If; + } + }; + + struct SyntaxUnary : public SyntaxNode { + + // Unary expression. We denote a few unary operators: + // ++ : Increment and return latest value + // -- : Decrement and return latest value + // & : Take the memory address of the expression on the right + // * : Dereference the expression on the right + // + : Assert RHS expression is positive + // - : Negate RHS expression + // ~ : Bitwise NOT + const SyntaxUnaryOp operation; + const Token *op; + + // Expression to apply unary operation + // Ex: ++some_var, &memory_address, *pointer_value, -some_value + const SyntaxNode *expression = nullptr; + + /// Convert `TokenKind` to `SyntaxUnaryOp` operation + static NG_INLINE const SyntaxUnaryOp MatchOp(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + return SyntaxUnaryOp::Positive; + case TokenKind::NOT: + return SyntaxUnaryOp::BitwiseNot; + case TokenKind::Minus: + return SyntaxUnaryOp::Negative; + case TokenKind::PlusPlus: + break; + case TokenKind::MinusMinus: + break; + case TokenKind::ExclamationPoint: + return SyntaxUnaryOp::ConditionalNot; + default: + UNREACHABLE; + } + UNREACHABLE; + } + + SyntaxUnary(const SyntaxUnaryOp op, + const Token *oo, + const SyntaxNode *expr) + : operation(op), op(oo), expression(expr) { + this->kind = SyntaxKind::Unary; + } + }; + + struct SyntaxFunctionParameter : public SyntaxNode { + + SyntaxFunctionParameter(const Token *pn, SyntaxType *ty) + : param_name(pn), type(ty) {} + + // Function parameter name + const Token *param_name = nullptr; + + // Function Parameter type. If no type is given, then it is inferred to + // be a generic parameter. + // Ex: + // fn some_func(param_1: int, param_2: str, $param_3) + // ^^^^^^^^ Generic Parameter + const SyntaxType *type; + }; + + struct SyntaxFunction : public SyntaxNode { + + // Function name + const Token *function_name = nullptr; + + // Function return type. If 'None' then infer return type. + Option function_type; + + // Function body + SyntaxBlock body; + + // Function parameters + ArenaVec parameters{nullptr, nullptr}; + + SyntaxFunction(const Token *name, + const Option &ty, + const SyntaxBlock &body, + const ArenaVec ¶ms) + : function_name(name), body(body), function_type(ty), + parameters(params) {} + }; + + struct SyntaxFunctionCall : public SyntaxNode { + + // NOTE: Function Declarations should be lowered + // before function call declarations. + // + // Ex: + // some_function(1, 2, 2, 3, 5) + const Token *function_name = nullptr; + ObjectVector parameters; + + SyntaxFunctionCall(const Token *name) + : function_name(name) { this->kind = SyntaxKind::FunctionCall; } + }; + + struct SyntaxForList : public SyntaxNode { + SyntaxForList(const Token *ln, const Token *lv, SyntaxBlock body) + : list_name(ln), list_var(lv), body(body) {} + + // For loop iteration over a variable of type list. + // Ex: + // for num in numbers { ... } + // ^^^ Loop var ^^^^^^^ List variable + const Token *list_name = nullptr; + const Token *list_var = nullptr; + SyntaxBlock body; + }; + + struct RangeFunction { + size_t set[3] = {0, 0, 0}; + }; + + struct SyntaxForRange : public SyntaxNode { + SyntaxForRange(const Token *list_var, const RangeFunction + range, SyntaxBlock body) : + list_var(list_var), range(range), body(body) {} + + // Range based for loop. A range consists of a few factors: + // start, end, and step. + // Start - the value that initializes the loop variable. + // End - the last value the variable should loop to. + // Step - is how much a loop variable should change. A step of 2, for + // example, will increment the loop variable by 2 each time. + + // Ex: + // for i in range(1, 5) { ... } OR + // for i in range(1, 232, 3) { ... } + const Token *list_var = nullptr; + const RangeFunction range; + SyntaxBlock body; + }; + + struct SyntaxStructMember { + const Token *name = nullptr; + SyntaxType *type; + + SyntaxStructMember(const Token *name, SyntaxType *ty) + : name(name), type(ty) {} + }; + + struct SyntaxStruct : public SyntaxNode { + const Token *name = nullptr; + ArenaVec members; + + SyntaxStruct(const Token *name, const ArenaVec + &members) : name(name), members(members) {} + }; + + } +} #endif //JET_JET_SYNTAX_NODES_H diff --git a/import/nextgen/jet/jet-token.h b/import/nextgen/jet/jet-token.h index 960be9f..51739ff 100644 --- a/import/nextgen/jet/jet-token.h +++ b/import/nextgen/jet/jet-token.h @@ -1,534 +1,537 @@ # ifndef NEXTGEN_TOKEN_H # define NEXTGEN_TOKEN_H + # include # include -namespace nextgen { namespace jet { - - enum TokenClassification : unsigned short { - Function = 1 << 2, - Assignment = 1 << 3, - Unsigned = 1 << 4, - Keyword = 1 << 5, - Literal = 1 << 6, - }; - - enum TokenKind { - - // Primitives --------------------------- - Integer = 1, - Zero = 1, - String, - Decimal, - Boolean, - //--------------------------------------- - - // Valid Expression Tokens (DO NOT CHANGE ORDER) --------------------------- - Identifier, // [a-z] && [A-Z] && _ - LessThan, // < - GreaterThan, // > - - Plus, // + - Minus, // - - Slash, // / - Star, // * - XOR, // ^ - AND, // & - NOT, // ~ - Pipe, // | - Percent,// % - Char, // \' - Dot, // . - - // TODO: Fix this ... Should not be apart of binary expression. It should - // be apart of the assignment modifier. - PlusEquals, - MinusEquals, - DivEquals, - PowEquals, - MulEquals, - ANDEquals, - XOREquals, - OREquals, - PercentEquals, - LeftShiftEquals, - RightShiftEquals, - GreaterThanEquals, - LessThanEquals, - - EqualsEquals, // == - LeftShift, // << - RightShift, // >> - Pow, // ** - //--------------------------------------- - - - // Keywords --------------------------- - KeywordIf, - KeywordElse, - KeywordElif, - KeywordWhile, - KeywordFor, - KeywordAnd, - KeywordOr, - KeywordBreak, - KeywordContinue, - KeywordDefer, - KeywordStruct, - KeywordEnum, - KeywordExport, - KeywordExtern, - KeywordFunction, - KeywordError, - KeywordVar, - KeywordTrue, - KeywordFalse, - KeywordNone, - KeywordReturn, - KeywordUnion, - KeywordMatch, - KeywordIn, - KeywordRange, - //--------------------------------------- - - // Typename --------------------------- - Typename_str, - Typename_i8, - Typename_i16, - Typename_i32, - Typename_i64, - Typename_u8, - Typename_u16, - Typename_u32, - Typename_u64, - Typename_box, - Typename_f32, - Typename_f64, - //--------------------------------------- - - // File Control --------------------------- - NewLine, - Whitespace, - StringEscapeStart, - //--------------------------------------- - - LParenthesis, // ( - RParenthesis, // ) - LCurlyBrace, // { - RCurlyBrace, // } - LBracket, // [ - RBracket, // ] - Colon, // : - Equals, // = - QuestionMark, // ? - Then, // ?? - At, // @ - SemiColon, // ; - ExclamationPoint, // ! - Comma, // , - - RangeSpan, // .. - Ellipsis, // ... - - PlusPlus, // ++ - MinusMinus, // -- - FunctionArrow,// => - - ColonEquals, // := - Arrow, // -> - - Error, - EOFToken, - }; - - struct TokenTraits { - - using SourceSpan = Range; - - struct SourceLocation { - size_t line; - size_t column; - SourceSpan source; - }; - - // We have to very careful here. We do not have access to std::variant<...> - // in C++11, so we must use a controlled union to get a Token's value. - struct Value { - union { - decltype(UINTPTR_MAX) integer; - bool boolean; - double float64; - char character; - }; - }; - - }; - - class Token { - private: - using TokenValue = TokenTraits::Value; - using SourceLocation = TokenTraits::SourceLocation; - - str id; - - unsigned flags = TokenClassification::Literal; - SourceLocation loc = {}; - - - - TokenKind kind; - public: - Token() = default; - - union { - decltype(UINTPTR_MAX) integer; - bool boolean; - double float64; - char character; - }; - - template - Token(const str &id, SourceLocation loc, T value, - TokenKind kind, unsigned flags = -1) - : id(id), flags(flags), loc(loc), kind(kind) { - setValue(value); - } - - // Note: This is only for lex errors. Failed tokens are important - // for the diagnostic class generate valid errors. - Token(str id, SourceLocation loc, TokenKind kind) - : id(id), loc(loc) { - setKind(kind); - } - - static NG_INLINE char const *GetTokenKindName(TokenKind const kind) { - switch(kind){ - case LessThan: - return "<"; - case GreaterThan: - return ">"; - case Plus: - return "+"; - case Minus: - return "-"; - case Slash: - return "/"; - case Star: - return "*"; - case XOR: - return "^"; - case AND: - return "&"; - case NOT: - return "~"; - case Pipe: - return "|"; - case Percent: - return "%"; - case Char: - return "'"; - case Dot: - return "."; - case PlusEquals: - return "+="; - case MinusEquals: - return "-="; - case DivEquals: - return "/="; - case PowEquals: - return "**="; - case MulEquals: - return "*="; - case GreaterThanEquals: - return ">="; - case LessThanEquals: - return "<="; - case LeftShiftEquals: - return "<<="; - case RightShiftEquals: - return ">>="; - case ANDEquals: - return "&="; - case XOREquals: - return "^="; - case OREquals: - return "|="; - case PercentEquals: - return "%="; - case EqualsEquals: - return "=="; - case LeftShift: - return "<<"; - case RightShift: - return ">>"; - case Pow: - return "**"; - case KeywordIf: - return "if"; - case KeywordElse: - return "else"; - case KeywordElif: - return "elif"; - case KeywordWhile: - return "while"; - case KeywordFor: - return "for"; - case KeywordAnd: - return "and"; - case KeywordOr: - return "or"; - case KeywordBreak: - return "break"; - case KeywordContinue: - return "continue"; - case KeywordDefer: - return "defer"; - case KeywordStruct: - return "struct"; - case KeywordEnum: - return "enum"; - case KeywordExport: - return "export"; - case KeywordExtern: - return "extern"; - case KeywordFunction: - return "fn"; - case KeywordError: - return "error"; - case KeywordTrue: - return "true"; - case KeywordFalse: - return "false"; - case KeywordNone: - return "None"; - case KeywordReturn: - return "return"; - case KeywordUnion: - return "union"; - case KeywordMatch: - return "match"; - case KeywordIn: - return "in"; - case KeywordRange: - return "range"; - case Typename_str: - return "str"; - case Typename_i8: - return "i8"; - case Typename_i16: - return "i16"; - case Typename_i32: - return "i32"; - case Typename_i64: - return "i64"; - case Typename_u8: - return "u8"; - case Typename_u16: - return "u16"; - case Typename_u32: - return "u32"; - case Typename_u64: - return "u64"; - case Typename_box: - return "box"; - case Typename_f32: - return "f32"; - case Typename_f64: - return "f64"; - case LParenthesis: - return "("; - case RParenthesis: - return ")"; - case LCurlyBrace: - return "{"; - case RCurlyBrace: - return "}"; - case LBracket: - return "["; - case RBracket: - return "]"; - case Colon: - return ":"; - case Equals: - return "="; - case QuestionMark: - return "?"; - case Then: - return "??"; - case At: - return "@"; - case SemiColon: - return ";"; - case ExclamationPoint: - return "!"; - case Comma: - return ","; - case RangeSpan: - return ".."; - case Ellipsis: - return "..."; - case PlusPlus: - return "++"; - case MinusMinus: - return "--"; - case FunctionArrow: - return "=>"; - case ColonEquals: - return ":="; - case Arrow: - return "->"; - case EOFToken: - return "EOF"; - default: - UNREACHABLE; - } - } - - size_t len() const { - return id.size(); - } - - str name() const { - return id; - } - - template - void setValue(T v) { - setInternalRepr(v); - } - - void setKind(TokenKind k) { - this->kind = k; - } - - TokenKind getKind() const { - return this->kind; - } - - bool isKeyword() const { - return flags & TokenClassification::Keyword; - } - - bool isLiteral() const { - return flags & TokenClassification::Literal; - } - - bool isClosingDelim() const { - return kind == TokenKind::RBracket || kind == TokenKind::RCurlyBrace - || kind == TokenKind::RParenthesis; - } - - bool isValidExpressionType() const { - return unsigned(kind) >= Integer && unsigned(kind) <= Pow; - } - - bool isValueAssignmentOp() const { - return unsigned(kind) >= PlusEquals && unsigned(kind) <= RightShiftEquals; - } - - bool isValidTypename() const { - return unsigned (kind) >= Typename_str && unsigned(kind) <= - Typename_f64 || kind == Identifier; - } - - bool isValidTypenameStart() const { - return kind == Star || kind == LBracket || kind == LParenthesis; - } - - SourceLocation getSourceLocation() const { - return loc; - } - - void setFlag(TokenClassification flag) { - flags |= flag; - } - - template - T getValue() const { - static_assert(std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value, - "Attempting to get invalid value from Token."); - - T v; setFromInternalRepr(v); - return v; - } - - /// Pretty Print individual token to the screen. - /// TODO: May not need this anymore ... - void PrettyPrint() const { - if (isKeyword()) { - Console::Log(Colors::RED); - } - auto Kind = getKind(); - switch (Kind) { - case String: - Console::Log(Colors::GREEN); - break; - case Integer: - case Decimal: - Console::Log(Colors::BLUE); - break; - case Char: - Console::Log(Colors::GREEN, "\'"); - break; - case Identifier: - Console::Log(Colors::YELLOW); - break; - default: - break; - } - Console::Log(name()); - Console::Log(Colors::RESET); - } - - bool assignment = false; // (ie: +=, -=, *=) - private: - - // This is the restricted usage of union values. - // We are guaranteed to know the actual type of the value from the - // [Token] constructor, therefore setting and retrieving values is not UB. - - - void setFromInternalRepr(double &V) const { - V = float64; - } - - void setFromInternalRepr(decltype(UINTPTR_MAX) &V) const { - V = integer; - } - - void setFromInternalRepr(char &V) const { - V = character; - } - - void setFromInternalRepr(bool &V) const { - V = boolean; - } - - void setInternalRepr(double V) { - float64 = V; - } - - void setInternalRepr(bool V) { - boolean = V; - } - - void setInternalRepr(char V) { - character = V; - } - - void setInternalRepr(decltype(UINTPTR_MAX) V) { - integer = V; - } - - void setInternalRepr(const char *V) { - integer = (unsigned long long) V; - } - - }; - -}} // namespace nextgen::jet +namespace nextgen { + namespace jet { + + enum TokenClassification : unsigned short { + Function = 1 << 2, + Assignment = 1 << 3, + Unsigned = 1 << 4, + Keyword = 1 << 5, + Literal = 1 << 6, + }; + + enum TokenKind { + + // Primitives --------------------------- + Integer = 1, + Zero = 1, + String, + Decimal, + Boolean, + //--------------------------------------- + + // Valid Expression Tokens (DO NOT CHANGE ORDER) --------------------------- + Identifier, // [a-z] && [A-Z] && _ + LessThan, // < + GreaterThan, // > + + Plus, // + + Minus, // - + Slash, // / + Star, // * + XOR, // ^ + AND, // & + NOT, // ~ + Pipe, // | + Percent,// % + Char, // \' + Dot, // . + + // TODO: Fix this ... Should not be apart of binary expression. It should + // be apart of the assignment modifier. + PlusEquals, + MinusEquals, + DivEquals, + PowEquals, + MulEquals, + ANDEquals, + XOREquals, + OREquals, + PercentEquals, + LeftShiftEquals, + RightShiftEquals, + GreaterThanEquals, + LessThanEquals, + + EqualsEquals, // == + LeftShift, // << + RightShift, // >> + Pow, // ** + //--------------------------------------- + + + // Keywords --------------------------- + KeywordIf, + KeywordElse, + KeywordElif, + KeywordWhile, + KeywordFor, + KeywordAnd, + KeywordOr, + KeywordBreak, + KeywordContinue, + KeywordDefer, + KeywordStruct, + KeywordEnum, + KeywordExport, + KeywordExtern, + KeywordFunction, + KeywordError, + KeywordVar, + KeywordTrue, + KeywordFalse, + KeywordNone, + KeywordReturn, + KeywordUnion, + KeywordMatch, + KeywordIn, + KeywordRange, + //--------------------------------------- + + // Typename --------------------------- + Typename_str, + Typename_i8, + Typename_i16, + Typename_i32, + Typename_i64, + Typename_u8, + Typename_u16, + Typename_u32, + Typename_u64, + Typename_box, + Typename_f32, + Typename_f64, + //--------------------------------------- + + // File Control --------------------------- + NewLine, + Whitespace, + StringEscapeStart, + //--------------------------------------- + + LParenthesis, // ( + RParenthesis, // ) + LCurlyBrace, // { + RCurlyBrace, // } + LBracket, // [ + RBracket, // ] + Colon, // : + Equals, // = + QuestionMark, // ? + Then, // ?? + At, // @ + SemiColon, // ; + ExclamationPoint, // ! + Comma, // , + + RangeSpan, // .. + Ellipsis, // ... + + PlusPlus, // ++ + MinusMinus, // -- + FunctionArrow,// => + + ColonEquals, // := + Arrow, // -> + + Error, + EOFToken, + }; + + struct TokenTraits { + + using SourceSpan = Range; + + struct SourceLocation { + size_t line; + size_t column; + SourceSpan source; + }; + + // We have to very careful here. We do not have access to std::variant<...> + // in C++11, so we must use a controlled union to get a Token's value. + struct Value { + union { + decltype(UINTPTR_MAX) integer; + bool boolean; + double float64; + char character; + }; + }; + + }; + + class Token { + private: + using TokenValue = TokenTraits::Value; + using SourceLocation = TokenTraits::SourceLocation; + + str id; + + unsigned flags = TokenClassification::Literal; + SourceLocation loc = {}; + + + TokenKind kind; + public: + Token() = default; + + union { + decltype(UINTPTR_MAX) integer; + bool boolean; + double float64; + char character; + }; + + template + Token(const str &id, SourceLocation loc, T value, + TokenKind kind, unsigned flags = -1) + : id(id), flags(flags), loc(loc), kind(kind) { + setValue(value); + } + + // Note: This is only for lex errors. Failed tokens are important + // for the diagnostic class generate valid errors. + Token(str id, SourceLocation loc, TokenKind kind) + : id(id), loc(loc) { + setKind(kind); + } + + static NG_INLINE char const *GetTokenKindName(TokenKind const kind) { + switch (kind) { + case LessThan: + return "<"; + case GreaterThan: + return ">"; + case Plus: + return "+"; + case Minus: + return "-"; + case Slash: + return "/"; + case Star: + return "*"; + case XOR: + return "^"; + case AND: + return "&"; + case NOT: + return "~"; + case Pipe: + return "|"; + case Percent: + return "%"; + case Char: + return "'"; + case Dot: + return "."; + case PlusEquals: + return "+="; + case MinusEquals: + return "-="; + case DivEquals: + return "/="; + case PowEquals: + return "**="; + case MulEquals: + return "*="; + case GreaterThanEquals: + return ">="; + case LessThanEquals: + return "<="; + case LeftShiftEquals: + return "<<="; + case RightShiftEquals: + return ">>="; + case ANDEquals: + return "&="; + case XOREquals: + return "^="; + case OREquals: + return "|="; + case PercentEquals: + return "%="; + case EqualsEquals: + return "=="; + case LeftShift: + return "<<"; + case RightShift: + return ">>"; + case Pow: + return "**"; + case KeywordIf: + return "if"; + case KeywordElse: + return "else"; + case KeywordElif: + return "elif"; + case KeywordWhile: + return "while"; + case KeywordFor: + return "for"; + case KeywordAnd: + return "and"; + case KeywordOr: + return "or"; + case KeywordBreak: + return "break"; + case KeywordContinue: + return "continue"; + case KeywordDefer: + return "defer"; + case KeywordStruct: + return "struct"; + case KeywordEnum: + return "enum"; + case KeywordExport: + return "export"; + case KeywordExtern: + return "extern"; + case KeywordFunction: + return "fn"; + case KeywordError: + return "error"; + case KeywordTrue: + return "true"; + case KeywordFalse: + return "false"; + case KeywordNone: + return "None"; + case KeywordReturn: + return "return"; + case KeywordUnion: + return "union"; + case KeywordMatch: + return "match"; + case KeywordIn: + return "in"; + case KeywordRange: + return "range"; + case Typename_str: + return "str"; + case Typename_i8: + return "i8"; + case Typename_i16: + return "i16"; + case Typename_i32: + return "i32"; + case Typename_i64: + return "i64"; + case Typename_u8: + return "u8"; + case Typename_u16: + return "u16"; + case Typename_u32: + return "u32"; + case Typename_u64: + return "u64"; + case Typename_box: + return "box"; + case Typename_f32: + return "f32"; + case Typename_f64: + return "f64"; + case LParenthesis: + return "("; + case RParenthesis: + return ")"; + case LCurlyBrace: + return "{"; + case RCurlyBrace: + return "}"; + case LBracket: + return "["; + case RBracket: + return "]"; + case Colon: + return ":"; + case Equals: + return "="; + case QuestionMark: + return "?"; + case Then: + return "??"; + case At: + return "@"; + case SemiColon: + return ";"; + case ExclamationPoint: + return "!"; + case Comma: + return ","; + case RangeSpan: + return ".."; + case Ellipsis: + return "..."; + case PlusPlus: + return "++"; + case MinusMinus: + return "--"; + case FunctionArrow: + return "=>"; + case ColonEquals: + return ":="; + case Arrow: + return "->"; + case EOFToken: + return "EOF"; + default: + UNREACHABLE; + } + } + + size_t len() const { + return id.size(); + } + + str name() const { + return id; + } + + template + void setValue(T v) { + setInternalRepr(v); + } + + void setKind(TokenKind k) { + this->kind = k; + } + + TokenKind getKind() const { + return this->kind; + } + + bool isKeyword() const { + return flags & TokenClassification::Keyword; + } + + bool isLiteral() const { + return flags & TokenClassification::Literal; + } + + bool isClosingDelim() const { + return kind == TokenKind::RBracket || kind == TokenKind::RCurlyBrace + || kind == TokenKind::RParenthesis; + } + + bool isValidExpressionType() const { + return unsigned(kind) >= Integer && unsigned(kind) <= Pow; + } + + bool isValueAssignmentOp() const { + return unsigned(kind) >= PlusEquals && unsigned(kind) <= RightShiftEquals; + } + + bool isValidTypename() const { + return unsigned(kind) >= Typename_str && unsigned(kind) <= + Typename_f64 || kind == Identifier; + } + + bool isValidTypenameStart() const { + return kind == Star || kind == LBracket || kind == LParenthesis; + } + + SourceLocation getSourceLocation() const { + return loc; + } + + void setFlag(TokenClassification flag) { + flags |= flag; + } + + template + T getValue() const { + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + "Attempting to get invalid value from Token."); + + T v; + setFromInternalRepr(v); + return v; + } + + /// Pretty Print individual token to the screen. + /// TODO: May not need this anymore ... + void PrettyPrint() const { + if (isKeyword()) { + Console::Log(Colors::RED); + } + auto Kind = getKind(); + switch (Kind) { + case String: + Console::Log(Colors::GREEN); + break; + case Integer: + case Decimal: + Console::Log(Colors::BLUE); + break; + case Char: + Console::Log(Colors::GREEN, "\'"); + break; + case Identifier: + Console::Log(Colors::YELLOW); + break; + default: + break; + } + Console::Log(name()); + Console::Log(Colors::RESET); + } + + bool assignment = false; // (ie: +=, -=, *=) + private: + + // This is the restricted usage of union values. + // We are guaranteed to know the actual type of the value from the + // [Token] constructor, therefore setting and retrieving values is not UB. + + + void setFromInternalRepr(double &V) const { + V = float64; + } + + void setFromInternalRepr(decltype(UINTPTR_MAX) &V) const { + V = integer; + } + + void setFromInternalRepr(char &V) const { + V = character; + } + + void setFromInternalRepr(bool &V) const { + V = boolean; + } + + void setInternalRepr(double V) { + float64 = V; + } + + void setInternalRepr(bool V) { + boolean = V; + } + + void setInternalRepr(char V) { + character = V; + } + + void setInternalRepr(decltype(UINTPTR_MAX) V) { + integer = V; + } + + void setInternalRepr(const char *V) { + integer = (unsigned long long) V; + } + + }; + + } +} // namespace nextgen::jet # endif //NEXTGEN_TOKEN_H diff --git a/import/nextgen/jet/jet-x64-assembler.h b/import/nextgen/jet/jet-x64-assembler.h index 061a6ae..2e9b166 100644 --- a/import/nextgen/jet/jet-x64-assembler.h +++ b/import/nextgen/jet/jet-x64-assembler.h @@ -7,7 +7,7 @@ #include "../core-util.h" -typedef uint8_t ubyte; +typedef uint8_t ubyte; typedef uint16_t ushort; typedef uint32_t uint; typedef uint64_t ulong; @@ -60,301 +60,363 @@ void name##ss(Register reg, const MemoryAddress &addr);\ void name##sd(Register dest, Register src);\ void name##sd(Register reg, const MemoryAddress &addr);\ -namespace nextgen { namespace jet { namespace x64 { - -typedef struct Imm8 { ubyte value; } Imm8; -typedef struct Imm16 { ushort value; } Imm16; -typedef struct Imm32 { uint value; } Imm32; -typedef struct Imm64 { ulong value; } Imm64; - -enum Condition { - Overflow = 0, - NotOverflow = 1, - Below = 2, - AboveOrEqual = 3, - Equal = 4, - NotEqual = 5, - BelowOrEqual = 6, - Above = 7, - Signed = 8, - NotSigned = 9, - LessThan = 0xC, - LessThanEqual = 0xE, - GreaterThan = 0xF, - GreaterThanEqual = 0xD, - }; - -enum REX { - REX_PREFIX = 0b01000000, - REX_W = 0b00001000, - REX_R = 0b00000100, - REX_X = 0b00000010, - REX_B = 0b00000001 -}; - -enum MOD { - MOD_INDIRECT = 0b00, - MOD_DISP8 = 0b01, - MOD_DISP32 = 0b10, - MOD_REG = 0b11 -}; - - -enum Register : ubyte { - AX = 0b000, CX = 0b001, DX = 0b010, BX = 0b011, - SP = 0b100, BP = 0b101, SI = 0b110, DI = 0b111, - - R8, R9, R10, R11, R12, R13, R14, R15, - - XMM0 = 0b000, XMM1 = 0b001, XMM2 = 0b010, XMM3 = 0b011, - XMM4 = 0b100, XMM5 = 0b101, XMM6 = 0b110, XMM7 = 0b111, - - XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15 -}; - - -enum OperandSize { - BYTE, - WORD, - DWORD, - QWORD, - }; +namespace nextgen { + namespace jet { + namespace x64 { + + typedef struct Imm8 { + ubyte value; + } Imm8; + typedef struct Imm16 { + ushort value; + } Imm16; + typedef struct Imm32 { + uint value; + } Imm32; + typedef struct Imm64 { + ulong value; + } Imm64; + + enum Condition { + Overflow = 0, + NotOverflow = 1, + Below = 2, + AboveOrEqual = 3, + Equal = 4, + NotEqual = 5, + BelowOrEqual = 6, + Above = 7, + Signed = 8, + NotSigned = 9, + LessThan = 0xC, + LessThanEqual = 0xE, + GreaterThan = 0xF, + GreaterThanEqual = 0xD, + }; + + enum REX { + REX_PREFIX = 0b01000000, + REX_W = 0b00001000, + REX_R = 0b00000100, + REX_X = 0b00000010, + REX_B = 0b00000001 + }; + + enum MOD { + MOD_INDIRECT = 0b00, + MOD_DISP8 = 0b01, + MOD_DISP32 = 0b10, + MOD_REG = 0b11 + }; + + + enum Register : ubyte { + AX = 0b000, CX = 0b001, DX = 0b010, BX = 0b011, + SP = 0b100, BP = 0b101, SI = 0b110, DI = 0b111, + + R8, R9, R10, R11, R12, R13, R14, R15, + + XMM0 = 0b000, XMM1 = 0b001, XMM2 = 0b010, XMM3 = 0b011, + XMM4 = 0b100, XMM5 = 0b101, XMM6 = 0b110, XMM7 = 0b111, + + XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15 + }; + + + enum OperandSize { + BYTE, + WORD, + DWORD, + QWORD, + }; // Reference: https://blog.yossarian.net/2020/06/13/How-x86_64-addresses-memory -enum MemoryAddressKind { - Disp32, // [disp32] - Base, // [base] - BaseIndex, // [base + index] - BaseDisp32, // [base + disp32] - BaseDisp8, // [base + disp8] - BaseIndexScale, // [base + (index * scale)] - BaseIndexDisp32, // [base + (index * scale) + disp32] - BaseIndexDisp8, // [base + (index * scale) + disp8] -}; - -enum SibScale : ubyte { - Scale1 = 0b00, - Scale2 = 0b01, - Scale4 = 0b10, - Scale8 = 0b11 -}; - -struct MemoryAddress { - bool has_disp8 = false; - bool has_disp32 = false; - bool has_sib = false; + enum MemoryAddressKind { + Disp32, // [disp32] + Base, // [base] + BaseIndex, // [base + index] + BaseDisp32, // [base + disp32] + BaseDisp8, // [base + disp8] + BaseIndexScale, // [base + (index * scale)] + BaseIndexDisp32, // [base + (index * scale) + disp32] + BaseIndexDisp8, // [base + (index * scale) + disp8] + }; - ubyte scale = Scale1; - ubyte mod = MOD_INDIRECT; - uint disp = 0; + enum SibScale : ubyte { + Scale1 = 0b00, + Scale2 = 0b01, + Scale4 = 0b10, + Scale8 = 0b11 + }; - Register base = BP; - Register index = SP; + struct MemoryAddress { + bool has_disp8 = false; + bool has_disp32 = false; + bool has_sib = false; - // [disp32] - explicit MemoryAddress(uint disp32); + ubyte scale = Scale1; + ubyte mod = MOD_INDIRECT; + uint disp = 0; - // [base] - explicit MemoryAddress(Register base); + Register base = BP; + Register index = SP; - // [base + disp8] - explicit MemoryAddress(Register base, ubyte disp8); + // [disp32] + explicit MemoryAddress(uint disp32); - // [base + disp32] - explicit MemoryAddress(Register base, uint disp32); + // [base] + explicit MemoryAddress(Register base); - // [base + (index * scale)] - explicit MemoryAddress(Register base, - Register index, - SibScale scale); + // [base + disp8] + explicit MemoryAddress(Register base, ubyte disp8); - // [base + (index * scale) + disp8] - explicit MemoryAddress(Register base, Register index, SibScale, - ubyte disp8); + // [base + disp32] + explicit MemoryAddress(Register base, uint disp32); - // [base + (index * scale) + disp32] - explicit MemoryAddress(Register base, Register index, SibScale, - uint disp32); -}; + // [base + (index * scale)] + explicit MemoryAddress(Register base, + Register index, + SibScale scale); -class Assembler { -private: - ubyte *memory; - size_t used = 0; + // [base + (index * scale) + disp8] + explicit MemoryAddress(Register base, Register index, SibScale, + ubyte disp8); - // Writing primitive values to buffer - void write(ubyte value); - void write_16(ushort value); - void write_32(uint value); - void write_64(ulong value); + // [base + (index * scale) + disp32] + explicit MemoryAddress(Register base, Register index, SibScale, + uint disp32); + }; - // Write Immediate values to the buffer - void write(Imm8 imm); - void write(Imm16 imm); - void write(Imm32 imm); - void write(Imm64 imm); + class Assembler { + private: + ubyte *memory; + size_t used = 0; - // Write a memory address given the register destination - /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html - void write_address(const MemoryAddress &addr, Register dest); + // Writing primitive values to buffer + void write(ubyte value); + void write_16(ushort value); - // Some useful overrides when writing multiple values. - // There is no need for a custom implementation because - // we don't write more than 4 values at a given time in a row. - // - // We depend on the compiler on choosing the correct function for the - // value that is passed in -- the choice should be clear given distinct - // parameter values. + void write_32(uint value); - template - void write(T f, K s); + void write_64(ulong value); - template - void write(T f, K s, L t); + // Write Immediate values to the buffer + void write(Imm8 imm); - template - void write(T f, K s, L t, Q fr); + void write(Imm16 imm); - /// Write rex-byte based on a single register for a BYTE operation. - void rex_optional_r1(Register reg); + void write(Imm32 imm); - /// Write rex-byte based on a single register for a WORD operation. - void rex_optional_r2(Register reg); + void write(Imm64 imm); - /// Write rex-byte based on a single register for a DWORD operation. - void rex_optional_r3(Register reg); + // Write a memory address given the register destination + /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html + void write_address(const MemoryAddress &addr, Register dest); - /// Writes rex-values for 64-bit argument operations. This applies for - /// the long-mode of x86-- which is x64. It also integrates rex-prefix - /// with special register R8-R15. - void rex_optional_r4(Register reg); - /// Binary Operation Rex Byte -- BYTE SIZE - void rex_optional_rr1(Register dest, Register src); + // Some useful overrides when writing multiple values. + // There is no need for a custom implementation because + // we don't write more than 4 values at a given time in a row. + // + // We depend on the compiler on choosing the correct function for the + // value that is passed in -- the choice should be clear given distinct + // parameter values. - /// Binary Operation Rex Byte -- WORD SIZE - void rex_optional_rr2(Register dest, Register src); + template + void write(T f, K s); + template + void write(T f, K s, L t); - /// Binary Operation Rex Byte -- DWORD SIZE - void rex_optional_rr3(Register dest, Register src); + template + void write(T f, K s, L t, Q fr); - /// Binary Operation Rex Byte -- QWORD SIZE - void rex_optional_rr4(Register dest, Register src); + /// Write rex-byte based on a single register for a BYTE operation. + void rex_optional_r1(Register reg); - /// Memory Operation Rex Byte -- BYTE SIZE - void rex_optional_rm1(Register reg, const MemoryAddress &addr); + /// Write rex-byte based on a single register for a WORD operation. + void rex_optional_r2(Register reg); - /// Memory Operation Rex Byte -- WORD SIZE - void rex_optional_rm2(Register reg, const MemoryAddress &addr); + /// Write rex-byte based on a single register for a DWORD operation. + void rex_optional_r3(Register reg); - /// Memory Operation Rex Byte -- DWORD SIZE - void rex_optional_rm3(Register reg, const MemoryAddress &addr); + /// Writes rex-values for 64-bit argument operations. This applies for + /// the long-mode of x86-- which is x64. It also integrates rex-prefix + /// with special register R8-R15. + void rex_optional_r4(Register reg); - /// Memory Operation Rex Byte -- QWORD SIZE - void rex_optional_rm4(Register reg, const MemoryAddress &addr); + /// Binary Operation Rex Byte -- BYTE SIZE + void rex_optional_rr1(Register dest, Register src); + /// Binary Operation Rex Byte -- WORD SIZE + void rex_optional_rr2(Register dest, Register src); -public: - explicit Assembler(size_t size); - explicit Assembler(size_t size, bool executable); + /// Binary Operation Rex Byte -- DWORD SIZE + void rex_optional_rr3(Register dest, Register src); - /// Spill the memory buffer containing machine code bytes written - ubyte *spill() const; + /// Binary Operation Rex Byte -- QWORD SIZE + void rex_optional_rr4(Register dest, Register src); - /// The number of bytes that have been written to the memory buffer - ulong bytes() const; + /// Memory Operation Rex Byte -- BYTE SIZE + void rex_optional_rm1(Register reg, const MemoryAddress &addr); - /// Get a given byte that has been generated by the assembler - ubyte operator[](size_t index) { return memory[index]; } + /// Memory Operation Rex Byte -- WORD SIZE + void rex_optional_rm2(Register reg, const MemoryAddress &addr); - void popq(Register reg); - void popw(Register reg); + /// Memory Operation Rex Byte -- DWORD SIZE + void rex_optional_rm3(Register reg, const MemoryAddress &addr); - void pushq(Register reg); - void pushw(Register reg); + /// Memory Operation Rex Byte -- QWORD SIZE + void rex_optional_rm4(Register reg, const MemoryAddress &addr); - void movb(Register reg, Imm8 imm); - template - void movw(Register reg, Imm imm); - template - void movd(Register reg, Imm imm); - template - void movq(Register reg, Imm imm); - void movb(Register reg, const MemoryAddress &addr); - void movw(Register reg, const MemoryAddress &addr); - void movd(Register reg, const MemoryAddress &addr); - void movq(Register reg, const MemoryAddress &addr); + public: - void movb(Register dest, Register src); - void movw(Register dest, Register src); - void movd(Register dest, Register src); - void movq(Register dest, Register src); + explicit Assembler(size_t size); - void shlb(Register dest, Imm8 count); - void shlw(Register dest, Imm8 count); - void shld(Register dest, Imm8 count); - void shlq(Register dest, Imm8 count); + explicit Assembler(size_t size, bool executable); - void shlb(Register dest); - void shlw(Register dest); - void shld(Register dest); - void shlq(Register dest); + /// Spill the memory buffer containing machine code bytes written + ubyte *spill() const; - void shrb(Register dest, Imm8 count); - void shrw(Register dest, Imm8 count); - void shrd(Register dest, Imm8 count); - void shrq(Register dest, Imm8 count); + /// The number of bytes that have been written to the memory buffer + ulong bytes() const; - void jmp(Imm8 rel8); - void jmp(Imm16 rel16); - void jmp(Imm32 rel32); + /// Get a given byte that has been generated by the assembler + ubyte operator[](size_t index) { return memory[index]; } - void jmpw(Register reg); - void jmpq(Register reg); + void popq(Register reg); - void jump_cond(Condition cond, Imm8 rel8); - void jump_cond(Condition cond, Imm16 rel16); - void jump_cond(Condition cond, Imm32 rel32); + void popw(Register reg); - // NOTE: imul instruction order for rex is (lib, dest), yes, this is - // correct. - void imulw(Register dest, Register src); - void imuld(Register dest, Register src); - void imulq(Register dest, Register src); + void pushq(Register reg); - void lea(Register reg, const MemoryAddress &addr, OperandSize size); + void pushw(Register reg); - void movss(Register dest, Register src); - void movss(Register dest, const MemoryAddress &addr); - void movss(const MemoryAddress& dest, Register src); + void movb(Register reg, Imm8 imm); - instruction_binary(add); - instruction_binary(sub); - instruction_binary(and); - instruction_binary(cmp); - instruction_binary(xor); - instruction_binary(or); + template + void movw(Register reg, Imm imm); - instruction_float_binary(add); - instruction_float_binary(sub); - instruction_float_binary(div); - instruction_float_binary(mul); - - instruction_unary(inc); - instruction_unary(dec); - instruction_unary(not); - instruction_unary(mul); - instruction_unary(neg); - instruction_unary(div); - instruction_unary(idiv); - - void ret(); -}; -}}} + template + void movd(Register reg, Imm imm); + + template + void movq(Register reg, Imm imm); + + void movb(Register reg, const MemoryAddress &addr); + + void movw(Register reg, const MemoryAddress &addr); + + void movd(Register reg, const MemoryAddress &addr); + + void movq(Register reg, const MemoryAddress &addr); + + void movb(Register dest, Register src); + + void movw(Register dest, Register src); + + void movd(Register dest, Register src); + + void movq(Register dest, Register src); + + void shlb(Register dest, Imm8 count); + + void shlw(Register dest, Imm8 count); + + void shld(Register dest, Imm8 count); + + void shlq(Register dest, Imm8 count); + + void shlb(Register dest); + + void shlw(Register dest); + + void shld(Register dest); + + void shlq(Register dest); + + void shrb(Register dest, Imm8 count); + + void shrw(Register dest, Imm8 count); + + void shrd(Register dest, Imm8 count); + + void shrq(Register dest, Imm8 count); + + void jmp(Imm8 rel8); + + void jmp(Imm16 rel16); + + void jmp(Imm32 rel32); + + void jmpw(Register reg); + + void jmpq(Register reg); + + void jump_cond(Condition cond, Imm8 rel8); + + void jump_cond(Condition cond, Imm16 rel16); + + void jump_cond(Condition cond, Imm32 rel32); + + // NOTE: imul instruction order for rex is (lib, dest), yes, this is + // correct. + void imulw(Register dest, Register src); + + void imuld(Register dest, Register src); + + void imulq(Register dest, Register src); + + void lea(Register reg, const MemoryAddress &addr, OperandSize size); + + void movss(Register dest, Register src); + + void movss(Register dest, const MemoryAddress &addr); + + void movss(const MemoryAddress &dest, Register src); + + instruction_binary(add); + + instruction_binary(sub); + + instruction_binary(and); + + instruction_binary(cmp); + + instruction_binary(xor); + + instruction_binary(or); + + instruction_float_binary(add); + + instruction_float_binary(sub); + + instruction_float_binary(div); + + instruction_float_binary(mul); + + instruction_unary(inc); + + instruction_unary(dec); + + instruction_unary(not); + + instruction_unary(mul); + + instruction_unary(neg); + + instruction_unary(div); + + instruction_unary(idiv); + + void ret(); + }; + } + } +} #endif //JET_X64_ASSEMBLER_H diff --git a/import/nextgen/os-config.h b/import/nextgen/os-config.h index e13000a..354a12c 100644 --- a/import/nextgen/os-config.h +++ b/import/nextgen/os-config.h @@ -2,7 +2,9 @@ # define NEXTGEN_CONFIG_H # if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) || defined(__WIN32) && !defined(__CYGWIN__) # define NG_OS_WINDOWS true + # include + # elif __APPLE__ # define NG_OS_APPLE true # include @@ -20,8 +22,8 @@ # define NG_INLINE inline # define NG_AINLINE inline __attribute__((always_inline)) # elif defined(_MSC_VER) -# define NG_INLINE __inline -# define NG_AINLINE __forceinline +# define NG_INLINE __inline +# define NG_AINLINE __forceinline # endif # if defined(_M_IX86) # define NG_FAST __fastcall @@ -50,26 +52,29 @@ /* Older versions of MSVC do not have stdint.h values that are properly defined from C99 */ # if defined(_MSC_VER) && (_MSC_VER < 1700) typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; # ifdef _WIN64 # define UINTPTR_MAX 0xffffffffffffffff - typedef __int64 intptr_t; - typedef unsigned __int64 uintptr_t; + typedef __int64 intptr_t; + typedef unsigned __int64 uintptr_t; # else # define UINTPTR_MAX 0xffffffff - typedef __int32 intptr_t; - typedef unsigned __int32 uintptr_t; + typedef __int32 intptr_t; + typedef unsigned __int32 uintptr_t; # endif # else # define HAS_STDINT + # include + # endif + # include # include # include @@ -82,6 +87,7 @@ typedef __int8 int8_t; # include # include # include + # ifdef HAS_STDINT /* This is non-specific compiler detection */ # if UINTPTR_MAX == 0xffffffff # ifndef BIT32 @@ -97,7 +103,7 @@ typedef __int8 int8_t; # endif # if !defined(BIT32) && !defined(BIT64) # ifdef _WIN64 /* Only windows has this here because MSVC sometimes does not have stdint.h */ - define BIT64 +define BIT64 # else # define BIT32 # endif diff --git a/import/nextgen/panic.h b/import/nextgen/panic.h index da1d5b5..6b96baa 100644 --- a/import/nextgen/panic.h +++ b/import/nextgen/panic.h @@ -1,5 +1,6 @@ #ifndef NEXTGEN_PANIC_H #define NEXTGEN_PANIC_H + #include "os-config.h" @@ -7,82 +8,84 @@ namespace nextgen { #if defined(NG_OS_LINUX) || defined(NG_OS_APPLE) - // ANSI Terminal Colors for Linux and MacOS. The colors are strictly - // pre-defined to be White, Red, Green, Yellow, Blue, and Cyan and nothing - // more that is needed to highlight something in the terminal. - - // NOTE: This is namespace because of weird debug issue with GCC. For some - // reason in debug mode, the symbols are not resolved by the linker but works - // in release mode. The effect is the same either way with a namespace so we - // leave it as such. - namespace Colors { - static constexpr auto WHITE = COLOR_WHITE; - static constexpr auto RED = COLOR_RED; - static constexpr auto GREEN = COLOR_GREEN; - static constexpr auto YELLOW = COLOR_YELLOW; - static constexpr auto BLUE = COLOR_BLUE; - static constexpr auto CYAN = COLOR_CYAN; - static constexpr auto RESET = COLOR_RESET; - }; + // ANSI Terminal Colors for Linux and MacOS. The colors are strictly + // pre-defined to be White, Red, Green, Yellow, Blue, and Cyan and nothing + // more that is needed to highlight something in the terminal. + + // NOTE: This is namespace because of weird debug issue with GCC. For some + // reason in debug mode, the symbols are not resolved by the linker but works + // in release mode. The effect is the same either way with a namespace so we + // leave it as such. + namespace Colors { + static constexpr auto WHITE = COLOR_WHITE; + static constexpr auto RED = COLOR_RED; + static constexpr auto GREEN = COLOR_GREEN; + static constexpr auto YELLOW = COLOR_YELLOW; + static constexpr auto BLUE = COLOR_BLUE; + static constexpr auto CYAN = COLOR_CYAN; + static constexpr auto RESET = COLOR_RESET; + }; # elif defined(NG_OS_WINDOWS) - // NOTE: These colors are number values designated as console "modes" for - // the Windows Terminal. Windows uses these modes to render text as a - // specific color. - enum Colors { - WHITE = COLOR_WHITE, - RED = COLOR_RED, - GREEN = COLOR_GREEN, - YELLOW = COLOR_YELLOW, - BLUE = COLOR_BLUE, - CYAN = COLOR_CYAN, - RESET = COLOR_RESET - }; + // NOTE: These colors are number values designated as console "modes" for + // the Windows Terminal. Windows uses these modes to render text as a + // specific color. + enum Colors { + WHITE = COLOR_WHITE, + RED = COLOR_RED, + GREEN = COLOR_GREEN, + YELLOW = COLOR_YELLOW, + BLUE = COLOR_BLUE, + CYAN = COLOR_CYAN, + RESET = COLOR_RESET + }; # else - namespace Colors { - static constexpr auto WHITE = ""; - static constexpr auto RED = ""; - static constexpr auto GREEN = ""; - static constexpr auto YELLOW = ""; - static constexpr auto BLUE = ""; - static constexpr auto CYAN = ""; - static constexpr auto RESET = ""; - }; + namespace Colors { + static constexpr auto WHITE = ""; + static constexpr auto RED = ""; + static constexpr auto GREEN = ""; + static constexpr auto YELLOW = ""; + static constexpr auto BLUE = ""; + static constexpr auto CYAN = ""; + static constexpr auto RESET = ""; + }; # endif - struct Console { + struct Console { # if defined(NG_OS_WINDOWS) - // A little guessing trick that has the compiler infer which function gets - // called. It chooses default template unless we use a color. - static NG_INLINE void Write(Colors color) { - static HANDLE GenericConsole = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(GenericConsole, color); - } - - // Wrapper around std::cout to mask terminal color settings - // in the other `Write` function. - template - static NG_INLINE void Write(Arg value) { - std::cout << value; - } + // A little guessing trick that has the compiler infer which function gets + // called. It chooses default template unless we use a color. + static NG_INLINE void Write(Colors color) { + static HANDLE GenericConsole = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(GenericConsole, color); + } + + // Wrapper around std::cout to mask terminal color settings + // in the other `Write` function. + template + static NG_INLINE void Write(Arg value) { + std::cout << value; + } + # endif - template - static void Log(Args ... args) { - using fold = int[]; + + template + static void Log(Args ... args) { + using fold = int[]; # if defined(NG_OS_LINUX) || defined(NG_OS_APPLE) - fold { - (std::cout << args, 0)... - }; + fold { + (std::cout << args, 0)... + }; # elif defined(NG_OS_WINDOWS) - fold { - ((Write(args)), 0)... - }; + fold{ + ((Write(args)), 0)... + }; # endif - } - }; + } + }; # define PANIC(msg) nextgen::PanicAt(msg, __FILE__, "PANIC("#msg");") @@ -99,41 +102,41 @@ nextgen::PanicAt("Expected value to be true", __FILE__, \ "ASSERT_TRUE("#cond");") - // Panic at a specific point in the code. This is simulated as an exception - // throw caught by main and returns a failure as to avoid misuse of - // "std::exit". It prints out the line number,and text of the panic message - // to have the user locate the failed point in the code. - template< - std::size_t N1, std::size_t N2, std::size_t N3, - int LINE, bool Assert> - static void PanicAt( - const char (&msg)[N1], /** @brief Panic Message */ - const char (&FILE)[N2], /** @brief File name of Error */ - const char (&dup)[N3]) /** @brief Line text of Panic */ { - - Console::Log(FILE, ":", LINE, " ", Colors::RED, "error: ", - Colors::WHITE, - msg); - Console::Log('\n', Colors::RESET); - - auto whitespace = (std::to_string(LINE) + " | ").length(); - if (Assert) { - Console::Log(Colors::RED, LINE, " | ", dup, Colors::WHITE, - " <-- ", - Colors::RED, "Assert Failed:", Colors::YELLOW, - " Thread panicked here", "\n", Colors::BLUE); - } else { - Console::Log(Colors::RED, LINE, " | ", dup, Colors::YELLOW, - " <-- Thread panicked here", "\n", - Colors::BLUE); - } - /* Leading Whitespace */ - FOR(i, whitespace) Console::Log(" ");; - /* Highlighting region */ - FOR(i, N3 - 1) Console::Log('~'); - Console::Log(Colors::RESET, '\n'); - throw std::exception(); - } + // Panic at a specific point in the code. This is simulated as an exception + // throw caught by main and returns a failure as to avoid misuse of + // "std::exit". It prints out the line number,and text of the panic message + // to have the user locate the failed point in the code. + template< + std::size_t N1, std::size_t N2, std::size_t N3, + int LINE, bool Assert> + static void PanicAt( + const char (&msg)[N1], /** @brief Panic Message */ + const char (&FILE)[N2], /** @brief File name of Error */ + const char (&dup)[N3]) /** @brief Line text of Panic */ { + + Console::Log(FILE, ":", LINE, " ", Colors::RED, "error: ", + Colors::WHITE, + msg); + Console::Log('\n', Colors::RESET); + + auto whitespace = (std::to_string(LINE) + " | ").length(); + if (Assert) { + Console::Log(Colors::RED, LINE, " | ", dup, Colors::WHITE, + " <-- ", + Colors::RED, "Assert Failed:", Colors::YELLOW, + " Thread panicked here", "\n", Colors::BLUE); + } else { + Console::Log(Colors::RED, LINE, " | ", dup, Colors::YELLOW, + " <-- Thread panicked here", "\n", + Colors::BLUE); + } + /* Leading Whitespace */ + FOR(i, whitespace) Console::Log(" ");; + /* Highlighting region */ + FOR(i, N3 - 1) Console::Log('~'); + Console::Log(Colors::RESET, '\n'); + throw std::exception(); + } } diff --git a/import/nextgen/str.h b/import/nextgen/str.h index c845e75..26769a1 100644 --- a/import/nextgen/str.h +++ b/import/nextgen/str.h @@ -1,194 +1,201 @@ # ifndef NEXTGEN_STR_H # define NEXTGEN_STR_H + # include "core-util.h" # include "allocator.h" -namespace nextgen { using namespace core; using namespace nextgen::mem; +namespace nextgen { + using namespace core; using namespace nextgen::mem; + + template + struct Range { + + T begin; + T end; + + Range(T begin, T end) : begin(begin), end(end) {} + + Range() = default; + + [[nodiscard]] + NG_INLINE size_t range() const { + return (size_t) (end - begin); + } + }; + + + class str { + public: + str() = default; + + struct intern_hash { // std::unordered_map + size_t operator()(str const &s) const { + return s.getHashCache(); + } + }; + + struct intern_eq { + bool operator()(str const &LHS, str const &RHS) const { + return strncmp(LHS.pointer, RHS.pointer, LHS.len) == 0; + } + }; + + + template + /*implicit*/ + constexpr str(const char(&data)[N]) : len(N), pointer(data) {} - template - struct Range { + /*implicit*/ str(const char *data) : len(strlen(data)), pointer(data) {} - T begin; - T end; + /*implicit*/ str(char c) : len(1) { pointer = &c; } - Range(T begin, T end) : begin(begin), end(end) {} + explicit str(const char *data, size_t len) : len(len), pointer(data) {} - Range() = default; + /*implicit*/ str(Range range) + : len(range.range()), pointer(range.begin) {} - [[nodiscard]] - NG_INLINE size_t range() const { - return (size_t) (end - begin); - } - }; + /*implicit*/ str(std::string &data) { // NOTE: Do not use init-list for this + len = data.length(); + pointer = data.c_str(); + } + [[nodiscard]] + size_t size() const { return len; } - class str { - public: - str() = default; + // String FNV-1a Hashing Algorithm (Internal) + [[nodiscard]] + size_t hash() const { + auto val = FNV_OFF; + for (auto i = 0; i < len; ++i) { + val ^= (pointer[i]); + val *= FNV_PRIME; + } + return val; + } - struct intern_hash { // std::unordered_map - size_t operator()(str const &s) const { - return s.getHashCache(); - } - }; + bool operator==(str const &other) const { + return pointer == other.pointer; + } - struct intern_eq { - bool operator()(str const &LHS, str const &RHS) const { - return strncmp(LHS.pointer, RHS.pointer, LHS.len) == 0; - } - }; - - - template /*implicit*/ - constexpr str(const char(&data)[N]) : len(N), pointer(data) {} - - /*implicit*/ str(const char *data) : len(strlen(data)), pointer(data) {} - /*implicit*/ str(char c) : len(1) { pointer = &c; } + char operator[](size_t index) const { + return (static_cast(pointer))[index]; + } - explicit str(const char *data, size_t len) : len(len), pointer(data) {} - /*implicit*/ str(Range range) - : len(range.range()), pointer(range.begin) {} - - /*implicit*/ str(std::string &data) { // NOTE: Do not use init-list for this - len = data.length(); - pointer = data.c_str(); - } - - [[nodiscard]] - size_t size() const { return len; } - - // String FNV-1a Hashing Algorithm (Internal) - [[nodiscard]] - size_t hash() const { - auto val = FNV_OFF; - for (auto i = 0; i < len; ++i) { - val ^= (pointer[i]); - val *= FNV_PRIME; - } - return val; - } - - bool operator==(str const &other) const { - return pointer == other.pointer; - } - - char operator[](size_t index) const { - return (static_cast(pointer))[index]; - } - - /*implicit*/ operator std::string() const { - return std::string((const char *) *this); - } - /*implicit*/ operator Range() const { - return Range(pointer, pointer + len); - } - - explicit operator const char *() const { - return static_cast(pointer); - } - - - str operator-(const size_t size) const { - return str { - pointer - size, - len - size - }; - } - - str operator+(int offset) const { - return str { - pointer + offset, - len + offset - }; - } - - char operator*() const { - return *pointer; - } - - const char *begin() const { - return pointer; - } - - const char *end() const { - return pointer + len; - } - - // You MUST set the hash value before interning a string, - // so it is not recomputed in str::intern_hash. - NG_AINLINE size_t getHashCache() const { - return hash_cache; - } - - // Mutation. - - NG_INLINE void setHash() { - hash_cache = hash(); - } - - NG_INLINE void setHash(size_t hash) { - hash_cache = hash; - } - - str operator++() { - pointer++; - return *this; - } - - str operator--() { - pointer--; - return *this; - } - - str operator+=(int offset) { - pointer += (size_t) (offset); - return *this; - } - - str operator-=(int offset) { - pointer -= (size_t) (offset); - return *this; - } - - const char *pointer{}; // char* data - //bool is_heap_allocated = false; - private: - size_t len{}; // String length - size_t hash_cache = 0; - }; - - - // Very thin wrapper around std::unordered_set for String interning. - // Mostly O(1) Retrieval (FNV-1a is fast and good String hash). - // Designed around the symbols retrieved for the token and not the token - // values. - class StringCache { - public: - using StringSet = std::unordered_set; - - static StringSet& Strings() { - static StringSet Strings; - return Strings; - } - - static str &Get(str &s) { - return const_cast(*(Strings().find(s))); - } - - static NG_INLINE str InsertOrGet(str &s) { - auto find = Strings().find(s); - if (find != Strings().end()) { - return *find; - } else { - auto new_string = (str) s; - new_string.setHash(s.getHashCache()); - return (*Strings().insert(new_string).first); - } - } - }; - - str operator""_intern(const char *s, size_t len); - std::ostream &operator<<(std::ostream &s, nextgen::str &str); + /*implicit*/ operator std::string() const { + return std::string((const char *) *this); + } + + /*implicit*/ operator Range() const { + return Range(pointer, pointer + len); + } + + explicit operator const char *() const { + return static_cast(pointer); + } + + + str operator-(const size_t size) const { + return str{ + pointer - size, + len - size + }; + } + + str operator+(int offset) const { + return str{ + pointer + offset, + len + offset + }; + } + + char operator*() const { + return *pointer; + } + + const char *begin() const { + return pointer; + } + + const char *end() const { + return pointer + len; + } + + // You MUST set the hash value before interning a string, + // so it is not recomputed in str::intern_hash. + NG_AINLINE size_t getHashCache() const { + return hash_cache; + } + + // Mutation. + + NG_INLINE void setHash() { + hash_cache = hash(); + } + + NG_INLINE void setHash(size_t hash) { + hash_cache = hash; + } + + str operator++() { + pointer++; + return *this; + } + + str operator--() { + pointer--; + return *this; + } + + str operator+=(int offset) { + pointer += (size_t) (offset); + return *this; + } + + str operator-=(int offset) { + pointer -= (size_t) (offset); + return *this; + } + + const char *pointer{}; // char* data + //bool is_heap_allocated = false; + private: + size_t len{}; // String length + size_t hash_cache = 0; + }; + + + // Very thin wrapper around std::unordered_set for String interning. + // Mostly O(1) Retrieval (FNV-1a is fast and good String hash). + // Designed around the symbols retrieved for the token and not the token + // values. + class StringCache { + public: + using StringSet = std::unordered_set; + + static StringSet &Strings() { + static StringSet Strings; + return Strings; + } + + static str &Get(str &s) { + return const_cast(*(Strings().find(s))); + } + + static NG_INLINE str InsertOrGet(str &s) { + auto find = Strings().find(s); + if (find != Strings().end()) { + return *find; + } else { + auto new_string = (str) s; + new_string.setHash(s.getHashCache()); + return (*Strings().insert(new_string).first); + } + } + }; + + str operator ""_intern(const char *s, size_t len); + + std::ostream &operator<<(std::ostream &s, nextgen::str &str); } #endif //NEXTGEN_STR_H diff --git a/lib/allocator.cpp b/lib/allocator.cpp index 0ae71f2..44bd83a 100644 --- a/lib/allocator.cpp +++ b/lib/allocator.cpp @@ -3,41 +3,41 @@ using namespace nextgen::mem; void *os::calloc(size_t mem, size_t size) { - if (mem == 0) return nullptr; - auto p = std::calloc(mem, size); - ASSERT(p, "Internal Error: Calloc Failed"); - return p; + if (mem == 0) return nullptr; + auto p = std::calloc(mem, size); + ASSERT(p, "Internal Error: Calloc Failed"); + return p; } void *os::malloc(size_t size) { - if (size == 0) return nullptr; - auto p = std::malloc(size); - ASSERT(p, "Internal Error: Malloc Failed"); - return p; + if (size == 0) return nullptr; + auto p = std::malloc(size); + ASSERT(p, "Internal Error: Malloc Failed"); + return p; } void *os::realloc(void *buf, size_t size) { - if (buf == nullptr && size == 0) return nullptr; - auto p = std::realloc(buf, size); - ASSERT(p, "Internal Error: Realloc Failed"); - return p; + if (buf == nullptr && size == 0) return nullptr; + auto p = std::realloc(buf, size); + ASSERT(p, "Internal Error: Realloc Failed"); + return p; } void *os::memmove(void *dest, const void *src, size_t size) { - auto p = std::memmove(dest, src, size); - ASSERT(p, "Internal Error: Memmove Failed"); - return p; + auto p = std::memmove(dest, src, size); + ASSERT(p, "Internal Error: Memmove Failed"); + return p; } void os::free(void *p) { - std::free(p); + std::free(p); } Allocator nextgen::mem::GLOBAL_OBJECT_ALLOC = Allocator( - arena::OVER_COMMIT_MEMORY_SIZE - arena::DATA_ALLOCATION_MEMORY_SIZE + arena::OVER_COMMIT_MEMORY_SIZE - arena::DATA_ALLOCATION_MEMORY_SIZE ); Allocator nextgen::mem::GLOBAL_DATA_ALLOC = Allocator( - arena::DATA_ALLOCATION_MEMORY_SIZE + arena::DATA_ALLOCATION_MEMORY_SIZE ); @@ -45,6 +45,7 @@ Allocator nextgen::mem::GLOBAL_DATA_ALLOC = Allocator( // the new and delete operator, but for this code base, it just makes the code // much cleaner, and since the arena pool is global, it should be okay. void *operator new(size_t size) { - return ::GLOBAL_OBJECT_ALLOC.allocate(size); + return ::GLOBAL_OBJECT_ALLOC.allocate(size); } + void operator delete([[maybe_unused]] void *p) noexcept {} diff --git a/lib/io.cpp b/lib/io.cpp index 3757fdc..bd9943a 100644 --- a/lib/io.cpp +++ b/lib/io.cpp @@ -19,12 +19,12 @@ void FileBuf::Output(FileBuf *files, FileID output) { } FileBuf nextgen::io::CreateFileBuffer(const char *FILE, FileID ID) { - auto read = std::ifstream(FILE, std::ios::binary); - auto buf = read.rdbuf(); - auto size = read.tellg(); + auto read = std::ifstream(FILE, std::ios::binary); + auto buf = read.rdbuf(); + auto size = read.tellg(); - auto block = static_cast(nextgen::mem::os::malloc(size)); - buf->sgetn(block, size); + auto block = static_cast(nextgen::mem::os::malloc(size)); + buf->sgetn(block, size); - return { block, ID }; + return {block, ID}; } diff --git a/lib/jet-analyzer.cpp b/lib/jet-analyzer.cpp index 8f83f09..c9bbef1 100644 --- a/lib/jet-analyzer.cpp +++ b/lib/jet-analyzer.cpp @@ -4,7 +4,7 @@ using namespace nextgen::jet; void Analyzer::analyze(const ParserOutput output_to_analyze) { - + } void Analyzer::analyze_function(const SyntaxFunction *function) { diff --git a/lib/jet-diagnostics.cpp b/lib/jet-diagnostics.cpp index 87cb34a..7e47db1 100644 --- a/lib/jet-diagnostics.cpp +++ b/lib/jet-diagnostics.cpp @@ -8,182 +8,182 @@ using nextgen::Range; // Color namespacing #ifdef NG_OS_WINDOWS - using nextgen::Colors; +using nextgen::Colors; #else - using namespace nextgen::Colors; +using namespace nextgen::Colors; #endif static Range - FileLines(size_t nth, const char *file_buf, size_t buf_len) { - int max_num_after_found_point = 3; - - bool found = false; - int count = 1; - - auto copy = file_buf; - const char *found_point; - const char *last_new_line; - - if (nth == 1) { - return Range(file_buf, file_buf + buf_len); - } - - - FOR(ch, buf_len) { - if (*copy == '\n' || *copy == '\r') { - if (found) { - return Range(file_buf, copy); - } - count++; - } - if (count == nth) { - if (!found) { - found = true; - } - } - copy++; - } +FileLines(size_t nth, const char *file_buf, size_t buf_len) { + int max_num_after_found_point = 3; + + bool found = false; + int count = 1; + + auto copy = file_buf; + const char *found_point; + const char *last_new_line; + + if (nth == 1) { + return Range(file_buf, file_buf + buf_len); + } + + + FOR(ch, buf_len) { + if (*copy == '\n' || *copy == '\r') { + if (found) { + return Range(file_buf, copy); + } + count++; + } + if (count == nth) { + if (!found) { + found = true; + } + } + copy++; + } } static str GetNthLineOfBuffer(size_t nth, const char *file_buf, size_t buf_len) { - bool found = false; - int count = 1; - const char *copy = file_buf; - const char *found_point; - - if (nth == 1) { - return str {file_buf, buf_len}; - } - - FOR(i, buf_len) { - if (*copy == '\n' || *copy == '\r' || *copy == '\0') { - if (found) { - return str(Range(found_point, copy)); - } - count++; - } - if (count == nth) { - if (!found) { - found_point = copy+1; - found = true; - } - } - copy++; - } - - return str {found_point, static_cast(copy - found_point)}; + bool found = false; + int count = 1; + const char *copy = file_buf; + const char *found_point; + + if (nth == 1) { + return str{file_buf, buf_len}; + } + + FOR(i, buf_len) { + if (*copy == '\n' || *copy == '\r' || *copy == '\0') { + if (found) { + return str(Range(found_point, copy)); + } + count++; + } + if (count == nth) { + if (!found) { + found_point = copy + 1; + found = true; + } + } + copy++; + } + + return str{found_point, static_cast(copy - found_point)}; } template -static void AddHint(std::string &line, Args&& ... hint) { +static void AddHint(std::string &line, Args &&... hint) { - // Align space - FOR(i, line.length() + 1) { - Console::Log(" "); - } + // Align space + FOR(i, line.length() + 1) { + Console::Log(" "); + } - Console::Log(hint..., nextgen::Colors::RESET); + Console::Log(hint..., nextgen::Colors::RESET); } void Diagnostic::build(LexError error) { - source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); - Console::Log(Colors::RED, - "Error ---------------------------------------------------------- ", - file_name, ":", error.location.line, ":", error.location.column, "\n", - Colors::RESET); - - switch (error.error) { - case MalformedUTF8: - break; - case InvalidChar: - Console::Log("Invalid Char"); - break; - case UnexpectedEOF: - Console::Log("EOF"); - break; - case OutOfRange: - Console::Log("Out of Buffer Range"); - break; - case Unreachable: - Console::Log("Unreachable"); - break; - case IntegerOverflow: - ErrorIntegerOverflow(error); - break; - case FloatingPointOverflow: - break; - case DigitOutOfRange: - ErrorDigitOutOfRange(error); - break; - case MissingClosingDelim: - ErrorMissingClosingDelim(error); - break; - case InvalidDot: - Console::Log("Invalid Dot"); - break; - case InvalidNumberValue: - Console::Log("Invalid Number"); - break; - case InvalidStringEscape: - ErrorInvalidStringEscape(error); - break; - case InvalidUnicodeEscapeClose: - break; - case HexEscapeOutOfRange: - ErrorHexEscapeOutOfRange(error); - break; - default: - UNREACHABLE; - } - Console::Log(Colors::RESET, "\n"); - - // Lexer errors are always fatal to a program's continuation. - send_exception(); + source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); + Console::Log(Colors::RED, + "Error ---------------------------------------------------------- ", + file_name, ":", error.location.line, ":", error.location.column, "\n", + Colors::RESET); + + switch (error.error) { + case MalformedUTF8: + break; + case InvalidChar: + Console::Log("Invalid Char"); + break; + case UnexpectedEOF: + Console::Log("EOF"); + break; + case OutOfRange: + Console::Log("Out of Buffer Range"); + break; + case Unreachable: + Console::Log("Unreachable"); + break; + case IntegerOverflow: + ErrorIntegerOverflow(error); + break; + case FloatingPointOverflow: + break; + case DigitOutOfRange: + ErrorDigitOutOfRange(error); + break; + case MissingClosingDelim: + ErrorMissingClosingDelim(error); + break; + case InvalidDot: + Console::Log("Invalid Dot"); + break; + case InvalidNumberValue: + Console::Log("Invalid Number"); + break; + case InvalidStringEscape: + ErrorInvalidStringEscape(error); + break; + case InvalidUnicodeEscapeClose: + break; + case HexEscapeOutOfRange: + ErrorHexEscapeOutOfRange(error); + break; + default: + UNREACHABLE; + } + Console::Log(Colors::RESET, "\n"); + + // Lexer errors are always fatal to a program's continuation. + send_exception(); } void Diagnostic::build(ParseError error) { - source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); - Console::Log(Colors::RED, - "Error ---------------------------------------------------------- ", - file_name, ":", error.location.line, ":", error.location.column, "\n", - Colors::RESET); - - switch (error.error) { - case ReservedIdentifierAsVariableName: - break; - case UnexpectedExpression: - Console::Log("Unexpected Expression"); - break; - case InvalidToken: - Console::Log("Invalid Token"); - break; - case ExpectedToken: - ErrorParseExpectedToken(error); - break; - case MissingForLoopVariable: - break; - case ExpectedIdentifierForFunctionParameter: - break; - case ExpectedIdentifierForStructProperty: - break; - case MissingClosingPair: - ErrorParseMissingClosingDelim(error); - break; - case UnexpectedEndOfFile: - Console::Log("EOF"); - break; - case MissingFunctionName: - break; - case MissingVariableName: - break; - case InvalidTokenAfterIdentInGlobalScope: - ErrorParseInvalidTokenAfterIdentInGlobalScope(error); - break; - } + source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); + Console::Log(Colors::RED, + "Error ---------------------------------------------------------- ", + file_name, ":", error.location.line, ":", error.location.column, "\n", + Colors::RESET); + + switch (error.error) { + case ReservedIdentifierAsVariableName: + break; + case UnexpectedExpression: + Console::Log("Unexpected Expression"); + break; + case InvalidToken: + Console::Log("Invalid Token"); + break; + case ExpectedToken: + ErrorParseExpectedToken(error); + break; + case MissingForLoopVariable: + break; + case ExpectedIdentifierForFunctionParameter: + break; + case ExpectedIdentifierForStructProperty: + break; + case MissingClosingPair: + ErrorParseMissingClosingDelim(error); + break; + case UnexpectedEndOfFile: + Console::Log("EOF"); + break; + case MissingFunctionName: + break; + case MissingVariableName: + break; + case InvalidTokenAfterIdentInGlobalScope: + ErrorParseInvalidTokenAfterIdentInGlobalScope(error); + break; + } } @@ -191,41 +191,41 @@ void Diagnostic::build(ParseError error) { void Diagnostic::ErrorMissingClosingDelim(nextgen::jet::LexError &error) { - Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); - auto Line = std::to_string(error.location.line); + Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); + auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Reached EOF Before Closing Delim", error); - AddHint(Line, Colors::CYAN, "= ", Colors::GREEN, "try: ", - Colors::RESET, "Add `", (char) error.metadata[0], "` to the end of " - "the statement"); + ErrorLexSetup(Line, "Reached EOF Before Closing Delim", error); + AddHint(Line, Colors::CYAN, "= ", Colors::GREEN, "try: ", + Colors::RESET, "Add `", (char) error.metadata[0], "` to the end of " + "the statement"); } void Diagnostic::ErrorIntegerOverflow(LexError &error) { - Console::Log(Colors::RESET, "Integer Value Too Large\n\n"); - auto Line = std::to_string(error.location.line); + Console::Log(Colors::RESET, "Integer Value Too Large\n\n"); + auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Overflow Occurs Here", error); + ErrorLexSetup(Line, "Overflow Occurs Here", error); - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", - Colors::RESET, "Integer values must be less than ", UINTPTR_MAX, - ".\n", - Colors::RESET, '\n'); + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", + Colors::RESET, "Integer values must be less than ", UINTPTR_MAX, + ".\n", + Colors::RESET, '\n'); - Console::Log("An ", Colors::RED, - "Integer Overflow", Colors::RESET, - " occurs when a numeric literal's value ", Colors::BLUE, - "exceeds ", Colors::RESET, - "the limit bound by their computer. \n", Colors::YELLOW); + Console::Log("An ", Colors::RED, + "Integer Overflow", Colors::RESET, + " occurs when a numeric literal's value ", Colors::BLUE, + "exceeds ", Colors::RESET, + "the limit bound by their computer. \n", Colors::YELLOW); # ifdef BIT32 - Console::Log("Your computer is 32-Bit", Colors::RESET, - " which has a limit of ", - Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", - "your value exceeds that."); + Console::Log("Your computer is 32-Bit", Colors::RESET, + " which has a limit of ", + Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", + "your value exceeds that."); # else - Console::Log("Your computer is 64-Bit", Colors::RESET, - " which has a limit of ", - Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", - "your value exceeds that."); + Console::Log("Your computer is 64-Bit", Colors::RESET, + " which has a limit of ", + Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", + "your value exceeds that."); # endif } @@ -239,204 +239,204 @@ void Diagnostic::ErrorMalformedUTF8(LexError &error) { void Diagnostic::ErrorDigitOutOfRange(LexError &error) { - Console::Log(Colors::RESET, "Digit Value Out Of Range\n\n"); - - auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Digit does not match Integer Radix", error); - - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", Colors::RESET, - "The integer value you wrote was ", - Colors::YELLOW, "Base ", error.metadata[0], Colors::RESET, - " which accepts digits "); - - switch (error.metadata[0]) { - case 10: - Console::Log("[", Colors::YELLOW, "0-9", Colors::RESET, "]"); - break; - case 16: - Console::Log(Colors::YELLOW, "0-9, a-f, and A-F", Colors::RESET, - " not ", Colors::RED, "'", source_line[error.location.column], - "'", Colors::RESET, '\n'); - break; - default: - UNREACHABLE; - } - AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::RESET, "Insert "); - error.failed_token.PrettyPrint(); - - switch (error.metadata[0]) { - case 10: - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9", Colors::RESET, "]\n"); - break; - case 16: - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n"); - break; - default: - UNREACHABLE; - } + Console::Log(Colors::RESET, "Digit Value Out Of Range\n\n"); + + auto Line = std::to_string(error.location.line); + ErrorLexSetup(Line, "Digit does not match Integer Radix", error); + + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", Colors::RESET, + "The integer value you wrote was ", + Colors::YELLOW, "Base ", error.metadata[0], Colors::RESET, + " which accepts digits "); + + switch (error.metadata[0]) { + case 10: + Console::Log("[", Colors::YELLOW, "0-9", Colors::RESET, "]"); + break; + case 16: + Console::Log(Colors::YELLOW, "0-9, a-f, and A-F", Colors::RESET, + " not ", Colors::RED, "'", source_line[error.location.column], + "'", Colors::RESET, '\n'); + break; + default: + UNREACHABLE; + } + AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::RESET, "Insert "); + error.failed_token.PrettyPrint(); + + switch (error.metadata[0]) { + case 10: + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9", Colors::RESET, "]\n"); + break; + case 16: + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n"); + break; + default: + UNREACHABLE; + } } void Diagnostic::ErrorHexEscapeOutOfRange(LexError &error) { - Console::Log(Colors::RESET, "Escape Digit Value Out Of Range\n\n"); - - auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Digit is not Base 16", error); - - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", - Colors::RESET, "Value started as Base 16" - " but value encountered was Base ", - error.metadata[0], '\n'); - - AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::RESET, "Insert "); - error.failed_token.PrettyPrint(); - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n\n"); - - Console::Log("Hex escapes turn", Colors::BLUE," 2 ", Colors::RESET, - "hexadecimal chars in to a single char; for " - "example:\n", Colors::GREEN, "\\x32 ", Colors::RESET, - "becomes the", Colors::GREEN, " ", Colors::RESET, - " character. Hex escapes accept digits", Colors::YELLOW, - " 0-9, a-f, and A-F ", Colors::RESET, "\nbut your value was " - "none of " - "these."); + Console::Log(Colors::RESET, "Escape Digit Value Out Of Range\n\n"); + + auto Line = std::to_string(error.location.line); + ErrorLexSetup(Line, "Digit is not Base 16", error); + + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", + Colors::RESET, "Value started as Base 16" + " but value encountered was Base ", + error.metadata[0], '\n'); + + AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::RESET, "Insert "); + error.failed_token.PrettyPrint(); + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n\n"); + + Console::Log("Hex escapes turn", Colors::BLUE, " 2 ", Colors::RESET, + "hexadecimal chars in to a single char; for " + "example:\n", Colors::GREEN, "\\x32 ", Colors::RESET, + "becomes the", Colors::GREEN, " ", Colors::RESET, + " character. Hex escapes accept digits", Colors::YELLOW, + " 0-9, a-f, and A-F ", Colors::RESET, "\nbut your value was " + "none of " + "these."); } void Diagnostic::ErrorInvalidStringEscape(LexError &error) { - Console::Log(Colors::RESET, "Not a Valid String Escape\n\n"); + Console::Log(Colors::RESET, "Not a Valid String Escape\n\n"); - auto line = std::to_string(error.location.line); - ErrorLexSetup(line, "Not a String Escape", error); + auto line = std::to_string(error.location.line); + ErrorLexSetup(line, "Not a String Escape", error); - AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::GREEN, - R"(\a, \b, \v, \n, \r, \', \", \x[0-9a-fA-F]{2}, \u{[0-9a-fA-F]}{1,6})", - "\n\n"); + AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::GREEN, + R"(\a, \b, \v, \n, \r, \', \", \x[0-9a-fA-F]{2}, \u{[0-9a-fA-F]}{1,6})", + "\n\n"); - Console::Log(Colors::BLUE, "String Escapes are used to escape " - "characters that ""are otherwise hard to display. \n", - Colors::RED, "'\\", - source_line[error.location.column - 1], "'", Colors::BLUE, - " is not a valid string escape. " - "View the complete list in the language documentation: \n", - "https://github.com/The-NextGen-Project/jet/blob/main/LANG.md", - Colors::RESET); + Console::Log(Colors::BLUE, "String Escapes are used to escape " + "characters that ""are otherwise hard to display. \n", + Colors::RED, "'\\", + source_line[error.location.column - 1], "'", Colors::BLUE, + " is not a valid string escape. " + "View the complete list in the language documentation: \n", + "https://github.com/The-NextGen-Project/jet/blob/main/LANG.md", + Colors::RESET); } void Diagnostic::ErrorParseInvalidTokenAfterIdentInGlobalScope(const ParseError &error) { - Console::Log(Colors::RESET, "Invalid Token\n\n"); - - auto err = error.metadata; - auto invalid_token = err.begin()->misc_tok; - - auto loc = invalid_token->getSourceLocation(); - auto line = std::to_string(loc.line); - ErrorParseSetup(loc.line, "Unexpected Token After Identifier", - invalid_token, error.location); - AddHint(line, Colors::BLUE, "= ", - Colors::CYAN, "try: ", - Colors::RESET, "Adding ", Colors::YELLOW, "=>", Colors::BLUE, " or " - "", - Colors::YELLOW, - ":=" - "\n", Colors::RESET); - - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "Expected one of ", Colors::BLUE, "':=', ", - "'=>', ", Colors::RESET, - "in global scope declaration."); - Console::Log("\n"); - FOR(i, line.length() + 9) Console::Log(" "); - Console::Log("Identifiers in the global scope only represent two values: " - "\n", Colors::GREEN); - FOR(i, line.length() + 9) Console::Log(" "); - Console::Log("Global Variable " - "Declaration ", - Colors::RESET, - "and ", - Colors::BLUE, - "Function " - "Declaration", - Colors::RESET, - " but your token was " - "", Colors::RED, - "none of these.\n"); + Console::Log(Colors::RESET, "Invalid Token\n\n"); + + auto err = error.metadata; + auto invalid_token = err.begin()->misc_tok; + + auto loc = invalid_token->getSourceLocation(); + auto line = std::to_string(loc.line); + ErrorParseSetup(loc.line, "Unexpected Token After Identifier", + invalid_token, error.location); + AddHint(line, Colors::BLUE, "= ", + Colors::CYAN, "try: ", + Colors::RESET, "Adding ", Colors::YELLOW, "=>", Colors::BLUE, " or " + "", + Colors::YELLOW, + ":=" + "\n", Colors::RESET); + + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "Expected one of ", Colors::BLUE, "':=', ", + "'=>', ", Colors::RESET, + "in global scope declaration."); + Console::Log("\n"); + FOR(i, line.length() + 9) Console::Log(" "); + Console::Log("Identifiers in the global scope only represent two values: " + "\n", Colors::GREEN); + FOR(i, line.length() + 9) Console::Log(" "); + Console::Log("Global Variable " + "Declaration ", + Colors::RESET, + "and ", + Colors::BLUE, + "Function " + "Declaration", + Colors::RESET, + " but your token was " + "", Colors::RED, + "none of these.\n"); } void Diagnostic::ErrorParseMissingClosingDelim(ParseError const &error) { - Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); - - auto err = error.metadata; - auto expected_kind = Token::GetTokenKindName(err.begin()->misc_kind); - auto token_got_instead = (err.begin()+1)->misc_tok; - auto token_loc = (err.begin()+2)->location; - - ErrorParseSetup(error.location.line, (std::string("Missing Closing Delim '") + - std::string(expected_kind) + std::string("'")).c_str(), token_got_instead, - error.location); - - std::string line = std::to_string(error.location.line); - AddHint(line, Colors::BLUE, "= ", - Colors::CYAN, "try: ", - Colors::RESET, "Adding ", Colors::YELLOW, "'", expected_kind, - "'\n", Colors::RESET); - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "Delimiters like ", Colors::BLUE, "'(', ", - "'[', ", "'{', ", Colors::RESET, - "must be met with their closing pairs.\n"); + Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); + + auto err = error.metadata; + auto expected_kind = Token::GetTokenKindName(err.begin()->misc_kind); + auto token_got_instead = (err.begin() + 1)->misc_tok; + auto token_loc = (err.begin() + 2)->location; + + ErrorParseSetup(error.location.line, (std::string("Missing Closing Delim '") + + std::string(expected_kind) + std::string("'")).c_str(), token_got_instead, + error.location); + + std::string line = std::to_string(error.location.line); + AddHint(line, Colors::BLUE, "= ", + Colors::CYAN, "try: ", + Colors::RESET, "Adding ", Colors::YELLOW, "'", expected_kind, + "'\n", Colors::RESET); + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "Delimiters like ", Colors::BLUE, "'(', ", + "'[', ", "'{', ", Colors::RESET, + "must be met with their closing pairs.\n"); } void Diagnostic::ErrorParseExpectedToken(ParseError const &error) { - Console::Log(Colors::RESET, "Expected Token\n\n"); - - auto err = error.metadata.begin()->expected_error; - auto line = std::to_string(error.location.line); - ErrorParseSetup(error.location.line, err.message, err.got, err - .got->getSourceLocation()); - AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::GREEN, "Adding ", Colors::YELLOW, "'", - Token::GetTokenKindName(err.expected), "'\n"); - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "View docs here: " - "https://github" - ".com/The-NextGen-Project/jet/blob/main/LANG.md for " - "syntax.\n"); + Console::Log(Colors::RESET, "Expected Token\n\n"); + + auto err = error.metadata.begin()->expected_error; + auto line = std::to_string(error.location.line); + ErrorParseSetup(error.location.line, err.message, err.got, err + .got->getSourceLocation()); + AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::GREEN, "Adding ", Colors::YELLOW, "'", + Token::GetTokenKindName(err.expected), "'\n"); + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "View docs here: " + "https://github" + ".com/The-NextGen-Project/jet/blob/main/LANG.md for " + "syntax.\n"); } -void Diagnostic::ErrorLexSetup(std::string& line, const char *message, +void Diagnostic::ErrorLexSetup(std::string &line, const char *message, LexError &error) { - auto printer = Lexer((const char *) source_line, - file_name, - source_line.size()); - // Log error line - Console::Log(Colors::RESET, line, " |\t "); - printer.lex(); // Print Failed Token - - Console::Log("\n"); - - // Align space - FOR(i, line.length() + 1) Console::Log(" "); - - Console::Log("\t", Colors::RED); - FOR(column, source_line.size()) { - if (column == error.failed_token.getSourceLocation().column) { - FOR(ch, error.failed_token.name().size()) { - Console::Log("^"); - } - break; - } - Console::Log(" "); - } - - Console::Log("_ <-- ", message); - Console::Log("\n\n"); + auto printer = Lexer((const char *) source_line, + file_name, + source_line.size()); + // Log error line + Console::Log(Colors::RESET, line, " |\t "); + printer.lex(); // Print Failed Token + + Console::Log("\n"); + + // Align space + FOR(i, line.length() + 1) Console::Log(" "); + + Console::Log("\t", Colors::RED); + FOR(column, source_line.size()) { + if (column == error.failed_token.getSourceLocation().column) { + FOR(ch, error.failed_token.name().size()) { + Console::Log("^"); + } + break; + } + Console::Log(" "); + } + + Console::Log("_ <-- ", message); + Console::Log("\n\n"); } template @@ -444,59 +444,58 @@ void Diagnostic::ErrorParseSetup(size_t const ln, const char *message, Token const *reported_token, TokenTraits::SourceLocation loc) { - std::string line = std::to_string(ln); - if (ln > 1) { - Console::Log(Colors::RESET, ln - 1, " | "); - - str new_line; - - if (ln - 1 == 1) { - new_line = str(file_buf, source_line.begin() - file_buf - 1); - } - else { - new_line = GetNthLineOfBuffer(ln - 1, file_buf, buf_len); - } - - Lexer(new_line.begin(), - file_name, - new_line.size()).lex(); - Console::Log("\n"); - } - - auto printer = Lexer((const char *) source_line, - file_name, - source_line.size()); - // Log error line - Console::Log(Colors::RESET, line, " | "); - printer.lex(); // Print Failed Token - - Console::Log("\n"); - - // Align space - FOR(i, line.length()+1) Console::Log(" "); - - Console::Log(" ", Colors::RED); - - auto size = reported_token->getKind() == EOFToken ? source_line.size()+2 : - source_line.size(); - - size_t nth_column; - if (POINT) nth_column = loc.column-1; - else nth_column = loc.column; - for(size_t column = 1; column <= size; ++column) { - Console::Log(" "); - if (column == loc.column-1) { - FOR(ch, reported_token->name().size()) { - Console::Log("^"); - } - break; - } - } - - if (!POINT) - Console::Log("_ <-- ", message); - else Console::Log(" <-- ", message); - Console::Log("\n\n"); + std::string line = std::to_string(ln); + if (ln > 1) { + Console::Log(Colors::RESET, ln - 1, " | "); + + str new_line; + + if (ln - 1 == 1) { + new_line = str(file_buf, source_line.begin() - file_buf - 1); + } else { + new_line = GetNthLineOfBuffer(ln - 1, file_buf, buf_len); + } + + Lexer(new_line.begin(), + file_name, + new_line.size()).lex(); + Console::Log("\n"); + } + + auto printer = Lexer((const char *) source_line, + file_name, + source_line.size()); + // Log error line + Console::Log(Colors::RESET, line, " | "); + printer.lex(); // Print Failed Token + + Console::Log("\n"); + + // Align space + FOR(i, line.length() + 1) Console::Log(" "); + + Console::Log(" ", Colors::RED); + + auto size = reported_token->getKind() == EOFToken ? source_line.size() + 2 : + source_line.size(); + + size_t nth_column; + if (POINT) nth_column = loc.column - 1; + else nth_column = loc.column; + for (size_t column = 1; column <= size; ++column) { + Console::Log(" "); + if (column == loc.column - 1) { + FOR(ch, reported_token->name().size()) { + Console::Log("^"); + } + break; + } + } + + if (!POINT) + Console::Log("_ <-- ", message); + else Console::Log(" <-- ", message); + Console::Log("\n\n"); } diff --git a/lib/jet-lexer-v1.cpp b/lib/jet-lexer-v1.cpp index 0c27117..2ad02e8 100644 --- a/lib/jet-lexer-v1.cpp +++ b/lib/jet-lexer-v1.cpp @@ -6,1260 +6,1309 @@ using namespace nextgen::jet; # define TOKEN(type, text, k, value) Token(text, {line, column} \ ,value, k) -static constexpr TokenKind TokenKindClass[256] { +static constexpr TokenKind TokenKindClass[256]{ - // Error code - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, - TokenKind::Whitespace, // \a - TokenKind::Whitespace, // \b - TokenKind::Whitespace, // \t - TokenKind::NewLine, // \n - TokenKind::Whitespace, // \v - TokenKind::Error, + // Error code + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Whitespace, // \a + TokenKind::Whitespace, // \b + TokenKind::Whitespace, // \t + TokenKind::NewLine, // \n + TokenKind::Whitespace, // \v + TokenKind::Error, - TokenKind::NewLine, // \r + TokenKind::NewLine, // \r - TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - - TokenKind::Whitespace, // - - TokenKind::ExclamationPoint, // ! - TokenKind::String, // '"' - TokenKind::Error, TokenKind::Error, - TokenKind::Percent, // '%' - TokenKind::AND, // '&' - TokenKind::Char, // '\'' - TokenKind::LParenthesis, // ( - TokenKind::RParenthesis, // ) - TokenKind::Star, // '*' - TokenKind::Plus, // '+' - TokenKind::Comma, // ',' - - TokenKind::Minus, // '-' - - - TokenKind::Dot, // '.' - - TokenKind::Slash, // '/' - - - // Digits - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - - - TokenKind::Colon, // ':' - - - TokenKind::SemiColon, // ';' - TokenKind::LessThan, // '<' - TokenKind::Equals, // '=' - TokenKind::GreaterThan, // '>' - TokenKind::QuestionMark, // '?' - TokenKind::At, // @ - - // Uppercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LBracket, // '[' - - TokenKind::StringEscapeStart, // \\ - - TokenKind::RBracket, // ']' - - TokenKind::XOR, // '^' - - TokenKind::Identifier,// '_' - - TokenKind::Error, - - // Lowercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LCurlyBrace, // '{' - TokenKind::Pipe, // '|' - TokenKind::RCurlyBrace, // '}' - TokenKind::NOT, // '~' - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + + TokenKind::Whitespace, // + + TokenKind::ExclamationPoint, // ! + TokenKind::String, // '"' + TokenKind::Error, TokenKind::Error, + TokenKind::Percent, // '%' + TokenKind::AND, // '&' + TokenKind::Char, // '\'' + TokenKind::LParenthesis, // ( + TokenKind::RParenthesis, // ) + TokenKind::Star, // '*' + TokenKind::Plus, // '+' + TokenKind::Comma, // ',' + + TokenKind::Minus, // '-' + + + TokenKind::Dot, // '.' + + TokenKind::Slash, // '/' + + + // Digits + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + + + TokenKind::Colon, // ':' + + + TokenKind::SemiColon, // ';' + TokenKind::LessThan, // '<' + TokenKind::Equals, // '=' + TokenKind::GreaterThan, // '>' + TokenKind::QuestionMark, // '?' + TokenKind::At, // @ + + // Uppercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LBracket, // '[' + + TokenKind::StringEscapeStart, // \\ + + TokenKind::RBracket, // ']' + + TokenKind::XOR, // '^' + + TokenKind::Identifier,// '_' + + TokenKind::Error, + + // Lowercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LCurlyBrace, // '{' + TokenKind::Pipe, // '|' + TokenKind::RCurlyBrace, // '}' + TokenKind::NOT, // '~' + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, }; constexpr static int IntegerBits[256] = { - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, - // Digits - 0,1,2,3,4,5,6,7,8,9, + // Digits + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - // Error Code - -3,-3,-3,-3,-3,-3,-3, + // Error Code + -3, -3, -3, -3, -3, -3, -3, - // Uppercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35, + // Uppercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, - // Lowercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35, + // Lowercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, }; struct JetKeyword { - str key; - TokenKind id; + str key; + TokenKind id; }; -static const struct JetKeyword Keywords[] { - { "if"_intern, KeywordIf }, - { "else"_intern, KeywordElse }, - { "elif"_intern, KeywordElif }, - { "while"_intern, KeywordWhile }, - { "for"_intern, KeywordFor }, - { "true"_intern, KeywordTrue }, - { "false"_intern, KeywordFalse }, - { "None"_intern, KeywordNone }, - { "fn"_intern, KeywordFunction }, - { "struct"_intern, KeywordStruct }, - { "export"_intern, KeywordExport }, - { "enum"_intern, KeywordEnum }, - { "extern"_intern, KeywordExtern }, - { "error"_intern, KeywordError }, - { "var"_intern, KeywordVar }, - { "return"_intern, KeywordReturn }, - { "and"_intern, KeywordAnd }, - { "or"_intern, KeywordOr }, - { "break"_intern, KeywordBreak }, - { "continue"_intern, KeywordContinue }, - { "defer"_intern, KeywordDefer }, - { "union"_intern, KeywordUnion }, - { "match"_intern, KeywordMatch }, - { "in"_intern, KeywordIn } +static const struct JetKeyword Keywords[]{ + {"if"_intern, KeywordIf}, + {"else"_intern, KeywordElse}, + {"elif"_intern, KeywordElif}, + {"while"_intern, KeywordWhile}, + {"for"_intern, KeywordFor}, + {"true"_intern, KeywordTrue}, + {"false"_intern, KeywordFalse}, + {"None"_intern, KeywordNone}, + {"fn"_intern, KeywordFunction}, + {"struct"_intern, KeywordStruct}, + {"export"_intern, KeywordExport}, + {"enum"_intern, KeywordEnum}, + {"extern"_intern, KeywordExtern}, + {"error"_intern, KeywordError}, + {"var"_intern, KeywordVar}, + {"return"_intern, KeywordReturn}, + {"and"_intern, KeywordAnd}, + {"or"_intern, KeywordOr}, + {"break"_intern, KeywordBreak}, + {"continue"_intern, KeywordContinue}, + {"defer"_intern, KeywordDefer}, + {"union"_intern, KeywordUnion}, + {"match"_intern, KeywordMatch}, + {"in"_intern, KeywordIn} }; /// Determine the validity of parsed string being a Keyword. The assumption /// made in this function is that `value` is an interned string therefore only /// direct pointer comparison is made resulting in O(1) comparison. static TokenKind ReturnValidKeyword(str &value) { - for (auto i = 0; i < SizeOfArray(Keywords); ++i) { - if (value == Keywords[i].key) - return Keywords[i].id; - } - return TokenKind::Identifier; + for (auto i = 0; i < SizeOfArray(Keywords); ++i) { + if (value == Keywords[i].key) + return Keywords[i].id; + } + return TokenKind::Identifier; } template -Token Lexer::next_token() { - TokenKind kind = TokenKindClass[curr()]; - Token instance = Token {}; - - goto start; - start: - { - switch (kind) { - - /// - /// Integers - /// - /// Pattern: Key Lexing Statutes - /// â€ĸ '_' is valid anywhere in the integer as long as it's not the - /// first character. Examples: - /// â€ĸ 123_232_2423 - /// â€ĸ 234__343 - /// â€ĸ 3__23_3_23_2_2 - /// â€ĸ Identifiers are not interned for integers. This is because they - /// do not provide any reasonable performance gain. - /// â€ĸ All number values with suffixes of: 'e' or 'p' are floating - /// point values unless casted (handled in Parser/AST). - /// - /// Valid Patterns: - /// â€ĸ 0x123 (Hexadecimal) - /// â€ĸ 12345 (Base 10) - /// â€ĸ 0b011 (Base 2) - /// â€ĸ 0#abc (Base 36) - /// - /// - case Integer: { - - // Integer Base - int radix = 10; - - // Classification for Unsigned values - auto flags = static_cast(TokenClassification::Literal); - - // Start of token - auto Begin = buffer; - auto BeginCol = column; - switch (curr()) { - case '0': - switch (next(1)) { - case 'b': - radix = 2; - next(1); - break; - case 'x': - radix = 16; - next(1); - break; - case '#': - radix = 36; - next(1); - break; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': - radix = 8; - break; - default: // TODO: Figure what needs to be done here - break; - } - break; - default: - break; - } - - - decltype(UINTPTR_MAX) IntegerValue = 0; - do { - - auto Current = curr(); - int Digit = IntegerBits[Current]; - - if (Digit < 0) { - break; - } - - if (Digit >= radix) { - diagnostics.build(LexError{ - LexErrorType::DigitOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, TokenKind::Integer), - {radix, Digit + 1} - }); - fatal = true; - return instance; - } - - if (IntegerValue > (UINTPTR_MAX - Digit) / radix) { - diagnostics.build(LexError{ - LexErrorType::IntegerOverflow, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::Integer), - }); - fatal = true; - return instance; - } - IntegerValue = IntegerValue * radix + Digit; - } while (next(1)); - - // Token Suffix - switch (curr()) { // TODO: Add Implementation - case 'p': // Power - break; - case 'u': // Unsigned - flags = TokenClassification::Unsigned; - next(1); - break; - case 'e': // x10^power - if (radix == 16) UNREACHABLE; - break; - } - - // End of Token - auto End = buffer; - - - // Important Note: When integer values are lexed, they do not hold - // a direct value stored in a buffer, rather just a pseudo range value - // with no allocation is substituted because integer value strings do - // need to be interned. - auto ID = str(Range(Begin, End)); - - instance = TOKEN(decltype(UINTPTR_MAX), ID, TokenKind::Integer, - IntegerValue); - instance.setFlag(flags); - return instance; - } - case String: { - auto Begin = this->buffer; - auto BeginCol = column; - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - char Current = next(1); - - // Equivalent to C99 String Standard - auto StringAllocator = allocator->allocator_for_space(4095); - - char *ID = (char*) StringAllocator->current_point(); - size_t Len = 0; - - do { - Current = curr(); - - if (Current == '\0') { - diagnostics.build(LexError{ - LexErrorType::MissingClosingDelim, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {'"'} - }); - fatal = true; - return instance; - } - - if (Current == '\\') { - switch (next(1)) { - case 'a': - Current = '\a'; - break; - case 'b': - Current = '\b'; - break; - case 'r': - Current = '\r'; - break; - case 'n': - Current = '\n'; - break; - case 'v': - Current = '\v'; - break; - case 't': - Current = '\t'; - break; - case 'x': { - auto V1 = IntegerBits[next(1)]; - if (V1 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {V1 + 1} // Pass in the invalid digit for the escape - }); - fatal = true; - return instance; - } - auto V2 = IntegerBits[next(1)]; - if (V2 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {V2 + 1} // Pass in the invalid digit for the escape - }); - fatal = true; - return instance; - } - - // Unroll Loop - int Value = V1; - Value = Value * 16 + V2; - - // @Safety Guaranteed! We know for a fact that this conversion - // is safe because of the validity. - Current = (char) Value; - break; - } - case 'u': // TODO: IDK how to do this properly but ... Unicode Char - break; - default: - diagnostics.build(LexError{ - LexErrorType::InvalidStringEscape, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - }); - fatal = true; - return instance; - } - } - - val ^= Current; - val *= FNV_PRIME; - - ID[Len++] = Current; - - - } while (next(1) != '"'); - - ID[Len++] = '\0'; - - auto S = str(ID, Len); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - instance = TOKEN(const char *, Intern, TokenKind::String, ""); - instance.setFlag(TokenClassification::Literal); - next(1); // Skip the last '"' - return instance; - } - case Decimal: - break; - case Identifier: { - - - auto Begin = this->buffer; - - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - char Current; - do { - Current = curr(); - - val ^= Current; - val *= FNV_PRIME; - - - } while (TokenKindClass[next(1)] == TokenKind::Identifier); - - auto End = this->buffer; - - auto S = str(Begin, End-Begin); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - // Ensure appropriate kind is assigned - auto Type = ReturnValidKeyword(Intern); - - instance = TOKEN(const char *, Intern, Type, ""); - if (Type != TokenKind::Identifier) - instance.setFlag(TokenClassification::Keyword); - instance.setFlag(TokenClassification::Literal); - - return instance; - } - case LessThan: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::LessThan, TokenKind::Equals): - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - break; - group(TokenKind::LessThan, TokenKind::LessThan): // << <-- - kind = TokenKind::LeftShift; - goto start; - default: - instance = TOKEN(char, "<", TokenKind::LessThan, "<"); - break; - } - - auto s2 = peek(1); - match ('<', s2) { - group('<', '='): - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - break; - group('<', '<'): // << <-- - kind = TokenKind::LeftShift; - goto start; - default: - instance = TOKEN(char, "<", TokenKind::LessThan, "<"); - break; - } - if (s2 == '=') { - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - } - break; - } - case LeftShift: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::LeftShift, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, "<<=", TokenKind::LeftShiftEquals, - "<<="); - break; - default: - instance = TOKEN(const char *, "<<", TokenKind::LeftShift, "<<"); - break; - } - break; - } - case RightShift: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::RightShift, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, ">>=", TokenKind::RightShiftEquals, - ">>="); - break; - default: - instance = TOKEN(const char *, ">>", TokenKind::RightShift, ">>"); - break; - } - break; - } - case GreaterThan: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::GreaterThan, TokenKind::Equals): - instance = TOKEN(const char *, ">=", TokenKind::GreaterThanEquals, - ">="); - instance.assignment = true; - break; - group(TokenKind::GreaterThan, TokenKind::GreaterThan): - kind = TokenKind::RightShift; - goto start; - default: - instance = TOKEN(char, ">", TokenKind::GreaterThan, ">"); - break; - } - break; - } - case Equals: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::Equals) { - instance = TOKEN(const char *, "==", TokenKind::EqualsEquals, "=="); - } else { - instance = TOKEN(char, "=", TokenKind::Equals, "="); - } - break; - } - case QuestionMark: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::QuestionMark) { - instance = TOKEN(const char *, "??", TokenKind::Then, "??"); - } else { - instance = TOKEN(char, "?", TokenKind::QuestionMark, "?"); - } - break; - } - case Plus: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Plus, TokenKind::Plus): - instance = TOKEN(const char *, "++", TokenKind::PlusPlus, "++"); - break; - group(TokenKind::Plus, TokenKind::Equals): - instance = TOKEN(const char *, "+=", TokenKind::PlusEquals, "+="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "+", TokenKind::Plus, "+"); - break; - } - break; - } - case Minus: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Minus, TokenKind::Minus): - instance = TOKEN(const char *, "--", TokenKind::MinusMinus, "--"); - break; - group(TokenKind::Minus, TokenKind::Equals): - instance = TOKEN(const char *, "-=", TokenKind::MinusEquals, "-="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "-", TokenKind::Minus, "-"); - break; - } - break; - } - case Slash: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Slash, TokenKind::Equals): - instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "/", TokenKind::Star, "/"); - break; - } - break; - } - case Star: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Star, TokenKind::Star): - kind = TokenKind::Pow; - goto start; - group(TokenKind::Star, TokenKind::Equals): - instance = TOKEN(const char *, "*=", TokenKind::MulEquals, "*="); - instance.assignment = true; - break; - default: - instance = TOKEN(const char *, "*", TokenKind::Star, "*"); - break; - } - break; - } - case Pow: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::Pow, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, "**=", TokenKind::PowEquals, "**="); - break; - default: - instance = TOKEN(const char *, "**", TokenKind::Pow, "**"); - break; - } - break; - } - case AND: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::AND, TokenKind::Equals): - instance = TOKEN(const char *, "&=", TokenKind::ANDEquals, "&="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "&", TokenKind::AND, "&"); - break; - } - break; - } - case Pipe: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Pipe, TokenKind::Equals): - instance = TOKEN(const char *, "|=", TokenKind::OREquals, "|="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "|", TokenKind::Pipe, "|"); - break; - } - break; - } - case XOR: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::XOR, TokenKind::Equals): - instance = TOKEN(const char *, "^=", TokenKind::XOREquals, "^="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "^", TokenKind::XOR, "^"); - break; - } - break; - } - case NewLine: // \n or \r - skip_new_line(); - kind = TokenKindClass[curr()]; - goto start; // Does this so 2 function calls aren't wasted. - case Char: { - auto Begin = buffer; - auto BeginCol = column; - instance = TOKEN(char, next(1), TokenKind::Char, curr()); - if (TokenKindClass[next(1)] != TokenKind::Char) { - diagnostics.build(LexError{ - LexErrorType::MissingClosingDelim, - {line, column}, // 'x[?] <-- Missing Closing! - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {'\''} - }); - fatal = true; - } - instance.setFlag(TokenClassification::Literal); - break; - } - case Whitespace: // Skip whitespace - next(1); - kind = TokenKindClass[curr()]; - goto start; - case Dot: { - auto Begin = buffer; - auto BeginCol = column; - - auto s1 = TokenKindClass[peek(1)]; - auto s2 = TokenKindClass[peek(2)]; - auto s3 = TokenKindClass[peek(3)]; - - match (kind, s1) { - group(TokenKind::Dot, TokenKind::Dot): - instance = s2 == TokenKind::Dot ? - (TOKEN(const char *, "...", - TokenKind::Ellipsis, "...")) : - (TOKEN(const char *, "..", - TokenKind::RangeSpan, "..")); - break; - default: - instance = TOKEN(char, ".", TokenKind::Dot, "."); - break; - } - // Cannot match token "...[.]" <-- Invalid Dot Here! - if (s3 == TokenKind::Dot && instance.getKind() == TokenKind::Ellipsis) { - diagnostics.build(LexError{ - LexErrorType::InvalidDot, - {line, column + 1}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::Ellipsis) - }); - fatal = true; - } - - break; - } - // Cannot be generated by the TokenKind table, assigned after checks - case EqualsEquals: - case Boolean: - UNREACHABLE; - break; - case LParenthesis: - instance = TOKEN(const char *, "(", kind, ")"); - break; - case RParenthesis: - instance = TOKEN(const char *, ")", kind, ")"); - break; - case LBracket: - instance = TOKEN(const char *, "[", kind, "["); - break; - case RBracket: - instance = TOKEN(const char *, "]", kind, "]"); - break; - case LCurlyBrace: - instance = TOKEN(const char *, "{", kind, "{"); - break; - case RCurlyBrace: - instance = TOKEN(const char *, "}", kind, "}"); - break; - case NOT: - instance = TOKEN(const char *, "~", kind, "~"); - break; - case Colon: - instance = TOKEN(const char *, ":", kind, ":"); - break; - case SemiColon: - instance = TOKEN(const char *, ";", kind, ";"); - break; - case ExclamationPoint: - instance = TOKEN(const char *, "!", kind, "!"); - break; - case At: - instance = TOKEN(const char *, "@", kind, "@"); - break; - case Percent: - instance = TOKEN(const char *, "%", kind, "%"); - break; - case Comma: - instance = TOKEN(const char *, ",", kind, ","); - break; - case Error: - if (curr() == '\0') { - buf_pos++; - return Token("\0", {line, column}, - TokenKind::EOFToken); - } - diagnostics.build(LexError{ - LexErrorType::InvalidChar, - {line, column} - }); - fatal = true; - default: - //instance = TOKEN(char, Curr(), kind, curr()); - break; - } - } - next(instance.len()); - return instance; +Token Lexer::next_token() { + TokenKind kind = TokenKindClass[curr()]; + Token instance = Token{}; + + goto start; + start: + { + switch (kind) { + + /// + /// Integers + /// + /// Pattern: Key Lexing Statutes + /// â€ĸ '_' is valid anywhere in the integer as long as it's not the + /// first character. Examples: + /// â€ĸ 123_232_2423 + /// â€ĸ 234__343 + /// â€ĸ 3__23_3_23_2_2 + /// â€ĸ Identifiers are not interned for integers. This is because they + /// do not provide any reasonable performance gain. + /// â€ĸ All number values with suffixes of: 'e' or 'p' are floating + /// point values unless casted (handled in Parser/AST). + /// + /// Valid Patterns: + /// â€ĸ 0x123 (Hexadecimal) + /// â€ĸ 12345 (Base 10) + /// â€ĸ 0b011 (Base 2) + /// â€ĸ 0#abc (Base 36) + /// + /// + case Integer: { + + // Integer Base + int radix = 10; + + // Classification for Unsigned values + auto flags = static_cast(TokenClassification::Literal); + + // Start of token + auto Begin = buffer; + auto BeginCol = column; + switch (curr()) { + case '0': + switch (next(1)) { + case 'b': + radix = 2; + next(1); + break; + case 'x': + radix = 16; + next(1); + break; + case '#': + radix = 36; + next(1); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + radix = 8; + break; + default: // TODO: Figure what needs to be done here + break; + } + break; + default: + break; + } + + + decltype(UINTPTR_MAX) IntegerValue = 0; + do { + + auto Current = curr(); + int Digit = IntegerBits[Current]; + + if (Digit < 0) { + break; + } + + if (Digit >= radix) { + diagnostics.build(LexError{ + LexErrorType::DigitOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, TokenKind::Integer), + {radix, Digit + 1} + }); + fatal = true; + return instance; + } + + if (IntegerValue > (UINTPTR_MAX - Digit) / radix) { + diagnostics.build(LexError{ + LexErrorType::IntegerOverflow, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::Integer), + }); + fatal = true; + return instance; + } + IntegerValue = IntegerValue * radix + Digit; + } while (next(1)); + + // Token Suffix + switch (curr()) { // TODO: Add Implementation + case 'p': // Power + break; + case 'u': // Unsigned + flags = TokenClassification::Unsigned; + next(1); + break; + case 'e': // x10^power + if (radix == 16) UNREACHABLE; + break; + } + + // End of Token + auto End = buffer; + + + // Important Note: When integer values are lexed, they do not hold + // a direct value stored in a buffer, rather just a pseudo range value + // with no allocation is substituted because integer value strings do + // need to be interned. + auto ID = str(Range(Begin, End)); + + instance = TOKEN(decltype(UINTPTR_MAX), ID, TokenKind::Integer, + IntegerValue); + instance.setFlag(flags); + return instance; + } + case String: { + auto Begin = this->buffer; + auto BeginCol = column; + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + char Current = next(1); + + // Equivalent to C99 String Standard + auto StringAllocator = allocator->allocator_for_space(4095); + + char *ID = (char *) StringAllocator->current_point(); + size_t Len = 0; + + do { + Current = curr(); + + if (Current == '\0') { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {'"'} + }); + fatal = true; + return instance; + } + + if (Current == '\\') { + switch (next(1)) { + case 'a': + Current = '\a'; + break; + case 'b': + Current = '\b'; + break; + case 'r': + Current = '\r'; + break; + case 'n': + Current = '\n'; + break; + case 'v': + Current = '\v'; + break; + case 't': + Current = '\t'; + break; + case 'x': { + auto V1 = IntegerBits[next(1)]; + if (V1 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {V1 + 1} // Pass in the invalid digit for the escape + }); + fatal = true; + return instance; + } + auto V2 = IntegerBits[next(1)]; + if (V2 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {V2 + 1} // Pass in the invalid digit for the escape + }); + fatal = true; + return instance; + } + + // Unroll Loop + int Value = V1; + Value = Value * 16 + V2; + + // @Safety Guaranteed! We know for a fact that this conversion + // is safe because of the validity. + Current = (char) Value; + break; + } + case 'u': // TODO: IDK how to do this properly but ... Unicode Char + break; + default: + diagnostics.build(LexError{ + LexErrorType::InvalidStringEscape, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + }); + fatal = true; + return instance; + } + } + + val ^= Current; + val *= FNV_PRIME; + + ID[Len++] = Current; + + + } while (next(1) != '"'); + + ID[Len++] = '\0'; + + auto S = str(ID, Len); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + instance = TOKEN(const char *, Intern, TokenKind::String, ""); + instance.setFlag(TokenClassification::Literal); + next(1); // Skip the last '"' + return instance; + } + case Decimal: + break; + case Identifier: { + + + auto Begin = this->buffer; + + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + char Current; + do { + Current = curr(); + + val ^= Current; + val *= FNV_PRIME; + + + } while (TokenKindClass[next(1)] == TokenKind::Identifier); + + auto End = this->buffer; + + auto S = str(Begin, End - Begin); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + // Ensure appropriate kind is assigned + auto Type = ReturnValidKeyword(Intern); + + instance = TOKEN(const char *, Intern, Type, ""); + if (Type != TokenKind::Identifier) + instance.setFlag(TokenClassification::Keyword); + instance.setFlag(TokenClassification::Literal); + + return instance; + } + case LessThan: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::LessThan, TokenKind::Equals): + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + break; + group(TokenKind::LessThan, TokenKind::LessThan): // << <-- + kind = TokenKind::LeftShift; + goto start; + default: + instance = TOKEN(char, "<", TokenKind::LessThan, "<"); + break; + } + + auto s2 = peek(1); + match('<', s2) + { + group('<', '='): + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + break; + group('<', '<'): // << <-- + kind = TokenKind::LeftShift; + goto start; + default: + instance = TOKEN(char, "<", TokenKind::LessThan, "<"); + break; + } + if (s2 == '=') { + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + } + break; + } + case LeftShift: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::LeftShift, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, "<<=", TokenKind::LeftShiftEquals, + "<<="); + break; + default: + instance = TOKEN(const char *, "<<", TokenKind::LeftShift, "<<"); + break; + } + break; + } + case RightShift: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::RightShift, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, ">>=", TokenKind::RightShiftEquals, + ">>="); + break; + default: + instance = TOKEN(const char *, ">>", TokenKind::RightShift, ">>"); + break; + } + break; + } + case GreaterThan: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::GreaterThan, TokenKind::Equals): + instance = TOKEN(const char *, ">=", TokenKind::GreaterThanEquals, + ">="); + instance.assignment = true; + break; + group(TokenKind::GreaterThan, TokenKind::GreaterThan): + kind = TokenKind::RightShift; + goto start; + default: + instance = TOKEN(char, ">", TokenKind::GreaterThan, ">"); + break; + } + break; + } + case Equals: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::Equals) { + instance = TOKEN(const char *, "==", TokenKind::EqualsEquals, "=="); + } else { + instance = TOKEN(char, "=", TokenKind::Equals, "="); + } + break; + } + case QuestionMark: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::QuestionMark) { + instance = TOKEN(const char *, "??", TokenKind::Then, "??"); + } else { + instance = TOKEN(char, "?", TokenKind::QuestionMark, "?"); + } + break; + } + case Plus: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Plus, TokenKind::Plus): + instance = TOKEN(const char *, "++", TokenKind::PlusPlus, "++"); + break; + group(TokenKind::Plus, TokenKind::Equals): + instance = TOKEN(const char *, "+=", TokenKind::PlusEquals, "+="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "+", TokenKind::Plus, "+"); + break; + } + break; + } + case Minus: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Minus, TokenKind::Minus): + instance = TOKEN(const char *, "--", TokenKind::MinusMinus, "--"); + break; + group(TokenKind::Minus, TokenKind::Equals): + instance = TOKEN(const char *, "-=", TokenKind::MinusEquals, "-="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "-", TokenKind::Minus, "-"); + break; + } + break; + } + case Slash: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Slash, TokenKind::Equals): + instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "/", TokenKind::Star, "/"); + break; + } + break; + } + case Star: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Star, TokenKind::Star): + kind = TokenKind::Pow; + goto start; + group(TokenKind::Star, TokenKind::Equals): + instance = TOKEN(const char *, "*=", TokenKind::MulEquals, "*="); + instance.assignment = true; + break; + default: + instance = TOKEN(const char *, "*", TokenKind::Star, "*"); + break; + } + break; + } + case Pow: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::Pow, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, "**=", TokenKind::PowEquals, "**="); + break; + default: + instance = TOKEN(const char *, "**", TokenKind::Pow, "**"); + break; + } + break; + } + case AND: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::AND, TokenKind::Equals): + instance = TOKEN(const char *, "&=", TokenKind::ANDEquals, "&="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "&", TokenKind::AND, "&"); + break; + } + break; + } + case Pipe: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Pipe, TokenKind::Equals): + instance = TOKEN(const char *, "|=", TokenKind::OREquals, "|="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "|", TokenKind::Pipe, "|"); + break; + } + break; + } + case XOR: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::XOR, TokenKind::Equals): + instance = TOKEN(const char *, "^=", TokenKind::XOREquals, "^="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "^", TokenKind::XOR, "^"); + break; + } + break; + } + case NewLine: // \n or \r + skip_new_line(); + kind = TokenKindClass[curr()]; + goto start; // Does this so 2 function calls aren't wasted. + case Char: { + auto Begin = buffer; + auto BeginCol = column; + instance = TOKEN(char, next(1), TokenKind::Char, curr()); + if (TokenKindClass[next(1)] != TokenKind::Char) { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, // 'x[?] <-- Missing Closing! + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {'\''} + }); + fatal = true; + } + instance.setFlag(TokenClassification::Literal); + break; + } + case Whitespace: // Skip whitespace + next(1); + kind = TokenKindClass[curr()]; + goto start; + case Dot: { + auto Begin = buffer; + auto BeginCol = column; + + auto s1 = TokenKindClass[peek(1)]; + auto s2 = TokenKindClass[peek(2)]; + auto s3 = TokenKindClass[peek(3)]; + + match(kind, s1) + { + group(TokenKind::Dot, TokenKind::Dot): + instance = s2 == TokenKind::Dot ? + (TOKEN(const char *, "...", + TokenKind::Ellipsis, "...")) : + (TOKEN(const char *, "..", + TokenKind::RangeSpan, "..")); + break; + default: + instance = TOKEN(char, ".", TokenKind::Dot, "."); + break; + } + // Cannot match token "...[.]" <-- Invalid Dot Here! + if (s3 == TokenKind::Dot && instance.getKind() == TokenKind::Ellipsis) { + diagnostics.build(LexError{ + LexErrorType::InvalidDot, + {line, column + 1}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::Ellipsis) + }); + fatal = true; + } + + break; + } + // Cannot be generated by the TokenKind table, assigned after checks + case EqualsEquals: + case Boolean: + UNREACHABLE; + break; + case LParenthesis: + instance = TOKEN(const char *, "(", kind, ")"); + break; + case RParenthesis: + instance = TOKEN(const char *, ")", kind, ")"); + break; + case LBracket: + instance = TOKEN(const char *, "[", kind, "["); + break; + case RBracket: + instance = TOKEN(const char *, "]", kind, "]"); + break; + case LCurlyBrace: + instance = TOKEN(const char *, "{", kind, "{"); + break; + case RCurlyBrace: + instance = TOKEN(const char *, "}", kind, "}"); + break; + case NOT: + instance = TOKEN(const char *, "~", kind, "~"); + break; + case Colon: + instance = TOKEN(const char *, ":", kind, ":"); + break; + case SemiColon: + instance = TOKEN(const char *, ";", kind, ";"); + break; + case ExclamationPoint: + instance = TOKEN(const char *, "!", kind, "!"); + break; + case At: + instance = TOKEN(const char *, "@", kind, "@"); + break; + case Percent: + instance = TOKEN(const char *, "%", kind, "%"); + break; + case Comma: + instance = TOKEN(const char *, ",", kind, ","); + break; + case Error: + if (curr() == '\0') { + buf_pos++; + return Token("\0", {line, column}, + TokenKind::EOFToken); + } + diagnostics.build(LexError{ + LexErrorType::InvalidChar, + {line, column} + }); + fatal = true; + default: + //instance = TOKEN(char, Curr(), kind, curr()); + break; + } + } + next(instance.len()); + return instance; } template void Lexer::PrintNextToken() { - TokenKind Kind = TokenKindClass[curr()]; - Token Instance = Token {}; - - goto start; - start: - { - if (!buffer) - return; - switch (Kind) { - - case Integer: { - - // Start of token - auto Begin = buffer; - - switch (curr()) { - case '0': - switch (next(1)) { - case 'b': - next(1); - break; - case 'x': - next(1); - break; - case '#': - next(1); - break; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': - break; - default: - break; - } - break; - default: - break; - } - - do { - - auto Current = curr(); - int Digit = IntegerBits[Current]; - - if (Digit < 0) { - break; - } - - } while (next(1)); - - // Token Suffix - switch (curr()) { // TODO: Add Implementation - case 'p': // Power - break; - case 'u': // Unsigned - break; - case 'e': // x10^power - break; - } - - auto End = buffer; - - auto Print = str(Begin, End - Begin); - Console::Log(Colors::BLUE, Print, Colors::RESET); - - break; - } - case String: { - auto Begin = this->buffer; - int Offset = 1; - Console::Log(Colors::GREEN); - - char Current; - do { - Current = curr(); - - if (Current == '\0') { - Offset = 0; - break; - } - } while (next(1) != '"'); - - - auto End = this->buffer; - - auto Print = str(Begin, End - Begin + Offset); - Console::Log(Print, Colors::RESET); - - next(1); // Skip the last '"' - break; - } - case Decimal: - break; - case Identifier: { // Maybe : Identifier length no more 256 characters? - - - auto Begin = this->buffer; - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - do { - - val ^= curr(); - val *= FNV_PRIME; - } while (TokenKindClass[next(1)] == TokenKind::Identifier); - - - auto End = this->buffer; - - auto S = str(Begin, End-Begin); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - // Ensure appropriate kind is assigned - auto Type = ReturnValidKeyword(Intern); - - if (Type != TokenKind::Identifier) { - Console::Log(Colors::RED, Intern, Colors::RESET); - } else { - - Console::Log(Colors::YELLOW, Intern, Colors::RESET); - } - break; - } - case LessThan: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::LessThan, TokenKind::Equals): - Console::Log("<="); - next(2); - break; - group(TokenKind::LessThan, TokenKind::LessThan): - Kind = TokenKind::LeftShift; - goto start; - default: - Console::Log("<"); - next(1); - break; - } - break; - } - case LeftShift: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::LeftShift, TokenKind::Equals): - Console::Log("<<="); - next(3); - break; - default: - Console::Log("<<"); - next(2); - break; - } - break; - } - case RightShift: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::RightShift, TokenKind::Equals): - Console::Log(">>="); - next(3); - break; - default: - Console::Log(">>"); - next(2); - break; - } - break; - } - case GreaterThan: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::GreaterThan, TokenKind::Equals): - Console::Log(">="); - next(2); - break; - group(TokenKind::GreaterThan, TokenKind::GreaterThan): - Kind = TokenKind::RightShift; - goto start; - default: - Console::Log(">"); - next(1); - break; - } - break; - } - case Equals: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::Equals) { - Console::Log("=="); - next(1); - } else { - Console::Log("="); - } - next(1); - break; - } - case QuestionMark: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::QuestionMark) { - Console::Log("??"); - next(2); - } else { - Console::Log("?"); - next(1); - } - break; - } - case Plus: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Plus, TokenKind::Plus): - Console::Log("++"); - next(2); - break; - group(TokenKind::Plus, TokenKind::Equals): - Console::Log("+="); - next(2); - break; - default: - Console::Log("+"); - next(1); - break; - } - break; - } - case Minus: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Minus, TokenKind::Minus): - Console::Log("--"); - next(2); - break; - group(TokenKind::Minus, TokenKind::Equals): - Console::Log("-="); - next(2); - break; - default: - Console::Log("-"); - next(1); - break; - } - break; - } - case Slash: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Slash, TokenKind::Equals): - Instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); - Instance.assignment = true; - break; - default: - Instance = TOKEN(const char *, "/", TokenKind::Slash, "/"); - break; - } - break; - } - case Star: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Star, TokenKind::Star): - Kind = TokenKind::Pow; - goto start; - group(TokenKind::Star, TokenKind::Equals): - Console::Log("*="); - break; - default: - Console::Log('*'); - break; - } - break; - } - case Pow: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::Pow, TokenKind::Equals): - Console::Log("**="); - next(3); - break; - default: - Console::Log("**"); - next(2); - break; - } - break; - } - case AND: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::AND, TokenKind::Equals): - Console::Log("/="); - next(2); - break; - default: - Console::Log('/'); - next(1); - break; - } - break; - } - case Pipe: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Pipe, TokenKind::Equals): - Console::Log("|="); - next(2); - break; - default: - Console::Log('|'); - next(1); - break; - } - break; - } - case XOR: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::XOR, TokenKind::Equals): - Console::Log("^="); - next(2); - break; - default: - Console::Log('^'); - next(1); - break; - } - break; - } - case NewLine: // \n or \r - Console::Log(curr()); - skip_new_line(); - Kind = TokenKindClass[curr()]; - goto start; // Does this so 2 function calls aren't wasted. - case Char: - Console::Log(Colors::GREEN, "'", next(1), Colors::RESET); - if (peek(1) == '\'') - Console::Log("'"); - next(1); - break; - case Whitespace: // Skip whitespace - Console::Log(curr()); - next(1); - Kind = TokenKindClass[curr()]; - goto start; - case Dot: { - auto s1 = TokenKindClass[peek(1)]; - auto s2 = TokenKindClass[peek(2)]; - - match (Kind, s1) { - group(TokenKind::Dot, TokenKind::Dot): - s2 == TokenKind::Dot ? - next(3), Console::Log("...") : - (next(2), Console::Log("..")); - break; - default: - Console::Log("."); - next(1); - break; - } - break; - } - // Cannot be generated by the TokenKind table, assigned after checks - case EqualsEquals: - case Boolean: - UNREACHABLE; - break; - case LParenthesis: - case RParenthesis: - case LBracket: - case RBracket: - case LCurlyBrace: - case RCurlyBrace: - case NOT: - case Colon: - case SemiColon: - case ExclamationPoint: - case At: - case Comma: - Console::Log(curr()); - next(1); - break; - case Error: - buffer = nullptr; - break; - default: - break; - } - } + TokenKind Kind = TokenKindClass[curr()]; + Token Instance = Token{}; + + goto start; + start: + { + if (!buffer) + return; + switch (Kind) { + + case Integer: { + + // Start of token + auto Begin = buffer; + + switch (curr()) { + case '0': + switch (next(1)) { + case 'b': + next(1); + break; + case 'x': + next(1); + break; + case '#': + next(1); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + break; + default: + break; + } + break; + default: + break; + } + + do { + + auto Current = curr(); + int Digit = IntegerBits[Current]; + + if (Digit < 0) { + break; + } + + } while (next(1)); + + // Token Suffix + switch (curr()) { // TODO: Add Implementation + case 'p': // Power + break; + case 'u': // Unsigned + break; + case 'e': // x10^power + break; + } + + auto End = buffer; + + auto Print = str(Begin, End - Begin); + Console::Log(Colors::BLUE, Print, Colors::RESET); + + break; + } + case String: { + auto Begin = this->buffer; + int Offset = 1; + Console::Log(Colors::GREEN); + + char Current; + do { + Current = curr(); + + if (Current == '\0') { + Offset = 0; + break; + } + } while (next(1) != '"'); + + + auto End = this->buffer; + + auto Print = str(Begin, End - Begin + Offset); + Console::Log(Print, Colors::RESET); + + next(1); // Skip the last '"' + break; + } + case Decimal: + break; + case Identifier: { // Maybe : Identifier length no more 256 characters? + + + auto Begin = this->buffer; + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + do { + + val ^= curr(); + val *= FNV_PRIME; + } while (TokenKindClass[next(1)] == TokenKind::Identifier); + + + auto End = this->buffer; + + auto S = str(Begin, End - Begin); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + // Ensure appropriate kind is assigned + auto Type = ReturnValidKeyword(Intern); + + if (Type != TokenKind::Identifier) { + Console::Log(Colors::RED, Intern, Colors::RESET); + } else { + + Console::Log(Colors::YELLOW, Intern, Colors::RESET); + } + break; + } + case LessThan: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::LessThan, TokenKind::Equals): + Console::Log("<="); + next(2); + break; + group(TokenKind::LessThan, TokenKind::LessThan): + Kind = TokenKind::LeftShift; + goto start; + default: + Console::Log("<"); + next(1); + break; + } + break; + } + case LeftShift: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::LeftShift, TokenKind::Equals): + Console::Log("<<="); + next(3); + break; + default: + Console::Log("<<"); + next(2); + break; + } + break; + } + case RightShift: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::RightShift, TokenKind::Equals): + Console::Log(">>="); + next(3); + break; + default: + Console::Log(">>"); + next(2); + break; + } + break; + } + case GreaterThan: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::GreaterThan, TokenKind::Equals): + Console::Log(">="); + next(2); + break; + group(TokenKind::GreaterThan, TokenKind::GreaterThan): + Kind = TokenKind::RightShift; + goto start; + default: + Console::Log(">"); + next(1); + break; + } + break; + } + case Equals: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::Equals) { + Console::Log("=="); + next(1); + } else { + Console::Log("="); + } + next(1); + break; + } + case QuestionMark: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::QuestionMark) { + Console::Log("??"); + next(2); + } else { + Console::Log("?"); + next(1); + } + break; + } + case Plus: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Plus, TokenKind::Plus): + Console::Log("++"); + next(2); + break; + group(TokenKind::Plus, TokenKind::Equals): + Console::Log("+="); + next(2); + break; + default: + Console::Log("+"); + next(1); + break; + } + break; + } + case Minus: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Minus, TokenKind::Minus): + Console::Log("--"); + next(2); + break; + group(TokenKind::Minus, TokenKind::Equals): + Console::Log("-="); + next(2); + break; + default: + Console::Log("-"); + next(1); + break; + } + break; + } + case Slash: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Slash, TokenKind::Equals): + Instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); + Instance.assignment = true; + break; + default: + Instance = TOKEN(const char *, "/", TokenKind::Slash, "/"); + break; + } + break; + } + case Star: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Star, TokenKind::Star): + Kind = TokenKind::Pow; + goto start; + group(TokenKind::Star, TokenKind::Equals): + Console::Log("*="); + break; + default: + Console::Log('*'); + break; + } + break; + } + case Pow: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::Pow, TokenKind::Equals): + Console::Log("**="); + next(3); + break; + default: + Console::Log("**"); + next(2); + break; + } + break; + } + case AND: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::AND, TokenKind::Equals): + Console::Log("/="); + next(2); + break; + default: + Console::Log('/'); + next(1); + break; + } + break; + } + case Pipe: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Pipe, TokenKind::Equals): + Console::Log("|="); + next(2); + break; + default: + Console::Log('|'); + next(1); + break; + } + break; + } + case XOR: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::XOR, TokenKind::Equals): + Console::Log("^="); + next(2); + break; + default: + Console::Log('^'); + next(1); + break; + } + break; + } + case NewLine: // \n or \r + Console::Log(curr()); + skip_new_line(); + Kind = TokenKindClass[curr()]; + goto start; // Does this so 2 function calls aren't wasted. + case Char: + Console::Log(Colors::GREEN, "'", next(1), Colors::RESET); + if (peek(1) == '\'') + Console::Log("'"); + next(1); + break; + case Whitespace: // Skip whitespace + Console::Log(curr()); + next(1); + Kind = TokenKindClass[curr()]; + goto start; + case Dot: { + auto s1 = TokenKindClass[peek(1)]; + auto s2 = TokenKindClass[peek(2)]; + + match(Kind, s1) + { + group(TokenKind::Dot, TokenKind::Dot): + s2 == TokenKind::Dot ? + next(3), Console::Log("...") : + (next(2), Console::Log("..")); + break; + default: + Console::Log("."); + next(1); + break; + } + break; + } + // Cannot be generated by the TokenKind table, assigned after checks + case EqualsEquals: + case Boolean: + UNREACHABLE; + break; + case LParenthesis: + case RParenthesis: + case LBracket: + case RBracket: + case LCurlyBrace: + case RCurlyBrace: + case NOT: + case Colon: + case SemiColon: + case ExclamationPoint: + case At: + case Comma: + Console::Log(curr()); + next(1); + break; + case Error: + buffer = nullptr; + break; + default: + break; + } + } +} + +namespace nextgen { + namespace jet { + template + class Lexer; + + template + class Lexer; + } } -namespace nextgen { namespace jet { - template class Lexer; - template class Lexer; -}} diff --git a/lib/jet-lexer-v2.cpp b/lib/jet-lexer-v2.cpp index 8257ea5..0b95405 100644 --- a/lib/jet-lexer-v2.cpp +++ b/lib/jet-lexer-v2.cpp @@ -15,300 +15,321 @@ if (OUTPUT_MODE) Console::Log(value), next((sizeof value) - 1); \ else (tokens.end = TOKEN_SHORT(value, kind), next((sizeof value) - 1)) \ \ -static const constexpr TokenKind MatchTokenKind[256] { +static const constexpr TokenKind MatchTokenKind[256]{ - // Error code - TokenKind::EOFToken, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, - TokenKind::Whitespace, // \a - TokenKind::Whitespace, // \b - TokenKind::Whitespace, // \t - TokenKind::NewLine, // \n - TokenKind::Whitespace, // \v - TokenKind::Error, + // Error code + TokenKind::EOFToken, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Whitespace, // \a + TokenKind::Whitespace, // \b + TokenKind::Whitespace, // \t + TokenKind::NewLine, // \n + TokenKind::Whitespace, // \v + TokenKind::Error, - TokenKind::NewLine, // \r + TokenKind::NewLine, // \r - TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - - TokenKind::Whitespace, // - - TokenKind::ExclamationPoint, // ! - TokenKind::String, // '"' - TokenKind::Error, TokenKind::Error, - TokenKind::Percent, // '%' - TokenKind::AND, // '&' - TokenKind::Char, // '\'' - TokenKind::LParenthesis, // ( - TokenKind::RParenthesis, // ) - TokenKind::Star, // '*' - TokenKind::Plus, // '+' - TokenKind::Comma, // ',' - - TokenKind::Minus, // '-' - - - TokenKind::Dot, // '.' - - TokenKind::Slash, // '/' - - - // Digits - TokenKind::Zero, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - - - TokenKind::Colon, // ':' - - - TokenKind::SemiColon, // ';' - TokenKind::LessThan, // '<' - TokenKind::Equals, // '=' - TokenKind::GreaterThan, // '>' - TokenKind::QuestionMark, // '?' - TokenKind::At, // @ - - // Uppercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LBracket, // '[' - - TokenKind::StringEscapeStart, // \\ - - TokenKind::RBracket, // ']' - - TokenKind::XOR, // '^' - - TokenKind::Identifier,// '_' - - TokenKind::Error, - - // Lowercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LCurlyBrace, // '{' - TokenKind::Pipe, // '|' - TokenKind::RCurlyBrace, // '}' - TokenKind::NOT, // '~' - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + + TokenKind::Whitespace, // + + TokenKind::ExclamationPoint, // ! + TokenKind::String, // '"' + TokenKind::Error, TokenKind::Error, + TokenKind::Percent, // '%' + TokenKind::AND, // '&' + TokenKind::Char, // '\'' + TokenKind::LParenthesis, // ( + TokenKind::RParenthesis, // ) + TokenKind::Star, // '*' + TokenKind::Plus, // '+' + TokenKind::Comma, // ',' + + TokenKind::Minus, // '-' + + + TokenKind::Dot, // '.' + + TokenKind::Slash, // '/' + + + // Digits + TokenKind::Zero, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + + + TokenKind::Colon, // ':' + + + TokenKind::SemiColon, // ';' + TokenKind::LessThan, // '<' + TokenKind::Equals, // '=' + TokenKind::GreaterThan, // '>' + TokenKind::QuestionMark, // '?' + TokenKind::At, // @ + + // Uppercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LBracket, // '[' + + TokenKind::StringEscapeStart, // \\ + + TokenKind::RBracket, // ']' + + TokenKind::XOR, // '^' + + TokenKind::Identifier,// '_' + + TokenKind::Error, + + // Lowercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LCurlyBrace, // '{' + TokenKind::Pipe, // '|' + TokenKind::RCurlyBrace, // '}' + TokenKind::NOT, // '~' + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, }; constexpr static int IntegerBits[256] = { - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, - // Digits - 0,1,2,3,4,5,6,7,8,9, + // Digits + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - // Error Code - -3,-3,-3,-3,-3,-3,-3, + // Error Code + -3, -3, -3, -3, -3, -3, -3, - // Uppercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35, + // Uppercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, - // Lowercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35, + // Lowercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, }; struct ReservedIdent { - const char *id; - TokenKind type; + const char *id; + TokenKind type; }; -static const struct ReservedIdent Reserved[] { - { "if", KeywordIf }, - { "else", KeywordElse }, - { "elif", KeywordElif }, - { "while", KeywordWhile }, - { "for", KeywordFor }, - { "true", KeywordTrue }, - { "false", KeywordFalse }, - { "None", KeywordNone }, - { "fn", KeywordFunction }, - { "struct", KeywordStruct }, - { "export", KeywordExport }, - { "enum", KeywordEnum }, - { "extern", KeywordExtern }, - { "error", KeywordError }, - { "var", KeywordVar }, - { "return", KeywordReturn }, - { "and", KeywordAnd }, - { "or", KeywordOr }, - { "break", KeywordBreak }, - { "continue", KeywordContinue }, - { "defer", KeywordDefer }, - { "union", KeywordUnion }, - { "match", KeywordMatch }, - { "in", KeywordIn }, - {"str", Typename_str }, - {"i8", Typename_i8 }, - {"i16", Typename_i16 }, - {"i32", Typename_i32 }, - {"i64", Typename_i64 }, - {"u8", Typename_u8 }, - {"u16", Typename_u16 }, - {"u32", Typename_u32 }, - {"u64", Typename_u64 }, - {"box", Typename_box }, - {"f32", Typename_f32 }, - {"f64", Typename_f64 }, +static const struct ReservedIdent Reserved[]{ + {"if", KeywordIf}, + {"else", KeywordElse}, + {"elif", KeywordElif}, + {"while", KeywordWhile}, + {"for", KeywordFor}, + {"true", KeywordTrue}, + {"false", KeywordFalse}, + {"None", KeywordNone}, + {"fn", KeywordFunction}, + {"struct", KeywordStruct}, + {"export", KeywordExport}, + {"enum", KeywordEnum}, + {"extern", KeywordExtern}, + {"error", KeywordError}, + {"var", KeywordVar}, + {"return", KeywordReturn}, + {"and", KeywordAnd}, + {"or", KeywordOr}, + {"break", KeywordBreak}, + {"continue", KeywordContinue}, + {"defer", KeywordDefer}, + {"union", KeywordUnion}, + {"match", KeywordMatch}, + {"in", KeywordIn}, + {"str", Typename_str}, + {"i8", Typename_i8}, + {"i16", Typename_i16}, + {"i32", Typename_i32}, + {"i64", Typename_i64}, + {"u8", Typename_u8}, + {"u16", Typename_u16}, + {"u32", Typename_u32}, + {"u64", Typename_u64}, + {"box", Typename_box}, + {"f32", Typename_f32}, + {"f64", Typename_f64}, }; static const TokenKind MatchIdentToReserved(const char *ident, size_t len) { - FOR(i, SizeOfArray(Reserved)) { - if (::strncmp(Reserved[i].id, ident, len) == 0) - return Reserved[i].type; - } - return Identifier; + FOR(i, SizeOfArray(Reserved)) { + if (::strncmp(Reserved[i].id, ident, len) == 0) + return Reserved[i].type; + } + return Identifier; } template void Lexer::lex_float(int skip, int start) { - // Initialize start of token - auto token_start = buffer-skip; - auto token_start_col = column; + // Initialize start of token + auto token_start = buffer - skip; + auto token_start_col = column; - next(skip-1); + next(skip - 1); - double decimal = 0; - double divisor = 1; - double value = 0; + double decimal = 0; + double divisor = 1; + double value = 0; - do { - auto digit = IntegerBits[curr()]; - if (digit < 0) break; - divisor *= 10; - value = value * 10 + digit ; - } while(next(1)); + do { + auto digit = IntegerBits[curr()]; + if (digit < 0) break; + divisor *= 10; + value = value * 10 + digit; + } while (next(1)); - decimal = (value / divisor); + decimal = (value / divisor); - auto token_end = buffer; + auto token_end = buffer; - auto literal_token_representation - = Range(token_start, token_end); + auto literal_token_representation + = Range(token_start, token_end); - if (OUTPUT_MODE) { - Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); - } + if (OUTPUT_MODE) { + Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); + } - tokens.end = new Token { - literal_token_representation, - {line, column}, - (double)(start + decimal), - TokenKind::Decimal, - TokenClassification::Literal - }; + tokens.end = new Token{ + literal_token_representation, + {line, column}, + (double) (start + decimal), + TokenKind::Decimal, + TokenClassification::Literal + }; } template void Lexer::lex_int() { - switch (curr()) { - case '0': - switch(peek(1)) { - case 'b': lex_int<2, 2>(); return; - case 'x': lex_int<16, 2>(); return; - case '#': lex_int<36, 2>(); return; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': lex_int<8, 1>(); return; - case '.': lex_float(2); return; - default: lex_int<10, 0>(); return; - } - default: lex_int<10, 0>(); - } + switch (curr()) { + case '0': + switch (peek(1)) { + case 'b': + lex_int < 2, 2 > (); + return; + case 'x': + lex_int < 16, 2 > (); + return; + case '#': + lex_int < 36, 2 > (); + return; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + lex_int < 8, 1 > (); + return; + case '.': + lex_float(2); + return; + default: + lex_int < 10, 0 > (); + return; + } + default: + lex_int < 10, 0 > (); + } } template @@ -316,494 +337,508 @@ template void Lexer::lex_int() { - // Initialize start of token - const char *token_start = buffer; - size_t token_start_col = column; - - // Skip any tokens, if any (for 0x, 0b, 00 cases) - next(skip); - - // Note: Keep the type! - decltype(UINTPTR_MAX) lexed_int = 0; - do { - auto digit = IntegerBits[curr()]; - if (digit < 0) break; - - // Invalid digit in number. For example, in base 10, - // we accept 1234 as a valid number but not 123f as 'f' is out of - // range for the accepted base. - if (digit >= radix) { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::DigitOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::Integer - }, - {radix, digit + 1} - }); - } - } - // Check whether value is about to overflow the maximum integer value. - // Ensures that all integers don't exceed 64-bit max. - if (lexed_int > (UINTPTR_MAX - digit) / radix) { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::IntegerOverflow, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::Integer - } - }); - } - } - lexed_int = lexed_int * radix + digit; - } while(next(1)); - - - if (curr() == '.') { // At this point, we've realized its f64, so we proceed - lex_float(buffer - token_start, lexed_int); - return; - } - - const char *token_end = buffer; - auto literal_token_representation - = Range(token_start, token_end); - - if (OUTPUT_MODE) { - Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); - } - - - tokens.end = new Token { - literal_token_representation, - {line, column}, - (decltype(UINTPTR_MAX)) lexed_int, - TokenKind::Integer, - TokenClassification::Literal - }; + // Initialize start of token + const char *token_start = buffer; + size_t token_start_col = column; + + // Skip any tokens, if any (for 0x, 0b, 00 cases) + next(skip); + + // Note: Keep the type! + decltype(UINTPTR_MAX) lexed_int = 0; + do { + auto digit = IntegerBits[curr()]; + if (digit < 0) break; + + // Invalid digit in number. For example, in base 10, + // we accept 1234 as a valid number but not 123f as 'f' is out of + // range for the accepted base. + if (digit >= radix) { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::DigitOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::Integer + }, + {radix, digit + 1} + }); + } + } + // Check whether value is about to overflow the maximum integer value. + // Ensures that all integers don't exceed 64-bit max. + if (lexed_int > (UINTPTR_MAX - digit) / radix) { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::IntegerOverflow, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::Integer + } + }); + } + } + lexed_int = lexed_int * radix + digit; + } while (next(1)); + + + if (curr() == '.') { // At this point, we've realized its f64, so we proceed + lex_float(buffer - token_start, lexed_int); + return; + } + + const char *token_end = buffer; + auto literal_token_representation + = Range(token_start, token_end); + + if (OUTPUT_MODE) { + Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); + } + + + tokens.end = new Token{ + literal_token_representation, + {line, column}, + (decltype(UINTPTR_MAX)) lexed_int, + TokenKind::Integer, + TokenClassification::Literal + }; } template void Lexer::lex_ident() { - const char *token_start = buffer; - auto token_start_col = column; - auto val = FNV_OFF; - do { - val ^= (curr()); - val *= FNV_PRIME; - } while(MatchTokenKind[next(1)] == TokenKind::Identifier); - const char *token_end = buffer; - - auto literal_token_representation = str(Range(token_start, - token_end)); - literal_token_representation.setHash(val); - TokenKind reserved = - MatchIdentToReserved(token_start, token_end - token_start); - - if (OUTPUT_MODE) { - if (reserved != TokenKind::Identifier) { - Console::Log(Colors::RED, literal_token_representation, - Colors::RESET); - } else { - Console::Log(Colors::YELLOW, literal_token_representation, Colors::RESET); - } - } - - tokens.end = new Token { - literal_token_representation, - {line, token_start_col}, - "", - reserved, - static_cast(reserved == Identifier ? Keyword : -1) - }; + const char *token_start = buffer; + auto token_start_col = column; + auto val = FNV_OFF; + do { + val ^= (curr()); + val *= FNV_PRIME; + } while (MatchTokenKind[next(1)] == TokenKind::Identifier); + const char *token_end = buffer; + + auto literal_token_representation = str(Range(token_start, + token_end)); + literal_token_representation.setHash(val); + TokenKind reserved = + MatchIdentToReserved(token_start, token_end - token_start); + + if (OUTPUT_MODE) { + if (reserved != TokenKind::Identifier) { + Console::Log(Colors::RED, literal_token_representation, + Colors::RESET); + } else { + Console::Log(Colors::YELLOW, literal_token_representation, Colors::RESET); + } + } + + tokens.end = new Token{ + literal_token_representation, + {line, token_start_col}, + "", + reserved, + static_cast(reserved == Identifier ? Keyword : -1) + }; } template void Lexer::lex_str() { - auto token_start = buffer; - auto token_start_col = column; - auto offset = 1; - - if (OUTPUT_MODE) Console::Log(Colors::GREEN); - - // @NOTE: Remember that this is DATA not OBJECT for allocation - auto lexed_string = (char*) (GLOBAL_DATA_ALLOC.current() + 1); - size_t len = 0; - - char current_char = next(1); - if (OUTPUT_MODE) Console::Log("\""); - do { - current_char = curr(); - if (OUTPUT_MODE) { - Console::Log(current_char); - if (current_char == '\0' || peek(1) == '\0') { - offset = 0; - break; - } - } else { - if (current_char == '\0') { - diagnostics.build(LexError { - LexErrorType::MissingClosingDelim, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {(int)('"' - )} - }); - } - - if (current_char == '\\') { - switch (next(1)) { - case 'a': current_char = '\a'; break; - case 'b': current_char = '\b'; break; - case 'r': current_char = '\r'; break; - case 'n': current_char = '\n'; break; - case 'v': current_char = '\v'; break; - case 't': current_char = '\t'; break; - case 'x': { - auto value = IntegerBits[next(1)]; - if (value >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {value + 1} // Pass in the invalid digit for the escape - }); - } - auto value2 = IntegerBits[next(1)]; - if (value2 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {value2 + 1} // Pass in the invalid digit for the escape - }); - } - int val = value; - val = val * 16 + value2; // @Opt: (val << 4) | value2 ? - - current_char = (char) val; - break; - } - default: - diagnostics.build(LexError { - LexErrorType::InvalidStringEscape, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - } - }); - } - } - lexed_string[len++] = current_char; - } - } while(next(1) != '"'); - - next(1); // Skip '"' - - // For missing closing expect_delim this needs to be here - if (OUTPUT_MODE && offset != 0) Console::Log("\""); - if (NORMAL_MODE) { - lexed_string[len] = '\0'; - - // We wrote to the data buffer without telling the size of the string - // because we did not know what it was, so we tell it the size of the - // string after lexing it. `reserve(size_t)` in this case is used to - // increment the number of bytes written. - GLOBAL_DATA_ALLOC.reserve(len); - tokens.end = new Token { - str {lexed_string, len}, - {line, column}, - "", - TokenKind::String, - TokenClassification::Literal - }; - } + auto token_start = buffer; + auto token_start_col = column; + auto offset = 1; + + if (OUTPUT_MODE) Console::Log(Colors::GREEN); + + // @NOTE: Remember that this is DATA not OBJECT for allocation + auto lexed_string = (char *) (GLOBAL_DATA_ALLOC.current() + 1); + size_t len = 0; + + char current_char = next(1); + if (OUTPUT_MODE) Console::Log("\""); + do { + current_char = curr(); + if (OUTPUT_MODE) { + Console::Log(current_char); + if (current_char == '\0' || peek(1) == '\0') { + offset = 0; + break; + } + } else { + if (current_char == '\0') { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {(int) ('"' + )} + }); + } + + if (current_char == '\\') { + switch (next(1)) { + case 'a': + current_char = '\a'; + break; + case 'b': + current_char = '\b'; + break; + case 'r': + current_char = '\r'; + break; + case 'n': + current_char = '\n'; + break; + case 'v': + current_char = '\v'; + break; + case 't': + current_char = '\t'; + break; + case 'x': { + auto value = IntegerBits[next(1)]; + if (value >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {value + 1} // Pass in the invalid digit for the escape + }); + } + auto value2 = IntegerBits[next(1)]; + if (value2 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {value2 + 1} // Pass in the invalid digit for the escape + }); + } + int val = value; + val = val * 16 + value2; // @Opt: (val << 4) | value2 ? + + current_char = (char) val; + break; + } + default: + diagnostics.build(LexError{ + LexErrorType::InvalidStringEscape, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + } + }); + } + } + lexed_string[len++] = current_char; + } + } while (next(1) != '"'); + + next(1); // Skip '"' + + // For missing closing expect_delim this needs to be here + if (OUTPUT_MODE && offset != 0) Console::Log("\""); + if (NORMAL_MODE) { + lexed_string[len] = '\0'; + + // We wrote to the data buffer without telling the size of the string + // because we did not know what it was, so we tell it the size of the + // string after lexing it. `reserve(size_t)` in this case is used to + // increment the number of bytes written. + GLOBAL_DATA_ALLOC.reserve(len); + tokens.end = new Token{ + str{lexed_string, len}, + {line, column}, + "", + TokenKind::String, + TokenClassification::Literal + }; + } } template ArenaVec Lexer::lex() { - tokens.begin = (Token *) GLOBAL_OBJECT_ALLOC.current(); - - TokenKind kind; - while(true) { - if (fatal) break; // Special case: Diagnostics may crash - - auto current_char = curr(); - kind = MatchTokenKind[current_char]; - if (OUTPUT_MODE) Console::Log(Colors::RESET); - - switch (kind) { - case EOFToken: { - tokens.end = new Token("a", {line, column}, TokenKind::EOFToken); - return tokens; - } - case Integer: lex_int(); break; - case String: lex_str(); break; - case Identifier: lex_ident(); break; - case NewLine: skip_new_line(); break; - case Whitespace: { - if (OUTPUT_MODE) - Console::Log(current_char); - next(1); - break; - } - case Char: { - auto token_start = buffer; - auto token_start_col = column; - if (OUTPUT_MODE) { - Console::Log(Colors::GREEN, "'"); - Console::Log(next(1), Colors::RESET); - } - else { - TOKEN_SHORT(next(1), TokenKind::Char); - } - if (curr() != '\'') { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::MissingClosingDelim, - {line, column}, // 'x[?] <-- Missing Closing! - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {(char)'\''} - }); - } - next(1); // Skip "'" - break; - } - else if (OUTPUT_MODE) Console::Log("'", Colors::RESET); - next(1); // Skip the "'" - break; - } - case LessThan: { // <, <=, <<, <<= - auto n = peek(1); - if (n == '<') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT("<<=", TokenKind::LeftShiftEquals); - break; - } - TOKEN_ADD_SHORT("<<", TokenKind::LeftShift); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT("<=",TokenKind::LessThanEquals); - break; - } - TOKEN_ADD_SHORT("<", TokenKind::LessThan); - break; - } - case GreaterThan: { - auto n = peek(1); - if (n == '>') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT(">>=", TokenKind::RightShiftEquals); - break; - } - TOKEN_ADD_SHORT(">>", TokenKind::RightShift); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT(">=",TokenKind::GreaterThanEquals); - break; - } - TOKEN_ADD_SHORT(">", TokenKind::GreaterThan); - break; - } - case Star: { - auto n = peek(1); - if (n == '*') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT("**=", TokenKind::PowEquals); - break; - } - TOKEN_ADD_SHORT("**", TokenKind::Pow); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT("*=",TokenKind::MulEquals); - break; - } - TOKEN_ADD_SHORT("*", TokenKind::Star); - break; - } - case Dot: { - auto n = peek(1); - if (n == '.') { - if (peek(2) == '.') { - TOKEN_ADD_SHORT("...", TokenKind::Ellipsis); - break; - } - TOKEN_ADD_SHORT("..", TokenKind::RangeSpan); - break; - } - TOKEN_ADD_SHORT(".", TokenKind::Dot); - break; - } - case Plus: { - auto n = peek(1); - if (n == '+') { TOKEN_ADD_SHORT("++", TokenKind::PlusPlus);} - else if (n == '=') { TOKEN_ADD_SHORT("+=", TokenKind::PlusEquals); } - else { - TOKEN_ADD_SHORT("+", TokenKind::Plus); - } - break; - } - case Minus: { - auto n = peek(1); - if (n == '-') { - TOKEN_ADD_SHORT("--", TokenKind::MinusMinus); - } - else if (n == '=') { - TOKEN_ADD_SHORT("-=", TokenKind::MinusMinus); - } - else if (n == '>') { - TOKEN_ADD_SHORT("->", TokenKind::Arrow); - } - else { - TOKEN_ADD_SHORT("+", TokenKind::Minus); - } - break; - } - case Slash: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("/=", TokenKind::DivEquals); - } - else if (n == '/') { - if (OUTPUT_MODE) Console::Log("/"); - while (MatchTokenKind[next(1)] != NewLine) { - if (OUTPUT_MODE) { - Console::Log(curr()); - } - } - skip_new_line(); - } - else { - TOKEN_ADD_SHORT("/", TokenKind::Slash); - } - break; - } - case Equals: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("==", TokenKind::EqualsEquals); - } else if (n == '>') { - TOKEN_ADD_SHORT("=>", TokenKind::FunctionArrow); - } else { - TOKEN_ADD_SHORT("=", TokenKind::Equals); - } - break; - } - case Percent: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("%=", TokenKind::PercentEquals); - } else { - TOKEN_ADD_SHORT("%", TokenKind::Percent); - } - break; - } - case AND: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("&=", TokenKind::AND); - } else { - TOKEN_ADD_SHORT("&", TokenKind::ANDEquals); - } - break; - } - case Pipe: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("|=", TokenKind::OREquals); - } else { - TOKEN_ADD_SHORT("|", TokenKind::Pipe); - } - break; - } - case XOR: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("^=", TokenKind::XOREquals); - } else { - TOKEN_ADD_SHORT("^", TokenKind::XOR); - } - break; - } - case LParenthesis: - TOKEN_ADD_SHORT("(", TokenKind::LParenthesis); - break; - case RParenthesis: - TOKEN_ADD_SHORT(")", TokenKind::RParenthesis); - break; - case LCurlyBrace: - TOKEN_ADD_SHORT("{", TokenKind::LCurlyBrace); - break; - case RCurlyBrace: - TOKEN_ADD_SHORT("}", TokenKind::RCurlyBrace); - break; - case LBracket: - TOKEN_ADD_SHORT("[", TokenKind::LBracket); - break; - case RBracket: - TOKEN_ADD_SHORT("]", TokenKind::RBracket); - break; - case SemiColon: - TOKEN_ADD_SHORT(";", TokenKind::SemiColon); - break; - case At: - TOKEN_ADD_SHORT("@", TokenKind::At); - break; - case NOT: - TOKEN_ADD_SHORT("~", TokenKind::NOT); - break; - case ExclamationPoint: - TOKEN_ADD_SHORT("!", TokenKind::ExclamationPoint); - break; - case QuestionMark: - TOKEN_ADD_SHORT("?", TokenKind::QuestionMark); - break; - case Colon: - if (peek(1) == '=') - TOKEN_ADD_SHORT(":=", TokenKind::ColonEquals); - else - TOKEN_ADD_SHORT(":", TokenKind::Colon); - break; - case Comma: - TOKEN_ADD_SHORT(",", TokenKind::Comma); - break; - default: - break; - } - } - return tokens; + tokens.begin = (Token *) GLOBAL_OBJECT_ALLOC.current(); + + TokenKind kind; + while (true) { + if (fatal) break; // Special case: Diagnostics may crash + + auto current_char = curr(); + kind = MatchTokenKind[current_char]; + if (OUTPUT_MODE) Console::Log(Colors::RESET); + + switch (kind) { + case EOFToken: { + tokens.end = new Token("a", {line, column}, TokenKind::EOFToken); + return tokens; + } + case Integer: + lex_int(); + break; + case String: + lex_str(); + break; + case Identifier: + lex_ident(); + break; + case NewLine: + skip_new_line(); + break; + case Whitespace: { + if (OUTPUT_MODE) + Console::Log(current_char); + next(1); + break; + } + case Char: { + auto token_start = buffer; + auto token_start_col = column; + if (OUTPUT_MODE) { + Console::Log(Colors::GREEN, "'"); + Console::Log(next(1), Colors::RESET); + } else { + TOKEN_SHORT(next(1), TokenKind::Char); + } + if (curr() != '\'') { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, // 'x[?] <-- Missing Closing! + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {(char) '\''} + }); + } + next(1); // Skip "'" + break; + } else if (OUTPUT_MODE) Console::Log("'", Colors::RESET); + next(1); // Skip the "'" + break; + } + case LessThan: { // <, <=, <<, <<= + auto n = peek(1); + if (n == '<') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT("<<=", TokenKind::LeftShiftEquals); + break; + } + TOKEN_ADD_SHORT("<<", TokenKind::LeftShift); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT("<=", TokenKind::LessThanEquals); + break; + } + TOKEN_ADD_SHORT("<", TokenKind::LessThan); + break; + } + case GreaterThan: { + auto n = peek(1); + if (n == '>') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT(">>=", TokenKind::RightShiftEquals); + break; + } + TOKEN_ADD_SHORT(">>", TokenKind::RightShift); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT(">=", TokenKind::GreaterThanEquals); + break; + } + TOKEN_ADD_SHORT(">", TokenKind::GreaterThan); + break; + } + case Star: { + auto n = peek(1); + if (n == '*') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT("**=", TokenKind::PowEquals); + break; + } + TOKEN_ADD_SHORT("**", TokenKind::Pow); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT("*=", TokenKind::MulEquals); + break; + } + TOKEN_ADD_SHORT("*", TokenKind::Star); + break; + } + case Dot: { + auto n = peek(1); + if (n == '.') { + if (peek(2) == '.') { + TOKEN_ADD_SHORT("...", TokenKind::Ellipsis); + break; + } + TOKEN_ADD_SHORT("..", TokenKind::RangeSpan); + break; + } + TOKEN_ADD_SHORT(".", TokenKind::Dot); + break; + } + case Plus: { + auto n = peek(1); + if (n == '+') { TOKEN_ADD_SHORT("++", TokenKind::PlusPlus); } + else if (n == '=') { TOKEN_ADD_SHORT("+=", TokenKind::PlusEquals); } + else { + TOKEN_ADD_SHORT("+", TokenKind::Plus); + } + break; + } + case Minus: { + auto n = peek(1); + if (n == '-') { + TOKEN_ADD_SHORT("--", TokenKind::MinusMinus); + } else if (n == '=') { + TOKEN_ADD_SHORT("-=", TokenKind::MinusMinus); + } else if (n == '>') { + TOKEN_ADD_SHORT("->", TokenKind::Arrow); + } else { + TOKEN_ADD_SHORT("+", TokenKind::Minus); + } + break; + } + case Slash: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("/=", TokenKind::DivEquals); + } else if (n == '/') { + if (OUTPUT_MODE) Console::Log("/"); + while (MatchTokenKind[next(1)] != NewLine) { + if (OUTPUT_MODE) { + Console::Log(curr()); + } + } + skip_new_line(); + } else { + TOKEN_ADD_SHORT("/", TokenKind::Slash); + } + break; + } + case Equals: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("==", TokenKind::EqualsEquals); + } else if (n == '>') { + TOKEN_ADD_SHORT("=>", TokenKind::FunctionArrow); + } else { + TOKEN_ADD_SHORT("=", TokenKind::Equals); + } + break; + } + case Percent: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("%=", TokenKind::PercentEquals); + } else { + TOKEN_ADD_SHORT("%", TokenKind::Percent); + } + break; + } + case AND: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("&=", TokenKind::AND); + } else { + TOKEN_ADD_SHORT("&", TokenKind::ANDEquals); + } + break; + } + case Pipe: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("|=", TokenKind::OREquals); + } else { + TOKEN_ADD_SHORT("|", TokenKind::Pipe); + } + break; + } + case XOR: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("^=", TokenKind::XOREquals); + } else { + TOKEN_ADD_SHORT("^", TokenKind::XOR); + } + break; + } + case LParenthesis: + TOKEN_ADD_SHORT("(", TokenKind::LParenthesis); + break; + case RParenthesis: + TOKEN_ADD_SHORT(")", TokenKind::RParenthesis); + break; + case LCurlyBrace: + TOKEN_ADD_SHORT("{", TokenKind::LCurlyBrace); + break; + case RCurlyBrace: + TOKEN_ADD_SHORT("}", TokenKind::RCurlyBrace); + break; + case LBracket: + TOKEN_ADD_SHORT("[", TokenKind::LBracket); + break; + case RBracket: + TOKEN_ADD_SHORT("]", TokenKind::RBracket); + break; + case SemiColon: + TOKEN_ADD_SHORT(";", TokenKind::SemiColon); + break; + case At: + TOKEN_ADD_SHORT("@", TokenKind::At); + break; + case NOT: + TOKEN_ADD_SHORT("~", TokenKind::NOT); + break; + case ExclamationPoint: + TOKEN_ADD_SHORT("!", TokenKind::ExclamationPoint); + break; + case QuestionMark: + TOKEN_ADD_SHORT("?", TokenKind::QuestionMark); + break; + case Colon: + if (peek(1) == '=') + TOKEN_ADD_SHORT(":=", TokenKind::ColonEquals); + else TOKEN_ADD_SHORT(":", TokenKind::Colon); + break; + case Comma: + TOKEN_ADD_SHORT(",", TokenKind::Comma); + break; + default: + break; + } + } + return tokens; } // Forces compilation of the two lexer classes // so that we can have definitions in a cpp file without // crowding the header file. -namespace nextgen { namespace jet { - template class Lexer; - template class Lexer; -}} \ No newline at end of file +namespace nextgen { + namespace jet { + template + class Lexer; + + template + class Lexer; + } +} \ No newline at end of file diff --git a/lib/jet-parser.cpp b/lib/jet-parser.cpp index 8f07a5b..9ba78c7 100644 --- a/lib/jet-parser.cpp +++ b/lib/jet-parser.cpp @@ -38,357 +38,355 @@ using namespace nextgen::jet; const ParserOutput Parser::parse() { - auto output = ParserOutput(); - - for (;;) { - - // Report up to 2 errors before exiting the program - if (fatal > 2) { - this->diagnostics.send_exception(); - } - - switch (curr()->getKind()) { - case TokenKind::EOFToken: - return output; - case TokenKind::Identifier: { - auto C1 = peek(1); - auto C2 = peek(2); - - if (C1->getKind() == TokenKind::FunctionArrow) { - switch (C2->getKind()) { - case TokenKind::LParenthesis: // Function - output.functions << parse_function(curr()); - break; - case TokenKind::KeywordStruct: // Struct - output.structures << parse_struct(curr()); - break; - case TokenKind::KeywordEnum: // Enum - break; - default: - UNREACHABLE; - break; - } - } - else if (C1->getKind() == TokenKind::ColonEquals) { - auto name = curr(); - skip(2); - output.global_variables << - (const SyntaxVariableAssignment*) - parse_variable_assignment(name); - } - else { - this->diagnostics.build(ParseError { - ParseErrorType::InvalidTokenAfterIdentInGlobalScope, - C1->getSourceLocation(), - { ParseError::Metadata { C1 }} - }); - this->diagnostics.send_exception(); - } - break; - } - default: // TODO: Figure out what to do here - break; - } - } - return output; + auto output = ParserOutput(); + + for (;;) { + + // Report up to 2 errors before exiting the program + if (fatal > 2) { + this->diagnostics.send_exception(); + } + + switch (curr()->getKind()) { + case TokenKind::EOFToken: + return output; + case TokenKind::Identifier: { + auto C1 = peek(1); + auto C2 = peek(2); + + if (C1->getKind() == TokenKind::FunctionArrow) { + switch (C2->getKind()) { + case TokenKind::LParenthesis: // Function + output.functions << parse_function(curr()); + break; + case TokenKind::KeywordStruct: // Struct + output.structures << parse_struct(curr()); + break; + case TokenKind::KeywordEnum: // Enum + break; + default: + UNREACHABLE; + break; + } + } else if (C1->getKind() == TokenKind::ColonEquals) { + auto name = curr(); + skip(2); + output.global_variables << + (const SyntaxVariableAssignment *) + parse_variable_assignment(name); + } else { + this->diagnostics.build(ParseError{ + ParseErrorType::InvalidTokenAfterIdentInGlobalScope, + C1->getSourceLocation(), + {ParseError::Metadata{C1}} + }); + this->diagnostics.send_exception(); + } + break; + } + default: // TODO: Figure out what to do here + break; + } + } + return output; } - const SyntaxNode * -Parser::parse_variable_assignment(const Token *name) { - - // Check 1: name: type = expr - // Check 2: name := expr - if (peek(1)->getKind() == TokenKind::Colon) { - auto E = new SyntaxVariableAssignment(name, Some(parse_type()), parse_expr()); - expect("Expected ';' after declaration"); - return E; - } else { - auto E = new SyntaxVariableAssignment(name, None, parse_expr()); - expect("Expected ';' after declaration"); - return E; - } +Parser::parse_variable_assignment(const Token *name) { + + // Check 1: name: type = expr + // Check 2: name := expr + if (peek(1)->getKind() == TokenKind::Colon) { + auto E = new SyntaxVariableAssignment(name, Some(parse_type()), parse_expr()); + expect("Expected ';' after declaration"); + return E; + } else { + auto E = new SyntaxVariableAssignment(name, None, parse_expr()); + expect("Expected ';' after declaration"); + return E; + } } /// Pratt Parsing Expression Technique (Precedence Matching) /// References: https://en.wikipedia.org/wiki/Operator-precedence_parser const SyntaxNode * -Parser::parse_expr(int previous_binding) { - SyntaxNode *lhs; - - auto first_token = curr(); - auto first_kind = first_token->getKind(); - - // ~ : Bitwise NOT - // + : Positive - // - : Negation - // & : allocator Reference - // * : pointer Dereference - // not : Logical Negation - auto unary_binding = Parser::UnaryOperatorBinding(first_kind); - if (unary_binding > previous_binding) { - const auto expr = parse_expr(unary_binding); - const auto op = SyntaxUnary::MatchOp(first_kind); - - const auto E = (SyntaxNode*)(new SyntaxUnary(op, first_token, expr)); - E->kind = SyntaxKind::Unary; - return expr; - } else { - lhs = (SyntaxNode*) match_expr(); - } - next(1); - do { - auto op = curr(); - auto op_kind = op->getKind(); - - const auto op_bindings = Parser::InfixOperatorBinding(op_kind); - if (/* LeftBindingPrecedence */ op_bindings[0] <= previous_binding || - op_bindings[0] == -1) { - break; - } - auto a = skip(1); - const auto rhs = parse_expr(/* RightBindingPrecedence */ - op_bindings[1]); - auto new_expr = new SyntaxBinary { - SyntaxBinary::MatchOp(op_kind), op, lhs, rhs - }; - lhs = new_expr; - lhs->kind = SyntaxKind::Binary; - } while (curr()->isValidExpressionType()); - return lhs; +Parser::parse_expr(int previous_binding) { + SyntaxNode *lhs; + + auto first_token = curr(); + auto first_kind = first_token->getKind(); + + // ~ : Bitwise NOT + // + : Positive + // - : Negation + // & : allocator Reference + // * : pointer Dereference + // not : Logical Negation + auto unary_binding = Parser::UnaryOperatorBinding(first_kind); + if (unary_binding > previous_binding) { + const auto expr = parse_expr(unary_binding); + const auto op = SyntaxUnary::MatchOp(first_kind); + + const auto E = (SyntaxNode *) (new SyntaxUnary(op, first_token, expr)); + E->kind = SyntaxKind::Unary; + return expr; + } else { + lhs = (SyntaxNode *) match_expr(); + } + next(1); + do { + auto op = curr(); + auto op_kind = op->getKind(); + + const auto op_bindings = Parser::InfixOperatorBinding(op_kind); + if (/* LeftBindingPrecedence */ op_bindings[0] <= previous_binding || + op_bindings[0] == -1) { + break; + } + auto a = skip(1); + const auto rhs = parse_expr(/* RightBindingPrecedence */ + op_bindings[1]); + auto new_expr = new SyntaxBinary{ + SyntaxBinary::MatchOp(op_kind), op, lhs, rhs + }; + lhs = new_expr; + lhs->kind = SyntaxKind::Binary; + } while (curr()->isValidExpressionType()); + return lhs; } const NG_INLINE SyntaxNode * Parser::match_expr() { - auto matched_token = curr(); - switch (matched_token->getKind()) { - case EOFToken: - this->diagnostics.build(ParseError { - ParseErrorType::UnexpectedEndOfFile, - matched_token->getSourceLocation() - }); - break; - case Identifier: { // Unresolved variable or Function Call - auto C1 = peek(1); - if (peek(1)->getKind() == TokenKind::LParenthesis) { - return parse_function_call(matched_token, C1); - } - auto E = (SyntaxNode*) new SyntaxLiteral(matched_token); - E->kind = SyntaxKind::LiteralValue; - return E; - } - case Integer: case String: case Char: case Decimal: case KeywordTrue: - case KeywordFalse: { - auto E = (SyntaxNode*) new SyntaxLiteral(matched_token); - E->kind = SyntaxKind::LiteralValue; - return E; - } - case LParenthesis: { - skip(1); - auto E = parse_expr(); - expect_delim(curr()->getSourceLocation()); - position--; - return E; - } - case LBracket: { // ex: [1, 2, 3] - auto current = next(1); - auto E = new SyntaxList; - - // Loop through with reaching the EOF or closing expect_delim - while (current->getKind() != TokenKind::RBracket - && current->getKind() != TokenKind::EOFToken) { - - // Skip separating expect_delim - if (current->getKind() == TokenKind::Comma) { - skip(1); - } - - E->values << parse_expr(); - current = curr(); - } - expect_delim(matched_token->getSourceLocation()); - position--; - auto node = (SyntaxNode*)E; - node->kind = SyntaxKind::List; - return node; - } - default: // TODO: Unexpected Expression Token - break; - } - return nullptr; + auto matched_token = curr(); + switch (matched_token->getKind()) { + case EOFToken: + this->diagnostics.build(ParseError{ + ParseErrorType::UnexpectedEndOfFile, + matched_token->getSourceLocation() + }); + break; + case Identifier: { // Unresolved variable or Function Call + auto C1 = peek(1); + if (peek(1)->getKind() == TokenKind::LParenthesis) { + return parse_function_call(matched_token, C1); + } + auto E = (SyntaxNode *) new SyntaxLiteral(matched_token); + E->kind = SyntaxKind::LiteralValue; + return E; + } + case Integer: + case String: + case Char: + case Decimal: + case KeywordTrue: + case KeywordFalse: { + auto E = (SyntaxNode *) new SyntaxLiteral(matched_token); + E->kind = SyntaxKind::LiteralValue; + return E; + } + case LParenthesis: { + skip(1); + auto E = parse_expr(); + expect_delim(curr()->getSourceLocation()); + position--; + return E; + } + case LBracket: { // ex: [1, 2, 3] + auto current = next(1); + auto E = new SyntaxList; + + // Loop through with reaching the EOF or closing expect_delim + while (current->getKind() != TokenKind::RBracket + && current->getKind() != TokenKind::EOFToken) { + + // Skip separating expect_delim + if (current->getKind() == TokenKind::Comma) { + skip(1); + } + + E->values << parse_expr(); + current = curr(); + } + expect_delim(matched_token->getSourceLocation()); + position--; + auto node = (SyntaxNode *) E; + node->kind = SyntaxKind::List; + return node; + } + default: // TODO: Unexpected Expression Token + break; + } + return nullptr; } -SyntaxType *Parser::parse_type() { - - // []i32 - // **i32 - // *****i32 - // (i32, i32) - - auto first_token = curr(); - switch(first_token->getKind()) { - case LBracket: { - expect("Sequence '[]' (representing array) must be" - " met with closing value ']'"); - SyntaxTypename ty_name; - auto ty_name_tok = curr(); - if (! ty_name_tok->isValidTypenameStart()) { - auto ty = parse_type(); - return new SyntaxType(SyntaxTypeAnnotation::ArrayType, None, ty); - } - else if (! ty_name_tok->isValidTypename()) { - // TODO: Add type error value here - } - else { - ty_name = SyntaxType::MatchTypename(ty_name_tok); - } - return new SyntaxType(SyntaxTypeAnnotation::ArrayType, ty_name, nullptr); - } - case LParenthesis: - // TODO: Add tuples - break; - case Star: { - SyntaxTypename ty_name; - auto ty_name_tok = next(1); - if (! ty_name_tok->isValidTypenameStart()) { - auto ty = parse_type(); - return new SyntaxType(SyntaxTypeAnnotation::Pointer, None, ty); - } - else if (! ty_name_tok->isValidTypename()) { - // TODO: Add type error value here - } - else { - ty_name = SyntaxType::MatchTypename(ty_name_tok); - } - return new SyntaxType(SyntaxTypeAnnotation::Pointer, ty_name, nullptr); - } - default: - UNREACHABLE; - } - return new SyntaxType(None, None, nullptr); +SyntaxType *Parser::parse_type() { + + // []i32 + // **i32 + // *****i32 + // (i32, i32) + + auto first_token = curr(); + switch (first_token->getKind()) { + case LBracket: { + expect("Sequence '[]' (representing array) must be" + " met with closing value ']'"); + SyntaxTypename ty_name; + auto ty_name_tok = curr(); + if (!ty_name_tok->isValidTypenameStart()) { + auto ty = parse_type(); + return new SyntaxType(SyntaxTypeAnnotation::ArrayType, None, ty); + } else if (!ty_name_tok->isValidTypename()) { + // TODO: Add type error value here + } else { + ty_name = SyntaxType::MatchTypename(ty_name_tok); + } + return new SyntaxType(SyntaxTypeAnnotation::ArrayType, ty_name, nullptr); + } + case LParenthesis: + // TODO: Add tuples + break; + case Star: { + SyntaxTypename ty_name; + auto ty_name_tok = next(1); + if (!ty_name_tok->isValidTypenameStart()) { + auto ty = parse_type(); + return new SyntaxType(SyntaxTypeAnnotation::Pointer, None, ty); + } else if (!ty_name_tok->isValidTypename()) { + // TODO: Add type error value here + } else { + ty_name = SyntaxType::MatchTypename(ty_name_tok); + } + return new SyntaxType(SyntaxTypeAnnotation::Pointer, ty_name, nullptr); + } + default: + UNREACHABLE; + } + return new SyntaxType(None, None, nullptr); } const SyntaxNode * -Parser::parse_for() { - auto loop_variable = skip(); - skip(); - auto list_or_range = skip(1); - switch (list_or_range->getKind()) { - case Identifier: { - auto body = parse_block(); - return new SyntaxForList(list_or_range, loop_variable, body); - } - case KeywordRange: { - expect(""); - RangeFunction func; - FOR(value, 3) { - if (curr()->getKind() == TokenKind::RParenthesis - || curr()->getKind() == TokenKind::EOFToken) - break; - func.set[value] = expect("Range function values " - "must " - "be " - "integers")->getValue(); - expect("Expected comma in between parameters."); - } - auto body = parse_block(); - return new SyntaxForRange(loop_variable, func, body); - } - default: - this->diagnostics.build(ParseError { - ParseErrorType::InvalidToken, - list_or_range->getSourceLocation(), - { ParseError::Metadata("Invalid token in for-loop context. Expected " - "a list variable or the range function.") } - }); - break; - } - return nullptr; +Parser::parse_for() { + auto loop_variable = skip(); + skip(); + auto list_or_range = skip(1); + switch (list_or_range->getKind()) { + case Identifier: { + auto body = parse_block(); + return new SyntaxForList(list_or_range, loop_variable, body); + } + case KeywordRange: { + expect(""); + RangeFunction func; + FOR(value, 3) { + if (curr()->getKind() == TokenKind::RParenthesis + || curr()->getKind() == TokenKind::EOFToken) + break; + func.set[value] = expect("Range function values " + "must " + "be " + "integers")->getValue(); + expect("Expected comma in between parameters."); + } + auto body = parse_block(); + return new SyntaxForRange(loop_variable, func, body); + } + default: + this->diagnostics.build(ParseError{ + ParseErrorType::InvalidToken, + list_or_range->getSourceLocation(), + {ParseError::Metadata("Invalid token in for-loop context. Expected " + "a list variable or the range function.")} + }); + break; + } + return nullptr; } // TODO: Finish match statement. Leave this for later as it will be difficult // to see the outlook of this right now. const NG_INLINE SyntaxNode * -Parser::parse_match() { - return nullptr; +Parser::parse_match() { + return nullptr; } + const NG_INLINE SyntaxNode * -Parser::parse_match_pair_value() { - return nullptr; +Parser::parse_match_pair_value() { + return nullptr; } nextgen::ArenaVec Parser::parse_function_param() { - skip(); - auto current = curr(); + skip(); + auto current = curr(); - SyntaxFunctionParameter *begin = (SyntaxFunctionParameter*)(GLOBAL_OBJECT_ALLOC.current()); - SyntaxFunctionParameter *end = nullptr; + SyntaxFunctionParameter *begin = (SyntaxFunctionParameter *) (GLOBAL_OBJECT_ALLOC.current()); + SyntaxFunctionParameter *end = nullptr; - while (current->getKind() != TokenKind::RParenthesis) { - auto param_name = skip(); - expect("Expected ':' before parameter type (this is not" - " go!)"); + while (current->getKind() != TokenKind::RParenthesis) { + auto param_name = skip(); + expect("Expected ':' before parameter type (this is not" + " go!)"); - end = new SyntaxFunctionParameter(param_name, parse_type()); - } - return mem::ArenaVec{begin, end}; + end = new SyntaxFunctionParameter(param_name, parse_type()); + } + return mem::ArenaVec{begin, end}; } const NG_INLINE SyntaxFunction * Parser::parse_function(const Token *function_name) { - return new SyntaxFunction(function_name, - parse_type(), - parse_block(), - parse_function_param()); + return new SyntaxFunction(function_name, + parse_type(), + parse_block(), + parse_function_param()); } const NG_INLINE SyntaxNode * Parser::parse_function_call(const Token *function_name, const Token *delim) { - auto E = new SyntaxFunctionCall(function_name); - while(curr()->getKind() != TokenKind::RParenthesis) { - - // Looks odd, right? What's happening here is that the arena will allocate - // nodes in its continuous memory block and the function will write to that - // memory, so we don't need to worry about pushing values to the arena. - // - // TODO: Refactor some parsing functions to be void because of the arena - // allocation. - E->parameters << parse_expr(); - } - expect_delim(delim->getSourceLocation()); - return E; + auto E = new SyntaxFunctionCall(function_name); + while (curr()->getKind() != TokenKind::RParenthesis) { + + // Looks odd, right? What's happening here is that the arena will allocate + // nodes in its continuous memory block and the function will write to that + // memory, so we don't need to worry about pushing values to the arena. + // + // TODO: Refactor some parsing functions to be void because of the arena + // allocation. + E->parameters << parse_expr(); + } + expect_delim(delim->getSourceLocation()); + return E; } -SyntaxBlock Parser::parse_block() { - expect("Expected '{' before beginning of " - "current"); - auto block = SyntaxBlock {}; - // We know for sure that we got right curly brace - while (curr()->getKind() != TokenKind::RCurlyBrace - && curr()->getKind() != TokenKind::EOFToken) { - block.statements << parse_stmt(); - } - expect_delim(curr()->getSourceLocation()); - next(1); - return block; +SyntaxBlock Parser::parse_block() { + expect("Expected '{' before beginning of " + "current"); + auto block = SyntaxBlock{}; + // We know for sure that we got right curly brace + while (curr()->getKind() != TokenKind::RCurlyBrace + && curr()->getKind() != TokenKind::EOFToken) { + block.statements << parse_stmt(); + } + expect_delim(curr()->getSourceLocation()); + next(1); + return block; } nextgen::ArenaVec Parser::parse_struct_data_members() { - auto members = mem::ArenaVec(); - members.begin = (SyntaxStructMember*) (GLOBAL_OBJECT_ALLOC.current()); - while (curr()->getKind() != TokenKind::RCurlyBrace) { - auto name = expect("Identifiers must be used for " - "struct name"); - expect("':' Must be followed after struct member name"); - auto type = parse_type(); - members.end = new SyntaxStructMember(name, type); - } - return members; + auto members = mem::ArenaVec(); + members.begin = (SyntaxStructMember *) (GLOBAL_OBJECT_ALLOC.current()); + while (curr()->getKind() != TokenKind::RCurlyBrace) { + auto name = expect("Identifiers must be used for " + "struct name"); + expect("':' Must be followed after struct member name"); + auto type = parse_type(); + members.end = new SyntaxStructMember(name, type); + } + return members; } \ No newline at end of file diff --git a/lib/jet-x64-assembler.cpp b/lib/jet-x64-assembler.cpp index bdea471..3811ca3 100644 --- a/lib/jet-x64-assembler.cpp +++ b/lib/jet-x64-assembler.cpp @@ -302,41 +302,42 @@ write(0x0F, float_prefix);\ write_address(addr, reg); \ } \ + // [disp32] MemoryAddress::MemoryAddress(uint disp32) -: has_disp32(true), mod(MOD_INDIRECT), disp(disp32) {} + : has_disp32(true), mod(MOD_INDIRECT), disp(disp32) {} // [base] MemoryAddress::MemoryAddress(const Register base) : base(base), mod(MOD_INDIRECT) { - has_sib = (base & DI) == SP; - if ((base & DI) == BP) { - // Encodes as [base+0x0] - has_disp8 = true; - mod = MOD_DISP8; - } + has_sib = (base & DI) == SP; + if ((base & DI) == BP) { + // Encodes as [base+0x0] + has_disp8 = true; + mod = MOD_DISP8; + } } // [base + disp8] MemoryAddress::MemoryAddress(const Register base, ubyte disp8) -: base(base), mod (MOD_DISP8), disp(disp8) { - has_sib = (base & DI) == SP; - has_disp8 = true; + : base(base), mod(MOD_DISP8), disp(disp8) { + has_sib = (base & DI) == SP; + has_disp8 = true; } // [base + disp32] MemoryAddress::MemoryAddress(const Register base, uint disp32) -: base(base), mod (MOD_DISP32), disp(disp32) { - has_sib = (base & DI) == SP; - has_disp32 = true; + : base(base), mod(MOD_DISP32), disp(disp32) { + has_sib = (base & DI) == SP; + has_disp32 = true; } // [base + (index * scale)] MemoryAddress::MemoryAddress(const Register base, const Register index, SibScale scale) -: base(base), index(index), scale(scale), has_sib(true), mod(MOD_INDIRECT) { - if ((base & DI) == BP) { - mod = MOD_DISP8; - has_disp8 = true; - } + : base(base), index(index), scale(scale), has_sib(true), mod(MOD_INDIRECT) { + if ((base & DI) == BP) { + mod = MOD_DISP8; + has_disp8 = true; + } } // [base + (index * scale) + disp8] @@ -344,53 +345,53 @@ MemoryAddress::MemoryAddress(const Register base, const Register index, SibScale scale, ubyte disp8) - : base(base), index(index), scale(scale), - mod(MOD_DISP8), has_disp8(true), disp(disp8), - has_sib(true) {} + : base(base), index(index), scale(scale), + mod(MOD_DISP8), has_disp8(true), disp(disp8), + has_sib(true) {} - // [base + (index * scale) + disp32] - MemoryAddress::MemoryAddress(const Register base, - const Register index, - SibScale scale, - uint disp32) - : base(base), index(index), scale(scale), mod(MOD_DISP32), - has_disp32 (true), disp(disp32), has_sib(true) {} +// [base + (index * scale) + disp32] +MemoryAddress::MemoryAddress(const Register base, + const Register index, + SibScale scale, + uint disp32) + : base(base), index(index), scale(scale), mod(MOD_DISP32), + has_disp32(true), disp(disp32), has_sib(true) {} - /// Encode ModRM (no values known) - static ubyte EncodeModRM(ubyte mod, const Register reg, const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); +/// Encode ModRM (no values known) +static ubyte EncodeModRM(ubyte mod, const Register reg, const Register rm) { + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Encode ModRM (mod value known) template static ubyte EncodeModRM(const Register reg, const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Encode ModRM (mod and reg value known) template static ubyte EncodeModRM(const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Check if a rex value is needed for a given register for /// argument sizes that are (16, 32, 64) bit. static bool rex_needed(const Register reg) { - switch (reg) { - case R8: - case R9: - case R10: - case R11: - case R12: - case R13: - case R14: - case R15: - return true; - default: - return false; - } + switch (reg) { + case R8: + case R9: + case R10: + case R11: + case R12: + case R13: + case R14: + case R15: + return true; + default: + return false; + } } /// This function is the same as the 'rex_needed', but the only difference @@ -398,140 +399,140 @@ static bool rex_needed(const Register reg) { /// a rex byte should be encoded. As per manual/reference, 8-bit registers /// ah, ch, dh, and bh require a rex byte. static bool rex_needed2(const Register reg) { - switch (reg) { - case R8: - case R9: - case R10: - case R11: - case R12: - case R13: - case R14: - case R15: - case SP: // AH - case BP: // CH - case SI: // DH - case DI: // BH - return true; - default: - return false; - } + switch (reg) { + case R8: + case R9: + case R10: + case R11: + case R12: + case R13: + case R14: + case R15: + case SP: // AH + case BP: // CH + case SI: // DH + case DI: // BH + return true; + default: + return false; + } } /// Allocate the assembler with given 'size' value. Assembler::Assembler(const size_t size) { - memory = (ubyte *) malloc(size); + memory = (ubyte *) malloc(size); } /// Allocate the virtual page with given 'size' or give potential /// executable permissions to allocated memory. Assembler::Assembler(const size_t size, bool executable) { - // TODO: Add mmap to the virtual memory allocation to allocate the memory - // as executable + // TODO: Add mmap to the virtual memory allocation to allocate the memory + // as executable #ifdef NG_OS_WINDOWS -memory = (ubyte*) VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, - executable ? - PAGE_EXECUTE_READWRITE : PAGE_READWRITE); + memory = (ubyte *) VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, + executable ? + PAGE_EXECUTE_READWRITE : PAGE_READWRITE); #endif } /// Spill the memory buffer containing machine code bytes written ubyte *Assembler::spill() const { - return memory; + return memory; } /// The number of bytes that have been written to the memory buffer ulong Assembler::bytes() const { - return used; + return used; } /// Write a byte to the memory buffer. void Assembler::write(ubyte value) { - memory[used] = value; - used += sizeof(value); + memory[used] = value; + used += sizeof(value); } /// Write a 16-bit value to the memory buffer. void Assembler::write_16(ushort value) { - write(ubyte(value)); - write(ubyte(value >> 8)); + write(ubyte(value)); + write(ubyte(value >> 8)); } /// Write a 32-bit value to the memory buffer. void Assembler::write_32(uint value) { - write(ubyte(value)); - write(ubyte(value >> 8)); - write(ubyte(value >> 16)); - write(ubyte(value >> 24)); + write(ubyte(value)); + write(ubyte(value >> 8)); + write(ubyte(value >> 16)); + write(ubyte(value >> 24)); } /// Write a 64-bit value to the memory buffer. void Assembler::write_64(ulong value) { - write(ubyte(value)); - write(ubyte(value >> 8)); - write(ubyte(value >> 16)); - write(ubyte(value >> 24)); - write(ubyte(value >> 32)); - write(ubyte(value >> 40)); - write(ubyte(value >> 48)); - write(ubyte(value >> 52)); + write(ubyte(value)); + write(ubyte(value >> 8)); + write(ubyte(value >> 16)); + write(ubyte(value >> 24)); + write(ubyte(value >> 32)); + write(ubyte(value >> 40)); + write(ubyte(value >> 48)); + write(ubyte(value >> 52)); } /// Write an immediate 8-bit value. void Assembler::write(Imm8 imm) { - write(ubyte(imm.value)); + write(ubyte(imm.value)); } /// Write an immediate 16-bit value. void Assembler::write(Imm16 imm) { - write_16(imm.value); + write_16(imm.value); } /// Write an immediate 32-bit value. void Assembler::write(Imm32 imm) { - write_32(imm.value); + write_32(imm.value); } /// Write an immediate 64-bit value. void Assembler::write(Imm64 imm) { - write_64(imm.value); + write_64(imm.value); } /// Write a memory address given the register destination /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html void Assembler::write_address(const MemoryAddress &addr, const Register dest) { - // Mod RM Byte Special Cases: - // 1. Address has SIB with no displacement R/M is 0b100 (SP) - // 2. Address has only displacement R/M is 0b101 (BP) - write((addr.mod << 6) | (dest & DI) << 3 | (addr.has_sib ? SP : (addr.base & DI))); + // Mod RM Byte Special Cases: + // 1. Address has SIB with no displacement R/M is 0b100 (SP) + // 2. Address has only displacement R/M is 0b101 (BP) + write((addr.mod << 6) | (dest & DI) << 3 | (addr.has_sib ? SP : (addr.base & DI))); - // Write SIB if it has one - if (addr.has_sib) - write((addr.scale << 6) | (addr.index & DI) << 3 | addr.base); + // Write SIB if it has one + if (addr.has_sib) + write((addr.scale << 6) | (addr.index & DI) << 3 | addr.base); - // Write Displacement if it has one - if (addr.has_disp32) write_32(addr.disp); - else if (addr.has_disp8) write(ubyte(addr.disp)); + // Write Displacement if it has one + if (addr.has_disp32) write_32(addr.disp); + else if (addr.has_disp8) write(ubyte(addr.disp)); } template void Assembler::write(T f, K s) { - write(f), write(s); + write(f), write(s); } template void Assembler::write(T f, K s, L t) { - write(f), write(s), write(t); + write(f), write(s), write(t); } template void Assembler::write(T f, K s, L t, Q fr) { - write(f), write(s), write(t), write(fr); + write(f), write(s), write(t), write(fr); } void Assembler::rex_optional_r1(const Register reg) { - if (rex_needed2(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + if (rex_needed2(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Write optional rex-values for argument sizes of 16-bits. @@ -539,381 +540,397 @@ void Assembler::rex_optional_r1(const Register reg) { /// If the register is R8-R15, it must have another rex byte /// that is written. void Assembler::rex_optional_r2(const Register reg) { - write(0x66); - if (rex_needed(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + write(0x66); + if (rex_needed(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Apply rex-byte to all operations that are not BYTE size. void Assembler::rex_optional_r3(const Register reg) { - if (rex_needed(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + if (rex_needed(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Writes rex-values for 64-bit argument operations. This applies for /// the long-mode of x86-- which is x64. It also integrates rex-prefix /// with special register R8-R15. void Assembler::rex_optional_r4(const Register reg) { - if (rex_needed(reg)) - write(REX_PREFIX| REX_W | (reg & 0b1000) >> 3); - else write(0x48); + if (rex_needed(reg)) + write(REX_PREFIX | REX_W | (reg & 0b1000) >> 3); + else write(0x48); } /// Two const Register -- BYTE SIZE void Assembler::rex_optional_rr1(const Register dest, const Register src) { - if (rex_needed2(src) || rex_needed2(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + if (rex_needed2(src) || rex_needed2(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- WORD SIZE void Assembler::rex_optional_rr2(const Register dest, const Register src) { - write(0x66); - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + write(0x66); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- DWORD SIZE void Assembler::rex_optional_rr3(const Register dest, const Register src) { - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- QWORD SIZE void Assembler::rex_optional_rr4(const Register dest, const Register src) { - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | REX_W | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); - else write(0x48); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | REX_W | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + else write(0x48); } /// Memory Operation Rex Byte -- BYTE SIZE void Assembler::rex_optional_rm1(const Register reg, const MemoryAddress &addr) { - if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- WORD SIZE void Assembler::rex_optional_rm2(const Register reg, const MemoryAddress &addr) { - write(0x66); - if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + write(0x66); + if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- DWORD SIZE void Assembler::rex_optional_rm3(const Register reg, const MemoryAddress &addr) { - if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- QWORD SIZE void Assembler::rex_optional_rm4(const Register reg, const MemoryAddress &addr) { - if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // W - | REX_W - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); - else write(0x48); + if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // W + | REX_W + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); + else write(0x48); } //====================== // Unary Instructions //====================== unary_instruction(inc, 0xFE, 0xFF, 0); + unary_instruction(dec, 0xFE, 0xFF, 1); + unary_instruction(not, 0xF6, 0xF7, 2); + unary_instruction(neg, 0xF6, 0xF7, 3); + unary_instruction(mul, 0xF6, 0xF7, 4); + unary_instruction(div, 0xF6, 0xF7, 6); + unary_instruction(idiv, 0xF6, 0xF7, 7); //====================== // Binary Instructions //====================== binary_instruction(add, 0x00, 0x01, 0x02, 0x03); + binary_instruction_imm(add, 0x04, 0x05, 0); binary_instruction(sub, 0x28, 0x29, 0x2A, 0x2B); + binary_instruction_imm(sub, 0x2C, 0x2D, 5); binary_instruction(and, 0x20, 0x21, 0x22, 0x23); + binary_instruction_imm(and, 0x24, 0x25, 4); binary_instruction(xor, 0x30, 0x31, 0x32, 0x33); + binary_instruction_imm(xor, 0x34, 0x35, 6); binary_instruction(cmp, 0x38, 0x39, 0x3A, 0x3B); + binary_instruction_imm(cmp, 0x3C, 0x3D, 7); binary_instruction(or, 0x08, 0x09, 0x0A, 0x0B); + binary_instruction_imm(or, 0x0C, 0x0D, 1); floating_point_binary(add, 0x58); + floating_point_binary(sub, 0x5C); + floating_point_binary(div, 0x5E); + floating_point_binary(mul, 0x59); void Assembler::popq(const Register reg) { - write(0x58+reg); + write(0x58 + reg); } void Assembler::popw(const Register reg) { - write(0x66, 0x58+reg); + write(0x66, 0x58 + reg); } void Assembler::pushq(const Register reg) { - rex_optional_r4(reg); - write(0x50 + (reg & DI)); + rex_optional_r4(reg); + write(0x50 + (reg & DI)); } void Assembler::pushw(const Register reg) { - rex_optional_r2(reg); - write(0x50 + (reg & DI)); + rex_optional_r2(reg); + write(0x50 + (reg & DI)); } void Assembler::movb(const Register reg, Imm8 imm) { - rex_optional_r1(reg); - write(0xB0 + (reg & DI), imm); + rex_optional_r1(reg); + write(0xB0 + (reg & DI), imm); } template void Assembler::movw(const Register reg, const Imm imm) { - rex_optional_r2(reg); - write(0xB8 + (reg & DI), imm); + rex_optional_r2(reg); + write(0xB8 + (reg & DI), imm); } template void Assembler::movd(const Register reg, const Imm imm) { - rex_optional_r3(reg); - write(0xB8 + (reg & DI), imm); + rex_optional_r3(reg); + write(0xB8 + (reg & DI), imm); } template void Assembler::movq(const Register reg, const Imm imm) { - rex_optional_r4(reg); - write(0xC7, EncodeModRM<0b11, 0>(reg), imm); + rex_optional_r4(reg); + write(0xC7, EncodeModRM<0b11, 0>(reg), imm); } void Assembler::movb(const Register reg, const MemoryAddress &addr) { - rex_optional_rm1(reg, addr); - write(0x8A); - write_address(addr, reg); + rex_optional_rm1(reg, addr); + write(0x8A); + write_address(addr, reg); } void Assembler::movw(const Register reg, const MemoryAddress &addr) { - rex_optional_rm2(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm2(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movd(const Register reg, const MemoryAddress &addr) { - rex_optional_rm3(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm3(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movq(const Register reg, const MemoryAddress &addr) { - rex_optional_rm4(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm4(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movb(const Register dest, const Register src) { - rex_optional_rr1(dest, src); - write(0x88, EncodeModRM<0b11>(src, dest)); + rex_optional_rr1(dest, src); + write(0x88, EncodeModRM<0b11>(src, dest)); } void Assembler::movw(const Register dest, const Register src) { - rex_optional_rr2(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr2(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::movd(const Register dest, const Register src) { - rex_optional_rr3(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr3(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::movq(const Register dest, const Register src) { - rex_optional_rr4(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr4(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::shlb(const Register dest, const Imm8 count) { - rex_optional_r1(dest); - write(0xC0, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r1(dest); + write(0xC0, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlw(const Register dest, const Imm8 count) { - rex_optional_r2(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r2(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shld(const Register dest, const Imm8 count) { - rex_optional_r3(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r3(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlq(const Register dest, const Imm8 count) { - rex_optional_r4(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r4(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlb(const Register dest) { - rex_optional_r1(dest); - write(0xD0, EncodeModRM<0b11, SP>(dest)); + rex_optional_r1(dest); + write(0xD0, EncodeModRM<0b11, SP>(dest)); } void Assembler::shlw(const Register dest) { - rex_optional_r2(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest)); + rex_optional_r2(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest)); } void Assembler::shld(const Register dest) { - rex_optional_r3(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest), 1); + rex_optional_r3(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest), 1); } void Assembler::shlq(const Register dest) { - rex_optional_r4(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest)); + rex_optional_r4(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest)); } void Assembler::shrb(const Register dest, Imm8 count) { - rex_optional_r1(dest); - write(0xC0, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r1(dest); + write(0xC0, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrw(const Register dest, Imm8 count) { - rex_optional_r2(dest); - write(0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r2(dest); + write(0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrd(const Register dest, Imm8 count) { - rex_optional_r3(dest); - write(0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r3(dest); + write(0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrq(const Register dest, Imm8 count) { - rex_optional_r4(dest); - write(0x48, 0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r4(dest); + write(0x48, 0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::jmp(Imm8 rel8) { - write(0xEB, rel8); + write(0xEB, rel8); } void Assembler::jmp(Imm16 rel16) { - write(0xE9, rel16); + write(0xE9, rel16); } void Assembler::jmp(Imm32 rel32) { - write(0xE9, rel32); + write(0xE9, rel32); } void Assembler::jmpw(const Register reg) { - rex_optional_r2(reg); - write(0xFF, EncodeModRM<0b11, SP>(reg)); + rex_optional_r2(reg); + write(0xFF, EncodeModRM<0b11, SP>(reg)); } void Assembler::jmpq(const Register reg) { - rex_optional_r4(reg); - write(0xFF, EncodeModRM<0b11, SP>(reg)); + rex_optional_r4(reg); + write(0xFF, EncodeModRM<0b11, SP>(reg)); } // NOTE: imul instruction order for rex is (src, dest), yes, this is // correct. void Assembler::imulw(const Register dest, const Register src) { - rex_optional_rr2(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr2(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::imuld(const Register dest, const Register src) { - rex_optional_rr3(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr3(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::imulq(const Register dest, const Register src) { - rex_optional_rr4(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr4(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::jump_cond(Condition cond, Imm8 rel8) { - write(0x70 | cond, rel8); + write(0x70 | cond, rel8); } void Assembler::jump_cond(Condition cond, Imm16 rel16) { - write(0x80 | cond, rel16); + write(0x80 | cond, rel16); } void Assembler::jump_cond(Condition cond, Imm32 rel32) { - write(0x80 | cond, rel32); + write(0x80 | cond, rel32); } void Assembler::lea(const Register reg, const MemoryAddress &addr, const OperandSize size) { - switch (size) { - case BYTE: break; - case WORD: - rex_optional_rm2(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - case DWORD: - rex_optional_rm3(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - case QWORD: - rex_optional_rm4(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - } + switch (size) { + case BYTE: + break; + case WORD: + rex_optional_rm2(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + case DWORD: + rex_optional_rm3(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + case QWORD: + rex_optional_rm4(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + } } void Assembler::movss(const Register dest, const Register src) { - rex_optional_rr3(dest, src); - write(0xF3, 0x0F, 0x10, EncodeModRM<0b11>(dest, src)); + rex_optional_rr3(dest, src); + write(0xF3, 0x0F, 0x10, EncodeModRM<0b11>(dest, src)); } void Assembler::movss(const Register dest, const MemoryAddress &addr) { - rex_optional_rm3(dest, addr); - write(0xF3, 0x0F, 0x10); - write_address(addr, dest); + rex_optional_rm3(dest, addr); + write(0xF3, 0x0F, 0x10); + write_address(addr, dest); } void Assembler::movss(const MemoryAddress &dest, Register src) { - rex_optional_rm3(src, dest); - write(0xF3, 0x0F, 0x11); - write_address(dest, src); + rex_optional_rm3(src, dest); + write(0xF3, 0x0F, 0x11); + write_address(dest, src); } void Assembler::ret() { - write(0xC3); + write(0xC3); } \ No newline at end of file diff --git a/lib/str.cpp b/lib/str.cpp index 9f58ae3..a3079fd 100644 --- a/lib/str.cpp +++ b/lib/str.cpp @@ -4,16 +4,16 @@ using nextgen::str; using nextgen::StringCache; // Inline in string intern on constant strings -str nextgen::operator""_intern(const char *s, size_t len) { - str ss = str { s, len }; - ss.setHash(); - return StringCache::InsertOrGet(ss); +str nextgen::operator ""_intern(const char *s, size_t len) { + str ss = str{s, len}; + ss.setHash(); + return StringCache::InsertOrGet(ss); } // We write using the length because `str` may contain shadowed data // to Range. std::ostream &nextgen::operator<<(std::ostream &s, str &str) { - s.write(str.begin(), str.size()); - return s; + s.write(str.begin(), str.size()); + return s; } diff --git a/main.cpp b/main.cpp index 376e806..1948a3c 100644 --- a/main.cpp +++ b/main.cpp @@ -1,18 +1,14 @@ #include #include - using namespace nextgen::mem; using namespace nextgen::jet; using namespace nextgen; - - int main(int argc, char **argv, char **envP) { + try { - try { - - } catch(std::exception&) { - mem::deinit(); - } + } catch (std::exception &) { + mem::deinit(); + } } \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a371dd2..ac3f90e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.0) project(tests) - # Setup Google Test Compilation Cases add_executable(tests @@ -31,7 +30,7 @@ add_executable(tests lex-test.cpp parse-test.cpp x64-test.cpp -) + ) #set(gtest_force_shared_crt ON CACHE BOOL "MSVC defaulting to CRT" FORCE) diff --git a/tests/lex-test.cpp b/tests/lex-test.cpp index 3d9af84..d7dc64f 100644 --- a/tests/lex-test.cpp +++ b/tests/lex-test.cpp @@ -1,24 +1,24 @@ #include #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif TEST(LexTest, NonFMTNumbers) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer ( "1211", "src/test.jet", 4); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("1211", "src/test.jet", 4); + auto token = lexer.lex()[0]; // ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); @@ -27,318 +27,319 @@ TEST(LexTest, NonFMTNumbers) { } TEST(LexTest, DecimalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer ( "23.246", "src/test.jet", 6); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("23.246", "src/test.jet", 6); + auto token = lexer.lex()[0]; - ASSERT_EQ(token->template getValue(), 23.246); - ASSERT_EQ(token->getKind(), jet::TokenKind::Decimal); + ASSERT_EQ(token->template getValue(), 23.246); + ASSERT_EQ(token->getKind(), jet::TokenKind::Decimal); } TEST(LexTest, HexadecimalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0x1111", "src/test.jet", 6); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("0x1111", "src/test.jet", 6); + auto token = lexer.lex()[0]; // ASSERT_EQ(token->getValue(), 0x1111); // ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, BinaryNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0b011101110", "src/test.jet", 11); - auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 0b011101110); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + auto lexer = jet::Lexer("0b011101110", "src/test.jet", 11); + auto token = lexer.lex()[0]; + ASSERT_EQ(token->getValue(), 0b011101110); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, OctalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0112022", "src/test.jet", 7); - auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 0112022); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + auto lexer = jet::Lexer("0112022", "src/test.jet", 7); + auto token = lexer.lex()[0]; + ASSERT_EQ(token->getValue(), 0112022); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, Base36Number) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0#helloworld", "src/test.jet", 12); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("0#helloworld", "src/test.jet", 12); + auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 1767707668033969); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + ASSERT_EQ(token->getValue(), 1767707668033969); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, Identifier) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "ident", "src/test.jet", 5); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("ident", "src/test.jet", 5); + auto token = lexer.lex()[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "ident", 5) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); + ASSERT_TRUE(::strncmp(token->name().begin(), "ident", 5) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); } TEST(LexTest, Keyword) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "while extern", "src/test.jet", - 12); + auto lexer = jet::Lexer("while extern", "src/test.jet", + 12); - auto tokens = lexer.lex(); - auto token = tokens[0]; + auto tokens = lexer.lex(); + auto token = tokens[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "while", 5) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordWhile); + ASSERT_TRUE(::strncmp(token->name().begin(), "while", 5) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordWhile); - token = tokens[1]; - ASSERT_TRUE(::strncmp(token->name().begin(), "extern", 6) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordExtern); + token = tokens[1]; + ASSERT_TRUE(::strncmp(token->name().begin(), "extern", 6) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordExtern); } TEST(LexTest, String) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( R"("Hello" ace "There")", - "src/test.jet", 19); + auto lexer = jet::Lexer(R"("Hello" ace "There")", + "src/test.jet", 19); - auto tokens = lexer.lex(); - auto token = tokens[0]; + auto tokens = lexer.lex(); + auto token = tokens[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "Hello", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::String); + ASSERT_TRUE(::strncmp(token->name().begin(), "Hello", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::String); - token = tokens[1]; + token = tokens[1]; - ASSERT_TRUE(::strncmp(token->name().begin(), "ace", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); + ASSERT_TRUE(::strncmp(token->name().begin(), "ace", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); - token = tokens[2]; + token = tokens[2]; - ASSERT_TRUE(::strncmp(token->name().begin(), "There", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::String); + ASSERT_TRUE(::strncmp(token->name().begin(), "There", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::String); } TEST(LexTest, StringEscape) { // TODO: Add Unicode Escape Later - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(var every_escape = "I have\x56 every \t thing\n that could \b\v in \r")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - ArenaVec tokens{nullptr, nullptr}; - try { - tokens = lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } - auto token = tokens[3]; - - auto compare = "I have\x56 every \t thing\n that could \b\v in \r"; - ASSERT_TRUE(::strncmp(token->name().begin(), compare, token->len()) == 0); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(var every_escape = "I have\x56 every \t thing\n that could \b\v in \r")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + ArenaVec tokens{nullptr, nullptr}; + try { + tokens = lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } + auto token = tokens[3]; + + auto compare = "I have\x56 every \t thing\n that could \b\v in \r"; + ASSERT_TRUE(::strncmp(token->name().begin(), compare, token->len()) == 0); } TEST(LexTest, AllTokens) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - - auto buf = "+ += ++ - -= -- * *= ** **= / /= > >= >> >>= < <= << <<= % %=" - "() {} " - "[] ~ ^ ^= " - "| |= & &= = == : ; @ ! ? . .. ..."; // 43 tokens - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - - - ArenaVec tokens{nullptr, nullptr}; - try { - tokens = lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } - ASSERT_EQ(tokens.len(), 45); - ///* ASSERT_EQ(tokens[0].getKind(), jet::TokenKind::Plus); - // ASSERT_EQ(tokens[1].getKind(), jet::TokenKind::PlusEquals); - // ASSERT_EQ(tokens[2].getKind(), jet::TokenKind::PlusPlus); - // ASSERT_EQ(tokens[3].getKind(), jet::TokenKind::Minus); - // ASSERT_EQ(tokens[4].getKind(), jet::TokenKind::MinusEquals); - // ASSERT_EQ(tokens[5].getKind(), jet::TokenKind::MinusMinus); - // ASSERT_EQ(tokens[6].getKind(), jet::TokenKind::Star); - // ASSERT_EQ(tokens[7].getKind(), jet::TokenKind::MulEquals); - // ASSERT_EQ(tokens[8].getKind(), jet::TokenKind::Pow); - // ASSERT_EQ(tokens[9].getKind(), jet::TokenKind::Slash);*//* + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + + auto buf = "+ += ++ - -= -- * *= ** **= / /= > >= >> >>= < <= << <<= % %=" + "() {} " + "[] ~ ^ ^= " + "| |= & &= = == : ; @ ! ? . .. ..."; // 43 tokens + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + + + ArenaVec tokens{nullptr, nullptr}; + try { + tokens = lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } + ASSERT_EQ(tokens.len(), 45); + ///* ASSERT_EQ(tokens[0].getKind(), jet::TokenKind::Plus); + // ASSERT_EQ(tokens[1].getKind(), jet::TokenKind::PlusEquals); + // ASSERT_EQ(tokens[2].getKind(), jet::TokenKind::PlusPlus); + // ASSERT_EQ(tokens[3].getKind(), jet::TokenKind::Minus); + // ASSERT_EQ(tokens[4].getKind(), jet::TokenKind::MinusEquals); + // ASSERT_EQ(tokens[5].getKind(), jet::TokenKind::MinusMinus); + // ASSERT_EQ(tokens[6].getKind(), jet::TokenKind::Star); + // ASSERT_EQ(tokens[7].getKind(), jet::TokenKind::MulEquals); + // ASSERT_EQ(tokens[8].getKind(), jet::TokenKind::Pow); + // ASSERT_EQ(tokens[9].getKind(), jet::TokenKind::Slash);*//* } TEST(LexTest, ErrorOverflow) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "test := 0xffffffffffffffffffaaaaaaa test"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = "test := 0xffffffffffffffffffaaaaaaa test"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } + TEST(LexTest, DigitOutOfRange) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "invalid := 0xfffzz"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); + auto buf = "invalid := 0xfffzz"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); - try { - lexer.lex(); - } catch(std::exception&) { - return; - } + try { + lexer.lex(); + } catch (std::exception &) { + return; + } } TEST(LexTest, HexEscapeOutOfRange) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = R"(failed_string := "Hello\x2z")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = R"(failed_string := "Hello\x2z")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, InvalidStringEscape) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = R"(export fail_again := "Hello Test\q")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = R"(export fail_again := "Hello Test\q")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, MissingClosingDelimString) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(export fail_again := "Hello Test\n)"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(export fail_again := "Hello Test\n)"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, MissingClosingDelimChar) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(test := 'a)"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(test := 'a)"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST_SUITE_MAIN(LexTest) { - TEST_CALL(LexTest, NonFMTNumbers); - TEST_CALL(LexTest, DecimalNumber); - TEST_CALL(LexTest, HexadecimalNumber); - TEST_CALL(LexTest, BinaryNumber); - TEST_CALL(LexTest, OctalNumber); - TEST_CALL(LexTest, Base36Number); - TEST_CALL(LexTest, Identifier); - TEST_CALL(LexTest, Keyword); - TEST_CALL(LexTest, String); - TEST_CALL(LexTest, StringEscape); - TEST_CALL(LexTest, AllTokens); - TEST_CALL(LexTest, ErrorOverflow); - TEST_CALL(LexTest, DigitOutOfRange); - TEST_CALL(LexTest, HexEscapeOutOfRange); - TEST_CALL(LexTest, InvalidStringEscape); - TEST_CALL(LexTest, MissingClosingDelimString); - TEST_CALL(LexTest, MissingClosingDelimChar); + TEST_CALL(LexTest, NonFMTNumbers); + TEST_CALL(LexTest, DecimalNumber); + TEST_CALL(LexTest, HexadecimalNumber); + TEST_CALL(LexTest, BinaryNumber); + TEST_CALL(LexTest, OctalNumber); + TEST_CALL(LexTest, Base36Number); + TEST_CALL(LexTest, Identifier); + TEST_CALL(LexTest, Keyword); + TEST_CALL(LexTest, String); + TEST_CALL(LexTest, StringEscape); + TEST_CALL(LexTest, AllTokens); + TEST_CALL(LexTest, ErrorOverflow); + TEST_CALL(LexTest, DigitOutOfRange); + TEST_CALL(LexTest, HexEscapeOutOfRange); + TEST_CALL(LexTest, InvalidStringEscape); + TEST_CALL(LexTest, MissingClosingDelimString); + TEST_CALL(LexTest, MissingClosingDelimChar); } \ No newline at end of file diff --git a/tests/parse-test.cpp b/tests/parse-test.cpp index 55c9b9d..9a8c53b 100644 --- a/tests/parse-test.cpp +++ b/tests/parse-test.cpp @@ -1,314 +1,321 @@ #include #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif TEST(ParseTest, VariableDeclLiteral) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "hello := 23;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = static_cast(expr); - ASSERT_EQ(expr->kind, SyntaxKind::VariableAssignment); - // //ASSERT_EQ(decl->name->name(), "hello"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "hello", decl->name->len()) == 0); - ASSERT_EQ(decl->expression->kind, jet::SyntaxKind::LiteralValue); - ASSERT_EQ(CAST(const SyntaxLiteral*, decl->expression) - ->literal->getValue - (), 23); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "hello := 23;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = static_cast(expr); + ASSERT_EQ(expr->kind, SyntaxKind::VariableAssignment); + // //ASSERT_EQ(decl->name->name(), "hello"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "hello", decl->name->len()) == 0); + ASSERT_EQ(decl->expression->kind, jet::SyntaxKind::LiteralValue); + ASSERT_EQ(CAST(const SyntaxLiteral*, decl->expression) + ->literal->getValue + (), 23); } TEST(ParseTest, VariableArrayDecl) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = "my_list := [11, 2, 55];"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto node = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, node); - auto list = CAST(SyntaxList*, decl->expression); - - - ASSERT_EQ(decl->kind, SyntaxKind::VariableAssignment); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[0]) - ->literal->getValue - (), 11); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[1]) - ->literal->getValue(), 2); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[2]) - ->literal->getValue - (), 55); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = "my_list := [11, 2, 55];"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto node = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, node); + auto list = CAST(SyntaxList*, decl->expression); + + + ASSERT_EQ(decl->kind, SyntaxKind::VariableAssignment); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[0]) + ->literal->getValue + (), 11); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[1]) + ->literal->getValue(), 2); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[2]) + ->literal->getValue + (), 55); } TEST(ParseTest, VariableDeclBinary) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "wow := 2 + 3 * 2;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); + auto buf = "wow := 2 + 3 * 2;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, expr); + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, expr); - auto binary = CAST(SyntaxBinary*, decl->expression); + auto binary = CAST(SyntaxBinary*, decl->expression); - auto lhs = CAST(SyntaxLiteral*, binary->lhs); - auto op = binary->operation; - auto rhs = CAST(SyntaxBinary*, binary->rhs); + auto lhs = CAST(SyntaxLiteral*, binary->lhs); + auto op = binary->operation; + auto rhs = CAST(SyntaxBinary*, binary->rhs); - auto rhs_lhs = CAST(SyntaxLiteral*, rhs->lhs); - auto rhs_rhs = CAST(SyntaxLiteral*, rhs->rhs); + auto rhs_lhs = CAST(SyntaxLiteral*, rhs->lhs); + auto rhs_rhs = CAST(SyntaxLiteral*, rhs->rhs); - ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); - //ASSERT_EQ(decl->name->name(), "wow"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "wow", decl->name->len()) == 0); - ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); + ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); + //ASSERT_EQ(decl->name->name(), "wow"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "wow", decl->name->len()) == 0); + ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); - ASSERT_EQ(op, SyntaxBinaryOp::Addition); - ASSERT_EQ(lhs->literal->getValue(),2); - ASSERT_EQ(rhs_lhs->literal->getValue(), 3); - ASSERT_EQ(rhs_rhs->literal->getValue(), 2); + ASSERT_EQ(op, SyntaxBinaryOp::Addition); + ASSERT_EQ(lhs->literal->getValue(), 2); + ASSERT_EQ(rhs_lhs->literal->getValue(), 3); + ASSERT_EQ(rhs_rhs->literal->getValue(), 2); } TEST(ParseTest, VariableDeclParenBinary) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "awesome := (2 + 3) * 2;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, expr); - - auto binary = CAST(SyntaxBinary*, decl->expression); - - auto lhs = CAST(SyntaxBinary*, binary->lhs); - auto lhs_lhs = CAST(SyntaxLiteral*, lhs->lhs); - auto lhs_rhs = CAST(SyntaxLiteral*, lhs->rhs); - - auto op = binary->operation; - auto rhs = CAST(SyntaxLiteral*, binary->rhs); - - ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); - //ASSERT_EQ(decl->name->name(), "wow"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "awesome", decl->name->len()) == 0); - ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); - - ASSERT_EQ(op, SyntaxBinaryOp::Multiplication); - ASSERT_EQ(lhs_lhs->literal->getValue(),2); - ASSERT_EQ(lhs_rhs->literal->getValue(), 3); - ASSERT_EQ(rhs->literal->getValue(), 2); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "awesome := (2 + 3) * 2;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, expr); + + auto binary = CAST(SyntaxBinary*, decl->expression); + + auto lhs = CAST(SyntaxBinary*, binary->lhs); + auto lhs_lhs = CAST(SyntaxLiteral*, lhs->lhs); + auto lhs_rhs = CAST(SyntaxLiteral*, lhs->rhs); + + auto op = binary->operation; + auto rhs = CAST(SyntaxLiteral*, binary->rhs); + + ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); + //ASSERT_EQ(decl->name->name(), "wow"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "awesome", decl->name->len()) == 0); + ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); + + ASSERT_EQ(op, SyntaxBinaryOp::Multiplication); + ASSERT_EQ(lhs_lhs->literal->getValue(), 2); + ASSERT_EQ(lhs_rhs->literal->getValue(), 3); + ASSERT_EQ(rhs->literal->getValue(), 2); } TEST(ParseTest, VariableValueAssignment) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "weather <<= 2223;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); + auto buf = "weather <<= 2223;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); - auto node = (SyntaxVariableValueAssignment*) parser.parse_stmt(); + auto node = (SyntaxVariableValueAssignment *) parser.parse_stmt(); - ASSERT_TRUE(::strncmp(node->name->name().begin(), "weather", node->name->len()) == 0); - ASSERT_EQ(node->op, SyntaxAssignmentOp::LShiftAssign); - ASSERT_EQ(((SyntaxLiteral*)(node->expression))->literal->getValue(), - 2223); + ASSERT_TRUE(::strncmp(node->name->name().begin(), "weather", node->name->len()) == 0); + ASSERT_EQ(node->op, SyntaxAssignmentOp::LShiftAssign); + ASSERT_EQ(((SyntaxLiteral *) (node->expression))->literal->getValue(), + 2223); } TEST(ParseTest, IfStatement) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "if value { nice := 23; }"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto buf = "if value { nice := 23; }"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_if(); - auto stmt = (SyntaxIf*) node; + auto node = parser.parse_if(); + auto stmt = (SyntaxIf *) node; - auto if_cond_node = stmt->condition; - auto if_cond = (SyntaxLiteral*) stmt->condition; + auto if_cond_node = stmt->condition; + auto if_cond = (SyntaxLiteral *) stmt->condition; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_EQ(if_cond_node->kind, SyntaxKind::LiteralValue); - ASSERT_TRUE(::strncmp(if_cond->literal->name().begin(), "value", - if_cond->literal->len()) == 0); + ASSERT_EQ(if_cond_node->kind, SyntaxKind::LiteralValue); + ASSERT_TRUE(::strncmp(if_cond->literal->name().begin(), "value", + if_cond->literal->len()) == 0); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "nice", variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression)->literal->getValue(), 23); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "nice", variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression)->literal->getValue(), 23); } TEST(ParseTest, WhileLoop) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "while some_value > 223 { another := 43; }"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto buf = "while some_value > 223 { another := 43; }"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_while(); - auto stmt = (SyntaxWhile*) node; + auto node = parser.parse_while(); + auto stmt = (SyntaxWhile *) node; - auto while_cond_node = stmt->condition; + auto while_cond_node = stmt->condition; - auto while_cond = (SyntaxBinary*) stmt->condition; - auto lhs_cond = (SyntaxLiteral*) while_cond->lhs; - auto bin_op = while_cond->operation; - auto rhs_cond = (SyntaxLiteral*) while_cond->rhs; + auto while_cond = (SyntaxBinary *) stmt->condition; + auto lhs_cond = (SyntaxLiteral *) while_cond->lhs; + auto bin_op = while_cond->operation; + auto rhs_cond = (SyntaxLiteral *) while_cond->rhs; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_EQ(while_cond_node->kind, SyntaxKind::Binary); - ASSERT_TRUE(::strncmp(lhs_cond->literal->name().begin(), "some_value", - lhs_cond->literal->len()) == 0); - ASSERT_EQ(bin_op, SyntaxBinaryOp::Greater); - ASSERT_EQ(rhs_cond->literal->getValue(), 223); + ASSERT_EQ(while_cond_node->kind, SyntaxKind::Binary); + ASSERT_TRUE(::strncmp(lhs_cond->literal->name().begin(), "some_value", + lhs_cond->literal->len()) == 0); + ASSERT_EQ(bin_op, SyntaxBinaryOp::Greater); + ASSERT_EQ(rhs_cond->literal->getValue(), 223); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "another", - variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression) - ->literal->getValue(), 43); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "another", + variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression) + ->literal->getValue(), 43); } TEST(ParseTest, ForStatement) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "for value in list_of_values { \n" - " do_something := 23.233;" - "}"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); + auto buf = "for value in list_of_values { \n" + " do_something := 23.233;" + "}"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_for(); - auto stmt = (SyntaxForList*) node; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto node = parser.parse_for(); + auto stmt = (SyntaxForList *) node; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_TRUE(::strncmp(stmt->list_var->name().begin(), "value", - stmt->list_var->len()) == 0); - ASSERT_TRUE(::strncmp(stmt->list_name->name().begin(), "list_of_values", - stmt->list_name->len()) == 0); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "do_something", - variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression) - ->literal->getValue(), 23.233); + ASSERT_TRUE(::strncmp(stmt->list_var->name().begin(), "value", + stmt->list_var->len()) == 0); + ASSERT_TRUE(::strncmp(stmt->list_name->name().begin(), "list_of_values", + stmt->list_name->len()) == 0); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "do_something", + variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression) + ->literal->getValue(), 23.233); } TEST(ParseTest, SyntaxSemiColonError) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "// this is a comment\ncool := 23 / (22+3);\nanother_var := " - "232 {"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - - try { - name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - } catch(std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "// this is a comment\ncool := 23 / (22+3);\nanother_var := " + "232 {"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + + try { + name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + } catch (std::exception &) {} } TEST(ParseTest, SyntaxMissingDelimError) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "some_value := 22 * 2 / (232 + 4 + (2 + (3*2) / 2)"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - try { - auto name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - } catch (std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "some_value := 22 * 2 / (232 + 4 + (2 + (3*2) / 2)"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + try { + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + } catch (std::exception &) {} } TEST(ParseTest, SyntaxInvalidGlobalDecl) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "some_value := 2 + 3 / 232;\n value << 23;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - try { - auto name = parser.curr(); - parser.skip(2); - parser.parse_variable_assignment(name); - parser.parse(); - } catch (std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "some_value := 2 + 3 / 232;\n value << 23;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + try { + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + parser.parse(); + } catch (std::exception &) {} } TEST_SUITE_MAIN(ParseTest) { - TEST_CALL(ParseTest, VariableDeclLiteral); - TEST_CALL(ParseTest, VariableArrayDecl); - TEST_CALL(ParseTest, VariableDeclBinary); - TEST_CALL(ParseTest, VariableDeclParenBinary); - TEST_CALL(ParseTest, VariableValueAssignment); - TEST_CALL(ParseTest, IfStatement); - TEST_CALL(ParseTest, WhileLoop); - TEST_CALL(ParseTest, SyntaxSemiColonError); - TEST_CALL(ParseTest, SyntaxMissingDelimError); - TEST_CALL(ParseTest, SyntaxInvalidGlobalDecl); - TEST_CALL(ParseTest, ForStatement); + TEST_CALL(ParseTest, VariableDeclLiteral); + TEST_CALL(ParseTest, VariableArrayDecl); + TEST_CALL(ParseTest, VariableDeclBinary); + TEST_CALL(ParseTest, VariableDeclParenBinary); + TEST_CALL(ParseTest, VariableValueAssignment); + TEST_CALL(ParseTest, IfStatement); + TEST_CALL(ParseTest, WhileLoop); + TEST_CALL(ParseTest, SyntaxSemiColonError); + TEST_CALL(ParseTest, SyntaxMissingDelimError); + TEST_CALL(ParseTest, SyntaxInvalidGlobalDecl); + TEST_CALL(ParseTest, ForStatement); } diff --git a/tests/test-main.cpp b/tests/test-main.cpp index 41782c6..cef904b 100644 --- a/tests/test-main.cpp +++ b/tests/test-main.cpp @@ -6,12 +6,14 @@ V(ParseTest) \ V(x64Test) \ + #include DEF_ALL_TESTS + int main(int argc, char **argv) { - TEST(LexTest); - TEST(ParseTest); - TEST(x64Test); - nextgen::mem::deinit(); + TEST(LexTest); + TEST(ParseTest); + TEST(x64Test); + nextgen::mem::deinit(); } diff --git a/tests/x64-test.cpp b/tests/x64-test.cpp index 566f2cd..2b0b925 100644 --- a/tests/x64-test.cpp +++ b/tests/x64-test.cpp @@ -1,11 +1,11 @@ #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif #include @@ -13,97 +13,97 @@ TEST(x64Test, Add) { - using namespace nextgen::jet::x64; - using namespace nextgen; + using namespace nextgen::jet::x64; + using namespace nextgen; - Assembler assembler = Assembler(1000); - for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) +1)) { - assembler.addb((Register)r, (Register)r); - assembler.addw((Register)r, (Register)r); - assembler.addd((Register)r, (Register)r); - assembler.addq((Register)r, (Register)r); - } + Assembler assembler = Assembler(1000); + for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) + 1)) { + assembler.addb((Register) r, (Register) r); + assembler.addw((Register) r, (Register) r); + assembler.addd((Register) r, (Register) r); + assembler.addq((Register) r, (Register) r); + } - auto number_of_bytes = assembler.bytes(); - auto bytes = assembler.spill(); + auto number_of_bytes = assembler.bytes(); + auto bytes = assembler.spill(); - ubyte correct_output[] = { 0x00, 0xC0, 0x66, 0x01, 0xC0, 0x01, 0xC0, 0x48, - 0x01, 0xC0, 0x00, 0xC9, 0x66, 0x01, 0xC9, 0x01, - 0xC9, 0x48, 0x01, 0xC9, 0x00, 0xD2, 0x66, 0x01, - 0xD2, 0x01, 0xD2, 0x48, 0x01, 0xD2, 0x00, 0xDB, - 0x66, 0x01, 0xDB, 0x01, 0xDB, 0x48, 0x01, 0xDB, - 0x40, 0x00, 0xE4, 0x66, 0x01, 0xE4, 0x01, 0xE4, - 0x48, 0x01, 0xE4, 0x40, 0x00, 0xED, 0x66, 0x01, - 0xED, 0x01, 0xED, 0x48, 0x01, 0xED, 0x40, 0x00, - 0xF6, 0x66, 0x01, 0xF6, 0x01, 0xF6, 0x48, 0x01, - 0xF6, 0x40, 0x00, 0xFF, 0x66, 0x01, 0xFF, 0x01, - 0xFF, 0x48, 0x01, 0xFF, 0x45, 0x00, 0xC0, 0x66, - 0x45, 0x01, 0xC0, 0x45, 0x01, 0xC0, 0x4D, 0x01, - 0xC0, 0x45, 0x00, 0xC9, 0x66, 0x45, 0x01, 0xC9, - 0x45, 0x01, 0xC9, 0x4D, 0x01, 0xC9, 0x45, 0x00, - 0xD2, 0x66, 0x45, 0x01, 0xD2, 0x45, 0x01, 0xD2, - 0x4D, 0x01, 0xD2, 0x45, 0x00, 0xDB, 0x66, 0x45, - 0x01, 0xDB, 0x45, 0x01, 0xDB, 0x4D, 0x01, 0xDB, - 0x45, 0x00, 0xE4, 0x66, 0x45, 0x01, 0xE4, 0x45, - 0x01, 0xE4, 0x4D, 0x01, 0xE4, 0x45, 0x00, 0xED, - 0x66, 0x45, 0x01, 0xED, 0x45, 0x01, 0xED, 0x4D, - 0x01, 0xED, 0x45, 0x00, 0xF6, 0x66, 0x45, 0x01, - 0xF6, 0x45, 0x01, 0xF6, 0x4D, 0x01, 0xF6, 0x45, - 0x00, 0xFF, 0x66, 0x45, 0x01, 0xFF, 0x45, 0x01, - 0xFF, 0x4D, 0x01, 0xFF } ; + ubyte correct_output[] = {0x00, 0xC0, 0x66, 0x01, 0xC0, 0x01, 0xC0, 0x48, + 0x01, 0xC0, 0x00, 0xC9, 0x66, 0x01, 0xC9, 0x01, + 0xC9, 0x48, 0x01, 0xC9, 0x00, 0xD2, 0x66, 0x01, + 0xD2, 0x01, 0xD2, 0x48, 0x01, 0xD2, 0x00, 0xDB, + 0x66, 0x01, 0xDB, 0x01, 0xDB, 0x48, 0x01, 0xDB, + 0x40, 0x00, 0xE4, 0x66, 0x01, 0xE4, 0x01, 0xE4, + 0x48, 0x01, 0xE4, 0x40, 0x00, 0xED, 0x66, 0x01, + 0xED, 0x01, 0xED, 0x48, 0x01, 0xED, 0x40, 0x00, + 0xF6, 0x66, 0x01, 0xF6, 0x01, 0xF6, 0x48, 0x01, + 0xF6, 0x40, 0x00, 0xFF, 0x66, 0x01, 0xFF, 0x01, + 0xFF, 0x48, 0x01, 0xFF, 0x45, 0x00, 0xC0, 0x66, + 0x45, 0x01, 0xC0, 0x45, 0x01, 0xC0, 0x4D, 0x01, + 0xC0, 0x45, 0x00, 0xC9, 0x66, 0x45, 0x01, 0xC9, + 0x45, 0x01, 0xC9, 0x4D, 0x01, 0xC9, 0x45, 0x00, + 0xD2, 0x66, 0x45, 0x01, 0xD2, 0x45, 0x01, 0xD2, + 0x4D, 0x01, 0xD2, 0x45, 0x00, 0xDB, 0x66, 0x45, + 0x01, 0xDB, 0x45, 0x01, 0xDB, 0x4D, 0x01, 0xDB, + 0x45, 0x00, 0xE4, 0x66, 0x45, 0x01, 0xE4, 0x45, + 0x01, 0xE4, 0x4D, 0x01, 0xE4, 0x45, 0x00, 0xED, + 0x66, 0x45, 0x01, 0xED, 0x45, 0x01, 0xED, 0x4D, + 0x01, 0xED, 0x45, 0x00, 0xF6, 0x66, 0x45, 0x01, + 0xF6, 0x45, 0x01, 0xF6, 0x4D, 0x01, 0xF6, 0x45, + 0x00, 0xFF, 0x66, 0x45, 0x01, 0xFF, 0x45, 0x01, + 0xFF, 0x4D, 0x01, 0xFF}; - FOR(i, number_of_bytes) { - ASSERT_EQ(bytes[i], correct_output[i]); + FOR(i, number_of_bytes) { + ASSERT_EQ(bytes[i], correct_output[i]); // Console::Log(std::hex, std::setfill('0'), std::setw(2), (ushort)bytes[i], "" // " "); - } + } } TEST(x64Test, Sub) { - using namespace nextgen::jet::x64; - using namespace nextgen; + using namespace nextgen::jet::x64; + using namespace nextgen; - Assembler assembler = Assembler(1000); - for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) +1)) { - assembler.subb((Register)r, (Register)r); - assembler.subw((Register)r, (Register)r); - assembler.subd((Register)r, (Register)r); - assembler.subq((Register)r, (Register)r); - } + Assembler assembler = Assembler(1000); + for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) + 1)) { + assembler.subb((Register) r, (Register) r); + assembler.subw((Register) r, (Register) r); + assembler.subd((Register) r, (Register) r); + assembler.subq((Register) r, (Register) r); + } - auto number_of_bytes = assembler.bytes(); - auto bytes = assembler.spill(); + auto number_of_bytes = assembler.bytes(); + auto bytes = assembler.spill(); - ubyte correct_output[] = { 0x28, 0xC0, 0x66, 0x29, 0xC0, 0x29, 0xC0, 0x48, 0x29, - 0xC0, 0x28, 0xC9, 0x66, 0x29, 0xC9, 0x29, 0xC9, 0x48, - 0x29, 0xC9, 0x28, 0xD2, 0x66, 0x29, 0xD2, 0x29, 0xD2, - 0x48, 0x29, 0xD2, 0x28, 0xDB, 0x66, 0x29, 0xDB, 0x29, - 0xDB, 0x48, 0x29, 0xDB, 0x40, 0x28, 0xE4, 0x66, 0x29, - 0xE4, 0x29, 0xE4, 0x48, 0x29, 0xE4, 0x40, 0x28, 0xED, - 0x66, 0x29, 0xED, 0x29, 0xED, 0x48, 0x29, 0xED, 0x40, - 0x28, 0xF6, 0x66, 0x29, 0xF6, 0x29, 0xF6, 0x48, 0x29, - 0xF6, - 0x40, 0x28, 0xFF, 0x66, 0x29, 0xFF, 0x29, 0xFF, 0x48, - 0x29, 0xFF, 0x45, 0x28, 0xC0, 0x66, 0x45, 0x29, 0xC0, - 0x45, 0x29, 0xC0, 0x4D, 0x29, 0xC0, 0x45, 0x28, 0xC9, - 0x66, 0x45, 0x29, 0xC9, 0x45, 0x29, 0xC9, 0x4D, 0x29, - 0xC9, 0x45, 0x28, 0xD2, 0x66, 0x45, 0x29, 0xD2, 0x45, 0x29, 0xD2, 0x4D, 0x29, 0xD2, 0x45, - 0x28, 0xDB, 0x66, 0x45, 0x29, 0xDB, 0x45, 0x29, - 0xDB, 0x4D, 0x29, 0xDB, 0x45, 0x28, 0xE4, 0x66, 0x45, 0x29, 0xE4, 0x45, 0x29, 0xE4, 0x4D, - 0x29, 0xE4, 0x45, 0x28, 0xED, 0x66, 0x45, 0x29, 0xED, 0x45, 0x29, 0xED, - 0x4D, 0x29, 0xED, 0x45, 0x28, 0xF6, 0x66, 0x45, 0x29, - 0xF6, 0x45, 0x29, 0xF6, 0x4D, 0x29, 0xF6, 0x45, 0x28, - 0xFF, 0x66, 0x45, 0x29, 0xFF, 0x45, 0x29, 0xFF, 0x4D, 0x29, 0xFF }; + ubyte correct_output[] = {0x28, 0xC0, 0x66, 0x29, 0xC0, 0x29, 0xC0, 0x48, 0x29, + 0xC0, 0x28, 0xC9, 0x66, 0x29, 0xC9, 0x29, 0xC9, 0x48, + 0x29, 0xC9, 0x28, 0xD2, 0x66, 0x29, 0xD2, 0x29, 0xD2, + 0x48, 0x29, 0xD2, 0x28, 0xDB, 0x66, 0x29, 0xDB, 0x29, + 0xDB, 0x48, 0x29, 0xDB, 0x40, 0x28, 0xE4, 0x66, 0x29, + 0xE4, 0x29, 0xE4, 0x48, 0x29, 0xE4, 0x40, 0x28, 0xED, + 0x66, 0x29, 0xED, 0x29, 0xED, 0x48, 0x29, 0xED, 0x40, + 0x28, 0xF6, 0x66, 0x29, 0xF6, 0x29, 0xF6, 0x48, 0x29, + 0xF6, + 0x40, 0x28, 0xFF, 0x66, 0x29, 0xFF, 0x29, 0xFF, 0x48, + 0x29, 0xFF, 0x45, 0x28, 0xC0, 0x66, 0x45, 0x29, 0xC0, + 0x45, 0x29, 0xC0, 0x4D, 0x29, 0xC0, 0x45, 0x28, 0xC9, + 0x66, 0x45, 0x29, 0xC9, 0x45, 0x29, 0xC9, 0x4D, 0x29, + 0xC9, 0x45, 0x28, 0xD2, 0x66, 0x45, 0x29, 0xD2, 0x45, 0x29, 0xD2, 0x4D, 0x29, 0xD2, 0x45, + 0x28, 0xDB, 0x66, 0x45, 0x29, 0xDB, 0x45, 0x29, + 0xDB, 0x4D, 0x29, 0xDB, 0x45, 0x28, 0xE4, 0x66, 0x45, 0x29, 0xE4, 0x45, 0x29, 0xE4, 0x4D, + 0x29, 0xE4, 0x45, 0x28, 0xED, 0x66, 0x45, 0x29, 0xED, 0x45, 0x29, 0xED, + 0x4D, 0x29, 0xED, 0x45, 0x28, 0xF6, 0x66, 0x45, 0x29, + 0xF6, 0x45, 0x29, 0xF6, 0x4D, 0x29, 0xF6, 0x45, 0x28, + 0xFF, 0x66, 0x45, 0x29, 0xFF, 0x45, 0x29, 0xFF, 0x4D, 0x29, 0xFF}; - FOR(i, number_of_bytes) { - ASSERT_EQ(bytes[i], correct_output[i]); + FOR(i, number_of_bytes) { + ASSERT_EQ(bytes[i], correct_output[i]); // Console::Log(std::hex, std::setfill('0'), std::setw(2), (ushort)bytes[i], "" // " "); - } + } } TEST_SUITE_MAIN(x64Test) { - TEST_CALL(x64Test, Add); - TEST_CALL(x64Test, Sub); + TEST_CALL(x64Test, Add); + TEST_CALL(x64Test, Sub); } From 06c525ad90aa0f3faa23a14015b1edd4619b202d Mon Sep 17 00:00:00 2001 From: Chinay-Domitrix Date: Thu, 28 Oct 2021 23:07:17 +0000 Subject: [PATCH 3/4] Undo :art: Reformat codebase Signed-off-by: Chinay-Domitrix <55807470+Chinay-Domitrix@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/jet----bug-report.md | 24 +- .github/workflows/codeql-analysis.yml | 50 +- .github/workflows/linux.yml | 22 +- .github/workflows/macos.yml | 22 +- .github/workflows/windows.yml | 22 +- CMakeLists.txt | 2 +- CODE_OF_CONDUCT.md | 114 +- CONTRIBUTING.md | 64 +- LANG.md | 188 +- README.md | 55 +- import/nextgen/allocator.h | 338 +-- import/nextgen/core-util.h | 139 +- import/nextgen/io.h | 74 +- import/nextgen/jet/jet-analyzer.h | 23 +- import/nextgen/jet/jet-ast.h | 329 +-- import/nextgen/jet/jet-diagnostics.h | 157 +- import/nextgen/jet/jet-lexer.h | 280 ++- import/nextgen/jet/jet-parser.h | 728 +++--- import/nextgen/jet/jet-syntax-nodes.h | 1283 +++++----- import/nextgen/jet/jet-token.h | 1057 ++++---- import/nextgen/jet/jet-x64-assembler.h | 544 ++-- import/nextgen/os-config.h | 34 +- import/nextgen/panic.h | 199 +- import/nextgen/str.h | 359 +-- lib/allocator.cpp | 39 +- lib/io.cpp | 12 +- lib/jet-analyzer.cpp | 2 +- lib/jet-diagnostics.cpp | 813 +++--- lib/jet-lexer-v1.cpp | 2495 ++++++++++--------- lib/jet-lexer-v2.cpp | 1495 +++++------ lib/jet-parser.cpp | 608 +++-- lib/jet-x64-assembler.cpp | 547 ++-- lib/str.cpp | 12 +- main.cpp | 12 +- tests/CMakeLists.txt | 3 +- tests/lex-test.cpp | 437 ++-- tests/parse-test.cpp | 475 ++-- tests/test-main.cpp | 10 +- tests/x64-test.cpp | 154 +- 39 files changed, 6745 insertions(+), 6476 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/jet----bug-report.md b/.github/ISSUE_TEMPLATE/jet----bug-report.md index bb2d33a..60fd1b9 100644 --- a/.github/ISSUE_TEMPLATE/jet----bug-report.md +++ b/.github/ISSUE_TEMPLATE/jet----bug-report.md @@ -1,9 +1,6 @@ --- -name: Jet ✈ī¸ Bug Report -about: Help Report bugs and any other security vulnerabilities. -title: '' -labels: bug -assignees: '' +name: Jet ✈ī¸ Bug Report about: Help Report bugs and any other security vulnerabilities. title: '' +labels: bug assignees: '' --- @@ -12,21 +9,22 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Find '...' 2. Click on '....' 3. Scroll down to '....' 4. Error occurs **Expected behavior** -A clear and concise description of what was supposed to happen. Also please described how the -unit tests reacted when you tested for this behavior. +A clear and concise description of what was supposed to happen. Also, please describe how the unit tests reacted when +you tested for this behavior. **Screenshots and Videos** -If you can, please add screenshots and videos to help us understand the problem. -Doing this is a sure fire way that we will be able to reproduce the error and likely get -the problem solved faster. +If you can, please add screenshots and videos to help us understand the problem. Doing this is a sure fire way that we +will be able to reproduce the error and likely get the problem solved faster. **Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] + +- OS: [e.g. iOS] +- Browser [e.g. chrome, safari] +- Version [e.g. 22] diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1919bd4..d842af4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,34 +38,34 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - #- run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 54a9065..f5a132c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,6 +1,6 @@ name: Linux -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && ./tests diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2986190..0a62ff4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,6 @@ name: MacOS -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: macos-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && ./tests diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a86fd39..e7ce79b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,6 +1,6 @@ name: Windows -on: [push, pull_request] +on: [ push, pull_request ] env: BUILD_TYPE: Release @@ -10,16 +10,16 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v1 - - name: Install gtest manually - run: git submodule init && git submodule update - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + - uses: actions/checkout@v1 + - name: Install gtest manually + run: git submodule init && git submodule update + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: cd tests && cd Release && ./tests + - name: Test + working-directory: ${{github.workspace}}/build + run: cd tests && cd Release && ./tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b5252b..0a2494f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ add_subdirectory(tests) add_executable(jet main.cpp - # Import Support Libraries. + # Import Support Libraries. lib/panic.cpp lib/allocator.cpp lib/str.cpp diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 705bb97..65c8b4d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,126 +1,100 @@ # Jet ✈ī¸ Code of Conduct - ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for +everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity +and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, +or sexual identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to a positive environment for our -community include: +Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexual language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take +appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, +issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for +moderation decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing +the community in public spaces. Examples of representing our community include using an official e-mail address, posting +via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -nextgenprojectofficial@gmail.com. -All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible +for enforcement at nextgenprojectofficial@gmail.com. All complaints will be reviewed and investigated promptly and +fairly. -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. +All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem +in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the +community. -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation +and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of actions. -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including +unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding +interactions in community spaces as well as external channels like social media. Violating these terms may lead to a +temporary or permanent ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified +period of time. No public or private interaction with the people involved, including unsolicited interaction with those +enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate +behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +**Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). +Community Impact Guidelines were inspired +by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d466ac5..c7d1879 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -👍 Thanks for considering to contribute to the project, we appreciate it. 👍 +👍 Thanks for considering contributing to the project, we appreciate it. 👍 Outline ----------------- @@ -23,16 +23,16 @@ Before starting the rest of the process, please take time read and understand th Spread the Word --------------- -Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on -some online forums to encourage a friendly discussion. The language is at its early stages, so -we want as much exposure as possible. We have currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) +Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on some online forums to +encourage a friendly discussion. The language is at its early stages, so we want as much exposure as possible. We +currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) where we discuss new changes and possible integrations with new technologies. Add your own Features --------------------- -Want to add amazing language features yourself? Submit your -own language syntax ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if -selected, the syntax/feature will be named after you! (or an appropriate name of your choosing). +Want to add amazing language features yourself? Submit your own language syntax +ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if selected, the syntax/feature will be named after +you! (or an appropriate name of your choosing). > **Note:** Existing Language Features/Syntax suggested will not be named after you. We love suggestions > of new features of the language, but please be mindful to see if your suggestion has been already been @@ -40,20 +40,23 @@ selected, the syntax/feature will be named after you! (or an appropriate name of Style Guidelines ---------------- -All programs under the NextGen Project follow the following C++ Style Guidlines. +All programs under the NextGen Project follow the following C++ Style Guidelines. * Getter and Setter functions are to be named using `camelCase` * Functions (including in namespaces and in classes) should be denoted with `MyFunction` notation -* Class and Struct member **variables** should be deonoted with `MyVariable` +* Class and Struct member **variables** should be denoted with `MyVariable` * Don't be *too* generous with your comments. Use them appropriately, especially when describing a complicated task -* Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistant with config -* Lines should be no longer than **80 characters**. We like to limit this line length for readibility +* Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistent with + config +* Lines should be no longer than **80 characters**. We like to limit this line length for readability * Mark implicit constructors with `/*implicit*/` indication -* Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format guidlines with a library -* Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in that scope, for example, - a class with that name requires all its methods and variables to be denoted as `snake_case`. +* Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format + guidelines with a library +* Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in + that scope, for example, a class with that name requires all its methods and variables to be denoted as `snake_case`. When writing nested namespace use the following setup to be consistent with other files: + ```c++ namespace A { namespace B { namespace C { @@ -62,8 +65,8 @@ namespace A { namespace B { namespace C ``` > **Note:** There are a few exceptions to the rules above: If we create an STL-like object, keep styling -> based on STL styling, and just make sure to use common sense. If something has a very long typename and -> has better readibility using trailing return type, use it, **prioritize readibility** over anything else. +> based on STL styling, and just make sure to use common sense. If something has a very long typename and +> has better readability using trailing return type, use it, **prioritize readability** over anything else. Committing Guidelines --------------------- @@ -71,25 +74,26 @@ All commits to the project's repository are follow the following Guidelines. * Please limit your commit messages to no more than **10** words * Start all of your commit messages with a capital letter after the designated icon (see below) -* **Don't** end your commiting messages with `.` +* **Don't** end your committing messages with `.` * Please use the following icons before your commit message to indicate changes made: - * :art: `:art:` when improving the format/structure of the code - * :heavy_plus_sign: `:heavy_plus_sign:` when adding new features to existing code - * :zap: `:zap:` when improving performance - * :book: `:book:` when writing docs - * :penguin: `:penguin:` when updating portability - * :bug: `:bug:` when fixing a bug - * :fire: `:fire:` when source files are moved, changed, or deleted - * :wrench: `:wrench:` when fixing build system (CMake, CI) - * :white_check_mark: `:white_check_mark:` when working on tests + * :art: `:art:` when improving the format/structure of the code + * :heavy_plus_sign: `:heavy_plus_sign:` when adding new features to existing code + * :zap: `:zap:` when improving performance + * :book: `:book:` when writing docs + * :penguin: `:penguin:` when updating portability + * :bug: `:bug:` when fixing a bug + * :fire: `:fire:` when source files are moved, changed, or deleted + * :wrench: `:wrench:` when fixing build system (CMake, CI) + * :white_check_mark: `:white_check_mark:` when working on tests Pull Requests --------------- -Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want create -a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to -merge into the branch you requested for. +Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want to +create a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to merge into +the branch you requested for. Licensing ---------- -As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise specified. +As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise +specified. diff --git a/LANG.md b/LANG.md index 2bec113..cda12ec 100644 --- a/LANG.md +++ b/LANG.md @@ -1,6 +1,7 @@ # Jet ✈ī¸ Language Documentation -The language specification is listed below, everything from function variables, and the build systems that are planned to -be implemented or are already implemented are below. + +The language specification is listed below, everything from function variables, and the build systems that are planned +to be implemented or are already implemented are below. > **Note:** To determine whether a feature currently exists check for the keywords: **Implemented**, **WIP**, and **Planned**. @@ -17,25 +18,18 @@ Outline * [Enum](#enum-declaration) * [Match Statement](#match-statement) - Primitives -------------- Current Status: **WIP** -| Core Type | Description | -| ------------- | ------------- | -| `i32` | By default, all integer types are designated to 32 bits unless specified differently. | -| `i64` | 64 bit integer is separate and is using for longer integer values. | -| `u32` | Unsigned 32 bit integer. | -| `u64` | Unsigned 64 bit integer. | -| `f32` | 32 bit decimal value. | -| `f64` | 64 bit decimal value. | -| `bool` | Boolean Value: true or false. | -| `fn(Arg Types...) -> Ret` | Function Lambda. | -| `char` | A 1 byte character. | -| `str` | Array of characters | -| `None` | Optional value for `None` | +| Core Type | Description | | ------------- | ------------- | | `i32` | By default, all integer types are +designated to 32 bits unless specified differently. | | `i64` | 64-bit integer is separate and is using for longer +integer values. | | `u32` | Unsigned 32-bit integer. | | `u64` | Unsigned 64-bit integer. | | `f32` | +32-bit decimal value. | | `f64` | 64-bit decimal value. | | `bool` | Boolean Value: true or false. | +| `fn(Arg Types...) -> Ret` | Function Lambda. | | `char` | A 1 byte character. | | `str` | Array +of characters | | `None` | Optional value for `None` | ### Type Annotations + | Annotation | Description | | ------------- | ------------- | | `box` | Designates a type as smart pointer value | @@ -46,22 +40,11 @@ Current Status: **WIP** Operators ----------- Current Status: **Completed** -| Operator | Description | -| ------------- | ------------- | -| `+` | Addition. | -| `-` | Subtraction. | -| `/` | Division. | -| `*` | Multiplication. | -| `%` | Modulus. | -| `^` | XOR. | -| `&` | Bitwise AND. | -| `<<` | Shift Left. | -| `>>` | Shift Right. | -| `~` | Bitwise Not and Concat. | -| `**` | Exponential Power. | -| `!` | Logical Not. | -======= -| `??` | Then operator | +| Operator | Description | | ------------- | ------------- | | `+` | Addition. | | `-` | +Subtraction. | | `/` | Division. | | `*` | Multiplication. | | `%` | Modulus. | | `^` +| XOR. | | `&` | Bitwise AND. | | `<<` | Shift Left. | | `>>` | Shift Right. | | `~` +| Bitwise Not and Concat. | | `**` | Exponential Power. | | `!` | Logical Not. | ======= | `??` +| Then operator | @@ -71,20 +54,24 @@ Current Status: **Completed** The programming language is gradually typed. Static typing is only required when the compiler cannot infer the type. Variables may be declared in two different ways: + ```v my_variable := 23; another_variable: bool = false; ``` + All variables are immutable by default (except for lists), and require the `mut` keyword in order to change the way you interact with that variable. + ```v cant_change_me := 2324; cant_change_me = 24354; // Compiler error! mut change_me := 3342; change_me = 23424; // Ok ``` -Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, the -specification is standard for all data types. + +Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, +the specification is standard for all data types. Arrays @@ -92,95 +79,115 @@ Arrays Current Status: **WIP** ### Initialization -We wanted arrays to be simple and easy to initalize like in Python. + +We wanted arrays to be simple and easy to initialize like in Python. + ```rust mut list := [1, 2, 3, 4, 5]; // Can now add values to the list fixed: [5,i32] = [1, 2, 3, 4, 5]; ``` ### Designated Initializers + Current Status: **Planned** -Lists are mutable by default (unlike most of the other data types) because they are most -commonly dynamic and require mutability. Since we like to acknowledge the strengths of some -languages and weaknesses of others, one may consider a similar example of initalizing an array -using designated initalizers in Jet. +Lists are mutable by default (unlike most of the other data types) because they are most commonly dynamic and require +mutability. Since we like to acknowledge the strengths of some languages and weaknesses of others, one may consider a +similar example of initializing an array using designated initializers in Jet. + ```v list: [256,i32] = [ ['>'] = 2323, ['3'] = 343, ... ] ``` -The advantages of such a method reduce excess array initalization code, and allow for better readability to -the programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. -We feel that this as a design principle is good for dealing with code that relies on jump comparisons. + +The advantages of such a method reduce excess array initialization code, and allow for better readability to the +programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. We feel +that this as a design principle is good for dealing with code that relies on jump comparisons. ### Length + Current Status: **Planned** ```python -len(list); +len(list) ``` -Length is checked in a similar global length function that can find the length of various types, including lists. +Length is checked in a similar global length function that can find the length of various types, including lists. ### Slices + Current Status: **Planned** + ``` list[ Initial : End : IndexJump ] ``` -List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the list. + +List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the +list. Functions ------------ Current Status: **WIP** -Functions like anything else are declared explicitly in structs or are declared -in the global module scope. If functions are not exported, they are only viewable to the members in -the module that it was declared in. The `export` keyword is equivalent to the `public` keyword in other -languages. +Functions like anything else are declared explicitly in structs or are declared in the global module scope. If functions +are not exported, they are only viewable to the members in the module that it was declared in. The `export` keyword is +equivalent to the `public` keyword in other languages. ### Function Declaration + Current Status: **WIP** + ```zig my_func => (a: i32, b: i32, c: str) { ... } ``` -parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would be -written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are also -supported: + +parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would +be written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are +also supported: + ```zig generics => ($a, $b) { ... } ``` -Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is guarnteed to be valid -otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and dynamic typing flexibility for types that can be -generalized for similar actions. To make it easier for anonymous trait building, it is best for the programmer to statically define their variables -to make it easier for the compiler. + +Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is +guaranteed to be valid otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and +dynamic typing flexibility for types that can be generalized for similar actions. To make it easier for anonymous trait +building, it is best for the programmer to statically define their variables to make it easier for the compiler. ### Default Values + ```zig default_values := (a: int = 0xffff, b: str) { ... } ``` -Default values allow for flexibility in programming, especially when you want to handle a case without passing in a parameter. Currently -it is not planned to have default values for parameters that don't have their type explicitely stated. + +Default values allow for flexibility in programming, especially when you want to handle a case without passing in a +parameter. Currently, it is not planned to have default values for parameters that don't have their type explicitly +stated. ### Lambdas + Current Status: **Planned** -Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally declared as a parameters -by the following method: +Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally +declared as a parameters by the following method: + ```zig func => (lambda: fn(i32) -> i32) { lambda(23); // Calling the function } ``` -You may pass in lambdas through generic parameters as well and call them as long as function definition generation works as intended. An Example -for this function call may be (in reference to the above example): + +You may pass in lambdas through generic parameters as well and call them as long as function definition generation works +as intended. An Example for this function call may be (in reference to the above example): + ```zig func(|x| => { return x * x @@ -191,52 +198,69 @@ Modules -------- Current Status: **WIP** -The way a module system of a programming language dictates its entire package functionality structure. In order to be able to give -the programmer a lot of control we have decided to make it explicit as to functions / types are exported out in the file's module. +The way a module system of a programming language dictates its entire package functionality structure. In order to be +able to give the programmer a lot of control we have decided to make it explicit as to functions / types are exported +out in the file's module. ### Imports + Current Status: **WIP** ```python import SomeModuleName ``` + This is how most modules will be imported directly. > **Note:** Sub module structure like `std.func` are still in debate as of now, and this will be updated in the future to reflect language > changes. -### Github Imports +### GitHub Imports + Current Status: **Planned** ```python import 'https://github.com/The-NextGen-Project/jet.git' as SomeModule ``` -This is especially a very interesting part of the programming language. We will have support for open source imports. These projects -will need to be Jet projects that can be recognizable and imported directly into your project. This may be a good way to consistantly -ensure that you have the latest stable version of the git repository imported into your programming language. + +This is especially a very interesting part of the programming language. We will have support for open source imports. +These projects will need to be Jet projects that can be recognizable and imported directly into your project. This may +be a good way to consistently ensure that you have the latest stable version of the git repository imported into your +programming language. ### Exporting values + Current Status: **Planned** Functions: + ```zig export my_func => (...) { ... } ``` + Types: + ```zig export MyStruct => struct { ... } ``` + Variables: + ```js -export MyGlobalVariable := 32; +export +MyGlobalVariable +: += 32; ``` -values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted by the prefix -of the `export` before the declaration. +values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted +by the prefix of the `export` before the declaration. ### Static Compilation (Generally JIT Compiled) + Examples: + ```zig // Evaluated at compile-time some_value := static { @@ -245,8 +269,9 @@ some_value := static { ``` ### Match Statement -Match statements allow for expressing complex situations in shorter and more readable code. -Examples: + +Match statements allow for expressing complex situations in shorter and more readable code. Examples: + ```rust match value { 0 -> { @@ -259,8 +284,10 @@ match value { ``` ### Struct Declaration -Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while allowing for zero cost abstractions to take place. -Examples: + +Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while +allowing for zero cost abstractions to take place. Examples: + ```zig MyStruct => struct { prop: str; @@ -269,8 +296,10 @@ MyStruct => struct { ``` ### Enum Declaration -Enum Declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as constant values may be implemented. -Regular Enum: + +Enum Declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as +constant values may be implemented. Regular Enum: + ```zig Opcodes => enum { Add, @@ -279,7 +308,9 @@ Opcodes => enum { Div } ``` + Tagged Enums: + ```zig Operation => enum { Add { @@ -296,4 +327,5 @@ Operation => enum { End ------- -You made it to the end? Wow, thank you for the interest in reading the language. Before we you lose your attention, please consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). +You made it to the end? Wow, thank you for the interest in reading the language. Before we lose your attention, please +consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). diff --git a/README.md b/README.md index 4735745..ece2584 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ ![Jet](https://github.com/The-NextGen-Project/jet/blob/main/.github/JETV4Large3.png) -[![GitHub license](https://img.shields.io/github/license/The-NextGen-Project/jet?color=%2320ba37&label=License&logo=%2320ba37&logoColor=%2320ba37&style=for-the-badge)](https://github.com/The-NextGen-Project/jet/blob/main/LICENSE) +[![GitHub license](https://img.shields.io/github/license/The-NextGen-Project/jet?color=%2320ba37&label=License&logo=%2320ba37&logoColor=%2320ba37&style=for-the-badge)](https://github.com/The-NextGen-Project/jet/blob/main/LICENSE) [![Linux](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Linux?label=Linux&style=for-the-badge&logo=github)]() [![Mac](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/MacOS?label=MacOS&style=for-the-badge&logo=github)]() [![Windows](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Windows?label=Windows&style=for-the-badge&logo=github)]() + #### Fast Language for Developing Consistent Native, Mobile App, and Web Applications. + > **NOTE:** The language is in its early development stages. Don't expect syntax to be fixed until v1.0. If you want to see the development branch, you can view it [here](https://github.com/The-NextGen-Project/jet/tree/dev). The main branch is the most stable with all tests passing on every platform. Resources @@ -17,8 +19,9 @@ Resources * [r/Programming Languages Discord Channel](https://discord.gg/tft9Nw5jm6) * [License](#license) -Download +Download --------- + ``` git clone https://github.com/The-NextGen-Project/jet.git cd jet @@ -27,15 +30,19 @@ git submodule update cmake -DCMAKE_BUILD_TYPE=Release -GYourBuildGenerator make ``` -> **Note:** For building the project on windows, it can be done by simply building with MSVC or, if you have MinGW installed it will work with that C compiler as well. For setting up a build system that is fast, I reccommend using [Ninja](https://ninja-build.org/) which works on Windows, Linux, and Mac as a CMake generator. + +> **Note:** For building the project on Windows, it can be done by simply building with MSVC or, if you have MinGW installed it will work with that C compiler as well. For setting up a build system that is fast, I recommend using [Ninja](https://ninja-build.org/) which works on Windows, Linux, and Mac as a CMake generator. A Test Flight ------------- Offering a little taste of the lang, here is some syntax. A hello world program is as follows: + ```zig print("Hello, World!") ``` + Fibonacci Implemented in the lang would be as follows: + ```zig fib => (n: i32) i32 { return match n { @@ -45,34 +52,39 @@ fib => (n: i32) i32 { }; } ``` -The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is by default, statically-typed, however, types are -usually inferred by the langugage most of the time, and generate clean and easy to read code. This way we get to appeal to both those who like static typing and those that -prefer mainly dynamic typing. + +The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is +by default, statically-typed, however, types are usually inferred by the language most of the time, and generate clean +and easy to read code. This way we get to appeal to both those who like static typing and those that prefer mainly +dynamic typing. ### Error Messages -Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why their code -is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer overflow**: + +Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why +their code is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer +overflow**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/LexErrorExample.png) -The error message points out the initial error, points out the offending line and character, gives a potential solution, and explains -why what the programmer wrote was invalid. These components identify the problem and give opportunities for a solution to help the -programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a hybrid error message type that -encapsulates all that the programmer needs to identify the problem and solve it. +The error message points out the initial error, points out the offending line and character, gives a potential solution, +and explains why what the programmer wrote was invalid. These components identify the problem and give opportunities for +a solution to help the programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a +hybrid error message type that encapsulates all that the programmer needs to identify the problem and solve it. Let's have a look at another example, for an **unexpected token in the global scope**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/ParseErrorExample1.png) -People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help provide clarity -and provide potential solutions to a problem they may not understand. By telling the programmer what is directly valid, it prevents programmers to lookup documentation -unless it is necessary to explain an error. +People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help +provide clarity and provide potential solutions to a problem they may not understand. By telling the programmer what is +directly valid, it prevents programmers to lookup documentation unless it is necessary to explain an error. Progress ---------- -- [x] Lexer -- [x] Parser + +- [x] Lexer +- [x] Parser - [x] Semantic Analysis (**WIP**) - [ ] IR-Gen - [ ] IR-Optimization @@ -83,11 +95,12 @@ Progress Contributing ------------- -We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a community effort -and we would love to have many of you who are willing to help. Before your start your wonderful contribution, have a look at our +We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a +community effort, and we would love to have many of you who are willing to help. Before your start your wonderful +contribution, have a look at our [Contributing Guidelines](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). License ---------- -We want the project to be able to integrate with other with a lot of other open source software and determined that -we will be using the **MIT License**. +We want the project to be able to integrate with other with a lot of other open source software and determined that we +will be using the **MIT License**. diff --git a/import/nextgen/allocator.h b/import/nextgen/allocator.h index bcfc428..bf67ff4 100644 --- a/import/nextgen/allocator.h +++ b/import/nextgen/allocator.h @@ -1,205 +1,213 @@ # ifndef NEXTGEN_ALLOCATOR_H # define NEXTGEN_ALLOCATOR_H + # include "core-util.h" +namespace nextgen { + namespace mem { + using namespace nextgen::core; + + namespace os { // System allocators/libc + + void *calloc(size_t, size_t); + + void *malloc(size_t); -namespace nextgen { namespace mem { using namespace nextgen::core; + void *realloc(void *, size_t); - namespace os { // System allocators/libc + void free(void *); - void *calloc(size_t, size_t); - void *malloc(size_t); - void *realloc(void *, size_t); - void free(void *); - void *memmove(void *dst, const void *src, size_t size); + void *memmove(void *dst, const void *src, size_t size); - } // namespace nextgen::mem::os + } // namespace nextgen::mem::os - namespace detail { - struct c_deleter { - void operator()(void *ptr) const { - os::free(ptr); - } - }; - } + namespace detail { + struct c_deleter { + void operator()(void *ptr) const { + os::free(ptr); + } + }; + } - template - using Box = std::unique_ptr; - template - using Rc = std::shared_ptr; + template + using Box = std::unique_ptr; + template + using Rc = std::shared_ptr; - namespace arena { + namespace arena { - static constexpr auto PAGE_ALLOCATION_SIZE = 65'536; + static constexpr auto PAGE_ALLOCATION_SIZE = 65'536; - /** @brief 100 GiB */ - static constexpr decltype(UINTPTR_MAX) - OVER_COMMIT_MEMORY_SIZE = 107374182400ULL; + /** @brief 100 GiB */ + static constexpr decltype(UINTPTR_MAX) + OVER_COMMIT_MEMORY_SIZE = 107374182400ULL; - /** @brief 10 GiB */ - static constexpr decltype(UINTPTR_MAX) - DATA_ALLOCATION_MEMORY_SIZE = 10737418240ULL; + /** @brief 10 GiB */ + static constexpr decltype(UINTPTR_MAX) + DATA_ALLOCATION_MEMORY_SIZE = 10737418240ULL; - static uint8_t *ArenaMemory = + static uint8_t *ArenaMemory = #ifdef NG_OS_WINDOWS - (uint8_t *) VirtualAlloc(nullptr,OVER_COMMIT_MEMORY_SIZE,MEM_RESERVE, PAGE_READWRITE); + (uint8_t *) VirtualAlloc(nullptr, OVER_COMMIT_MEMORY_SIZE, MEM_RESERVE, PAGE_READWRITE); #endif #ifdef NG_OS_LINUX - (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | - PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); + (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | + PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); #endif #ifdef NG_OS_APPLE - (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | - PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + (uint8_t*) mmap(nullptr, OVER_COMMIT_MEMORY_SIZE, PROT_READ | + PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); #endif - NG_INLINE size_t &length() { - static size_t BytesWritten = 0; - return BytesWritten; - } + NG_INLINE size_t &length() { + static size_t BytesWritten = 0; + return BytesWritten; + } - /** @summary Give portion of memory from the arena */ - NG_INLINE uint8_t *give(decltype(UINTPTR_MAX) memory) { - uint8_t * bytes; - switch (memory) { - case 0: return nullptr; - default: - auto written_bytes = arena::length(); + /** @summary Give portion of memory from the arena */ + NG_INLINE uint8_t *give(decltype(UINTPTR_MAX) memory) { + uint8_t *bytes; + switch (memory) { + case 0: + return nullptr; + default: + auto written_bytes = arena::length(); - bytes = ArenaMemory + written_bytes; - arena::length() += memory; - return bytes; - } - } + bytes = ArenaMemory + written_bytes; + arena::length() += memory; + return bytes; + } + } - /** @summary Get the memory from the arena and ensure the OS maps it */ - // TODO: Setup Linux and macOS memory allocation - NG_INLINE uint8_t *give_and_commit(decltype(UINTPTR_MAX) memory) { - uint8_t *mem = arena::give(memory); + /** @summary Get the memory from the arena and ensure the OS maps it */ + // TODO: Setup Linux and macOS memory allocation + NG_INLINE uint8_t *give_and_commit(decltype(UINTPTR_MAX) memory) { + uint8_t *mem = arena::give(memory); #ifdef NG_OS_WINDOWS - // Windows is a weird specimen. We cannot over-commit, so we need to - // reserve an impossible amount of memory and then request it from the - // same buffer. - return (uint8_t *) VirtualAlloc(mem, memory, MEM_COMMIT, PAGE_READWRITE); + // Windows is a weird specimen. We cannot over-commit, so we need to + // reserve an impossible amount of memory and then request it from the + // same buffer. + return (uint8_t *) VirtualAlloc(mem, memory, MEM_COMMIT, PAGE_READWRITE); #else - return mem; + return mem; #endif - } + } - // TODO: Setup Linux and macOS memory allocation - NG_INLINE void commit(uint8_t *arena, decltype(UINTPTR_MAX) memory) { + // TODO: Setup Linux and macOS memory allocation + NG_INLINE void commit(uint8_t *arena, decltype(UINTPTR_MAX) memory) { #ifdef NG_OS_WINDOWS - VirtualAlloc(arena, memory, MEM_COMMIT, - PAGE_READWRITE); + VirtualAlloc(arena, memory, MEM_COMMIT, + PAGE_READWRITE); #endif - } - } - - class Allocator { - uint8_t *bytes = nullptr; - size_t bytes_written = 0; - size_t capacity = 0; - public: - explicit Allocator(size_t reserved_memory) - : bytes(arena::give_and_commit(arena::PAGE_ALLOCATION_SIZE)), - bytes_written(0), - capacity(reserved_memory) {} - - auto current() const { - return bytes + bytes_written; - } - - auto length() const { - return bytes_written; - } - - auto reserve(size_t size) { - auto curr = current(); - bytes_written += size; - return curr; - } - - auto allocate(size_t allocation_size) { - auto ret_value = bytes_written + bytes; - if (bytes_written + allocation_size > capacity) { - arena::commit(ret_value, arena::PAGE_ALLOCATION_SIZE); - } - bytes_written += allocation_size; - return ret_value; - } - - template - auto allocate(size_t n) { - return (T*) (allocate(sizeof(T) * n)); - } - }; - - extern Allocator GLOBAL_OBJECT_ALLOC; - extern Allocator GLOBAL_DATA_ALLOC; - - NG_INLINE void deinit() { + } + } + + class Allocator { + uint8_t *bytes = nullptr; + size_t bytes_written = 0; + size_t capacity = 0; + public: + explicit Allocator(size_t reserved_memory) + : bytes(arena::give_and_commit(arena::PAGE_ALLOCATION_SIZE)), + bytes_written(0), + capacity(reserved_memory) {} + + auto current() const { + return bytes + bytes_written; + } + + auto length() const { + return bytes_written; + } + + auto reserve(size_t size) { + auto curr = current(); + bytes_written += size; + return curr; + } + + auto allocate(size_t allocation_size) { + auto ret_value = bytes_written + bytes; + if (bytes_written + allocation_size > capacity) { + arena::commit(ret_value, arena::PAGE_ALLOCATION_SIZE); + } + bytes_written += allocation_size; + return ret_value; + } + + template + auto allocate(size_t n) { + return (T *) (allocate(sizeof(T) * n)); + } + }; + + extern Allocator GLOBAL_OBJECT_ALLOC; + extern Allocator GLOBAL_DATA_ALLOC; + + NG_INLINE void deinit() { #ifdef NG_OS_WINDOWS - VirtualFree(arena::ArenaMemory, arena::length(), MEM_DECOMMIT); + VirtualFree(arena::ArenaMemory, arena::length(), MEM_DECOMMIT); #else - munmap(arena::ArenaMemory, GLOBAL_DATA_ALLOC.length() - +GLOBAL_OBJECT_ALLOC.length()); + munmap(arena::ArenaMemory, GLOBAL_DATA_ALLOC.length() + +GLOBAL_OBJECT_ALLOC.length()); #endif - } - - - template - class ObjectVector { - T *objects = GLOBAL_OBJECT_ALLOC.template allocate(N); - size_t len = 0; - size_t cap = N; - public: - - T operator[](size_t index) { - return objects[index]; - } - - void operator<<(T const &elem) { - if (len >= cap) { - cap *= 2; - auto new_ptr = GLOBAL_OBJECT_ALLOC.template allocate(cap); - FOR(i, len) { - new_ptr[i] = objects[i]; - } - objects = new_ptr; - } - objects[len++] = elem; - } - }; - - template - struct ArenaVec { - /** @brief Start of vector */ - T *begin; - /** @brief End of vector */ - T *end; - - /** @brief Indexes the value of the vector. This is slightly more - * expensive than normal array access */ - NG_INLINE T *operator[](size_t index) { - return (T*) (((uint8_t *)begin) + (sizeof(T) * index)); - } - - /** @brief Iterates over the vector with any other potential factors */ - template - NG_AINLINE void iter(Lambda lambda) { - FOR(i, len()) lambda(); - } - - /** @brief Span of the vector in memory */ - auto NG_AINLINE len() { - return end-begin; - } - }; - -}} // namespace nextgen::mem + } + + + template + class ObjectVector { + T *objects = GLOBAL_OBJECT_ALLOC.template allocate(N); + size_t len = 0; + size_t cap = N; + public: + + T operator[](size_t index) { + return objects[index]; + } + + void operator<<(T const &elem) { + if (len >= cap) { + cap *= 2; + auto new_ptr = GLOBAL_OBJECT_ALLOC.template allocate(cap); + FOR(i, len) { + new_ptr[i] = objects[i]; + } + objects = new_ptr; + } + objects[len++] = elem; + } + }; + + template + struct ArenaVec { + /** @brief Start of vector */ + T *begin; + /** @brief End of vector */ + T *end; + + /** @brief Indexes the value of the vector. This is slightly more + * expensive than normal array access */ + NG_INLINE T *operator[](size_t index) { + return (T *) (((uint8_t *) begin) + (sizeof(T) * index)); + } + + /** @brief Iterates over the vector with any other potential factors */ + template + NG_AINLINE void iter(Lambda lambda) { + FOR(i, len()) lambda(); + } + + /** @brief Span of the vector in memory */ + auto NG_AINLINE len() { + return end - begin; + } + }; + + } +} // namespace nextgen::mem # endif //NEXTGEN_ALLOCATOR_H diff --git a/import/nextgen/core-util.h b/import/nextgen/core-util.h index e098259..31b5e73 100644 --- a/import/nextgen/core-util.h +++ b/import/nextgen/core-util.h @@ -1,79 +1,82 @@ #ifndef NEXTGEN_CORE_H #define NEXTGEN_CORE_H + #include "os-config.h" #include "panic.h" namespace nextgen { - enum class NoneValue { None = 1 }; - static constexpr NoneValue None = NoneValue::None; - - template - struct Array { - size_t len; - const T arr[]; - }; - - template - constexpr size_t SizeOfArray(const T(&arr)[N]) { - return N; - } - - namespace core { - - /// Optional Type Value - Emulates Rust's style of an optional value in C++11. - /// We must ensure that values are owned by the value when passed. - template - class Option { - public: - // None value by default - Option() : is(false) {}; - - // Direct Copy - Option(Option const &other) : is(other.is), Some(other.Some) {} - - Option(T const &value) : - Some(std::move(value)), is(true) {} - - /*implicit*/ Option(NoneValue) : - is(false) {} - - NG_AINLINE bool isSome() const { - return is; - } - - NG_AINLINE bool isNone() const { - return !is; - } - - template - bool contains(U x) const { - if (is) return x == Some; - return false; - } - - auto unwrap() const { - ASSERT(is, "Unwrapped on None Value"); - return std::move(Some); - } - - template - auto unwrap_or_else(Lambda f) { - if (is) return std::move(Some); - return f(); - } - - private: - T Some; - bool is; - }; - - template - static NG_AINLINE auto Some(T value) { - return Option(value); // NOTE: Make explicit - } - } + enum class NoneValue { + None = 1 + }; + static constexpr NoneValue None = NoneValue::None; + + template + struct Array { + size_t len; + const T arr[]; + }; + + template + constexpr size_t SizeOfArray(const T(&arr)[N]) { + return N; + } + + namespace core { + + /// Optional Type Value - Emulates Rust's style of an optional value in C++11. + /// We must ensure that values are owned by the value when passed. + template + class Option { + public: + // None value by default + Option() : is(false) {}; + + // Direct Copy + Option(Option const &other) : is(other.is), Some(other.Some) {} + + Option(T const &value) : + Some(std::move(value)), is(true) {} + + /*implicit*/ Option(NoneValue) : + is(false) {} + + NG_AINLINE bool isSome() const { + return is; + } + + NG_AINLINE bool isNone() const { + return !is; + } + + template + bool contains(U x) const { + if (is) return x == Some; + return false; + } + + auto unwrap() const { + ASSERT(is, "Unwrapped on None Value"); + return std::move(Some); + } + + template + auto unwrap_or_else(Lambda f) { + if (is) return std::move(Some); + return f(); + } + + private: + T Some; + bool is; + }; + + template + static NG_AINLINE auto Some(T value) { + return Option(value); // NOTE: Make explicit + } + } } // namespace nextgen::core diff --git a/import/nextgen/io.h b/import/nextgen/io.h index e726aad..2b1228e 100644 --- a/import/nextgen/io.h +++ b/import/nextgen/io.h @@ -1,53 +1,57 @@ # ifndef NEXTGEN_IO_H # define NEXTGEN_IO_H + # include "allocator.h" -namespace nextgen { namespace io { +namespace nextgen { + namespace io { + + enum FileID { + JetSourceCode, + CSourceCode, + Mach_O, + ELF, + PE + }; - enum FileID { - JetSourceCode, - CSourceCode, - Mach_O, - ELF, - PE - }; + class FileBuf { + public: + using Files = FileBuf[]; - class FileBuf { - public: - using Files = FileBuf[]; + FileBuf(char *buffer, FileID id) : buffer(buffer), id(id) {} - FileBuf(char *buffer, FileID id) : buffer(buffer), id(id) {} + FileID getFileID() const { + return id; + } - FileID getFileID() const { - return id; - } + char *getFileBuffer() const { + return buffer; + } - char *getFileBuffer() const { - return buffer; - } + // Given a list of files, output the type of file given. For example, + // given a list of source code for Jet, and set it to output a binary + // ELF, it will output an ELF binary for the given source input. + // + // Example: + // FileBuf files[] = ... + // FileBuf::Output(files, FileID::CSourceCode); + // + // NOTE: The types of files need to be able to be linked together to form + // the resulting FileID file type. + static void Output(Files files, FileID output); - // Given a list of files, output the type of file given. For example, - // given a list of source code for Jet, and set it to output a binary - // ELF, it will output an ELF binary for the given source input. - // - // Example: - // FileBuf files[] = ... - // FileBuf::Output(files, FileID::CSourceCode); - // - // NOTE: The types of files need to be able to be linked together to form - // the resulting FileID file type. - static void Output(Files files, FileID output); - private: - char *buffer; // Buffer pointer to the source text of a file - FileID id; // File Type to know how file should be read - }; + private: + char *buffer; // Buffer pointer to the source text of a file + FileID id; // File Type to know how file should be read + }; - FileBuf CreateFileBuffer(const char *FILE, FileID id); + FileBuf CreateFileBuffer(const char *FILE, FileID id); -}} // namespace nextgen::io + } +} // namespace nextgen::io diff --git a/import/nextgen/jet/jet-analyzer.h b/import/nextgen/jet/jet-analyzer.h index 57281f5..3767d2b 100644 --- a/import/nextgen/jet/jet-analyzer.h +++ b/import/nextgen/jet/jet-analyzer.h @@ -4,19 +4,22 @@ #include "jet-parser.h" #include "jet-ast.h" -namespace nextgen { namespace jet { +namespace nextgen { + namespace jet { - class Analyzer { - VariableMap global_variables; - FunctionMap functions; - public: - void analyze(const ParserOutput output_to_analyze); - void analyze_function(const SyntaxFunction *function); - void analyze_syntax_node(const SyntaxNode *node); - }; -}} + class Analyzer { + VariableMap global_variables; + FunctionMap functions; + public: + void analyze(const ParserOutput output_to_analyze); + void analyze_function(const SyntaxFunction *function); + + void analyze_syntax_node(const SyntaxNode *node); + }; + } +} #endif //JET_JET_ANALYZER_H diff --git a/import/nextgen/jet/jet-ast.h b/import/nextgen/jet/jet-ast.h index 5281a28..bd8fb2e 100644 --- a/import/nextgen/jet/jet-ast.h +++ b/import/nextgen/jet/jet-ast.h @@ -1,169 +1,172 @@ # ifndef JET_AST_H # define JET_AST_H -namespace nextgen { namespace jet { - - enum ASTKind { - BinaryExprNode, - LiteralExprNode, - IfNode, - ElseNode, - ElifNode, - WhileNode, - ForListNode, - ForRangeNode, - Structure, - FunctionNode, - MatchNode - }; - - struct AST { - ASTKind kind; - }; - - enum TypeTag { - Void, - I8, - I16, - I32, - I65, - U8, - U16, - U32, - U64, - Str, - PointerTy, - ArrayView, - SmartPointer - }; - - struct Type { - TypeTag tag; - Type *ref; - }; - - struct Pointer : public Type { - Type pointer_of; - /*implicit*/ Pointer(const Type &p) : pointer_of(p) {} - }; - - struct ArrayView : public Type { - Type array_of; - size_t size = 0; - bool has_size = false; - /*implicit*/ ArrayView(const Type &a) : array_of(a) {} - }; - - struct Variable { - const Token *initial_decl; - const Type type; - bool is_mutable = false; - - struct intern_hash { - size_t operator()(Variable const &s) const { - return s.initial_decl->name().getHashCache(); - } - }; - - struct intern_eq { - bool operator()(Variable const &LHS, Variable const &RHS) const { - auto lhs = LHS.initial_decl->name(); - auto rhs = RHS.initial_decl->name(); - return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; - } - }; - }; - - struct Scope { - Scope *parent; - std::unordered_set variables; - }; - - struct NodeLiteral { - Type type; - const Token *literal_token; - }; - - struct NodeBinary { - enum Op { - MemberGet, // struct_instance.member_that_is_struct.its_member - Add, - Sub, - Div, - Mul, - Power, - Gtr, - GtrEq, - LessThanValue, - LessEq, - LShft, - LShftEq, - RShft, - RShftEq, - l_AND, // and - l_OR, // or - _XOR, // ^ - _AND, // & - _OR // | - }; - - const AST *lhs = nullptr; - const AST *rhs = nullptr; - Op op; - }; - - struct NodeWhile { - const AST *cond; - ArenaVec statements; - }; - - struct NodeElse { - ArenaVec statements; - }; - - struct NodeElif { - ArenaVec statements; - }; - - struct NodeIf { - const AST *cond; - NodeElif *else_; - Option> elif_branches; - }; - - struct FunctionParameter { - const Token *param_name; - const Type type; - }; - - struct NodeFunction { - const Token *name = nullptr; - ArenaVec statements; - ArenaVec parameters; - - struct intern_hash { - size_t operator()(NodeFunction const &s) const { - return s.name->name().getHashCache(); - } - }; - - struct intern_eq { - bool operator()(NodeFunction const &LHS, NodeFunction const &RHS) const { - auto lhs = LHS.name->name(); - auto rhs = RHS.name->name(); - return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; - } - }; - }; - - using VariableMap = std::unordered_set ; - using FunctionMap = std::unordered_set ; - -}} - +namespace nextgen { + namespace jet { + + enum ASTKind { + BinaryExprNode, + LiteralExprNode, + IfNode, + ElseNode, + ElifNode, + WhileNode, + ForListNode, + ForRangeNode, + Structure, + FunctionNode, + MatchNode + }; + + struct AST { + ASTKind kind; + }; + + enum TypeTag { + Void, + I8, + I16, + I32, + I65, + U8, + U16, + U32, + U64, + Str, + PointerTy, + ArrayView, + SmartPointer + }; + + struct Type { + TypeTag tag; + Type *ref; + }; + + struct Pointer : public Type { + Type pointer_of; + + /*implicit*/ Pointer(const Type &p) : pointer_of(p) {} + }; + + struct ArrayView : public Type { + Type array_of; + size_t size = 0; + bool has_size = false; + + /*implicit*/ ArrayView(const Type &a) : array_of(a) {} + }; + + struct Variable { + const Token *initial_decl; + const Type type; + bool is_mutable = false; + + struct intern_hash { + size_t operator()(Variable const &s) const { + return s.initial_decl->name().getHashCache(); + } + }; + + struct intern_eq { + bool operator()(Variable const &LHS, Variable const &RHS) const { + auto lhs = LHS.initial_decl->name(); + auto rhs = RHS.initial_decl->name(); + return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; + } + }; + }; + + struct Scope { + Scope *parent; + std::unordered_set variables; + }; + + struct NodeLiteral { + Type type; + const Token *literal_token; + }; + + struct NodeBinary { + enum Op { + MemberGet, // struct_instance.member_that_is_struct.its_member + Add, + Sub, + Div, + Mul, + Power, + Gtr, + GtrEq, + LessThanValue, + LessEq, + LShft, + LShftEq, + RShft, + RShftEq, + l_AND, // and + l_OR, // or + _XOR, // ^ + _AND, // & + _OR // | + }; + + const AST *lhs = nullptr; + const AST *rhs = nullptr; + Op op; + }; + + struct NodeWhile { + const AST *cond; + ArenaVec statements; + }; + + struct NodeElse { + ArenaVec statements; + }; + + struct NodeElif { + ArenaVec statements; + }; + + struct NodeIf { + const AST *cond; + NodeElif *else_; + Option > elif_branches; + }; + + struct FunctionParameter { + const Token *param_name; + const Type type; + }; + + struct NodeFunction { + const Token *name = nullptr; + ArenaVec statements; + ArenaVec parameters; + + struct intern_hash { + size_t operator()(NodeFunction const &s) const { + return s.name->name().getHashCache(); + } + }; + + struct intern_eq { + bool operator()(NodeFunction const &LHS, NodeFunction const &RHS) const { + auto lhs = LHS.name->name(); + auto rhs = RHS.name->name(); + return strncmp(lhs.begin(), rhs.begin(), lhs.size()) == 0; + } + }; + }; + + using VariableMap = std::unordered_set; + using FunctionMap = std::unordered_set; + + } +} # endif //JET_AST_H \ No newline at end of file diff --git a/import/nextgen/jet/jet-diagnostics.h b/import/nextgen/jet/jet-diagnostics.h index c94b76f..e36cf62 100644 --- a/import/nextgen/jet/jet-diagnostics.h +++ b/import/nextgen/jet/jet-diagnostics.h @@ -1,86 +1,97 @@ #ifndef JET_JET_DIAGNOSTICS_H #define JET_JET_DIAGNOSTICS_H + #include "nextgen/io.h" #include "nextgen/allocator.h" #include "nextgen/str.h" #include "nextgen/panic.h" #include "jet-token.h" -namespace nextgen { namespace jet { - - // Lex Errors - struct LexError; - - // Printing Mode re-analyzes the file, while printing out tokens with - // syntax-highlighting to the console. This allows for pretty-error messages - // with colored output. - enum LexMode { - TokenMode, - PrintingMode - }; - - template - class Lexer; - - // Parsing Errors - struct ParseError; - class Parser; - - class Diagnostic { - private: - const char *file_buf; - const char *file_name; - size_t buf_len; - - // Line where error has occurred - str source_line; - - public: - Diagnostic() = default; - - Diagnostic(const char *buffer, size_t buf_len, const - char *file_name) - : file_buf(buffer), file_name(file_name), buf_len(buf_len) {} - - /// Build error Message for a Lexing error - void build(LexError error); - - /// Build error Message for a Lexing error - void build(ParseError error); - - void send_exception() { - Console::Log(Colors::RED, "error: ", Colors::RESET, "Aborting due to " - "previous errors\n"); - throw std::exception(); - } - private: - // Helpers - void ErrorLexSetup(std::string &line, const char *message, LexError &error); - - template - void ErrorParseSetup(size_t const ln, - const char *message, - Token const *reported_token, - TokenTraits::SourceLocation loc); - - // Lex Errors (We will keep these PascalCase) - void ErrorIntegerOverflow(LexError &error); - void ErrorDigitOutOfRange(LexError &error); - void ErrorInvalidChar(LexError &error); - void ErrorMalformedUTF8(LexError &error); - void ErrorMissingClosingDelim(LexError &error); - void ErrorHexEscapeOutOfRange(LexError &error); - void ErrorInvalidStringEscape(LexError &error); - - // Parse Errors (We will keep these PascalCase) - void ErrorParseExpectedToken(ParseError const &error); - void ErrorParseMissingClosingDelim(ParseError const &error); - void ErrorParseInvalidTokenAfterIdentInGlobalScope(ParseError const &error); - }; - -}} +namespace nextgen { + namespace jet { + + // Lex Errors + struct LexError; + + // Printing Mode re-analyzes the file, while printing out tokens with + // syntax-highlighting to the console. This allows for pretty-error messages + // with colored output. + enum LexMode { + TokenMode, + PrintingMode + }; + + template + class Lexer; + + // Parsing Errors + struct ParseError; + + class Parser; + + class Diagnostic { + private: + const char *file_buf; + const char *file_name; + size_t buf_len; + + // Line where error has occurred + str source_line; + + public: + Diagnostic() = default; + + Diagnostic(const char *buffer, size_t buf_len, const + char *file_name) + : file_buf(buffer), file_name(file_name), buf_len(buf_len) {} + + /// Build error Message for a Lexing error + void build(LexError error); + + /// Build error Message for a Lexing error + void build(ParseError error); + + void send_exception() { + Console::Log(Colors::RED, "error: ", Colors::RESET, "Aborting due to " + "previous errors\n"); + throw std::exception(); + } + + private: + // Helpers + void ErrorLexSetup(std::string &line, const char *message, LexError &error); + + template + void ErrorParseSetup(size_t const ln, + const char *message, + Token const *reported_token, + TokenTraits::SourceLocation loc); + + // Lex Errors (We will keep these PascalCase) + void ErrorIntegerOverflow(LexError &error); + + void ErrorDigitOutOfRange(LexError &error); + + void ErrorInvalidChar(LexError &error); + + void ErrorMalformedUTF8(LexError &error); + + void ErrorMissingClosingDelim(LexError &error); + + void ErrorHexEscapeOutOfRange(LexError &error); + + void ErrorInvalidStringEscape(LexError &error); + + // Parse Errors (We will keep these PascalCase) + void ErrorParseExpectedToken(ParseError const &error); + + void ErrorParseMissingClosingDelim(ParseError const &error); + void ErrorParseInvalidTokenAfterIdentInGlobalScope(ParseError const &error); + }; + } +} #endif //JET_JET_DIAGNOSTICS_H diff --git a/import/nextgen/jet/jet-lexer.h b/import/nextgen/jet/jet-lexer.h index a8f744b..b5171d8 100644 --- a/import/nextgen/jet/jet-lexer.h +++ b/import/nextgen/jet/jet-lexer.h @@ -4,146 +4,152 @@ # include "jet-token.h" # include "jet-diagnostics.h" -namespace nextgen { namespace jet { using namespace nextgen::core; - - enum LexErrorType { - // Other - UnexpectedEOF, - OutOfRange, - Unreachable, - - // Strings - MalformedUTF8, - InvalidChar, - InvalidStringEscape, - InvalidUnicodeEscapeClose, - HexEscapeOutOfRange, - MissingClosingDelim, - - // Numbers - DigitOutOfRange, - InvalidDot, - InvalidNumberValue, - IntegerOverflow, - FloatingPointOverflow, - }; - - struct LexError { - // The type of error - LexErrorType error; - - // Error location - TokenTraits::SourceLocation location; - - // Token value lexed until error - Token failed_token; - - // Error metadata - int metadata[5]; - }; - - // Lexical analysis has two different modes for two purposes. Depending on - // the case, the lexer will either register tokens or it will simply, print - // tokens to the terminal. - // - // TokenMode: Specifies the mode in which tokens are lexed and output is - // returned. - template - class Lexer { - using File = char; - - const File *buffer; - const size_t buf_size; - - size_t line = 1; - size_t column = 1; - size_t buf_pos = 0; - - Diagnostic diagnostics; - ArenaVec tokens; - - bool fatal = false; - - // Ensure that branches are taken away during compile-time with guaranteed - // compile-time value if statements. This is used so that both printing mode - // and normal mode do not have to be separate functions. - static constexpr bool OUTPUT_MODE = Mode == LexMode::PrintingMode; - static constexpr bool NORMAL_MODE = Mode == LexMode::TokenMode; - - public: - friend class Parser; - - Lexer(const File *buffer, const char *file_name, - const size_t buffer_size) - : buffer(buffer), buf_size(buffer_size) { - diagnostics = Diagnostic(buffer, buffer_size, file_name); - } - - void lex_int(); - template - void lex_int(); - void lex_float(int skip, int start=0); - - void lex_str(); - void lex_ident(); - - ArenaVec lex(); - - - /// peek `NChars` characters in the file buffer. - char peek(size_t NChars) { - return *(buffer + NChars); - } - - /// Get the latest character in the file buffer - char curr() { - return *buffer; - } - - /// Move the lexer forward by `n` and fill in `Error` for - /// unexpected EOF or Out of Range Error that may be encountered. - char next(size_t n) { - - if (buf_pos >= buf_size) { - if (Mode == TokenMode) { - diagnostics.build(LexError { - LexErrorType::OutOfRange, - { line, column }, - Token("\0", {line, column}, TokenKind::EOFToken) - }); - UNREACHABLE; - } else fatal = true; - } - - column += n; - buf_pos += n; - buffer += n; - return *buffer; - } - - private: - - /// Matches a new line encountered in the buffer and handles - /// them appropriately depending on \n or \r. - void skip_new_line() { - line++; - column = 1; - - next(1); +namespace nextgen { + namespace jet { + using namespace nextgen::core; + + enum LexErrorType { + // Other + UnexpectedEOF, + OutOfRange, + Unreachable, + + // Strings + MalformedUTF8, + InvalidChar, + InvalidStringEscape, + InvalidUnicodeEscapeClose, + HexEscapeOutOfRange, + MissingClosingDelim, + + // Numbers + DigitOutOfRange, + InvalidDot, + InvalidNumberValue, + IntegerOverflow, + FloatingPointOverflow, + }; + + struct LexError { + // The type of error + LexErrorType error; + + // Error location + TokenTraits::SourceLocation location; + + // Token value lexed until error + Token failed_token; + + // Error metadata + int metadata[5]; + }; + + // Lexical analysis has two different modes for two purposes. Depending on + // the case, the lexer will either register tokens or it will simply, print + // tokens to the terminal. + // + // TokenMode: Specifies the mode in which tokens are lexed and output is + // returned. + template + class Lexer { + using File = char; + + const File *buffer; + const size_t buf_size; + + size_t line = 1; + size_t column = 1; + size_t buf_pos = 0; + + Diagnostic diagnostics; + ArenaVec tokens; + + bool fatal = false; + + // Ensure that branches are taken away during compile-time with guaranteed + // compile-time value if statements. This is used so that both printing mode + // and normal mode do not have to be separate functions. + static constexpr bool OUTPUT_MODE = Mode == LexMode::PrintingMode; + static constexpr bool NORMAL_MODE = Mode == LexMode::TokenMode; + + public: + friend class Parser; + + Lexer(const File *buffer, const char *file_name, + const size_t buffer_size) + : buffer(buffer), buf_size(buffer_size) { + diagnostics = Diagnostic(buffer, buffer_size, file_name); + } + + void lex_int(); + + template + void lex_int(); + + void lex_float(int skip, int start = 0); + + void lex_str(); + + void lex_ident(); + + ArenaVec lex(); + + + /// peek `NChars` characters in the file buffer. + char peek(size_t NChars) { + return *(buffer + NChars); + } + + /// Get the latest character in the file buffer + char curr() { + return *buffer; + } + + /// Move the lexer forward by `n` and fill in `Error` for + /// unexpected EOF or Out of Range Error that may be encountered. + char next(size_t n) { + + if (buf_pos >= buf_size) { + if (Mode == TokenMode) { + diagnostics.build(LexError{ + LexErrorType::OutOfRange, + {line, column}, + Token("\0", {line, column}, TokenKind::EOFToken) + }); + UNREACHABLE; + } else fatal = true; + } - char current = *buffer; - char next_ch = peek(1); + column += n; + buf_pos += n; + buffer += n; + return *buffer; + } - if (next_ch != current && (next_ch == '\n' || next_ch == '\r')) { - if (Mode == LexMode::PrintingMode) Console::Log(next_ch); - ++line, column = 1, next(1); - } - } - - }; + private: - -}} // namespace nextgen::jet + /// Matches a new line encountered in the buffer and handles + /// them appropriately depending on \n or \r. + void skip_new_line() { + line++; + column = 1; + + next(1); + + char current = *buffer; + char next_ch = peek(1); + + if (next_ch != current && (next_ch == '\n' || next_ch == '\r')) { + if (Mode == LexMode::PrintingMode) Console::Log(next_ch); + ++line, column = 1, next(1); + } + } + + }; + + + } +} // namespace nextgen::jet diff --git a/import/nextgen/jet/jet-parser.h b/import/nextgen/jet/jet-parser.h index fef7555..6998b63 100644 --- a/import/nextgen/jet/jet-parser.h +++ b/import/nextgen/jet/jet-parser.h @@ -1,362 +1,382 @@ # ifndef JET_JET_PARSER_H # define JET_JET_PARSER_H + # include "jet-lexer.h" # include "jet-syntax-nodes.h" -namespace nextgen { namespace jet { using namespace nextgen::core; - - enum ParseErrorType { - ReservedIdentifierAsVariableName, - UnexpectedExpression, - InvalidToken, - ExpectedToken, - - MissingFunctionName, - MissingVariableName, - MissingForLoopVariable, - - ExpectedIdentifierForFunctionParameter, - ExpectedIdentifierForStructProperty, - InvalidTokenAfterIdentInGlobalScope, - - MissingClosingPair, - UnexpectedEndOfFile - }; - - enum ParseContext { - AnonymousBlock, - StatementBlock, - LoopBlock, - LambdaBlock, - GeneralScope - }; - - - struct ParserOutput { - ObjectVector functions; - ObjectVector structures; - ObjectVector global_variables; - }; - - struct ParseError { - union Metadata; - - ParseErrorType error; - TokenTraits::SourceLocation location; - std::initializer_list metadata; - - /** @brief Contains specific error information pertaining to the error - * message. Only a few parser errors have specific items, while most are - * invalid or missing tokens - * */ - union Metadata { - struct { - TokenKind expected; - Token const *got; - const char *message; - } expected_error; - TokenTraits::SourceLocation location; - TokenKind const misc_kind; - Token const *misc_tok; - const char *context_message; - - // GCC and CLANG let us get away with not using union constructors, - // but we need them for MSVC, and it's probably more clear anyway - Metadata(TokenTraits::SourceLocation loc) : location(loc) {} - Metadata(TokenKind const kind) : misc_kind(kind) {} - Metadata(Token const *tok) : misc_tok(tok) {} - Metadata(const char *msg) : context_message(msg) {} - Metadata(TokenKind const expected, Token const *got, const char - *message) { - expected_error.expected = expected; - expected_error.got = got; - expected_error.message = message; - } - }; - - ParseError(ParseErrorType error, TokenTraits::SourceLocation loc) - : error(error), location(loc) {} - - ParseError(ParseErrorType error, TokenTraits::SourceLocation loc, - const std::initializer_list &metadata) - : error(error), location(loc), metadata(metadata) {} - }; - - class Parser { - - // Parser's position - size_t position = 0; - - // Parsing context (valid statements per context scope) - // TODO: Set this up so stuff actually works with this... - ParseContext context = GeneralScope; - - // Parsing errors - Diagnostic diagnostics; - - // Encountered error that results in stopped compilation - char fatal = 0; - - - public: - ArenaVec tokens; - Parser() = default; - - // Why are we passing a pointer here? Each pass in the frontend is done - // through the main function, thus, when we go ahead and pass the lexer, - // we know that it has the lifetime of the program. - explicit Parser(Lexer *lexer) - : tokens(lexer->lex()), diagnostics(lexer->diagnostics), - fatal(0) {} - - /// Parse all statements - auto parse() -> const ParserOutput; - - // ========= Parsing Language Generalizations ========== - auto NG_AINLINE curr() { return tokens[position]; } - - // ========= Parsing Utils ========== - - /** @brief Lookahead 'n' amount of times in the list */ - auto NG_AINLINE peek(size_t n) { - return tokens[position + n]; - } - - /** @brief Returns the current token and skips 'n' tokens ahead */ - auto NG_INLINE skip(size_t n) { - auto *ret = tokens[position]; - position += n; - return ret; - } - - /** @brief Skips 'n' tokens and returns the latest token after the skip */ - auto NG_INLINE next(size_t n) { - position += n; - return tokens[position]; - } - - /** @brief Asserts next token kind is 'TK' or errors with type 'PE' */ - template - auto skip() -> const Token* { - auto next = tokens[(position++)]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError( - PE, - next->getSourceLocation() - )); - } - return next; - } - - /** @brief Expect the next token to be 'kind', if not, build an error. */ - template - auto expect(char const (&msg)[N]) -> const Token* { - const Token *next = tokens[(position++)]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError( - ParseErrorType::ExpectedToken, - next->getSourceLocation(), - { - ParseError::Metadata { TK, next, msg } - } - )); - if (curr()->getKind() == EOFToken) - this->diagnostics.send_exception(); - else - fatal++; - } - return next; - } - - /** @brief Expected a closing delim in a statement or expression */ - template - void expect_delim(const TokenTraits::SourceLocation &loc) { - Token *next = tokens[position++]; - if (next->getKind() != TK) { - this->diagnostics.build(ParseError ( - ParseErrorType::MissingClosingPair, - next->getSourceLocation(), - { - ParseError::Metadata { TK }, - ParseError::Metadata { next }, - ParseError::Metadata { loc } - } - )); - this->diagnostics.send_exception(); - } - } - - - /** @brief Unary operator precedence */ - static NG_INLINE int UnaryOperatorBinding(TokenKind kind) { - switch (kind) { - case TokenKind::Plus: - case TokenKind::Minus: - case TokenKind::ExclamationPoint: - case TokenKind::NOT: - return 17; - default: - return -1; - } - } - - /** @brief Infix operator binding for expressions */ - static NG_INLINE - std::array InfixOperatorBinding(TokenKind kind) { - switch (kind) { - case TokenKind::Dot: - return {16, 15}; // Property value operator - case TokenKind::Star: - case TokenKind::Slash: - case TokenKind::Percent: - return {13, 14}; - case TokenKind::Plus: - case TokenKind::Minus: - return {11, 12}; - case TokenKind::GreaterThan: - case TokenKind::LessThan: - return {9, 10}; - case TokenKind::EqualsEquals: - return {7, 8}; - case TokenKind::KeywordAnd: - return {5, 6}; - case TokenKind::KeywordOr: - return {3, 4}; - case TokenKind::AND: - case TokenKind::NOT: - case TokenKind::XOR: - case TokenKind::Pipe: - return {1, 2}; - default: - return {-1, -1}; - } - } - auto parse_expr(int previous_binding = -1) -> const SyntaxNode*; - auto match_expr() -> const SyntaxNode*; - - // ========= Parsing Language Constructs ========== - - auto parse_block() -> SyntaxBlock; - auto parse_type() -> SyntaxType*; - auto parse_function_param() -> ArenaVec; - auto parse_variable_assignment(const Token *name) -> const SyntaxNode*; - - auto parse_function(const Token *name) -> const SyntaxFunction*; - auto parse_struct_data_members() -> ArenaVec; - auto parse_function_call(const Token *name, const Token *delim) -> const SyntaxNode*; - - auto parse_for() -> const SyntaxNode*; - auto parse_match() -> const SyntaxNode*; - auto parse_match_pair_value() -> const SyntaxNode*; - - //========== INLINED FUNCTIONS ========== - - - template - auto parse_if() -> const SyntaxNode* { - auto cond = parse_expr(); - auto body = parse_block(); - - SyntaxElse *else_ = nullptr; SyntaxElif *elif = nullptr; - auto is_else = next(1); - if (is_else->getKind() == TokenKind::KeywordElse) { - else_ = new SyntaxElse(parse_block()); - } - else if (is_else->getKind() == TokenKind::KeywordElif) { - elif = (SyntaxElif*)((SyntaxNode*) parse_if()); - } - auto E = (SyntaxNode*) new SyntaxIf(cond, body, else_, elif); - if (ELIF) - E->kind = SyntaxKind::Elif; - return E; - } - - auto NG_INLINE - parse_while() { - auto cond = parse_expr(); - auto body = parse_block(); - return new SyntaxWhile(cond, body); - } - - auto NG_INLINE - parse_variable_value_assignment(const Token *name, SyntaxAssignmentOp op) { - auto ret = new SyntaxVariableValueAssignment(name, parse_expr(), op); - expect("Expected ';' after declaration"); - return ret; - } - - auto parse_struct(const Token *name) -> const SyntaxStruct* { - expect("Opening curly brace is required " - "before struct member declarations"); - return new SyntaxStruct(name, parse_struct_data_members()); - } - - auto parse_stmt() -> const SyntaxNode* { - switch (curr()->getKind()) { - case KeywordIf: - skip(1); - return parse_if(); - case KeywordWhile: - context = ParseContext::LoopBlock; - skip(1); - return parse_while(); - case KeywordFor: - context = ParseContext::LoopBlock; - skip(1); - return parse_for(); - case KeywordBreak: { - if (context != ParseContext::LoopBlock) { - // TODO: Add Error - } - break; - } - case KeywordContinue: { - if (context != ParseContext::LoopBlock) { - // TODO: Add Error - } - break; - } - case KeywordDefer: - break; - case KeywordNone: - break; - case KeywordReturn: - break; - case KeywordMatch: - break; - case Identifier: { - auto C1 = peek(1); - if (C1->getKind() == TokenKind::ColonEquals) { - auto name = curr(); - skip(2); - return parse_variable_assignment(name); - } - if (C1->isValueAssignmentOp()) { - auto name = curr(); - skip(2); - return parse_variable_value_assignment(name, - SyntaxVariableValueAssignment::MatchOp(C1->getKind())); - } - break; - } - case LCurlyBrace: - break; - case Then: - break; - case At: - break; - case Error: - break; - case EOFToken: - break; - default: break; - } - next(1); - return nullptr; - } - - }; - }} +namespace nextgen { + namespace jet { + using namespace nextgen::core; + + enum ParseErrorType { + ReservedIdentifierAsVariableName, + UnexpectedExpression, + InvalidToken, + ExpectedToken, + + MissingFunctionName, + MissingVariableName, + MissingForLoopVariable, + + ExpectedIdentifierForFunctionParameter, + ExpectedIdentifierForStructProperty, + InvalidTokenAfterIdentInGlobalScope, + + MissingClosingPair, + UnexpectedEndOfFile + }; + + enum ParseContext { + AnonymousBlock, + StatementBlock, + LoopBlock, + LambdaBlock, + GeneralScope + }; + + + struct ParserOutput { + ObjectVector functions; + ObjectVector structures; + ObjectVector global_variables; + }; + + struct ParseError { + union Metadata; + + ParseErrorType error; + TokenTraits::SourceLocation location; + std::initializer_list metadata; + + /** @brief Contains specific error information pertaining to the error + * message. Only a few parser errors have specific items, while most are + * invalid or missing tokens + * */ + union Metadata { + struct { + TokenKind expected; + Token const *got; + const char *message; + } expected_error; + TokenTraits::SourceLocation location; + TokenKind const misc_kind; + Token const *misc_tok; + const char *context_message; + + // GCC and CLANG let us get away with not using union constructors, + // but we need them for MSVC, and it's probably more clear anyway + Metadata(TokenTraits::SourceLocation loc) : location(loc) {} + + Metadata(TokenKind const kind) : misc_kind(kind) {} + + Metadata(Token const *tok) : misc_tok(tok) {} + + Metadata(const char *msg) : context_message(msg) {} + + Metadata(TokenKind const expected, Token const *got, const char + *message) { + expected_error.expected = expected; + expected_error.got = got; + expected_error.message = message; + } + }; + + ParseError(ParseErrorType error, TokenTraits::SourceLocation loc) + : error(error), location(loc) {} + + ParseError(ParseErrorType error, TokenTraits::SourceLocation loc, + const std::initializer_list &metadata) + : error(error), location(loc), metadata(metadata) {} + }; + + class Parser { + + // Parser's position + size_t position = 0; + + // Parsing context (valid statements per context scope) + // TODO: Set this up so stuff actually works with this... + ParseContext context = GeneralScope; + + // Parsing errors + Diagnostic diagnostics; + + // Encountered error that results in stopped compilation + char fatal = 0; + + + public: + ArenaVec tokens; + + Parser() = default; + + // Why are we passing a pointer here? Each pass in the frontend is done + // through the main function, thus, when we go ahead and pass the lexer, + // we know that it has the lifetime of the program. + explicit Parser(Lexer *lexer) + : tokens(lexer->lex()), diagnostics(lexer->diagnostics), + fatal(0) {} + + /// Parse all statements + auto parse() -> const ParserOutput; + + // ========= Parsing Language Generalizations ========== + auto NG_AINLINE curr() { return tokens[position]; } + + // ========= Parsing Utils ========== + + /** @brief Lookahead 'n' amount of times in the list */ + auto NG_AINLINE peek(size_t n) { + return tokens[position + n]; + } + + /** @brief Returns the current token and skips 'n' tokens ahead */ + auto NG_INLINE skip(size_t n) { + auto *ret = tokens[position]; + position += n; + return ret; + } + + /** @brief Skips 'n' tokens and returns the latest token after the skip */ + auto NG_INLINE next(size_t n) { + position += n; + return tokens[position]; + } + + /** @brief Asserts next token kind is 'TK' or errors with type 'PE' */ + template + auto skip() -> const Token * { + auto next = tokens[(position++)]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + PE, + next->getSourceLocation() + )); + } + return next; + } + + /** @brief Expect the next token to be 'kind', if not, build an error. */ + template + auto expect(char const (&msg)[N]) -> const Token * { + const Token *next = tokens[(position++)]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + ParseErrorType::ExpectedToken, + next->getSourceLocation(), + { + ParseError::Metadata{TK, next, msg} + } + )); + if (curr()->getKind() == EOFToken) + this->diagnostics.send_exception(); + else + fatal++; + } + return next; + } + + /** @brief Expected a closing delim in a statement or expression */ + template + void expect_delim(const TokenTraits::SourceLocation &loc) { + Token *next = tokens[position++]; + if (next->getKind() != TK) { + this->diagnostics.build(ParseError( + ParseErrorType::MissingClosingPair, + next->getSourceLocation(), + { + ParseError::Metadata{TK}, + ParseError::Metadata{next}, + ParseError::Metadata{loc} + } + )); + this->diagnostics.send_exception(); + } + } + + + /** @brief Unary operator precedence */ + static NG_INLINE int UnaryOperatorBinding(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + case TokenKind::Minus: + case TokenKind::ExclamationPoint: + case TokenKind::NOT: + return 17; + default: + return -1; + } + } + + /** @brief Infix operator binding for expressions */ + static NG_INLINE + std::array InfixOperatorBinding(TokenKind kind) { + switch (kind) { + case TokenKind::Dot: + return {16, 15}; // Property value operator + case TokenKind::Star: + case TokenKind::Slash: + case TokenKind::Percent: + return {13, 14}; + case TokenKind::Plus: + case TokenKind::Minus: + return {11, 12}; + case TokenKind::GreaterThan: + case TokenKind::LessThan: + return {9, 10}; + case TokenKind::EqualsEquals: + return {7, 8}; + case TokenKind::KeywordAnd: + return {5, 6}; + case TokenKind::KeywordOr: + return {3, 4}; + case TokenKind::AND: + case TokenKind::NOT: + case TokenKind::XOR: + case TokenKind::Pipe: + return {1, 2}; + default: + return {-1, -1}; + } + } + + auto parse_expr(int previous_binding = -1) -> const SyntaxNode *; + + auto match_expr() -> const SyntaxNode *; + + // ========= Parsing Language Constructs ========== + + auto parse_block() -> SyntaxBlock; + + auto parse_type() -> SyntaxType *; + + auto parse_function_param() -> ArenaVec; + + auto parse_variable_assignment(const Token *name) -> const SyntaxNode *; + + auto parse_function(const Token *name) -> const SyntaxFunction *; + + auto parse_struct_data_members() -> ArenaVec; + + auto parse_function_call(const Token *name, const Token *delim) -> const SyntaxNode *; + + auto parse_for() -> const SyntaxNode *; + + auto parse_match() -> const SyntaxNode *; + + auto parse_match_pair_value() -> const SyntaxNode *; + + //========== INLINED FUNCTIONS ========== + + + template + auto parse_if() -> const SyntaxNode * { + auto cond = parse_expr(); + auto body = parse_block(); + + SyntaxElse *else_ = nullptr; + SyntaxElif *elif = nullptr; + auto is_else = next(1); + if (is_else->getKind() == TokenKind::KeywordElse) { + else_ = new SyntaxElse(parse_block()); + } else if (is_else->getKind() == TokenKind::KeywordElif) { + elif = (SyntaxElif *) ((SyntaxNode *) parse_if()); + } + auto E = (SyntaxNode *) new SyntaxIf(cond, body, else_, elif); + if (ELIF) + E->kind = SyntaxKind::Elif; + return E; + } + + auto NG_INLINE + parse_while() { + auto cond = parse_expr(); + auto body = parse_block(); + return new SyntaxWhile(cond, body); + } + + auto NG_INLINE + parse_variable_value_assignment(const Token *name, SyntaxAssignmentOp op) { + auto ret = new SyntaxVariableValueAssignment(name, parse_expr(), op); + expect("Expected ';' after declaration"); + return ret; + } + + auto parse_struct(const Token *name) -> const SyntaxStruct * { + expect("Opening curly brace is required " + "before struct member declarations"); + return new SyntaxStruct(name, parse_struct_data_members()); + } + + auto parse_stmt() -> const SyntaxNode * { + switch (curr()->getKind()) { + case KeywordIf: + skip(1); + return parse_if(); + case KeywordWhile: + context = ParseContext::LoopBlock; + skip(1); + return parse_while(); + case KeywordFor: + context = ParseContext::LoopBlock; + skip(1); + return parse_for(); + case KeywordBreak: { + if (context != ParseContext::LoopBlock) { + // TODO: Add Error + } + break; + } + case KeywordContinue: { + if (context != ParseContext::LoopBlock) { + // TODO: Add Error + } + break; + } + case KeywordDefer: + break; + case KeywordNone: + break; + case KeywordReturn: + break; + case KeywordMatch: + break; + case Identifier: { + auto C1 = peek(1); + if (C1->getKind() == TokenKind::ColonEquals) { + auto name = curr(); + skip(2); + return parse_variable_assignment(name); + } + if (C1->isValueAssignmentOp()) { + auto name = curr(); + skip(2); + return parse_variable_value_assignment(name, + SyntaxVariableValueAssignment::MatchOp( + C1->getKind())); + } + break; + } + case LCurlyBrace: + break; + case Then: + break; + case At: + break; + case Error: + break; + case EOFToken: + break; + default: + break; + } + next(1); + return nullptr; + } + + }; + } +} # endif //JET_JET_PARSER_H \ No newline at end of file diff --git a/import/nextgen/jet/jet-syntax-nodes.h b/import/nextgen/jet/jet-syntax-nodes.h index 2e1ca56..c2b89a9 100644 --- a/import/nextgen/jet/jet-syntax-nodes.h +++ b/import/nextgen/jet/jet-syntax-nodes.h @@ -1,644 +1,649 @@ #ifndef JET_JET_SYNTAX_NODES_H #define JET_JET_SYNTAX_NODES_H + # include "jet-token.h" -namespace nextgen { namespace jet { - - enum SyntaxKind { - LiteralValue, - Binary, - Unary, - List, - If, - Else, - Elif, - For, - While, - FunctionCall, - Struct, - ForList, - ForRange, - Enum, - VariableAssignment, - VariableValueAssignment, - VariableReassignment, - ArrayInitialization, - Assert, - FunctionDefault, - FunctionStructProperty, - Match, - }; - - enum SyntaxUnaryOp { - BitwiseNot, - Positive, - Negative, - ConditionalNot, - MemoryReference, - Inc, - Dec - }; - - enum SyntaxBinaryOp { - Addition, - Subtraction, - Division, - Multiplication, - Modulus, - Greater, - Less, - GreaterEqualTo, - LessEqualTo, - BooleanOr, - BooleanAnd, - BinaryShLeft, - BinaryShRight, - BinaryXOR, - BinaryAND, - BinaryOR - }; - - enum SyntaxTypeAnnotation { - Optional, // ?int - Reference, // &int - Pointer, // *int - Box, // box int - ArrayType // [] int - }; - - enum SyntaxTypenameKind { - Integer8, - Integer16, - Integer32, - Integer64, - UInteger8, - UInteger16, - UInteger32, - UInteger64, - Boxed, - Float32, - Float64, - StringValue, - UserDefined - }; - - enum SyntaxAssignmentOp { - AddAssign, - SubAssign, - MulAssign, - DivAssign, - ModAssign, - PowAssign, - LShiftAssign, - RShiftAssign, - ORAssign, - XORAssign, - ANDAssign, - }; - - struct SyntaxTypename { - SyntaxTypenameKind kind; - - // When dealing with user-defined structs, when expanding the type - // we need to ensure that the name is recorded somewhere so that - // type-checking can proceed in the third pass of the compiler. - Option> user_defined_typename; - - SyntaxTypename() = default; - SyntaxTypename(SyntaxTypenameKind kind, - const Option> &optional) - : kind(kind), user_defined_typename(optional) - {} - - bool operator ==(const SyntaxTypename &other) const { - if ( - kind == SyntaxTypenameKind::UserDefined - && - other.kind == SyntaxTypenameKind::UserDefined - ) { - auto first = user_defined_typename.unwrap(); - auto second = other.user_defined_typename.unwrap(); - return strncmp(first.begin, second.begin, first.range()); - } - return other.kind == kind; - } - }; - - struct SyntaxNode { - SyntaxKind kind; - SyntaxNode() = default; - }; - - - struct SyntaxLiteral : public SyntaxNode { - - explicit SyntaxLiteral(const Token *l) : literal(l) {} - - - // contains the literal token value. Token's already contain the actual - // data for the literal, and therefore nothing else is needed. The token - // is required for any diagnostics that could be recorded as they contain - // source location & pretty printing options. - // - // Literal examples can be seen in the [Lexer] class. - const Token *literal; - }; - - - struct SyntaxBinary : public SyntaxNode { - - // Binary expression syntax operation. - // Ex: - // + : Addition - // - : Subtraction - // * : Multiplication - // / : Division - // ** : Exponential - // << : Left Shift - // >> : Right Shift - // ^ : XOR - // | : OR - // & : AND - const SyntaxBinaryOp operation; - const Token *op; - - // Left and right grouped expressions. - const SyntaxNode *lhs; - const SyntaxNode *rhs; - - /// Convert a `TokenKind` to `SyntaxBinaryOp` kind - static NG_INLINE const SyntaxBinaryOp MatchOp(TokenKind kind) { - switch(kind) { - case TokenKind::Plus: - return SyntaxBinaryOp::Addition; - case TokenKind::Minus: - return SyntaxBinaryOp::Subtraction; - case TokenKind::Star: - return SyntaxBinaryOp::Multiplication; - case TokenKind::Slash: - return SyntaxBinaryOp::Division; - case TokenKind::Percent: - return SyntaxBinaryOp::Modulus; - case TokenKind::AND: - return SyntaxBinaryOp::BinaryAND; - case TokenKind::Pipe: - return SyntaxBinaryOp::BinaryOR; - case TokenKind::XOR: - return SyntaxBinaryOp::BinaryXOR; - case TokenKind::LeftShift: - return SyntaxBinaryOp::BinaryShLeft; - case TokenKind::RightShift: - return SyntaxBinaryOp::BinaryShRight; - case TokenKind::GreaterThan: - return SyntaxBinaryOp::Greater; - case TokenKind::GreaterThanEquals: - return SyntaxBinaryOp::GreaterEqualTo; - case TokenKind::LessThan: - return SyntaxBinaryOp::Less; - case TokenKind::LessThanEquals: - return SyntaxBinaryOp::LessEqualTo; - case TokenKind::KeywordOr: - return SyntaxBinaryOp::BooleanOr; - case TokenKind::KeywordAnd: - return SyntaxBinaryOp::BooleanAnd; - default: - UNREACHABLE; - } - } - - SyntaxBinary(const SyntaxBinaryOp op, - const Token *oo, - const SyntaxNode *lhs, - const SyntaxNode*rhs) - : operation(op), op(oo), lhs(lhs), rhs(rhs) {} - }; - - struct SyntaxType : public SyntaxNode { - SyntaxType() = default; - SyntaxType(const Option &modifier, - const Option &ty, - const SyntaxType *type) - : modifier(modifier), ty_name(ty), type(type) {} - - - const Option modifier = None; - const Option ty_name = None; - const SyntaxType *type = nullptr; - - static NG_INLINE SyntaxTypename MatchTypename(const Token *kind) { - switch (kind->getKind()) { - case Typename_str: - return SyntaxTypename(SyntaxTypenameKind::StringValue, None); - case Typename_i8: - return SyntaxTypename(SyntaxTypenameKind::Integer8, None); - case Typename_i16: - return SyntaxTypename(SyntaxTypenameKind::Integer16, None); - case Typename_i32: - return SyntaxTypename(SyntaxTypenameKind::Integer32, None); - case Typename_i64: - return SyntaxTypename(SyntaxTypenameKind::Integer64, None); - case Typename_u8: - return SyntaxTypename(SyntaxTypenameKind::UInteger8, None); - case Typename_u16: - return SyntaxTypename(SyntaxTypenameKind::UInteger16, None); - case Typename_u32: - return SyntaxTypename(SyntaxTypenameKind::UInteger32, None); - case Typename_u64: - return SyntaxTypename(SyntaxTypenameKind::UInteger64, None); - case Typename_box: - return SyntaxTypename(SyntaxTypenameKind::Boxed, None); - case Identifier: - return SyntaxTypename(SyntaxTypenameKind::UserDefined, - (Range)kind->name()); - default: - UNREACHABLE; - } - } - - - bool has_typename() const { - return ty_name.isSome(); - } - - bool has_modifier() const { - return modifier.isSome(); - } - - bool has_type() const { - return type; - } - - bool operator ==(const SyntaxType &other) const { - auto ret = (other.has_typename() && has_typename()) - && (other.has_modifier() && has_modifier()) - && (other.has_type()); - if (ret) { - return (*type) == (*other.type) - && ty_name.contains(other.ty_name.unwrap()) - && modifier.contains(other.modifier.unwrap()); - } - return ret; - } - - const SyntaxTypename get_typename() const { - if (!has_typename()) { - type->get_typename(); - } - return ty_name.unwrap(); - } - - - // *int = SyntaxType { modifier: Pointer, ty_name: None, - // type: SyntaxType { - // modifier: None, - // ty_name: Integer32, - // type: None - // } - - }; - - struct SyntaxList : public SyntaxNode { - ObjectVector values; - }; - - struct SyntaxVariableValueAssignment : public SyntaxNode { - const Token *name = nullptr; - const SyntaxNode *expression = nullptr; - const SyntaxAssignmentOp op; - - static NG_INLINE SyntaxAssignmentOp MatchOp(TokenKind kind) { - switch(kind) { - case PlusEquals: - return SyntaxAssignmentOp::AddAssign; - case MinusEquals: - return SyntaxAssignmentOp::SubAssign; - case DivEquals: - return SyntaxAssignmentOp::DivAssign; - case PowEquals: - return SyntaxAssignmentOp::PowAssign; - case MulEquals: - return SyntaxAssignmentOp::MulAssign; - case LeftShiftEquals: - return SyntaxAssignmentOp::LShiftAssign; - case RightShiftEquals: - return SyntaxAssignmentOp::RShiftAssign; - case ANDEquals: - return SyntaxAssignmentOp::ANDAssign; - case XOREquals: - return SyntaxAssignmentOp::XORAssign; - case OREquals: - return SyntaxAssignmentOp::ORAssign; - case PercentEquals: - return SyntaxAssignmentOp::ModAssign; - default: - break; - } - } - - SyntaxVariableValueAssignment(const Token *name, - const SyntaxNode *expr, - const SyntaxAssignmentOp op) - : name(name), expression(expr), op(op) - { this->kind = SyntaxKind::VariableAssignment; } - - }; - - struct SyntaxVariableAssignment : public SyntaxNode { - - SyntaxVariableAssignment(const Token *name, - const Option &ty, - const SyntaxNode*expr) - : name(name), type(ty), expression(expr) { - this->kind = SyntaxKind::VariableAssignment; - } - - // The actual variable name; ex: var my_variable_name = ... - // ^^^^^^^^^^^^^^^^ - const Token *name = nullptr; - - // Optional variable type. It may need to be inferred so it can be set - // to 'None'. ex: (var ident: int) = 23 OR (var ident = 23) - // ^^^ ^ No type here - Option type = None; - - // Node representing the actual value that the variable contains - const SyntaxNode *expression; - }; - - - struct SyntaxVariableReassignment : public SyntaxNode { - - SyntaxVariableReassignment(const Token *v, const SyntaxNode *expr) - : variable(v), expression(expr) {} - - // Identifier representing the variable that is being reassigned. - // Ex: var some_var = 23 - // ^^^^^^^^ First declared here - // some_var = 242 - // ^^^^^^^^ Reassigned here. Note: You cannot reassign variables with - // values that are not the same as its first initialization. For example, - // the variable `some_var` is declared as an 'int' and cannot be - // reassigned with a value of 'str'. - const Token *variable = nullptr; - - // Node representing the actual value that the variable contains - const SyntaxNode *expression = nullptr; - }; - - struct SyntaxBlock : public SyntaxNode { - - // A sequential list of statements categorized as a current node. - // Ex: - // { - // x := 23 - // x = x ** x - // a_function_call(x) - // } - // There are 3 statements here, and the current will contain 3 statements. - // Note: There is an exception to parsing of this node, in statements such - // as 'while', 'if', 'elif', 'else', 'for' may contain a single statement - // after them without a set of corresponding '{}'. - ObjectVector statements; - - - // Special statement for function / lambda return check. Allows for void - // function type inference. - bool contains_return = false; - }; - - struct SyntaxElse : public SyntaxNode { - - explicit SyntaxElse(SyntaxBlock block) : body(block) {} - - - // Else statement current in an if-elif-else statement chain - // Ex: - // if cond { ... } - // else { ... } - // ^^^^ Declared here. Note that the curly braces may be omitted for one - // statement bodies. - SyntaxBlock body; - }; - - struct SyntaxElif : public SyntaxNode { - - explicit SyntaxElif(SyntaxBlock block) : body(block) {} - - - // Elif statement current in an if-elif-else statement chain - // Ex: - // if cond { ... } - // elif cond { ... } - // ^^^^ Declared here. Note that the curly braces may be omitted for one - // statement bodies. - SyntaxBlock body; - }; - - struct SyntaxWhile : public SyntaxNode { - // Boolean expression representing the while condition. - // Ex: while cond - // ^^^^ Declared here - const SyntaxNode *condition; - // Statement list of commands executed upon the condition above being true - // Ex: while cond { - // variable := "Hello" - // } - SyntaxBlock body; - - SyntaxWhile(const SyntaxNode *cond, const SyntaxBlock &body) - : condition(cond), body(body) { - this->kind = SyntaxKind::While; - } - }; - - struct SyntaxMatch : public SyntaxNode { - - struct MatchPair { - const SyntaxNode *value; - SyntaxBlock action; - }; - - const SyntaxNode *value; - ArenaVec pairs; - }; - - struct SyntaxIf : public SyntaxNode { - - // Boolean expression representing the if condition. - // Ex: if cond - // ^^^^ Declared here - const SyntaxNode *condition; - - // Statement list of commands executed upon the condition above being true - // Ex: if cond { - // flag = true - // } - SyntaxBlock body; - - // See `SyntaxElse` - const SyntaxElse *else_ = nullptr; - - // See `SyntaxElif` - const SyntaxNode *elif = nullptr; - - SyntaxIf(const SyntaxNode *cond, - const SyntaxBlock &body, - const SyntaxElse *else_, - const SyntaxNode *elif) - : condition(cond), body(body), else_(else_), elif(elif) { - this->kind = SyntaxKind::If; - } - }; - - struct SyntaxUnary : public SyntaxNode { - - // Unary expression. We denote a few unary operators: - // ++ : Increment and return latest value - // -- : Decrement and return latest value - // & : Take the memory address of the expression on the right - // * : Dereference the expression on the right - // + : Assert RHS expression is positive - // - : Negate RHS expression - // ~ : Bitwise NOT - const SyntaxUnaryOp operation; - const Token *op; - - // Expression to apply unary operation - // Ex: ++some_var, &memory_address, *pointer_value, -some_value - const SyntaxNode *expression = nullptr; - - /// Convert `TokenKind` to `SyntaxUnaryOp` operation - static NG_INLINE const SyntaxUnaryOp MatchOp(TokenKind kind) { - switch(kind) { - case TokenKind::Plus: - return SyntaxUnaryOp::Positive; - case TokenKind::NOT: - return SyntaxUnaryOp::BitwiseNot; - case TokenKind::Minus: - return SyntaxUnaryOp::Negative; - case TokenKind::PlusPlus: - break; - case TokenKind::MinusMinus: - break; - case TokenKind::ExclamationPoint: - return SyntaxUnaryOp::ConditionalNot; - default: - UNREACHABLE; - } - UNREACHABLE; - } - - SyntaxUnary(const SyntaxUnaryOp op, - const Token *oo, - const SyntaxNode *expr) - : operation(op), op(oo), expression(expr) { - this->kind = SyntaxKind::Unary; - } - }; - - struct SyntaxFunctionParameter : public SyntaxNode { - - SyntaxFunctionParameter(const Token *pn, SyntaxType *ty) - : param_name(pn), type(ty) {} - - // Function parameter name - const Token *param_name = nullptr; - - // Function Parameter type. If no type is given, then it is inferred to - // be a generic parameter. - // Ex: - // fn some_func(param_1: int, param_2: str, $param_3) - // ^^^^^^^^ Generic Parameter - const SyntaxType *type; - }; - - struct SyntaxFunction : public SyntaxNode { - - // Function name - const Token *function_name = nullptr; - - // Function return type. If 'None' then infer return type. - Option function_type; - - // Function body - SyntaxBlock body; - - // Function parameters - ArenaVec parameters{nullptr, nullptr}; - - SyntaxFunction(const Token *name, - const Option &ty, - const SyntaxBlock &body, - const ArenaVec ¶ms) - : function_name(name), body(body), function_type(ty), - parameters(params){} - }; - - struct SyntaxFunctionCall : public SyntaxNode { - - // NOTE: Function Declarations should be lowered - // before function call declarations. - // - // Ex: - // some_function(1, 2, 2, 3, 5) - const Token *function_name = nullptr; - ObjectVector parameters; - - SyntaxFunctionCall(const Token *name) - : function_name(name) - { this->kind = SyntaxKind::FunctionCall; } - }; - - struct SyntaxForList : public SyntaxNode { - SyntaxForList(const Token *ln, const Token *lv, SyntaxBlock body) - : list_name(ln), list_var(lv), body(body) {} - - // For loop iteration over a variable of type list. - // Ex: - // for num in numbers { ... } - // ^^^ Loop var ^^^^^^^ List variable - const Token *list_name = nullptr; - const Token *list_var = nullptr; - SyntaxBlock body; - }; - - struct RangeFunction { - size_t set[3] = {0, 0, 0}; - }; - - struct SyntaxForRange : public SyntaxNode { - SyntaxForRange(const Token *list_var, const RangeFunction - range, SyntaxBlock body) : - list_var(list_var), range(range), body(body) {} - - // Range based for loop. A range consists of a few factors: - // start, end, and step. - // Start - the value that initializes the loop variable. - // End - the last value the variable should loop to. - // Step - is how much a loop variable should change. A step of 2, for - // example, will increment the loop variable by 2 each time. - - // Ex: - // for i in range(1, 5) { ... } OR - // for i in range(1, 232, 3) { ... } - const Token *list_var = nullptr; - const RangeFunction range; - SyntaxBlock body; - }; - - struct SyntaxStructMember { - const Token *name = nullptr; - SyntaxType *type; - SyntaxStructMember(const Token *name, SyntaxType *ty) - : name(name), type(ty) {} - }; - - struct SyntaxStruct : public SyntaxNode { - const Token *name = nullptr; - ArenaVec members; - SyntaxStruct(const Token *name, const ArenaVec - &members) : name(name), members(members) {} - }; - -}} +namespace nextgen { + namespace jet { + + enum SyntaxKind { + LiteralValue, + Binary, + Unary, + List, + If, + Else, + Elif, + For, + While, + FunctionCall, + Struct, + ForList, + ForRange, + Enum, + VariableAssignment, + VariableValueAssignment, + VariableReassignment, + ArrayInitialization, + Assert, + FunctionDefault, + FunctionStructProperty, + Match, + }; + + enum SyntaxUnaryOp { + BitwiseNot, + Positive, + Negative, + ConditionalNot, + MemoryReference, + Inc, + Dec + }; + + enum SyntaxBinaryOp { + Addition, + Subtraction, + Division, + Multiplication, + Modulus, + Greater, + Less, + GreaterEqualTo, + LessEqualTo, + BooleanOr, + BooleanAnd, + BinaryShLeft, + BinaryShRight, + BinaryXOR, + BinaryAND, + BinaryOR + }; + + enum SyntaxTypeAnnotation { + Optional, // ?int + Reference, // &int + Pointer, // *int + Box, // box int + ArrayType // [] int + }; + + enum SyntaxTypenameKind { + Integer8, + Integer16, + Integer32, + Integer64, + UInteger8, + UInteger16, + UInteger32, + UInteger64, + Boxed, + Float32, + Float64, + StringValue, + UserDefined + }; + + enum SyntaxAssignmentOp { + AddAssign, + SubAssign, + MulAssign, + DivAssign, + ModAssign, + PowAssign, + LShiftAssign, + RShiftAssign, + ORAssign, + XORAssign, + ANDAssign, + }; + + struct SyntaxTypename { + SyntaxTypenameKind kind; + + // When dealing with user-defined structs, when expanding the type + // we need to ensure that the name is recorded somewhere so that + // type-checking can proceed in the third pass of the compiler. + Option > user_defined_typename; + + SyntaxTypename() = default; + + SyntaxTypename(SyntaxTypenameKind kind, + const Option > &optional) + : kind(kind), user_defined_typename(optional) {} + + bool operator==(const SyntaxTypename &other) const { + if ( + kind == SyntaxTypenameKind::UserDefined + && + other.kind == SyntaxTypenameKind::UserDefined + ) { + auto first = user_defined_typename.unwrap(); + auto second = other.user_defined_typename.unwrap(); + return strncmp(first.begin, second.begin, first.range()); + } + return other.kind == kind; + } + }; + + struct SyntaxNode { + SyntaxKind kind; + + SyntaxNode() = default; + }; + + + struct SyntaxLiteral : public SyntaxNode { + + explicit SyntaxLiteral(const Token *l) : literal(l) {} + + + // contains the literal token value. Token's already contain the actual + // data for the literal, and therefore nothing else is needed. The token + // is required for any diagnostics that could be recorded as they contain + // source location & pretty printing options. + // + // Literal examples can be seen in the [Lexer] class. + const Token *literal; + }; + + + struct SyntaxBinary : public SyntaxNode { + + // Binary expression syntax operation. + // Ex: + // + : Addition + // - : Subtraction + // * : Multiplication + // / : Division + // ** : Exponential + // << : Left Shift + // >> : Right Shift + // ^ : XOR + // | : OR + // & : AND + const SyntaxBinaryOp operation; + const Token *op; + + // Left and right grouped expressions. + const SyntaxNode *lhs; + const SyntaxNode *rhs; + + /// Convert a `TokenKind` to `SyntaxBinaryOp` kind + static NG_INLINE const SyntaxBinaryOp MatchOp(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + return SyntaxBinaryOp::Addition; + case TokenKind::Minus: + return SyntaxBinaryOp::Subtraction; + case TokenKind::Star: + return SyntaxBinaryOp::Multiplication; + case TokenKind::Slash: + return SyntaxBinaryOp::Division; + case TokenKind::Percent: + return SyntaxBinaryOp::Modulus; + case TokenKind::AND: + return SyntaxBinaryOp::BinaryAND; + case TokenKind::Pipe: + return SyntaxBinaryOp::BinaryOR; + case TokenKind::XOR: + return SyntaxBinaryOp::BinaryXOR; + case TokenKind::LeftShift: + return SyntaxBinaryOp::BinaryShLeft; + case TokenKind::RightShift: + return SyntaxBinaryOp::BinaryShRight; + case TokenKind::GreaterThan: + return SyntaxBinaryOp::Greater; + case TokenKind::GreaterThanEquals: + return SyntaxBinaryOp::GreaterEqualTo; + case TokenKind::LessThan: + return SyntaxBinaryOp::Less; + case TokenKind::LessThanEquals: + return SyntaxBinaryOp::LessEqualTo; + case TokenKind::KeywordOr: + return SyntaxBinaryOp::BooleanOr; + case TokenKind::KeywordAnd: + return SyntaxBinaryOp::BooleanAnd; + default: + UNREACHABLE; + } + } + + SyntaxBinary(const SyntaxBinaryOp op, + const Token *oo, + const SyntaxNode *lhs, + const SyntaxNode *rhs) + : operation(op), op(oo), lhs(lhs), rhs(rhs) {} + }; + + struct SyntaxType : public SyntaxNode { + SyntaxType() = default; + + SyntaxType(const Option &modifier, + const Option &ty, + const SyntaxType *type) + : modifier(modifier), ty_name(ty), type(type) {} + + + const Option modifier = None; + const Option ty_name = None; + const SyntaxType *type = nullptr; + + static NG_INLINE SyntaxTypename MatchTypename(const Token *kind) { + switch (kind->getKind()) { + case Typename_str: + return SyntaxTypename(SyntaxTypenameKind::StringValue, None); + case Typename_i8: + return SyntaxTypename(SyntaxTypenameKind::Integer8, None); + case Typename_i16: + return SyntaxTypename(SyntaxTypenameKind::Integer16, None); + case Typename_i32: + return SyntaxTypename(SyntaxTypenameKind::Integer32, None); + case Typename_i64: + return SyntaxTypename(SyntaxTypenameKind::Integer64, None); + case Typename_u8: + return SyntaxTypename(SyntaxTypenameKind::UInteger8, None); + case Typename_u16: + return SyntaxTypename(SyntaxTypenameKind::UInteger16, None); + case Typename_u32: + return SyntaxTypename(SyntaxTypenameKind::UInteger32, None); + case Typename_u64: + return SyntaxTypename(SyntaxTypenameKind::UInteger64, None); + case Typename_box: + return SyntaxTypename(SyntaxTypenameKind::Boxed, None); + case Identifier: + return SyntaxTypename(SyntaxTypenameKind::UserDefined, + (Range) kind->name()); + default: + UNREACHABLE; + } + } + + + bool has_typename() const { + return ty_name.isSome(); + } + + bool has_modifier() const { + return modifier.isSome(); + } + + bool has_type() const { + return type; + } + + bool operator==(const SyntaxType &other) const { + auto ret = (other.has_typename() && has_typename()) + && (other.has_modifier() && has_modifier()) + && (other.has_type()); + if (ret) { + return (*type) == (*other.type) + && ty_name.contains(other.ty_name.unwrap()) + && modifier.contains(other.modifier.unwrap()); + } + return ret; + } + + const SyntaxTypename get_typename() const { + if (!has_typename()) { + type->get_typename(); + } + return ty_name.unwrap(); + } + + + // *int = SyntaxType { modifier: Pointer, ty_name: None, + // type: SyntaxType { + // modifier: None, + // ty_name: Integer32, + // type: None + // } + + }; + + struct SyntaxList : public SyntaxNode { + ObjectVector values; + }; + + struct SyntaxVariableValueAssignment : public SyntaxNode { + const Token *name = nullptr; + const SyntaxNode *expression = nullptr; + const SyntaxAssignmentOp op; + + static NG_INLINE SyntaxAssignmentOp MatchOp(TokenKind kind) { + switch (kind) { + case PlusEquals: + return SyntaxAssignmentOp::AddAssign; + case MinusEquals: + return SyntaxAssignmentOp::SubAssign; + case DivEquals: + return SyntaxAssignmentOp::DivAssign; + case PowEquals: + return SyntaxAssignmentOp::PowAssign; + case MulEquals: + return SyntaxAssignmentOp::MulAssign; + case LeftShiftEquals: + return SyntaxAssignmentOp::LShiftAssign; + case RightShiftEquals: + return SyntaxAssignmentOp::RShiftAssign; + case ANDEquals: + return SyntaxAssignmentOp::ANDAssign; + case XOREquals: + return SyntaxAssignmentOp::XORAssign; + case OREquals: + return SyntaxAssignmentOp::ORAssign; + case PercentEquals: + return SyntaxAssignmentOp::ModAssign; + default: + break; + } + } + + SyntaxVariableValueAssignment(const Token *name, + const SyntaxNode *expr, + const SyntaxAssignmentOp op) + : name(name), expression(expr), op(op) { this->kind = SyntaxKind::VariableAssignment; } + + }; + + struct SyntaxVariableAssignment : public SyntaxNode { + + SyntaxVariableAssignment(const Token *name, + const Option &ty, + const SyntaxNode *expr) + : name(name), type(ty), expression(expr) { + this->kind = SyntaxKind::VariableAssignment; + } + + // The actual variable name; ex: var my_variable_name = ... + // ^^^^^^^^^^^^^^^^ + const Token *name = nullptr; + + // Optional variable type. It may need to be inferred so it can be set + // to 'None'. ex: (var ident: int) = 23 OR (var ident = 23) + // ^^^ ^ No type here + Option type = None; + + // Node representing the actual value that the variable contains + const SyntaxNode *expression; + }; + + + struct SyntaxVariableReassignment : public SyntaxNode { + + SyntaxVariableReassignment(const Token *v, const SyntaxNode *expr) + : variable(v), expression(expr) {} + + // Identifier representing the variable that is being reassigned. + // Ex: var some_var = 23 + // ^^^^^^^^ First declared here + // some_var = 242 + // ^^^^^^^^ Reassigned here. Note: You cannot reassign variables with + // values that are not the same as its first initialization. For example, + // the variable `some_var` is declared as an 'int' and cannot be + // reassigned with a value of 'str'. + const Token *variable = nullptr; + + // Node representing the actual value that the variable contains + const SyntaxNode *expression = nullptr; + }; + + struct SyntaxBlock : public SyntaxNode { + + // A sequential list of statements categorized as a current node. + // Ex: + // { + // x := 23 + // x = x ** x + // a_function_call(x) + // } + // There are 3 statements here, and the current will contain 3 statements. + // Note: There is an exception to parsing of this node, in statements such + // as 'while', 'if', 'elif', 'else', 'for' may contain a single statement + // after them without a set of corresponding '{}'. + ObjectVector statements; + + + // Special statement for function / lambda return check. Allows for void + // function type inference. + bool contains_return = false; + }; + + struct SyntaxElse : public SyntaxNode { + + explicit SyntaxElse(SyntaxBlock block) : body(block) {} + + + // Else statement current in an if-elif-else statement chain + // Ex: + // if cond { ... } + // else { ... } + // ^^^^ Declared here. Note that the curly braces may be omitted for one + // statement bodies. + SyntaxBlock body; + }; + + struct SyntaxElif : public SyntaxNode { + + explicit SyntaxElif(SyntaxBlock block) : body(block) {} + + + // Elif statement current in an if-elif-else statement chain + // Ex: + // if cond { ... } + // elif cond { ... } + // ^^^^ Declared here. Note that the curly braces may be omitted for one + // statement bodies. + SyntaxBlock body; + }; + + struct SyntaxWhile : public SyntaxNode { + // Boolean expression representing the while condition. + // Ex: while cond + // ^^^^ Declared here + const SyntaxNode *condition; + // Statement list of commands executed upon the condition above being true + // Ex: while cond { + // variable := "Hello" + // } + SyntaxBlock body; + + SyntaxWhile(const SyntaxNode *cond, const SyntaxBlock &body) + : condition(cond), body(body) { + this->kind = SyntaxKind::While; + } + }; + + struct SyntaxMatch : public SyntaxNode { + + struct MatchPair { + const SyntaxNode *value; + SyntaxBlock action; + }; + + const SyntaxNode *value; + ArenaVec pairs; + }; + + struct SyntaxIf : public SyntaxNode { + + // Boolean expression representing the if condition. + // Ex: if cond + // ^^^^ Declared here + const SyntaxNode *condition; + + // Statement list of commands executed upon the condition above being true + // Ex: if cond { + // flag = true + // } + SyntaxBlock body; + + // See `SyntaxElse` + const SyntaxElse *else_ = nullptr; + + // See `SyntaxElif` + const SyntaxNode *elif = nullptr; + + SyntaxIf(const SyntaxNode *cond, + const SyntaxBlock &body, + const SyntaxElse *else_, + const SyntaxNode *elif) + : condition(cond), body(body), else_(else_), elif(elif) { + this->kind = SyntaxKind::If; + } + }; + + struct SyntaxUnary : public SyntaxNode { + + // Unary expression. We denote a few unary operators: + // ++ : Increment and return latest value + // -- : Decrement and return latest value + // & : Take the memory address of the expression on the right + // * : Dereference the expression on the right + // + : Assert RHS expression is positive + // - : Negate RHS expression + // ~ : Bitwise NOT + const SyntaxUnaryOp operation; + const Token *op; + + // Expression to apply unary operation + // Ex: ++some_var, &memory_address, *pointer_value, -some_value + const SyntaxNode *expression = nullptr; + + /// Convert `TokenKind` to `SyntaxUnaryOp` operation + static NG_INLINE const SyntaxUnaryOp MatchOp(TokenKind kind) { + switch (kind) { + case TokenKind::Plus: + return SyntaxUnaryOp::Positive; + case TokenKind::NOT: + return SyntaxUnaryOp::BitwiseNot; + case TokenKind::Minus: + return SyntaxUnaryOp::Negative; + case TokenKind::PlusPlus: + break; + case TokenKind::MinusMinus: + break; + case TokenKind::ExclamationPoint: + return SyntaxUnaryOp::ConditionalNot; + default: + UNREACHABLE; + } + UNREACHABLE; + } + + SyntaxUnary(const SyntaxUnaryOp op, + const Token *oo, + const SyntaxNode *expr) + : operation(op), op(oo), expression(expr) { + this->kind = SyntaxKind::Unary; + } + }; + + struct SyntaxFunctionParameter : public SyntaxNode { + + SyntaxFunctionParameter(const Token *pn, SyntaxType *ty) + : param_name(pn), type(ty) {} + + // Function parameter name + const Token *param_name = nullptr; + + // Function Parameter type. If no type is given, then it is inferred to + // be a generic parameter. + // Ex: + // fn some_func(param_1: int, param_2: str, $param_3) + // ^^^^^^^^ Generic Parameter + const SyntaxType *type; + }; + + struct SyntaxFunction : public SyntaxNode { + + // Function name + const Token *function_name = nullptr; + + // Function return type. If 'None' then infer return type. + Option function_type; + + // Function body + SyntaxBlock body; + + // Function parameters + ArenaVec parameters{nullptr, nullptr}; + + SyntaxFunction(const Token *name, + const Option &ty, + const SyntaxBlock &body, + const ArenaVec ¶ms) + : function_name(name), body(body), function_type(ty), + parameters(params) {} + }; + + struct SyntaxFunctionCall : public SyntaxNode { + + // NOTE: Function Declarations should be lowered + // before function call declarations. + // + // Ex: + // some_function(1, 2, 2, 3, 5) + const Token *function_name = nullptr; + ObjectVector parameters; + + SyntaxFunctionCall(const Token *name) + : function_name(name) { this->kind = SyntaxKind::FunctionCall; } + }; + + struct SyntaxForList : public SyntaxNode { + SyntaxForList(const Token *ln, const Token *lv, SyntaxBlock body) + : list_name(ln), list_var(lv), body(body) {} + + // For loop iteration over a variable of type list. + // Ex: + // for num in numbers { ... } + // ^^^ Loop var ^^^^^^^ List variable + const Token *list_name = nullptr; + const Token *list_var = nullptr; + SyntaxBlock body; + }; + + struct RangeFunction { + size_t set[3] = {0, 0, 0}; + }; + + struct SyntaxForRange : public SyntaxNode { + SyntaxForRange(const Token *list_var, const RangeFunction + range, SyntaxBlock body) : + list_var(list_var), range(range), body(body) {} + + // Range based for loop. A range consists of a few factors: + // start, end, and step. + // Start - the value that initializes the loop variable. + // End - the last value the variable should loop to. + // Step - is how much a loop variable should change. A step of 2, for + // example, will increment the loop variable by 2 each time. + + // Ex: + // for i in range(1, 5) { ... } OR + // for i in range(1, 232, 3) { ... } + const Token *list_var = nullptr; + const RangeFunction range; + SyntaxBlock body; + }; + + struct SyntaxStructMember { + const Token *name = nullptr; + SyntaxType *type; + + SyntaxStructMember(const Token *name, SyntaxType *ty) + : name(name), type(ty) {} + }; + + struct SyntaxStruct : public SyntaxNode { + const Token *name = nullptr; + ArenaVec members; + + SyntaxStruct(const Token *name, const ArenaVec + &members) : name(name), members(members) {} + }; + + } +} #endif //JET_JET_SYNTAX_NODES_H diff --git a/import/nextgen/jet/jet-token.h b/import/nextgen/jet/jet-token.h index 960be9f..51739ff 100644 --- a/import/nextgen/jet/jet-token.h +++ b/import/nextgen/jet/jet-token.h @@ -1,534 +1,537 @@ # ifndef NEXTGEN_TOKEN_H # define NEXTGEN_TOKEN_H + # include # include -namespace nextgen { namespace jet { - - enum TokenClassification : unsigned short { - Function = 1 << 2, - Assignment = 1 << 3, - Unsigned = 1 << 4, - Keyword = 1 << 5, - Literal = 1 << 6, - }; - - enum TokenKind { - - // Primitives --------------------------- - Integer = 1, - Zero = 1, - String, - Decimal, - Boolean, - //--------------------------------------- - - // Valid Expression Tokens (DO NOT CHANGE ORDER) --------------------------- - Identifier, // [a-z] && [A-Z] && _ - LessThan, // < - GreaterThan, // > - - Plus, // + - Minus, // - - Slash, // / - Star, // * - XOR, // ^ - AND, // & - NOT, // ~ - Pipe, // | - Percent,// % - Char, // \' - Dot, // . - - // TODO: Fix this ... Should not be apart of binary expression. It should - // be apart of the assignment modifier. - PlusEquals, - MinusEquals, - DivEquals, - PowEquals, - MulEquals, - ANDEquals, - XOREquals, - OREquals, - PercentEquals, - LeftShiftEquals, - RightShiftEquals, - GreaterThanEquals, - LessThanEquals, - - EqualsEquals, // == - LeftShift, // << - RightShift, // >> - Pow, // ** - //--------------------------------------- - - - // Keywords --------------------------- - KeywordIf, - KeywordElse, - KeywordElif, - KeywordWhile, - KeywordFor, - KeywordAnd, - KeywordOr, - KeywordBreak, - KeywordContinue, - KeywordDefer, - KeywordStruct, - KeywordEnum, - KeywordExport, - KeywordExtern, - KeywordFunction, - KeywordError, - KeywordVar, - KeywordTrue, - KeywordFalse, - KeywordNone, - KeywordReturn, - KeywordUnion, - KeywordMatch, - KeywordIn, - KeywordRange, - //--------------------------------------- - - // Typename --------------------------- - Typename_str, - Typename_i8, - Typename_i16, - Typename_i32, - Typename_i64, - Typename_u8, - Typename_u16, - Typename_u32, - Typename_u64, - Typename_box, - Typename_f32, - Typename_f64, - //--------------------------------------- - - // File Control --------------------------- - NewLine, - Whitespace, - StringEscapeStart, - //--------------------------------------- - - LParenthesis, // ( - RParenthesis, // ) - LCurlyBrace, // { - RCurlyBrace, // } - LBracket, // [ - RBracket, // ] - Colon, // : - Equals, // = - QuestionMark, // ? - Then, // ?? - At, // @ - SemiColon, // ; - ExclamationPoint, // ! - Comma, // , - - RangeSpan, // .. - Ellipsis, // ... - - PlusPlus, // ++ - MinusMinus, // -- - FunctionArrow,// => - - ColonEquals, // := - Arrow, // -> - - Error, - EOFToken, - }; - - struct TokenTraits { - - using SourceSpan = Range; - - struct SourceLocation { - size_t line; - size_t column; - SourceSpan source; - }; - - // We have to very careful here. We do not have access to std::variant<...> - // in C++11, so we must use a controlled union to get a Token's value. - struct Value { - union { - decltype(UINTPTR_MAX) integer; - bool boolean; - double float64; - char character; - }; - }; - - }; - - class Token { - private: - using TokenValue = TokenTraits::Value; - using SourceLocation = TokenTraits::SourceLocation; - - str id; - - unsigned flags = TokenClassification::Literal; - SourceLocation loc = {}; - - - - TokenKind kind; - public: - Token() = default; - - union { - decltype(UINTPTR_MAX) integer; - bool boolean; - double float64; - char character; - }; - - template - Token(const str &id, SourceLocation loc, T value, - TokenKind kind, unsigned flags = -1) - : id(id), flags(flags), loc(loc), kind(kind) { - setValue(value); - } - - // Note: This is only for lex errors. Failed tokens are important - // for the diagnostic class generate valid errors. - Token(str id, SourceLocation loc, TokenKind kind) - : id(id), loc(loc) { - setKind(kind); - } - - static NG_INLINE char const *GetTokenKindName(TokenKind const kind) { - switch(kind){ - case LessThan: - return "<"; - case GreaterThan: - return ">"; - case Plus: - return "+"; - case Minus: - return "-"; - case Slash: - return "/"; - case Star: - return "*"; - case XOR: - return "^"; - case AND: - return "&"; - case NOT: - return "~"; - case Pipe: - return "|"; - case Percent: - return "%"; - case Char: - return "'"; - case Dot: - return "."; - case PlusEquals: - return "+="; - case MinusEquals: - return "-="; - case DivEquals: - return "/="; - case PowEquals: - return "**="; - case MulEquals: - return "*="; - case GreaterThanEquals: - return ">="; - case LessThanEquals: - return "<="; - case LeftShiftEquals: - return "<<="; - case RightShiftEquals: - return ">>="; - case ANDEquals: - return "&="; - case XOREquals: - return "^="; - case OREquals: - return "|="; - case PercentEquals: - return "%="; - case EqualsEquals: - return "=="; - case LeftShift: - return "<<"; - case RightShift: - return ">>"; - case Pow: - return "**"; - case KeywordIf: - return "if"; - case KeywordElse: - return "else"; - case KeywordElif: - return "elif"; - case KeywordWhile: - return "while"; - case KeywordFor: - return "for"; - case KeywordAnd: - return "and"; - case KeywordOr: - return "or"; - case KeywordBreak: - return "break"; - case KeywordContinue: - return "continue"; - case KeywordDefer: - return "defer"; - case KeywordStruct: - return "struct"; - case KeywordEnum: - return "enum"; - case KeywordExport: - return "export"; - case KeywordExtern: - return "extern"; - case KeywordFunction: - return "fn"; - case KeywordError: - return "error"; - case KeywordTrue: - return "true"; - case KeywordFalse: - return "false"; - case KeywordNone: - return "None"; - case KeywordReturn: - return "return"; - case KeywordUnion: - return "union"; - case KeywordMatch: - return "match"; - case KeywordIn: - return "in"; - case KeywordRange: - return "range"; - case Typename_str: - return "str"; - case Typename_i8: - return "i8"; - case Typename_i16: - return "i16"; - case Typename_i32: - return "i32"; - case Typename_i64: - return "i64"; - case Typename_u8: - return "u8"; - case Typename_u16: - return "u16"; - case Typename_u32: - return "u32"; - case Typename_u64: - return "u64"; - case Typename_box: - return "box"; - case Typename_f32: - return "f32"; - case Typename_f64: - return "f64"; - case LParenthesis: - return "("; - case RParenthesis: - return ")"; - case LCurlyBrace: - return "{"; - case RCurlyBrace: - return "}"; - case LBracket: - return "["; - case RBracket: - return "]"; - case Colon: - return ":"; - case Equals: - return "="; - case QuestionMark: - return "?"; - case Then: - return "??"; - case At: - return "@"; - case SemiColon: - return ";"; - case ExclamationPoint: - return "!"; - case Comma: - return ","; - case RangeSpan: - return ".."; - case Ellipsis: - return "..."; - case PlusPlus: - return "++"; - case MinusMinus: - return "--"; - case FunctionArrow: - return "=>"; - case ColonEquals: - return ":="; - case Arrow: - return "->"; - case EOFToken: - return "EOF"; - default: - UNREACHABLE; - } - } - - size_t len() const { - return id.size(); - } - - str name() const { - return id; - } - - template - void setValue(T v) { - setInternalRepr(v); - } - - void setKind(TokenKind k) { - this->kind = k; - } - - TokenKind getKind() const { - return this->kind; - } - - bool isKeyword() const { - return flags & TokenClassification::Keyword; - } - - bool isLiteral() const { - return flags & TokenClassification::Literal; - } - - bool isClosingDelim() const { - return kind == TokenKind::RBracket || kind == TokenKind::RCurlyBrace - || kind == TokenKind::RParenthesis; - } - - bool isValidExpressionType() const { - return unsigned(kind) >= Integer && unsigned(kind) <= Pow; - } - - bool isValueAssignmentOp() const { - return unsigned(kind) >= PlusEquals && unsigned(kind) <= RightShiftEquals; - } - - bool isValidTypename() const { - return unsigned (kind) >= Typename_str && unsigned(kind) <= - Typename_f64 || kind == Identifier; - } - - bool isValidTypenameStart() const { - return kind == Star || kind == LBracket || kind == LParenthesis; - } - - SourceLocation getSourceLocation() const { - return loc; - } - - void setFlag(TokenClassification flag) { - flags |= flag; - } - - template - T getValue() const { - static_assert(std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value, - "Attempting to get invalid value from Token."); - - T v; setFromInternalRepr(v); - return v; - } - - /// Pretty Print individual token to the screen. - /// TODO: May not need this anymore ... - void PrettyPrint() const { - if (isKeyword()) { - Console::Log(Colors::RED); - } - auto Kind = getKind(); - switch (Kind) { - case String: - Console::Log(Colors::GREEN); - break; - case Integer: - case Decimal: - Console::Log(Colors::BLUE); - break; - case Char: - Console::Log(Colors::GREEN, "\'"); - break; - case Identifier: - Console::Log(Colors::YELLOW); - break; - default: - break; - } - Console::Log(name()); - Console::Log(Colors::RESET); - } - - bool assignment = false; // (ie: +=, -=, *=) - private: - - // This is the restricted usage of union values. - // We are guaranteed to know the actual type of the value from the - // [Token] constructor, therefore setting and retrieving values is not UB. - - - void setFromInternalRepr(double &V) const { - V = float64; - } - - void setFromInternalRepr(decltype(UINTPTR_MAX) &V) const { - V = integer; - } - - void setFromInternalRepr(char &V) const { - V = character; - } - - void setFromInternalRepr(bool &V) const { - V = boolean; - } - - void setInternalRepr(double V) { - float64 = V; - } - - void setInternalRepr(bool V) { - boolean = V; - } - - void setInternalRepr(char V) { - character = V; - } - - void setInternalRepr(decltype(UINTPTR_MAX) V) { - integer = V; - } - - void setInternalRepr(const char *V) { - integer = (unsigned long long) V; - } - - }; - -}} // namespace nextgen::jet +namespace nextgen { + namespace jet { + + enum TokenClassification : unsigned short { + Function = 1 << 2, + Assignment = 1 << 3, + Unsigned = 1 << 4, + Keyword = 1 << 5, + Literal = 1 << 6, + }; + + enum TokenKind { + + // Primitives --------------------------- + Integer = 1, + Zero = 1, + String, + Decimal, + Boolean, + //--------------------------------------- + + // Valid Expression Tokens (DO NOT CHANGE ORDER) --------------------------- + Identifier, // [a-z] && [A-Z] && _ + LessThan, // < + GreaterThan, // > + + Plus, // + + Minus, // - + Slash, // / + Star, // * + XOR, // ^ + AND, // & + NOT, // ~ + Pipe, // | + Percent,// % + Char, // \' + Dot, // . + + // TODO: Fix this ... Should not be apart of binary expression. It should + // be apart of the assignment modifier. + PlusEquals, + MinusEquals, + DivEquals, + PowEquals, + MulEquals, + ANDEquals, + XOREquals, + OREquals, + PercentEquals, + LeftShiftEquals, + RightShiftEquals, + GreaterThanEquals, + LessThanEquals, + + EqualsEquals, // == + LeftShift, // << + RightShift, // >> + Pow, // ** + //--------------------------------------- + + + // Keywords --------------------------- + KeywordIf, + KeywordElse, + KeywordElif, + KeywordWhile, + KeywordFor, + KeywordAnd, + KeywordOr, + KeywordBreak, + KeywordContinue, + KeywordDefer, + KeywordStruct, + KeywordEnum, + KeywordExport, + KeywordExtern, + KeywordFunction, + KeywordError, + KeywordVar, + KeywordTrue, + KeywordFalse, + KeywordNone, + KeywordReturn, + KeywordUnion, + KeywordMatch, + KeywordIn, + KeywordRange, + //--------------------------------------- + + // Typename --------------------------- + Typename_str, + Typename_i8, + Typename_i16, + Typename_i32, + Typename_i64, + Typename_u8, + Typename_u16, + Typename_u32, + Typename_u64, + Typename_box, + Typename_f32, + Typename_f64, + //--------------------------------------- + + // File Control --------------------------- + NewLine, + Whitespace, + StringEscapeStart, + //--------------------------------------- + + LParenthesis, // ( + RParenthesis, // ) + LCurlyBrace, // { + RCurlyBrace, // } + LBracket, // [ + RBracket, // ] + Colon, // : + Equals, // = + QuestionMark, // ? + Then, // ?? + At, // @ + SemiColon, // ; + ExclamationPoint, // ! + Comma, // , + + RangeSpan, // .. + Ellipsis, // ... + + PlusPlus, // ++ + MinusMinus, // -- + FunctionArrow,// => + + ColonEquals, // := + Arrow, // -> + + Error, + EOFToken, + }; + + struct TokenTraits { + + using SourceSpan = Range; + + struct SourceLocation { + size_t line; + size_t column; + SourceSpan source; + }; + + // We have to very careful here. We do not have access to std::variant<...> + // in C++11, so we must use a controlled union to get a Token's value. + struct Value { + union { + decltype(UINTPTR_MAX) integer; + bool boolean; + double float64; + char character; + }; + }; + + }; + + class Token { + private: + using TokenValue = TokenTraits::Value; + using SourceLocation = TokenTraits::SourceLocation; + + str id; + + unsigned flags = TokenClassification::Literal; + SourceLocation loc = {}; + + + TokenKind kind; + public: + Token() = default; + + union { + decltype(UINTPTR_MAX) integer; + bool boolean; + double float64; + char character; + }; + + template + Token(const str &id, SourceLocation loc, T value, + TokenKind kind, unsigned flags = -1) + : id(id), flags(flags), loc(loc), kind(kind) { + setValue(value); + } + + // Note: This is only for lex errors. Failed tokens are important + // for the diagnostic class generate valid errors. + Token(str id, SourceLocation loc, TokenKind kind) + : id(id), loc(loc) { + setKind(kind); + } + + static NG_INLINE char const *GetTokenKindName(TokenKind const kind) { + switch (kind) { + case LessThan: + return "<"; + case GreaterThan: + return ">"; + case Plus: + return "+"; + case Minus: + return "-"; + case Slash: + return "/"; + case Star: + return "*"; + case XOR: + return "^"; + case AND: + return "&"; + case NOT: + return "~"; + case Pipe: + return "|"; + case Percent: + return "%"; + case Char: + return "'"; + case Dot: + return "."; + case PlusEquals: + return "+="; + case MinusEquals: + return "-="; + case DivEquals: + return "/="; + case PowEquals: + return "**="; + case MulEquals: + return "*="; + case GreaterThanEquals: + return ">="; + case LessThanEquals: + return "<="; + case LeftShiftEquals: + return "<<="; + case RightShiftEquals: + return ">>="; + case ANDEquals: + return "&="; + case XOREquals: + return "^="; + case OREquals: + return "|="; + case PercentEquals: + return "%="; + case EqualsEquals: + return "=="; + case LeftShift: + return "<<"; + case RightShift: + return ">>"; + case Pow: + return "**"; + case KeywordIf: + return "if"; + case KeywordElse: + return "else"; + case KeywordElif: + return "elif"; + case KeywordWhile: + return "while"; + case KeywordFor: + return "for"; + case KeywordAnd: + return "and"; + case KeywordOr: + return "or"; + case KeywordBreak: + return "break"; + case KeywordContinue: + return "continue"; + case KeywordDefer: + return "defer"; + case KeywordStruct: + return "struct"; + case KeywordEnum: + return "enum"; + case KeywordExport: + return "export"; + case KeywordExtern: + return "extern"; + case KeywordFunction: + return "fn"; + case KeywordError: + return "error"; + case KeywordTrue: + return "true"; + case KeywordFalse: + return "false"; + case KeywordNone: + return "None"; + case KeywordReturn: + return "return"; + case KeywordUnion: + return "union"; + case KeywordMatch: + return "match"; + case KeywordIn: + return "in"; + case KeywordRange: + return "range"; + case Typename_str: + return "str"; + case Typename_i8: + return "i8"; + case Typename_i16: + return "i16"; + case Typename_i32: + return "i32"; + case Typename_i64: + return "i64"; + case Typename_u8: + return "u8"; + case Typename_u16: + return "u16"; + case Typename_u32: + return "u32"; + case Typename_u64: + return "u64"; + case Typename_box: + return "box"; + case Typename_f32: + return "f32"; + case Typename_f64: + return "f64"; + case LParenthesis: + return "("; + case RParenthesis: + return ")"; + case LCurlyBrace: + return "{"; + case RCurlyBrace: + return "}"; + case LBracket: + return "["; + case RBracket: + return "]"; + case Colon: + return ":"; + case Equals: + return "="; + case QuestionMark: + return "?"; + case Then: + return "??"; + case At: + return "@"; + case SemiColon: + return ";"; + case ExclamationPoint: + return "!"; + case Comma: + return ","; + case RangeSpan: + return ".."; + case Ellipsis: + return "..."; + case PlusPlus: + return "++"; + case MinusMinus: + return "--"; + case FunctionArrow: + return "=>"; + case ColonEquals: + return ":="; + case Arrow: + return "->"; + case EOFToken: + return "EOF"; + default: + UNREACHABLE; + } + } + + size_t len() const { + return id.size(); + } + + str name() const { + return id; + } + + template + void setValue(T v) { + setInternalRepr(v); + } + + void setKind(TokenKind k) { + this->kind = k; + } + + TokenKind getKind() const { + return this->kind; + } + + bool isKeyword() const { + return flags & TokenClassification::Keyword; + } + + bool isLiteral() const { + return flags & TokenClassification::Literal; + } + + bool isClosingDelim() const { + return kind == TokenKind::RBracket || kind == TokenKind::RCurlyBrace + || kind == TokenKind::RParenthesis; + } + + bool isValidExpressionType() const { + return unsigned(kind) >= Integer && unsigned(kind) <= Pow; + } + + bool isValueAssignmentOp() const { + return unsigned(kind) >= PlusEquals && unsigned(kind) <= RightShiftEquals; + } + + bool isValidTypename() const { + return unsigned(kind) >= Typename_str && unsigned(kind) <= + Typename_f64 || kind == Identifier; + } + + bool isValidTypenameStart() const { + return kind == Star || kind == LBracket || kind == LParenthesis; + } + + SourceLocation getSourceLocation() const { + return loc; + } + + void setFlag(TokenClassification flag) { + flags |= flag; + } + + template + T getValue() const { + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + "Attempting to get invalid value from Token."); + + T v; + setFromInternalRepr(v); + return v; + } + + /// Pretty Print individual token to the screen. + /// TODO: May not need this anymore ... + void PrettyPrint() const { + if (isKeyword()) { + Console::Log(Colors::RED); + } + auto Kind = getKind(); + switch (Kind) { + case String: + Console::Log(Colors::GREEN); + break; + case Integer: + case Decimal: + Console::Log(Colors::BLUE); + break; + case Char: + Console::Log(Colors::GREEN, "\'"); + break; + case Identifier: + Console::Log(Colors::YELLOW); + break; + default: + break; + } + Console::Log(name()); + Console::Log(Colors::RESET); + } + + bool assignment = false; // (ie: +=, -=, *=) + private: + + // This is the restricted usage of union values. + // We are guaranteed to know the actual type of the value from the + // [Token] constructor, therefore setting and retrieving values is not UB. + + + void setFromInternalRepr(double &V) const { + V = float64; + } + + void setFromInternalRepr(decltype(UINTPTR_MAX) &V) const { + V = integer; + } + + void setFromInternalRepr(char &V) const { + V = character; + } + + void setFromInternalRepr(bool &V) const { + V = boolean; + } + + void setInternalRepr(double V) { + float64 = V; + } + + void setInternalRepr(bool V) { + boolean = V; + } + + void setInternalRepr(char V) { + character = V; + } + + void setInternalRepr(decltype(UINTPTR_MAX) V) { + integer = V; + } + + void setInternalRepr(const char *V) { + integer = (unsigned long long) V; + } + + }; + + } +} // namespace nextgen::jet # endif //NEXTGEN_TOKEN_H diff --git a/import/nextgen/jet/jet-x64-assembler.h b/import/nextgen/jet/jet-x64-assembler.h index 061a6ae..2e9b166 100644 --- a/import/nextgen/jet/jet-x64-assembler.h +++ b/import/nextgen/jet/jet-x64-assembler.h @@ -7,7 +7,7 @@ #include "../core-util.h" -typedef uint8_t ubyte; +typedef uint8_t ubyte; typedef uint16_t ushort; typedef uint32_t uint; typedef uint64_t ulong; @@ -60,301 +60,363 @@ void name##ss(Register reg, const MemoryAddress &addr);\ void name##sd(Register dest, Register src);\ void name##sd(Register reg, const MemoryAddress &addr);\ -namespace nextgen { namespace jet { namespace x64 { - -typedef struct Imm8 { ubyte value; } Imm8; -typedef struct Imm16 { ushort value; } Imm16; -typedef struct Imm32 { uint value; } Imm32; -typedef struct Imm64 { ulong value; } Imm64; - -enum Condition { - Overflow = 0, - NotOverflow = 1, - Below = 2, - AboveOrEqual = 3, - Equal = 4, - NotEqual = 5, - BelowOrEqual = 6, - Above = 7, - Signed = 8, - NotSigned = 9, - LessThan = 0xC, - LessThanEqual = 0xE, - GreaterThan = 0xF, - GreaterThanEqual = 0xD, - }; - -enum REX { - REX_PREFIX = 0b01000000, - REX_W = 0b00001000, - REX_R = 0b00000100, - REX_X = 0b00000010, - REX_B = 0b00000001 -}; - -enum MOD { - MOD_INDIRECT = 0b00, - MOD_DISP8 = 0b01, - MOD_DISP32 = 0b10, - MOD_REG = 0b11 -}; - - -enum Register : ubyte { - AX = 0b000, CX = 0b001, DX = 0b010, BX = 0b011, - SP = 0b100, BP = 0b101, SI = 0b110, DI = 0b111, - - R8, R9, R10, R11, R12, R13, R14, R15, - - XMM0 = 0b000, XMM1 = 0b001, XMM2 = 0b010, XMM3 = 0b011, - XMM4 = 0b100, XMM5 = 0b101, XMM6 = 0b110, XMM7 = 0b111, - - XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15 -}; - - -enum OperandSize { - BYTE, - WORD, - DWORD, - QWORD, - }; +namespace nextgen { + namespace jet { + namespace x64 { + + typedef struct Imm8 { + ubyte value; + } Imm8; + typedef struct Imm16 { + ushort value; + } Imm16; + typedef struct Imm32 { + uint value; + } Imm32; + typedef struct Imm64 { + ulong value; + } Imm64; + + enum Condition { + Overflow = 0, + NotOverflow = 1, + Below = 2, + AboveOrEqual = 3, + Equal = 4, + NotEqual = 5, + BelowOrEqual = 6, + Above = 7, + Signed = 8, + NotSigned = 9, + LessThan = 0xC, + LessThanEqual = 0xE, + GreaterThan = 0xF, + GreaterThanEqual = 0xD, + }; + + enum REX { + REX_PREFIX = 0b01000000, + REX_W = 0b00001000, + REX_R = 0b00000100, + REX_X = 0b00000010, + REX_B = 0b00000001 + }; + + enum MOD { + MOD_INDIRECT = 0b00, + MOD_DISP8 = 0b01, + MOD_DISP32 = 0b10, + MOD_REG = 0b11 + }; + + + enum Register : ubyte { + AX = 0b000, CX = 0b001, DX = 0b010, BX = 0b011, + SP = 0b100, BP = 0b101, SI = 0b110, DI = 0b111, + + R8, R9, R10, R11, R12, R13, R14, R15, + + XMM0 = 0b000, XMM1 = 0b001, XMM2 = 0b010, XMM3 = 0b011, + XMM4 = 0b100, XMM5 = 0b101, XMM6 = 0b110, XMM7 = 0b111, + + XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15 + }; + + + enum OperandSize { + BYTE, + WORD, + DWORD, + QWORD, + }; // Reference: https://blog.yossarian.net/2020/06/13/How-x86_64-addresses-memory -enum MemoryAddressKind { - Disp32, // [disp32] - Base, // [base] - BaseIndex, // [base + index] - BaseDisp32, // [base + disp32] - BaseDisp8, // [base + disp8] - BaseIndexScale, // [base + (index * scale)] - BaseIndexDisp32, // [base + (index * scale) + disp32] - BaseIndexDisp8, // [base + (index * scale) + disp8] -}; - -enum SibScale : ubyte { - Scale1 = 0b00, - Scale2 = 0b01, - Scale4 = 0b10, - Scale8 = 0b11 -}; - -struct MemoryAddress { - bool has_disp8 = false; - bool has_disp32 = false; - bool has_sib = false; + enum MemoryAddressKind { + Disp32, // [disp32] + Base, // [base] + BaseIndex, // [base + index] + BaseDisp32, // [base + disp32] + BaseDisp8, // [base + disp8] + BaseIndexScale, // [base + (index * scale)] + BaseIndexDisp32, // [base + (index * scale) + disp32] + BaseIndexDisp8, // [base + (index * scale) + disp8] + }; - ubyte scale = Scale1; - ubyte mod = MOD_INDIRECT; - uint disp = 0; + enum SibScale : ubyte { + Scale1 = 0b00, + Scale2 = 0b01, + Scale4 = 0b10, + Scale8 = 0b11 + }; - Register base = BP; - Register index = SP; + struct MemoryAddress { + bool has_disp8 = false; + bool has_disp32 = false; + bool has_sib = false; - // [disp32] - explicit MemoryAddress(uint disp32); + ubyte scale = Scale1; + ubyte mod = MOD_INDIRECT; + uint disp = 0; - // [base] - explicit MemoryAddress(Register base); + Register base = BP; + Register index = SP; - // [base + disp8] - explicit MemoryAddress(Register base, ubyte disp8); + // [disp32] + explicit MemoryAddress(uint disp32); - // [base + disp32] - explicit MemoryAddress(Register base, uint disp32); + // [base] + explicit MemoryAddress(Register base); - // [base + (index * scale)] - explicit MemoryAddress(Register base, - Register index, - SibScale scale); + // [base + disp8] + explicit MemoryAddress(Register base, ubyte disp8); - // [base + (index * scale) + disp8] - explicit MemoryAddress(Register base, Register index, SibScale, - ubyte disp8); + // [base + disp32] + explicit MemoryAddress(Register base, uint disp32); - // [base + (index * scale) + disp32] - explicit MemoryAddress(Register base, Register index, SibScale, - uint disp32); -}; + // [base + (index * scale)] + explicit MemoryAddress(Register base, + Register index, + SibScale scale); -class Assembler { -private: - ubyte *memory; - size_t used = 0; + // [base + (index * scale) + disp8] + explicit MemoryAddress(Register base, Register index, SibScale, + ubyte disp8); - // Writing primitive values to buffer - void write(ubyte value); - void write_16(ushort value); - void write_32(uint value); - void write_64(ulong value); + // [base + (index * scale) + disp32] + explicit MemoryAddress(Register base, Register index, SibScale, + uint disp32); + }; - // Write Immediate values to the buffer - void write(Imm8 imm); - void write(Imm16 imm); - void write(Imm32 imm); - void write(Imm64 imm); + class Assembler { + private: + ubyte *memory; + size_t used = 0; - // Write a memory address given the register destination - /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html - void write_address(const MemoryAddress &addr, Register dest); + // Writing primitive values to buffer + void write(ubyte value); + void write_16(ushort value); - // Some useful overrides when writing multiple values. - // There is no need for a custom implementation because - // we don't write more than 4 values at a given time in a row. - // - // We depend on the compiler on choosing the correct function for the - // value that is passed in -- the choice should be clear given distinct - // parameter values. + void write_32(uint value); - template - void write(T f, K s); + void write_64(ulong value); - template - void write(T f, K s, L t); + // Write Immediate values to the buffer + void write(Imm8 imm); - template - void write(T f, K s, L t, Q fr); + void write(Imm16 imm); - /// Write rex-byte based on a single register for a BYTE operation. - void rex_optional_r1(Register reg); + void write(Imm32 imm); - /// Write rex-byte based on a single register for a WORD operation. - void rex_optional_r2(Register reg); + void write(Imm64 imm); - /// Write rex-byte based on a single register for a DWORD operation. - void rex_optional_r3(Register reg); + // Write a memory address given the register destination + /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html + void write_address(const MemoryAddress &addr, Register dest); - /// Writes rex-values for 64-bit argument operations. This applies for - /// the long-mode of x86-- which is x64. It also integrates rex-prefix - /// with special register R8-R15. - void rex_optional_r4(Register reg); - /// Binary Operation Rex Byte -- BYTE SIZE - void rex_optional_rr1(Register dest, Register src); + // Some useful overrides when writing multiple values. + // There is no need for a custom implementation because + // we don't write more than 4 values at a given time in a row. + // + // We depend on the compiler on choosing the correct function for the + // value that is passed in -- the choice should be clear given distinct + // parameter values. - /// Binary Operation Rex Byte -- WORD SIZE - void rex_optional_rr2(Register dest, Register src); + template + void write(T f, K s); + template + void write(T f, K s, L t); - /// Binary Operation Rex Byte -- DWORD SIZE - void rex_optional_rr3(Register dest, Register src); + template + void write(T f, K s, L t, Q fr); - /// Binary Operation Rex Byte -- QWORD SIZE - void rex_optional_rr4(Register dest, Register src); + /// Write rex-byte based on a single register for a BYTE operation. + void rex_optional_r1(Register reg); - /// Memory Operation Rex Byte -- BYTE SIZE - void rex_optional_rm1(Register reg, const MemoryAddress &addr); + /// Write rex-byte based on a single register for a WORD operation. + void rex_optional_r2(Register reg); - /// Memory Operation Rex Byte -- WORD SIZE - void rex_optional_rm2(Register reg, const MemoryAddress &addr); + /// Write rex-byte based on a single register for a DWORD operation. + void rex_optional_r3(Register reg); - /// Memory Operation Rex Byte -- DWORD SIZE - void rex_optional_rm3(Register reg, const MemoryAddress &addr); + /// Writes rex-values for 64-bit argument operations. This applies for + /// the long-mode of x86-- which is x64. It also integrates rex-prefix + /// with special register R8-R15. + void rex_optional_r4(Register reg); - /// Memory Operation Rex Byte -- QWORD SIZE - void rex_optional_rm4(Register reg, const MemoryAddress &addr); + /// Binary Operation Rex Byte -- BYTE SIZE + void rex_optional_rr1(Register dest, Register src); + /// Binary Operation Rex Byte -- WORD SIZE + void rex_optional_rr2(Register dest, Register src); -public: - explicit Assembler(size_t size); - explicit Assembler(size_t size, bool executable); + /// Binary Operation Rex Byte -- DWORD SIZE + void rex_optional_rr3(Register dest, Register src); - /// Spill the memory buffer containing machine code bytes written - ubyte *spill() const; + /// Binary Operation Rex Byte -- QWORD SIZE + void rex_optional_rr4(Register dest, Register src); - /// The number of bytes that have been written to the memory buffer - ulong bytes() const; + /// Memory Operation Rex Byte -- BYTE SIZE + void rex_optional_rm1(Register reg, const MemoryAddress &addr); - /// Get a given byte that has been generated by the assembler - ubyte operator[](size_t index) { return memory[index]; } + /// Memory Operation Rex Byte -- WORD SIZE + void rex_optional_rm2(Register reg, const MemoryAddress &addr); - void popq(Register reg); - void popw(Register reg); + /// Memory Operation Rex Byte -- DWORD SIZE + void rex_optional_rm3(Register reg, const MemoryAddress &addr); - void pushq(Register reg); - void pushw(Register reg); + /// Memory Operation Rex Byte -- QWORD SIZE + void rex_optional_rm4(Register reg, const MemoryAddress &addr); - void movb(Register reg, Imm8 imm); - template - void movw(Register reg, Imm imm); - template - void movd(Register reg, Imm imm); - template - void movq(Register reg, Imm imm); - void movb(Register reg, const MemoryAddress &addr); - void movw(Register reg, const MemoryAddress &addr); - void movd(Register reg, const MemoryAddress &addr); - void movq(Register reg, const MemoryAddress &addr); + public: - void movb(Register dest, Register src); - void movw(Register dest, Register src); - void movd(Register dest, Register src); - void movq(Register dest, Register src); + explicit Assembler(size_t size); - void shlb(Register dest, Imm8 count); - void shlw(Register dest, Imm8 count); - void shld(Register dest, Imm8 count); - void shlq(Register dest, Imm8 count); + explicit Assembler(size_t size, bool executable); - void shlb(Register dest); - void shlw(Register dest); - void shld(Register dest); - void shlq(Register dest); + /// Spill the memory buffer containing machine code bytes written + ubyte *spill() const; - void shrb(Register dest, Imm8 count); - void shrw(Register dest, Imm8 count); - void shrd(Register dest, Imm8 count); - void shrq(Register dest, Imm8 count); + /// The number of bytes that have been written to the memory buffer + ulong bytes() const; - void jmp(Imm8 rel8); - void jmp(Imm16 rel16); - void jmp(Imm32 rel32); + /// Get a given byte that has been generated by the assembler + ubyte operator[](size_t index) { return memory[index]; } - void jmpw(Register reg); - void jmpq(Register reg); + void popq(Register reg); - void jump_cond(Condition cond, Imm8 rel8); - void jump_cond(Condition cond, Imm16 rel16); - void jump_cond(Condition cond, Imm32 rel32); + void popw(Register reg); - // NOTE: imul instruction order for rex is (lib, dest), yes, this is - // correct. - void imulw(Register dest, Register src); - void imuld(Register dest, Register src); - void imulq(Register dest, Register src); + void pushq(Register reg); - void lea(Register reg, const MemoryAddress &addr, OperandSize size); + void pushw(Register reg); - void movss(Register dest, Register src); - void movss(Register dest, const MemoryAddress &addr); - void movss(const MemoryAddress& dest, Register src); + void movb(Register reg, Imm8 imm); - instruction_binary(add); - instruction_binary(sub); - instruction_binary(and); - instruction_binary(cmp); - instruction_binary(xor); - instruction_binary(or); + template + void movw(Register reg, Imm imm); - instruction_float_binary(add); - instruction_float_binary(sub); - instruction_float_binary(div); - instruction_float_binary(mul); - - instruction_unary(inc); - instruction_unary(dec); - instruction_unary(not); - instruction_unary(mul); - instruction_unary(neg); - instruction_unary(div); - instruction_unary(idiv); - - void ret(); -}; -}}} + template + void movd(Register reg, Imm imm); + + template + void movq(Register reg, Imm imm); + + void movb(Register reg, const MemoryAddress &addr); + + void movw(Register reg, const MemoryAddress &addr); + + void movd(Register reg, const MemoryAddress &addr); + + void movq(Register reg, const MemoryAddress &addr); + + void movb(Register dest, Register src); + + void movw(Register dest, Register src); + + void movd(Register dest, Register src); + + void movq(Register dest, Register src); + + void shlb(Register dest, Imm8 count); + + void shlw(Register dest, Imm8 count); + + void shld(Register dest, Imm8 count); + + void shlq(Register dest, Imm8 count); + + void shlb(Register dest); + + void shlw(Register dest); + + void shld(Register dest); + + void shlq(Register dest); + + void shrb(Register dest, Imm8 count); + + void shrw(Register dest, Imm8 count); + + void shrd(Register dest, Imm8 count); + + void shrq(Register dest, Imm8 count); + + void jmp(Imm8 rel8); + + void jmp(Imm16 rel16); + + void jmp(Imm32 rel32); + + void jmpw(Register reg); + + void jmpq(Register reg); + + void jump_cond(Condition cond, Imm8 rel8); + + void jump_cond(Condition cond, Imm16 rel16); + + void jump_cond(Condition cond, Imm32 rel32); + + // NOTE: imul instruction order for rex is (lib, dest), yes, this is + // correct. + void imulw(Register dest, Register src); + + void imuld(Register dest, Register src); + + void imulq(Register dest, Register src); + + void lea(Register reg, const MemoryAddress &addr, OperandSize size); + + void movss(Register dest, Register src); + + void movss(Register dest, const MemoryAddress &addr); + + void movss(const MemoryAddress &dest, Register src); + + instruction_binary(add); + + instruction_binary(sub); + + instruction_binary(and); + + instruction_binary(cmp); + + instruction_binary(xor); + + instruction_binary(or); + + instruction_float_binary(add); + + instruction_float_binary(sub); + + instruction_float_binary(div); + + instruction_float_binary(mul); + + instruction_unary(inc); + + instruction_unary(dec); + + instruction_unary(not); + + instruction_unary(mul); + + instruction_unary(neg); + + instruction_unary(div); + + instruction_unary(idiv); + + void ret(); + }; + } + } +} #endif //JET_X64_ASSEMBLER_H diff --git a/import/nextgen/os-config.h b/import/nextgen/os-config.h index e13000a..354a12c 100644 --- a/import/nextgen/os-config.h +++ b/import/nextgen/os-config.h @@ -2,7 +2,9 @@ # define NEXTGEN_CONFIG_H # if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) || defined(__WIN32) && !defined(__CYGWIN__) # define NG_OS_WINDOWS true + # include + # elif __APPLE__ # define NG_OS_APPLE true # include @@ -20,8 +22,8 @@ # define NG_INLINE inline # define NG_AINLINE inline __attribute__((always_inline)) # elif defined(_MSC_VER) -# define NG_INLINE __inline -# define NG_AINLINE __forceinline +# define NG_INLINE __inline +# define NG_AINLINE __forceinline # endif # if defined(_M_IX86) # define NG_FAST __fastcall @@ -50,26 +52,29 @@ /* Older versions of MSVC do not have stdint.h values that are properly defined from C99 */ # if defined(_MSC_VER) && (_MSC_VER < 1700) typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; # ifdef _WIN64 # define UINTPTR_MAX 0xffffffffffffffff - typedef __int64 intptr_t; - typedef unsigned __int64 uintptr_t; + typedef __int64 intptr_t; + typedef unsigned __int64 uintptr_t; # else # define UINTPTR_MAX 0xffffffff - typedef __int32 intptr_t; - typedef unsigned __int32 uintptr_t; + typedef __int32 intptr_t; + typedef unsigned __int32 uintptr_t; # endif # else # define HAS_STDINT + # include + # endif + # include # include # include @@ -82,6 +87,7 @@ typedef __int8 int8_t; # include # include # include + # ifdef HAS_STDINT /* This is non-specific compiler detection */ # if UINTPTR_MAX == 0xffffffff # ifndef BIT32 @@ -97,7 +103,7 @@ typedef __int8 int8_t; # endif # if !defined(BIT32) && !defined(BIT64) # ifdef _WIN64 /* Only windows has this here because MSVC sometimes does not have stdint.h */ - define BIT64 +define BIT64 # else # define BIT32 # endif diff --git a/import/nextgen/panic.h b/import/nextgen/panic.h index da1d5b5..6b96baa 100644 --- a/import/nextgen/panic.h +++ b/import/nextgen/panic.h @@ -1,5 +1,6 @@ #ifndef NEXTGEN_PANIC_H #define NEXTGEN_PANIC_H + #include "os-config.h" @@ -7,82 +8,84 @@ namespace nextgen { #if defined(NG_OS_LINUX) || defined(NG_OS_APPLE) - // ANSI Terminal Colors for Linux and MacOS. The colors are strictly - // pre-defined to be White, Red, Green, Yellow, Blue, and Cyan and nothing - // more that is needed to highlight something in the terminal. - - // NOTE: This is namespace because of weird debug issue with GCC. For some - // reason in debug mode, the symbols are not resolved by the linker but works - // in release mode. The effect is the same either way with a namespace so we - // leave it as such. - namespace Colors { - static constexpr auto WHITE = COLOR_WHITE; - static constexpr auto RED = COLOR_RED; - static constexpr auto GREEN = COLOR_GREEN; - static constexpr auto YELLOW = COLOR_YELLOW; - static constexpr auto BLUE = COLOR_BLUE; - static constexpr auto CYAN = COLOR_CYAN; - static constexpr auto RESET = COLOR_RESET; - }; + // ANSI Terminal Colors for Linux and MacOS. The colors are strictly + // pre-defined to be White, Red, Green, Yellow, Blue, and Cyan and nothing + // more that is needed to highlight something in the terminal. + + // NOTE: This is namespace because of weird debug issue with GCC. For some + // reason in debug mode, the symbols are not resolved by the linker but works + // in release mode. The effect is the same either way with a namespace so we + // leave it as such. + namespace Colors { + static constexpr auto WHITE = COLOR_WHITE; + static constexpr auto RED = COLOR_RED; + static constexpr auto GREEN = COLOR_GREEN; + static constexpr auto YELLOW = COLOR_YELLOW; + static constexpr auto BLUE = COLOR_BLUE; + static constexpr auto CYAN = COLOR_CYAN; + static constexpr auto RESET = COLOR_RESET; + }; # elif defined(NG_OS_WINDOWS) - // NOTE: These colors are number values designated as console "modes" for - // the Windows Terminal. Windows uses these modes to render text as a - // specific color. - enum Colors { - WHITE = COLOR_WHITE, - RED = COLOR_RED, - GREEN = COLOR_GREEN, - YELLOW = COLOR_YELLOW, - BLUE = COLOR_BLUE, - CYAN = COLOR_CYAN, - RESET = COLOR_RESET - }; + // NOTE: These colors are number values designated as console "modes" for + // the Windows Terminal. Windows uses these modes to render text as a + // specific color. + enum Colors { + WHITE = COLOR_WHITE, + RED = COLOR_RED, + GREEN = COLOR_GREEN, + YELLOW = COLOR_YELLOW, + BLUE = COLOR_BLUE, + CYAN = COLOR_CYAN, + RESET = COLOR_RESET + }; # else - namespace Colors { - static constexpr auto WHITE = ""; - static constexpr auto RED = ""; - static constexpr auto GREEN = ""; - static constexpr auto YELLOW = ""; - static constexpr auto BLUE = ""; - static constexpr auto CYAN = ""; - static constexpr auto RESET = ""; - }; + namespace Colors { + static constexpr auto WHITE = ""; + static constexpr auto RED = ""; + static constexpr auto GREEN = ""; + static constexpr auto YELLOW = ""; + static constexpr auto BLUE = ""; + static constexpr auto CYAN = ""; + static constexpr auto RESET = ""; + }; # endif - struct Console { + struct Console { # if defined(NG_OS_WINDOWS) - // A little guessing trick that has the compiler infer which function gets - // called. It chooses default template unless we use a color. - static NG_INLINE void Write(Colors color) { - static HANDLE GenericConsole = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(GenericConsole, color); - } - - // Wrapper around std::cout to mask terminal color settings - // in the other `Write` function. - template - static NG_INLINE void Write(Arg value) { - std::cout << value; - } + // A little guessing trick that has the compiler infer which function gets + // called. It chooses default template unless we use a color. + static NG_INLINE void Write(Colors color) { + static HANDLE GenericConsole = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(GenericConsole, color); + } + + // Wrapper around std::cout to mask terminal color settings + // in the other `Write` function. + template + static NG_INLINE void Write(Arg value) { + std::cout << value; + } + # endif - template - static void Log(Args ... args) { - using fold = int[]; + + template + static void Log(Args ... args) { + using fold = int[]; # if defined(NG_OS_LINUX) || defined(NG_OS_APPLE) - fold { - (std::cout << args, 0)... - }; + fold { + (std::cout << args, 0)... + }; # elif defined(NG_OS_WINDOWS) - fold { - ((Write(args)), 0)... - }; + fold{ + ((Write(args)), 0)... + }; # endif - } - }; + } + }; # define PANIC(msg) nextgen::PanicAt(msg, __FILE__, "PANIC("#msg");") @@ -99,41 +102,41 @@ nextgen::PanicAt("Expected value to be true", __FILE__, \ "ASSERT_TRUE("#cond");") - // Panic at a specific point in the code. This is simulated as an exception - // throw caught by main and returns a failure as to avoid misuse of - // "std::exit". It prints out the line number,and text of the panic message - // to have the user locate the failed point in the code. - template< - std::size_t N1, std::size_t N2, std::size_t N3, - int LINE, bool Assert> - static void PanicAt( - const char (&msg)[N1], /** @brief Panic Message */ - const char (&FILE)[N2], /** @brief File name of Error */ - const char (&dup)[N3]) /** @brief Line text of Panic */ { - - Console::Log(FILE, ":", LINE, " ", Colors::RED, "error: ", - Colors::WHITE, - msg); - Console::Log('\n', Colors::RESET); - - auto whitespace = (std::to_string(LINE) + " | ").length(); - if (Assert) { - Console::Log(Colors::RED, LINE, " | ", dup, Colors::WHITE, - " <-- ", - Colors::RED, "Assert Failed:", Colors::YELLOW, - " Thread panicked here", "\n", Colors::BLUE); - } else { - Console::Log(Colors::RED, LINE, " | ", dup, Colors::YELLOW, - " <-- Thread panicked here", "\n", - Colors::BLUE); - } - /* Leading Whitespace */ - FOR(i, whitespace) Console::Log(" ");; - /* Highlighting region */ - FOR(i, N3 - 1) Console::Log('~'); - Console::Log(Colors::RESET, '\n'); - throw std::exception(); - } + // Panic at a specific point in the code. This is simulated as an exception + // throw caught by main and returns a failure as to avoid misuse of + // "std::exit". It prints out the line number,and text of the panic message + // to have the user locate the failed point in the code. + template< + std::size_t N1, std::size_t N2, std::size_t N3, + int LINE, bool Assert> + static void PanicAt( + const char (&msg)[N1], /** @brief Panic Message */ + const char (&FILE)[N2], /** @brief File name of Error */ + const char (&dup)[N3]) /** @brief Line text of Panic */ { + + Console::Log(FILE, ":", LINE, " ", Colors::RED, "error: ", + Colors::WHITE, + msg); + Console::Log('\n', Colors::RESET); + + auto whitespace = (std::to_string(LINE) + " | ").length(); + if (Assert) { + Console::Log(Colors::RED, LINE, " | ", dup, Colors::WHITE, + " <-- ", + Colors::RED, "Assert Failed:", Colors::YELLOW, + " Thread panicked here", "\n", Colors::BLUE); + } else { + Console::Log(Colors::RED, LINE, " | ", dup, Colors::YELLOW, + " <-- Thread panicked here", "\n", + Colors::BLUE); + } + /* Leading Whitespace */ + FOR(i, whitespace) Console::Log(" ");; + /* Highlighting region */ + FOR(i, N3 - 1) Console::Log('~'); + Console::Log(Colors::RESET, '\n'); + throw std::exception(); + } } diff --git a/import/nextgen/str.h b/import/nextgen/str.h index c845e75..26769a1 100644 --- a/import/nextgen/str.h +++ b/import/nextgen/str.h @@ -1,194 +1,201 @@ # ifndef NEXTGEN_STR_H # define NEXTGEN_STR_H + # include "core-util.h" # include "allocator.h" -namespace nextgen { using namespace core; using namespace nextgen::mem; +namespace nextgen { + using namespace core; using namespace nextgen::mem; + + template + struct Range { + + T begin; + T end; + + Range(T begin, T end) : begin(begin), end(end) {} + + Range() = default; + + [[nodiscard]] + NG_INLINE size_t range() const { + return (size_t) (end - begin); + } + }; + + + class str { + public: + str() = default; + + struct intern_hash { // std::unordered_map + size_t operator()(str const &s) const { + return s.getHashCache(); + } + }; + + struct intern_eq { + bool operator()(str const &LHS, str const &RHS) const { + return strncmp(LHS.pointer, RHS.pointer, LHS.len) == 0; + } + }; + + + template + /*implicit*/ + constexpr str(const char(&data)[N]) : len(N), pointer(data) {} - template - struct Range { + /*implicit*/ str(const char *data) : len(strlen(data)), pointer(data) {} - T begin; - T end; + /*implicit*/ str(char c) : len(1) { pointer = &c; } - Range(T begin, T end) : begin(begin), end(end) {} + explicit str(const char *data, size_t len) : len(len), pointer(data) {} - Range() = default; + /*implicit*/ str(Range range) + : len(range.range()), pointer(range.begin) {} - [[nodiscard]] - NG_INLINE size_t range() const { - return (size_t) (end - begin); - } - }; + /*implicit*/ str(std::string &data) { // NOTE: Do not use init-list for this + len = data.length(); + pointer = data.c_str(); + } + [[nodiscard]] + size_t size() const { return len; } - class str { - public: - str() = default; + // String FNV-1a Hashing Algorithm (Internal) + [[nodiscard]] + size_t hash() const { + auto val = FNV_OFF; + for (auto i = 0; i < len; ++i) { + val ^= (pointer[i]); + val *= FNV_PRIME; + } + return val; + } - struct intern_hash { // std::unordered_map - size_t operator()(str const &s) const { - return s.getHashCache(); - } - }; + bool operator==(str const &other) const { + return pointer == other.pointer; + } - struct intern_eq { - bool operator()(str const &LHS, str const &RHS) const { - return strncmp(LHS.pointer, RHS.pointer, LHS.len) == 0; - } - }; - - - template /*implicit*/ - constexpr str(const char(&data)[N]) : len(N), pointer(data) {} - - /*implicit*/ str(const char *data) : len(strlen(data)), pointer(data) {} - /*implicit*/ str(char c) : len(1) { pointer = &c; } + char operator[](size_t index) const { + return (static_cast(pointer))[index]; + } - explicit str(const char *data, size_t len) : len(len), pointer(data) {} - /*implicit*/ str(Range range) - : len(range.range()), pointer(range.begin) {} - - /*implicit*/ str(std::string &data) { // NOTE: Do not use init-list for this - len = data.length(); - pointer = data.c_str(); - } - - [[nodiscard]] - size_t size() const { return len; } - - // String FNV-1a Hashing Algorithm (Internal) - [[nodiscard]] - size_t hash() const { - auto val = FNV_OFF; - for (auto i = 0; i < len; ++i) { - val ^= (pointer[i]); - val *= FNV_PRIME; - } - return val; - } - - bool operator==(str const &other) const { - return pointer == other.pointer; - } - - char operator[](size_t index) const { - return (static_cast(pointer))[index]; - } - - /*implicit*/ operator std::string() const { - return std::string((const char *) *this); - } - /*implicit*/ operator Range() const { - return Range(pointer, pointer + len); - } - - explicit operator const char *() const { - return static_cast(pointer); - } - - - str operator-(const size_t size) const { - return str { - pointer - size, - len - size - }; - } - - str operator+(int offset) const { - return str { - pointer + offset, - len + offset - }; - } - - char operator*() const { - return *pointer; - } - - const char *begin() const { - return pointer; - } - - const char *end() const { - return pointer + len; - } - - // You MUST set the hash value before interning a string, - // so it is not recomputed in str::intern_hash. - NG_AINLINE size_t getHashCache() const { - return hash_cache; - } - - // Mutation. - - NG_INLINE void setHash() { - hash_cache = hash(); - } - - NG_INLINE void setHash(size_t hash) { - hash_cache = hash; - } - - str operator++() { - pointer++; - return *this; - } - - str operator--() { - pointer--; - return *this; - } - - str operator+=(int offset) { - pointer += (size_t) (offset); - return *this; - } - - str operator-=(int offset) { - pointer -= (size_t) (offset); - return *this; - } - - const char *pointer{}; // char* data - //bool is_heap_allocated = false; - private: - size_t len{}; // String length - size_t hash_cache = 0; - }; - - - // Very thin wrapper around std::unordered_set for String interning. - // Mostly O(1) Retrieval (FNV-1a is fast and good String hash). - // Designed around the symbols retrieved for the token and not the token - // values. - class StringCache { - public: - using StringSet = std::unordered_set; - - static StringSet& Strings() { - static StringSet Strings; - return Strings; - } - - static str &Get(str &s) { - return const_cast(*(Strings().find(s))); - } - - static NG_INLINE str InsertOrGet(str &s) { - auto find = Strings().find(s); - if (find != Strings().end()) { - return *find; - } else { - auto new_string = (str) s; - new_string.setHash(s.getHashCache()); - return (*Strings().insert(new_string).first); - } - } - }; - - str operator""_intern(const char *s, size_t len); - std::ostream &operator<<(std::ostream &s, nextgen::str &str); + /*implicit*/ operator std::string() const { + return std::string((const char *) *this); + } + + /*implicit*/ operator Range() const { + return Range(pointer, pointer + len); + } + + explicit operator const char *() const { + return static_cast(pointer); + } + + + str operator-(const size_t size) const { + return str{ + pointer - size, + len - size + }; + } + + str operator+(int offset) const { + return str{ + pointer + offset, + len + offset + }; + } + + char operator*() const { + return *pointer; + } + + const char *begin() const { + return pointer; + } + + const char *end() const { + return pointer + len; + } + + // You MUST set the hash value before interning a string, + // so it is not recomputed in str::intern_hash. + NG_AINLINE size_t getHashCache() const { + return hash_cache; + } + + // Mutation. + + NG_INLINE void setHash() { + hash_cache = hash(); + } + + NG_INLINE void setHash(size_t hash) { + hash_cache = hash; + } + + str operator++() { + pointer++; + return *this; + } + + str operator--() { + pointer--; + return *this; + } + + str operator+=(int offset) { + pointer += (size_t) (offset); + return *this; + } + + str operator-=(int offset) { + pointer -= (size_t) (offset); + return *this; + } + + const char *pointer{}; // char* data + //bool is_heap_allocated = false; + private: + size_t len{}; // String length + size_t hash_cache = 0; + }; + + + // Very thin wrapper around std::unordered_set for String interning. + // Mostly O(1) Retrieval (FNV-1a is fast and good String hash). + // Designed around the symbols retrieved for the token and not the token + // values. + class StringCache { + public: + using StringSet = std::unordered_set; + + static StringSet &Strings() { + static StringSet Strings; + return Strings; + } + + static str &Get(str &s) { + return const_cast(*(Strings().find(s))); + } + + static NG_INLINE str InsertOrGet(str &s) { + auto find = Strings().find(s); + if (find != Strings().end()) { + return *find; + } else { + auto new_string = (str) s; + new_string.setHash(s.getHashCache()); + return (*Strings().insert(new_string).first); + } + } + }; + + str operator ""_intern(const char *s, size_t len); + + std::ostream &operator<<(std::ostream &s, nextgen::str &str); } #endif //NEXTGEN_STR_H diff --git a/lib/allocator.cpp b/lib/allocator.cpp index 0ae71f2..44bd83a 100644 --- a/lib/allocator.cpp +++ b/lib/allocator.cpp @@ -3,41 +3,41 @@ using namespace nextgen::mem; void *os::calloc(size_t mem, size_t size) { - if (mem == 0) return nullptr; - auto p = std::calloc(mem, size); - ASSERT(p, "Internal Error: Calloc Failed"); - return p; + if (mem == 0) return nullptr; + auto p = std::calloc(mem, size); + ASSERT(p, "Internal Error: Calloc Failed"); + return p; } void *os::malloc(size_t size) { - if (size == 0) return nullptr; - auto p = std::malloc(size); - ASSERT(p, "Internal Error: Malloc Failed"); - return p; + if (size == 0) return nullptr; + auto p = std::malloc(size); + ASSERT(p, "Internal Error: Malloc Failed"); + return p; } void *os::realloc(void *buf, size_t size) { - if (buf == nullptr && size == 0) return nullptr; - auto p = std::realloc(buf, size); - ASSERT(p, "Internal Error: Realloc Failed"); - return p; + if (buf == nullptr && size == 0) return nullptr; + auto p = std::realloc(buf, size); + ASSERT(p, "Internal Error: Realloc Failed"); + return p; } void *os::memmove(void *dest, const void *src, size_t size) { - auto p = std::memmove(dest, src, size); - ASSERT(p, "Internal Error: Memmove Failed"); - return p; + auto p = std::memmove(dest, src, size); + ASSERT(p, "Internal Error: Memmove Failed"); + return p; } void os::free(void *p) { - std::free(p); + std::free(p); } Allocator nextgen::mem::GLOBAL_OBJECT_ALLOC = Allocator( - arena::OVER_COMMIT_MEMORY_SIZE - arena::DATA_ALLOCATION_MEMORY_SIZE + arena::OVER_COMMIT_MEMORY_SIZE - arena::DATA_ALLOCATION_MEMORY_SIZE ); Allocator nextgen::mem::GLOBAL_DATA_ALLOC = Allocator( - arena::DATA_ALLOCATION_MEMORY_SIZE + arena::DATA_ALLOCATION_MEMORY_SIZE ); @@ -45,6 +45,7 @@ Allocator nextgen::mem::GLOBAL_DATA_ALLOC = Allocator( // the new and delete operator, but for this code base, it just makes the code // much cleaner, and since the arena pool is global, it should be okay. void *operator new(size_t size) { - return ::GLOBAL_OBJECT_ALLOC.allocate(size); + return ::GLOBAL_OBJECT_ALLOC.allocate(size); } + void operator delete([[maybe_unused]] void *p) noexcept {} diff --git a/lib/io.cpp b/lib/io.cpp index 3757fdc..bd9943a 100644 --- a/lib/io.cpp +++ b/lib/io.cpp @@ -19,12 +19,12 @@ void FileBuf::Output(FileBuf *files, FileID output) { } FileBuf nextgen::io::CreateFileBuffer(const char *FILE, FileID ID) { - auto read = std::ifstream(FILE, std::ios::binary); - auto buf = read.rdbuf(); - auto size = read.tellg(); + auto read = std::ifstream(FILE, std::ios::binary); + auto buf = read.rdbuf(); + auto size = read.tellg(); - auto block = static_cast(nextgen::mem::os::malloc(size)); - buf->sgetn(block, size); + auto block = static_cast(nextgen::mem::os::malloc(size)); + buf->sgetn(block, size); - return { block, ID }; + return {block, ID}; } diff --git a/lib/jet-analyzer.cpp b/lib/jet-analyzer.cpp index 8f83f09..c9bbef1 100644 --- a/lib/jet-analyzer.cpp +++ b/lib/jet-analyzer.cpp @@ -4,7 +4,7 @@ using namespace nextgen::jet; void Analyzer::analyze(const ParserOutput output_to_analyze) { - + } void Analyzer::analyze_function(const SyntaxFunction *function) { diff --git a/lib/jet-diagnostics.cpp b/lib/jet-diagnostics.cpp index 87cb34a..7e47db1 100644 --- a/lib/jet-diagnostics.cpp +++ b/lib/jet-diagnostics.cpp @@ -8,182 +8,182 @@ using nextgen::Range; // Color namespacing #ifdef NG_OS_WINDOWS - using nextgen::Colors; +using nextgen::Colors; #else - using namespace nextgen::Colors; +using namespace nextgen::Colors; #endif static Range - FileLines(size_t nth, const char *file_buf, size_t buf_len) { - int max_num_after_found_point = 3; - - bool found = false; - int count = 1; - - auto copy = file_buf; - const char *found_point; - const char *last_new_line; - - if (nth == 1) { - return Range(file_buf, file_buf + buf_len); - } - - - FOR(ch, buf_len) { - if (*copy == '\n' || *copy == '\r') { - if (found) { - return Range(file_buf, copy); - } - count++; - } - if (count == nth) { - if (!found) { - found = true; - } - } - copy++; - } +FileLines(size_t nth, const char *file_buf, size_t buf_len) { + int max_num_after_found_point = 3; + + bool found = false; + int count = 1; + + auto copy = file_buf; + const char *found_point; + const char *last_new_line; + + if (nth == 1) { + return Range(file_buf, file_buf + buf_len); + } + + + FOR(ch, buf_len) { + if (*copy == '\n' || *copy == '\r') { + if (found) { + return Range(file_buf, copy); + } + count++; + } + if (count == nth) { + if (!found) { + found = true; + } + } + copy++; + } } static str GetNthLineOfBuffer(size_t nth, const char *file_buf, size_t buf_len) { - bool found = false; - int count = 1; - const char *copy = file_buf; - const char *found_point; - - if (nth == 1) { - return str {file_buf, buf_len}; - } - - FOR(i, buf_len) { - if (*copy == '\n' || *copy == '\r' || *copy == '\0') { - if (found) { - return str(Range(found_point, copy)); - } - count++; - } - if (count == nth) { - if (!found) { - found_point = copy+1; - found = true; - } - } - copy++; - } - - return str {found_point, static_cast(copy - found_point)}; + bool found = false; + int count = 1; + const char *copy = file_buf; + const char *found_point; + + if (nth == 1) { + return str{file_buf, buf_len}; + } + + FOR(i, buf_len) { + if (*copy == '\n' || *copy == '\r' || *copy == '\0') { + if (found) { + return str(Range(found_point, copy)); + } + count++; + } + if (count == nth) { + if (!found) { + found_point = copy + 1; + found = true; + } + } + copy++; + } + + return str{found_point, static_cast(copy - found_point)}; } template -static void AddHint(std::string &line, Args&& ... hint) { +static void AddHint(std::string &line, Args &&... hint) { - // Align space - FOR(i, line.length() + 1) { - Console::Log(" "); - } + // Align space + FOR(i, line.length() + 1) { + Console::Log(" "); + } - Console::Log(hint..., nextgen::Colors::RESET); + Console::Log(hint..., nextgen::Colors::RESET); } void Diagnostic::build(LexError error) { - source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); - Console::Log(Colors::RED, - "Error ---------------------------------------------------------- ", - file_name, ":", error.location.line, ":", error.location.column, "\n", - Colors::RESET); - - switch (error.error) { - case MalformedUTF8: - break; - case InvalidChar: - Console::Log("Invalid Char"); - break; - case UnexpectedEOF: - Console::Log("EOF"); - break; - case OutOfRange: - Console::Log("Out of Buffer Range"); - break; - case Unreachable: - Console::Log("Unreachable"); - break; - case IntegerOverflow: - ErrorIntegerOverflow(error); - break; - case FloatingPointOverflow: - break; - case DigitOutOfRange: - ErrorDigitOutOfRange(error); - break; - case MissingClosingDelim: - ErrorMissingClosingDelim(error); - break; - case InvalidDot: - Console::Log("Invalid Dot"); - break; - case InvalidNumberValue: - Console::Log("Invalid Number"); - break; - case InvalidStringEscape: - ErrorInvalidStringEscape(error); - break; - case InvalidUnicodeEscapeClose: - break; - case HexEscapeOutOfRange: - ErrorHexEscapeOutOfRange(error); - break; - default: - UNREACHABLE; - } - Console::Log(Colors::RESET, "\n"); - - // Lexer errors are always fatal to a program's continuation. - send_exception(); + source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); + Console::Log(Colors::RED, + "Error ---------------------------------------------------------- ", + file_name, ":", error.location.line, ":", error.location.column, "\n", + Colors::RESET); + + switch (error.error) { + case MalformedUTF8: + break; + case InvalidChar: + Console::Log("Invalid Char"); + break; + case UnexpectedEOF: + Console::Log("EOF"); + break; + case OutOfRange: + Console::Log("Out of Buffer Range"); + break; + case Unreachable: + Console::Log("Unreachable"); + break; + case IntegerOverflow: + ErrorIntegerOverflow(error); + break; + case FloatingPointOverflow: + break; + case DigitOutOfRange: + ErrorDigitOutOfRange(error); + break; + case MissingClosingDelim: + ErrorMissingClosingDelim(error); + break; + case InvalidDot: + Console::Log("Invalid Dot"); + break; + case InvalidNumberValue: + Console::Log("Invalid Number"); + break; + case InvalidStringEscape: + ErrorInvalidStringEscape(error); + break; + case InvalidUnicodeEscapeClose: + break; + case HexEscapeOutOfRange: + ErrorHexEscapeOutOfRange(error); + break; + default: + UNREACHABLE; + } + Console::Log(Colors::RESET, "\n"); + + // Lexer errors are always fatal to a program's continuation. + send_exception(); } void Diagnostic::build(ParseError error) { - source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); - Console::Log(Colors::RED, - "Error ---------------------------------------------------------- ", - file_name, ":", error.location.line, ":", error.location.column, "\n", - Colors::RESET); - - switch (error.error) { - case ReservedIdentifierAsVariableName: - break; - case UnexpectedExpression: - Console::Log("Unexpected Expression"); - break; - case InvalidToken: - Console::Log("Invalid Token"); - break; - case ExpectedToken: - ErrorParseExpectedToken(error); - break; - case MissingForLoopVariable: - break; - case ExpectedIdentifierForFunctionParameter: - break; - case ExpectedIdentifierForStructProperty: - break; - case MissingClosingPair: - ErrorParseMissingClosingDelim(error); - break; - case UnexpectedEndOfFile: - Console::Log("EOF"); - break; - case MissingFunctionName: - break; - case MissingVariableName: - break; - case InvalidTokenAfterIdentInGlobalScope: - ErrorParseInvalidTokenAfterIdentInGlobalScope(error); - break; - } + source_line = GetNthLineOfBuffer(error.location.line, file_buf, buf_len); + Console::Log(Colors::RED, + "Error ---------------------------------------------------------- ", + file_name, ":", error.location.line, ":", error.location.column, "\n", + Colors::RESET); + + switch (error.error) { + case ReservedIdentifierAsVariableName: + break; + case UnexpectedExpression: + Console::Log("Unexpected Expression"); + break; + case InvalidToken: + Console::Log("Invalid Token"); + break; + case ExpectedToken: + ErrorParseExpectedToken(error); + break; + case MissingForLoopVariable: + break; + case ExpectedIdentifierForFunctionParameter: + break; + case ExpectedIdentifierForStructProperty: + break; + case MissingClosingPair: + ErrorParseMissingClosingDelim(error); + break; + case UnexpectedEndOfFile: + Console::Log("EOF"); + break; + case MissingFunctionName: + break; + case MissingVariableName: + break; + case InvalidTokenAfterIdentInGlobalScope: + ErrorParseInvalidTokenAfterIdentInGlobalScope(error); + break; + } } @@ -191,41 +191,41 @@ void Diagnostic::build(ParseError error) { void Diagnostic::ErrorMissingClosingDelim(nextgen::jet::LexError &error) { - Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); - auto Line = std::to_string(error.location.line); + Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); + auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Reached EOF Before Closing Delim", error); - AddHint(Line, Colors::CYAN, "= ", Colors::GREEN, "try: ", - Colors::RESET, "Add `", (char) error.metadata[0], "` to the end of " - "the statement"); + ErrorLexSetup(Line, "Reached EOF Before Closing Delim", error); + AddHint(Line, Colors::CYAN, "= ", Colors::GREEN, "try: ", + Colors::RESET, "Add `", (char) error.metadata[0], "` to the end of " + "the statement"); } void Diagnostic::ErrorIntegerOverflow(LexError &error) { - Console::Log(Colors::RESET, "Integer Value Too Large\n\n"); - auto Line = std::to_string(error.location.line); + Console::Log(Colors::RESET, "Integer Value Too Large\n\n"); + auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Overflow Occurs Here", error); + ErrorLexSetup(Line, "Overflow Occurs Here", error); - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", - Colors::RESET, "Integer values must be less than ", UINTPTR_MAX, - ".\n", - Colors::RESET, '\n'); + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", + Colors::RESET, "Integer values must be less than ", UINTPTR_MAX, + ".\n", + Colors::RESET, '\n'); - Console::Log("An ", Colors::RED, - "Integer Overflow", Colors::RESET, - " occurs when a numeric literal's value ", Colors::BLUE, - "exceeds ", Colors::RESET, - "the limit bound by their computer. \n", Colors::YELLOW); + Console::Log("An ", Colors::RED, + "Integer Overflow", Colors::RESET, + " occurs when a numeric literal's value ", Colors::BLUE, + "exceeds ", Colors::RESET, + "the limit bound by their computer. \n", Colors::YELLOW); # ifdef BIT32 - Console::Log("Your computer is 32-Bit", Colors::RESET, - " which has a limit of ", - Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", - "your value exceeds that."); + Console::Log("Your computer is 32-Bit", Colors::RESET, + " which has a limit of ", + Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", + "your value exceeds that."); # else - Console::Log("Your computer is 64-Bit", Colors::RESET, - " which has a limit of ", - Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", - "your value exceeds that."); + Console::Log("Your computer is 64-Bit", Colors::RESET, + " which has a limit of ", + Colors::GREEN, UINTPTR_MAX, Colors::RESET, " but ", + "your value exceeds that."); # endif } @@ -239,204 +239,204 @@ void Diagnostic::ErrorMalformedUTF8(LexError &error) { void Diagnostic::ErrorDigitOutOfRange(LexError &error) { - Console::Log(Colors::RESET, "Digit Value Out Of Range\n\n"); - - auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Digit does not match Integer Radix", error); - - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", Colors::RESET, - "The integer value you wrote was ", - Colors::YELLOW, "Base ", error.metadata[0], Colors::RESET, - " which accepts digits "); - - switch (error.metadata[0]) { - case 10: - Console::Log("[", Colors::YELLOW, "0-9", Colors::RESET, "]"); - break; - case 16: - Console::Log(Colors::YELLOW, "0-9, a-f, and A-F", Colors::RESET, - " not ", Colors::RED, "'", source_line[error.location.column], - "'", Colors::RESET, '\n'); - break; - default: - UNREACHABLE; - } - AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::RESET, "Insert "); - error.failed_token.PrettyPrint(); - - switch (error.metadata[0]) { - case 10: - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9", Colors::RESET, "]\n"); - break; - case 16: - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n"); - break; - default: - UNREACHABLE; - } + Console::Log(Colors::RESET, "Digit Value Out Of Range\n\n"); + + auto Line = std::to_string(error.location.line); + ErrorLexSetup(Line, "Digit does not match Integer Radix", error); + + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", Colors::RESET, + "The integer value you wrote was ", + Colors::YELLOW, "Base ", error.metadata[0], Colors::RESET, + " which accepts digits "); + + switch (error.metadata[0]) { + case 10: + Console::Log("[", Colors::YELLOW, "0-9", Colors::RESET, "]"); + break; + case 16: + Console::Log(Colors::YELLOW, "0-9, a-f, and A-F", Colors::RESET, + " not ", Colors::RED, "'", source_line[error.location.column], + "'", Colors::RESET, '\n'); + break; + default: + UNREACHABLE; + } + AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::RESET, "Insert "); + error.failed_token.PrettyPrint(); + + switch (error.metadata[0]) { + case 10: + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9", Colors::RESET, "]\n"); + break; + case 16: + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n"); + break; + default: + UNREACHABLE; + } } void Diagnostic::ErrorHexEscapeOutOfRange(LexError &error) { - Console::Log(Colors::RESET, "Escape Digit Value Out Of Range\n\n"); - - auto Line = std::to_string(error.location.line); - ErrorLexSetup(Line, "Digit is not Base 16", error); - - AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", - Colors::RESET, "Value started as Base 16" - " but value encountered was Base ", - error.metadata[0], '\n'); - - AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::RESET, "Insert "); - error.failed_token.PrettyPrint(); - Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", - Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n\n"); - - Console::Log("Hex escapes turn", Colors::BLUE," 2 ", Colors::RESET, - "hexadecimal chars in to a single char; for " - "example:\n", Colors::GREEN, "\\x32 ", Colors::RESET, - "becomes the", Colors::GREEN, " ", Colors::RESET, - " character. Hex escapes accept digits", Colors::YELLOW, - " 0-9, a-f, and A-F ", Colors::RESET, "\nbut your value was " - "none of " - "these."); + Console::Log(Colors::RESET, "Escape Digit Value Out Of Range\n\n"); + + auto Line = std::to_string(error.location.line); + ErrorLexSetup(Line, "Digit is not Base 16", error); + + AddHint(Line, Colors::BLUE, "= ", Colors::GREEN, "hint: ", + Colors::RESET, "Value started as Base 16" + " but value encountered was Base ", + error.metadata[0], '\n'); + + AddHint(Line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::RESET, "Insert "); + error.failed_token.PrettyPrint(); + Console::Log(Colors::RED, "_ <-- ", Colors::RESET, "[", + Colors::YELLOW, "0-9a-fA-F", Colors::RESET, "]\n\n"); + + Console::Log("Hex escapes turn", Colors::BLUE, " 2 ", Colors::RESET, + "hexadecimal chars in to a single char; for " + "example:\n", Colors::GREEN, "\\x32 ", Colors::RESET, + "becomes the", Colors::GREEN, " ", Colors::RESET, + " character. Hex escapes accept digits", Colors::YELLOW, + " 0-9, a-f, and A-F ", Colors::RESET, "\nbut your value was " + "none of " + "these."); } void Diagnostic::ErrorInvalidStringEscape(LexError &error) { - Console::Log(Colors::RESET, "Not a Valid String Escape\n\n"); + Console::Log(Colors::RESET, "Not a Valid String Escape\n\n"); - auto line = std::to_string(error.location.line); - ErrorLexSetup(line, "Not a String Escape", error); + auto line = std::to_string(error.location.line); + ErrorLexSetup(line, "Not a String Escape", error); - AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::GREEN, - R"(\a, \b, \v, \n, \r, \', \", \x[0-9a-fA-F]{2}, \u{[0-9a-fA-F]}{1,6})", - "\n\n"); + AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::GREEN, + R"(\a, \b, \v, \n, \r, \', \", \x[0-9a-fA-F]{2}, \u{[0-9a-fA-F]}{1,6})", + "\n\n"); - Console::Log(Colors::BLUE, "String Escapes are used to escape " - "characters that ""are otherwise hard to display. \n", - Colors::RED, "'\\", - source_line[error.location.column - 1], "'", Colors::BLUE, - " is not a valid string escape. " - "View the complete list in the language documentation: \n", - "https://github.com/The-NextGen-Project/jet/blob/main/LANG.md", - Colors::RESET); + Console::Log(Colors::BLUE, "String Escapes are used to escape " + "characters that ""are otherwise hard to display. \n", + Colors::RED, "'\\", + source_line[error.location.column - 1], "'", Colors::BLUE, + " is not a valid string escape. " + "View the complete list in the language documentation: \n", + "https://github.com/The-NextGen-Project/jet/blob/main/LANG.md", + Colors::RESET); } void Diagnostic::ErrorParseInvalidTokenAfterIdentInGlobalScope(const ParseError &error) { - Console::Log(Colors::RESET, "Invalid Token\n\n"); - - auto err = error.metadata; - auto invalid_token = err.begin()->misc_tok; - - auto loc = invalid_token->getSourceLocation(); - auto line = std::to_string(loc.line); - ErrorParseSetup(loc.line, "Unexpected Token After Identifier", - invalid_token, error.location); - AddHint(line, Colors::BLUE, "= ", - Colors::CYAN, "try: ", - Colors::RESET, "Adding ", Colors::YELLOW, "=>", Colors::BLUE, " or " - "", - Colors::YELLOW, - ":=" - "\n", Colors::RESET); - - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "Expected one of ", Colors::BLUE, "':=', ", - "'=>', ", Colors::RESET, - "in global scope declaration."); - Console::Log("\n"); - FOR(i, line.length() + 9) Console::Log(" "); - Console::Log("Identifiers in the global scope only represent two values: " - "\n", Colors::GREEN); - FOR(i, line.length() + 9) Console::Log(" "); - Console::Log("Global Variable " - "Declaration ", - Colors::RESET, - "and ", - Colors::BLUE, - "Function " - "Declaration", - Colors::RESET, - " but your token was " - "", Colors::RED, - "none of these.\n"); + Console::Log(Colors::RESET, "Invalid Token\n\n"); + + auto err = error.metadata; + auto invalid_token = err.begin()->misc_tok; + + auto loc = invalid_token->getSourceLocation(); + auto line = std::to_string(loc.line); + ErrorParseSetup(loc.line, "Unexpected Token After Identifier", + invalid_token, error.location); + AddHint(line, Colors::BLUE, "= ", + Colors::CYAN, "try: ", + Colors::RESET, "Adding ", Colors::YELLOW, "=>", Colors::BLUE, " or " + "", + Colors::YELLOW, + ":=" + "\n", Colors::RESET); + + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "Expected one of ", Colors::BLUE, "':=', ", + "'=>', ", Colors::RESET, + "in global scope declaration."); + Console::Log("\n"); + FOR(i, line.length() + 9) Console::Log(" "); + Console::Log("Identifiers in the global scope only represent two values: " + "\n", Colors::GREEN); + FOR(i, line.length() + 9) Console::Log(" "); + Console::Log("Global Variable " + "Declaration ", + Colors::RESET, + "and ", + Colors::BLUE, + "Function " + "Declaration", + Colors::RESET, + " but your token was " + "", Colors::RED, + "none of these.\n"); } void Diagnostic::ErrorParseMissingClosingDelim(ParseError const &error) { - Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); - - auto err = error.metadata; - auto expected_kind = Token::GetTokenKindName(err.begin()->misc_kind); - auto token_got_instead = (err.begin()+1)->misc_tok; - auto token_loc = (err.begin()+2)->location; - - ErrorParseSetup(error.location.line, (std::string("Missing Closing Delim '") + - std::string(expected_kind) + std::string("'")).c_str(), token_got_instead, - error.location); - - std::string line = std::to_string(error.location.line); - AddHint(line, Colors::BLUE, "= ", - Colors::CYAN, "try: ", - Colors::RESET, "Adding ", Colors::YELLOW, "'", expected_kind, - "'\n", Colors::RESET); - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "Delimiters like ", Colors::BLUE, "'(', ", - "'[', ", "'{', ", Colors::RESET, - "must be met with their closing pairs.\n"); + Console::Log(Colors::RESET, "Missing Closing Delim\n\n"); + + auto err = error.metadata; + auto expected_kind = Token::GetTokenKindName(err.begin()->misc_kind); + auto token_got_instead = (err.begin() + 1)->misc_tok; + auto token_loc = (err.begin() + 2)->location; + + ErrorParseSetup(error.location.line, (std::string("Missing Closing Delim '") + + std::string(expected_kind) + std::string("'")).c_str(), token_got_instead, + error.location); + + std::string line = std::to_string(error.location.line); + AddHint(line, Colors::BLUE, "= ", + Colors::CYAN, "try: ", + Colors::RESET, "Adding ", Colors::YELLOW, "'", expected_kind, + "'\n", Colors::RESET); + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "Delimiters like ", Colors::BLUE, "'(', ", + "'[', ", "'{', ", Colors::RESET, + "must be met with their closing pairs.\n"); } void Diagnostic::ErrorParseExpectedToken(ParseError const &error) { - Console::Log(Colors::RESET, "Expected Token\n\n"); - - auto err = error.metadata.begin()->expected_error; - auto line = std::to_string(error.location.line); - ErrorParseSetup(error.location.line, err.message, err.got, err - .got->getSourceLocation()); - AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", - Colors::GREEN, "Adding ", Colors::YELLOW, "'", - Token::GetTokenKindName(err.expected), "'\n"); - AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", - Colors::RESET, "View docs here: " - "https://github" - ".com/The-NextGen-Project/jet/blob/main/LANG.md for " - "syntax.\n"); + Console::Log(Colors::RESET, "Expected Token\n\n"); + + auto err = error.metadata.begin()->expected_error; + auto line = std::to_string(error.location.line); + ErrorParseSetup(error.location.line, err.message, err.got, err + .got->getSourceLocation()); + AddHint(line, Colors::BLUE, "= ", Colors::CYAN, "try: ", + Colors::GREEN, "Adding ", Colors::YELLOW, "'", + Token::GetTokenKindName(err.expected), "'\n"); + AddHint(line, Colors::BLUE, "= ", Colors::GREEN, "help: ", + Colors::RESET, "View docs here: " + "https://github" + ".com/The-NextGen-Project/jet/blob/main/LANG.md for " + "syntax.\n"); } -void Diagnostic::ErrorLexSetup(std::string& line, const char *message, +void Diagnostic::ErrorLexSetup(std::string &line, const char *message, LexError &error) { - auto printer = Lexer((const char *) source_line, - file_name, - source_line.size()); - // Log error line - Console::Log(Colors::RESET, line, " |\t "); - printer.lex(); // Print Failed Token - - Console::Log("\n"); - - // Align space - FOR(i, line.length() + 1) Console::Log(" "); - - Console::Log("\t", Colors::RED); - FOR(column, source_line.size()) { - if (column == error.failed_token.getSourceLocation().column) { - FOR(ch, error.failed_token.name().size()) { - Console::Log("^"); - } - break; - } - Console::Log(" "); - } - - Console::Log("_ <-- ", message); - Console::Log("\n\n"); + auto printer = Lexer((const char *) source_line, + file_name, + source_line.size()); + // Log error line + Console::Log(Colors::RESET, line, " |\t "); + printer.lex(); // Print Failed Token + + Console::Log("\n"); + + // Align space + FOR(i, line.length() + 1) Console::Log(" "); + + Console::Log("\t", Colors::RED); + FOR(column, source_line.size()) { + if (column == error.failed_token.getSourceLocation().column) { + FOR(ch, error.failed_token.name().size()) { + Console::Log("^"); + } + break; + } + Console::Log(" "); + } + + Console::Log("_ <-- ", message); + Console::Log("\n\n"); } template @@ -444,59 +444,58 @@ void Diagnostic::ErrorParseSetup(size_t const ln, const char *message, Token const *reported_token, TokenTraits::SourceLocation loc) { - std::string line = std::to_string(ln); - if (ln > 1) { - Console::Log(Colors::RESET, ln - 1, " | "); - - str new_line; - - if (ln - 1 == 1) { - new_line = str(file_buf, source_line.begin() - file_buf - 1); - } - else { - new_line = GetNthLineOfBuffer(ln - 1, file_buf, buf_len); - } - - Lexer(new_line.begin(), - file_name, - new_line.size()).lex(); - Console::Log("\n"); - } - - auto printer = Lexer((const char *) source_line, - file_name, - source_line.size()); - // Log error line - Console::Log(Colors::RESET, line, " | "); - printer.lex(); // Print Failed Token - - Console::Log("\n"); - - // Align space - FOR(i, line.length()+1) Console::Log(" "); - - Console::Log(" ", Colors::RED); - - auto size = reported_token->getKind() == EOFToken ? source_line.size()+2 : - source_line.size(); - - size_t nth_column; - if (POINT) nth_column = loc.column-1; - else nth_column = loc.column; - for(size_t column = 1; column <= size; ++column) { - Console::Log(" "); - if (column == loc.column-1) { - FOR(ch, reported_token->name().size()) { - Console::Log("^"); - } - break; - } - } - - if (!POINT) - Console::Log("_ <-- ", message); - else Console::Log(" <-- ", message); - Console::Log("\n\n"); + std::string line = std::to_string(ln); + if (ln > 1) { + Console::Log(Colors::RESET, ln - 1, " | "); + + str new_line; + + if (ln - 1 == 1) { + new_line = str(file_buf, source_line.begin() - file_buf - 1); + } else { + new_line = GetNthLineOfBuffer(ln - 1, file_buf, buf_len); + } + + Lexer(new_line.begin(), + file_name, + new_line.size()).lex(); + Console::Log("\n"); + } + + auto printer = Lexer((const char *) source_line, + file_name, + source_line.size()); + // Log error line + Console::Log(Colors::RESET, line, " | "); + printer.lex(); // Print Failed Token + + Console::Log("\n"); + + // Align space + FOR(i, line.length() + 1) Console::Log(" "); + + Console::Log(" ", Colors::RED); + + auto size = reported_token->getKind() == EOFToken ? source_line.size() + 2 : + source_line.size(); + + size_t nth_column; + if (POINT) nth_column = loc.column - 1; + else nth_column = loc.column; + for (size_t column = 1; column <= size; ++column) { + Console::Log(" "); + if (column == loc.column - 1) { + FOR(ch, reported_token->name().size()) { + Console::Log("^"); + } + break; + } + } + + if (!POINT) + Console::Log("_ <-- ", message); + else Console::Log(" <-- ", message); + Console::Log("\n\n"); } diff --git a/lib/jet-lexer-v1.cpp b/lib/jet-lexer-v1.cpp index 0c27117..2ad02e8 100644 --- a/lib/jet-lexer-v1.cpp +++ b/lib/jet-lexer-v1.cpp @@ -6,1260 +6,1309 @@ using namespace nextgen::jet; # define TOKEN(type, text, k, value) Token(text, {line, column} \ ,value, k) -static constexpr TokenKind TokenKindClass[256] { +static constexpr TokenKind TokenKindClass[256]{ - // Error code - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, - TokenKind::Whitespace, // \a - TokenKind::Whitespace, // \b - TokenKind::Whitespace, // \t - TokenKind::NewLine, // \n - TokenKind::Whitespace, // \v - TokenKind::Error, + // Error code + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Whitespace, // \a + TokenKind::Whitespace, // \b + TokenKind::Whitespace, // \t + TokenKind::NewLine, // \n + TokenKind::Whitespace, // \v + TokenKind::Error, - TokenKind::NewLine, // \r + TokenKind::NewLine, // \r - TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - - TokenKind::Whitespace, // - - TokenKind::ExclamationPoint, // ! - TokenKind::String, // '"' - TokenKind::Error, TokenKind::Error, - TokenKind::Percent, // '%' - TokenKind::AND, // '&' - TokenKind::Char, // '\'' - TokenKind::LParenthesis, // ( - TokenKind::RParenthesis, // ) - TokenKind::Star, // '*' - TokenKind::Plus, // '+' - TokenKind::Comma, // ',' - - TokenKind::Minus, // '-' - - - TokenKind::Dot, // '.' - - TokenKind::Slash, // '/' - - - // Digits - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - - - TokenKind::Colon, // ':' - - - TokenKind::SemiColon, // ';' - TokenKind::LessThan, // '<' - TokenKind::Equals, // '=' - TokenKind::GreaterThan, // '>' - TokenKind::QuestionMark, // '?' - TokenKind::At, // @ - - // Uppercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LBracket, // '[' - - TokenKind::StringEscapeStart, // \\ - - TokenKind::RBracket, // ']' - - TokenKind::XOR, // '^' - - TokenKind::Identifier,// '_' - - TokenKind::Error, - - // Lowercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LCurlyBrace, // '{' - TokenKind::Pipe, // '|' - TokenKind::RCurlyBrace, // '}' - TokenKind::NOT, // '~' - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + + TokenKind::Whitespace, // + + TokenKind::ExclamationPoint, // ! + TokenKind::String, // '"' + TokenKind::Error, TokenKind::Error, + TokenKind::Percent, // '%' + TokenKind::AND, // '&' + TokenKind::Char, // '\'' + TokenKind::LParenthesis, // ( + TokenKind::RParenthesis, // ) + TokenKind::Star, // '*' + TokenKind::Plus, // '+' + TokenKind::Comma, // ',' + + TokenKind::Minus, // '-' + + + TokenKind::Dot, // '.' + + TokenKind::Slash, // '/' + + + // Digits + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + + + TokenKind::Colon, // ':' + + + TokenKind::SemiColon, // ';' + TokenKind::LessThan, // '<' + TokenKind::Equals, // '=' + TokenKind::GreaterThan, // '>' + TokenKind::QuestionMark, // '?' + TokenKind::At, // @ + + // Uppercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LBracket, // '[' + + TokenKind::StringEscapeStart, // \\ + + TokenKind::RBracket, // ']' + + TokenKind::XOR, // '^' + + TokenKind::Identifier,// '_' + + TokenKind::Error, + + // Lowercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LCurlyBrace, // '{' + TokenKind::Pipe, // '|' + TokenKind::RCurlyBrace, // '}' + TokenKind::NOT, // '~' + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, }; constexpr static int IntegerBits[256] = { - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, - // Digits - 0,1,2,3,4,5,6,7,8,9, + // Digits + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - // Error Code - -3,-3,-3,-3,-3,-3,-3, + // Error Code + -3, -3, -3, -3, -3, -3, -3, - // Uppercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35, + // Uppercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, - // Lowercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35, + // Lowercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, }; struct JetKeyword { - str key; - TokenKind id; + str key; + TokenKind id; }; -static const struct JetKeyword Keywords[] { - { "if"_intern, KeywordIf }, - { "else"_intern, KeywordElse }, - { "elif"_intern, KeywordElif }, - { "while"_intern, KeywordWhile }, - { "for"_intern, KeywordFor }, - { "true"_intern, KeywordTrue }, - { "false"_intern, KeywordFalse }, - { "None"_intern, KeywordNone }, - { "fn"_intern, KeywordFunction }, - { "struct"_intern, KeywordStruct }, - { "export"_intern, KeywordExport }, - { "enum"_intern, KeywordEnum }, - { "extern"_intern, KeywordExtern }, - { "error"_intern, KeywordError }, - { "var"_intern, KeywordVar }, - { "return"_intern, KeywordReturn }, - { "and"_intern, KeywordAnd }, - { "or"_intern, KeywordOr }, - { "break"_intern, KeywordBreak }, - { "continue"_intern, KeywordContinue }, - { "defer"_intern, KeywordDefer }, - { "union"_intern, KeywordUnion }, - { "match"_intern, KeywordMatch }, - { "in"_intern, KeywordIn } +static const struct JetKeyword Keywords[]{ + {"if"_intern, KeywordIf}, + {"else"_intern, KeywordElse}, + {"elif"_intern, KeywordElif}, + {"while"_intern, KeywordWhile}, + {"for"_intern, KeywordFor}, + {"true"_intern, KeywordTrue}, + {"false"_intern, KeywordFalse}, + {"None"_intern, KeywordNone}, + {"fn"_intern, KeywordFunction}, + {"struct"_intern, KeywordStruct}, + {"export"_intern, KeywordExport}, + {"enum"_intern, KeywordEnum}, + {"extern"_intern, KeywordExtern}, + {"error"_intern, KeywordError}, + {"var"_intern, KeywordVar}, + {"return"_intern, KeywordReturn}, + {"and"_intern, KeywordAnd}, + {"or"_intern, KeywordOr}, + {"break"_intern, KeywordBreak}, + {"continue"_intern, KeywordContinue}, + {"defer"_intern, KeywordDefer}, + {"union"_intern, KeywordUnion}, + {"match"_intern, KeywordMatch}, + {"in"_intern, KeywordIn} }; /// Determine the validity of parsed string being a Keyword. The assumption /// made in this function is that `value` is an interned string therefore only /// direct pointer comparison is made resulting in O(1) comparison. static TokenKind ReturnValidKeyword(str &value) { - for (auto i = 0; i < SizeOfArray(Keywords); ++i) { - if (value == Keywords[i].key) - return Keywords[i].id; - } - return TokenKind::Identifier; + for (auto i = 0; i < SizeOfArray(Keywords); ++i) { + if (value == Keywords[i].key) + return Keywords[i].id; + } + return TokenKind::Identifier; } template -Token Lexer::next_token() { - TokenKind kind = TokenKindClass[curr()]; - Token instance = Token {}; - - goto start; - start: - { - switch (kind) { - - /// - /// Integers - /// - /// Pattern: Key Lexing Statutes - /// â€ĸ '_' is valid anywhere in the integer as long as it's not the - /// first character. Examples: - /// â€ĸ 123_232_2423 - /// â€ĸ 234__343 - /// â€ĸ 3__23_3_23_2_2 - /// â€ĸ Identifiers are not interned for integers. This is because they - /// do not provide any reasonable performance gain. - /// â€ĸ All number values with suffixes of: 'e' or 'p' are floating - /// point values unless casted (handled in Parser/AST). - /// - /// Valid Patterns: - /// â€ĸ 0x123 (Hexadecimal) - /// â€ĸ 12345 (Base 10) - /// â€ĸ 0b011 (Base 2) - /// â€ĸ 0#abc (Base 36) - /// - /// - case Integer: { - - // Integer Base - int radix = 10; - - // Classification for Unsigned values - auto flags = static_cast(TokenClassification::Literal); - - // Start of token - auto Begin = buffer; - auto BeginCol = column; - switch (curr()) { - case '0': - switch (next(1)) { - case 'b': - radix = 2; - next(1); - break; - case 'x': - radix = 16; - next(1); - break; - case '#': - radix = 36; - next(1); - break; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': - radix = 8; - break; - default: // TODO: Figure what needs to be done here - break; - } - break; - default: - break; - } - - - decltype(UINTPTR_MAX) IntegerValue = 0; - do { - - auto Current = curr(); - int Digit = IntegerBits[Current]; - - if (Digit < 0) { - break; - } - - if (Digit >= radix) { - diagnostics.build(LexError{ - LexErrorType::DigitOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, TokenKind::Integer), - {radix, Digit + 1} - }); - fatal = true; - return instance; - } - - if (IntegerValue > (UINTPTR_MAX - Digit) / radix) { - diagnostics.build(LexError{ - LexErrorType::IntegerOverflow, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::Integer), - }); - fatal = true; - return instance; - } - IntegerValue = IntegerValue * radix + Digit; - } while (next(1)); - - // Token Suffix - switch (curr()) { // TODO: Add Implementation - case 'p': // Power - break; - case 'u': // Unsigned - flags = TokenClassification::Unsigned; - next(1); - break; - case 'e': // x10^power - if (radix == 16) UNREACHABLE; - break; - } - - // End of Token - auto End = buffer; - - - // Important Note: When integer values are lexed, they do not hold - // a direct value stored in a buffer, rather just a pseudo range value - // with no allocation is substituted because integer value strings do - // need to be interned. - auto ID = str(Range(Begin, End)); - - instance = TOKEN(decltype(UINTPTR_MAX), ID, TokenKind::Integer, - IntegerValue); - instance.setFlag(flags); - return instance; - } - case String: { - auto Begin = this->buffer; - auto BeginCol = column; - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - char Current = next(1); - - // Equivalent to C99 String Standard - auto StringAllocator = allocator->allocator_for_space(4095); - - char *ID = (char*) StringAllocator->current_point(); - size_t Len = 0; - - do { - Current = curr(); - - if (Current == '\0') { - diagnostics.build(LexError{ - LexErrorType::MissingClosingDelim, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {'"'} - }); - fatal = true; - return instance; - } - - if (Current == '\\') { - switch (next(1)) { - case 'a': - Current = '\a'; - break; - case 'b': - Current = '\b'; - break; - case 'r': - Current = '\r'; - break; - case 'n': - Current = '\n'; - break; - case 'v': - Current = '\v'; - break; - case 't': - Current = '\t'; - break; - case 'x': { - auto V1 = IntegerBits[next(1)]; - if (V1 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {V1 + 1} // Pass in the invalid digit for the escape - }); - fatal = true; - return instance; - } - auto V2 = IntegerBits[next(1)]; - if (V2 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {V2 + 1} // Pass in the invalid digit for the escape - }); - fatal = true; - return instance; - } - - // Unroll Loop - int Value = V1; - Value = Value * 16 + V2; - - // @Safety Guaranteed! We know for a fact that this conversion - // is safe because of the validity. - Current = (char) Value; - break; - } - case 'u': // TODO: IDK how to do this properly but ... Unicode Char - break; - default: - diagnostics.build(LexError{ - LexErrorType::InvalidStringEscape, - {line, column}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - }); - fatal = true; - return instance; - } - } - - val ^= Current; - val *= FNV_PRIME; - - ID[Len++] = Current; - - - } while (next(1) != '"'); - - ID[Len++] = '\0'; - - auto S = str(ID, Len); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - instance = TOKEN(const char *, Intern, TokenKind::String, ""); - instance.setFlag(TokenClassification::Literal); - next(1); // Skip the last '"' - return instance; - } - case Decimal: - break; - case Identifier: { - - - auto Begin = this->buffer; - - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - char Current; - do { - Current = curr(); - - val ^= Current; - val *= FNV_PRIME; - - - } while (TokenKindClass[next(1)] == TokenKind::Identifier); - - auto End = this->buffer; - - auto S = str(Begin, End-Begin); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - // Ensure appropriate kind is assigned - auto Type = ReturnValidKeyword(Intern); - - instance = TOKEN(const char *, Intern, Type, ""); - if (Type != TokenKind::Identifier) - instance.setFlag(TokenClassification::Keyword); - instance.setFlag(TokenClassification::Literal); - - return instance; - } - case LessThan: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::LessThan, TokenKind::Equals): - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - break; - group(TokenKind::LessThan, TokenKind::LessThan): // << <-- - kind = TokenKind::LeftShift; - goto start; - default: - instance = TOKEN(char, "<", TokenKind::LessThan, "<"); - break; - } - - auto s2 = peek(1); - match ('<', s2) { - group('<', '='): - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - break; - group('<', '<'): // << <-- - kind = TokenKind::LeftShift; - goto start; - default: - instance = TOKEN(char, "<", TokenKind::LessThan, "<"); - break; - } - if (s2 == '=') { - instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, - "<="); - instance.assignment = true; - } - break; - } - case LeftShift: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::LeftShift, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, "<<=", TokenKind::LeftShiftEquals, - "<<="); - break; - default: - instance = TOKEN(const char *, "<<", TokenKind::LeftShift, "<<"); - break; - } - break; - } - case RightShift: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::RightShift, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, ">>=", TokenKind::RightShiftEquals, - ">>="); - break; - default: - instance = TOKEN(const char *, ">>", TokenKind::RightShift, ">>"); - break; - } - break; - } - case GreaterThan: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::GreaterThan, TokenKind::Equals): - instance = TOKEN(const char *, ">=", TokenKind::GreaterThanEquals, - ">="); - instance.assignment = true; - break; - group(TokenKind::GreaterThan, TokenKind::GreaterThan): - kind = TokenKind::RightShift; - goto start; - default: - instance = TOKEN(char, ">", TokenKind::GreaterThan, ">"); - break; - } - break; - } - case Equals: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::Equals) { - instance = TOKEN(const char *, "==", TokenKind::EqualsEquals, "=="); - } else { - instance = TOKEN(char, "=", TokenKind::Equals, "="); - } - break; - } - case QuestionMark: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::QuestionMark) { - instance = TOKEN(const char *, "??", TokenKind::Then, "??"); - } else { - instance = TOKEN(char, "?", TokenKind::QuestionMark, "?"); - } - break; - } - case Plus: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Plus, TokenKind::Plus): - instance = TOKEN(const char *, "++", TokenKind::PlusPlus, "++"); - break; - group(TokenKind::Plus, TokenKind::Equals): - instance = TOKEN(const char *, "+=", TokenKind::PlusEquals, "+="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "+", TokenKind::Plus, "+"); - break; - } - break; - } - case Minus: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Minus, TokenKind::Minus): - instance = TOKEN(const char *, "--", TokenKind::MinusMinus, "--"); - break; - group(TokenKind::Minus, TokenKind::Equals): - instance = TOKEN(const char *, "-=", TokenKind::MinusEquals, "-="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "-", TokenKind::Minus, "-"); - break; - } - break; - } - case Slash: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Slash, TokenKind::Equals): - instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "/", TokenKind::Star, "/"); - break; - } - break; - } - case Star: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Star, TokenKind::Star): - kind = TokenKind::Pow; - goto start; - group(TokenKind::Star, TokenKind::Equals): - instance = TOKEN(const char *, "*=", TokenKind::MulEquals, "*="); - instance.assignment = true; - break; - default: - instance = TOKEN(const char *, "*", TokenKind::Star, "*"); - break; - } - break; - } - case Pow: { - auto s1 = TokenKindClass[peek(2)]; - match (kind, s1) { - group(TokenKind::Pow, TokenKind::Equals): - instance.assignment = true; - instance = TOKEN(const char *, "**=", TokenKind::PowEquals, "**="); - break; - default: - instance = TOKEN(const char *, "**", TokenKind::Pow, "**"); - break; - } - break; - } - case AND: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::AND, TokenKind::Equals): - instance = TOKEN(const char *, "&=", TokenKind::ANDEquals, "&="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "&", TokenKind::AND, "&"); - break; - } - break; - } - case Pipe: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::Pipe, TokenKind::Equals): - instance = TOKEN(const char *, "|=", TokenKind::OREquals, "|="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "|", TokenKind::Pipe, "|"); - break; - } - break; - } - case XOR: { - auto s1 = TokenKindClass[peek(1)]; - match (kind, s1) { - group(TokenKind::XOR, TokenKind::Equals): - instance = TOKEN(const char *, "^=", TokenKind::XOREquals, "^="); - instance.assignment = true; - break; - default: - instance = TOKEN(char, "^", TokenKind::XOR, "^"); - break; - } - break; - } - case NewLine: // \n or \r - skip_new_line(); - kind = TokenKindClass[curr()]; - goto start; // Does this so 2 function calls aren't wasted. - case Char: { - auto Begin = buffer; - auto BeginCol = column; - instance = TOKEN(char, next(1), TokenKind::Char, curr()); - if (TokenKindClass[next(1)] != TokenKind::Char) { - diagnostics.build(LexError{ - LexErrorType::MissingClosingDelim, - {line, column}, // 'x[?] <-- Missing Closing! - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::String), - {'\''} - }); - fatal = true; - } - instance.setFlag(TokenClassification::Literal); - break; - } - case Whitespace: // Skip whitespace - next(1); - kind = TokenKindClass[curr()]; - goto start; - case Dot: { - auto Begin = buffer; - auto BeginCol = column; - - auto s1 = TokenKindClass[peek(1)]; - auto s2 = TokenKindClass[peek(2)]; - auto s3 = TokenKindClass[peek(3)]; - - match (kind, s1) { - group(TokenKind::Dot, TokenKind::Dot): - instance = s2 == TokenKind::Dot ? - (TOKEN(const char *, "...", - TokenKind::Ellipsis, "...")) : - (TOKEN(const char *, "..", - TokenKind::RangeSpan, "..")); - break; - default: - instance = TOKEN(char, ".", TokenKind::Dot, "."); - break; - } - // Cannot match token "...[.]" <-- Invalid Dot Here! - if (s3 == TokenKind::Dot && instance.getKind() == TokenKind::Ellipsis) { - diagnostics.build(LexError{ - LexErrorType::InvalidDot, - {line, column + 1}, - Token(str(Range(Begin, buffer)), - {line, BeginCol}, - TokenKind::Ellipsis) - }); - fatal = true; - } - - break; - } - // Cannot be generated by the TokenKind table, assigned after checks - case EqualsEquals: - case Boolean: - UNREACHABLE; - break; - case LParenthesis: - instance = TOKEN(const char *, "(", kind, ")"); - break; - case RParenthesis: - instance = TOKEN(const char *, ")", kind, ")"); - break; - case LBracket: - instance = TOKEN(const char *, "[", kind, "["); - break; - case RBracket: - instance = TOKEN(const char *, "]", kind, "]"); - break; - case LCurlyBrace: - instance = TOKEN(const char *, "{", kind, "{"); - break; - case RCurlyBrace: - instance = TOKEN(const char *, "}", kind, "}"); - break; - case NOT: - instance = TOKEN(const char *, "~", kind, "~"); - break; - case Colon: - instance = TOKEN(const char *, ":", kind, ":"); - break; - case SemiColon: - instance = TOKEN(const char *, ";", kind, ";"); - break; - case ExclamationPoint: - instance = TOKEN(const char *, "!", kind, "!"); - break; - case At: - instance = TOKEN(const char *, "@", kind, "@"); - break; - case Percent: - instance = TOKEN(const char *, "%", kind, "%"); - break; - case Comma: - instance = TOKEN(const char *, ",", kind, ","); - break; - case Error: - if (curr() == '\0') { - buf_pos++; - return Token("\0", {line, column}, - TokenKind::EOFToken); - } - diagnostics.build(LexError{ - LexErrorType::InvalidChar, - {line, column} - }); - fatal = true; - default: - //instance = TOKEN(char, Curr(), kind, curr()); - break; - } - } - next(instance.len()); - return instance; +Token Lexer::next_token() { + TokenKind kind = TokenKindClass[curr()]; + Token instance = Token{}; + + goto start; + start: + { + switch (kind) { + + /// + /// Integers + /// + /// Pattern: Key Lexing Statutes + /// â€ĸ '_' is valid anywhere in the integer as long as it's not the + /// first character. Examples: + /// â€ĸ 123_232_2423 + /// â€ĸ 234__343 + /// â€ĸ 3__23_3_23_2_2 + /// â€ĸ Identifiers are not interned for integers. This is because they + /// do not provide any reasonable performance gain. + /// â€ĸ All number values with suffixes of: 'e' or 'p' are floating + /// point values unless casted (handled in Parser/AST). + /// + /// Valid Patterns: + /// â€ĸ 0x123 (Hexadecimal) + /// â€ĸ 12345 (Base 10) + /// â€ĸ 0b011 (Base 2) + /// â€ĸ 0#abc (Base 36) + /// + /// + case Integer: { + + // Integer Base + int radix = 10; + + // Classification for Unsigned values + auto flags = static_cast(TokenClassification::Literal); + + // Start of token + auto Begin = buffer; + auto BeginCol = column; + switch (curr()) { + case '0': + switch (next(1)) { + case 'b': + radix = 2; + next(1); + break; + case 'x': + radix = 16; + next(1); + break; + case '#': + radix = 36; + next(1); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + radix = 8; + break; + default: // TODO: Figure what needs to be done here + break; + } + break; + default: + break; + } + + + decltype(UINTPTR_MAX) IntegerValue = 0; + do { + + auto Current = curr(); + int Digit = IntegerBits[Current]; + + if (Digit < 0) { + break; + } + + if (Digit >= radix) { + diagnostics.build(LexError{ + LexErrorType::DigitOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, TokenKind::Integer), + {radix, Digit + 1} + }); + fatal = true; + return instance; + } + + if (IntegerValue > (UINTPTR_MAX - Digit) / radix) { + diagnostics.build(LexError{ + LexErrorType::IntegerOverflow, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::Integer), + }); + fatal = true; + return instance; + } + IntegerValue = IntegerValue * radix + Digit; + } while (next(1)); + + // Token Suffix + switch (curr()) { // TODO: Add Implementation + case 'p': // Power + break; + case 'u': // Unsigned + flags = TokenClassification::Unsigned; + next(1); + break; + case 'e': // x10^power + if (radix == 16) UNREACHABLE; + break; + } + + // End of Token + auto End = buffer; + + + // Important Note: When integer values are lexed, they do not hold + // a direct value stored in a buffer, rather just a pseudo range value + // with no allocation is substituted because integer value strings do + // need to be interned. + auto ID = str(Range(Begin, End)); + + instance = TOKEN(decltype(UINTPTR_MAX), ID, TokenKind::Integer, + IntegerValue); + instance.setFlag(flags); + return instance; + } + case String: { + auto Begin = this->buffer; + auto BeginCol = column; + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + char Current = next(1); + + // Equivalent to C99 String Standard + auto StringAllocator = allocator->allocator_for_space(4095); + + char *ID = (char *) StringAllocator->current_point(); + size_t Len = 0; + + do { + Current = curr(); + + if (Current == '\0') { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {'"'} + }); + fatal = true; + return instance; + } + + if (Current == '\\') { + switch (next(1)) { + case 'a': + Current = '\a'; + break; + case 'b': + Current = '\b'; + break; + case 'r': + Current = '\r'; + break; + case 'n': + Current = '\n'; + break; + case 'v': + Current = '\v'; + break; + case 't': + Current = '\t'; + break; + case 'x': { + auto V1 = IntegerBits[next(1)]; + if (V1 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {V1 + 1} // Pass in the invalid digit for the escape + }); + fatal = true; + return instance; + } + auto V2 = IntegerBits[next(1)]; + if (V2 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {V2 + 1} // Pass in the invalid digit for the escape + }); + fatal = true; + return instance; + } + + // Unroll Loop + int Value = V1; + Value = Value * 16 + V2; + + // @Safety Guaranteed! We know for a fact that this conversion + // is safe because of the validity. + Current = (char) Value; + break; + } + case 'u': // TODO: IDK how to do this properly but ... Unicode Char + break; + default: + diagnostics.build(LexError{ + LexErrorType::InvalidStringEscape, + {line, column}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + }); + fatal = true; + return instance; + } + } + + val ^= Current; + val *= FNV_PRIME; + + ID[Len++] = Current; + + + } while (next(1) != '"'); + + ID[Len++] = '\0'; + + auto S = str(ID, Len); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + instance = TOKEN(const char *, Intern, TokenKind::String, ""); + instance.setFlag(TokenClassification::Literal); + next(1); // Skip the last '"' + return instance; + } + case Decimal: + break; + case Identifier: { + + + auto Begin = this->buffer; + + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + char Current; + do { + Current = curr(); + + val ^= Current; + val *= FNV_PRIME; + + + } while (TokenKindClass[next(1)] == TokenKind::Identifier); + + auto End = this->buffer; + + auto S = str(Begin, End - Begin); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + // Ensure appropriate kind is assigned + auto Type = ReturnValidKeyword(Intern); + + instance = TOKEN(const char *, Intern, Type, ""); + if (Type != TokenKind::Identifier) + instance.setFlag(TokenClassification::Keyword); + instance.setFlag(TokenClassification::Literal); + + return instance; + } + case LessThan: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::LessThan, TokenKind::Equals): + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + break; + group(TokenKind::LessThan, TokenKind::LessThan): // << <-- + kind = TokenKind::LeftShift; + goto start; + default: + instance = TOKEN(char, "<", TokenKind::LessThan, "<"); + break; + } + + auto s2 = peek(1); + match('<', s2) + { + group('<', '='): + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + break; + group('<', '<'): // << <-- + kind = TokenKind::LeftShift; + goto start; + default: + instance = TOKEN(char, "<", TokenKind::LessThan, "<"); + break; + } + if (s2 == '=') { + instance = TOKEN(const char *, "<=", TokenKind::LessThanEquals, + "<="); + instance.assignment = true; + } + break; + } + case LeftShift: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::LeftShift, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, "<<=", TokenKind::LeftShiftEquals, + "<<="); + break; + default: + instance = TOKEN(const char *, "<<", TokenKind::LeftShift, "<<"); + break; + } + break; + } + case RightShift: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::RightShift, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, ">>=", TokenKind::RightShiftEquals, + ">>="); + break; + default: + instance = TOKEN(const char *, ">>", TokenKind::RightShift, ">>"); + break; + } + break; + } + case GreaterThan: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::GreaterThan, TokenKind::Equals): + instance = TOKEN(const char *, ">=", TokenKind::GreaterThanEquals, + ">="); + instance.assignment = true; + break; + group(TokenKind::GreaterThan, TokenKind::GreaterThan): + kind = TokenKind::RightShift; + goto start; + default: + instance = TOKEN(char, ">", TokenKind::GreaterThan, ">"); + break; + } + break; + } + case Equals: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::Equals) { + instance = TOKEN(const char *, "==", TokenKind::EqualsEquals, "=="); + } else { + instance = TOKEN(char, "=", TokenKind::Equals, "="); + } + break; + } + case QuestionMark: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::QuestionMark) { + instance = TOKEN(const char *, "??", TokenKind::Then, "??"); + } else { + instance = TOKEN(char, "?", TokenKind::QuestionMark, "?"); + } + break; + } + case Plus: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Plus, TokenKind::Plus): + instance = TOKEN(const char *, "++", TokenKind::PlusPlus, "++"); + break; + group(TokenKind::Plus, TokenKind::Equals): + instance = TOKEN(const char *, "+=", TokenKind::PlusEquals, "+="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "+", TokenKind::Plus, "+"); + break; + } + break; + } + case Minus: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Minus, TokenKind::Minus): + instance = TOKEN(const char *, "--", TokenKind::MinusMinus, "--"); + break; + group(TokenKind::Minus, TokenKind::Equals): + instance = TOKEN(const char *, "-=", TokenKind::MinusEquals, "-="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "-", TokenKind::Minus, "-"); + break; + } + break; + } + case Slash: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Slash, TokenKind::Equals): + instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "/", TokenKind::Star, "/"); + break; + } + break; + } + case Star: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Star, TokenKind::Star): + kind = TokenKind::Pow; + goto start; + group(TokenKind::Star, TokenKind::Equals): + instance = TOKEN(const char *, "*=", TokenKind::MulEquals, "*="); + instance.assignment = true; + break; + default: + instance = TOKEN(const char *, "*", TokenKind::Star, "*"); + break; + } + break; + } + case Pow: { + auto s1 = TokenKindClass[peek(2)]; + match(kind, s1) + { + group(TokenKind::Pow, TokenKind::Equals): + instance.assignment = true; + instance = TOKEN(const char *, "**=", TokenKind::PowEquals, "**="); + break; + default: + instance = TOKEN(const char *, "**", TokenKind::Pow, "**"); + break; + } + break; + } + case AND: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::AND, TokenKind::Equals): + instance = TOKEN(const char *, "&=", TokenKind::ANDEquals, "&="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "&", TokenKind::AND, "&"); + break; + } + break; + } + case Pipe: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::Pipe, TokenKind::Equals): + instance = TOKEN(const char *, "|=", TokenKind::OREquals, "|="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "|", TokenKind::Pipe, "|"); + break; + } + break; + } + case XOR: { + auto s1 = TokenKindClass[peek(1)]; + match(kind, s1) + { + group(TokenKind::XOR, TokenKind::Equals): + instance = TOKEN(const char *, "^=", TokenKind::XOREquals, "^="); + instance.assignment = true; + break; + default: + instance = TOKEN(char, "^", TokenKind::XOR, "^"); + break; + } + break; + } + case NewLine: // \n or \r + skip_new_line(); + kind = TokenKindClass[curr()]; + goto start; // Does this so 2 function calls aren't wasted. + case Char: { + auto Begin = buffer; + auto BeginCol = column; + instance = TOKEN(char, next(1), TokenKind::Char, curr()); + if (TokenKindClass[next(1)] != TokenKind::Char) { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, // 'x[?] <-- Missing Closing! + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::String), + {'\''} + }); + fatal = true; + } + instance.setFlag(TokenClassification::Literal); + break; + } + case Whitespace: // Skip whitespace + next(1); + kind = TokenKindClass[curr()]; + goto start; + case Dot: { + auto Begin = buffer; + auto BeginCol = column; + + auto s1 = TokenKindClass[peek(1)]; + auto s2 = TokenKindClass[peek(2)]; + auto s3 = TokenKindClass[peek(3)]; + + match(kind, s1) + { + group(TokenKind::Dot, TokenKind::Dot): + instance = s2 == TokenKind::Dot ? + (TOKEN(const char *, "...", + TokenKind::Ellipsis, "...")) : + (TOKEN(const char *, "..", + TokenKind::RangeSpan, "..")); + break; + default: + instance = TOKEN(char, ".", TokenKind::Dot, "."); + break; + } + // Cannot match token "...[.]" <-- Invalid Dot Here! + if (s3 == TokenKind::Dot && instance.getKind() == TokenKind::Ellipsis) { + diagnostics.build(LexError{ + LexErrorType::InvalidDot, + {line, column + 1}, + Token(str(Range(Begin, buffer)), + {line, BeginCol}, + TokenKind::Ellipsis) + }); + fatal = true; + } + + break; + } + // Cannot be generated by the TokenKind table, assigned after checks + case EqualsEquals: + case Boolean: + UNREACHABLE; + break; + case LParenthesis: + instance = TOKEN(const char *, "(", kind, ")"); + break; + case RParenthesis: + instance = TOKEN(const char *, ")", kind, ")"); + break; + case LBracket: + instance = TOKEN(const char *, "[", kind, "["); + break; + case RBracket: + instance = TOKEN(const char *, "]", kind, "]"); + break; + case LCurlyBrace: + instance = TOKEN(const char *, "{", kind, "{"); + break; + case RCurlyBrace: + instance = TOKEN(const char *, "}", kind, "}"); + break; + case NOT: + instance = TOKEN(const char *, "~", kind, "~"); + break; + case Colon: + instance = TOKEN(const char *, ":", kind, ":"); + break; + case SemiColon: + instance = TOKEN(const char *, ";", kind, ";"); + break; + case ExclamationPoint: + instance = TOKEN(const char *, "!", kind, "!"); + break; + case At: + instance = TOKEN(const char *, "@", kind, "@"); + break; + case Percent: + instance = TOKEN(const char *, "%", kind, "%"); + break; + case Comma: + instance = TOKEN(const char *, ",", kind, ","); + break; + case Error: + if (curr() == '\0') { + buf_pos++; + return Token("\0", {line, column}, + TokenKind::EOFToken); + } + diagnostics.build(LexError{ + LexErrorType::InvalidChar, + {line, column} + }); + fatal = true; + default: + //instance = TOKEN(char, Curr(), kind, curr()); + break; + } + } + next(instance.len()); + return instance; } template void Lexer::PrintNextToken() { - TokenKind Kind = TokenKindClass[curr()]; - Token Instance = Token {}; - - goto start; - start: - { - if (!buffer) - return; - switch (Kind) { - - case Integer: { - - // Start of token - auto Begin = buffer; - - switch (curr()) { - case '0': - switch (next(1)) { - case 'b': - next(1); - break; - case 'x': - next(1); - break; - case '#': - next(1); - break; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': - break; - default: - break; - } - break; - default: - break; - } - - do { - - auto Current = curr(); - int Digit = IntegerBits[Current]; - - if (Digit < 0) { - break; - } - - } while (next(1)); - - // Token Suffix - switch (curr()) { // TODO: Add Implementation - case 'p': // Power - break; - case 'u': // Unsigned - break; - case 'e': // x10^power - break; - } - - auto End = buffer; - - auto Print = str(Begin, End - Begin); - Console::Log(Colors::BLUE, Print, Colors::RESET); - - break; - } - case String: { - auto Begin = this->buffer; - int Offset = 1; - Console::Log(Colors::GREEN); - - char Current; - do { - Current = curr(); - - if (Current == '\0') { - Offset = 0; - break; - } - } while (next(1) != '"'); - - - auto End = this->buffer; - - auto Print = str(Begin, End - Begin + Offset); - Console::Log(Print, Colors::RESET); - - next(1); // Skip the last '"' - break; - } - case Decimal: - break; - case Identifier: { // Maybe : Identifier length no more 256 characters? - - - auto Begin = this->buffer; - - // FNV-1a Hash on String as Lexing as to not do it twice - // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function - auto val = FNV_OFF; - do { - - val ^= curr(); - val *= FNV_PRIME; - } while (TokenKindClass[next(1)] == TokenKind::Identifier); - - - auto End = this->buffer; - - auto S = str(Begin, End-Begin); - S.setHash(val); - auto Intern = StringCache::InsertOrGet(S); - - // Ensure appropriate kind is assigned - auto Type = ReturnValidKeyword(Intern); - - if (Type != TokenKind::Identifier) { - Console::Log(Colors::RED, Intern, Colors::RESET); - } else { - - Console::Log(Colors::YELLOW, Intern, Colors::RESET); - } - break; - } - case LessThan: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::LessThan, TokenKind::Equals): - Console::Log("<="); - next(2); - break; - group(TokenKind::LessThan, TokenKind::LessThan): - Kind = TokenKind::LeftShift; - goto start; - default: - Console::Log("<"); - next(1); - break; - } - break; - } - case LeftShift: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::LeftShift, TokenKind::Equals): - Console::Log("<<="); - next(3); - break; - default: - Console::Log("<<"); - next(2); - break; - } - break; - } - case RightShift: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::RightShift, TokenKind::Equals): - Console::Log(">>="); - next(3); - break; - default: - Console::Log(">>"); - next(2); - break; - } - break; - } - case GreaterThan: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::GreaterThan, TokenKind::Equals): - Console::Log(">="); - next(2); - break; - group(TokenKind::GreaterThan, TokenKind::GreaterThan): - Kind = TokenKind::RightShift; - goto start; - default: - Console::Log(">"); - next(1); - break; - } - break; - } - case Equals: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::Equals) { - Console::Log("=="); - next(1); - } else { - Console::Log("="); - } - next(1); - break; - } - case QuestionMark: { - auto s1 = TokenKindClass[peek(1)]; - if (s1 == TokenKind::QuestionMark) { - Console::Log("??"); - next(2); - } else { - Console::Log("?"); - next(1); - } - break; - } - case Plus: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Plus, TokenKind::Plus): - Console::Log("++"); - next(2); - break; - group(TokenKind::Plus, TokenKind::Equals): - Console::Log("+="); - next(2); - break; - default: - Console::Log("+"); - next(1); - break; - } - break; - } - case Minus: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Minus, TokenKind::Minus): - Console::Log("--"); - next(2); - break; - group(TokenKind::Minus, TokenKind::Equals): - Console::Log("-="); - next(2); - break; - default: - Console::Log("-"); - next(1); - break; - } - break; - } - case Slash: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Slash, TokenKind::Equals): - Instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); - Instance.assignment = true; - break; - default: - Instance = TOKEN(const char *, "/", TokenKind::Slash, "/"); - break; - } - break; - } - case Star: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Star, TokenKind::Star): - Kind = TokenKind::Pow; - goto start; - group(TokenKind::Star, TokenKind::Equals): - Console::Log("*="); - break; - default: - Console::Log('*'); - break; - } - break; - } - case Pow: { - auto s1 = TokenKindClass[peek(2)]; - match (Kind, s1) { - group(TokenKind::Pow, TokenKind::Equals): - Console::Log("**="); - next(3); - break; - default: - Console::Log("**"); - next(2); - break; - } - break; - } - case AND: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::AND, TokenKind::Equals): - Console::Log("/="); - next(2); - break; - default: - Console::Log('/'); - next(1); - break; - } - break; - } - case Pipe: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::Pipe, TokenKind::Equals): - Console::Log("|="); - next(2); - break; - default: - Console::Log('|'); - next(1); - break; - } - break; - } - case XOR: { - auto s1 = TokenKindClass[peek(1)]; - match (Kind, s1) { - group(TokenKind::XOR, TokenKind::Equals): - Console::Log("^="); - next(2); - break; - default: - Console::Log('^'); - next(1); - break; - } - break; - } - case NewLine: // \n or \r - Console::Log(curr()); - skip_new_line(); - Kind = TokenKindClass[curr()]; - goto start; // Does this so 2 function calls aren't wasted. - case Char: - Console::Log(Colors::GREEN, "'", next(1), Colors::RESET); - if (peek(1) == '\'') - Console::Log("'"); - next(1); - break; - case Whitespace: // Skip whitespace - Console::Log(curr()); - next(1); - Kind = TokenKindClass[curr()]; - goto start; - case Dot: { - auto s1 = TokenKindClass[peek(1)]; - auto s2 = TokenKindClass[peek(2)]; - - match (Kind, s1) { - group(TokenKind::Dot, TokenKind::Dot): - s2 == TokenKind::Dot ? - next(3), Console::Log("...") : - (next(2), Console::Log("..")); - break; - default: - Console::Log("."); - next(1); - break; - } - break; - } - // Cannot be generated by the TokenKind table, assigned after checks - case EqualsEquals: - case Boolean: - UNREACHABLE; - break; - case LParenthesis: - case RParenthesis: - case LBracket: - case RBracket: - case LCurlyBrace: - case RCurlyBrace: - case NOT: - case Colon: - case SemiColon: - case ExclamationPoint: - case At: - case Comma: - Console::Log(curr()); - next(1); - break; - case Error: - buffer = nullptr; - break; - default: - break; - } - } + TokenKind Kind = TokenKindClass[curr()]; + Token Instance = Token{}; + + goto start; + start: + { + if (!buffer) + return; + switch (Kind) { + + case Integer: { + + // Start of token + auto Begin = buffer; + + switch (curr()) { + case '0': + switch (next(1)) { + case 'b': + next(1); + break; + case 'x': + next(1); + break; + case '#': + next(1); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + break; + default: + break; + } + break; + default: + break; + } + + do { + + auto Current = curr(); + int Digit = IntegerBits[Current]; + + if (Digit < 0) { + break; + } + + } while (next(1)); + + // Token Suffix + switch (curr()) { // TODO: Add Implementation + case 'p': // Power + break; + case 'u': // Unsigned + break; + case 'e': // x10^power + break; + } + + auto End = buffer; + + auto Print = str(Begin, End - Begin); + Console::Log(Colors::BLUE, Print, Colors::RESET); + + break; + } + case String: { + auto Begin = this->buffer; + int Offset = 1; + Console::Log(Colors::GREEN); + + char Current; + do { + Current = curr(); + + if (Current == '\0') { + Offset = 0; + break; + } + } while (next(1) != '"'); + + + auto End = this->buffer; + + auto Print = str(Begin, End - Begin + Offset); + Console::Log(Print, Colors::RESET); + + next(1); // Skip the last '"' + break; + } + case Decimal: + break; + case Identifier: { // Maybe : Identifier length no more 256 characters? + + + auto Begin = this->buffer; + + // FNV-1a Hash on String as Lexing as to not do it twice + // https://en.wikipedia.org/wiki/Fowler–Noll–Vo_function + auto val = FNV_OFF; + do { + + val ^= curr(); + val *= FNV_PRIME; + } while (TokenKindClass[next(1)] == TokenKind::Identifier); + + + auto End = this->buffer; + + auto S = str(Begin, End - Begin); + S.setHash(val); + auto Intern = StringCache::InsertOrGet(S); + + // Ensure appropriate kind is assigned + auto Type = ReturnValidKeyword(Intern); + + if (Type != TokenKind::Identifier) { + Console::Log(Colors::RED, Intern, Colors::RESET); + } else { + + Console::Log(Colors::YELLOW, Intern, Colors::RESET); + } + break; + } + case LessThan: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::LessThan, TokenKind::Equals): + Console::Log("<="); + next(2); + break; + group(TokenKind::LessThan, TokenKind::LessThan): + Kind = TokenKind::LeftShift; + goto start; + default: + Console::Log("<"); + next(1); + break; + } + break; + } + case LeftShift: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::LeftShift, TokenKind::Equals): + Console::Log("<<="); + next(3); + break; + default: + Console::Log("<<"); + next(2); + break; + } + break; + } + case RightShift: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::RightShift, TokenKind::Equals): + Console::Log(">>="); + next(3); + break; + default: + Console::Log(">>"); + next(2); + break; + } + break; + } + case GreaterThan: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::GreaterThan, TokenKind::Equals): + Console::Log(">="); + next(2); + break; + group(TokenKind::GreaterThan, TokenKind::GreaterThan): + Kind = TokenKind::RightShift; + goto start; + default: + Console::Log(">"); + next(1); + break; + } + break; + } + case Equals: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::Equals) { + Console::Log("=="); + next(1); + } else { + Console::Log("="); + } + next(1); + break; + } + case QuestionMark: { + auto s1 = TokenKindClass[peek(1)]; + if (s1 == TokenKind::QuestionMark) { + Console::Log("??"); + next(2); + } else { + Console::Log("?"); + next(1); + } + break; + } + case Plus: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Plus, TokenKind::Plus): + Console::Log("++"); + next(2); + break; + group(TokenKind::Plus, TokenKind::Equals): + Console::Log("+="); + next(2); + break; + default: + Console::Log("+"); + next(1); + break; + } + break; + } + case Minus: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Minus, TokenKind::Minus): + Console::Log("--"); + next(2); + break; + group(TokenKind::Minus, TokenKind::Equals): + Console::Log("-="); + next(2); + break; + default: + Console::Log("-"); + next(1); + break; + } + break; + } + case Slash: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Slash, TokenKind::Equals): + Instance = TOKEN(const char *, "/=", TokenKind::DivEquals, "/="); + Instance.assignment = true; + break; + default: + Instance = TOKEN(const char *, "/", TokenKind::Slash, "/"); + break; + } + break; + } + case Star: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Star, TokenKind::Star): + Kind = TokenKind::Pow; + goto start; + group(TokenKind::Star, TokenKind::Equals): + Console::Log("*="); + break; + default: + Console::Log('*'); + break; + } + break; + } + case Pow: { + auto s1 = TokenKindClass[peek(2)]; + match(Kind, s1) + { + group(TokenKind::Pow, TokenKind::Equals): + Console::Log("**="); + next(3); + break; + default: + Console::Log("**"); + next(2); + break; + } + break; + } + case AND: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::AND, TokenKind::Equals): + Console::Log("/="); + next(2); + break; + default: + Console::Log('/'); + next(1); + break; + } + break; + } + case Pipe: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::Pipe, TokenKind::Equals): + Console::Log("|="); + next(2); + break; + default: + Console::Log('|'); + next(1); + break; + } + break; + } + case XOR: { + auto s1 = TokenKindClass[peek(1)]; + match(Kind, s1) + { + group(TokenKind::XOR, TokenKind::Equals): + Console::Log("^="); + next(2); + break; + default: + Console::Log('^'); + next(1); + break; + } + break; + } + case NewLine: // \n or \r + Console::Log(curr()); + skip_new_line(); + Kind = TokenKindClass[curr()]; + goto start; // Does this so 2 function calls aren't wasted. + case Char: + Console::Log(Colors::GREEN, "'", next(1), Colors::RESET); + if (peek(1) == '\'') + Console::Log("'"); + next(1); + break; + case Whitespace: // Skip whitespace + Console::Log(curr()); + next(1); + Kind = TokenKindClass[curr()]; + goto start; + case Dot: { + auto s1 = TokenKindClass[peek(1)]; + auto s2 = TokenKindClass[peek(2)]; + + match(Kind, s1) + { + group(TokenKind::Dot, TokenKind::Dot): + s2 == TokenKind::Dot ? + next(3), Console::Log("...") : + (next(2), Console::Log("..")); + break; + default: + Console::Log("."); + next(1); + break; + } + break; + } + // Cannot be generated by the TokenKind table, assigned after checks + case EqualsEquals: + case Boolean: + UNREACHABLE; + break; + case LParenthesis: + case RParenthesis: + case LBracket: + case RBracket: + case LCurlyBrace: + case RCurlyBrace: + case NOT: + case Colon: + case SemiColon: + case ExclamationPoint: + case At: + case Comma: + Console::Log(curr()); + next(1); + break; + case Error: + buffer = nullptr; + break; + default: + break; + } + } +} + +namespace nextgen { + namespace jet { + template + class Lexer; + + template + class Lexer; + } } -namespace nextgen { namespace jet { - template class Lexer; - template class Lexer; -}} diff --git a/lib/jet-lexer-v2.cpp b/lib/jet-lexer-v2.cpp index 8257ea5..0b95405 100644 --- a/lib/jet-lexer-v2.cpp +++ b/lib/jet-lexer-v2.cpp @@ -15,300 +15,321 @@ if (OUTPUT_MODE) Console::Log(value), next((sizeof value) - 1); \ else (tokens.end = TOKEN_SHORT(value, kind), next((sizeof value) - 1)) \ \ -static const constexpr TokenKind MatchTokenKind[256] { +static const constexpr TokenKind MatchTokenKind[256]{ - // Error code - TokenKind::EOFToken, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, - TokenKind::Whitespace, // \a - TokenKind::Whitespace, // \b - TokenKind::Whitespace, // \t - TokenKind::NewLine, // \n - TokenKind::Whitespace, // \v - TokenKind::Error, + // Error code + TokenKind::EOFToken, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Whitespace, // \a + TokenKind::Whitespace, // \b + TokenKind::Whitespace, // \t + TokenKind::NewLine, // \n + TokenKind::Whitespace, // \v + TokenKind::Error, - TokenKind::NewLine, // \r + TokenKind::NewLine, // \r - TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - - TokenKind::Whitespace, // - - TokenKind::ExclamationPoint, // ! - TokenKind::String, // '"' - TokenKind::Error, TokenKind::Error, - TokenKind::Percent, // '%' - TokenKind::AND, // '&' - TokenKind::Char, // '\'' - TokenKind::LParenthesis, // ( - TokenKind::RParenthesis, // ) - TokenKind::Star, // '*' - TokenKind::Plus, // '+' - TokenKind::Comma, // ',' - - TokenKind::Minus, // '-' - - - TokenKind::Dot, // '.' - - TokenKind::Slash, // '/' - - - // Digits - TokenKind::Zero, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, - TokenKind::Integer, TokenKind::Integer, - - - TokenKind::Colon, // ':' - - - TokenKind::SemiColon, // ';' - TokenKind::LessThan, // '<' - TokenKind::Equals, // '=' - TokenKind::GreaterThan, // '>' - TokenKind::QuestionMark, // '?' - TokenKind::At, // @ - - // Uppercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LBracket, // '[' - - TokenKind::StringEscapeStart, // \\ - - TokenKind::RBracket, // ']' - - TokenKind::XOR, // '^' - - TokenKind::Identifier,// '_' - - TokenKind::Error, - - // Lowercase letters - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, - TokenKind::Identifier, TokenKind::Identifier, - - - TokenKind::LCurlyBrace, // '{' - TokenKind::Pipe, // '|' - TokenKind::RCurlyBrace, // '}' - TokenKind::NOT, // '~' - - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, TokenKind::Error, - TokenKind::Error, TokenKind::Error, + TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + + TokenKind::Whitespace, // + + TokenKind::ExclamationPoint, // ! + TokenKind::String, // '"' + TokenKind::Error, TokenKind::Error, + TokenKind::Percent, // '%' + TokenKind::AND, // '&' + TokenKind::Char, // '\'' + TokenKind::LParenthesis, // ( + TokenKind::RParenthesis, // ) + TokenKind::Star, // '*' + TokenKind::Plus, // '+' + TokenKind::Comma, // ',' + + TokenKind::Minus, // '-' + + + TokenKind::Dot, // '.' + + TokenKind::Slash, // '/' + + + // Digits + TokenKind::Zero, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, TokenKind::Integer, + TokenKind::Integer, TokenKind::Integer, + + + TokenKind::Colon, // ':' + + + TokenKind::SemiColon, // ';' + TokenKind::LessThan, // '<' + TokenKind::Equals, // '=' + TokenKind::GreaterThan, // '>' + TokenKind::QuestionMark, // '?' + TokenKind::At, // @ + + // Uppercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LBracket, // '[' + + TokenKind::StringEscapeStart, // \\ + + TokenKind::RBracket, // ']' + + TokenKind::XOR, // '^' + + TokenKind::Identifier,// '_' + + TokenKind::Error, + + // Lowercase letters + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, TokenKind::Identifier, + TokenKind::Identifier, TokenKind::Identifier, + + + TokenKind::LCurlyBrace, // '{' + TokenKind::Pipe, // '|' + TokenKind::RCurlyBrace, // '}' + TokenKind::NOT, // '~' + + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, TokenKind::Error, + TokenKind::Error, TokenKind::Error, }; constexpr static int IntegerBits[256] = { - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, - // Digits - 0,1,2,3,4,5,6,7,8,9, + // Digits + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - // Error Code - -3,-3,-3,-3,-3,-3,-3, + // Error Code + -3, -3, -3, -3, -3, -3, -3, - // Uppercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35, + // Uppercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, - // Lowercase values - 10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35, + // Lowercase values + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - // Error code - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, - -3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3, + // Error code + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, }; struct ReservedIdent { - const char *id; - TokenKind type; + const char *id; + TokenKind type; }; -static const struct ReservedIdent Reserved[] { - { "if", KeywordIf }, - { "else", KeywordElse }, - { "elif", KeywordElif }, - { "while", KeywordWhile }, - { "for", KeywordFor }, - { "true", KeywordTrue }, - { "false", KeywordFalse }, - { "None", KeywordNone }, - { "fn", KeywordFunction }, - { "struct", KeywordStruct }, - { "export", KeywordExport }, - { "enum", KeywordEnum }, - { "extern", KeywordExtern }, - { "error", KeywordError }, - { "var", KeywordVar }, - { "return", KeywordReturn }, - { "and", KeywordAnd }, - { "or", KeywordOr }, - { "break", KeywordBreak }, - { "continue", KeywordContinue }, - { "defer", KeywordDefer }, - { "union", KeywordUnion }, - { "match", KeywordMatch }, - { "in", KeywordIn }, - {"str", Typename_str }, - {"i8", Typename_i8 }, - {"i16", Typename_i16 }, - {"i32", Typename_i32 }, - {"i64", Typename_i64 }, - {"u8", Typename_u8 }, - {"u16", Typename_u16 }, - {"u32", Typename_u32 }, - {"u64", Typename_u64 }, - {"box", Typename_box }, - {"f32", Typename_f32 }, - {"f64", Typename_f64 }, +static const struct ReservedIdent Reserved[]{ + {"if", KeywordIf}, + {"else", KeywordElse}, + {"elif", KeywordElif}, + {"while", KeywordWhile}, + {"for", KeywordFor}, + {"true", KeywordTrue}, + {"false", KeywordFalse}, + {"None", KeywordNone}, + {"fn", KeywordFunction}, + {"struct", KeywordStruct}, + {"export", KeywordExport}, + {"enum", KeywordEnum}, + {"extern", KeywordExtern}, + {"error", KeywordError}, + {"var", KeywordVar}, + {"return", KeywordReturn}, + {"and", KeywordAnd}, + {"or", KeywordOr}, + {"break", KeywordBreak}, + {"continue", KeywordContinue}, + {"defer", KeywordDefer}, + {"union", KeywordUnion}, + {"match", KeywordMatch}, + {"in", KeywordIn}, + {"str", Typename_str}, + {"i8", Typename_i8}, + {"i16", Typename_i16}, + {"i32", Typename_i32}, + {"i64", Typename_i64}, + {"u8", Typename_u8}, + {"u16", Typename_u16}, + {"u32", Typename_u32}, + {"u64", Typename_u64}, + {"box", Typename_box}, + {"f32", Typename_f32}, + {"f64", Typename_f64}, }; static const TokenKind MatchIdentToReserved(const char *ident, size_t len) { - FOR(i, SizeOfArray(Reserved)) { - if (::strncmp(Reserved[i].id, ident, len) == 0) - return Reserved[i].type; - } - return Identifier; + FOR(i, SizeOfArray(Reserved)) { + if (::strncmp(Reserved[i].id, ident, len) == 0) + return Reserved[i].type; + } + return Identifier; } template void Lexer::lex_float(int skip, int start) { - // Initialize start of token - auto token_start = buffer-skip; - auto token_start_col = column; + // Initialize start of token + auto token_start = buffer - skip; + auto token_start_col = column; - next(skip-1); + next(skip - 1); - double decimal = 0; - double divisor = 1; - double value = 0; + double decimal = 0; + double divisor = 1; + double value = 0; - do { - auto digit = IntegerBits[curr()]; - if (digit < 0) break; - divisor *= 10; - value = value * 10 + digit ; - } while(next(1)); + do { + auto digit = IntegerBits[curr()]; + if (digit < 0) break; + divisor *= 10; + value = value * 10 + digit; + } while (next(1)); - decimal = (value / divisor); + decimal = (value / divisor); - auto token_end = buffer; + auto token_end = buffer; - auto literal_token_representation - = Range(token_start, token_end); + auto literal_token_representation + = Range(token_start, token_end); - if (OUTPUT_MODE) { - Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); - } + if (OUTPUT_MODE) { + Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); + } - tokens.end = new Token { - literal_token_representation, - {line, column}, - (double)(start + decimal), - TokenKind::Decimal, - TokenClassification::Literal - }; + tokens.end = new Token{ + literal_token_representation, + {line, column}, + (double) (start + decimal), + TokenKind::Decimal, + TokenClassification::Literal + }; } template void Lexer::lex_int() { - switch (curr()) { - case '0': - switch(peek(1)) { - case 'b': lex_int<2, 2>(); return; - case 'x': lex_int<16, 2>(); return; - case '#': lex_int<36, 2>(); return; - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': lex_int<8, 1>(); return; - case '.': lex_float(2); return; - default: lex_int<10, 0>(); return; - } - default: lex_int<10, 0>(); - } + switch (curr()) { + case '0': + switch (peek(1)) { + case 'b': + lex_int < 2, 2 > (); + return; + case 'x': + lex_int < 16, 2 > (); + return; + case '#': + lex_int < 36, 2 > (); + return; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + lex_int < 8, 1 > (); + return; + case '.': + lex_float(2); + return; + default: + lex_int < 10, 0 > (); + return; + } + default: + lex_int < 10, 0 > (); + } } template @@ -316,494 +337,508 @@ template void Lexer::lex_int() { - // Initialize start of token - const char *token_start = buffer; - size_t token_start_col = column; - - // Skip any tokens, if any (for 0x, 0b, 00 cases) - next(skip); - - // Note: Keep the type! - decltype(UINTPTR_MAX) lexed_int = 0; - do { - auto digit = IntegerBits[curr()]; - if (digit < 0) break; - - // Invalid digit in number. For example, in base 10, - // we accept 1234 as a valid number but not 123f as 'f' is out of - // range for the accepted base. - if (digit >= radix) { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::DigitOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::Integer - }, - {radix, digit + 1} - }); - } - } - // Check whether value is about to overflow the maximum integer value. - // Ensures that all integers don't exceed 64-bit max. - if (lexed_int > (UINTPTR_MAX - digit) / radix) { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::IntegerOverflow, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::Integer - } - }); - } - } - lexed_int = lexed_int * radix + digit; - } while(next(1)); - - - if (curr() == '.') { // At this point, we've realized its f64, so we proceed - lex_float(buffer - token_start, lexed_int); - return; - } - - const char *token_end = buffer; - auto literal_token_representation - = Range(token_start, token_end); - - if (OUTPUT_MODE) { - Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); - } - - - tokens.end = new Token { - literal_token_representation, - {line, column}, - (decltype(UINTPTR_MAX)) lexed_int, - TokenKind::Integer, - TokenClassification::Literal - }; + // Initialize start of token + const char *token_start = buffer; + size_t token_start_col = column; + + // Skip any tokens, if any (for 0x, 0b, 00 cases) + next(skip); + + // Note: Keep the type! + decltype(UINTPTR_MAX) lexed_int = 0; + do { + auto digit = IntegerBits[curr()]; + if (digit < 0) break; + + // Invalid digit in number. For example, in base 10, + // we accept 1234 as a valid number but not 123f as 'f' is out of + // range for the accepted base. + if (digit >= radix) { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::DigitOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::Integer + }, + {radix, digit + 1} + }); + } + } + // Check whether value is about to overflow the maximum integer value. + // Ensures that all integers don't exceed 64-bit max. + if (lexed_int > (UINTPTR_MAX - digit) / radix) { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::IntegerOverflow, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::Integer + } + }); + } + } + lexed_int = lexed_int * radix + digit; + } while (next(1)); + + + if (curr() == '.') { // At this point, we've realized its f64, so we proceed + lex_float(buffer - token_start, lexed_int); + return; + } + + const char *token_end = buffer; + auto literal_token_representation + = Range(token_start, token_end); + + if (OUTPUT_MODE) { + Console::Log(Colors::BLUE, str(literal_token_representation), Colors::RESET); + } + + + tokens.end = new Token{ + literal_token_representation, + {line, column}, + (decltype(UINTPTR_MAX)) lexed_int, + TokenKind::Integer, + TokenClassification::Literal + }; } template void Lexer::lex_ident() { - const char *token_start = buffer; - auto token_start_col = column; - auto val = FNV_OFF; - do { - val ^= (curr()); - val *= FNV_PRIME; - } while(MatchTokenKind[next(1)] == TokenKind::Identifier); - const char *token_end = buffer; - - auto literal_token_representation = str(Range(token_start, - token_end)); - literal_token_representation.setHash(val); - TokenKind reserved = - MatchIdentToReserved(token_start, token_end - token_start); - - if (OUTPUT_MODE) { - if (reserved != TokenKind::Identifier) { - Console::Log(Colors::RED, literal_token_representation, - Colors::RESET); - } else { - Console::Log(Colors::YELLOW, literal_token_representation, Colors::RESET); - } - } - - tokens.end = new Token { - literal_token_representation, - {line, token_start_col}, - "", - reserved, - static_cast(reserved == Identifier ? Keyword : -1) - }; + const char *token_start = buffer; + auto token_start_col = column; + auto val = FNV_OFF; + do { + val ^= (curr()); + val *= FNV_PRIME; + } while (MatchTokenKind[next(1)] == TokenKind::Identifier); + const char *token_end = buffer; + + auto literal_token_representation = str(Range(token_start, + token_end)); + literal_token_representation.setHash(val); + TokenKind reserved = + MatchIdentToReserved(token_start, token_end - token_start); + + if (OUTPUT_MODE) { + if (reserved != TokenKind::Identifier) { + Console::Log(Colors::RED, literal_token_representation, + Colors::RESET); + } else { + Console::Log(Colors::YELLOW, literal_token_representation, Colors::RESET); + } + } + + tokens.end = new Token{ + literal_token_representation, + {line, token_start_col}, + "", + reserved, + static_cast(reserved == Identifier ? Keyword : -1) + }; } template void Lexer::lex_str() { - auto token_start = buffer; - auto token_start_col = column; - auto offset = 1; - - if (OUTPUT_MODE) Console::Log(Colors::GREEN); - - // @NOTE: Remember that this is DATA not OBJECT for allocation - auto lexed_string = (char*) (GLOBAL_DATA_ALLOC.current() + 1); - size_t len = 0; - - char current_char = next(1); - if (OUTPUT_MODE) Console::Log("\""); - do { - current_char = curr(); - if (OUTPUT_MODE) { - Console::Log(current_char); - if (current_char == '\0' || peek(1) == '\0') { - offset = 0; - break; - } - } else { - if (current_char == '\0') { - diagnostics.build(LexError { - LexErrorType::MissingClosingDelim, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {(int)('"' - )} - }); - } - - if (current_char == '\\') { - switch (next(1)) { - case 'a': current_char = '\a'; break; - case 'b': current_char = '\b'; break; - case 'r': current_char = '\r'; break; - case 'n': current_char = '\n'; break; - case 'v': current_char = '\v'; break; - case 't': current_char = '\t'; break; - case 'x': { - auto value = IntegerBits[next(1)]; - if (value >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {value + 1} // Pass in the invalid digit for the escape - }); - } - auto value2 = IntegerBits[next(1)]; - if (value2 >= 16) { - diagnostics.build(LexError{ - LexErrorType::HexEscapeOutOfRange, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {value2 + 1} // Pass in the invalid digit for the escape - }); - } - int val = value; - val = val * 16 + value2; // @Opt: (val << 4) | value2 ? - - current_char = (char) val; - break; - } - default: - diagnostics.build(LexError { - LexErrorType::InvalidStringEscape, - {line, column}, - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - } - }); - } - } - lexed_string[len++] = current_char; - } - } while(next(1) != '"'); - - next(1); // Skip '"' - - // For missing closing expect_delim this needs to be here - if (OUTPUT_MODE && offset != 0) Console::Log("\""); - if (NORMAL_MODE) { - lexed_string[len] = '\0'; - - // We wrote to the data buffer without telling the size of the string - // because we did not know what it was, so we tell it the size of the - // string after lexing it. `reserve(size_t)` in this case is used to - // increment the number of bytes written. - GLOBAL_DATA_ALLOC.reserve(len); - tokens.end = new Token { - str {lexed_string, len}, - {line, column}, - "", - TokenKind::String, - TokenClassification::Literal - }; - } + auto token_start = buffer; + auto token_start_col = column; + auto offset = 1; + + if (OUTPUT_MODE) Console::Log(Colors::GREEN); + + // @NOTE: Remember that this is DATA not OBJECT for allocation + auto lexed_string = (char *) (GLOBAL_DATA_ALLOC.current() + 1); + size_t len = 0; + + char current_char = next(1); + if (OUTPUT_MODE) Console::Log("\""); + do { + current_char = curr(); + if (OUTPUT_MODE) { + Console::Log(current_char); + if (current_char == '\0' || peek(1) == '\0') { + offset = 0; + break; + } + } else { + if (current_char == '\0') { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {(int) ('"' + )} + }); + } + + if (current_char == '\\') { + switch (next(1)) { + case 'a': + current_char = '\a'; + break; + case 'b': + current_char = '\b'; + break; + case 'r': + current_char = '\r'; + break; + case 'n': + current_char = '\n'; + break; + case 'v': + current_char = '\v'; + break; + case 't': + current_char = '\t'; + break; + case 'x': { + auto value = IntegerBits[next(1)]; + if (value >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {value + 1} // Pass in the invalid digit for the escape + }); + } + auto value2 = IntegerBits[next(1)]; + if (value2 >= 16) { + diagnostics.build(LexError{ + LexErrorType::HexEscapeOutOfRange, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {value2 + 1} // Pass in the invalid digit for the escape + }); + } + int val = value; + val = val * 16 + value2; // @Opt: (val << 4) | value2 ? + + current_char = (char) val; + break; + } + default: + diagnostics.build(LexError{ + LexErrorType::InvalidStringEscape, + {line, column}, + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + } + }); + } + } + lexed_string[len++] = current_char; + } + } while (next(1) != '"'); + + next(1); // Skip '"' + + // For missing closing expect_delim this needs to be here + if (OUTPUT_MODE && offset != 0) Console::Log("\""); + if (NORMAL_MODE) { + lexed_string[len] = '\0'; + + // We wrote to the data buffer without telling the size of the string + // because we did not know what it was, so we tell it the size of the + // string after lexing it. `reserve(size_t)` in this case is used to + // increment the number of bytes written. + GLOBAL_DATA_ALLOC.reserve(len); + tokens.end = new Token{ + str{lexed_string, len}, + {line, column}, + "", + TokenKind::String, + TokenClassification::Literal + }; + } } template ArenaVec Lexer::lex() { - tokens.begin = (Token *) GLOBAL_OBJECT_ALLOC.current(); - - TokenKind kind; - while(true) { - if (fatal) break; // Special case: Diagnostics may crash - - auto current_char = curr(); - kind = MatchTokenKind[current_char]; - if (OUTPUT_MODE) Console::Log(Colors::RESET); - - switch (kind) { - case EOFToken: { - tokens.end = new Token("a", {line, column}, TokenKind::EOFToken); - return tokens; - } - case Integer: lex_int(); break; - case String: lex_str(); break; - case Identifier: lex_ident(); break; - case NewLine: skip_new_line(); break; - case Whitespace: { - if (OUTPUT_MODE) - Console::Log(current_char); - next(1); - break; - } - case Char: { - auto token_start = buffer; - auto token_start_col = column; - if (OUTPUT_MODE) { - Console::Log(Colors::GREEN, "'"); - Console::Log(next(1), Colors::RESET); - } - else { - TOKEN_SHORT(next(1), TokenKind::Char); - } - if (curr() != '\'') { - if (NORMAL_MODE) { - diagnostics.build(LexError { - LexErrorType::MissingClosingDelim, - {line, column}, // 'x[?] <-- Missing Closing! - Token { - Range(token_start, buffer), - {line, token_start_col}, - TokenKind::String - }, - {(char)'\''} - }); - } - next(1); // Skip "'" - break; - } - else if (OUTPUT_MODE) Console::Log("'", Colors::RESET); - next(1); // Skip the "'" - break; - } - case LessThan: { // <, <=, <<, <<= - auto n = peek(1); - if (n == '<') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT("<<=", TokenKind::LeftShiftEquals); - break; - } - TOKEN_ADD_SHORT("<<", TokenKind::LeftShift); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT("<=",TokenKind::LessThanEquals); - break; - } - TOKEN_ADD_SHORT("<", TokenKind::LessThan); - break; - } - case GreaterThan: { - auto n = peek(1); - if (n == '>') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT(">>=", TokenKind::RightShiftEquals); - break; - } - TOKEN_ADD_SHORT(">>", TokenKind::RightShift); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT(">=",TokenKind::GreaterThanEquals); - break; - } - TOKEN_ADD_SHORT(">", TokenKind::GreaterThan); - break; - } - case Star: { - auto n = peek(1); - if (n == '*') { - if (peek(2) == '=') { - TOKEN_ADD_SHORT("**=", TokenKind::PowEquals); - break; - } - TOKEN_ADD_SHORT("**", TokenKind::Pow); - break; - } - else if (n == '=') { - TOKEN_ADD_SHORT("*=",TokenKind::MulEquals); - break; - } - TOKEN_ADD_SHORT("*", TokenKind::Star); - break; - } - case Dot: { - auto n = peek(1); - if (n == '.') { - if (peek(2) == '.') { - TOKEN_ADD_SHORT("...", TokenKind::Ellipsis); - break; - } - TOKEN_ADD_SHORT("..", TokenKind::RangeSpan); - break; - } - TOKEN_ADD_SHORT(".", TokenKind::Dot); - break; - } - case Plus: { - auto n = peek(1); - if (n == '+') { TOKEN_ADD_SHORT("++", TokenKind::PlusPlus);} - else if (n == '=') { TOKEN_ADD_SHORT("+=", TokenKind::PlusEquals); } - else { - TOKEN_ADD_SHORT("+", TokenKind::Plus); - } - break; - } - case Minus: { - auto n = peek(1); - if (n == '-') { - TOKEN_ADD_SHORT("--", TokenKind::MinusMinus); - } - else if (n == '=') { - TOKEN_ADD_SHORT("-=", TokenKind::MinusMinus); - } - else if (n == '>') { - TOKEN_ADD_SHORT("->", TokenKind::Arrow); - } - else { - TOKEN_ADD_SHORT("+", TokenKind::Minus); - } - break; - } - case Slash: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("/=", TokenKind::DivEquals); - } - else if (n == '/') { - if (OUTPUT_MODE) Console::Log("/"); - while (MatchTokenKind[next(1)] != NewLine) { - if (OUTPUT_MODE) { - Console::Log(curr()); - } - } - skip_new_line(); - } - else { - TOKEN_ADD_SHORT("/", TokenKind::Slash); - } - break; - } - case Equals: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("==", TokenKind::EqualsEquals); - } else if (n == '>') { - TOKEN_ADD_SHORT("=>", TokenKind::FunctionArrow); - } else { - TOKEN_ADD_SHORT("=", TokenKind::Equals); - } - break; - } - case Percent: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("%=", TokenKind::PercentEquals); - } else { - TOKEN_ADD_SHORT("%", TokenKind::Percent); - } - break; - } - case AND: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("&=", TokenKind::AND); - } else { - TOKEN_ADD_SHORT("&", TokenKind::ANDEquals); - } - break; - } - case Pipe: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("|=", TokenKind::OREquals); - } else { - TOKEN_ADD_SHORT("|", TokenKind::Pipe); - } - break; - } - case XOR: { - auto n = peek(1); - if (n == '=') { - TOKEN_ADD_SHORT("^=", TokenKind::XOREquals); - } else { - TOKEN_ADD_SHORT("^", TokenKind::XOR); - } - break; - } - case LParenthesis: - TOKEN_ADD_SHORT("(", TokenKind::LParenthesis); - break; - case RParenthesis: - TOKEN_ADD_SHORT(")", TokenKind::RParenthesis); - break; - case LCurlyBrace: - TOKEN_ADD_SHORT("{", TokenKind::LCurlyBrace); - break; - case RCurlyBrace: - TOKEN_ADD_SHORT("}", TokenKind::RCurlyBrace); - break; - case LBracket: - TOKEN_ADD_SHORT("[", TokenKind::LBracket); - break; - case RBracket: - TOKEN_ADD_SHORT("]", TokenKind::RBracket); - break; - case SemiColon: - TOKEN_ADD_SHORT(";", TokenKind::SemiColon); - break; - case At: - TOKEN_ADD_SHORT("@", TokenKind::At); - break; - case NOT: - TOKEN_ADD_SHORT("~", TokenKind::NOT); - break; - case ExclamationPoint: - TOKEN_ADD_SHORT("!", TokenKind::ExclamationPoint); - break; - case QuestionMark: - TOKEN_ADD_SHORT("?", TokenKind::QuestionMark); - break; - case Colon: - if (peek(1) == '=') - TOKEN_ADD_SHORT(":=", TokenKind::ColonEquals); - else - TOKEN_ADD_SHORT(":", TokenKind::Colon); - break; - case Comma: - TOKEN_ADD_SHORT(",", TokenKind::Comma); - break; - default: - break; - } - } - return tokens; + tokens.begin = (Token *) GLOBAL_OBJECT_ALLOC.current(); + + TokenKind kind; + while (true) { + if (fatal) break; // Special case: Diagnostics may crash + + auto current_char = curr(); + kind = MatchTokenKind[current_char]; + if (OUTPUT_MODE) Console::Log(Colors::RESET); + + switch (kind) { + case EOFToken: { + tokens.end = new Token("a", {line, column}, TokenKind::EOFToken); + return tokens; + } + case Integer: + lex_int(); + break; + case String: + lex_str(); + break; + case Identifier: + lex_ident(); + break; + case NewLine: + skip_new_line(); + break; + case Whitespace: { + if (OUTPUT_MODE) + Console::Log(current_char); + next(1); + break; + } + case Char: { + auto token_start = buffer; + auto token_start_col = column; + if (OUTPUT_MODE) { + Console::Log(Colors::GREEN, "'"); + Console::Log(next(1), Colors::RESET); + } else { + TOKEN_SHORT(next(1), TokenKind::Char); + } + if (curr() != '\'') { + if (NORMAL_MODE) { + diagnostics.build(LexError{ + LexErrorType::MissingClosingDelim, + {line, column}, // 'x[?] <-- Missing Closing! + Token{ + Range(token_start, buffer), + {line, token_start_col}, + TokenKind::String + }, + {(char) '\''} + }); + } + next(1); // Skip "'" + break; + } else if (OUTPUT_MODE) Console::Log("'", Colors::RESET); + next(1); // Skip the "'" + break; + } + case LessThan: { // <, <=, <<, <<= + auto n = peek(1); + if (n == '<') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT("<<=", TokenKind::LeftShiftEquals); + break; + } + TOKEN_ADD_SHORT("<<", TokenKind::LeftShift); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT("<=", TokenKind::LessThanEquals); + break; + } + TOKEN_ADD_SHORT("<", TokenKind::LessThan); + break; + } + case GreaterThan: { + auto n = peek(1); + if (n == '>') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT(">>=", TokenKind::RightShiftEquals); + break; + } + TOKEN_ADD_SHORT(">>", TokenKind::RightShift); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT(">=", TokenKind::GreaterThanEquals); + break; + } + TOKEN_ADD_SHORT(">", TokenKind::GreaterThan); + break; + } + case Star: { + auto n = peek(1); + if (n == '*') { + if (peek(2) == '=') { + TOKEN_ADD_SHORT("**=", TokenKind::PowEquals); + break; + } + TOKEN_ADD_SHORT("**", TokenKind::Pow); + break; + } else if (n == '=') { + TOKEN_ADD_SHORT("*=", TokenKind::MulEquals); + break; + } + TOKEN_ADD_SHORT("*", TokenKind::Star); + break; + } + case Dot: { + auto n = peek(1); + if (n == '.') { + if (peek(2) == '.') { + TOKEN_ADD_SHORT("...", TokenKind::Ellipsis); + break; + } + TOKEN_ADD_SHORT("..", TokenKind::RangeSpan); + break; + } + TOKEN_ADD_SHORT(".", TokenKind::Dot); + break; + } + case Plus: { + auto n = peek(1); + if (n == '+') { TOKEN_ADD_SHORT("++", TokenKind::PlusPlus); } + else if (n == '=') { TOKEN_ADD_SHORT("+=", TokenKind::PlusEquals); } + else { + TOKEN_ADD_SHORT("+", TokenKind::Plus); + } + break; + } + case Minus: { + auto n = peek(1); + if (n == '-') { + TOKEN_ADD_SHORT("--", TokenKind::MinusMinus); + } else if (n == '=') { + TOKEN_ADD_SHORT("-=", TokenKind::MinusMinus); + } else if (n == '>') { + TOKEN_ADD_SHORT("->", TokenKind::Arrow); + } else { + TOKEN_ADD_SHORT("+", TokenKind::Minus); + } + break; + } + case Slash: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("/=", TokenKind::DivEquals); + } else if (n == '/') { + if (OUTPUT_MODE) Console::Log("/"); + while (MatchTokenKind[next(1)] != NewLine) { + if (OUTPUT_MODE) { + Console::Log(curr()); + } + } + skip_new_line(); + } else { + TOKEN_ADD_SHORT("/", TokenKind::Slash); + } + break; + } + case Equals: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("==", TokenKind::EqualsEquals); + } else if (n == '>') { + TOKEN_ADD_SHORT("=>", TokenKind::FunctionArrow); + } else { + TOKEN_ADD_SHORT("=", TokenKind::Equals); + } + break; + } + case Percent: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("%=", TokenKind::PercentEquals); + } else { + TOKEN_ADD_SHORT("%", TokenKind::Percent); + } + break; + } + case AND: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("&=", TokenKind::AND); + } else { + TOKEN_ADD_SHORT("&", TokenKind::ANDEquals); + } + break; + } + case Pipe: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("|=", TokenKind::OREquals); + } else { + TOKEN_ADD_SHORT("|", TokenKind::Pipe); + } + break; + } + case XOR: { + auto n = peek(1); + if (n == '=') { + TOKEN_ADD_SHORT("^=", TokenKind::XOREquals); + } else { + TOKEN_ADD_SHORT("^", TokenKind::XOR); + } + break; + } + case LParenthesis: + TOKEN_ADD_SHORT("(", TokenKind::LParenthesis); + break; + case RParenthesis: + TOKEN_ADD_SHORT(")", TokenKind::RParenthesis); + break; + case LCurlyBrace: + TOKEN_ADD_SHORT("{", TokenKind::LCurlyBrace); + break; + case RCurlyBrace: + TOKEN_ADD_SHORT("}", TokenKind::RCurlyBrace); + break; + case LBracket: + TOKEN_ADD_SHORT("[", TokenKind::LBracket); + break; + case RBracket: + TOKEN_ADD_SHORT("]", TokenKind::RBracket); + break; + case SemiColon: + TOKEN_ADD_SHORT(";", TokenKind::SemiColon); + break; + case At: + TOKEN_ADD_SHORT("@", TokenKind::At); + break; + case NOT: + TOKEN_ADD_SHORT("~", TokenKind::NOT); + break; + case ExclamationPoint: + TOKEN_ADD_SHORT("!", TokenKind::ExclamationPoint); + break; + case QuestionMark: + TOKEN_ADD_SHORT("?", TokenKind::QuestionMark); + break; + case Colon: + if (peek(1) == '=') + TOKEN_ADD_SHORT(":=", TokenKind::ColonEquals); + else TOKEN_ADD_SHORT(":", TokenKind::Colon); + break; + case Comma: + TOKEN_ADD_SHORT(",", TokenKind::Comma); + break; + default: + break; + } + } + return tokens; } // Forces compilation of the two lexer classes // so that we can have definitions in a cpp file without // crowding the header file. -namespace nextgen { namespace jet { - template class Lexer; - template class Lexer; -}} \ No newline at end of file +namespace nextgen { + namespace jet { + template + class Lexer; + + template + class Lexer; + } +} \ No newline at end of file diff --git a/lib/jet-parser.cpp b/lib/jet-parser.cpp index 8f07a5b..9ba78c7 100644 --- a/lib/jet-parser.cpp +++ b/lib/jet-parser.cpp @@ -38,357 +38,355 @@ using namespace nextgen::jet; const ParserOutput Parser::parse() { - auto output = ParserOutput(); - - for (;;) { - - // Report up to 2 errors before exiting the program - if (fatal > 2) { - this->diagnostics.send_exception(); - } - - switch (curr()->getKind()) { - case TokenKind::EOFToken: - return output; - case TokenKind::Identifier: { - auto C1 = peek(1); - auto C2 = peek(2); - - if (C1->getKind() == TokenKind::FunctionArrow) { - switch (C2->getKind()) { - case TokenKind::LParenthesis: // Function - output.functions << parse_function(curr()); - break; - case TokenKind::KeywordStruct: // Struct - output.structures << parse_struct(curr()); - break; - case TokenKind::KeywordEnum: // Enum - break; - default: - UNREACHABLE; - break; - } - } - else if (C1->getKind() == TokenKind::ColonEquals) { - auto name = curr(); - skip(2); - output.global_variables << - (const SyntaxVariableAssignment*) - parse_variable_assignment(name); - } - else { - this->diagnostics.build(ParseError { - ParseErrorType::InvalidTokenAfterIdentInGlobalScope, - C1->getSourceLocation(), - { ParseError::Metadata { C1 }} - }); - this->diagnostics.send_exception(); - } - break; - } - default: // TODO: Figure out what to do here - break; - } - } - return output; + auto output = ParserOutput(); + + for (;;) { + + // Report up to 2 errors before exiting the program + if (fatal > 2) { + this->diagnostics.send_exception(); + } + + switch (curr()->getKind()) { + case TokenKind::EOFToken: + return output; + case TokenKind::Identifier: { + auto C1 = peek(1); + auto C2 = peek(2); + + if (C1->getKind() == TokenKind::FunctionArrow) { + switch (C2->getKind()) { + case TokenKind::LParenthesis: // Function + output.functions << parse_function(curr()); + break; + case TokenKind::KeywordStruct: // Struct + output.structures << parse_struct(curr()); + break; + case TokenKind::KeywordEnum: // Enum + break; + default: + UNREACHABLE; + break; + } + } else if (C1->getKind() == TokenKind::ColonEquals) { + auto name = curr(); + skip(2); + output.global_variables << + (const SyntaxVariableAssignment *) + parse_variable_assignment(name); + } else { + this->diagnostics.build(ParseError{ + ParseErrorType::InvalidTokenAfterIdentInGlobalScope, + C1->getSourceLocation(), + {ParseError::Metadata{C1}} + }); + this->diagnostics.send_exception(); + } + break; + } + default: // TODO: Figure out what to do here + break; + } + } + return output; } - const SyntaxNode * -Parser::parse_variable_assignment(const Token *name) { - - // Check 1: name: type = expr - // Check 2: name := expr - if (peek(1)->getKind() == TokenKind::Colon) { - auto E = new SyntaxVariableAssignment(name, Some(parse_type()), parse_expr()); - expect("Expected ';' after declaration"); - return E; - } else { - auto E = new SyntaxVariableAssignment(name, None, parse_expr()); - expect("Expected ';' after declaration"); - return E; - } +Parser::parse_variable_assignment(const Token *name) { + + // Check 1: name: type = expr + // Check 2: name := expr + if (peek(1)->getKind() == TokenKind::Colon) { + auto E = new SyntaxVariableAssignment(name, Some(parse_type()), parse_expr()); + expect("Expected ';' after declaration"); + return E; + } else { + auto E = new SyntaxVariableAssignment(name, None, parse_expr()); + expect("Expected ';' after declaration"); + return E; + } } /// Pratt Parsing Expression Technique (Precedence Matching) /// References: https://en.wikipedia.org/wiki/Operator-precedence_parser const SyntaxNode * -Parser::parse_expr(int previous_binding) { - SyntaxNode *lhs; - - auto first_token = curr(); - auto first_kind = first_token->getKind(); - - // ~ : Bitwise NOT - // + : Positive - // - : Negation - // & : allocator Reference - // * : pointer Dereference - // not : Logical Negation - auto unary_binding = Parser::UnaryOperatorBinding(first_kind); - if (unary_binding > previous_binding) { - const auto expr = parse_expr(unary_binding); - const auto op = SyntaxUnary::MatchOp(first_kind); - - const auto E = (SyntaxNode*)(new SyntaxUnary(op, first_token, expr)); - E->kind = SyntaxKind::Unary; - return expr; - } else { - lhs = (SyntaxNode*) match_expr(); - } - next(1); - do { - auto op = curr(); - auto op_kind = op->getKind(); - - const auto op_bindings = Parser::InfixOperatorBinding(op_kind); - if (/* LeftBindingPrecedence */ op_bindings[0] <= previous_binding || - op_bindings[0] == -1) { - break; - } - auto a = skip(1); - const auto rhs = parse_expr(/* RightBindingPrecedence */ - op_bindings[1]); - auto new_expr = new SyntaxBinary { - SyntaxBinary::MatchOp(op_kind), op, lhs, rhs - }; - lhs = new_expr; - lhs->kind = SyntaxKind::Binary; - } while (curr()->isValidExpressionType()); - return lhs; +Parser::parse_expr(int previous_binding) { + SyntaxNode *lhs; + + auto first_token = curr(); + auto first_kind = first_token->getKind(); + + // ~ : Bitwise NOT + // + : Positive + // - : Negation + // & : allocator Reference + // * : pointer Dereference + // not : Logical Negation + auto unary_binding = Parser::UnaryOperatorBinding(first_kind); + if (unary_binding > previous_binding) { + const auto expr = parse_expr(unary_binding); + const auto op = SyntaxUnary::MatchOp(first_kind); + + const auto E = (SyntaxNode *) (new SyntaxUnary(op, first_token, expr)); + E->kind = SyntaxKind::Unary; + return expr; + } else { + lhs = (SyntaxNode *) match_expr(); + } + next(1); + do { + auto op = curr(); + auto op_kind = op->getKind(); + + const auto op_bindings = Parser::InfixOperatorBinding(op_kind); + if (/* LeftBindingPrecedence */ op_bindings[0] <= previous_binding || + op_bindings[0] == -1) { + break; + } + auto a = skip(1); + const auto rhs = parse_expr(/* RightBindingPrecedence */ + op_bindings[1]); + auto new_expr = new SyntaxBinary{ + SyntaxBinary::MatchOp(op_kind), op, lhs, rhs + }; + lhs = new_expr; + lhs->kind = SyntaxKind::Binary; + } while (curr()->isValidExpressionType()); + return lhs; } const NG_INLINE SyntaxNode * Parser::match_expr() { - auto matched_token = curr(); - switch (matched_token->getKind()) { - case EOFToken: - this->diagnostics.build(ParseError { - ParseErrorType::UnexpectedEndOfFile, - matched_token->getSourceLocation() - }); - break; - case Identifier: { // Unresolved variable or Function Call - auto C1 = peek(1); - if (peek(1)->getKind() == TokenKind::LParenthesis) { - return parse_function_call(matched_token, C1); - } - auto E = (SyntaxNode*) new SyntaxLiteral(matched_token); - E->kind = SyntaxKind::LiteralValue; - return E; - } - case Integer: case String: case Char: case Decimal: case KeywordTrue: - case KeywordFalse: { - auto E = (SyntaxNode*) new SyntaxLiteral(matched_token); - E->kind = SyntaxKind::LiteralValue; - return E; - } - case LParenthesis: { - skip(1); - auto E = parse_expr(); - expect_delim(curr()->getSourceLocation()); - position--; - return E; - } - case LBracket: { // ex: [1, 2, 3] - auto current = next(1); - auto E = new SyntaxList; - - // Loop through with reaching the EOF or closing expect_delim - while (current->getKind() != TokenKind::RBracket - && current->getKind() != TokenKind::EOFToken) { - - // Skip separating expect_delim - if (current->getKind() == TokenKind::Comma) { - skip(1); - } - - E->values << parse_expr(); - current = curr(); - } - expect_delim(matched_token->getSourceLocation()); - position--; - auto node = (SyntaxNode*)E; - node->kind = SyntaxKind::List; - return node; - } - default: // TODO: Unexpected Expression Token - break; - } - return nullptr; + auto matched_token = curr(); + switch (matched_token->getKind()) { + case EOFToken: + this->diagnostics.build(ParseError{ + ParseErrorType::UnexpectedEndOfFile, + matched_token->getSourceLocation() + }); + break; + case Identifier: { // Unresolved variable or Function Call + auto C1 = peek(1); + if (peek(1)->getKind() == TokenKind::LParenthesis) { + return parse_function_call(matched_token, C1); + } + auto E = (SyntaxNode *) new SyntaxLiteral(matched_token); + E->kind = SyntaxKind::LiteralValue; + return E; + } + case Integer: + case String: + case Char: + case Decimal: + case KeywordTrue: + case KeywordFalse: { + auto E = (SyntaxNode *) new SyntaxLiteral(matched_token); + E->kind = SyntaxKind::LiteralValue; + return E; + } + case LParenthesis: { + skip(1); + auto E = parse_expr(); + expect_delim(curr()->getSourceLocation()); + position--; + return E; + } + case LBracket: { // ex: [1, 2, 3] + auto current = next(1); + auto E = new SyntaxList; + + // Loop through with reaching the EOF or closing expect_delim + while (current->getKind() != TokenKind::RBracket + && current->getKind() != TokenKind::EOFToken) { + + // Skip separating expect_delim + if (current->getKind() == TokenKind::Comma) { + skip(1); + } + + E->values << parse_expr(); + current = curr(); + } + expect_delim(matched_token->getSourceLocation()); + position--; + auto node = (SyntaxNode *) E; + node->kind = SyntaxKind::List; + return node; + } + default: // TODO: Unexpected Expression Token + break; + } + return nullptr; } -SyntaxType *Parser::parse_type() { - - // []i32 - // **i32 - // *****i32 - // (i32, i32) - - auto first_token = curr(); - switch(first_token->getKind()) { - case LBracket: { - expect("Sequence '[]' (representing array) must be" - " met with closing value ']'"); - SyntaxTypename ty_name; - auto ty_name_tok = curr(); - if (! ty_name_tok->isValidTypenameStart()) { - auto ty = parse_type(); - return new SyntaxType(SyntaxTypeAnnotation::ArrayType, None, ty); - } - else if (! ty_name_tok->isValidTypename()) { - // TODO: Add type error value here - } - else { - ty_name = SyntaxType::MatchTypename(ty_name_tok); - } - return new SyntaxType(SyntaxTypeAnnotation::ArrayType, ty_name, nullptr); - } - case LParenthesis: - // TODO: Add tuples - break; - case Star: { - SyntaxTypename ty_name; - auto ty_name_tok = next(1); - if (! ty_name_tok->isValidTypenameStart()) { - auto ty = parse_type(); - return new SyntaxType(SyntaxTypeAnnotation::Pointer, None, ty); - } - else if (! ty_name_tok->isValidTypename()) { - // TODO: Add type error value here - } - else { - ty_name = SyntaxType::MatchTypename(ty_name_tok); - } - return new SyntaxType(SyntaxTypeAnnotation::Pointer, ty_name, nullptr); - } - default: - UNREACHABLE; - } - return new SyntaxType(None, None, nullptr); +SyntaxType *Parser::parse_type() { + + // []i32 + // **i32 + // *****i32 + // (i32, i32) + + auto first_token = curr(); + switch (first_token->getKind()) { + case LBracket: { + expect("Sequence '[]' (representing array) must be" + " met with closing value ']'"); + SyntaxTypename ty_name; + auto ty_name_tok = curr(); + if (!ty_name_tok->isValidTypenameStart()) { + auto ty = parse_type(); + return new SyntaxType(SyntaxTypeAnnotation::ArrayType, None, ty); + } else if (!ty_name_tok->isValidTypename()) { + // TODO: Add type error value here + } else { + ty_name = SyntaxType::MatchTypename(ty_name_tok); + } + return new SyntaxType(SyntaxTypeAnnotation::ArrayType, ty_name, nullptr); + } + case LParenthesis: + // TODO: Add tuples + break; + case Star: { + SyntaxTypename ty_name; + auto ty_name_tok = next(1); + if (!ty_name_tok->isValidTypenameStart()) { + auto ty = parse_type(); + return new SyntaxType(SyntaxTypeAnnotation::Pointer, None, ty); + } else if (!ty_name_tok->isValidTypename()) { + // TODO: Add type error value here + } else { + ty_name = SyntaxType::MatchTypename(ty_name_tok); + } + return new SyntaxType(SyntaxTypeAnnotation::Pointer, ty_name, nullptr); + } + default: + UNREACHABLE; + } + return new SyntaxType(None, None, nullptr); } const SyntaxNode * -Parser::parse_for() { - auto loop_variable = skip(); - skip(); - auto list_or_range = skip(1); - switch (list_or_range->getKind()) { - case Identifier: { - auto body = parse_block(); - return new SyntaxForList(list_or_range, loop_variable, body); - } - case KeywordRange: { - expect(""); - RangeFunction func; - FOR(value, 3) { - if (curr()->getKind() == TokenKind::RParenthesis - || curr()->getKind() == TokenKind::EOFToken) - break; - func.set[value] = expect("Range function values " - "must " - "be " - "integers")->getValue(); - expect("Expected comma in between parameters."); - } - auto body = parse_block(); - return new SyntaxForRange(loop_variable, func, body); - } - default: - this->diagnostics.build(ParseError { - ParseErrorType::InvalidToken, - list_or_range->getSourceLocation(), - { ParseError::Metadata("Invalid token in for-loop context. Expected " - "a list variable or the range function.") } - }); - break; - } - return nullptr; +Parser::parse_for() { + auto loop_variable = skip(); + skip(); + auto list_or_range = skip(1); + switch (list_or_range->getKind()) { + case Identifier: { + auto body = parse_block(); + return new SyntaxForList(list_or_range, loop_variable, body); + } + case KeywordRange: { + expect(""); + RangeFunction func; + FOR(value, 3) { + if (curr()->getKind() == TokenKind::RParenthesis + || curr()->getKind() == TokenKind::EOFToken) + break; + func.set[value] = expect("Range function values " + "must " + "be " + "integers")->getValue(); + expect("Expected comma in between parameters."); + } + auto body = parse_block(); + return new SyntaxForRange(loop_variable, func, body); + } + default: + this->diagnostics.build(ParseError{ + ParseErrorType::InvalidToken, + list_or_range->getSourceLocation(), + {ParseError::Metadata("Invalid token in for-loop context. Expected " + "a list variable or the range function.")} + }); + break; + } + return nullptr; } // TODO: Finish match statement. Leave this for later as it will be difficult // to see the outlook of this right now. const NG_INLINE SyntaxNode * -Parser::parse_match() { - return nullptr; +Parser::parse_match() { + return nullptr; } + const NG_INLINE SyntaxNode * -Parser::parse_match_pair_value() { - return nullptr; +Parser::parse_match_pair_value() { + return nullptr; } nextgen::ArenaVec Parser::parse_function_param() { - skip(); - auto current = curr(); + skip(); + auto current = curr(); - SyntaxFunctionParameter *begin = (SyntaxFunctionParameter*)(GLOBAL_OBJECT_ALLOC.current()); - SyntaxFunctionParameter *end = nullptr; + SyntaxFunctionParameter *begin = (SyntaxFunctionParameter *) (GLOBAL_OBJECT_ALLOC.current()); + SyntaxFunctionParameter *end = nullptr; - while (current->getKind() != TokenKind::RParenthesis) { - auto param_name = skip(); - expect("Expected ':' before parameter type (this is not" - " go!)"); + while (current->getKind() != TokenKind::RParenthesis) { + auto param_name = skip(); + expect("Expected ':' before parameter type (this is not" + " go!)"); - end = new SyntaxFunctionParameter(param_name, parse_type()); - } - return mem::ArenaVec{begin, end}; + end = new SyntaxFunctionParameter(param_name, parse_type()); + } + return mem::ArenaVec{begin, end}; } const NG_INLINE SyntaxFunction * Parser::parse_function(const Token *function_name) { - return new SyntaxFunction(function_name, - parse_type(), - parse_block(), - parse_function_param()); + return new SyntaxFunction(function_name, + parse_type(), + parse_block(), + parse_function_param()); } const NG_INLINE SyntaxNode * Parser::parse_function_call(const Token *function_name, const Token *delim) { - auto E = new SyntaxFunctionCall(function_name); - while(curr()->getKind() != TokenKind::RParenthesis) { - - // Looks odd, right? What's happening here is that the arena will allocate - // nodes in its continuous memory block and the function will write to that - // memory, so we don't need to worry about pushing values to the arena. - // - // TODO: Refactor some parsing functions to be void because of the arena - // allocation. - E->parameters << parse_expr(); - } - expect_delim(delim->getSourceLocation()); - return E; + auto E = new SyntaxFunctionCall(function_name); + while (curr()->getKind() != TokenKind::RParenthesis) { + + // Looks odd, right? What's happening here is that the arena will allocate + // nodes in its continuous memory block and the function will write to that + // memory, so we don't need to worry about pushing values to the arena. + // + // TODO: Refactor some parsing functions to be void because of the arena + // allocation. + E->parameters << parse_expr(); + } + expect_delim(delim->getSourceLocation()); + return E; } -SyntaxBlock Parser::parse_block() { - expect("Expected '{' before beginning of " - "current"); - auto block = SyntaxBlock {}; - // We know for sure that we got right curly brace - while (curr()->getKind() != TokenKind::RCurlyBrace - && curr()->getKind() != TokenKind::EOFToken) { - block.statements << parse_stmt(); - } - expect_delim(curr()->getSourceLocation()); - next(1); - return block; +SyntaxBlock Parser::parse_block() { + expect("Expected '{' before beginning of " + "current"); + auto block = SyntaxBlock{}; + // We know for sure that we got right curly brace + while (curr()->getKind() != TokenKind::RCurlyBrace + && curr()->getKind() != TokenKind::EOFToken) { + block.statements << parse_stmt(); + } + expect_delim(curr()->getSourceLocation()); + next(1); + return block; } nextgen::ArenaVec Parser::parse_struct_data_members() { - auto members = mem::ArenaVec(); - members.begin = (SyntaxStructMember*) (GLOBAL_OBJECT_ALLOC.current()); - while (curr()->getKind() != TokenKind::RCurlyBrace) { - auto name = expect("Identifiers must be used for " - "struct name"); - expect("':' Must be followed after struct member name"); - auto type = parse_type(); - members.end = new SyntaxStructMember(name, type); - } - return members; + auto members = mem::ArenaVec(); + members.begin = (SyntaxStructMember *) (GLOBAL_OBJECT_ALLOC.current()); + while (curr()->getKind() != TokenKind::RCurlyBrace) { + auto name = expect("Identifiers must be used for " + "struct name"); + expect("':' Must be followed after struct member name"); + auto type = parse_type(); + members.end = new SyntaxStructMember(name, type); + } + return members; } \ No newline at end of file diff --git a/lib/jet-x64-assembler.cpp b/lib/jet-x64-assembler.cpp index bdea471..3811ca3 100644 --- a/lib/jet-x64-assembler.cpp +++ b/lib/jet-x64-assembler.cpp @@ -302,41 +302,42 @@ write(0x0F, float_prefix);\ write_address(addr, reg); \ } \ + // [disp32] MemoryAddress::MemoryAddress(uint disp32) -: has_disp32(true), mod(MOD_INDIRECT), disp(disp32) {} + : has_disp32(true), mod(MOD_INDIRECT), disp(disp32) {} // [base] MemoryAddress::MemoryAddress(const Register base) : base(base), mod(MOD_INDIRECT) { - has_sib = (base & DI) == SP; - if ((base & DI) == BP) { - // Encodes as [base+0x0] - has_disp8 = true; - mod = MOD_DISP8; - } + has_sib = (base & DI) == SP; + if ((base & DI) == BP) { + // Encodes as [base+0x0] + has_disp8 = true; + mod = MOD_DISP8; + } } // [base + disp8] MemoryAddress::MemoryAddress(const Register base, ubyte disp8) -: base(base), mod (MOD_DISP8), disp(disp8) { - has_sib = (base & DI) == SP; - has_disp8 = true; + : base(base), mod(MOD_DISP8), disp(disp8) { + has_sib = (base & DI) == SP; + has_disp8 = true; } // [base + disp32] MemoryAddress::MemoryAddress(const Register base, uint disp32) -: base(base), mod (MOD_DISP32), disp(disp32) { - has_sib = (base & DI) == SP; - has_disp32 = true; + : base(base), mod(MOD_DISP32), disp(disp32) { + has_sib = (base & DI) == SP; + has_disp32 = true; } // [base + (index * scale)] MemoryAddress::MemoryAddress(const Register base, const Register index, SibScale scale) -: base(base), index(index), scale(scale), has_sib(true), mod(MOD_INDIRECT) { - if ((base & DI) == BP) { - mod = MOD_DISP8; - has_disp8 = true; - } + : base(base), index(index), scale(scale), has_sib(true), mod(MOD_INDIRECT) { + if ((base & DI) == BP) { + mod = MOD_DISP8; + has_disp8 = true; + } } // [base + (index * scale) + disp8] @@ -344,53 +345,53 @@ MemoryAddress::MemoryAddress(const Register base, const Register index, SibScale scale, ubyte disp8) - : base(base), index(index), scale(scale), - mod(MOD_DISP8), has_disp8(true), disp(disp8), - has_sib(true) {} + : base(base), index(index), scale(scale), + mod(MOD_DISP8), has_disp8(true), disp(disp8), + has_sib(true) {} - // [base + (index * scale) + disp32] - MemoryAddress::MemoryAddress(const Register base, - const Register index, - SibScale scale, - uint disp32) - : base(base), index(index), scale(scale), mod(MOD_DISP32), - has_disp32 (true), disp(disp32), has_sib(true) {} +// [base + (index * scale) + disp32] +MemoryAddress::MemoryAddress(const Register base, + const Register index, + SibScale scale, + uint disp32) + : base(base), index(index), scale(scale), mod(MOD_DISP32), + has_disp32(true), disp(disp32), has_sib(true) {} - /// Encode ModRM (no values known) - static ubyte EncodeModRM(ubyte mod, const Register reg, const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); +/// Encode ModRM (no values known) +static ubyte EncodeModRM(ubyte mod, const Register reg, const Register rm) { + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Encode ModRM (mod value known) template static ubyte EncodeModRM(const Register reg, const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Encode ModRM (mod and reg value known) template static ubyte EncodeModRM(const Register rm) { - return (mod << 6) | ((reg & DI) << 3) | (rm & DI); + return (mod << 6) | ((reg & DI) << 3) | (rm & DI); } /// Check if a rex value is needed for a given register for /// argument sizes that are (16, 32, 64) bit. static bool rex_needed(const Register reg) { - switch (reg) { - case R8: - case R9: - case R10: - case R11: - case R12: - case R13: - case R14: - case R15: - return true; - default: - return false; - } + switch (reg) { + case R8: + case R9: + case R10: + case R11: + case R12: + case R13: + case R14: + case R15: + return true; + default: + return false; + } } /// This function is the same as the 'rex_needed', but the only difference @@ -398,140 +399,140 @@ static bool rex_needed(const Register reg) { /// a rex byte should be encoded. As per manual/reference, 8-bit registers /// ah, ch, dh, and bh require a rex byte. static bool rex_needed2(const Register reg) { - switch (reg) { - case R8: - case R9: - case R10: - case R11: - case R12: - case R13: - case R14: - case R15: - case SP: // AH - case BP: // CH - case SI: // DH - case DI: // BH - return true; - default: - return false; - } + switch (reg) { + case R8: + case R9: + case R10: + case R11: + case R12: + case R13: + case R14: + case R15: + case SP: // AH + case BP: // CH + case SI: // DH + case DI: // BH + return true; + default: + return false; + } } /// Allocate the assembler with given 'size' value. Assembler::Assembler(const size_t size) { - memory = (ubyte *) malloc(size); + memory = (ubyte *) malloc(size); } /// Allocate the virtual page with given 'size' or give potential /// executable permissions to allocated memory. Assembler::Assembler(const size_t size, bool executable) { - // TODO: Add mmap to the virtual memory allocation to allocate the memory - // as executable + // TODO: Add mmap to the virtual memory allocation to allocate the memory + // as executable #ifdef NG_OS_WINDOWS -memory = (ubyte*) VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, - executable ? - PAGE_EXECUTE_READWRITE : PAGE_READWRITE); + memory = (ubyte *) VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, + executable ? + PAGE_EXECUTE_READWRITE : PAGE_READWRITE); #endif } /// Spill the memory buffer containing machine code bytes written ubyte *Assembler::spill() const { - return memory; + return memory; } /// The number of bytes that have been written to the memory buffer ulong Assembler::bytes() const { - return used; + return used; } /// Write a byte to the memory buffer. void Assembler::write(ubyte value) { - memory[used] = value; - used += sizeof(value); + memory[used] = value; + used += sizeof(value); } /// Write a 16-bit value to the memory buffer. void Assembler::write_16(ushort value) { - write(ubyte(value)); - write(ubyte(value >> 8)); + write(ubyte(value)); + write(ubyte(value >> 8)); } /// Write a 32-bit value to the memory buffer. void Assembler::write_32(uint value) { - write(ubyte(value)); - write(ubyte(value >> 8)); - write(ubyte(value >> 16)); - write(ubyte(value >> 24)); + write(ubyte(value)); + write(ubyte(value >> 8)); + write(ubyte(value >> 16)); + write(ubyte(value >> 24)); } /// Write a 64-bit value to the memory buffer. void Assembler::write_64(ulong value) { - write(ubyte(value)); - write(ubyte(value >> 8)); - write(ubyte(value >> 16)); - write(ubyte(value >> 24)); - write(ubyte(value >> 32)); - write(ubyte(value >> 40)); - write(ubyte(value >> 48)); - write(ubyte(value >> 52)); + write(ubyte(value)); + write(ubyte(value >> 8)); + write(ubyte(value >> 16)); + write(ubyte(value >> 24)); + write(ubyte(value >> 32)); + write(ubyte(value >> 40)); + write(ubyte(value >> 48)); + write(ubyte(value >> 52)); } /// Write an immediate 8-bit value. void Assembler::write(Imm8 imm) { - write(ubyte(imm.value)); + write(ubyte(imm.value)); } /// Write an immediate 16-bit value. void Assembler::write(Imm16 imm) { - write_16(imm.value); + write_16(imm.value); } /// Write an immediate 32-bit value. void Assembler::write(Imm32 imm) { - write_32(imm.value); + write_32(imm.value); } /// Write an immediate 64-bit value. void Assembler::write(Imm64 imm) { - write_64(imm.value); + write_64(imm.value); } /// Write a memory address given the register destination /// Reference: http://www.c-jump.com/CIS77/CPU/x86/lecture.html void Assembler::write_address(const MemoryAddress &addr, const Register dest) { - // Mod RM Byte Special Cases: - // 1. Address has SIB with no displacement R/M is 0b100 (SP) - // 2. Address has only displacement R/M is 0b101 (BP) - write((addr.mod << 6) | (dest & DI) << 3 | (addr.has_sib ? SP : (addr.base & DI))); + // Mod RM Byte Special Cases: + // 1. Address has SIB with no displacement R/M is 0b100 (SP) + // 2. Address has only displacement R/M is 0b101 (BP) + write((addr.mod << 6) | (dest & DI) << 3 | (addr.has_sib ? SP : (addr.base & DI))); - // Write SIB if it has one - if (addr.has_sib) - write((addr.scale << 6) | (addr.index & DI) << 3 | addr.base); + // Write SIB if it has one + if (addr.has_sib) + write((addr.scale << 6) | (addr.index & DI) << 3 | addr.base); - // Write Displacement if it has one - if (addr.has_disp32) write_32(addr.disp); - else if (addr.has_disp8) write(ubyte(addr.disp)); + // Write Displacement if it has one + if (addr.has_disp32) write_32(addr.disp); + else if (addr.has_disp8) write(ubyte(addr.disp)); } template void Assembler::write(T f, K s) { - write(f), write(s); + write(f), write(s); } template void Assembler::write(T f, K s, L t) { - write(f), write(s), write(t); + write(f), write(s), write(t); } template void Assembler::write(T f, K s, L t, Q fr) { - write(f), write(s), write(t), write(fr); + write(f), write(s), write(t), write(fr); } void Assembler::rex_optional_r1(const Register reg) { - if (rex_needed2(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + if (rex_needed2(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Write optional rex-values for argument sizes of 16-bits. @@ -539,381 +540,397 @@ void Assembler::rex_optional_r1(const Register reg) { /// If the register is R8-R15, it must have another rex byte /// that is written. void Assembler::rex_optional_r2(const Register reg) { - write(0x66); - if (rex_needed(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + write(0x66); + if (rex_needed(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Apply rex-byte to all operations that are not BYTE size. void Assembler::rex_optional_r3(const Register reg) { - if (rex_needed(reg)) - write(REX_PREFIX | (reg & 0b1000) >> 3); + if (rex_needed(reg)) + write(REX_PREFIX | (reg & 0b1000) >> 3); } /// Writes rex-values for 64-bit argument operations. This applies for /// the long-mode of x86-- which is x64. It also integrates rex-prefix /// with special register R8-R15. void Assembler::rex_optional_r4(const Register reg) { - if (rex_needed(reg)) - write(REX_PREFIX| REX_W | (reg & 0b1000) >> 3); - else write(0x48); + if (rex_needed(reg)) + write(REX_PREFIX | REX_W | (reg & 0b1000) >> 3); + else write(0x48); } /// Two const Register -- BYTE SIZE void Assembler::rex_optional_rr1(const Register dest, const Register src) { - if (rex_needed2(src) || rex_needed2(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + if (rex_needed2(src) || rex_needed2(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- WORD SIZE void Assembler::rex_optional_rr2(const Register dest, const Register src) { - write(0x66); - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + write(0x66); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- DWORD SIZE void Assembler::rex_optional_rr3(const Register dest, const Register src) { - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); } /// Two const Register -- QWORD SIZE void Assembler::rex_optional_rr4(const Register dest, const Register src) { - if (rex_needed(src) || rex_needed(dest)) - write(REX_PREFIX | REX_W | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); - else write(0x48); + if (rex_needed(src) || rex_needed(dest)) + write(REX_PREFIX | REX_W | (src & 0b1000) >> 1 | (dest & 0b1000) >> 3); + else write(0x48); } /// Memory Operation Rex Byte -- BYTE SIZE void Assembler::rex_optional_rm1(const Register reg, const MemoryAddress &addr) { - if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- WORD SIZE void Assembler::rex_optional_rm2(const Register reg, const MemoryAddress &addr) { - write(0x66); - if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + write(0x66); + if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- DWORD SIZE void Assembler::rex_optional_rm3(const Register reg, const MemoryAddress &addr) { - if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); + if (rex_needed(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); } /// Memory Operation Rex Byte -- QWORD SIZE void Assembler::rex_optional_rm4(const Register reg, const MemoryAddress &addr) { - if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) - write( - REX_PREFIX - // W - | REX_W - // R - | (reg & 0b1000) >> 1 - // X - | (addr.index & 0b1000) >> 2 - // B - | (addr.base & 0b1000) >> 3); - else write(0x48); + if (rex_needed2(reg) || rex_needed(addr.base) || rex_needed(addr.index)) + write( + REX_PREFIX + // W + | REX_W + // R + | (reg & 0b1000) >> 1 + // X + | (addr.index & 0b1000) >> 2 + // B + | (addr.base & 0b1000) >> 3); + else write(0x48); } //====================== // Unary Instructions //====================== unary_instruction(inc, 0xFE, 0xFF, 0); + unary_instruction(dec, 0xFE, 0xFF, 1); + unary_instruction(not, 0xF6, 0xF7, 2); + unary_instruction(neg, 0xF6, 0xF7, 3); + unary_instruction(mul, 0xF6, 0xF7, 4); + unary_instruction(div, 0xF6, 0xF7, 6); + unary_instruction(idiv, 0xF6, 0xF7, 7); //====================== // Binary Instructions //====================== binary_instruction(add, 0x00, 0x01, 0x02, 0x03); + binary_instruction_imm(add, 0x04, 0x05, 0); binary_instruction(sub, 0x28, 0x29, 0x2A, 0x2B); + binary_instruction_imm(sub, 0x2C, 0x2D, 5); binary_instruction(and, 0x20, 0x21, 0x22, 0x23); + binary_instruction_imm(and, 0x24, 0x25, 4); binary_instruction(xor, 0x30, 0x31, 0x32, 0x33); + binary_instruction_imm(xor, 0x34, 0x35, 6); binary_instruction(cmp, 0x38, 0x39, 0x3A, 0x3B); + binary_instruction_imm(cmp, 0x3C, 0x3D, 7); binary_instruction(or, 0x08, 0x09, 0x0A, 0x0B); + binary_instruction_imm(or, 0x0C, 0x0D, 1); floating_point_binary(add, 0x58); + floating_point_binary(sub, 0x5C); + floating_point_binary(div, 0x5E); + floating_point_binary(mul, 0x59); void Assembler::popq(const Register reg) { - write(0x58+reg); + write(0x58 + reg); } void Assembler::popw(const Register reg) { - write(0x66, 0x58+reg); + write(0x66, 0x58 + reg); } void Assembler::pushq(const Register reg) { - rex_optional_r4(reg); - write(0x50 + (reg & DI)); + rex_optional_r4(reg); + write(0x50 + (reg & DI)); } void Assembler::pushw(const Register reg) { - rex_optional_r2(reg); - write(0x50 + (reg & DI)); + rex_optional_r2(reg); + write(0x50 + (reg & DI)); } void Assembler::movb(const Register reg, Imm8 imm) { - rex_optional_r1(reg); - write(0xB0 + (reg & DI), imm); + rex_optional_r1(reg); + write(0xB0 + (reg & DI), imm); } template void Assembler::movw(const Register reg, const Imm imm) { - rex_optional_r2(reg); - write(0xB8 + (reg & DI), imm); + rex_optional_r2(reg); + write(0xB8 + (reg & DI), imm); } template void Assembler::movd(const Register reg, const Imm imm) { - rex_optional_r3(reg); - write(0xB8 + (reg & DI), imm); + rex_optional_r3(reg); + write(0xB8 + (reg & DI), imm); } template void Assembler::movq(const Register reg, const Imm imm) { - rex_optional_r4(reg); - write(0xC7, EncodeModRM<0b11, 0>(reg), imm); + rex_optional_r4(reg); + write(0xC7, EncodeModRM<0b11, 0>(reg), imm); } void Assembler::movb(const Register reg, const MemoryAddress &addr) { - rex_optional_rm1(reg, addr); - write(0x8A); - write_address(addr, reg); + rex_optional_rm1(reg, addr); + write(0x8A); + write_address(addr, reg); } void Assembler::movw(const Register reg, const MemoryAddress &addr) { - rex_optional_rm2(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm2(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movd(const Register reg, const MemoryAddress &addr) { - rex_optional_rm3(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm3(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movq(const Register reg, const MemoryAddress &addr) { - rex_optional_rm4(reg, addr); - write(0x8B); - write_address(addr, reg); + rex_optional_rm4(reg, addr); + write(0x8B); + write_address(addr, reg); } void Assembler::movb(const Register dest, const Register src) { - rex_optional_rr1(dest, src); - write(0x88, EncodeModRM<0b11>(src, dest)); + rex_optional_rr1(dest, src); + write(0x88, EncodeModRM<0b11>(src, dest)); } void Assembler::movw(const Register dest, const Register src) { - rex_optional_rr2(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr2(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::movd(const Register dest, const Register src) { - rex_optional_rr3(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr3(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::movq(const Register dest, const Register src) { - rex_optional_rr4(dest, src); - write(0x89, EncodeModRM<0b11>(src, dest)); + rex_optional_rr4(dest, src); + write(0x89, EncodeModRM<0b11>(src, dest)); } void Assembler::shlb(const Register dest, const Imm8 count) { - rex_optional_r1(dest); - write(0xC0, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r1(dest); + write(0xC0, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlw(const Register dest, const Imm8 count) { - rex_optional_r2(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r2(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shld(const Register dest, const Imm8 count) { - rex_optional_r3(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r3(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlq(const Register dest, const Imm8 count) { - rex_optional_r4(dest); - write(0xC1, EncodeModRM<0b11, SP>(dest), count); + rex_optional_r4(dest); + write(0xC1, EncodeModRM<0b11, SP>(dest), count); } void Assembler::shlb(const Register dest) { - rex_optional_r1(dest); - write(0xD0, EncodeModRM<0b11, SP>(dest)); + rex_optional_r1(dest); + write(0xD0, EncodeModRM<0b11, SP>(dest)); } void Assembler::shlw(const Register dest) { - rex_optional_r2(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest)); + rex_optional_r2(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest)); } void Assembler::shld(const Register dest) { - rex_optional_r3(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest), 1); + rex_optional_r3(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest), 1); } void Assembler::shlq(const Register dest) { - rex_optional_r4(dest); - write(0xD1, EncodeModRM<0b11, SP>(dest)); + rex_optional_r4(dest); + write(0xD1, EncodeModRM<0b11, SP>(dest)); } void Assembler::shrb(const Register dest, Imm8 count) { - rex_optional_r1(dest); - write(0xC0, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r1(dest); + write(0xC0, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrw(const Register dest, Imm8 count) { - rex_optional_r2(dest); - write(0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r2(dest); + write(0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrd(const Register dest, Imm8 count) { - rex_optional_r3(dest); - write(0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r3(dest); + write(0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::shrq(const Register dest, Imm8 count) { - rex_optional_r4(dest); - write(0x48, 0xC1, EncodeModRM<0b11, BX>(dest), count); + rex_optional_r4(dest); + write(0x48, 0xC1, EncodeModRM<0b11, BX>(dest), count); } void Assembler::jmp(Imm8 rel8) { - write(0xEB, rel8); + write(0xEB, rel8); } void Assembler::jmp(Imm16 rel16) { - write(0xE9, rel16); + write(0xE9, rel16); } void Assembler::jmp(Imm32 rel32) { - write(0xE9, rel32); + write(0xE9, rel32); } void Assembler::jmpw(const Register reg) { - rex_optional_r2(reg); - write(0xFF, EncodeModRM<0b11, SP>(reg)); + rex_optional_r2(reg); + write(0xFF, EncodeModRM<0b11, SP>(reg)); } void Assembler::jmpq(const Register reg) { - rex_optional_r4(reg); - write(0xFF, EncodeModRM<0b11, SP>(reg)); + rex_optional_r4(reg); + write(0xFF, EncodeModRM<0b11, SP>(reg)); } // NOTE: imul instruction order for rex is (src, dest), yes, this is // correct. void Assembler::imulw(const Register dest, const Register src) { - rex_optional_rr2(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr2(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::imuld(const Register dest, const Register src) { - rex_optional_rr3(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr3(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::imulq(const Register dest, const Register src) { - rex_optional_rr4(src, dest); - write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); + rex_optional_rr4(src, dest); + write(0x0F, 0xAF, EncodeModRM<0b11>(src, dest)); } void Assembler::jump_cond(Condition cond, Imm8 rel8) { - write(0x70 | cond, rel8); + write(0x70 | cond, rel8); } void Assembler::jump_cond(Condition cond, Imm16 rel16) { - write(0x80 | cond, rel16); + write(0x80 | cond, rel16); } void Assembler::jump_cond(Condition cond, Imm32 rel32) { - write(0x80 | cond, rel32); + write(0x80 | cond, rel32); } void Assembler::lea(const Register reg, const MemoryAddress &addr, const OperandSize size) { - switch (size) { - case BYTE: break; - case WORD: - rex_optional_rm2(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - case DWORD: - rex_optional_rm3(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - case QWORD: - rex_optional_rm4(reg, addr); - write(0x8D); - write_address(addr, reg); - break; - } + switch (size) { + case BYTE: + break; + case WORD: + rex_optional_rm2(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + case DWORD: + rex_optional_rm3(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + case QWORD: + rex_optional_rm4(reg, addr); + write(0x8D); + write_address(addr, reg); + break; + } } void Assembler::movss(const Register dest, const Register src) { - rex_optional_rr3(dest, src); - write(0xF3, 0x0F, 0x10, EncodeModRM<0b11>(dest, src)); + rex_optional_rr3(dest, src); + write(0xF3, 0x0F, 0x10, EncodeModRM<0b11>(dest, src)); } void Assembler::movss(const Register dest, const MemoryAddress &addr) { - rex_optional_rm3(dest, addr); - write(0xF3, 0x0F, 0x10); - write_address(addr, dest); + rex_optional_rm3(dest, addr); + write(0xF3, 0x0F, 0x10); + write_address(addr, dest); } void Assembler::movss(const MemoryAddress &dest, Register src) { - rex_optional_rm3(src, dest); - write(0xF3, 0x0F, 0x11); - write_address(dest, src); + rex_optional_rm3(src, dest); + write(0xF3, 0x0F, 0x11); + write_address(dest, src); } void Assembler::ret() { - write(0xC3); + write(0xC3); } \ No newline at end of file diff --git a/lib/str.cpp b/lib/str.cpp index 9f58ae3..a3079fd 100644 --- a/lib/str.cpp +++ b/lib/str.cpp @@ -4,16 +4,16 @@ using nextgen::str; using nextgen::StringCache; // Inline in string intern on constant strings -str nextgen::operator""_intern(const char *s, size_t len) { - str ss = str { s, len }; - ss.setHash(); - return StringCache::InsertOrGet(ss); +str nextgen::operator ""_intern(const char *s, size_t len) { + str ss = str{s, len}; + ss.setHash(); + return StringCache::InsertOrGet(ss); } // We write using the length because `str` may contain shadowed data // to Range. std::ostream &nextgen::operator<<(std::ostream &s, str &str) { - s.write(str.begin(), str.size()); - return s; + s.write(str.begin(), str.size()); + return s; } diff --git a/main.cpp b/main.cpp index 376e806..1948a3c 100644 --- a/main.cpp +++ b/main.cpp @@ -1,18 +1,14 @@ #include #include - using namespace nextgen::mem; using namespace nextgen::jet; using namespace nextgen; - - int main(int argc, char **argv, char **envP) { + try { - try { - - } catch(std::exception&) { - mem::deinit(); - } + } catch (std::exception &) { + mem::deinit(); + } } \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a371dd2..ac3f90e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.0) project(tests) - # Setup Google Test Compilation Cases add_executable(tests @@ -31,7 +30,7 @@ add_executable(tests lex-test.cpp parse-test.cpp x64-test.cpp -) + ) #set(gtest_force_shared_crt ON CACHE BOOL "MSVC defaulting to CRT" FORCE) diff --git a/tests/lex-test.cpp b/tests/lex-test.cpp index 3d9af84..d7dc64f 100644 --- a/tests/lex-test.cpp +++ b/tests/lex-test.cpp @@ -1,24 +1,24 @@ #include #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif TEST(LexTest, NonFMTNumbers) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer ( "1211", "src/test.jet", 4); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("1211", "src/test.jet", 4); + auto token = lexer.lex()[0]; // ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); @@ -27,318 +27,319 @@ TEST(LexTest, NonFMTNumbers) { } TEST(LexTest, DecimalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer ( "23.246", "src/test.jet", 6); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("23.246", "src/test.jet", 6); + auto token = lexer.lex()[0]; - ASSERT_EQ(token->template getValue(), 23.246); - ASSERT_EQ(token->getKind(), jet::TokenKind::Decimal); + ASSERT_EQ(token->template getValue(), 23.246); + ASSERT_EQ(token->getKind(), jet::TokenKind::Decimal); } TEST(LexTest, HexadecimalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0x1111", "src/test.jet", 6); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("0x1111", "src/test.jet", 6); + auto token = lexer.lex()[0]; // ASSERT_EQ(token->getValue(), 0x1111); // ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, BinaryNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0b011101110", "src/test.jet", 11); - auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 0b011101110); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + auto lexer = jet::Lexer("0b011101110", "src/test.jet", 11); + auto token = lexer.lex()[0]; + ASSERT_EQ(token->getValue(), 0b011101110); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, OctalNumber) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0112022", "src/test.jet", 7); - auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 0112022); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + auto lexer = jet::Lexer("0112022", "src/test.jet", 7); + auto token = lexer.lex()[0]; + ASSERT_EQ(token->getValue(), 0112022); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, Base36Number) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "0#helloworld", "src/test.jet", 12); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("0#helloworld", "src/test.jet", 12); + auto token = lexer.lex()[0]; - ASSERT_EQ(token->getValue(), 1767707668033969); - ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); + ASSERT_EQ(token->getValue(), 1767707668033969); + ASSERT_EQ(token->getKind(), jet::TokenKind::Integer); } TEST(LexTest, Identifier) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "ident", "src/test.jet", 5); - auto token = lexer.lex()[0]; + auto lexer = jet::Lexer("ident", "src/test.jet", 5); + auto token = lexer.lex()[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "ident", 5) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); + ASSERT_TRUE(::strncmp(token->name().begin(), "ident", 5) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); } TEST(LexTest, Keyword) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( "while extern", "src/test.jet", - 12); + auto lexer = jet::Lexer("while extern", "src/test.jet", + 12); - auto tokens = lexer.lex(); - auto token = tokens[0]; + auto tokens = lexer.lex(); + auto token = tokens[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "while", 5) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordWhile); + ASSERT_TRUE(::strncmp(token->name().begin(), "while", 5) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordWhile); - token = tokens[1]; - ASSERT_TRUE(::strncmp(token->name().begin(), "extern", 6) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordExtern); + token = tokens[1]; + ASSERT_TRUE(::strncmp(token->name().begin(), "extern", 6) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::KeywordExtern); } TEST(LexTest, String) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto lexer = jet::Lexer( R"("Hello" ace "There")", - "src/test.jet", 19); + auto lexer = jet::Lexer(R"("Hello" ace "There")", + "src/test.jet", 19); - auto tokens = lexer.lex(); - auto token = tokens[0]; + auto tokens = lexer.lex(); + auto token = tokens[0]; - ASSERT_TRUE(::strncmp(token->name().begin(), "Hello", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::String); + ASSERT_TRUE(::strncmp(token->name().begin(), "Hello", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::String); - token = tokens[1]; + token = tokens[1]; - ASSERT_TRUE(::strncmp(token->name().begin(), "ace", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); + ASSERT_TRUE(::strncmp(token->name().begin(), "ace", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::Identifier); - token = tokens[2]; + token = tokens[2]; - ASSERT_TRUE(::strncmp(token->name().begin(), "There", token->len()) == 0); - ASSERT_EQ(token->getKind(), jet::TokenKind::String); + ASSERT_TRUE(::strncmp(token->name().begin(), "There", token->len()) == 0); + ASSERT_EQ(token->getKind(), jet::TokenKind::String); } TEST(LexTest, StringEscape) { // TODO: Add Unicode Escape Later - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(var every_escape = "I have\x56 every \t thing\n that could \b\v in \r")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - ArenaVec tokens{nullptr, nullptr}; - try { - tokens = lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } - auto token = tokens[3]; - - auto compare = "I have\x56 every \t thing\n that could \b\v in \r"; - ASSERT_TRUE(::strncmp(token->name().begin(), compare, token->len()) == 0); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(var every_escape = "I have\x56 every \t thing\n that could \b\v in \r")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + ArenaVec tokens{nullptr, nullptr}; + try { + tokens = lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } + auto token = tokens[3]; + + auto compare = "I have\x56 every \t thing\n that could \b\v in \r"; + ASSERT_TRUE(::strncmp(token->name().begin(), compare, token->len()) == 0); } TEST(LexTest, AllTokens) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - - auto buf = "+ += ++ - -= -- * *= ** **= / /= > >= >> >>= < <= << <<= % %=" - "() {} " - "[] ~ ^ ^= " - "| |= & &= = == : ; @ ! ? . .. ..."; // 43 tokens - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - - - ArenaVec tokens{nullptr, nullptr}; - try { - tokens = lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } - ASSERT_EQ(tokens.len(), 45); - ///* ASSERT_EQ(tokens[0].getKind(), jet::TokenKind::Plus); - // ASSERT_EQ(tokens[1].getKind(), jet::TokenKind::PlusEquals); - // ASSERT_EQ(tokens[2].getKind(), jet::TokenKind::PlusPlus); - // ASSERT_EQ(tokens[3].getKind(), jet::TokenKind::Minus); - // ASSERT_EQ(tokens[4].getKind(), jet::TokenKind::MinusEquals); - // ASSERT_EQ(tokens[5].getKind(), jet::TokenKind::MinusMinus); - // ASSERT_EQ(tokens[6].getKind(), jet::TokenKind::Star); - // ASSERT_EQ(tokens[7].getKind(), jet::TokenKind::MulEquals); - // ASSERT_EQ(tokens[8].getKind(), jet::TokenKind::Pow); - // ASSERT_EQ(tokens[9].getKind(), jet::TokenKind::Slash);*//* + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + + auto buf = "+ += ++ - -= -- * *= ** **= / /= > >= >> >>= < <= << <<= % %=" + "() {} " + "[] ~ ^ ^= " + "| |= & &= = == : ; @ ! ? . .. ..."; // 43 tokens + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + + + ArenaVec tokens{nullptr, nullptr}; + try { + tokens = lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } + ASSERT_EQ(tokens.len(), 45); + ///* ASSERT_EQ(tokens[0].getKind(), jet::TokenKind::Plus); + // ASSERT_EQ(tokens[1].getKind(), jet::TokenKind::PlusEquals); + // ASSERT_EQ(tokens[2].getKind(), jet::TokenKind::PlusPlus); + // ASSERT_EQ(tokens[3].getKind(), jet::TokenKind::Minus); + // ASSERT_EQ(tokens[4].getKind(), jet::TokenKind::MinusEquals); + // ASSERT_EQ(tokens[5].getKind(), jet::TokenKind::MinusMinus); + // ASSERT_EQ(tokens[6].getKind(), jet::TokenKind::Star); + // ASSERT_EQ(tokens[7].getKind(), jet::TokenKind::MulEquals); + // ASSERT_EQ(tokens[8].getKind(), jet::TokenKind::Pow); + // ASSERT_EQ(tokens[9].getKind(), jet::TokenKind::Slash);*//* } TEST(LexTest, ErrorOverflow) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "test := 0xffffffffffffffffffaaaaaaa test"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = "test := 0xffffffffffffffffffaaaaaaa test"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } + TEST(LexTest, DigitOutOfRange) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "invalid := 0xfffzz"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); + auto buf = "invalid := 0xfffzz"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); - try { - lexer.lex(); - } catch(std::exception&) { - return; - } + try { + lexer.lex(); + } catch (std::exception &) { + return; + } } TEST(LexTest, HexEscapeOutOfRange) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = R"(failed_string := "Hello\x2z")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = R"(failed_string := "Hello\x2z")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, InvalidStringEscape) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = R"(export fail_again := "Hello Test\q")"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + auto buf = R"(export fail_again := "Hello Test\q")"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, MissingClosingDelimString) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(export fail_again := "Hello Test\n)"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(export fail_again := "Hello Test\n)"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST(LexTest, MissingClosingDelimChar) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = R"(test := 'a)"; - auto len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", len); - try { - lexer.lex(); - } catch (std::exception &) { - // Do nothing ... - return; - } + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = R"(test := 'a)"; + auto len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", len); + try { + lexer.lex(); + } catch (std::exception &) { + // Do nothing ... + return; + } } TEST_SUITE_MAIN(LexTest) { - TEST_CALL(LexTest, NonFMTNumbers); - TEST_CALL(LexTest, DecimalNumber); - TEST_CALL(LexTest, HexadecimalNumber); - TEST_CALL(LexTest, BinaryNumber); - TEST_CALL(LexTest, OctalNumber); - TEST_CALL(LexTest, Base36Number); - TEST_CALL(LexTest, Identifier); - TEST_CALL(LexTest, Keyword); - TEST_CALL(LexTest, String); - TEST_CALL(LexTest, StringEscape); - TEST_CALL(LexTest, AllTokens); - TEST_CALL(LexTest, ErrorOverflow); - TEST_CALL(LexTest, DigitOutOfRange); - TEST_CALL(LexTest, HexEscapeOutOfRange); - TEST_CALL(LexTest, InvalidStringEscape); - TEST_CALL(LexTest, MissingClosingDelimString); - TEST_CALL(LexTest, MissingClosingDelimChar); + TEST_CALL(LexTest, NonFMTNumbers); + TEST_CALL(LexTest, DecimalNumber); + TEST_CALL(LexTest, HexadecimalNumber); + TEST_CALL(LexTest, BinaryNumber); + TEST_CALL(LexTest, OctalNumber); + TEST_CALL(LexTest, Base36Number); + TEST_CALL(LexTest, Identifier); + TEST_CALL(LexTest, Keyword); + TEST_CALL(LexTest, String); + TEST_CALL(LexTest, StringEscape); + TEST_CALL(LexTest, AllTokens); + TEST_CALL(LexTest, ErrorOverflow); + TEST_CALL(LexTest, DigitOutOfRange); + TEST_CALL(LexTest, HexEscapeOutOfRange); + TEST_CALL(LexTest, InvalidStringEscape); + TEST_CALL(LexTest, MissingClosingDelimString); + TEST_CALL(LexTest, MissingClosingDelimChar); } \ No newline at end of file diff --git a/tests/parse-test.cpp b/tests/parse-test.cpp index 55c9b9d..9a8c53b 100644 --- a/tests/parse-test.cpp +++ b/tests/parse-test.cpp @@ -1,314 +1,321 @@ #include #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif TEST(ParseTest, VariableDeclLiteral) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "hello := 23;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = static_cast(expr); - ASSERT_EQ(expr->kind, SyntaxKind::VariableAssignment); - // //ASSERT_EQ(decl->name->name(), "hello"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "hello", decl->name->len()) == 0); - ASSERT_EQ(decl->expression->kind, jet::SyntaxKind::LiteralValue); - ASSERT_EQ(CAST(const SyntaxLiteral*, decl->expression) - ->literal->getValue - (), 23); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "hello := 23;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = static_cast(expr); + ASSERT_EQ(expr->kind, SyntaxKind::VariableAssignment); + // //ASSERT_EQ(decl->name->name(), "hello"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "hello", decl->name->len()) == 0); + ASSERT_EQ(decl->expression->kind, jet::SyntaxKind::LiteralValue); + ASSERT_EQ(CAST(const SyntaxLiteral*, decl->expression) + ->literal->getValue + (), 23); } TEST(ParseTest, VariableArrayDecl) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - - auto buf = "my_list := [11, 2, 55];"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto node = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, node); - auto list = CAST(SyntaxList*, decl->expression); - - - ASSERT_EQ(decl->kind, SyntaxKind::VariableAssignment); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[0]) - ->literal->getValue - (), 11); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[1]) - ->literal->getValue(), 2); - ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[2]) - ->literal->getValue - (), 55); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + + auto buf = "my_list := [11, 2, 55];"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto node = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, node); + auto list = CAST(SyntaxList*, decl->expression); + + + ASSERT_EQ(decl->kind, SyntaxKind::VariableAssignment); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[0]) + ->literal->getValue + (), 11); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[1]) + ->literal->getValue(), 2); + ASSERT_EQ(CAST(const SyntaxLiteral*, list->values[2]) + ->literal->getValue + (), 55); } TEST(ParseTest, VariableDeclBinary) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "wow := 2 + 3 * 2;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); + auto buf = "wow := 2 + 3 * 2;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, expr); + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, expr); - auto binary = CAST(SyntaxBinary*, decl->expression); + auto binary = CAST(SyntaxBinary*, decl->expression); - auto lhs = CAST(SyntaxLiteral*, binary->lhs); - auto op = binary->operation; - auto rhs = CAST(SyntaxBinary*, binary->rhs); + auto lhs = CAST(SyntaxLiteral*, binary->lhs); + auto op = binary->operation; + auto rhs = CAST(SyntaxBinary*, binary->rhs); - auto rhs_lhs = CAST(SyntaxLiteral*, rhs->lhs); - auto rhs_rhs = CAST(SyntaxLiteral*, rhs->rhs); + auto rhs_lhs = CAST(SyntaxLiteral*, rhs->lhs); + auto rhs_rhs = CAST(SyntaxLiteral*, rhs->rhs); - ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); - //ASSERT_EQ(decl->name->name(), "wow"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "wow", decl->name->len()) == 0); - ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); + ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); + //ASSERT_EQ(decl->name->name(), "wow"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "wow", decl->name->len()) == 0); + ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); - ASSERT_EQ(op, SyntaxBinaryOp::Addition); - ASSERT_EQ(lhs->literal->getValue(),2); - ASSERT_EQ(rhs_lhs->literal->getValue(), 3); - ASSERT_EQ(rhs_rhs->literal->getValue(), 2); + ASSERT_EQ(op, SyntaxBinaryOp::Addition); + ASSERT_EQ(lhs->literal->getValue(), 2); + ASSERT_EQ(rhs_lhs->literal->getValue(), 3); + ASSERT_EQ(rhs_rhs->literal->getValue(), 2); } TEST(ParseTest, VariableDeclParenBinary) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "awesome := (2 + 3) * 2;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - auto expr = parser.parse_variable_assignment(name); - auto decl = CAST(SyntaxVariableAssignment*, expr); - - auto binary = CAST(SyntaxBinary*, decl->expression); - - auto lhs = CAST(SyntaxBinary*, binary->lhs); - auto lhs_lhs = CAST(SyntaxLiteral*, lhs->lhs); - auto lhs_rhs = CAST(SyntaxLiteral*, lhs->rhs); - - auto op = binary->operation; - auto rhs = CAST(SyntaxLiteral*, binary->rhs); - - ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); - //ASSERT_EQ(decl->name->name(), "wow"_intern); - ASSERT_TRUE(::strncmp(decl->name->name().begin(), "awesome", decl->name->len()) == 0); - ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); - - ASSERT_EQ(op, SyntaxBinaryOp::Multiplication); - ASSERT_EQ(lhs_lhs->literal->getValue(),2); - ASSERT_EQ(lhs_rhs->literal->getValue(), 3); - ASSERT_EQ(rhs->literal->getValue(), 2); + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "awesome := (2 + 3) * 2;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + auto expr = parser.parse_variable_assignment(name); + auto decl = CAST(SyntaxVariableAssignment*, expr); + + auto binary = CAST(SyntaxBinary*, decl->expression); + + auto lhs = CAST(SyntaxBinary*, binary->lhs); + auto lhs_lhs = CAST(SyntaxLiteral*, lhs->lhs); + auto lhs_rhs = CAST(SyntaxLiteral*, lhs->rhs); + + auto op = binary->operation; + auto rhs = CAST(SyntaxLiteral*, binary->rhs); + + ASSERT_EQ(expr->kind, jet::SyntaxKind::VariableAssignment); + //ASSERT_EQ(decl->name->name(), "wow"_intern); + ASSERT_TRUE(::strncmp(decl->name->name().begin(), "awesome", decl->name->len()) == 0); + ASSERT_EQ(binary->kind, jet::SyntaxKind::Binary); + + ASSERT_EQ(op, SyntaxBinaryOp::Multiplication); + ASSERT_EQ(lhs_lhs->literal->getValue(), 2); + ASSERT_EQ(lhs_rhs->literal->getValue(), 3); + ASSERT_EQ(rhs->literal->getValue(), 2); } TEST(ParseTest, VariableValueAssignment) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "weather <<= 2223;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); + auto buf = "weather <<= 2223;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); - auto node = (SyntaxVariableValueAssignment*) parser.parse_stmt(); + auto node = (SyntaxVariableValueAssignment *) parser.parse_stmt(); - ASSERT_TRUE(::strncmp(node->name->name().begin(), "weather", node->name->len()) == 0); - ASSERT_EQ(node->op, SyntaxAssignmentOp::LShiftAssign); - ASSERT_EQ(((SyntaxLiteral*)(node->expression))->literal->getValue(), - 2223); + ASSERT_TRUE(::strncmp(node->name->name().begin(), "weather", node->name->len()) == 0); + ASSERT_EQ(node->op, SyntaxAssignmentOp::LShiftAssign); + ASSERT_EQ(((SyntaxLiteral *) (node->expression))->literal->getValue(), + 2223); } TEST(ParseTest, IfStatement) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "if value { nice := 23; }"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto buf = "if value { nice := 23; }"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_if(); - auto stmt = (SyntaxIf*) node; + auto node = parser.parse_if(); + auto stmt = (SyntaxIf *) node; - auto if_cond_node = stmt->condition; - auto if_cond = (SyntaxLiteral*) stmt->condition; + auto if_cond_node = stmt->condition; + auto if_cond = (SyntaxLiteral *) stmt->condition; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_EQ(if_cond_node->kind, SyntaxKind::LiteralValue); - ASSERT_TRUE(::strncmp(if_cond->literal->name().begin(), "value", - if_cond->literal->len()) == 0); + ASSERT_EQ(if_cond_node->kind, SyntaxKind::LiteralValue); + ASSERT_TRUE(::strncmp(if_cond->literal->name().begin(), "value", + if_cond->literal->len()) == 0); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "nice", variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression)->literal->getValue(), 23); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "nice", variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression)->literal->getValue(), 23); } TEST(ParseTest, WhileLoop) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "while some_value > 223 { another := 43; }"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto buf = "while some_value > 223 { another := 43; }"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_while(); - auto stmt = (SyntaxWhile*) node; + auto node = parser.parse_while(); + auto stmt = (SyntaxWhile *) node; - auto while_cond_node = stmt->condition; + auto while_cond_node = stmt->condition; - auto while_cond = (SyntaxBinary*) stmt->condition; - auto lhs_cond = (SyntaxLiteral*) while_cond->lhs; - auto bin_op = while_cond->operation; - auto rhs_cond = (SyntaxLiteral*) while_cond->rhs; + auto while_cond = (SyntaxBinary *) stmt->condition; + auto lhs_cond = (SyntaxLiteral *) while_cond->lhs; + auto bin_op = while_cond->operation; + auto rhs_cond = (SyntaxLiteral *) while_cond->rhs; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_EQ(while_cond_node->kind, SyntaxKind::Binary); - ASSERT_TRUE(::strncmp(lhs_cond->literal->name().begin(), "some_value", - lhs_cond->literal->len()) == 0); - ASSERT_EQ(bin_op, SyntaxBinaryOp::Greater); - ASSERT_EQ(rhs_cond->literal->getValue(), 223); + ASSERT_EQ(while_cond_node->kind, SyntaxKind::Binary); + ASSERT_TRUE(::strncmp(lhs_cond->literal->name().begin(), "some_value", + lhs_cond->literal->len()) == 0); + ASSERT_EQ(bin_op, SyntaxBinaryOp::Greater); + ASSERT_EQ(rhs_cond->literal->getValue(), 223); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "another", - variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression) - ->literal->getValue(), 43); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "another", + variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression) + ->literal->getValue(), 43); } TEST(ParseTest, ForStatement) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; - auto buf = "for value in list_of_values { \n" - " do_something := 23.233;" - "}"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); + auto buf = "for value in list_of_values { \n" + " do_something := 23.233;" + "}"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - parser.skip(1); + auto parser = jet::Parser(&lexer); + parser.skip(1); - auto node = parser.parse_for(); - auto stmt = (SyntaxForList*) node; - auto variable_decl = (SyntaxVariableAssignment*) stmt->body.statements[0]; + auto node = parser.parse_for(); + auto stmt = (SyntaxForList *) node; + auto variable_decl = (SyntaxVariableAssignment *) stmt->body.statements[0]; - ASSERT_TRUE(::strncmp(stmt->list_var->name().begin(), "value", - stmt->list_var->len()) == 0); - ASSERT_TRUE(::strncmp(stmt->list_name->name().begin(), "list_of_values", - stmt->list_name->len()) == 0); - ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "do_something", - variable_decl->name->len()) == 0); - ASSERT_EQ(CAST(SyntaxLiteral*, variable_decl->expression) - ->literal->getValue(), 23.233); + ASSERT_TRUE(::strncmp(stmt->list_var->name().begin(), "value", + stmt->list_var->len()) == 0); + ASSERT_TRUE(::strncmp(stmt->list_name->name().begin(), "list_of_values", + stmt->list_name->len()) == 0); + ASSERT_TRUE(::strncmp(variable_decl->name->name().begin(), "do_something", + variable_decl->name->len()) == 0); + ASSERT_EQ(CAST(SyntaxLiteral * , variable_decl->expression) + ->literal->getValue(), 23.233); } TEST(ParseTest, SyntaxSemiColonError) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "// this is a comment\ncool := 23 / (22+3);\nanother_var := " - "232 {"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - auto name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - - try { - name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - } catch(std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "// this is a comment\ncool := 23 / (22+3);\nanother_var := " + "232 {"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + + try { + name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + } catch (std::exception &) {} } TEST(ParseTest, SyntaxMissingDelimError) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "some_value := 22 * 2 / (232 + 4 + (2 + (3*2) / 2)"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - try { - auto name = parser.curr(); parser.skip(2); - parser.parse_variable_assignment(name); - } catch (std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "some_value := 22 * 2 / (232 + 4 + (2 + (3*2) / 2)"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + try { + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + } catch (std::exception &) {} } TEST(ParseTest, SyntaxInvalidGlobalDecl) { - using namespace nextgen; - using namespace nextgen::core; - using namespace nextgen::jet; - - auto buf = "some_value := 2 + 3 / 232;\n value << 23;"; - auto buf_len = strlen(buf); - auto lexer = jet::Lexer( buf, "src/test.jet", buf_len); - auto parser = jet::Parser(&lexer); - - try { - auto name = parser.curr(); - parser.skip(2); - parser.parse_variable_assignment(name); - parser.parse(); - } catch (std::exception&) {} + using namespace nextgen; + using namespace nextgen::core; + using namespace nextgen::jet; + + auto buf = "some_value := 2 + 3 / 232;\n value << 23;"; + auto buf_len = strlen(buf); + auto lexer = jet::Lexer(buf, "src/test.jet", buf_len); + auto parser = jet::Parser(&lexer); + + try { + auto name = parser.curr(); + parser.skip(2); + parser.parse_variable_assignment(name); + parser.parse(); + } catch (std::exception &) {} } TEST_SUITE_MAIN(ParseTest) { - TEST_CALL(ParseTest, VariableDeclLiteral); - TEST_CALL(ParseTest, VariableArrayDecl); - TEST_CALL(ParseTest, VariableDeclBinary); - TEST_CALL(ParseTest, VariableDeclParenBinary); - TEST_CALL(ParseTest, VariableValueAssignment); - TEST_CALL(ParseTest, IfStatement); - TEST_CALL(ParseTest, WhileLoop); - TEST_CALL(ParseTest, SyntaxSemiColonError); - TEST_CALL(ParseTest, SyntaxMissingDelimError); - TEST_CALL(ParseTest, SyntaxInvalidGlobalDecl); - TEST_CALL(ParseTest, ForStatement); + TEST_CALL(ParseTest, VariableDeclLiteral); + TEST_CALL(ParseTest, VariableArrayDecl); + TEST_CALL(ParseTest, VariableDeclBinary); + TEST_CALL(ParseTest, VariableDeclParenBinary); + TEST_CALL(ParseTest, VariableValueAssignment); + TEST_CALL(ParseTest, IfStatement); + TEST_CALL(ParseTest, WhileLoop); + TEST_CALL(ParseTest, SyntaxSemiColonError); + TEST_CALL(ParseTest, SyntaxMissingDelimError); + TEST_CALL(ParseTest, SyntaxInvalidGlobalDecl); + TEST_CALL(ParseTest, ForStatement); } diff --git a/tests/test-main.cpp b/tests/test-main.cpp index 41782c6..cef904b 100644 --- a/tests/test-main.cpp +++ b/tests/test-main.cpp @@ -6,12 +6,14 @@ V(ParseTest) \ V(x64Test) \ + #include DEF_ALL_TESTS + int main(int argc, char **argv) { - TEST(LexTest); - TEST(ParseTest); - TEST(x64Test); - nextgen::mem::deinit(); + TEST(LexTest); + TEST(ParseTest); + TEST(x64Test); + nextgen::mem::deinit(); } diff --git a/tests/x64-test.cpp b/tests/x64-test.cpp index 566f2cd..2b0b925 100644 --- a/tests/x64-test.cpp +++ b/tests/x64-test.cpp @@ -1,11 +1,11 @@ #ifndef TEST - #define TEST(suite, name) void suite##_##name##_##test() +#define TEST(suite, name) void suite##_##name##_##test() #endif #ifndef TEST_SUITE_MAIN - #define TEST_SUITE_MAIN(suite) void main_##suite() +#define TEST_SUITE_MAIN(suite) void main_##suite() #endif #ifndef TEST_CALL - #define TEST_CALL(suite, name) suite##_##name##_##test() +#define TEST_CALL(suite, name) suite##_##name##_##test() #endif #include @@ -13,97 +13,97 @@ TEST(x64Test, Add) { - using namespace nextgen::jet::x64; - using namespace nextgen; + using namespace nextgen::jet::x64; + using namespace nextgen; - Assembler assembler = Assembler(1000); - for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) +1)) { - assembler.addb((Register)r, (Register)r); - assembler.addw((Register)r, (Register)r); - assembler.addd((Register)r, (Register)r); - assembler.addq((Register)r, (Register)r); - } + Assembler assembler = Assembler(1000); + for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) + 1)) { + assembler.addb((Register) r, (Register) r); + assembler.addw((Register) r, (Register) r); + assembler.addd((Register) r, (Register) r); + assembler.addq((Register) r, (Register) r); + } - auto number_of_bytes = assembler.bytes(); - auto bytes = assembler.spill(); + auto number_of_bytes = assembler.bytes(); + auto bytes = assembler.spill(); - ubyte correct_output[] = { 0x00, 0xC0, 0x66, 0x01, 0xC0, 0x01, 0xC0, 0x48, - 0x01, 0xC0, 0x00, 0xC9, 0x66, 0x01, 0xC9, 0x01, - 0xC9, 0x48, 0x01, 0xC9, 0x00, 0xD2, 0x66, 0x01, - 0xD2, 0x01, 0xD2, 0x48, 0x01, 0xD2, 0x00, 0xDB, - 0x66, 0x01, 0xDB, 0x01, 0xDB, 0x48, 0x01, 0xDB, - 0x40, 0x00, 0xE4, 0x66, 0x01, 0xE4, 0x01, 0xE4, - 0x48, 0x01, 0xE4, 0x40, 0x00, 0xED, 0x66, 0x01, - 0xED, 0x01, 0xED, 0x48, 0x01, 0xED, 0x40, 0x00, - 0xF6, 0x66, 0x01, 0xF6, 0x01, 0xF6, 0x48, 0x01, - 0xF6, 0x40, 0x00, 0xFF, 0x66, 0x01, 0xFF, 0x01, - 0xFF, 0x48, 0x01, 0xFF, 0x45, 0x00, 0xC0, 0x66, - 0x45, 0x01, 0xC0, 0x45, 0x01, 0xC0, 0x4D, 0x01, - 0xC0, 0x45, 0x00, 0xC9, 0x66, 0x45, 0x01, 0xC9, - 0x45, 0x01, 0xC9, 0x4D, 0x01, 0xC9, 0x45, 0x00, - 0xD2, 0x66, 0x45, 0x01, 0xD2, 0x45, 0x01, 0xD2, - 0x4D, 0x01, 0xD2, 0x45, 0x00, 0xDB, 0x66, 0x45, - 0x01, 0xDB, 0x45, 0x01, 0xDB, 0x4D, 0x01, 0xDB, - 0x45, 0x00, 0xE4, 0x66, 0x45, 0x01, 0xE4, 0x45, - 0x01, 0xE4, 0x4D, 0x01, 0xE4, 0x45, 0x00, 0xED, - 0x66, 0x45, 0x01, 0xED, 0x45, 0x01, 0xED, 0x4D, - 0x01, 0xED, 0x45, 0x00, 0xF6, 0x66, 0x45, 0x01, - 0xF6, 0x45, 0x01, 0xF6, 0x4D, 0x01, 0xF6, 0x45, - 0x00, 0xFF, 0x66, 0x45, 0x01, 0xFF, 0x45, 0x01, - 0xFF, 0x4D, 0x01, 0xFF } ; + ubyte correct_output[] = {0x00, 0xC0, 0x66, 0x01, 0xC0, 0x01, 0xC0, 0x48, + 0x01, 0xC0, 0x00, 0xC9, 0x66, 0x01, 0xC9, 0x01, + 0xC9, 0x48, 0x01, 0xC9, 0x00, 0xD2, 0x66, 0x01, + 0xD2, 0x01, 0xD2, 0x48, 0x01, 0xD2, 0x00, 0xDB, + 0x66, 0x01, 0xDB, 0x01, 0xDB, 0x48, 0x01, 0xDB, + 0x40, 0x00, 0xE4, 0x66, 0x01, 0xE4, 0x01, 0xE4, + 0x48, 0x01, 0xE4, 0x40, 0x00, 0xED, 0x66, 0x01, + 0xED, 0x01, 0xED, 0x48, 0x01, 0xED, 0x40, 0x00, + 0xF6, 0x66, 0x01, 0xF6, 0x01, 0xF6, 0x48, 0x01, + 0xF6, 0x40, 0x00, 0xFF, 0x66, 0x01, 0xFF, 0x01, + 0xFF, 0x48, 0x01, 0xFF, 0x45, 0x00, 0xC0, 0x66, + 0x45, 0x01, 0xC0, 0x45, 0x01, 0xC0, 0x4D, 0x01, + 0xC0, 0x45, 0x00, 0xC9, 0x66, 0x45, 0x01, 0xC9, + 0x45, 0x01, 0xC9, 0x4D, 0x01, 0xC9, 0x45, 0x00, + 0xD2, 0x66, 0x45, 0x01, 0xD2, 0x45, 0x01, 0xD2, + 0x4D, 0x01, 0xD2, 0x45, 0x00, 0xDB, 0x66, 0x45, + 0x01, 0xDB, 0x45, 0x01, 0xDB, 0x4D, 0x01, 0xDB, + 0x45, 0x00, 0xE4, 0x66, 0x45, 0x01, 0xE4, 0x45, + 0x01, 0xE4, 0x4D, 0x01, 0xE4, 0x45, 0x00, 0xED, + 0x66, 0x45, 0x01, 0xED, 0x45, 0x01, 0xED, 0x4D, + 0x01, 0xED, 0x45, 0x00, 0xF6, 0x66, 0x45, 0x01, + 0xF6, 0x45, 0x01, 0xF6, 0x4D, 0x01, 0xF6, 0x45, + 0x00, 0xFF, 0x66, 0x45, 0x01, 0xFF, 0x45, 0x01, + 0xFF, 0x4D, 0x01, 0xFF}; - FOR(i, number_of_bytes) { - ASSERT_EQ(bytes[i], correct_output[i]); + FOR(i, number_of_bytes) { + ASSERT_EQ(bytes[i], correct_output[i]); // Console::Log(std::hex, std::setfill('0'), std::setw(2), (ushort)bytes[i], "" // " "); - } + } } TEST(x64Test, Sub) { - using namespace nextgen::jet::x64; - using namespace nextgen; + using namespace nextgen::jet::x64; + using namespace nextgen; - Assembler assembler = Assembler(1000); - for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) +1)) { - assembler.subb((Register)r, (Register)r); - assembler.subw((Register)r, (Register)r); - assembler.subd((Register)r, (Register)r); - assembler.subq((Register)r, (Register)r); - } + Assembler assembler = Assembler(1000); + for (Register r = Register::AX; r <= Register::R15; r = Register(ubyte(r) + 1)) { + assembler.subb((Register) r, (Register) r); + assembler.subw((Register) r, (Register) r); + assembler.subd((Register) r, (Register) r); + assembler.subq((Register) r, (Register) r); + } - auto number_of_bytes = assembler.bytes(); - auto bytes = assembler.spill(); + auto number_of_bytes = assembler.bytes(); + auto bytes = assembler.spill(); - ubyte correct_output[] = { 0x28, 0xC0, 0x66, 0x29, 0xC0, 0x29, 0xC0, 0x48, 0x29, - 0xC0, 0x28, 0xC9, 0x66, 0x29, 0xC9, 0x29, 0xC9, 0x48, - 0x29, 0xC9, 0x28, 0xD2, 0x66, 0x29, 0xD2, 0x29, 0xD2, - 0x48, 0x29, 0xD2, 0x28, 0xDB, 0x66, 0x29, 0xDB, 0x29, - 0xDB, 0x48, 0x29, 0xDB, 0x40, 0x28, 0xE4, 0x66, 0x29, - 0xE4, 0x29, 0xE4, 0x48, 0x29, 0xE4, 0x40, 0x28, 0xED, - 0x66, 0x29, 0xED, 0x29, 0xED, 0x48, 0x29, 0xED, 0x40, - 0x28, 0xF6, 0x66, 0x29, 0xF6, 0x29, 0xF6, 0x48, 0x29, - 0xF6, - 0x40, 0x28, 0xFF, 0x66, 0x29, 0xFF, 0x29, 0xFF, 0x48, - 0x29, 0xFF, 0x45, 0x28, 0xC0, 0x66, 0x45, 0x29, 0xC0, - 0x45, 0x29, 0xC0, 0x4D, 0x29, 0xC0, 0x45, 0x28, 0xC9, - 0x66, 0x45, 0x29, 0xC9, 0x45, 0x29, 0xC9, 0x4D, 0x29, - 0xC9, 0x45, 0x28, 0xD2, 0x66, 0x45, 0x29, 0xD2, 0x45, 0x29, 0xD2, 0x4D, 0x29, 0xD2, 0x45, - 0x28, 0xDB, 0x66, 0x45, 0x29, 0xDB, 0x45, 0x29, - 0xDB, 0x4D, 0x29, 0xDB, 0x45, 0x28, 0xE4, 0x66, 0x45, 0x29, 0xE4, 0x45, 0x29, 0xE4, 0x4D, - 0x29, 0xE4, 0x45, 0x28, 0xED, 0x66, 0x45, 0x29, 0xED, 0x45, 0x29, 0xED, - 0x4D, 0x29, 0xED, 0x45, 0x28, 0xF6, 0x66, 0x45, 0x29, - 0xF6, 0x45, 0x29, 0xF6, 0x4D, 0x29, 0xF6, 0x45, 0x28, - 0xFF, 0x66, 0x45, 0x29, 0xFF, 0x45, 0x29, 0xFF, 0x4D, 0x29, 0xFF }; + ubyte correct_output[] = {0x28, 0xC0, 0x66, 0x29, 0xC0, 0x29, 0xC0, 0x48, 0x29, + 0xC0, 0x28, 0xC9, 0x66, 0x29, 0xC9, 0x29, 0xC9, 0x48, + 0x29, 0xC9, 0x28, 0xD2, 0x66, 0x29, 0xD2, 0x29, 0xD2, + 0x48, 0x29, 0xD2, 0x28, 0xDB, 0x66, 0x29, 0xDB, 0x29, + 0xDB, 0x48, 0x29, 0xDB, 0x40, 0x28, 0xE4, 0x66, 0x29, + 0xE4, 0x29, 0xE4, 0x48, 0x29, 0xE4, 0x40, 0x28, 0xED, + 0x66, 0x29, 0xED, 0x29, 0xED, 0x48, 0x29, 0xED, 0x40, + 0x28, 0xF6, 0x66, 0x29, 0xF6, 0x29, 0xF6, 0x48, 0x29, + 0xF6, + 0x40, 0x28, 0xFF, 0x66, 0x29, 0xFF, 0x29, 0xFF, 0x48, + 0x29, 0xFF, 0x45, 0x28, 0xC0, 0x66, 0x45, 0x29, 0xC0, + 0x45, 0x29, 0xC0, 0x4D, 0x29, 0xC0, 0x45, 0x28, 0xC9, + 0x66, 0x45, 0x29, 0xC9, 0x45, 0x29, 0xC9, 0x4D, 0x29, + 0xC9, 0x45, 0x28, 0xD2, 0x66, 0x45, 0x29, 0xD2, 0x45, 0x29, 0xD2, 0x4D, 0x29, 0xD2, 0x45, + 0x28, 0xDB, 0x66, 0x45, 0x29, 0xDB, 0x45, 0x29, + 0xDB, 0x4D, 0x29, 0xDB, 0x45, 0x28, 0xE4, 0x66, 0x45, 0x29, 0xE4, 0x45, 0x29, 0xE4, 0x4D, + 0x29, 0xE4, 0x45, 0x28, 0xED, 0x66, 0x45, 0x29, 0xED, 0x45, 0x29, 0xED, + 0x4D, 0x29, 0xED, 0x45, 0x28, 0xF6, 0x66, 0x45, 0x29, + 0xF6, 0x45, 0x29, 0xF6, 0x4D, 0x29, 0xF6, 0x45, 0x28, + 0xFF, 0x66, 0x45, 0x29, 0xFF, 0x45, 0x29, 0xFF, 0x4D, 0x29, 0xFF}; - FOR(i, number_of_bytes) { - ASSERT_EQ(bytes[i], correct_output[i]); + FOR(i, number_of_bytes) { + ASSERT_EQ(bytes[i], correct_output[i]); // Console::Log(std::hex, std::setfill('0'), std::setw(2), (ushort)bytes[i], "" // " "); - } + } } TEST_SUITE_MAIN(x64Test) { - TEST_CALL(x64Test, Add); - TEST_CALL(x64Test, Sub); + TEST_CALL(x64Test, Add); + TEST_CALL(x64Test, Sub); } From ee1678d0428cf833a971ce40316958b857f5aba8 Mon Sep 17 00:00:00 2001 From: Chinay-Domitrix Date: Thu, 28 Oct 2021 23:55:01 +0000 Subject: [PATCH 4/4] :art: Clean up Markdown and YAML files --- .github/ISSUE_TEMPLATE/jet----bug-report.md | 11 +- .github/workflows/codeql-analysis.yml | 8 +- .github/workflows/linux.yml | 3 +- .github/workflows/macos.yml | 3 +- .github/workflows/windows.yml | 3 +- CODE_OF_CONDUCT.md | 72 +++----- CONTRIBUTING.md | 128 ++++++-------- LANG.md | 183 +++++++++----------- README.md | 72 +++----- 9 files changed, 205 insertions(+), 278 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/jet----bug-report.md b/.github/ISSUE_TEMPLATE/jet----bug-report.md index 60fd1b9..8cbda3c 100644 --- a/.github/ISSUE_TEMPLATE/jet----bug-report.md +++ b/.github/ISSUE_TEMPLATE/jet----bug-report.md @@ -1,27 +1,26 @@ --- name: Jet ✈ī¸ Bug Report about: Help Report bugs and any other security vulnerabilities. title: '' labels: bug assignees: '' - --- +# Bug Reporting + **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: -1. Find '...' +1. Find '...' 2. Click on '....' 3. Scroll down to '....' 4. Error occurs **Expected behavior** -A clear and concise description of what was supposed to happen. Also, please describe how the unit tests reacted when -you tested for this behavior. +A clear and concise description of what was supposed to happen. Also, please describe how the unit tests reacted when you tested for this behavior. **Screenshots and Videos** -If you can, please add screenshots and videos to help us understand the problem. Doing this is a sure fire way that we -will be able to reproduce the error and likely get the problem solved faster. +If you can, please add screenshots and videos to help us understand the problem. Doing this is a sure fire way that we will be able to reproduce the error and likely get the problem solved faster. **Desktop (please complete the following information):** diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d842af4..132d41d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,12 +13,12 @@ name: "CodeQL" on: push: - branches: [ main ] + branches: [main] pull_request: # The branches below must be a subset of the branches above - branches: [ main ] + branches: [main] schedule: - - cron: '38 18 * * 3' + - cron: "38 18 * * 3" jobs: analyze: @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'cpp' ] + language: ["cpp"] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f5a132c..334445c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,6 +1,6 @@ name: Linux -on: [ push, pull_request ] +on: [push, pull_request] env: BUILD_TYPE: Release @@ -22,4 +22,3 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build run: cd tests && ./tests - diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0a62ff4..36b2dca 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,6 @@ name: MacOS -on: [ push, pull_request ] +on: [push, pull_request] env: BUILD_TYPE: Release @@ -22,4 +22,3 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build run: cd tests && ./tests - diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e7ce79b..b2ea37f 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,6 +1,6 @@ name: Windows -on: [ push, pull_request ] +on: [push, pull_request] env: BUILD_TYPE: Release @@ -22,4 +22,3 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build run: cd tests && cd Release && ./tests - diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 65c8b4d..d041e88 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,96 +2,72 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for -everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity -and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, -or sexual identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall community +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: -* The use of sexual language or imagery, and sexual attention or advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting +- The use of sexual language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take -appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. -Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, -issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for -moderation decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing -the community in public spaces. Examples of representing our community include using an official e-mail address, posting -via an official social media account, or acting as an appointed representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible -for enforcement at nextgenprojectofficial@gmail.com. All complaints will be reviewed and investigated promptly and -fairly. - -All community leaders are obligated to respect the privacy and security of the reporter of any incident. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at nextgenprojectofficial@gmail.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem -in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the -community. +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. -**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation -and an explanation of why the behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. -**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including -unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding -interactions in community spaces as well as external channels like social media. Violating these terms may lead to a -temporary or permanent ban. +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified -period of time. No public or private interaction with the people involved, including unsolicited interaction with those -enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate -behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). @@ -99,5 +75,5 @@ by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/div [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c7d1879..b830eef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,58 +2,45 @@ 👍 Thanks for considering contributing to the project, we appreciate it. 👍 -Outline ------------------ - -* [Code of Conduct](https://github.com/The-NextGen-Project/jet/blob/main/CODE_OF_CONDUCT.md) -* [Spread the Word About Us](#spread-the-word) -* [Add your own Feature](#add-your-own-features) -* [Code Style Guidelines](#style-guidelines) -* [Commit Style Guidelines](#committing-guidelines) -* [Bug Reports](https://github.com/The-NextGen-Project/jet/blob/main/.github/ISSUE_TEMPLATE/jet----bug-report.md) -* [Pull Requests](#pull-requests) -* [Licensing](#licensing) - -Code of Conduct. ----------------- +## Outline + +- [Code of Conduct](https://github.com/The-NextGen-Project/jet/blob/main/CODE_OF_CONDUCT.md) +- [Spread the Word About Us](#spread-the-word) +- [Add your own Feature](#add-your-own-features) +- [Code Style Guidelines](#style-guidelines) +- [Commit Style Guidelines](#committing-guidelines) +- [Bug Reports](https://github.com/The-NextGen-Project/jet/blob/main/.github/ISSUE_TEMPLATE/jet----bug-report.md) +- [Pull Requests](#pull-requests) +- [Licensing](#licensing) + +## Code of Conduct + Before starting the rest of the process, please take time read and understand the contributing [Code of Conduct](https://github.com/The-NextGen-Project/jet/blob/main/CODE_OF_CONDUCT.md). +## Spread the Word -Spread the Word ---------------- +Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on some online forums to encourage a friendly discussion. The language is at its early stages, so we want as much exposure as possible. We currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) where we discuss new changes and possible integrations with new technologies. -Another way to contribute is to write about Jet and the NextGen Project, or speak about Jet on some online forums to -encourage a friendly discussion. The language is at its early stages, so we want as much exposure as possible. We -currently have a channel on the [Programming Language Discord Server](https://discord.gg/tft9Nw5jm6) -where we discuss new changes and possible integrations with new technologies. +## Add your own Features -Add your own Features ---------------------- -Want to add amazing language features yourself? Submit your own language syntax -ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if selected, the syntax/feature will be named after -you! (or an appropriate name of your choosing). +Want to add amazing language features yourself? Submit your own language syntax ideas [here](https://github.com/The-NextGen-Project/jet/issues) and if selected, the syntax/feature will be named after you! (or an appropriate name of your choosing). -> **Note:** Existing Language Features/Syntax suggested will not be named after you. We love suggestions -> of new features of the language, but please be mindful to see if your suggestion has been already been -> accepted or an existing or planned language [feature](https://github.com/The-NextGen-Project/jet/blob/main/LANG.md). +> **Note:** Existing Language Features/Syntax suggested will not be named after you. We love suggestions of new features of the language, but please be mindful to see if your suggestion has been already been accepted or an existing or planned language [feature](https://github.com/The-NextGen-Project/jet/blob/main/LANG.md). + +## Style Guidelines -Style Guidelines ----------------- All programs under the NextGen Project follow the following C++ Style Guidelines. -* Getter and Setter functions are to be named using `camelCase` -* Functions (including in namespaces and in classes) should be denoted with `MyFunction` notation -* Class and Struct member **variables** should be denoted with `MyVariable` -* Don't be *too* generous with your comments. Use them appropriately, especially when describing a complicated task -* Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistent with - config -* Lines should be no longer than **80 characters**. We like to limit this line length for readability -* Mark implicit constructors with `/*implicit*/` indication -* Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format - guidelines with a library -* Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in - that scope, for example, a class with that name requires all its methods and variables to be denoted as `snake_case`. +- Getter and Setter functions are to be named using `camelCase` +- Functions (including in namespaces and in classes) should be denoted with `MyFunction` notation +- Class and Struct member **variables** should be denoted with `MyVariable` +- Don't be _too_ generous with your comments. Use them appropriately, especially when describing a complicated task +- Preprocessors have a `\t` between the '#'. For example: `# include `. This is done to be consistent with config +- Lines should be no longer than **80 characters**. We like to limit this line length for readability +- Mark implicit constructors with `/*implicit*/` indication +- Class and Struct names in-general should be denoted with `MyClass` notation unless we follow specific format guidelines with a library +- Use the `// STD NAMING` convention appropriately. This means that we are to use only `snake_case` when naming items in that scope, for example, a class with that name requires all its methods and variables to be denoted as `snake_case`. When writing nested namespace use the following setup to be consistent with other files: @@ -64,36 +51,33 @@ namespace A { namespace B { namespace C }}} ``` -> **Note:** There are a few exceptions to the rules above: If we create an STL-like object, keep styling -> based on STL styling, and just make sure to use common sense. If something has a very long typename and -> has better readability using trailing return type, use it, **prioritize readability** over anything else. +> **Note:** There are a few exceptions to the rules above: If we create an STL-like object, keep styling based on STL styling, and just make sure to use common sense. If something has a very long typename and has better readability using trailing return type, use it, **prioritize readability** over anything else. + +## Committing Guidelines -Committing Guidelines ---------------------- All commits to the project's repository are follow the following Guidelines. -* Please limit your commit messages to no more than **10** words -* Start all of your commit messages with a capital letter after the designated icon (see below) -* **Don't** end your committing messages with `.` -* Please use the following icons before your commit message to indicate changes made: - * :art: `:art:` when improving the format/structure of the code - * :heavy_plus_sign: `:heavy_plus_sign:` when adding new features to existing code - * :zap: `:zap:` when improving performance - * :book: `:book:` when writing docs - * :penguin: `:penguin:` when updating portability - * :bug: `:bug:` when fixing a bug - * :fire: `:fire:` when source files are moved, changed, or deleted - * :wrench: `:wrench:` when fixing build system (CMake, CI) - * :white_check_mark: `:white_check_mark:` when working on tests - -Pull Requests ---------------- -Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want to -create a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to merge into -the branch you requested for. - - -Licensing ----------- -As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise -specified. +- Please limit your commit messages to no more than **10** words +- Start all of your commit messages with a capital letter after the designated icon (see below) +- **Don't** end your committing messages with `.` +- Please use the following icons before your commit message to indicate changes made: + +| Icon | Markdown Code | When to Use | +| ------------------ | -------------------- | ------------------------------------------------ | +| :art: | `:art:` | when improving the format/structure of the code | +| :heavy_plus_sign: | `:heavy_plus_sign:` | when adding new features to existing code | +| :zap: | `:zap:` | when improving performance | +| :book: | `:book:` | when writing docs | +| :penguin: | `:penguin:` | when updating portability | +| :bug: | `:bug:` | when fixing a bug | +| :fire: | `:fire:` | when source files are moved, changed, or deleted | +| :wrench: | `:wrench:` | when fixing build system (CMake, CI) | +| :white_check_mark: | `:white_check_mark:` | when working on tests | + +## Pull Requests + +Pull Requests are pretty straightforward. You have a change to the existing source code, great :), now you want to create a pull request directly. If workflow succeeds and code is checked by the team, then you are ready to merge into the branch you requested for. + +## Licensing + +As a contributor, any changes you contribute are to be done under the terms of the project's license, unless otherwise specified. diff --git a/LANG.md b/LANG.md index cda12ec..a796d10 100644 --- a/LANG.md +++ b/LANG.md @@ -2,66 +2,80 @@ The language specification is listed below, everything from function variables, and the build systems that are planned to be implemented or are already implemented are below. + > **Note:** To determine whether a feature currently exists check for the keywords: **Implemented**, **WIP**, and **Planned**. +## Outline -Outline ------------------ +- [Primitives](#primitives) +- [Operators](#operators) +- [Modules](#modules) +- [Arrays](#arrays) +- [Variables](#variables) +- [Functions](#functions) +- [Structs](#struct-declaration) +- [Enum](#enum-declaration) +- [Match Statement](#match-statement) -* [Primitives](#primitives) -* [Operators](#operators) -* [Modules](#modules) -* [Arrays](#arrays) -* [Variables](#variables) -* [Functions](#functions) -* [Structs](#struct-declaration) -* [Enum](#enum-declaration) -* [Match Statement](#match-statement) +## Primitives -Primitives --------------- Current Status: **WIP** -| Core Type | Description | | ------------- | ------------- | | `i32` | By default, all integer types are -designated to 32 bits unless specified differently. | | `i64` | 64-bit integer is separate and is using for longer -integer values. | | `u32` | Unsigned 32-bit integer. | | `u64` | Unsigned 64-bit integer. | | `f32` | -32-bit decimal value. | | `f64` | 64-bit decimal value. | | `bool` | Boolean Value: true or false. | -| `fn(Arg Types...) -> Ret` | Function Lambda. | | `char` | A 1 byte character. | | `str` | Array -of characters | | `None` | Optional value for `None` | + +| Core Type | Description | +| ------------------------- | ------------------------------------------------------------------------------------- | +| `i32` | By default, all integer types are designated to 32 bits unless specified differently. | +| `i64` | 64-bit integer is separate and is using for longer integer values. | +| `u32` | Unsigned 32-bit integer. | +| `u64` | Unsigned 64-bit integer. | +| `f32` | 32-bit decimal value. | +| `f64` | 64-bit decimal value. | +| `bool` | Boolean Value: true or false. | +| `fn(Arg Types...) -> Ret` | Function Lambda. | +| `char` | A 1 byte character. | +| `str` | Array of characters | +| `None` | Optional value for `None` | ### Type Annotations -| Annotation | Description | -| ------------- | ------------- | -| `box` | Designates a type as smart pointer value | -| `*` | Type is a pointer. | -| `?` | Optional type. Either None or the value. | -| `&` | allocator reference to the location of an object (or a pass by reference). | +| Annotation | Description | +| ---------- | -------------------------------------------------------------------------- | +| `box` | Designates a type as smart pointer value | +| `*` | Type is a pointer. | +| `?` | Optional type. Either None or the value. | +| `&` | allocator reference to the location of an object (or a pass by reference). | -Operators ------------ -Current Status: **Completed** -| Operator | Description | | ------------- | ------------- | | `+` | Addition. | | `-` | -Subtraction. | | `/` | Division. | | `*` | Multiplication. | | `%` | Modulus. | | `^` -| XOR. | | `&` | Bitwise AND. | | `<<` | Shift Left. | | `>>` | Shift Right. | | `~` -| Bitwise Not and Concat. | | `**` | Exponential Power. | | `!` | Logical Not. | ======= | `??` -| Then operator | +## Operators +Current Status: **Completed** +| Operator | Description | +| -------- | ---------------------- | +| `+` | Addition | +| `-` | Subtraction | +| `/` | Division | +| `*` | Multiplication | +| `%` | Modulus | +| `^` | XOR | +| `&` | Bitwise AND | +| `<<` | Shift Left | +| `>>` | Shift Right | +| `~` | Bitwise Not and Concat | +| `**` | Exponential Power | +| `!` | Logical Not | +| `??` | Then operator | + +## Variables -Variables ---------- Current Status: **Completed** -The programming language is gradually typed. Static typing is only required when the compiler cannot infer the type. -Variables may be declared in two different ways: +The programming language is gradually typed. Static typing is only required when the compiler cannot infer the type. Variables may be declared in two different ways: ```v my_variable := 23; another_variable: bool = false; ``` -All variables are immutable by default (except for lists), and require the `mut` keyword in order to change the way you -interact with that variable. +All variables are immutable by default (except for lists), and require the `mut` keyword in order to change the way you interact with that variable. ```v cant_change_me := 2324; @@ -70,12 +84,10 @@ mut change_me := 3342; change_me = 23424; // Ok ``` -Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, -the specification is standard for all data types. +Custom initialization is specified in with different primitive datatypes (ie: lists and tuples). Other than these two, the specification is standard for all data types. +## Arrays -Arrays --------- Current Status: **WIP** ### Initialization @@ -91,9 +103,7 @@ fixed: [5,i32] = [1, 2, 3, 4, 5]; Current Status: **Planned** -Lists are mutable by default (unlike most of the other data types) because they are most commonly dynamic and require -mutability. Since we like to acknowledge the strengths of some languages and weaknesses of others, one may consider a -similar example of initializing an array using designated initializers in Jet. +Lists are mutable by default (unlike most of the other data types) because they are most commonly dynamic and require mutability. Since we like to acknowledge the strengths of some languages and weaknesses of others, one may consider a similar example of initializing an array using designated initializers in Jet. ```v list: [256,i32] = [ @@ -102,9 +112,7 @@ list: [256,i32] = [ ] ``` -The advantages of such a method reduce excess array initialization code, and allow for better readability to the -programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. We feel -that this as a design principle is good for dealing with code that relies on jump comparisons. +The advantages of such a method reduce excess array initialization code, and allow for better readability to the programmer. This feature is currently not available in Rust or in C++, while only existing as a C99 extension. We feel that this as a design principle is good for dealing with code that relies on jump comparisons. ### Length @@ -120,20 +128,17 @@ Length is checked in a similar global length function that can find the length o Current Status: **Planned** -``` +```zig list[ Initial : End : IndexJump ] ``` -List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the -list. +List slicing works just as expected in Python and has the same exact effect. For example, `list[::-1]` reverses the list. + +## Functions -Functions ------------- Current Status: **WIP** -Functions like anything else are declared explicitly in structs or are declared in the global module scope. If functions -are not exported, they are only viewable to the members in the module that it was declared in. The `export` keyword is -equivalent to the `public` keyword in other languages. +Functions like anything else are declared explicitly in structs or are declared in the global module scope. If functions are not exported, they are only viewable to the members in the module that it was declared in. The `export` keyword is equivalent to the `public` keyword in other languages. ### Function Declaration @@ -145,9 +150,7 @@ my_func => (a: i32, b: i32, c: str) { } ``` -parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would -be written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are -also supported: +parameters are ordered in the format of VariableName -> Typename. A variable of type `str` with a name of `name` would be written as `name str`. For those familiar with Golang, it follows a similar naming convention. Generic functions are also supported: ```zig generics => ($a, $b) { @@ -155,10 +158,7 @@ generics => ($a, $b) { } ``` -Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is -guaranteed to be valid otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and -dynamic typing flexibility for types that can be generalized for similar actions. To make it easier for anonymous trait -building, it is best for the programmer to statically define their variables to make it easier for the compiler. +Generic function parameters have anonymous traits bounded to them which means that their functions and generated code is guaranteed to be valid otherwise a compile-time error will be thrown otherwise. This allows for a mix of static and dynamic typing flexibility for types that can be generalized for similar actions. To make it easier for anonymous trait building, it is best for the programmer to statically define their variables to make it easier for the compiler. ### Default Values @@ -168,16 +168,13 @@ default_values := (a: int = 0xffff, b: str) { } ``` -Default values allow for flexibility in programming, especially when you want to handle a case without passing in a -parameter. Currently, it is not planned to have default values for parameters that don't have their type explicitly -stated. +Default values allow for flexibility in programming, especially when you want to handle a case without passing in a parameter. Currently, it is not planned to have default values for parameters that don't have their type explicitly stated. ### Lambdas Current Status: **Planned** -Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally -declared as a parameters by the following method: +Lambdas are small anonymous functions that can be passed in as callbacks into functions to be called. They are generally declared as a parameters by the following method: ```zig func => (lambda: fn(i32) -> i32) { @@ -185,8 +182,7 @@ func => (lambda: fn(i32) -> i32) { } ``` -You may pass in lambdas through generic parameters as well and call them as long as function definition generation works -as intended. An Example for this function call may be (in reference to the above example): +You may pass in lambdas through generic parameters as well and call them as long as function definition generation works as intended. An Example for this function call may be (in reference to the above example): ```zig func(|x| => { @@ -194,13 +190,11 @@ func(|x| => { }) ``` -Modules --------- +## Modules + Current Status: **WIP** -The way a module system of a programming language dictates its entire package functionality structure. In order to be -able to give the programmer a lot of control we have decided to make it explicit as to functions / types are exported -out in the file's module. +The way a module system of a programming language dictates its entire package functionality structure. In order to be able to give the programmer a lot of control we have decided to make it explicit as to functions / types are exported out in the file's module. ### Imports @@ -211,8 +205,8 @@ import SomeModuleName ``` This is how most modules will be imported directly. -> **Note:** Sub module structure like `std.func` are still in debate as of now, and this will be updated in the future to reflect language -> changes. + +> **Note:** Sub module structure like `std.func` are still in debate as of now, and this will be updated in the future to reflect language changes. ### GitHub Imports @@ -222,22 +216,19 @@ Current Status: **Planned** import 'https://github.com/The-NextGen-Project/jet.git' as SomeModule ``` -This is especially a very interesting part of the programming language. We will have support for open source imports. -These projects will need to be Jet projects that can be recognizable and imported directly into your project. This may -be a good way to consistently ensure that you have the latest stable version of the git repository imported into your -programming language. +This is especially a very interesting part of the programming language. We will have support for open source imports. These projects will need to be Jet projects that can be recognizable and imported directly into your project. This may be a good way to consistently ensure that you have the latest stable version of the git repository imported into your programming language. ### Exporting values Current Status: **Planned** -Functions: +#### Functions ```zig -export my_func => (...) { ... } +export my_func => (...) { ... } ``` -Types: +#### Types ```zig export MyStruct => struct { @@ -245,7 +236,7 @@ export MyStruct => struct { } ``` -Variables: +#### Variables ```js export @@ -254,8 +245,7 @@ MyGlobalVariable = 32; ``` -values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted -by the prefix of the `export` before the declaration. +Values are explicitly exported from the file (each file is a module with exported functions and types). This is denoted by the prefix of the `export` before the declaration. ### Static Compilation (Generally JIT Compiled) @@ -285,8 +275,7 @@ match value { ### Struct Declaration -Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while -allowing for zero cost abstractions to take place. Examples: +Struct declarations will emulate C in which they are handled. The language aims to be as simple as possible while allowing for zero cost abstractions to take place. Examples: ```zig MyStruct => struct { @@ -297,8 +286,9 @@ MyStruct => struct { ### Enum Declaration -Enum Declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as -constant values may be implemented. Regular Enum: +Enum declarations will be the core foundation for applying complex principles in the language. Tagged unions as well as constant values may be implemented. + +#### Regular Enum ```zig Opcodes => enum { @@ -309,7 +299,7 @@ Opcodes => enum { } ``` -Tagged Enums: +#### Tagged Enums ```zig Operation => enum { @@ -325,7 +315,6 @@ Operation => enum { } ``` -End -------- -You made it to the end? Wow, thank you for the interest in reading the language. Before we lose your attention, please -consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). +## End + +You made it to the end? Wow, thank you for the interest in reading the language. Before we lose your attention, please consider [contributing](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). diff --git a/README.md b/README.md index ece2584..e6721aa 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,24 @@ -![Jet](https://github.com/The-NextGen-Project/jet/blob/main/.github/JETV4Large3.png) +![Jet](.github/JETV4Large3.png) [![GitHub license](https://img.shields.io/github/license/The-NextGen-Project/jet?color=%2320ba37&label=License&logo=%2320ba37&logoColor=%2320ba37&style=for-the-badge)](https://github.com/The-NextGen-Project/jet/blob/main/LICENSE) [![Linux](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Linux?label=Linux&style=for-the-badge&logo=github)]() [![Mac](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/MacOS?label=MacOS&style=for-the-badge&logo=github)]() [![Windows](https://img.shields.io/github/workflow/status/The-NextGen-Project/jet/Windows?label=Windows&style=for-the-badge&logo=github)]() -#### Fast Language for Developing Consistent Native, Mobile App, and Web Applications. +#### Fast Language for Developing Consistent Native, Mobile App, and Web Applications > **NOTE:** The language is in its early development stages. Don't expect syntax to be fixed until v1.0. If you want to see the development branch, you can view it [here](https://github.com/The-NextGen-Project/jet/tree/dev). The main branch is the most stable with all tests passing on every platform. -Resources ------------------ +## Resources -* [Download](#download) -* [Introduction To The Language](#a-test-flight) -* [Documentation](https://github.com/The-NextGen-Project/jet/blob/main/LANG.md) -* [Contributing](#contributing) -* [Discord Server](https://discord.gg/8cnuzFxZGx) -* [r/Programming Languages Discord Channel](https://discord.gg/tft9Nw5jm6) -* [License](#license) +- [Download](#download) +- [Introduction To The Language](#a-test-flight) +- [Documentation](https://github.com/The-NextGen-Project/jet/blob/main/LANG.md) +- [Contributing](#contributing) +- [Discord Server](https://discord.gg/8cnuzFxZGx) +- [r/Programming Languages Discord Channel](https://discord.gg/tft9Nw5jm6) +- [License](#license) -Download ---------- +## Download ``` git clone https://github.com/The-NextGen-Project/jet.git @@ -33,15 +31,15 @@ make > **Note:** For building the project on Windows, it can be done by simply building with MSVC or, if you have MinGW installed it will work with that C compiler as well. For setting up a build system that is fast, I recommend using [Ninja](https://ninja-build.org/) which works on Windows, Linux, and Mac as a CMake generator. -A Test Flight -------------- -Offering a little taste of the lang, here is some syntax. A hello world program is as follows: +## A Test Flight + +Offering a little taste of Jet, here is some syntax. A hello world program is as follows: ```zig print("Hello, World!") ``` -Fibonacci Implemented in the lang would be as follows: +Fibonacci implemented in Jet would be as follows: ```zig fib => (n: i32) i32 { @@ -53,35 +51,23 @@ fib => (n: i32) i32 { } ``` -The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is -by default, statically-typed, however, types are usually inferred by the language most of the time, and generate clean -and easy to read code. This way we get to appeal to both those who like static typing and those that prefer mainly -dynamic typing. +The language will be using advanced type-inference in order to limit the amount of static typing that is needed. Jet is by default, statically-typed, however, types are usually inferred by the language most of the time, and generate clean and easy to read code. This way we get to appeal to both those who like static typing and those that prefer mainly dynamic typing. ### Error Messages -Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why -their code is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer -overflow**: +Jet has **amazing** error messages. Even for simple error, Jet makes sure that the programmer knows the error and why their code is causing the error is invalid. For example, look at a lexer error message pertaining to an **integer overflow**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/LexErrorExample.png) -The error message points out the initial error, points out the offending line and character, gives a potential solution, -and explains why what the programmer wrote was invalid. These components identify the problem and give opportunities for -a solution to help the programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a -hybrid error message type that encapsulates all that the programmer needs to identify the problem and solve it. +The error message points out the initial error, points out the offending line and character, gives a potential solution, and explains why what the programmer wrote was invalid. These components identify the problem and give opportunities for a solution to help the programmer debug and fix their error. We took inspiration from both Rust and Elm to produce a hybrid error message type that encapsulates all that the programmer needs to identify the problem and solve it. Let's have a look at another example, for an **unexpected token in the global scope**: ![](https://github.com/The-NextGen-Project/jet/blob/main/.github/ParseErrorExample1.png) -People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help -provide clarity and provide potential solutions to a problem they may not understand. By telling the programmer what is -directly valid, it prevents programmers to lookup documentation unless it is necessary to explain an error. - +People, especially beginners, do not understand where and why they are getting errors. Written errors like this one help provide clarity and provide potential solutions to a problem they may not understand. By telling the programmer what is directly valid, it prevents programmers to lookup documentation unless it is necessary to explain an error. -Progress ----------- +## Progress - [x] Lexer - [x] Parser @@ -93,14 +79,10 @@ Progress - [ ] ARM64 Code Generation (**WIP**) - [ ] Link -Contributing -------------- -We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a -community effort, and we would love to have many of you who are willing to help. Before your start your wonderful -contribution, have a look at our -[Contributing Guidelines](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). - -License ----------- -We want the project to be able to integrate with other with a lot of other open source software and determined that we -will be using the **MIT License**. +## Contributing + +We are very excited that you have considered contributing to Jet as part of the NextGen Project. This project is a community effort, and we would love to have many of you who are willing to help. Before your start your wonderful contribution, have a look at our [Contributing Guidelines](https://github.com/The-NextGen-Project/jet/blob/main/CONTRIBUTING.md). + +## License + +We want the project to be able to integrate with other with a lot of other open source software and determined that we will be using the **MIT License**.