90 lines
2.4 KiB
Nim
90 lines
2.4 KiB
Nim
# Copyright 2022 Mattia Giambirtone & All Contributors
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import std/strformat
|
|
import std/strutils
|
|
|
|
|
|
type
|
|
|
|
TokenType* {.pure.} = enum
|
|
## Token types enumeration
|
|
|
|
# Booleans
|
|
True, False,
|
|
|
|
# Other singleton types
|
|
Infinity, NotANumber, Nil
|
|
|
|
# Control flow statements
|
|
If, Else,
|
|
|
|
# Looping statements
|
|
While, For,
|
|
|
|
# Keywords
|
|
Function, Break, Continue,
|
|
Var, Let, Const, Return,
|
|
Coroutine, Generator, Import,
|
|
Raise, Assert, Await, Foreach,
|
|
Yield, Defer, Try, Except,
|
|
Finally, Type, Operator, Case,
|
|
Enum, From, Ptr, Ref, Object
|
|
|
|
# Literal types
|
|
Integer, Float, String, Identifier,
|
|
Binary, Octal, Hex, Char
|
|
|
|
# Brackets, parentheses,
|
|
# operators and others
|
|
|
|
LeftParen, RightParen, # ()
|
|
LeftBrace, RightBrace, # {}
|
|
LeftBracket, RightBracket, # []
|
|
Dot, Semicolon, Comma, # . ; ,
|
|
|
|
# Miscellaneous
|
|
|
|
EndOfFile, # Marks the end of the token stream
|
|
NoMatch, # Used internally by the symbol table
|
|
Comment, # Useful for documentation comments, pragmas, etc.
|
|
Symbol, # A generic symbol
|
|
Pragma,
|
|
|
|
|
|
|
|
|
|
Token* = ref object
|
|
## A token object
|
|
kind*: TokenType # Type of the token
|
|
lexeme*: string # The lexeme associated to the token
|
|
line*: int # The line where the token appears
|
|
pos*: tuple[start, stop: int] # The absolute position in the source file
|
|
# (0-indexed and inclusive at the beginning)
|
|
spaces*: int # Number of spaces before this token
|
|
|
|
|
|
|
|
proc `$`*(self: Token): string =
|
|
## Strinfifies
|
|
if self != nil:
|
|
result = &"Token(kind={self.kind}, lexeme={self.lexeme.escape()}, line={self.line}, pos=({self.pos.start}, {self.pos.stop}), spaces={self.spaces})"
|
|
else:
|
|
result = "nil"
|
|
|
|
|
|
proc `==`*(self, other: Token): bool =
|
|
## Returns self == other
|
|
return self.kind == other.kind and self.lexeme == other.lexeme
|