diff --git a/src/backend/lexer.nim b/src/backend/lexer.nim index a45e2ed..974067a 100644 --- a/src/backend/lexer.nim +++ b/src/backend/lexer.nim @@ -80,6 +80,7 @@ const reserved = to_table({ "lambda": TokenType.Lambda, "class": TokenType.Class, "async": TokenType.Async, "import": TokenType.Import, "isnot": TokenType.IsNot, "from": TokenType.From, + "let": TokenType.Let, "const": TokenType.Const }) type @@ -95,22 +96,20 @@ type errorMessage*: string -func newLexer*(self: Lexer = nil): Lexer = +func initLexer*(self: Lexer = nil): Lexer = ## Initializes the lexer or resets ## the state of an existing one - if self == nil: - result = Lexer(source: "", tokens: @[], line: 1, start: 0, current: 0, - errored: false, file: "", errorMessage: "") - else: - self.source = "" - self.tokens = @[] - self.line = 1 - self.start = 0 - self.current = 0 - self.errored = false - self.file = "" - self.errorMessage = "" + new(result) + if self != nil: result = self + result.source = "" + result.tokens = @[] + result.line = 1 + result.start = 0 + result.current = 0 + result.errored = false + result.file = "" + result.errorMessage = "" func done(self: Lexer): bool = @@ -150,6 +149,8 @@ func error(self: Lexer, message: string) = ## for the lexer. The lex method will not ## continue tokenizing if it finds out ## an error occurred + if self.errored: + return self.errored = true self.errorMessage = &"A fatal error occurred while parsing '{self.file}', line {self.line} at '{self.peek()}' -> {message}" @@ -181,7 +182,6 @@ func check(self: Lexer, what: string): bool = return true - func check(self: Lexer, what: openarray[char]): bool = ## Calls self.check() in a loop with ## each character from the given seq of @@ -306,7 +306,7 @@ func parseNumber(self: Lexer) = self.createToken(kind) -proc parseIdentifier(self: Lexer) = +func parseIdentifier(self: Lexer) = ## Parses identifiers. Note that ## multi-character tokens such as ## UTF runes are not supported @@ -321,7 +321,7 @@ proc parseIdentifier(self: Lexer) = self.createToken(TokenType.Identifier) -proc next(self: Lexer) = +func next(self: Lexer) = ## Scans a single token. This method is ## called iteratively until the source ## file reaches EOF @@ -380,13 +380,13 @@ proc next(self: Lexer) = self.error(&"Unexpected token '{single}'") -proc lex*(self: Lexer, source, file: string): seq[Token] = +func lex*(self: Lexer, source, file: string): seq[Token] = ## Lexes a source file, converting a stream ## of characters into a series of tokens. ## If an error occurs, this procedure ## returns an empty sequence and the lexer's ## errored and errorMessage fields will be set - discard self.newLexer() + discard self.initLexer() self.source = source self.file = file while not self.done(): @@ -396,4 +396,4 @@ proc lex*(self: Lexer, source, file: string): seq[Token] = return @[] self.tokens.add(Token(kind: TokenType.EndOfFile, lexeme: "EOF", line: self.line)) - return self.tokens + return self.tokens \ No newline at end of file diff --git a/src/backend/meta/token.nim b/src/backend/meta/token.nim index 85df85b..7f9fcdc 100644 --- a/src/backend/meta/token.nim +++ b/src/backend/meta/token.nim @@ -56,11 +56,11 @@ type LessOrEqual, GreaterOrEqual, # >= <= ExclamationMark, DoubleEqual, # ! == NotEqual, RightShift, LeftShift, # != >> << - LogicalAnd, LogicalOr, FloorDiv, # && || // - InplaceAdd, InplaceSub, InplaceDiv, # += -= /= - InplaceMod, InplaceMul, InplaceXor, # %= *= ^= - InplaceAnd, InplaceOr, InplaceNot, # &= |= ~= - InplaceFloorDiv, InplacePow # //= **= + LogicalAnd, LogicalOr, FloorDiv, # && || // + InplaceAdd, InplaceSub, InplaceDiv, # += -= /= + InplaceMod, InplaceMul, InplaceXor, # %= *= ^= + InplaceAnd, InplaceOr, InplaceNot, # &= |= ~= + InplaceFloorDiv, InplacePow # //= **= # Miscellaneous