Added support for FloorDiv (//) and removed redundant variable from Lexer.next()
This commit is contained in:
parent
eb538f9d4b
commit
099109daa6
|
@ -49,6 +49,7 @@ const double = to_table({"**": TokenType.DoubleAsterisk,
|
||||||
"!=": TokenType.NotEqual,
|
"!=": TokenType.NotEqual,
|
||||||
">=": TokenType.GreaterOrEqual,
|
">=": TokenType.GreaterOrEqual,
|
||||||
"<=": TokenType.LessOrEqual,
|
"<=": TokenType.LessOrEqual,
|
||||||
|
"//": TokenType.FloorDiv
|
||||||
})
|
})
|
||||||
|
|
||||||
# Constant table storing all the reserved keywords (parsed as identifiers)
|
# Constant table storing all the reserved keywords (parsed as identifiers)
|
||||||
|
@ -212,14 +213,14 @@ func createToken(self: Lexer, tokenType: TokenType) =
|
||||||
func parseString(self: Lexer, delimiter: char, mode: string = "single") =
|
func parseString(self: Lexer, delimiter: char, mode: string = "single") =
|
||||||
## Parses string literals
|
## Parses string literals
|
||||||
while not self.check(delimiter) and not self.done():
|
while not self.check(delimiter) and not self.done():
|
||||||
if self.match('\n') and mode == "multi":
|
if self.check('\n') and mode == "multi":
|
||||||
self.line = self.line + 1
|
self.line = self.line + 1
|
||||||
else:
|
else:
|
||||||
self.error("Unexpected EOL while parsing string literal")
|
self.error("Unexpected EOL while parsing string literal")
|
||||||
return
|
return
|
||||||
if mode in ["raw", "multi"]:
|
if mode in ["raw", "multi"]:
|
||||||
discard self.step()
|
discard self.step()
|
||||||
elif self.match('\\'):
|
elif self.check('\\'):
|
||||||
# Escape sequences.
|
# Escape sequences.
|
||||||
# We currently support only the basic
|
# We currently support only the basic
|
||||||
# ones, so stuff line \nnn, \xhhh, \uhhhh and
|
# ones, so stuff line \nnn, \xhhh, \uhhhh and
|
||||||
|
@ -296,7 +297,6 @@ func next(self: Lexer) =
|
||||||
if self.done():
|
if self.done():
|
||||||
return
|
return
|
||||||
var single = self.step()
|
var single = self.step()
|
||||||
var multi = false
|
|
||||||
if single in [' ', '\t', '\r', '\f',
|
if single in [' ', '\t', '\r', '\f',
|
||||||
'\e']: # We skip whitespaces, tabs and other useless characters
|
'\e']: # We skip whitespaces, tabs and other useless characters
|
||||||
return
|
return
|
||||||
|
@ -328,14 +328,14 @@ func next(self: Lexer) =
|
||||||
while not self.check('\n'):
|
while not self.check('\n'):
|
||||||
discard self.step()
|
discard self.step()
|
||||||
return
|
return
|
||||||
|
# We start by checking for multi-character tokens
|
||||||
for key in double.keys():
|
for key in double.keys():
|
||||||
if key[0] == single and key[1] == self.peek():
|
if key[0] == single and key[1] == self.peek():
|
||||||
discard self.step()
|
discard self.step()
|
||||||
multi = true
|
|
||||||
self.createToken(double[key])
|
self.createToken(double[key])
|
||||||
return
|
return
|
||||||
if not multi:
|
# Eventually we emit a single token
|
||||||
self.createToken(tokens[single])
|
self.createToken(tokens[single])
|
||||||
else:
|
else:
|
||||||
self.error(&"Unexpected token '{single}'")
|
self.error(&"Unexpected token '{single}'")
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue