nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

derete the fil #2

Closed
N00nehere wants to merge 49 commits from (deleted):n00nehere-patch-1 into master
pull from: (deleted):n00nehere-patch-1
merge into: nocturn9x:master
Conversation 0 Commits 49 Files Changed 18 +56 -43
3 changed files with 56 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ce82517f14 - Show all commits

16
src/frontend/lexer.nim
View File

@ -400,10 +400,6 @@ proc parseBinary(self: Lexer) =
if not self.check(["0", "1"]): if not self.check(["0", "1"]):
self.error(&"invalid digit '{self.peek()}' in binary literal") self.error(&"invalid digit '{self.peek()}' in binary literal")
discard self.step() discard self.step()
self.createToken(Binary)
# To make our life easier, we pad the binary number in here already
while (self.tokens[^1].lexeme.len() - 2) mod 8 != 0:
self.tokens[^1].lexeme = "0b" & "0" & self.tokens[^1].lexeme[2..^1]
proc parseOctal(self: Lexer) = proc parseOctal(self: Lexer) =
@ -412,7 +408,6 @@ proc parseOctal(self: Lexer) =
if self.peek() notin "0".."7": if self.peek() notin "0".."7":
self.error(&"invalid digit '{self.peek()}' in octal literal") self.error(&"invalid digit '{self.peek()}' in octal literal")
discard self.step() discard self.step()
self.createToken(Octal)
proc parseHex(self: Lexer) = proc parseHex(self: Lexer) =
@ -421,7 +416,6 @@ proc parseHex(self: Lexer) =
if not self.peek().isDigit() and self.peek().toLowerAscii() notin "a".."f": if not self.peek().isDigit() and self.peek().toLowerAscii() notin "a".."f":
self.error(&"invalid hexadecimal literal") self.error(&"invalid hexadecimal literal")
discard self.step() discard self.step()
self.createToken(Hex)
proc parseNumber(self: Lexer) = proc parseNumber(self: Lexer) =
@ -442,11 +436,14 @@ proc parseNumber(self: Lexer) =
case self.peek(): case self.peek():
of "b": of "b":
discard self.step() discard self.step()
kind = Binary
self.parseBinary() self.parseBinary()
of "x": of "x":
kind = Hex
discard self.step() discard self.step()
self.parseHex() self.parseHex()
of "o": of "o":
kind = Octal
discard self.step() discard self.step()
self.parseOctal() self.parseOctal()
else: else:
@ -475,6 +472,10 @@ proc parseNumber(self: Lexer) =
while (self.peek().isAlphaNumeric() or self.check("_")) and not self.done(): while (self.peek().isAlphaNumeric() or self.check("_")) and not self.done():
discard self.step() discard self.step()
self.createToken(kind) self.createToken(kind)
if kind == Binary:
# To make our life easier, we pad the binary number in here already
while (self.tokens[^1].lexeme.len() - 2) mod 8 != 0:
self.tokens[^1].lexeme = "0b" & "0" & self.tokens[^1].lexeme[2..^1]
@ -520,7 +521,8 @@ proc next(self: Lexer) =
mode = "multi" mode = "multi"
self.parseString(self.peek(-1), mode) self.parseString(self.peek(-1), mode)
elif self.peek().isDigit(): elif self.peek().isDigit():
discard self.step() discard self.step() # Needed because parseNumber reads the next
# character to tell the base of the number
# Number literal # Number literal
self.parseNumber() self.parseNumber()
elif self.peek().isAlphaNumeric() and self.check(["\"", "'"], 1): elif self.peek().isAlphaNumeric() and self.check(["\"", "'"], 1):

5
src/frontend/meta/token.nim
View File

@ -39,7 +39,7 @@ type
IsNot, Raise, Assert, Await, IsNot, Raise, Assert, Await,
Foreach, Yield, Public, As, Foreach, Yield, Public, As,
Of, Defer, Try, Except, Finally, Of, Defer, Try, Except, Finally,
Type, Operator, Case, Enum Type, Operator, Case, Enum, From,
# Literal types # Literal types
Integer, Float, String, Identifier, Integer, Float, String, Identifier,
@ -76,6 +76,9 @@ type
# stuff I haven't thought about yet # stuff I haven't thought about yet
Whitespace, Whitespace,
Tab, Tab,
UnaryOperator, # Arbitrary user-defined unary operator
BinaryOperator # Arbitrary user-defined binary operator
Token* = ref object Token* = ref object

78
src/frontend/parser.nim
View File

@ -187,7 +187,7 @@ proc expression(self: Parser): ASTNode
proc expressionStatement(self: Parser): ASTNode proc expressionStatement(self: Parser): ASTNode
proc statement(self: Parser): ASTNode proc statement(self: Parser): ASTNode
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode
proc declaration(self: Parser): ASTNode proc declaration(self: Parser): ASTNode
@ -349,9 +349,8 @@ proc makeCall(self: Parser, callee: ASTNode): ASTNode =
proc call(self: Parser): ASTNode = proc call(self: Parser): ASTNode =
## Parses calls, object field ## Parses func5ion calls, object field
## field accessing and slicing ## accessing and slicing expressions
## expressions
result = self.primary() result = self.primary()
while true: while true:
if self.match(LeftParen): if self.match(LeftParen):
@ -360,13 +359,17 @@ proc call(self: Parser): ASTNode =
self.expect(Identifier, "expecting attribute name after '.'") self.expect(Identifier, "expecting attribute name after '.'")
result = newGetItemExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1)) result = newGetItemExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1))
elif self.match(LeftBracket): elif self.match(LeftBracket):
# Slicing such as a[1:2]
let tok = self.peek(-1) let tok = self.peek(-1)
var ends: seq[ASTNode] = @[] var ends: seq[ASTNode] = @[]
while not self.match(RightBracket) and ends.len() < 3: while not self.check(RightBracket) and not self.done():
ends.add(self.expression()) if self.check(Colon):
discard self.match(Colon) ends.add(newNilExpr(Token()))
if ends.len() < 1: discard self.step()
self.error("invalid syntax") else:
ends.add(self.expression())
discard self.match(Colon)
self.expect(RightBracket, "expecting ']'")
result = newSliceExpr(result, ends, tok) result = newSliceExpr(result, ends, tok)
else: else:
break break
@ -501,7 +504,7 @@ proc assignment(self: Parser): ASTNode =
proc assertStmt(self: Parser): ASTNode = proc assertStmt(self: Parser): ASTNode =
## Parses "assert" statements, ## Parses "assert" statements, which
## raise an error if the expression ## raise an error if the expression
## fed into them is falsey ## fed into them is falsey
let tok = self.peek(-1) let tok = self.peek(-1)
@ -511,12 +514,12 @@ proc assertStmt(self: Parser): ASTNode =
proc beginScope(self: Parser) = proc beginScope(self: Parser) =
## Begins a new syntactical scope ## Begins a new lexical scope
inc(self.scopeDepth) inc(self.scopeDepth)
proc endScope(self: Parser) = proc endScope(self: Parser) =
## Ends a new syntactical scope ## Ends a new lexical scope
dec(self.scopeDepth) dec(self.scopeDepth)
@ -524,13 +527,12 @@ proc blockStmt(self: Parser): ASTNode =
## Parses block statements. A block ## Parses block statements. A block
## statement simply opens a new local ## statement simply opens a new local
## scope ## scope
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
var code: seq[ASTNode] = @[] var code: seq[ASTNode] = @[]
while not self.check(RightBrace) and not self.done(): while not self.check(RightBrace) and not self.done():
code.add(self.declaration()) code.add(self.declaration())
self.expect(RightBrace, "unterminated block statement") self.expect(RightBrace, "expecting '}'")
result = newBlockStmt(code, tok) result = newBlockStmt(code, tok)
self.endScope() self.endScope()
@ -579,14 +581,12 @@ proc returnStmt(self: Parser): ASTNode =
proc yieldStmt(self: Parser): ASTNode = proc yieldStmt(self: Parser): ASTNode =
## Parses yield Statements ## Parses yield statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
self.error("'yield' cannot be outside functions") self.error("'yield' cannot be outside functions")
if self.currentFunction.kind == NodeKind.funDecl: elif self.currentFunction.token.kind != Generator:
FunDecl(self.currentFunction).isGenerator = true self.error("'yield' can only be used inside generators")
else:
LambdaExpr(self.currentFunction).isGenerator = true
if not self.check(Semicolon): if not self.check(Semicolon):
result = newYieldStmt(self.expression(), tok) result = newYieldStmt(self.expression(), tok)
else: else:
@ -595,14 +595,14 @@ proc yieldStmt(self: Parser): ASTNode =
proc awaitStmt(self: Parser): ASTNode = proc awaitStmt(self: Parser): ASTNode =
## Parses yield Statements ## Parses await statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
self.error("'await' cannot be used outside functions") self.error("'await' cannot be used outside functions")
if self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isAsync: if self.currentFunction.token.kind != Coroutine:
self.error("'await' can only be used inside async functions") self.error("'await' can only be used inside coroutines")
result = newAwaitStmt(self.expression(), tok) result = newAwaitStmt(self.expression(), tok)
endOfLine("missing semicolon after yield statement") endOfLine("missing semicolon after await statement")
proc raiseStmt(self: Parser): ASTNode = proc raiseStmt(self: Parser): ASTNode =
@ -633,17 +633,22 @@ proc forEachStmt(self: Parser): ASTNode =
self.currentLoop = enclosingLoop self.currentLoop = enclosingLoop
proc importStmt(self: Parser): ASTNode = proc importStmt(self: Parser, fromStmt: bool = false): ASTNode =
## Parses import statements ## Parses import statements
let tok = self.peek(-1) var tok: Token
if fromStmt:
tok = self.peek(-2)
else:
tok = self.peek(-1)
# TODO: New AST node
self.expect(Identifier, "expecting module name(s) after import statement") self.expect(Identifier, "expecting module name(s) after import statement")
result = newImportStmt(self.expression(), tok) result = newImportStmt(newIdentExpr(self.peek(-1)), tok)
endOfLine("missing semicolon after import statement") endOfLine("missing semicolon after import statement")
proc tryStmt(self: Parser): ASTNode = proc tryStmt(self: Parser): ASTNode =
## Parses try/except/finally/else blocks ## Parses try/except/else/finally blocks
let tok = self.peek(-1) let tok = self.peek(-1)
var body = self.statement() var body = self.statement()
var handlers: seq[tuple[body, exc, name: ASTNode]] = @[] var handlers: seq[tuple[body, exc, name: ASTNode]] = @[]
@ -655,7 +660,9 @@ proc tryStmt(self: Parser): ASTNode =
while self.match(Except): while self.match(Except):
excName = self.expression() excName = self.expression()
if excName.kind == identExpr: if excName.kind == identExpr:
continue handlerBody = self.statement()
handlers.add((body: handlerBody, exc: excName, name: asName))
asName = nil
elif excName.kind == binaryExpr and BinaryExpr(excName).operator.kind == As: elif excName.kind == binaryExpr and BinaryExpr(excName).operator.kind == As:
asName = BinaryExpr(excName).b asName = BinaryExpr(excName).b
if BinaryExpr(excName).b.kind != identExpr: if BinaryExpr(excName).b.kind != identExpr:
@ -674,15 +681,12 @@ proc tryStmt(self: Parser): ASTNode =
continue continue
else: else:
excName = nil excName = nil
handlerBody = self.statement()
handlers.add((body: handlerBody, exc: excName, name: asName))
asName = nil
if self.match(Else): if self.match(Else):
elseClause = self.statement() elseClause = self.statement()
if self.match(Finally): if self.match(Finally):
finallyClause = self.statement() finallyClause = self.statement()
if handlers.len() == 0 and elseClause == nil and finallyClause == nil: if handlers.len() == 0 and elseClause == nil and finallyClause == nil:
self.error("expecting 'except', 'finally' or 'else' statements after 'try' block") self.error("expecting 'except', 'finally' or 'else' statement after 'try' block")
for i, handler in handlers: for i, handler in handlers:
if handler.exc == nil and i != handlers.high(): if handler.exc == nil and i != handlers.high():
self.error("catch-all exception handler with bare 'except' must come last in try statement") self.error("catch-all exception handler with bare 'except' must come last in try statement")
@ -822,9 +826,8 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
discard # Unreachable discard # Unreachable
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode = proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode =
## Parses functions, coroutines, generators and anonymous functions declarations (the latter is ## Parses functions, coroutines, generators, anonymous functions and custom operators
## technically an expression, not a declaration)
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingFunction = self.currentFunction var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[] var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[]
@ -941,6 +944,11 @@ proc statement(self: Parser): ASTNode =
of Import: of Import:
discard self.step() discard self.step()
result = self.importStmt() result = self.importStmt()
of From:
# TODO
# from module import a [, b, c as d]
discard self.step()
result = self.importStmt(fromStmt=true)
of While: of While:
discard self.step() discard self.step()
result = self.whileStmt() result = self.whileStmt()