nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

also derete this #3

Closed
N00nehere wants to merge 49 commits from (deleted):n00nehere-patch-2 into master
pull from: (deleted):n00nehere-patch-2
merge into: nocturn9x:master
Conversation 0 Commits 49 Files Changed 18 +56 -43
3 changed files with 56 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ce82517f14 - Show all commits

16
src/frontend/lexer.nim
View File

@ -400,10 +400,6 @@ proc parseBinary(self: Lexer) =
if not self.check(["0", "1"]):
self.error(&"invalid digit '{self.peek()}' in binary literal")
discard self.step()
self.createToken(Binary)
# To make our life easier, we pad the binary number in here already
while (self.tokens[^1].lexeme.len() - 2) mod 8 != 0:
self.tokens[^1].lexeme = "0b" & "0" & self.tokens[^1].lexeme[2..^1]
proc parseOctal(self: Lexer) =
@ -412,7 +408,6 @@ proc parseOctal(self: Lexer) =
if self.peek() notin "0".."7":
self.error(&"invalid digit '{self.peek()}' in octal literal")
discard self.step()
self.createToken(Octal)
proc parseHex(self: Lexer) =
@ -421,7 +416,6 @@ proc parseHex(self: Lexer) =
if not self.peek().isDigit() and self.peek().toLowerAscii() notin "a".."f":
self.error(&"invalid hexadecimal literal")
discard self.step()
self.createToken(Hex)
proc parseNumber(self: Lexer) =
@ -442,11 +436,14 @@ proc parseNumber(self: Lexer) =
case self.peek():
of "b":
discard self.step()
kind = Binary
self.parseBinary()
of "x":
kind = Hex
discard self.step()
self.parseHex()
of "o":
kind = Octal
discard self.step()
self.parseOctal()
else:
@ -475,6 +472,10 @@ proc parseNumber(self: Lexer) =
while (self.peek().isAlphaNumeric() or self.check("_")) and not self.done():
discard self.step()
self.createToken(kind)
if kind == Binary:
# To make our life easier, we pad the binary number in here already
while (self.tokens[^1].lexeme.len() - 2) mod 8 != 0:
self.tokens[^1].lexeme = "0b" & "0" & self.tokens[^1].lexeme[2..^1]
@ -520,7 +521,8 @@ proc next(self: Lexer) =
mode = "multi"
self.parseString(self.peek(-1), mode)
elif self.peek().isDigit():
discard self.step()
discard self.step() # Needed because parseNumber reads the next
# character to tell the base of the number
# Number literal
self.parseNumber()
elif self.peek().isAlphaNumeric() and self.check(["\"", "'"], 1):

5
src/frontend/meta/token.nim
View File

@ -39,7 +39,7 @@ type
IsNot, Raise, Assert, Await,
Foreach, Yield, Public, As,
Of, Defer, Try, Except, Finally,
Type, Operator, Case, Enum
Type, Operator, Case, Enum, From,
# Literal types
Integer, Float, String, Identifier,
@ -76,6 +76,9 @@ type
# stuff I haven't thought about yet
Whitespace,
Tab,
UnaryOperator, # Arbitrary user-defined unary operator
BinaryOperator # Arbitrary user-defined binary operator
Token* = ref object

78
src/frontend/parser.nim
View File

@ -187,7 +187,7 @@ proc expression(self: Parser): ASTNode
proc expressionStatement(self: Parser): ASTNode
proc statement(self: Parser): ASTNode
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode
proc declaration(self: Parser): ASTNode
@ -349,9 +349,8 @@ proc makeCall(self: Parser, callee: ASTNode): ASTNode =
proc call(self: Parser): ASTNode =
## Parses calls, object field
## field accessing and slicing
## expressions
## Parses func5ion calls, object field
## accessing and slicing expressions
result = self.primary()
while true:
if self.match(LeftParen):
@ -360,13 +359,17 @@ proc call(self: Parser): ASTNode =
self.expect(Identifier, "expecting attribute name after '.'")
result = newGetItemExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1))
elif self.match(LeftBracket):
# Slicing such as a[1:2]
let tok = self.peek(-1)
var ends: seq[ASTNode] = @[]
while not self.match(RightBracket) and ends.len() < 3:
ends.add(self.expression())
discard self.match(Colon)
if ends.len() < 1:
self.error("invalid syntax")
while not self.check(RightBracket) and not self.done():
if self.check(Colon):
ends.add(newNilExpr(Token()))
discard self.step()
else:
ends.add(self.expression())
discard self.match(Colon)
self.expect(RightBracket, "expecting ']'")
result = newSliceExpr(result, ends, tok)
else:
break
@ -501,7 +504,7 @@ proc assignment(self: Parser): ASTNode =
proc assertStmt(self: Parser): ASTNode =
## Parses "assert" statements,
## Parses "assert" statements, which
## raise an error if the expression
## fed into them is falsey
let tok = self.peek(-1)
@ -511,12 +514,12 @@ proc assertStmt(self: Parser): ASTNode =
proc beginScope(self: Parser) =
## Begins a new syntactical scope
## Begins a new lexical scope
inc(self.scopeDepth)
proc endScope(self: Parser) =
## Ends a new syntactical scope
## Ends a new lexical scope
dec(self.scopeDepth)
@ -524,13 +527,12 @@ proc blockStmt(self: Parser): ASTNode =
## Parses block statements. A block
## statement simply opens a new local
## scope
self.beginScope()
let tok = self.peek(-1)
var code: seq[ASTNode] = @[]
while not self.check(RightBrace) and not self.done():
code.add(self.declaration())
self.expect(RightBrace, "unterminated block statement")
self.expect(RightBrace, "expecting '}'")
result = newBlockStmt(code, tok)
self.endScope()
@ -579,14 +581,12 @@ proc returnStmt(self: Parser): ASTNode =
proc yieldStmt(self: Parser): ASTNode =
## Parses yield Statements
## Parses yield statements
let tok = self.peek(-1)
if self.currentFunction == nil:
self.error("'yield' cannot be outside functions")
if self.currentFunction.kind == NodeKind.funDecl:
FunDecl(self.currentFunction).isGenerator = true
else:
LambdaExpr(self.currentFunction).isGenerator = true
elif self.currentFunction.token.kind != Generator:
self.error("'yield' can only be used inside generators")
if not self.check(Semicolon):
result = newYieldStmt(self.expression(), tok)
else:
@ -595,14 +595,14 @@ proc yieldStmt(self: Parser): ASTNode =
proc awaitStmt(self: Parser): ASTNode =
## Parses yield Statements
## Parses await statements
let tok = self.peek(-1)
if self.currentFunction == nil:
self.error("'await' cannot be used outside functions")
if self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isAsync:
self.error("'await' can only be used inside async functions")
if self.currentFunction.token.kind != Coroutine:
self.error("'await' can only be used inside coroutines")
result = newAwaitStmt(self.expression(), tok)
endOfLine("missing semicolon after yield statement")
endOfLine("missing semicolon after await statement")
proc raiseStmt(self: Parser): ASTNode =
@ -633,17 +633,22 @@ proc forEachStmt(self: Parser): ASTNode =
self.currentLoop = enclosingLoop
proc importStmt(self: Parser): ASTNode =
proc importStmt(self: Parser, fromStmt: bool = false): ASTNode =
## Parses import statements
let tok = self.peek(-1)
var tok: Token
if fromStmt:
tok = self.peek(-2)
else:
tok = self.peek(-1)
# TODO: New AST node
self.expect(Identifier, "expecting module name(s) after import statement")
result = newImportStmt(self.expression(), tok)
result = newImportStmt(newIdentExpr(self.peek(-1)), tok)
endOfLine("missing semicolon after import statement")
proc tryStmt(self: Parser): ASTNode =
## Parses try/except/finally/else blocks
## Parses try/except/else/finally blocks
let tok = self.peek(-1)
var body = self.statement()
var handlers: seq[tuple[body, exc, name: ASTNode]] = @[]
@ -655,7 +660,9 @@ proc tryStmt(self: Parser): ASTNode =
while self.match(Except):
excName = self.expression()
if excName.kind == identExpr:
continue
handlerBody = self.statement()
handlers.add((body: handlerBody, exc: excName, name: asName))
asName = nil
elif excName.kind == binaryExpr and BinaryExpr(excName).operator.kind == As:
asName = BinaryExpr(excName).b
if BinaryExpr(excName).b.kind != identExpr:
@ -674,15 +681,12 @@ proc tryStmt(self: Parser): ASTNode =
continue
else:
excName = nil
handlerBody = self.statement()
handlers.add((body: handlerBody, exc: excName, name: asName))
asName = nil
if self.match(Else):
elseClause = self.statement()
if self.match(Finally):
finallyClause = self.statement()
if handlers.len() == 0 and elseClause == nil and finallyClause == nil:
self.error("expecting 'except', 'finally' or 'else' statements after 'try' block")
self.error("expecting 'except', 'finally' or 'else' statement after 'try' block")
for i, handler in handlers:
if handler.exc == nil and i != handlers.high():
self.error("catch-all exception handler with bare 'except' must come last in try statement")
@ -822,9 +826,8 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
discard # Unreachable
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode =
## Parses functions, coroutines, generators and anonymous functions declarations (the latter is
## technically an expression, not a declaration)
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode =
## Parses functions, coroutines, generators, anonymous functions and custom operators
let tok = self.peek(-1)
var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[]
@ -941,6 +944,11 @@ proc statement(self: Parser): ASTNode =
of Import:
discard self.step()
result = self.importStmt()
of From:
# TODO
# from module import a [, b, c as d]
discard self.step()
result = self.importStmt(fromStmt=true)
of While:
discard self.step()
result = self.whileStmt()