From e83cd424f94fcaedb0a4709915bd7a719b94c376 Mon Sep 17 00:00:00 2001 From: prod2 <95874442+prod2@users.noreply.github.com> Date: Fri, 2 Dec 2022 20:47:31 +0100 Subject: [PATCH] backtrack in parser + ampersand op support --- src/ndspkg/compv2/parser.nim | 66 +++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/src/ndspkg/compv2/parser.nim b/src/ndspkg/compv2/parser.nim index e3cf00b..de17448 100644 --- a/src/ndspkg/compv2/parser.nim +++ b/src/ndspkg/compv2/parser.nim @@ -12,6 +12,7 @@ import strutils import bitops import sequtils import sugar +import options # TYPEDEF @@ -21,7 +22,11 @@ type scanner: Scanner source: string current: Token - previous: Token + previous: Option[Token] + next: Option[Token] + # if there is a next set, advance won't trigger the scanner + # it will use next instead + hold: Node # temporary hold, used to implement ampersand op # errors hadError*: bool @@ -32,6 +37,8 @@ proc newParser*(name: string, source: string): Parser = result.source = source result.hadError = false result.panicMode = false + result.next = none[Token]() + result.previous = none[Token]() # UTILS @@ -49,7 +56,7 @@ proc errorAt(parser: Parser, line: int, msg: string, at: string = "") = parser.panicMode = true proc error(parser: Parser, msg: string) = - parser.errorAt(parser.previous.line, msg) + parser.errorAt(parser.previous.get().line, msg) proc errorAtCurrent(parser: Parser, msg: string) = parser.errorAt(parser.current.line, msg) @@ -57,15 +64,27 @@ proc errorAtCurrent(parser: Parser, msg: string) = # scanning for tokens proc advance(parser: Parser) = - parser.previous = parser.current + parser.previous = some(parser.current) while true: - parser.current = parser.scanner.scanToken() + if parser.next.isSome(): + parser.current = parser.next.get() + parser.next = none[Token]() + else: + parser.current = parser.scanner.scanToken() when debugScanner: parser.current.debugPrint() if (parser.current.tokenType != tkError): break parser.errorAtCurrent(parser.current.text) +proc backtrack(parser: Parser) = + parser.next = some(parser.current) + parser.current = parser.previous.get() + # danger - danger.previous is undefined here + # luckily thanks to options if that ever happens + # we get a crash + parser.previous = none[Token]() + proc match(parser: Parser, tokenType: TokenType): bool = if parser.current.tokenType == tokenType: parser.advance() @@ -93,7 +112,7 @@ proc peekMatch(parser: Parser, tokenType: TokenType): bool = proc synchronize(parser: Parser) = parser.panicMode = false while parser.current.tokenType != tkEof: - if parser.previous.tokenType in {tkSemicolon, tkRightBrace}: + if parser.previous.get().tokenType in {tkSemicolon, tkRightBrace}: return if parser.current.tokenType in {tkFunct, tkVar, tkFor, tkIf, tkWhile}: return @@ -127,7 +146,7 @@ proc parseTable(parser: Parser): Node = parser.consume(tkRightBracket, "']' expected after table key.") # key = syntax elif parser.match(tkIdentifier): - result.keys.add(Node(kind: nkConst, constant: parser.previous.text.fromNimString())) + result.keys.add(Node(kind: nkConst, constant: parser.previous.get().text.fromNimString())) else: parser.errorAtCurrent("Key expected (have you forgotten to put the key in brackets?).") parser.consume(tkEqual, "'=' expected after key.") @@ -146,9 +165,9 @@ proc primary(parser: Parser): Node = if parser.match(tkNil): return Node(kind: nkConst, constant: ndNil) if parser.match(tkNumber): - return Node(kind: nkConst, constant: fromFloat(parseFloat(parser.previous.text))) + return Node(kind: nkConst, constant: fromFloat(parseFloat(parser.previous.get().text))) if parser.match(tkString): - return Node(kind: nkConst, constant: fromNimString(parser.previous.text[1..^2])) + return Node(kind: nkConst, constant: fromNimString(parser.previous.get().text[1..^2])) if parser.match(tkLeftParen): let grouped = parser.expression() parser.consume(tkRightParen, "Expect ')' after expression.") @@ -158,7 +177,11 @@ proc primary(parser: Parser): Node = if parser.match(tkStartTable): return parser.parseTable() if parser.match(tkIdentifier): - return Node(kind: nkVarGet, gVarName: parser.previous.text) + return Node(kind: nkVarGet, gVarName: parser.previous.get().text) + if parser.match(tkAmpersand): + result = parser.hold + parser.hold = nil + return result parser.errorAtCurrent("Primary expected, but something else found.") @@ -179,12 +202,12 @@ proc parseIndex(parser: Parser): Node = while parser.match({tkLeftBracket, tkDot, tkIdentifier}): # NOTE: :index is counted as a single identifier, so two identifiers after eachother will be handled here - if parser.previous.tokenType == tkLeftBracket: + if parser.previous.get().tokenType == tkLeftBracket: let index = parser.expression() parser.consume(tkRightBracket, "']' after index.") result = Node(kind: nkGetIndex, gCollection: result, gIndex: index) - elif parser.previous.tokenType == tkIdentifier: - let identText = parser.previous.text + elif parser.previous.get().tokenType == tkIdentifier: + let identText = parser.previous.get().text if identText[0] != ':': parser.errorAtCurrent("';' expected after expression statement.") # update this with whatever the original error when two idents follow eachother is @@ -199,7 +222,7 @@ proc parseIndex(parser: Parser): Node = else: # dot parser.consume(tkIdentifier, "Identifier expected after '.' index operator.") - result = Node(kind: nkGetIndex, gCollection: result, gIndex: Node(kind: nkConst, constant: parser.previous.text.fromNimString())) + result = Node(kind: nkGetIndex, gCollection: result, gIndex: Node(kind: nkConst, constant: parser.previous.get().text.fromNimString())) proc parseCall(parser: Parser): Node = result = parser.parseIndex() @@ -230,7 +253,7 @@ proc unary(parser: Parser): Node = # unary level for unary operators, plus some control flow is here too const unaryOps = {tkBang, tkMinus, tkIf, tkWhile, tkHashtag} if parser.match(unaryOps): - let op = parser.previous + let op = parser.previous.get() case op.tokenType: of tkBang: let right = parser.unary() @@ -253,7 +276,7 @@ proc factor(parser: Parser): Node = result = parser.unary() while parser.match({tkSlash, tkStar}): - let op = parser.previous + let op = parser.previous.get() let right = parser.unary() if op.tokenType == tkSlash: result = Node(kind: nkDiv, left: result, right: right) @@ -264,7 +287,7 @@ proc term(parser: Parser): Node = result = parser.factor() while parser.match({tkMinus, tkPlus}): - let op = parser.previous + let op = parser.previous.get() let right = parser.factor() if op.tokenType == tkMinus: result = Node(kind: nkMinus, left: result, right: right) @@ -275,7 +298,7 @@ proc comparison(parser: Parser): Node = result = parser.term() while parser.match({tkGreater, tkGreaterEqual, tkLess, tkLessEqual}): - let op = parser.previous + let op = parser.previous.get() let right = parser.term() case op.tokenType: of tkGreater: @@ -293,7 +316,7 @@ proc equality(parser: Parser): Node = result = parser.comparison() while parser.match({tkBangEqual, tkEqualEqual}): - let op = parser.previous + let op = parser.previous.get() let right = parser.comparison() if op.tokenType == tkBangEqual: result = Node(kind: nkNeq, left: result, right: right) @@ -344,7 +367,7 @@ proc parseAssign(parser: Parser): Node = if parser.match(tkEqual): # check if result is assignable const assignable = {nkVarGet, nkGetIndex} - let right = parser.expression() + let right = parser.parseAssign() if result.kind notin assignable: parser.errorAtCurrent("Attempt to assign to invalid target.") return @@ -357,7 +380,10 @@ proc parseAssign(parser: Parser): Node = proc parseAmpersand(parser: Parser): Node = result = parser.parseAssign() - # TODO + if parser.match(tkAmpersand): + parser.hold = Node(kind: nkExpr, expression: result) + parser.backtrack() + return parser.parseAmpersand() proc expression(parser: Parser): Node = parser.parseAmpersand()