backtrack in parser + ampersand op support

src/ndspkg/compv2/parser.nim

@@ -12,6 +12,7 @@ import strutils
 import bitops
 import sequtils
 import sugar
+import options
 
 # TYPEDEF
 
@@ -21,7 +22,11 @@ type
     scanner: Scanner
     source: string
     current: Token
-    previous: Token
+    previous: Option[Token]
+    next: Option[Token]
+    # if there is a next set, advance won't trigger the scanner
+    # it will use next instead
+    hold: Node # temporary hold, used to implement ampersand op
 
     # errors
     hadError*: bool
@@ -32,6 +37,8 @@ proc newParser*(name: string, source: string): Parser =
   result.source = source
   result.hadError = false
   result.panicMode = false
+  result.next = none[Token]()
+  result.previous = none[Token]()
 
 # UTILS
 
@@ -49,7 +56,7 @@ proc errorAt(parser: Parser, line: int, msg: string, at: string = "") =
   parser.panicMode = true
 
 proc error(parser: Parser, msg: string) =
-  parser.errorAt(parser.previous.line, msg)
+  parser.errorAt(parser.previous.get().line, msg)
 
 proc errorAtCurrent(parser: Parser, msg: string) =
   parser.errorAt(parser.current.line, msg)
@@ -57,15 +64,27 @@ proc errorAtCurrent(parser: Parser, msg: string) =
 # scanning for tokens
 
 proc advance(parser: Parser) =
-  parser.previous = parser.current
+  parser.previous = some(parser.current)
   while true:
-    parser.current = parser.scanner.scanToken()
+    if parser.next.isSome():
+      parser.current = parser.next.get()
+      parser.next = none[Token]()
+    else:
+      parser.current = parser.scanner.scanToken()
     when debugScanner:
       parser.current.debugPrint()
     if (parser.current.tokenType != tkError):
       break
     parser.errorAtCurrent(parser.current.text)
 
+proc backtrack(parser: Parser) =
+  parser.next = some(parser.current)
+  parser.current = parser.previous.get()
+  # danger - danger.previous is undefined here
+  # luckily thanks to options if that ever happens
+  # we get a crash
+  parser.previous = none[Token]()
+
 proc match(parser: Parser, tokenType: TokenType): bool =
   if parser.current.tokenType == tokenType:
     parser.advance()
@@ -93,7 +112,7 @@ proc peekMatch(parser: Parser, tokenType: TokenType): bool =
 proc synchronize(parser: Parser) =
   parser.panicMode = false
   while parser.current.tokenType != tkEof:
-    if parser.previous.tokenType in {tkSemicolon, tkRightBrace}:
+    if parser.previous.get().tokenType in {tkSemicolon, tkRightBrace}:
       return
     if parser.current.tokenType in {tkFunct, tkVar, tkFor, tkIf, tkWhile}:
       return
@@ -127,7 +146,7 @@ proc parseTable(parser: Parser): Node =
       parser.consume(tkRightBracket, "']' expected after table key.")
     # key = syntax
     elif parser.match(tkIdentifier):
-      result.keys.add(Node(kind: nkConst, constant: parser.previous.text.fromNimString()))
+      result.keys.add(Node(kind: nkConst, constant: parser.previous.get().text.fromNimString()))
     else:
       parser.errorAtCurrent("Key expected (have you forgotten to put the key in brackets?).")
     parser.consume(tkEqual, "'=' expected after key.")
@@ -146,9 +165,9 @@ proc primary(parser: Parser): Node =
   if parser.match(tkNil):
     return Node(kind: nkConst, constant: ndNil)
   if parser.match(tkNumber):
-    return Node(kind: nkConst, constant: fromFloat(parseFloat(parser.previous.text)))
+    return Node(kind: nkConst, constant: fromFloat(parseFloat(parser.previous.get().text)))
   if parser.match(tkString):
-    return Node(kind: nkConst, constant: fromNimString(parser.previous.text[1..^2]))
+    return Node(kind: nkConst, constant: fromNimString(parser.previous.get().text[1..^2]))
   if parser.match(tkLeftParen):
     let grouped = parser.expression()
     parser.consume(tkRightParen, "Expect ')' after expression.")
@@ -158,7 +177,11 @@ proc primary(parser: Parser): Node =
   if parser.match(tkStartTable):
     return parser.parseTable()
   if parser.match(tkIdentifier):
-    return Node(kind: nkVarGet, gVarName: parser.previous.text)
+    return Node(kind: nkVarGet, gVarName: parser.previous.get().text)
+  if parser.match(tkAmpersand):
+    result = parser.hold
+    parser.hold = nil
+    return result
   
   parser.errorAtCurrent("Primary expected, but something else found.")
 
@@ -179,12 +202,12 @@ proc parseIndex(parser: Parser): Node =
 
   while parser.match({tkLeftBracket, tkDot, tkIdentifier}):
     # NOTE: :index is counted as a single identifier, so two identifiers after eachother will be handled here
-    if parser.previous.tokenType == tkLeftBracket:
+    if parser.previous.get().tokenType == tkLeftBracket:
       let index = parser.expression()
       parser.consume(tkRightBracket, "']' after index.")
       result = Node(kind: nkGetIndex, gCollection: result, gIndex: index)
-    elif parser.previous.tokenType == tkIdentifier:
+    elif parser.previous.get().tokenType == tkIdentifier:
-      let identText = parser.previous.text
+      let identText = parser.previous.get().text
       if identText[0] != ':':
         parser.errorAtCurrent("';' expected after expression statement.")
         # update this with whatever the original error when two idents follow eachother is
@@ -199,7 +222,7 @@ proc parseIndex(parser: Parser): Node =
     else:
       # dot
       parser.consume(tkIdentifier, "Identifier expected after '.' index operator.")
-      result = Node(kind: nkGetIndex, gCollection: result, gIndex: Node(kind: nkConst, constant: parser.previous.text.fromNimString()))
+      result = Node(kind: nkGetIndex, gCollection: result, gIndex: Node(kind: nkConst, constant: parser.previous.get().text.fromNimString()))
 
 proc parseCall(parser: Parser): Node =
   result = parser.parseIndex()
@@ -230,7 +253,7 @@ proc unary(parser: Parser): Node =
   # unary level for unary operators, plus some control flow is here too
   const unaryOps = {tkBang, tkMinus, tkIf, tkWhile, tkHashtag}
   if parser.match(unaryOps):
-    let op = parser.previous
+    let op = parser.previous.get()
     case op.tokenType:
     of tkBang:
       let right = parser.unary()
@@ -253,7 +276,7 @@ proc factor(parser: Parser): Node =
   result = parser.unary()
   
   while parser.match({tkSlash, tkStar}):
-    let op = parser.previous
+    let op = parser.previous.get()
     let right = parser.unary()
     if op.tokenType == tkSlash:
       result = Node(kind: nkDiv, left: result, right: right)
@@ -264,7 +287,7 @@ proc term(parser: Parser): Node =
   result = parser.factor()
 
   while parser.match({tkMinus, tkPlus}):
-    let op = parser.previous
+    let op = parser.previous.get()
     let right = parser.factor()
     if op.tokenType == tkMinus:
       result = Node(kind: nkMinus, left: result, right: right)
@@ -275,7 +298,7 @@ proc comparison(parser: Parser): Node =
   result = parser.term()
 
   while parser.match({tkGreater, tkGreaterEqual, tkLess, tkLessEqual}):
-    let op = parser.previous
+    let op = parser.previous.get()
     let right = parser.term()
     case op.tokenType:
     of tkGreater:
@@ -293,7 +316,7 @@ proc equality(parser: Parser): Node =
   result = parser.comparison()
 
   while parser.match({tkBangEqual, tkEqualEqual}):
-    let op = parser.previous
+    let op = parser.previous.get()
     let right = parser.comparison()
     if op.tokenType == tkBangEqual:
       result = Node(kind: nkNeq, left: result, right: right)
@@ -344,7 +367,7 @@ proc parseAssign(parser: Parser): Node =
   if parser.match(tkEqual):
     # check if result is assignable
     const assignable = {nkVarGet, nkGetIndex}
-    let right = parser.expression()
+    let right = parser.parseAssign()
     if result.kind notin assignable:
       parser.errorAtCurrent("Attempt to assign to invalid target.")
       return
@@ -357,7 +380,10 @@ proc parseAssign(parser: Parser): Node =
 
 proc parseAmpersand(parser: Parser): Node =
   result = parser.parseAssign()
-  # TODO
+  if parser.match(tkAmpersand):
+    parser.hold = Node(kind: nkExpr, expression: result)
+    parser.backtrack()
+    return parser.parseAmpersand()
 
 proc expression(parser: Parser): Node =
   parser.parseAmpersand()