Experimental support for arbitrary user-defined operators

src/frontend/lexer.nim

@@ -64,6 +64,9 @@ proc addSymbol*(self: SymbolTable, lexeme: string, token: TokenType) =
     self.symbols[lexeme] = token
 
 
+proc existsSymbol*(self: SymbolTable, lexeme: string): bool {.inline.} = lexeme in self.symbols
+
+
 proc removeSymbol*(self: SymbolTable, lexeme: string)  =
     ## Removes a symbol from the symbol table
     ## (does nothing if it does not exist)

src/frontend/parser.nim

@@ -62,6 +62,8 @@ type
         currentFunction: ASTNode
         # Stores the current scope depth (0 = global, > 0 local)
         scopeDepth: int
+        # We store user-defined operators for later use
+        operators: seq[string]
 
 
 proc newParser*(): Parser = 
@@ -384,14 +386,23 @@ proc unary(self: Parser): ASTNode =
         result = self.call()
 
 
+proc customUnaryOperator(self: Parser): ASTNode =
+    ## Parses user-defined unary expressions
+    if self.peek().lexeme in self.operators:
+        discard self.step()
+        result = newUnaryExpr(self.peek(-1), self.customUnaryOperator())
+    else:
+        result = self.unary()
+
+
 proc pow(self: Parser): ASTNode =
     ## Parses exponentiation expressions
-    result = self.unary()
+    result = self.customUnaryOperator()
     var operator: Token
     var right: ASTNode
     while self.match(DoubleStar):
         operator = self.peek(-1)
-        right = self.unary()
+        right = self.customUnaryOperator()
         result = newBinaryExpr(result, operator, right)
 
 
@@ -486,11 +497,22 @@ proc bitwiseOr(self: Parser): ASTNode =
         result = newBinaryExpr(result, operator, right)
 
 
+proc customBinaryOperator(self: Parser): ASTNode =
+    ## Parses user-defined binary operators
+    result = self.bitwiseOr()
+    var operator: Token
+    var right: ASTNode
+    while self.peek().lexeme in self.operators:
+        operator = self.step()
+        right = self.bitwiseOr()
+        result = newBinaryExpr(result, operator, right)
+
+
 proc assignment(self: Parser): ASTNode =
     ## Parses assignment, the highest-level
     ## expression (including stuff like a.b = 1).
     ## Slice assignments are also parsed here
-    result = self.bitwiseOr()
+    result = self.customBinaryOperator()
     if self.match([Equal, InplaceAdd, InplaceSub, InplaceDiv, InplaceMod,
                    InplacePow, InplaceMul, InplaceXor, InplaceAnd, InplaceOr,
                    InplaceFloorDiv, InplaceRightShift, InplaceLeftShift]):
@@ -880,6 +902,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
                 self.error("cannot have more than 255 arguments in function declaration")
             self.expect(Identifier, "expecting parameter name")
             parameter.name = newIdentExpr(self.peek(-1))
+            self.expect(Colon, "expecting ':' after parameter name")
             self.expect(Identifier, "expecting parameter type")
             parameter.valueType = newIdentExpr(self.peek(-1))
             if parameter in arguments:
@@ -892,6 +915,10 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
             if not self.match(Comma):
                 break
         self.expect(RightParen)
+        if self.match(Colon):
+            # Function's return type
+            self.expect(Identifier, "expecting return type after ':'")
+            returnType = newIdentExpr(self.peek(-1))
         self.expect(LeftBrace)
     if self.currentFunction.kind == funDecl:
         if not self.match(Semicolon):
@@ -905,6 +932,14 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
         LambdaExpr(self.currentFunction).arguments = arguments
         LambdaExpr(self.currentFunction).returnType = returnType
     result = self.currentFunction
+    if isOperator:
+        # isOperator is only true for functions
+        # with a name (since nameless operators
+        # don't make much sense)
+        if arguments.len() == 0:
+            self.error("cannot declare argument-less operator")
+        elif arguments.len() > 2:
+            self.error("cannot declare operator with more than 2 arguments")
     self.currentFunction = enclosingFunction
 
 
@@ -993,6 +1028,9 @@ proc declaration(self: Parser): ASTNode =
         of Generator:
             discard self.step()
             result = self.funDecl(isGenerator=true)
+        of Operator:
+            discard self.step()
+            result = self.funDecl(isOperator=true)
         of Type, Comment, Whitespace, Tab:
            discard self.step()  # TODO
         else:
@@ -1007,5 +1045,20 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
     self.currentLoop = None
     self.currentFunction = nil
     self.scopeDepth = 0
+    self.operators = @[]
+    for i, token in self.tokens:
+        # We do a first pass over the tokens
+        # to find user-defined operators.
+        # Note that this relies on the lexer
+        # ending the input with an EOF token
+        if token.kind == Operator:
+            if i == self.tokens.high():
+                self.error("invalid state: found malformed tokenizer input while looking for operators")
+            elif self.tokens[i + 1].kind != Identifier:
+                # The input is probably malformed,
+                # we'll catch this error later
+                break
+            else:
+                self.operators.add(self.tokens[i + 1].lexeme)
     while not self.done():
-        result.add(self.declaration())
+        result.add(self.declaration())

src/test.nim

@@ -42,11 +42,13 @@ when isMainModule:
             if input.len() > 0:
                 # Currently the parser doesn't handle these tokens well
                 tokens = filter(tokenizer.lex(input, "<stdin>"), proc (x: Token): bool = x.kind notin {Whitespace, Tab})
+                echo "Tokenization step:"
                 for i, token in tokens:
                     if i == tokens.high():
                         # Who cares about EOF?
                         break
                     echo token
+                echo "Parsing step:"
                 for node in parser.parse(tokens, "<stdin>"):
                     echo node
         except IOError: