Compiler support for BuildList, BuildSet, BuildDict and BuildTuple

src/backend/compiler.nim

@@ -16,10 +16,12 @@ import meta/ast
 import meta/errors
 import meta/bytecode
 import ../config
+import ../util/multibyte
 
 
 import strformat
 import parseutils
+import sequtils
 
 
 export ast
@@ -124,7 +126,7 @@ proc emitByte(self: Compiler, byt: OpCode|uint8) =
     ## to the current chunk being compiled
     when DEBUG_TRACE_COMPILER:
         echo &"DEBUG - Compiler: Emitting {$byt}"
-    self.chunk.write(uint8 byt, self.peek(-1).token.line)
+    self.chunk.write(uint8 byt, self.peek().token.line)
 
 
 proc emitBytes(self: Compiler, byt1: OpCode|uint8, byt2: OpCode|uint8) =
@@ -156,7 +158,7 @@ proc emitConstant(self: Compiler, obj: ASTNode) =
     self.emitBytes(self.makeConstant(obj))
 
 
-proc literal(self: Compiler, node: LiteralExpr) =
+proc literal(self: Compiler, node: ASTNode) =
     ## Emits instructions for literals such
     ## as singletons, strings, numbers and
     ## collections
@@ -188,7 +190,7 @@ proc literal(self: Compiler, node: LiteralExpr) =
         # will collapse all these other literals
         # to nodes of kind intExpr, that can be
         # disabled. This also allows us to catch
-        # overflow errors before running any code
+        # basic overflow errors before running any code
         of hexExpr:
             var x: int
             var y = HexExpr(node)
@@ -221,6 +223,31 @@ proc literal(self: Compiler, node: LiteralExpr) =
             except ValueError:
                 self.error("floating point value out of range")
             self.emitConstant(y)
+        of listExpr:
+            var y = ListExpr(node)
+            self.emitByte(BuildList)
+            self.emitBytes(y.members.len().toTriple())  # 24-bit integer, meaning list literals can have up to 2^24 elements
+            for member in y.members:
+                self.expression(member)
+        of tupleExpr:
+            var y = TupleExpr(node)
+            self.emitByte(BuildTuple)
+            self.emitBytes(y.members.len().toTriple())
+            for member in y.members:
+                self.expression(member)
+        of setExpr:
+            var y = SetExpr(node)
+            self.emitByte(BuildSet)
+            self.emitBytes(y.members.len().toTriple())
+            for member in y.members:
+                self.expression(member)
+        of dictExpr:
+            var y = DictExpr(node)
+            self.emitByte(BuildDict)
+            self.emitBytes(y.keys.len().toTriple())
+            for (key, value) in zip(y.keys, y.values):
+                self.expression(key)
+                self.expression(value)
         else:
             self.error(&"invalid AST node of kind {node.kind} at literal(): {node} (This is an internal error and most likely a bug)")
 
@@ -289,9 +316,12 @@ proc expression(self: Compiler, node: ASTNode) =
             self.unary(UnaryExpr(node))
         of binaryExpr:
             self.binary(BinaryExpr(node))
-        of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr,
+        of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr:
-            tupleExpr, dictExpr, setExpr, listExpr:
             self.literal(LiteralExpr(node))
+        of tupleExpr, setExpr, listExpr:
+            self.literal(ListExpr(node))
+        of dictExpr:
+            self.literal(DictExpr(node))
         else:
             self.error(&"invalid AST node of kind {node.kind} at expression(): {node} (This is an internal error and most likely a bug)")  # TODO
 

src/backend/meta/ast.nim

@@ -312,6 +312,7 @@ proc isLiteral*(self: ASTNode): bool {.inline.} = self.isConst() or self.kind in
 proc newIntExpr*(literal: Token): IntExpr =
     result = IntExpr(kind: intExpr)
     result.literal = literal
+    result.token = literal
 
 
 proc newOctExpr*(literal: Token): OctExpr =
@@ -625,7 +626,7 @@ proc `$`*(self: ASTNode): string =
             result &= &"SetItem(obj={self.obj}, name={self.value}, value={self.value})"
         of callExpr:
             var self = CallExpr(self)
-            result &= &"Call({self.callee}, arguments=(positionals=[{self.arguments.positionals.join(\", \")}], keyword=[{self.arguments.keyword.join(\", \")}]))"
+            result &= &"""Call({self.callee}, arguments=(positionals=[{self.arguments.positionals.join(", ")}], keyword=[{self.arguments.keyword.join(", ")}]))"""
         of unaryExpr:
             var self = UnaryExpr(self)
             result &= &"Unary(Operator('{self.operator.lexeme}'), {self.a})"
@@ -643,7 +644,7 @@ proc `$`*(self: ASTNode): string =
             result &= &"Import({self.moduleName})"
         of fromImportStmt:
             var self = FromImportStmt(self)
-            result &= &"FromImport(fromModule={self.fromModule}, fromAttributes=[{self.fromAttributes.join(\", \")}])"
+            result &= &"""FromImport(fromModule={self.fromModule}, fromAttributes=[{self.fromAttributes.join(", ")}])"""
         of delStmt:
             var self = DelStmt(self)
             result &= &"Del({self.name})"
@@ -655,7 +656,7 @@ proc `$`*(self: ASTNode): string =
             result &= &"Raise({self.exception})"
         of blockStmt:
             var self = BlockStmt(self)
-            result &= &"Block([{self.code.join(\", \")}])"
+            result &= &"""Block([{self.code.join(", ")}])"""
         of whileStmt:
             var self = WhileStmt(self)
             result &= &"While(condition={self.condition}, body={self.body})"
@@ -688,31 +689,31 @@ proc `$`*(self: ASTNode): string =
             result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})"
         of funDecl:
             var self = FunDecl(self)
-            result &= &"FunDecl(name={self.name}, body={self.body}, arguments=[{self.arguments.join(\", \")}], defaults=[{self.defaults.join(\", \")}], async={self.isAsync}, generator={self.isGenerator}, static={self.isStatic}, private={self.isPrivate})"
+            result &= &"""FunDecl(name={self.name}, body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, static={self.isStatic}, private={self.isPrivate})"""
         of classDecl:
             var self = ClassDecl(self)
-            result &= &"Class(name={self.name}, body={self.body}, parents=[{self.parents.join(\", \")}], static={self.isStatic}, private={self.isPrivate})"
+            result &= &"""Class(name={self.name}, body={self.body}, parents=[{self.parents.join(", ")}], static={self.isStatic}, private={self.isPrivate})"""
         of tupleExpr:
             var self = TupleExpr(self)
-            result &= &"Tuple([{self.members.join(\", \")}])"
+            result &= &"""Tuple([{self.members.join(", ")}])"""
         of setExpr:
             var self = SetExpr(self)
-            result &= &"Set([{self.members.join(\", \")}])"
+            result &= &"""Set([{self.members.join(", ")}])"""
         of listExpr:
             var self = ListExpr(self)
-            result &= &"List([{self.members.join(\", \")}])"
+            result &= &"""List([{self.members.join(", ")}])"""
         of dictExpr:
             var self = DictExpr(self)
-            result &= &"Dict(keys=[{self.keys.join(\", \")}], values=[{self.values.join(\", \")}])"
+            result &= &"""Dict(keys=[{self.keys.join(", ")}], values=[{self.values.join(", ")}])"""
         of lambdaExpr:
             var self = LambdaExpr(self)
-            result &= &"Lambda(body={self.body}, arguments=[{self.arguments.join(\", \")}], defaults=[{self.defaults.join(\", \")}], generator={self.isGenerator})"
+            result &= &"""Lambda(body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator})"""
         of deferStmt:
             var self = DeferStmt(self)
             result &= &"Defer({self.deferred})"
         of sliceExpr:
             var self = SliceExpr(self)
-            result &= &"Slice({self.slicee}, ends=[{self.ends.join(\", \")}])"
+            result &= &"""Slice({self.slicee}, ends=[{self.ends.join(", ")}])"""
         of tryStmt:
             var self = TryStmt(self)
             result &= &"TryStmt(body={self.body}, handlers={self.handlers}"

src/backend/meta/bytecode.nim

@@ -15,6 +15,10 @@ import ast
 import ../../util/multibyte
 
 
+import strutils
+import strformat
+
+
 export ast
 
 
@@ -47,7 +51,10 @@ type
         # argument to unary opcodes, while
         # a and b represent arguments to binary
         # opcodes. Other variable names may be
-        # used for more complex opcodes        
+        # used for more complex opcodes. All
+        # arguments to opcodes (if they take
+        # arguments) come from popping off the
+        # stack   
         LoadConstant = 0u8,  # Pushes constant at position x in the constant table onto the stack
         # Binary operators
         UnaryNegate,  # Pushes the result of -x onto the stack
@@ -132,8 +139,6 @@ type
         BuildTuple
 
 
-
-
 const simpleInstructions* = {Return, BinaryAdd, BinaryMultiply,
                              BinaryDivide, BinarySubtract,
                              BinaryMod, BinaryPow, Nil,
@@ -147,14 +152,14 @@ const simpleInstructions* = {Return, BinaryAdd, BinaryMultiply,
                              UnaryNot, InPlaceAdd, InPlaceDivide,
                              InPlaceFloorDiv, InPlaceMod, InPlaceMultiply,
                              InPlaceSubtract, BinaryFloorDiv, BinaryOf, Raise,
-                             ReRaise, BeginTry, FinishTry,
+                             ReRaise, BeginTry, FinishTry, Yield, Await}
-                             Yield, Await}
 const constantInstructions* = {LoadConstant, DeclareName,
                                LoadName, UpdateName,
                                DeleteName}
 const byteInstructions* = {UpdateNameFast, LoadNameFast, 
                            DeleteNameFast, Call}
 const jumpInstructions* = {JumpIfFalse, Jump}
+const collectionInstructions* = {BuildList, BuildDict, BuildSet, BuildTuple}
 
 
 proc newChunk*(): Chunk =
@@ -162,9 +167,13 @@ proc newChunk*(): Chunk =
     result = Chunk(consts: @[], code: @[], lines: @[])
 
 
+proc `$`*(self: Chunk): string = &"""Chunk(consts=[{self.consts.join(", ")}], code=[{self.code.join(", ")}], lines=[{self.lines.join(", ")}])"""
+
+
 proc write*(self: Chunk, newByte: uint8, line: int) =
     ## Adds the given instruction at the provided line number
     ## to the given chunk object
+    assert line > 0
     if self.lines.high() >= 1 and self.lines[^2] == line:
         self.lines[^1] += 1
     else:

src/backend/parser.nim

@@ -213,6 +213,7 @@ proc primary(self: Parser): ASTNode =
             result = newFloatExpr(self.step())
         of Integer:
             result = newIntExpr(self.step())
+            echo result.token
         of Identifier:
             result = newIdentExpr(self.step())
         of LeftParen:

src/backend/serializer.nim

@@ -106,7 +106,7 @@ proc dumpBytes*(self: Serializer, chunk: Chunk, file, filename: string): seq[byt
     result.extend(self.toBytes(computeSHA256(file)))
     for constant in chunk.consts:
         case constant.kind:
-            of intExpr:
+            of intExpr, floatExpr:
                 result.add(0x1)
                 result.add(byte(len(constant.token.lexeme)))
                 result.extend(self.toBytes(constant.token.lexeme))

src/config.nim

@@ -20,7 +20,7 @@ const HEAP_GROW_FACTOR* = 2   # How much extra memory to allocate for dynamic ar
 const MAX_STACK_FRAMES* = 800   # The maximum number of stack frames at any one time. Acts as a recursion limiter (1 frame = 1 call)
 const JAPL_VERSION* = (major: 0, minor: 4, patch: 0)
 const JAPL_RELEASE* = "alpha"
-const JAPL_COMMIT_HASH* = "b252749d0e5448b8fef64150299d8318362bc08c"
+const JAPL_COMMIT_HASH* = "5f33af5b2c909293bc4d0530b4d3d69645410153"
 const JAPL_BRANCH* = "master"
 const DEBUG_TRACE_VM* = false # Traces VM execution
 const SKIP_STDLIB_INIT* = false # Skips stdlib initialization (can be imported manually)

src/main.nim

@@ -93,14 +93,16 @@ proc main() =
 
             compiled = compiler.compile(optimized.tree, filename)
             echo "Compilation step:"
-            echo &"\tRaw byte stream: [{compiled.code.join(\", \")}]"
+            stdout.write("\t")
-            echo "\tBytecode disassembler output below:\n"
+            echo &"""Raw byte stream: [{compiled.code.join(", ")}]"""
+            echo "\n\nBytecode disassembler output below:\n"
             disassembleChunk(compiled, filename)
             echo ""
             
             serializedRaw = serializer.dumpBytes(compiled, source, filename)
             echo "Serialization step: "
-            echo &"\tRaw hex output: {serializedRaw.mapIt(toHex(it)).join(\"\").toLowerAscii()}"
+            stdout.write("\t")
+            echo &"""Raw hex output: {serializedRaw.mapIt(toHex(it)).join("").toLowerAscii()}"""
             echo ""
 
             serialized = serializer.loadBytes(serializedRaw)
@@ -111,6 +113,7 @@ proc main() =
             compileDate = fromUnix(serialized.compileDate).format("d/M/yyyy H:mm:ss")
             echo &"\t\t- Compilation date & time: {compileDate}"
         except:
+            raise
             echo &"A Nim runtime exception occurred: {getCurrentExceptionMsg()}"
             continue