Fixed parsing code in funDecl for arguments, fixed funDecl in compiler

docs/bytecode.md

@@ -67,6 +67,7 @@ After the modifier follows the string encoded in UTF-8, __without__ quotes.
 
 
 ### List-like collections (sets, lists and tuples)
+
 List-like collections (or _sequences_)-- namely sets, lists and tuples-- encode their length first: for lists and sets this only denotes the _starting_ size of the container, while a tuple's size is fixed once it is created. The length may be 0, in which case it is interpreted as the sequence being empty; After the length, which expresses the __number of elements__ in the collection (just the count!), follows a number of compile-time objects equal to the specified length, with their respective encoding.
 
 __TODO__: Currently the compiler does not emit constant instructions for collections using only constants: it will just emit a bunch of `LoadConstant` instructions and
@@ -82,8 +83,8 @@ Mappings (also called _associative arrays_ or, more informally, _dictionaries_)
 
 An object file starts with the headers, namely:
 
-- A 13-byte constant string with the value `"JAPL_BYTECODE"` (without quotes) encoded as a sequence of integers corresponding to their value in the ASCII table
-- A 3-byte version header composed of 3 unsigned integers representing the major, minor and patch version of the compiler used to generate the file, respectively. JAPL follows the SemVer standard for versioning
+- A 13-byte constant string with the value `"JAPL_BYTECODE"` (without quotes) encoded as a sequence of integers corresponding to the ordinal value of each character in the ASCII table
+- A 3-byte version header composed by 3 unsigned integers representing the major, minor and patch version of the compiler used to generate the file, respectively. JAPL follows the SemVer standard for versioning
 - A string representing the branch name of the git repo from which JAPL was compiled, prepended with its size represented as a single 8-bit unsigned integer. Due to this encoding the branch name can't be longer than 256 characters, which is a length deemed appropriate for this purpose
 - A 40 bytes hexadecimal string, pinpointing the version of the compiler down to the exact commit hash in the JAPL repository, particularly useful when testing development versions
 - An 8 byte (64 bit) UNIX timestamp (starting from the Unix Epoch of January 1st 1970 at 00:00), representing the date and time when the file was created
@@ -91,8 +92,7 @@ An object file starts with the headers, namely:
 
 ### Constant section
 
-This section of the file follows the headers and is meant to store all constants needed upon startup by the JAPL virtual machine. For example, the code `var x = 1;` would have the number one as a constant. Constants are just an ordered sequence of compile-time types as described in the sections above. The constant section's end is marked with
-the byte `0x59`.
+This section of the file follows the headers and is meant to store all constants needed upon startup by the JAPL virtual machine. For example, the code `var x = 1;` would have the number one as a constant. Constants are just an ordered sequence of compile-time types as described in the sections above. The constant section's end is marked with the byte `0x59`.
 
 ### Code section
 

src/backend/types/arrayList.nim

@@ -11,9 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 # Implementation of a custom list data type for JAPL objects (used also internally by the VM)
 
-{.experimental: "implicitDeref".}
 import iterable
 import ../../memory/allocator
 import baseObject

src/backend/types/baseObject.nim

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+## The base JAPL object
+
 import ../../memory/allocator
 
 
@@ -28,7 +30,8 @@ type
         ## The base object for all
         ## JAPL types. Every object
         ## in JAPL implicitly inherits
-        ## from this base type
+        ## from this base type and extends
+        ## its functionality
         kind*: ObjectType
         hashValue*: uint64
 
@@ -36,7 +39,7 @@ type
 ## Object constructors and allocators
 
 proc allocateObject*(size: int, kind: ObjectType): ptr Obj =
-    ## Wrapper around memory.reallocate to create a new generic JAPL object
+    ## Wrapper around reallocate() to create a new generic JAPL object
     result = cast[ptr Obj](reallocate(nil, 0, size))
     result.kind = kind
 
@@ -50,13 +53,32 @@ template allocateObj*(kind: untyped, objType: ObjectType): untyped =
 proc newObj*: ptr Obj =
     ## Allocates a generic JAPL object
     result = allocateObj(Obj, ObjectType.BaseObject)
+    result.hashValue = 0x123FFFF
 
 
-proc asObj*(self: ptr Obj): ptr Obj = 
-    ## Casts a specific JAPL object into a generic
-    ## pointer to Obj
-    result = cast[ptr Obj](self)
+## Default object methods implementations
 
+# In JAPL code, this method will be called
+# stringify()
+proc `$`*(self: ptr Obj): string = "<object>"
+proc stringify*(self: ptr Obj): string = $self
 
-proc hash*(self: ptr Obj): uint64 = 0x123FFFF  # Constant hash value
-proc `$`*(self: ptr Obj): string = "<object>"
+proc hash*(self: ptr Obj): int64 = 0x123FFAA  # Constant hash value
+# I could've used mul, sub and div, but "div" is a reserved
+# keyword and using `div` looks ugly. So to keep everything
+# consistent I just made all names long
+proc multiply*(self, other: ptr Obj): ptr Obj = nil
+proc sum*(self, other: ptr Obj): ptr Obj = nil
+proc divide*(self, other: ptr Obj): ptr Obj = nil
+proc subtract*(self, other: ptr Obj): ptr Obj = nil
+# Returns 0 if self == other, a negative number if self < other
+# and a positive number if self > other. This is a convenience
+# method to implement all basic comparison operators in one
+# method 
+proc compare*(self, other: ptr Obj): ptr Obj = nil
+# Specific methods for each comparison
+proc equalTo*(self, other: ptr Obj): ptr Obj = nil
+proc greaterThan*(self, other: ptr Obj): ptr Obj = nil
+proc lessThan*(self, other: ptr Obj): ptr Obj = nil
+proc greaterOrEqual*(self, other: ptr Obj): ptr Obj = nil
+proc lessOrEqual*(self, other: ptr Obj): ptr Obj = nil

src/backend/types/dispatch.nim

@@ -0,0 +1,48 @@
+# Copyright 2022 Mattia Giambirtone & All Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Type dispatching module
+import baseObject
+import intObject
+import floatObject
+
+
+proc dispatch*(obj: ptr Obj, p: proc (self: ptr Obj): ptr Obj): ptr Obj =
+    ## Dispatches a given one-argument procedure according to 
+    ## the provided object's runtime type and returns its result
+    case obj.kind:
+        of BaseObject:
+            result = p(obj)
+        of ObjectType.Float:
+            result = p(cast[ptr Float](obj))
+        of ObjectType.Integer:
+            result = p(cast[ptr Integer](obj))
+        else:
+            discard
+
+
+proc dispatch*(a, b: ptr Obj, p: proc (self: ptr Obj, other: ptr Obj): ptr Obj): ptr Obj =
+    ## Dispatches a given two-argument procedure according to 
+    ## the provided object's runtime type and returns its result
+    case a.kind:
+        of BaseObject:
+            result = p(a, b)
+        of ObjectType.Float:
+            # Further type casting for b is expected to occur later
+            # in the given procedure
+            result = p(cast[ptr Float](a), b)
+        of ObjectType.Integer:
+            result = p(cast[ptr Integer](a), b)
+        else:
+            discard

src/backend/types/floatObject.nim

@@ -0,0 +1,49 @@
+# Copyright 2022 Mattia Giambirtone & All Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Implementation of integer types
+
+import baseObject
+import lenientops
+
+
+type Float* = object of Obj
+    value: float64
+
+
+proc newFloat*(value: float): ptr Float =
+    ## Initializes a new JAPL
+    ## float object from
+    ## a machine native float
+    result = allocateObj(Float, ObjectType.Float)
+    result.value = value
+
+
+proc toNativeFloat*(self: ptr Float): float =
+    ## Returns the float's machine
+    ## native underlying value
+    result = self.value
+
+
+proc `$`*(self: ptr Float): string = $self.value
+
+
+proc hash*(self: ptr Float): int64 =
+    ## Implements hashing
+    ## for the given float
+    if self.value - int(self.value) == self.value:
+        result = int(self.value)
+    else:
+        result = 2166136261 xor int(self.value)   # TODO: Improve this
+        result *= 16777619

src/backend/types/intObject.nim

@@ -0,0 +1,40 @@
+# Copyright 2022 Mattia Giambirtone & All Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Implementation of integer types
+
+import baseObject
+
+
+type Integer* = object of Obj
+    value: int64
+
+
+proc newInteger*(value: int64): ptr Integer =
+    ## Initializes a new JAPL
+    ## integer object from
+    ## a machine native integer
+    result = allocateObj(Integer, ObjectType.Integer)
+    result.value = value
+
+
+proc toNativeInteger*(self: ptr Integer): int64 =
+    ## Returns the integer's machine
+    ## native underlying value
+    result = self.value
+
+
+proc `$`*(self: ptr Integer): string = $self.value 
+proc hash*(self: ptr Integer): int64 = self.value
+

src/backend/types/stringObject.nim

@@ -0,0 +1,15 @@
+# Copyright 2022 Mattia Giambirtone & All Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# JAPL string implementations

src/backend/vm.nim

@@ -0,0 +1,20 @@
+# Copyright 2022 Mattia Giambirtone & All Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## The JAPL runtime environment
+
+
+type 
+    VM* = ref object
+        stack: 

src/frontend/compiler.nim

@@ -233,7 +233,7 @@ proc patchJump(self: Compiler, offset: int) =
     ## offset and changes the bytecode instruction if possible
     ## (i.e. jump is in 16 bit range), but the converse is also
     ## true (i.e. it might change a regular jump into a long one)
-    let jump: int = self.chunk.code.len() - offset - 4
+    let jump: int = self.chunk.code.len() - offset
     if jump > 16777215:
         self.error("cannot jump more than 16777215 bytecode instructions")
     if jump < uint16.high().int:
@@ -473,6 +473,9 @@ proc binary(self: Compiler, node: BinaryExpr) =
             self.error(&"invalid AST node of kind {node.kind} at binary(): {node} (This is an internal error and most likely a bug)")
 
 
+proc identifier(self: Compiler, node: IdentExpr)
+
+
 proc declareName(self: Compiler, node: ASTNode) =
     ## Compiles all name declarations (constants, static,
     ## and dynamic)
@@ -491,11 +494,23 @@ proc declareName(self: Compiler, node: ASTNode) =
                     # slap myself 100 times with a sign saying "I'm dumb". Mark my words
                     self.error("cannot declare more than 16777215 static variables at a time")
                 self.names.add(Name(depth: self.scopeDepth, name: IdentExpr(node.name),
-                                                isPrivate: node.isPrivate,
+                                                        isPrivate: node.isPrivate,
                                                         owner: node.owner,
                                                         isConst: node.isConst))
+        of funDecl:
+            var node = FunDecl(node)
+            # Declares the function's name in the
+            # current (outer) scope...
+            self.declareName(node.name)
+            # ... but its arguments in an inner one!
+            # (this ugly part is needed because
+            # self.blockStmt() already increments
+            # and decrements the scope depth)
+            for argument in node.arguments:
+                self.names.add(Name(depth: self.scopeDepth + 1, isPrivate: true, owner: "", isConst: false, name: IdentExpr(argument)))
+            # TODO: Default arguments and unpacking
         else:
-            discard # TODO: Classes, functions
+            discard  # TODO: Classes
 
 
 proc varDecl(self: Compiler, node: VarDecl) =
@@ -660,10 +675,10 @@ proc endScope(self: Compiler) =
         # emit another batch of plain ol' Pop instructions for the rest
         if popped <= uint16.high().int():
             self.emitByte(PopN)
-            self.emitBytes(popped.toTriple())
+            self.emitBytes(popped.toDouble())
         else:
             self.emitByte(PopN)
-            self.emitBytes(uint16.high().int.toTriple())
+            self.emitBytes(uint16.high().int.toDouble())
             for i in countdown(self.names.high(), popped - uint16.high().int()):
                 if self.names[i].depth > self.scopeDepth:
                     self.emitByte(Pop)
@@ -928,22 +943,11 @@ proc funDecl(self: Compiler, node: FunDecl) =
     # We store the current function
     var function = self.currentFunction
     self.currentFunction = node
-    # Declares the function's name in the
-    # outer scope...
-    self.declareName(node.name)
-    self.scopeDepth += 1
-    # ... but its arguments in an inner one!
-    # (this ugly part is needed because
-    # self.blockStmt() already increments
-    # and decrements the scope depth)
-    for argument in node.arguments:
-        self.declareName(IdentExpr(argument))
-    self.scopeDepth -= 1
-    # TODO: Default arguments
-
     # A function's code is just compiled linearly
     # and then jumped over
     let jmp = self.emitJump(JumpForwards)
+    echo jmp
+    self.declareName(node)
 
     # Since the deferred array is a linear 
     # sequence of instructions and we want
@@ -969,21 +973,24 @@ proc funDecl(self: Compiler, node: FunDecl) =
     # that's about it
     
     # All functions implicitly return nil. This code
-    # will not execute if there's an explicit return
-    # and I cannot figure out an elegant and simple
-    # way to tell if a function already returns
-    # or not, so we just play it safe
+    # will not be executed by the VM in all but the simplest
+    # cases where there is an explicit return statement, but
+    # I cannot figure out an elegant and simple way to tell
+    # if a function already returned or not, so we play it safe
 
     if not self.enableOptimizations:
-        self.emitBytes(OpCode.Nil, OpCode.Return)
+        if OpCode(self.chunk.code[^1]) != OpCode.Return:
+            self.emitBytes(OpCode.Nil, OpCode.Return)
     else:
-        self.emitBytes(ImplicitReturn)
+        if OpCode(self.chunk.code[^1]) != OpCode.Return:
+            self.emitByte(ImplicitReturn)
 
     # Currently defer is not functional so we
     # just pop the instructions
     for i in countup(deferStart, self.deferred.len(), 1):
         self.deferred.delete(i)
-    
+
+    echo self.chunk.code.len() - jmp
     self.patchJump(jmp)
     # This makes us compile nested functions correctly
     self.currentFunction = function

src/frontend/parser.nim

@@ -724,10 +724,10 @@ proc tryStmt(self: Parser): ASTNode =
         handlerBody = self.statement()
         handlers.add((body: handlerBody, exc: excName, name: asName))
         asName = nil
-    if self.match(Finally):
-        finallyClause = self.statement()
     if self.match(Else):
         elseClause = self.statement()
+    if self.match(Finally):
+        finallyClause = self.statement()
     if handlers.len() == 0 and elseClause == nil and finallyClause == nil:
         self.error("expecting 'except', 'finally' or 'else' statements after 'try' block")
     for i, handler in handlers:
@@ -765,7 +765,8 @@ proc forStmt(self: Parser): ASTNode =
     # increment variable which doesn't really make sense, but still
     # allow people that like verbosity (for *some* reason) to use
     # private static var declarations as well as just private var 
-    # and static var
+    # and static var as well as providing decently specific error
+    # messages
     if self.match(Semicolon):
         discard
     elif self.match(Dynamic):
@@ -808,6 +809,19 @@ proc forStmt(self: Parser): ASTNode =
         # Nested blocks, so the initializer is
         # only executed once
         body = newBlockStmt(@[initializer, body], tok)
+    # This desgugars the following code:
+    # for (var i = 0; i < 10; i += 1) {
+    #     print(i);
+    # }
+    # To the semantically equivalent snippet
+    # below:
+    # {
+    #     private static var i = 0;
+    #     while (i < 10) {
+    #         print(i);
+    #         i += 1;
+    #     }
+    # }
     result = body
     self.currentLoop = enclosingLoop
     self.endScope()
@@ -826,7 +840,7 @@ proc ifStmt(self: Parser): ASTNode =
     result = newIfStmt(condition, thenBranch, elseBranch, tok)
 
 
-proc checkDecl(self: Parser, isStatic, isPrivate: bool) =
+template checkDecl(self: Parser, isStatic, isPrivate: bool) =
     ## Handy utility function that avoids us from copy
     ## pasting the same checks to all declaration handlers
     if not isStatic and self.currentFunction != nil:
@@ -911,8 +925,10 @@ proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPriva
         self.expect(LeftBrace)
     if not isLambda:
         FunDecl(self.currentFunction).body = self.blockStmt()
+        FunDecl(self.currentFunction).arguments = arguments
     else:
         LambdaExpr(self.currentFunction).body = self.blockStmt()
+        LambdaExpr(self.currentFunction).arguments = arguments
     result = self.currentFunction
     self.currentFunction = enclosingFunction
 

src/frontend/serializer.nim

@@ -122,23 +122,23 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
                 stream.extend(self.toBytes(constant.token.lexeme))
             of strExpr:
                 stream.add(0x2)
-                var temp: seq[byte] = @[]
+                var temp: byte
                 var strip: int = 2
                 var offset: int = 1
                 case constant.token.lexeme[0]:
                     of 'f':
                         strip = 3
                         inc(offset)
-                        temp.add(0x2)
+                        temp = 0x2
                     of 'b':
                         strip = 3
                         inc(offset)
-                        temp.add(0x1)
+                        temp = 0x1
                     else:
                         strip = 2
-                        temp.add(0x0)
+                        temp = 0x0
                 stream.extend((len(constant.token.lexeme) - strip).toTriple())  # Removes the quotes from the length count as they're not written
-                stream.extend(temp)
+                stream.add(temp)
                 stream.add(self.toBytes(constant.token.lexeme[offset..^2]))
             of identExpr:
                 stream.add(0x0)

src/main.nim

@@ -31,25 +31,26 @@ import jale/multiline
 import config
 
 
-import strformat
-import strutils
-import sequtils
-import times
-import nimSHA2
-
 const debugLexer = false
-const debugParser = false
+const debugParser = true
 const debugOptimizer = false
 const debugCompiler = true
-const debugSerializer = true
+const debugSerializer = false
+
+import strformat
+import strutils
+when debugSerializer:
+    import sequtils
+    import times
+    import nimSHA2
 
 
 proc getLineEditor: LineEditor =
     result = newLineEditor()
     result.prompt = "=> "
-    result.populateDefaults()  # setup default keybindings
-    let hist = result.plugHistory()  # create history object
-    result.bindHistory(hist)  # set default history keybindings
+    result.populateDefaults()        # Setup default keybindings
+    let hist = result.plugHistory()  # Create history object
+    result.bindHistory(hist)         # Set default history keybindings
 
 
 proc main =
@@ -59,15 +60,17 @@ proc main =
     var tree: seq[ASTNode]
     var optimized: tuple[tree: seq[ASTNode], warnings: seq[Warning]]
     var compiled: Chunk
-    var serialized: Serialized
-    var serializedRaw: seq[byte]
+    when debugSerializer:
+        var serialized: Serialized
+        var serializedRaw: seq[byte]
     var keep = true
 
     var lexer = initLexer()
     var parser = initParser()
     var optimizer = initOptimizer(foldConstants=false)
     var compiler = initCompiler()
-    var serializer = initSerializer()
+    when debugSerializer:
+        var serializer = initSerializer()
     let lineEditor = getLineEditor()
     lineEditor.bindEvent(jeQuit):
         keep = false

src/memory/allocator.nim

@@ -22,21 +22,21 @@ when DEBUG_TRACE_ALLOCATION:
     import strformat
 
 
-proc reallocate*(pointr: pointer, oldSize: int, newSize: int): pointer =
+proc reallocate*(p: pointer, oldSize: int, newSize: int): pointer =
     ## Wrapper around realloc/dealloc
     try:
-        if newSize == 0 and pointr != nil:   # pointr is awful, but clashing with builtins is even more awful
+        if newSize == 0 and p != nil:
             when DEBUG_TRACE_ALLOCATION:
                 if oldSize > 1:
                     echo &"DEBUG - Memory manager: Deallocating {oldSize} bytes"
                 else:
                     echo "DEBUG - Memory manager: Deallocating 1 byte"
-            dealloc(pointr)
+            dealloc(p)
             return nil
         when DEBUG_TRACE_ALLOCATION:
             if pointr == nil and newSize == 0:
                 echo &"DEBUG - Memory manager: Warning, asked to dealloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request"
-        if oldSize > 0 and pointr != nil or oldSize == 0:
+        if oldSize > 0 and p != nil or oldSize == 0:
             when DEBUG_TRACE_ALLOCATION:
                 if oldSize == 0:
                     if newSize > 1:
@@ -45,7 +45,7 @@ proc reallocate*(pointr: pointer, oldSize: int, newSize: int): pointer =
                         echo "DEBUG - Memory manager: Allocating 1 byte of memory"
                 else:
                     echo &"DEBUG - Memory manager: Resizing {oldSize} bytes of memory to {newSize} bytes"
-            result = realloc(pointr, newSize)
+            result = realloc(p, newSize)
         when DEBUG_TRACE_ALLOCATION:
             if oldSize > 0 and pointr == nil:
                 echo &"DEBUG - Memory manager: Warning, asked to realloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request"