Merge pull request #43 from Productive2/master

Fix for.jpl, testing improvements

build.py

@@ -196,7 +196,7 @@ def build(path: str, flags: Optional[Dict[str, str]] = {}, options: Optional[Dic
             logging.debug("Running tests")
             start = time()
             # TODO: Find a better way of running the test suite
-            process = run_command(f"{tests_path} {'--stdout' if verbose else ''}", mode="run", shell=True, stderr=PIPE)
+            process = run_command(f"{tests_path} {'-e' if verbose else ''}", mode="run", shell=True, stderr=PIPE)
             if status != 0:
                 logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}")
                 return False

resources/profiles/debug_compiler.json

@@ -2,8 +2,7 @@
 // from nim itself (the nim compiler options are identical to those of
 // production.json)
 {"flags": {
-        "gc": "none",
-        "d": "danger"
+        "gc": "none"
         },
  "verbose": true,
  "override_config": true,

resources/profiles/debug_vm.json

@@ -2,8 +2,7 @@
 // from nim itself (the nim compiler options are identical to those of
 // production.json)
 {"flags": {
-        "gc": "none",
-        "d": "danger"
+        "gc": "none"
         },
  "verbose": true,
  "override_config": true,

src/compiler.nim

@@ -36,6 +36,8 @@ import config
 when isMainModule:
     import util/debug
 import types/methods
+when DEBUG_TRACE_COMPILER:
+    import terminal
 
 
 type
@@ -212,7 +214,11 @@ proc emitByte(self: Compiler, byt: OpCode|uint8) =
     ## Emits a single bytecode instruction and writes it
     ## to the current chunk being compiled
     when DEBUG_TRACE_COMPILER:
-        echo "DEBUG - Compiler: Emitting " & $byt & " (uint8 value of " & $(uint8 byt) & ")"
+        write stdout, &"DEBUG - Compiler: Emitting {$byt} (uint8 value of  {$(uint8 byt)}"
+        if byt.int() <= OpCode.high().int():
+          write stdout, &"; opcode value of {$byt.OpCode}"
+        write stdout, ")\n"
+          
     self.currentChunk.writeChunk(uint8 byt, self.parser.previous.line)
 
 
@@ -652,6 +658,10 @@ proc emitJump(self: Compiler, opcode: OpCode): int =
     self.emitByte(opcode)
     self.emitByte(0xff)
     self.emitByte(0xff)
+    when DEBUG_TRACE_COMPILER:
+        setForegroundColor(fgYellow)
+        write stdout, &"DEBUG - Compiler: emit jump @ {self.currentChunk.code.len-2}\n"
+        setForegroundColor(fgDefault)
     return self.currentChunk.code.len - 2
 
 
@@ -669,14 +679,23 @@ proc patchJump(self: Compiler, offset: int) =
     ## be jumped over, so the size of the if/else conditions
     ## or loops is limited (hopefully 65 thousands and change
     ## instructions are enough for everyone)
+
+    when DEBUG_TRACE_COMPILER:
+        setForegroundColor(fgYellow)
+        write stdout, &"DEBUG - Compiler: patching jump @ {offset}"
     let jump = self.currentChunk.code.len - offset - 2
     if jump > (int uint16.high):
+        when DEBUG_TRACE_COMPILER:
+            setForegroundColor(fgDefault)
+            write stdout, "\n"
         self.compileError("too much code to jump over")
     else:
         let casted = toDouble(jump)
         self.currentChunk.code[offset] = casted[0]
         self.currentChunk.code[offset + 1] = casted[1]
-
+        when DEBUG_TRACE_COMPILER:
+            write stdout, &" points to {casted[0]}, {casted[1]} = {jump}\n"
+            setForegroundColor(fgDefault)
 
 proc ifStatement(self: Compiler) =
     ## Parses if statements in a C-style fashion
@@ -703,14 +722,22 @@ proc ifStatement(self: Compiler) =
 
 proc emitLoop(self: Compiler, start: int) =
     ## Creates a loop and emits related instructions.
+    when DEBUG_TRACE_COMPILER:
+        setForegroundColor(fgYellow)
+        write stdout, &"DEBUG - Compiler: emitting loop at start {start} "
     self.emitByte(OpCode.Loop)
     var offset = self.currentChunk.code.len - start + 2
     if offset > (int uint16.high):
+        when DEBUG_TRACE_COMPILER:
+            setForegroundColor(fgDefault)
+            write stdout, "\n"
         self.compileError("loop body is too large")
     else:
         let offsetBytes = toDouble(offset)
         self.emitByte(offsetBytes[0])
         self.emitByte(offsetBytes[1])
+        when DEBUG_TRACE_COMPILER:
+            write stdout, &"pointing to {offsetBytes[0]}, {offsetBytes[1]} = {offset}\n"
 
 
 proc endLooping(self: Compiler) =
@@ -720,14 +747,15 @@ proc endLooping(self: Compiler) =
     if self.loop.loopEnd != -1:
         self.patchJump(self.loop.loopEnd)
         self.emitByte(OpCode.Pop)
-    var i = self.loop.body
-    while i < self.currentChunk.code.len:
-        if self.currentChunk.code[i] == uint OpCode.Break:
-            self.currentChunk.code[i] = uint8 OpCode.Jump
-            self.patchJump(i + 1)
-            i += 3
-        else:
-            i += 1
+
+    for brk in self.loop.breaks:
+        when DEBUG_TRACE_COMPILER:
+            setForegroundColor(fgYellow)
+            write stdout, &"DEBUG - Compiler: patching break at {brk}\n"
+            setForegroundColor(fgDefault)
+        self.currentChunk.code[brk] = OpCode.Jump.uint8
+        self.patchJump(brk + 1)
+        
     self.loop = self.loop.outer
 
 
@@ -816,6 +844,7 @@ proc parseBreak(self: Compiler) =
             self.emitByte(OpCode.Pop)
             i -= 1
         discard self.emitJump(OpCode.Break)
+        self.loop.breaks.add(self.currentChunk.code.len() - 3)
 
 
 proc parseAnd(self: Compiler, canAssign: bool) =

src/meta/frame.nim

@@ -37,7 +37,7 @@ proc clear*(self: CallFrame): int =
         inc result
 
 proc getView*(self: CallFrame): ptr ArrayList[ptr Obj] =
-    result = self.stack[self.slot..self.stack.high()]
+    result = self.stack[self.slot..self.stack.len()]
 
 
 proc len*(self: CallFrame): int =

src/meta/looptype.nim

@@ -21,3 +21,4 @@ type Loop* = ref object
     alive*: bool
     body*: int
     loopEnd*: int
+    breaks*: seq[int]

src/meta/opcode.nim

@@ -83,7 +83,9 @@ const simpleInstructions* = {OpCode.Return, OpCode.Add, OpCode.Multiply,
                              OpCode.Xor, OpCode.Not, OpCode.Equal,
                              OpCode.Greater, OpCode.Less, OpCode.GetItem,
                              OpCode.Slice, OpCode.Pop, OpCode.Negate,
-                             OpCode.Is, OpCode.As, GreaterOrEqual, LessOrEqual}
+                             OpCode.Is, OpCode.As, OpCode.GreaterOrEqual,
+                             OpCode.LessOrEqual, OpCode.Bor, OpCode.Band,
+                             OpCode.Bnot}
 const constantInstructions* = {OpCode.Constant, OpCode.DefineGlobal,
                                OpCode.GetGlobal, OpCode.SetGlobal,
                                OpCode.DeleteGlobal}

src/types/arraylist.nim

@@ -84,7 +84,7 @@ proc `[]`*[T](self: ptr ArrayList[T], slice: Hslice[int, int]): ptr ArrayList[T]
     ## of the slice
     if self.length == 0:
         raise newException(IndexDefect, "ArrayList index out of bounds")
-    if slice.a notin 0..self.length - 1 or slice.b notin 0..self.length - 1:
+    if slice.a notin 0..self.length - 1 or slice.b notin 0..self.length:
         raise newException(IndexDefect, "ArrayList index out of bounds")
     result = newArrayList[T]()
     for i in countup(slice.a, slice.b - 1):
@@ -189,4 +189,4 @@ proc `$`*[T](self: ptr ArrayList[T]): string =
 proc getIter*[T](self: ptr ArrayList[T]): Iterator = 
     ## Returns the iterator object of the
     ## arraylist
-    result = allocate(Iterator, )
+    result = allocate(Iterator, )

src/util/debug.nim

@@ -20,17 +20,30 @@ import ../types/baseObject
 import ../types/methods
 import ../types/arraylist
 import strformat
+import terminal
+import ../multibyte
 
+proc printName(name: string) =
+    setForegroundColor(fgGreen)
+    write stdout, name
+    setForegroundColor(fgDefault)
 
+proc nl =
+    write stdout, "\n"
 
 proc simpleInstruction(name: string, index: int): int =
-    echo &"DEBUG - VM:\tInstruction -> {name}"
+    write stdout, &"DEBUG - VM:\tInstruction -> "
+    printName(name)
+    nl()
     return index + 1
 
 
 proc byteInstruction(name: string, chunk: Chunk, offset: int): int =
     var slot = chunk.code[offset + 1]
-    echo &"DEBUG - VM:\tInstruction -> {name}, points to slot {slot}"
+    write stdout, &"DEBUG - VM:\tInstruction -> "
+    printName(name)
+    write stdout, &", points to slot {slot}"
+    nl()
     return offset + 2
 
 
@@ -39,7 +52,10 @@ proc constantInstruction(name: string, chunk: Chunk, offset: int): int =
     var constantArray: array[3, uint8] = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]]
     var constant: int
     copyMem(constant.addr, constantArray.addr, sizeof(constantArray))
-    echo &"DEBUG - VM:\tInstruction -> {name}, points to slot {constant}"
+    write stdout, &"DEBUG - VM:\tInstruction -> "
+    printName(name)
+    write stdout, &", points to slot {constant}"
+    nl()
     let obj = chunk.consts[constant]
     echo &"DEBUG - VM:\tOperand -> {stringify(obj)}\nDEBUG - VM:\tValue kind -> {obj.kind}"
     return offset + 4
@@ -47,9 +63,10 @@ proc constantInstruction(name: string, chunk: Chunk, offset: int): int =
 
 proc jumpInstruction(name: string, chunk: Chunk, offset: int): int =
     var jumpArray: array[2, uint8] = [chunk.code[offset + 1], chunk.code[offset + 2]]
-    var jump: int
-    copyMem(jump.addr, jumpArray.addr, sizeof(uint16))
-    echo &"DEBUG - VM:\tInstruction -> {name}\nDEBUG - VM:\tJump size -> {jump}"
+    write stdout, &"DEBUG - VM:\tInstruction -> "
+    printName(name)
+    write stdout, &"\nDEBUG - VM:\tJump size -> {jumpArray.fromDouble()} ( = {$jumpArray[0]}, {$jumpArray[1]})"
+    nl()
     return offset + 3
 
 

src/vm.nim

@@ -38,12 +38,16 @@ import types/typeutils
 import types/function
 import types/native
 import types/arraylist
+import multibyte
 # We always import it to
 # avoid the compiler complaining
 # about functions not existing
 # in production builds
 import util/debug
 
+when DEBUG_TRACE_VM:
+  import terminal
+
 
 type
     KeyboardInterrupt* = object of CatchableError
@@ -278,9 +282,7 @@ proc readBytes(self: CallFrame): int =
 proc readShort(self: CallFrame): uint16 =
     ## Reads a 16 bit number from the
     ## given frame's chunk
-    let arr = [self.readByte(), self.readByte()]
-    copyMem(result.addr, unsafeAddr(arr), sizeof(uint16))
-
+    fromDouble([self.readByte(), self.readByte()])
 
 proc readConstant(self: CallFrame): ptr Obj =
     ## Reads a constant from the given
@@ -292,44 +294,49 @@ proc readConstant(self: CallFrame): ptr Obj =
 
 
 
-proc showRuntime*(self: VM, frame: CallFrame, iteration: uint64) = 
-    ## Shows debug information about the current
-    ## state of the virtual machine
-    let view = frame.getView()
-    stdout.write("DEBUG - VM: General information\n")
-    stdout.write(&"DEBUG - VM:\tIteration -> {iteration}\nDEBUG - VM:\tStack -> [")
-    for i, v in self.stack:
-        stdout.write(stringify(v))
-        if i < self.stack.high():
-            stdout.write(", ")
-    stdout.write("]\nDEBUG - VM: \tGlobals -> {")
-    for i, (k, v) in enumerate(self.globals.pairs()):
-        stdout.write(&"'{k}': {stringify(v)}")
-        if i < self.globals.len() - 1:
-            stdout.write(", ")
-    stdout.write("}\nDEBUG - VM: Frame information\n")
-    stdout.write("DEBUG - VM:\tType -> ")
-    if frame.function.name == nil:
-        stdout.write("main\n")
-    else:
-        stdout.write(&"function, '{frame.function.name.stringify()}'\n")
-    echo &"DEBUG - VM:\tCount -> {self.frames.len()}"
-    echo &"DEBUG - VM:\tLength -> {view.len}"
-    stdout.write("DEBUG - VM:\tTable -> ")
-    stdout.write("[")
-    for i, e in frame.function.chunk.consts:
-        stdout.write(stringify(e))
-        if i < len(frame.function.chunk.consts) - 1:
-            stdout.write(", ")
-    stdout.write("]\nDEBUG - VM:\tStack view -> ")
-    stdout.write("[")
-    for i, e in view:
-        stdout.write(stringify(e))
-        if i < len(view) - 1:
-            stdout.write(", ")
-    stdout.write("]\n")
-    echo "DEBUG - VM: Current instruction"
-    discard disassembleInstruction(frame.function.chunk, frame.ip - 1)
+when DEBUG_TRACE_VM:
+    proc showRuntime*(self: VM, frame: CallFrame, iteration: uint64) = 
+        ## Shows debug information about the current
+        ## state of the virtual machine
+
+        let view = frame.getView()
+        setForegroundColor(fgYellow)
+        stdout.write("DEBUG - VM: General information\n")
+        stdout.write(&"DEBUG - VM:\tIteration -> {iteration}\n")
+        setForegroundColor(fgDefault)
+        stdout.write("DEBUG - VM:\tStack -> [")
+        for i, v in self.stack:
+            stdout.write(stringify(v))
+            if i < self.stack.high():
+                stdout.write(", ")
+        stdout.write("]\nDEBUG - VM: \tGlobals -> {")
+        for i, (k, v) in enumerate(self.globals.pairs()):
+            stdout.write(&"'{k}': {stringify(v)}")
+            if i < self.globals.len() - 1:
+                stdout.write(", ")
+        stdout.write("}\nDEBUG - VM: Frame information\n")
+        stdout.write("DEBUG - VM:\tType -> ")
+        if frame.function.name == nil:
+            stdout.write("main\n")
+        else:
+            stdout.write(&"function, '{frame.function.name.stringify()}'\n")
+        echo &"DEBUG - VM:\tCount -> {self.frames.len()}"
+        echo &"DEBUG - VM:\tLength -> {view.len}"
+        stdout.write("DEBUG - VM:\tTable -> ")
+        stdout.write("[")
+        for i, e in frame.function.chunk.consts:
+            stdout.write(stringify(e))
+            if i < len(frame.function.chunk.consts) - 1:
+                stdout.write(", ")
+        stdout.write("]\nDEBUG - VM:\tStack view -> ")
+        stdout.write("[")
+        for i, e in view:
+            stdout.write(stringify(e))
+            if i < len(view) - 1:
+                stdout.write(", ")
+        stdout.write("]\n")
+        echo "DEBUG - VM: Current instruction"
+        discard disassembleInstruction(frame.function.chunk, frame.ip - 1)
 
 
 proc run(self: VM): InterpretResult =

tests/japl/all.jpl

@@ -1,6 +1,8 @@
+/*
 [Test: all]
 [skip]
-[source: mixed]
+*/
+//[source: mixed]
 // Example file to test JAPL's syntax
 
 // Mathematical expressions
@@ -140,5 +142,5 @@ mark.greet();
 "implicit start"[:5]; // From 0 to 5
 "hello" + " world"; // Strings are immutable!
 "hello" * 3; //hellohellohello
-[end]
-[end]
+//[end]
+//[end]

tests/japl/arithmetic.jpl

@@ -1,5 +1,5 @@
-[Test: arithmetic]
-[source: mixed]
+//[Test: arithmetic]
+//[source: mixed]
 //int arithmetic
 
 print(7+5); //stdout:12
@@ -34,5 +34,5 @@ print(64/-64);//stdout:-1.0
 print(8/0);//stdout:inf
 print(8/-0);//stdout:inf
 print(-8/0);//stdout:-inf
-[end]
-[end]
+//[end]
+//[end]

tests/japl/assignexpr.jpl

@@ -0,0 +1,8 @@
+//[Test: assignment expressions]
+//[source: mixed]
+var x;
+var a = x = 5;
+print(x);//stdout:5
+print(a);//stdout:5
+//[end]
+//[end]

tests/japl/bitwise.jpl

@@ -1,5 +1,5 @@
-[Test: bitwise]
-[source: mixed]
+//[Test: bitwise]
+//[source: mixed]
 print(~5 | 5);//stdout:-1
 print(1 | 2);//stdout:3
 print(1 & 2);//stdout:0
@@ -9,5 +9,5 @@ print(32 | 64);//stdout:96
 print(96 & 32);//stdout:32
 print(~0);//stdout:-1
 print(~356);//stdout:-357
-[end]
-[end]
+//[end]
+//[end]

tests/japl/booleans.jpl

@@ -1,5 +1,5 @@
-[Test: booleans]
-[source: mixed]
+//[Test: booleans]
+//[source: mixed]
 print(2 or 3);//stdout:2
 print(2 and 3);//stdout:3
 print(false or true);//stdout:true
@@ -21,5 +21,5 @@ print(not false);//stdout:true
 print(not 1);//stdout:false
 print(not 1 and not 2);//stdout:false
 print(not (1 and false));//stdout:true
-[end]
-[end]
+//[end]
+//[end]

tests/japl/break.jpl

@@ -0,0 +1,23 @@
+//[Test: breaks]
+//[source:mixed]
+var x = 5;
+while (true) {
+  var a = 1;
+  x = x - a;
+  if (x < 0) {
+    break;
+  }
+  print(x);
+}
+//[end]
+/*
+[stdout]
+4
+3
+2
+1
+0
+[end]
+[end]
+*/
+

tests/japl/callchain.jpl

@@ -1,5 +1,7 @@
+/*
 [Test: callchain]
-[source: mixed]
+*/
+//[source: mixed]
 fun add2(x)
 {
     return x + 2;
@@ -14,8 +16,10 @@ fun mul2(x)
 }
 
 print(add2(sub2(mul2(sub2(5)))));
-//5-2=3
-//3*2=6
-//stdout:6
+//[end]
+/*
+[stdout]
+6
 [end]
 [end]
+*/

tests/japl/comparisons.jpl

@@ -1,5 +1,5 @@
-[Test: comparisons]
-[source: mixed]
+//[Test: comparisons]
+//[source: mixed]
 var x = 4;
 var y = 5;
 var z = 6;
@@ -42,5 +42,25 @@ if (8 <= z)
     print("15");
 else
     print("16");//stdout:16
-[end]
-[end]
+//[end]
+//[end]
+
+//[Test: multicomparisons]
+//[skip]
+//[source: mixed]
+var x = 2 < 3 < 4;
+print(x);//stdout:4
+
+x = 1 > 3 < 5;
+print(x);//stdout:false
+
+x = 1 >= 0 >= 0;
+print(x);//stdout:0
+
+if ( 4 < x < 6 ) {
+  print("inside");
+} else {
+  print("outside");//stdout:outside
+}
+//[end]
+//[end]

tests/japl/compile_time_interning.jpl

@@ -1,5 +1,5 @@
-[Test: compile_time_intern]
-[source: mixed]
+//[Test: compile_time_intern]
+//[source: mixed]
 //compile time interning
 
 var a = "hello";
@@ -11,6 +11,6 @@ print(a is b);//stdout:true
 var x = "ex";
 var y = "ey";
 print(x is y);//stdout:false
-[end]
-[end]
+//[end]
+//[end]
 

tests/japl/defaults.jpl

@@ -0,0 +1,11 @@
+//[Test: defaults]
+//[skip]
+//[source:mixed]
+var a = 3;
+fun test(b = a) {
+  print (b);
+}
+a = 9;
+test();//stdout:3
+//[end]
+//[end]

tests/japl/errors/read_in_own_init.jpl

@@ -1,8 +1,8 @@
-[Test: read_in_own_init_regex]
-[source: raw]
+//[Test: read_in_own_init_regex]
+//[source: raw]
 var a = 1; { var a = a; }
 [end]
 [stderr: re]
 [[^\-]*-> cannot read local variable in its own initializer
-[end]
-[end]
+//[end]
+//[end]

tests/japl/errors/undefname.jpl

@@ -1,10 +1,12 @@
-[Test: undefname_raw]
-[source: raw]
+//[Test: undefname_raw]
+//[source: raw]
 var a = b;
-[end]
+//[end]
+/*
 [stderr]
 An unhandled exception occurred, traceback below:
   File '''', line 1, in <module>:
 ReferenceError: undefined name 'b'
 [end]
-[end]
+*/
+//[end]

tests/japl/errors/unsup_binary_intstr.jpl

@@ -1,10 +1,12 @@
-[Test: unsup_binary_instr]
-[source: raw]
+//[Test: unsup_binary_instr]
+//[source: raw]
 var a = 2 + "hey";
-[end]
+//[end]
+/*
 [stderr]
 An unhandled exception occurred, traceback below:
   File '''', line 1, in <module>:
 TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string'
 [end]
-[end]
+*/
+//[end]

tests/japl/euler/problem1.jpl

@@ -1,5 +1,5 @@
-[Test: problem1]
-[source: mixed]
+//[Test: problem1]
+//[source: mixed]
 // Task: find the multiples of 3 and 5 below 1000, find their sum
 
 var sum = 0;
@@ -11,5 +11,5 @@ for (var x = 3; x < 1001; x = x + 1)
     }
 }
 print(sum);//stdout:234168
-[end]
-[end]
+//[end]
+//[end]

tests/japl/euler/problem2.jpl

@@ -1,5 +1,5 @@
-[Test: problem2]
-[source: mixed]
+//[Test: problem2]
+//[source: mixed]
 // Sum of even valued fibonacci numbers that don't exceed 4M
 
 var a = 1;
@@ -16,5 +16,5 @@ while (a < 4000000)
     b = c;
 }
 print(sum);//stdout:4613732
-[end]
-[end]
+//[end]
+//[end]

tests/japl/euler/problem4.jpl

@@ -1,6 +1,6 @@
-[Test: problem4]
-[skip]
-[source: mixed]
+//[Test: problem4]
+//[skip]
+//[source: mixed]
 // Find the largest palindrome that is a product of two 3 digit numbers
 
 fun isPalindrome(n)
@@ -66,5 +66,5 @@ for (var i = 100; i < 1000; i = i + 1)
     }
 }
 print(largest);
-[end]
-[end]
+//[end]
+//[end]

tests/japl/fib.jpl

@@ -1,5 +1,7 @@
-[Test: fibonacci]
-[source: raw]
+
+//[Test: fibonacci]
+
+//[source: raw]
 fun fib(n) {
     if (n < 2)
         return n;
@@ -14,7 +16,8 @@ print(fib(6));
 print(fib(7));
 print(fib(8));
 print(fib(9));
-[end]
+//[end]
+/*
 [stdout]
 1
 1
@@ -26,4 +29,5 @@ print(fib(9));
 21
 34
 [end]
-[end]
+*/
+//[end]

tests/japl/for.jpl

@@ -1,10 +1,10 @@
-[Test: for]
-[source: mixed]
+//[Test: for]
+//[source: mixed]
 for (var x = 0; x < 2; x = x + 1)
 {
     print(x);
     //stdout:0
     //stdout:1
 }
-[end]
-[end]
+//[end]
+//[end]

tests/japl/for_with_function.jpl

@@ -1,6 +1,6 @@
-[Test: forwithfunction]
-[skip]
-[source: mixed]
+//[Test: forwithfunction]
+//[skip]
+//[source: mixed]
 var y = 0; //a global to keep track of state
 //does not need closures for this to work yet
 
@@ -84,5 +84,5 @@ for (var i = 0; i != -1; i = next(i))
 // y = 9
 //stdout:10
 // y = 10
-[end]
-[end]
+//[end]
+//[end]

tests/japl/hellojapl.jpl

@@ -1,15 +1,17 @@
-[Test: hellojapl]
-[source: mixed]
+//[Test: hellojapl]
+//[source: mixed]
 print("Hello, JAPL.");
 //stdout:Hello, JAPL.
-[end]
-[end]
+//[end]
+//[end]
 
-[Test: hello_second_way]
-[source: raw]
+//[Test: hello_second_way]
+//[source: raw]
 print("Hello, JAPL.");
-[end]
+//[end]
+/*
 [stdout]
 Hello, JAPL.
 [end]
-[end]
+*/
+//[end]

tests/japl/if.jpl

@@ -1,5 +1,5 @@
-[Test: if]
-[source: mixed]
+//[Test: if]
+//[source: mixed]
 var x = 5;
 if (x > 2)
 {
@@ -28,5 +28,5 @@ if (2 == x)
     print("2");
 else
     print("not 2");//stdout:not 2
-[end]
-[end]
+//[end]
+//[end]

tests/japl/ifchain.jpl

@@ -1,5 +1,5 @@
-[Test: ifchain]
-[source: mixed]
+//[Test: ifchain]
+//[source: mixed]
 fun printInt(x) {
     if (x == 1)
         print("one");
@@ -22,5 +22,5 @@ x = 7;
 printInt(x);
 x = 1;
 printInt(x);//stdout:one
-[end]
-[end]
+//[end]
+//[end]

tests/japl/input.jpl

@@ -1,8 +1,10 @@
+/*
 [Test: inputtesttwo]
-[source: raw]
+*/
+//[source: raw]
 print(readLine());
-[end]
-
+//[end]
+/*
 [stdin]
 Hello world!
 [end]
@@ -11,3 +13,4 @@ Hello world!
 Hello world!
 [end]
 [end]
+*/

tests/japl/is.jpl

@@ -1,5 +1,5 @@
-[Test: is]
-[source:mixed]
+//[Test: is]
+//[source:mixed]
 var x = 4;
 var y = x;
 
@@ -21,5 +21,5 @@ print((l is z) is l);//stdout:true
 
 var k;
 print(k is nil);//stdout:true
-[end]
-[end]
+//[end]
+//[end]

tests/japl/lambdachain.jpl

@@ -1,21 +0,0 @@
-[Test: lambdachain]
-[source: mixed]
-var add2 = lambda(x)
-{
-    return x + 2;
-};
-var sub2 = lambda(x)
-{
-    return x - 2;
-};
-var mul2 = lambda(x)
-{
-    return x * 2;
-};
-
-print(add2(sub2(mul2(sub2(5)))));
-//5-2=3
-//3*2=6
-//stdout:6
-[end]
-[end]

tests/japl/lambdas.jpl

@@ -0,0 +1,50 @@
+/*
+[Test: lambdachain]
+*/
+//[source: raw]
+var add2 = lambda(x)
+{
+    return x + 2;
+};
+var sub2 = lambda(x)
+{
+    return x - 2;
+};
+var mul2 = lambda(x)
+{
+    return x * 2;
+};
+
+print(add2(sub2(mul2(sub2(5)))));
+//[end]
+/*
+[stdout]
+6
+[end]
+[end]
+[Test: simple lambdas]
+*/
+//[source: raw]
+var identity = lambda(x) { return x; };
+var comparison = lambda(x, y) { 
+  if (x > y) {
+    return x;
+  } else {
+    return y;
+  }
+};
+var max = lambda(x, y, z) { return identity(comparison(comparison(x, y), z)); };
+print(max(1, 5, 6));
+print(max(6, 2, 9));
+print(max(1.3, 7, 9.0));
+print(max(-4, 3, 2));
+//[end]
+/*
+[stdout]
+6
+9
+9.0
+3
+[end]
+[end]
+*/

tests/japl/longs/constant_long.jpl

@@ -1,5 +1,5 @@
-[Test: constant_long]
-[source: mixed]
+//[Test: constant_long]
+//[source: mixed]
 // Test for constants
 
 var v_1 = 1;
@@ -130,5 +130,5 @@ var v_125 = 1;
 var v_126 = 1;
 var v_127 = 1;
 var v_128 = 1;
-[end]
-[end]
+//[end]
+//[end]

tests/japl/longs/globAssgnRead.jpl

@@ -1,5 +1,5 @@
-[Test: glob_assgn_read]
-[source: mixed]
+//[Test: glob_assgn_read]
+//[source: mixed]
 var a0 = 451;
 var a1 = 5098;
 var a2 = 469;
@@ -1501,5 +1501,5 @@ print(a151);//stdout:4839
 print(a975);//stdout:7651
 print(a7);//stdout:2979
 print(a661);//stdout:8235
-[end]
-[end]
+//[end]
+//[end]

tests/japl/longs/globWithSets.jpl

@@ -1,5 +1,5 @@
-[Test: glob_with_sets]
-[source: mixed]
+//[Test: glob_with_sets]
+//[source: mixed]
 var a0 = 829;
 var a1 = 6820;
 var a2 = 114;
@@ -5238,5 +5238,5 @@ print(a87);//stdout:1282
 print(a445);//stdout:1726
 print(a790);//stdout:1140
 print(a961);//stdout:1708
-[end]
-[end]
+//[end]
+//[end]

tests/japl/longs/locAssgnRead.jpl

@@ -1,5 +1,5 @@
-[Test: loc_assgn_read]
-[source: mixed]
+//[Test: loc_assgn_read]
+//[source: mixed]
 {
 var a0 = 9103;
 var a1 = 4565;
@@ -1503,5 +1503,5 @@ print(a722);//stdout:5380
 print(a538);//stdout:8625
 print(a809);//stdout:4506
 }
-[end]
-[end]
+//[end]
+//[end]

tests/japl/longs/locWithSets.jpl

@@ -1,5 +1,5 @@
-[Test: loc_with_sets]
-[source: mixed]
+//[Test: loc_with_sets]
+//[source: mixed]
 {
 var a0 = 8313;
 var a1 = 3509;
@@ -5186,5 +5186,5 @@ print(a380);//stdout:2145
 print(a125);//stdout:4280
 print(a55);//stdout:6992
 }
-[end]
-[end]
+//[end]
+//[end]

tests/japl/meta/empty.jpl

@@ -1,4 +1,4 @@
-
+/*
 [; This is a comment
 
 This is a comment as well.
@@ -17,3 +17,4 @@ the test builder, but it should be a test with
 the name "".
 
 [end]
+*/

tests/japl/meta/mixed.jpl

@@ -1,19 +1,21 @@
+/*
 [test: mixed]
-[source: mixed]
+*/
+//[source: mixed]
 print("Hello", readLine());
 //stdout:Hello world
 //stdin:world
 
 print("aaaaaa");
-//stdoutre:a*
+//matchout:a*
 
 printErr("Hello", readLine());
 //stderr:Hello error
 //stdin:error
 
 printErr("bbbbbb");
-//stderrre:b*
-[end]
-[end]
+//matcherr:b*
+//[end]
+//[end]
 
 

tests/japl/meta/nw.jpl

@@ -1,12 +1,13 @@
-[Test: nw]
-[source]
+//[Test: nw]
+//[source]
 print("hey");
 print("second line");
 printErr("hey there");
 print("abcde");
 printErr("12345");
 printErr("0123456789.");
-[end]
+//[end]
+/*
 [stdout: nw]
   hey  
 	second line
@@ -21,4 +22,5 @@ printErr("0123456789.");
 [[0-9]*
 	[0-9]*.
 [end]
-[end]
+*/
+//[end]

tests/japl/meta/raw.jpl

@@ -1,11 +1,12 @@
-[Test: raw]
-[source: raw]
+//[Test: raw]
+//[source: raw]
 print("Hi", readLine());
 print("aaaaaaa");
 printErr("Bye", readLine());
 printErr("bbbbbbb");
 //stdout:This is not a part of the expected output
-[end]
+//[end]
+/*
 [stdin]
 person
 very important person
@@ -23,3 +24,4 @@ Bye very important person
 b*
 [end]
 [end]
+*/

tests/japl/meta/skip.jpl

@@ -1,7 +1,8 @@
-[test: skipped]
-[skip]
-
+//[test: skipped]
+//[skip]
+/*
 [stdout]
 Hello this text won't be matched.
 [end]
-[end]
+*/
+//[end]

tests/japl/nan.jpl

@@ -1,9 +1,11 @@
-[Test: nan]
-[source: raw]
+//[Test: nan]
+//[source: raw]
 print((5/0)*0);
-[end]
+//[end]
+/*
 [stdout]
 nan
 [end]
-[end]
+*/
+//[end]
 

tests/japl/nested.jpl

@@ -0,0 +1,13 @@
+//[Test: nested multiline comments]
+//[source: raw]
+/* first
+/* second
+/* third
+/* fourth
+*/
+*/
+*/
+*/
+
+//[end]
+//[end]

tests/japl/procedures.jpl

@@ -0,0 +1,13 @@
+//[Test: functions without parentheses]
+//[source: mixed]
+fun sayhi {
+  print("hi");
+}
+fun saybye {
+  print("bye");
+}
+
+sayhi();//stdout:hi
+saybye();//stdout:bye
+//[end]
+//[end]

tests/japl/reassignment.jpl

@@ -1,5 +1,5 @@
-[Test: reassignment]
-[source: mixed]
+//[Test: reassignment]
+//[source: mixed]
 {
     var x = 5;
     var y = x;
@@ -25,5 +25,5 @@ fun resetter(x) {
 var q = 5;
 resetter(q);//stdout:7
 print(q);//stdout:5
-[end]
-[end]
+//[end]
+//[end]

tests/japl/runtime_interning.jpl

@@ -1,6 +1,6 @@
-[Test: runtimeinterning]
-[skip]
-[source: mixed]
+//[Test: runtimeinterning]
+//[skip]
+//[source: mixed]
 //runtime interning
 
 var f = "leafy";
@@ -14,5 +14,5 @@ print(h is j);//stdout:true
 var x = "ex";
 var y = "ey";
 print(x is y);//stdout:false
-[end]
-[end]
+//[end]
+//[end]

tests/japl/shadowing.jpl

@@ -1,5 +1,5 @@
-[Test: shadowing]
-[source: mixed]
+//[Test: shadowing]
+//[source: mixed]
 //similar to vars.jpl, but more focused on shadowing
 
 // simple shadowing
@@ -76,5 +76,5 @@ eat();//stdout:nom nom nom
     print(eat);//stdout:5
 }
 eat();//stdout:nom nom nom
-[end]
-[end]
+//[end]
+//[end]

tests/japl/strings.jpl

@@ -1,5 +1,5 @@
-[Test: strings]
-[source: mixed]
+//[Test: strings]
+//[source: mixed]
 var left = "left";
 var right = "right";
 var directions = left + " " + right;
@@ -12,5 +12,15 @@ left = left + " side";
 print(left);//stdout:left side
 right = "side: " + right;
 print(right);//stdout:side: right
-[end]
-[end]
+//[end]
+//[end]
+//[Test: string slicing]
+//[skip]
+//[source: mixed]
+var longstring = "a very long string that will be sliced";
+var part = longstring[0:5];
+print(part);//stdout:a ver
+var part2 = longstring[0..4];
+print(part2);//stdout:a ver
+//[end]
+//[end]

tests/japl/vars.jpl

@@ -1,5 +1,5 @@
-[Test: vars]
-[source: mixed]
+//[Test: vars]
+//[source: mixed]
 var x = 1;
 var y = 2;
 print(x);//stdout:1
@@ -30,5 +30,5 @@ longName = "hello";
 print(longName); //stdout:hello
 longName = longName + " world";
 print(longName); //stdout:hello world
-[end]
-[end]
+//[end]
+//[end]

tests/japl/while.jpl

@@ -1,5 +1,5 @@
-[Test: while]
-[source: mixed]
+//[Test: while]
+//[source: mixed]
 var x = 5;
 while (x > 0)
 {
@@ -20,5 +20,5 @@ while (x < 10)
     string = string + "A";
 }
 print(string);//stdout:hAAAAAAAAAA
-[end]
-[end]
+//[end]
+//[end]

tests/jats.nim

@@ -152,6 +152,7 @@ Debug output flags:
 Test behavior flags:
 -j:<parallel test count> (or --jobs:<parallel test count>) to specify number of tests to run parallel
 -t:<test file or dir> (or --test:<path> or --tests:<path>) to specify where tests are
+--timeout <timeout in seconds> to specify when to kill tests
 -f (or --force) will run skipped tests
 Miscellaneous flags:
 -h (or --help) displays this help message

tests/logutils.nim

@@ -107,6 +107,7 @@ type Buffer* = ref object
     ## Represents an updateable line on the terminal
     contents: string
     previous: string
+    termwidth: int
 
 proc newBuffer*: Buffer =
     ## Creates a Buffer, hides the cursor
@@ -131,6 +132,8 @@ proc updateProgressBar*(buf: Buffer, text: string, total: int, current: int) =
     let w = terminalWidth()
     if w > newline.len():
         newline &= " ".repeat(w - newline.len() - 1)
+    else:
+        newline = newline[0..w-2]
     buf.contents = newline
 
 proc clearLineAndWrite(text: string, oldsize: int) =

tests/testbuilder.nim

@@ -30,8 +30,8 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
     ## if comment is true, the returned value has to be ignored
     # when non modal, mode becomes the line
     # when comment is true, it must not do anything to whenever it is exported
-    let line = line
     # initialize result
+    var start = 0
     result.modal = false
     result.mode = ""
     result.detail = ""
@@ -48,14 +48,21 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
                 result.modal = true
                 return result
         result.modal = true
+        start = 1
     # not modal line early return
+    elif line.len() >= 3 and line[0..2] == "//[":
+        if line.len() > 3:
+            result.modal = true
+            start = 3
+        else:
+            fatal "Invalid line //[, no mode defined."
     else:
         result.mode = line
         return result
 
     # normal modal line:
     var colon = false # if there has been a colon already
-    for i in countup(0, line.high()):
+    for i in countup(start, line.high()):
         let ch = line[i]
         if ch in Letters or ch in Digits or ch in {'_', '-'}:
             # legal characters
@@ -187,8 +194,7 @@ proc buildTests*(testDir: string): seq[Test] =
         else:
             fatal "test dir/file doesn't exist"
 
-    for candidateObj in walkDir(testDir):
-        let candidate = candidateObj.path
+    for kind, candidate in walkDir(testDir):
         if dirExists(candidate):
             log(LogLevel.Debug, &"Descending into dir {candidate}")
             result &= buildTests(candidate)

tests/testmarkup.md

@@ -36,16 +36,18 @@ Must not contain a BOM. Line endings must be a single
 ### Mode syntax
 
 The modes are constructed from modelines,
-which are lines starting with the character '['.
+which are lines starting with the character '[', or alternatively
+they can also start with the sequence "//[".
 Modelines also have to be closed by a ']' character
 on the end of this line. These lines may not contain
-whitespace before the opening '[' nor after then ending
+whitespace before the opening '[' or "//[" nor after then ending
 ']' characters. Inside the brackets, letters (case
 insensitive), numbers, underscores and dashes form\
 a name describing what the modeline does.
 
 ```
 [ name ]
+//[ name ]
 ```
 
 Optionally, an argument may be passed, which is 
@@ -53,6 +55,7 @@ separated by a colon.
 
 ```
 [ name : detail ]
+//[name: detail]
 ```
 
 Whitespace inside the brackets is ignored (even inside
@@ -144,13 +147,13 @@ added to add lines to the expected stdout/stderr or
 the stdin of the test using the legacy test format.
 
 They are defined by the sequences `//stdout:`, 
-`//stderr:`, `//stdin:`, `//stdoutre:` and 
-`//stderrre:`. Every character after the colon and
+`//stderr:`, `//stdin:`, `//matchout:` and 
+`//matcherr:`. Every character after the colon and
 before the end of the line is appended to the respective
 field of the test. `stdout` adds a raw line to be
-matched to the expected stdout of the test. `stdoutre`
+matched to the expected stdout of the test. `matchout`
 adds a regex to match a line of the stdout of the test.
-`stderr` and `stderrre` are the stderr equivalents.
+`stderr` and `matcherr` are the stderr equivalents.
 `stdin` adds a line to the stdin that the JAPL source
 can read from.
 
@@ -209,3 +212,10 @@ Hello there
 
 Coming soon.
 
+# Best practices
+
+Tests should be written so that they are valid jpl code. The test title
+and modes surrounding source code should be prefixed with `//`. Stdin/stdout 
+and other raw non-jpl sources should be inside `/* */` blocks. Single line
+commands such as skips should be either prefixed with `//` or inside a `/* */`
+block.

tests/testobject.nim

@@ -71,10 +71,10 @@ proc compileExpectedOutput(source: string, rawkw: string, rekw: string): seq[Exp
             result &= genEL(matches[0], ExpectedLineKind.Regex)
 
 proc compileExpectedOutput(source: string): seq[ExpectedLine] =
-    compileExpectedOutput(source, "stdout", "stdoutre")
+    compileExpectedOutput(source, "stdout", "matchout")
 
 proc compileExpectedError(source: string): seq[ExpectedLine] =
-  compileExpectedOutput(source, "stderr", "stderrre")
+  compileExpectedOutput(source, "stderr", "matcherr")
 
 proc compileInput(source: string): string =
     for line in source.split('\n'):