peon/src/frontend/compiler.nim

# Copyright 2022 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import meta/token
import meta/ast
import meta/errors
import meta/bytecode
import ../config
import ../util/multibyte


import strformat
import algorithm
import parseutils
import sequtils
import strutils


export ast
export bytecode
export token
export multibyte


type
    TypeKind = enum
        ## An enumeration of compile-time
        ## types
        Int8, UInt8, Int16, UInt16, Int32,
        UInt32, Int64, UInt64, Float32, Float64,
        Char, Byte, String, Function, CustomType,
        Dict, List, Tuple, Set, Nil, Nan, Bool,
        Inf
    Type = ref object
        ## A wrapper around
        ## compile-time types
        node: ASTNode
        case kind: TypeKind:
            of Function:
                returnType: Type
            of List, Tuple, Set:
                memberType: Type
            of Dict:
                keyType: Type
                valueType: Type
            else:
                discard
    Name = ref object
        ## A compile-time wrapper around
        ## statically resolved names.
        ## Depth indicates to which scope
        ## the variable belongs, zero meaning
        ## the global one
        name: IdentExpr        # Name of the identifier
        owner: string          # Owner of the identifier
        depth: int             # Scope depth
        isPrivate: bool        # Is this name private?
        isConst: bool          # Is this a constant?
        isLet: bool            # Can this name's value be mutated?
        valueType: Type        # The name's type
        codePos: int           # The position in the bytecode
                               # where this name's StoreVar
                               # instruction was emitted. This 
                               # is kept so that once we detect 
                               # this name as a closed-over variable 
                               # we can change the StoreVar into a StoreHeap
    Loop = object
        ## A "loop object" used
        ## by the compiler to emit
        ## appropriate jump offsets
        ## for continue and break
        ## statements
        start: int          # Position in the bytecode where the loop starts
        depth: int          # Scope depth where the loop is located
        breakPos: seq[int]  # List of positions where 

    Compiler* = ref object
        ## A wrapper around the compiler's state

        # The bytecode chunk where we write code to
        chunk: Chunk
        # The output of our parser (AST)
        ast: seq[ASTNode]
        # The current AST node we're looking at
        current: int
        # The current file being compiled (used only for
        # error reporting)
        file: string
        # Compile-time "simulation" of the stack at
        # runtime to load variables that have stack
        # behavior more efficiently
        names: seq[Name]
        # The current scope depth. If > 0, we're
        # in a local scope, otherwise it's global
        scopeDepth: int
        # The current function being compiled
        currentFunction: FunDecl
        # Are optimizations turned on?
        enableOptimizations*: bool
        # The current loop being compiled (used to
        # keep track of where to jump)
        currentLoop: Loop
        # The current module being compiled
        # (used to restrict access to statically
        # defined variables at compile time)
        currentModule: string
        # Each time a defer statement is
        # compiled, its code is emitted
        # here. Later, if there is any code
        # to defer in the current function,
        # funDecl will wrap the function's code
        # inside an implicit try/finally block
        # and add this code in the finally branch.
        # This sequence is emptied each time a
        # fun declaration is compiled and stores only
        # deferred code for the current function (may
        # be empty)
        deferred: seq[uint8]
        # List of closed-over variables
        closedOver: seq[IdentExpr]


proc newCompiler*(enableOptimizations: bool = true): Compiler =
    ## Initializes a new Compiler object
    new(result)
    result.ast = @[]
    result.current = 0
    result.file = ""
    result.names = @[]
    result.scopeDepth = 0
    result.currentFunction = nil
    result.enableOptimizations = enableOptimizations
    result.currentModule = ""


## Forward declarations
proc expression(self: Compiler, node: ASTNode)
proc statement(self: Compiler, node: ASTNode)
proc declaration(self: Compiler, node: ASTNode)
proc peek(self: Compiler, distance: int = 0): ASTNode
proc identifier(self: Compiler, node: IdentExpr)
proc varDecl(self: Compiler, node: VarDecl)
proc inferValueType(self: Compiler, node: ASTNode): Type
proc inferExprType(self: Compiler, node: ASTNode): Type
## End of forward declarations

## Public getter for nicer error formatting
proc getCurrentNode*(self: Compiler): ASTNode = (if self.current >=
        self.ast.len(): self.ast[^1] else: self.ast[self.current - 1])


## Utility functions
proc peek(self: Compiler, distance: int = 0): ASTNode =
    ## Peeks at the AST node at the given distance.
    ## If the distance is out of bounds, the last
    ## AST node in the tree is returned. A negative
    ## distance may be used to retrieve previously
    ## consumed AST nodes
    if self.ast.high() == -1 or self.current + distance > self.ast.high() or
            self.current + distance < 0:
        result = self.ast[^1]
    else:
        result = self.ast[self.current + distance]


proc done(self: Compiler): bool =
    ## Returns true if the compiler is done
    ## compiling, false otherwise
    result = self.current > self.ast.high()


proc error(self: Compiler, message: string) {.raises: [CompileError, ValueError].} =
    ## Raises a formatted CompileError exception
    var tok = self.getCurrentNode().token
    raise newException(CompileError, &"A fatal error occurred while compiling '{self.file}', module '{self.currentModule}' line {tok.line} at '{tok.lexeme}' -> {message}")


proc step(self: Compiler): ASTNode =
    ## Steps to the next node and returns
    ## the consumed one
    result = self.peek()
    if not self.done():
        self.current += 1


proc emitByte(self: Compiler, byt: OpCode|uint8) =
    ## Emits a single byte, writing it to
    ## the current chunk being compiled
    when DEBUG_TRACE_COMPILER:
        echo &"DEBUG - Compiler: Emitting {$byt}"
    self.chunk.write(uint8 byt, self.peek().token.line)


proc emitBytes(self: Compiler, byt1: OpCode|uint8, byt2: OpCode|uint8) =
    ## Emits multiple bytes instead of a single one. This is useful
    ## to emit operators along with their operands or for multi-byte
    ## instructions that are longer than one byte
    self.emitByte(uint8 byt1)
    self.emitByte(uint8 byt2)


proc emitBytes(self: Compiler, bytarr: array[2, uint8]) =
    ## Handy helper method to write an array of 2 bytes into
    ## the current chunk, calling emitByte on each of its
    ## elements
    self.emitBytes(bytarr[0], bytarr[1])


proc emitBytes(self: Compiler, bytarr: array[3, uint8]) =
    ## Handy helper method to write an array of 3 bytes into
    ## the current chunk, calling emitByte on each of its
    ## elements
    self.emitBytes(bytarr[0], bytarr[1])
    self.emitByte(bytarr[2])



proc makeConstant(self: Compiler, val: LiteralExpr): array[3, uint8] =
    ## Adds a constant to the current chunk's constant table
    ## and returns its index as a 3-byte array of uint8s
    result = self.chunk.addConstant(val)


proc emitConstant(self: Compiler, obj: LiteralExpr) =
    ## Emits a LoadConstant instruction along
    ## with its operand
    case self.inferExprType(obj).kind:
        of Int64:
            self.emitByte(LoadInt64)
        else:
            discard  # TODO
    self.emitBytes(self.makeConstant(obj))


proc emitJump(self: Compiler, opcode: OpCode): int =
    ## Emits a dummy jump offset to be patched later. Assumes
    ## the largest offset (emits 4 bytes, one for the given jump
    ## opcode, while the other 3 are for the jump offset which is set
    ## to the maximum unsigned 24 bit integer). If the shorter
    ## 16 bit alternative is later found to be better suited, patchJump
    ## will fix this. This function returns the absolute index into the
    ## chunk's bytecode array where the given placeholder instruction was written
    self.emitByte(opcode)
    self.emitBytes((0xffffff).toTriple())
    result = self.chunk.code.len() - 4


proc patchJump(self: Compiler, offset: int) =
    ## Patches a previously emitted relative 
    ## jump using emitJump. Since emitJump assumes
    ## a long jump, this also shrinks the jump
    ## offset and changes the bytecode instruction if possible
    ## (i.e. jump is in 16 bit range), but the converse is also
    ## true (i.e. it might change a regular jump into a long one)
    let jump: int = self.chunk.code.len() - offset
    if jump > 16777215:
        self.error("cannot jump more than 16777216 bytecode instructions")
    if jump < uint16.high().int:
        case OpCode(self.chunk.code[offset]):
            of LongJumpForwards:
                self.chunk.code[offset] = JumpForwards.uint8()
            of LongJumpBackwards:
                self.chunk.code[offset] = JumpBackwards.uint8()
            of LongJumpIfFalse:
                self.chunk.code[offset] = JumpIfFalse.uint8()
            of LongJumpIfFalsePop:
                self.chunk.code[offset] = JumpIfFalsePop.uint8()
            of LongJumpIfFalseOrPop:
                self.chunk.code[offset] = JumpIfFalseOrPop.uint8()
            else:
                discard
        self.chunk.code.delete(offset + 1)      # Discards the first 8 bits of the jump offset (which are empty)
        let offsetArray = (jump - 1).toDouble() # -1 since we got rid of 1 byte!
        self.chunk.code[offset + 1] = offsetArray[0]
        self.chunk.code[offset + 2] = offsetArray[1]
    else:
        case OpCode(self.chunk.code[offset]):
            of JumpForwards:
                self.chunk.code[offset] = LongJumpForwards.uint8()
            of JumpBackwards:
                self.chunk.code[offset] = LongJumpBackwards.uint8()
            of JumpIfFalse:
                self.chunk.code[offset] = LongJumpIfFalse.uint8()
            of JumpIfFalsePop:
                self.chunk.code[offset] = LongJumpIfFalsePop.uint8()
            of JumpIfFalseOrPop:
                self.chunk.code[offset] = LongJumpIfFalseOrPop.uint8()
            else:
                discard
        let offsetArray = jump.toTriple()
        self.chunk.code[offset + 1] = offsetArray[0]
        self.chunk.code[offset + 2] = offsetArray[1]
        self.chunk.code[offset + 3] = offsetArray[2]


proc resolve(self: Compiler, name: IdentExpr,
        depth: int = self.scopeDepth): Name =
    ## Traverses self.names backwards and returns the
    ## first name object with the given name. Returns
    ## nil when the name can't be found. This function
    ## has no concept of scope depth, because getStackPos
    ## does that job. Note that private names declared in
    ## other modules will not be resolved!
    for obj in reversed(self.names):
        if obj.name.token.lexeme == name.token.lexeme:
            if obj.isPrivate and obj.owner != self.currentModule:
                continue   # There may be a name in the current module that
                           # matches, so we skip this
            return obj
    return nil


proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] =
    ## Iterates the internal list of declared names backwards and
    ## returns a tuple (closedOver, pos) that tells the caller whether the
    ## the name is to be emitted as a closure as well as its predicted
    ## stack/closure array position. Returns (false, -1) if the variable's 
    ## location can not be determined at compile time (this is an error!).
    ## Note that private names declared in other modules will not be resolved!
    var i: int = self.names.high()
    for variable in reversed(self.names):
        if name.name.lexeme == variable.name.name.lexeme:
            if variable.isPrivate and variable.owner != self.currentModule:
                continue
            if variable.depth == depth or variable.depth == 0:
                # variable.depth == 0 for globals!
                return (false, i)
            elif variable.depth > 0:
                for j, closure in reversed(self.closedOver):
                    if closure.name.lexeme == name.name.lexeme:
                        return (true, j)
        dec(i)
    return (false, -1)


proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth) =
    ## Detects if the given name is used in a local scope deeper
    ## than the given one and modifies the code emitted for it
    ## to store it as a closure variable if it is. Does nothing if the name
    ## hasn't been declared yet or is unreachable (for example if it's
    ## declared as private in another module), if the name itself is a
    ## global variable and if either the current or the outer scope are 
    ## the global (outermost) one. This function must be called each
    ## time a name is referenced in order for closed-over variables 
    ## to be emitted properly, otherwise the runtime may behave 
    ## unpredictably or crash
    if depth == 0 or depth - 1 == 0:
        return
    let entry = self.resolve(name)
    if entry == nil:
        return
    if entry.depth == 0:
        return
    if entry.depth < depth:
        # Ding! The given name is closed over: we need to
        # change the StoreVar instruction that created this
        # name entry into a StoreHeap. We don't need to change
        # other pieces of code because self.identifier() already
        # emits LoadHeap if it detects the variable is closed over,
        # whether or not this function is called
        self.closedOver.add(entry.name)
        if self.closedOver.len() >= 16777216:
            self.error("too many consecutive closure-over variables (max is 16777216)")
        let idx = self.closedOver.high().toTriple()
        self.chunk.code[entry.codePos] = StoreHeap.uint8
        self.chunk.code[entry.codePos + 1] = idx[0]
        self.chunk.code[entry.codePos + 2] = idx[1]
        self.chunk.code[entry.codePos + 3] = idx[2]


proc toIntrinsic(name: string): Type =
    ## Converts a string to an intrinsic 
    ## type if it is valid and returns nil
    ## otherwise        
    if name in ["int", "int64", "i64"]:
        return Type(kind: Int64)
    elif name in ["uint64", "u64"]:
        return Type(kind: UInt64)
    elif name in ["int32", "i32"]:
        return Type(kind: Int32)
    elif name in ["uint32", "u32"]:
        return Type(kind: UInt32)
    elif name in ["int16", "i16"]:
        return Type(kind: Int16)
    elif name in ["uint16", "u16"]:
        return Type(kind: UInt16)
    elif name in ["int8", "i8"]:
        return Type(kind: Int8)
    elif name in ["uint8", "u8"]:
        return Type(kind: UInt8)
    elif name in ["f64", "float", "float64"]:
        return Type(kind: Float64)
    elif name in ["f32", "float32"]:
        return Type(kind: Float32)
    elif name == "byte":
        return Type(kind: Byte)
    elif name == "char":
        return Type(kind: Char)
    elif name == "nan":
        return Type(kind: Nan)
    elif name == "nil":
        return Type(kind: Nil)
    elif name == "inf":
        return Type(kind: Inf)
    elif name == "bool":
        return Type(kind: Bool)
    else:
        return nil


proc toIntrinsic(typ: Expression): Type = 
    ## Gets an expression's 
    ## intrinsic type, if possible
    if typ == nil:
        return nil
    case typ.kind:
        of identExpr:
            return typ.token.lexeme.toIntrinsic()
        else:
            discard


proc inferValueType(self: Compiler, node: ASTNode): Type =
    ## Infers the type of a given literal expression
    case node.kind:
        of listExpr:
            return Type(kind: List, memberType: self.inferExprType(ListExpr(node).valueType))
        of tupleExpr:
            return Type(kind: Tuple, memberType: self.inferExprType(TupleExpr(node).valueType))
        of setExpr:
            return Type(kind: Set, memberType: self.inferExprType(SetExpr(node).valueType))
        of dictExpr:
            let node = DictExpr(node)
            return Type(kind: Dict, keyType: self.inferExprType(node.valueType), valueType: self.inferExprType(node.valueType))
        of intExpr, binExpr, octExpr, hexExpr:
            let node = LiteralExpr(node)
            let size = node.token.lexeme.split("'")
            if len(size) notin 1..2:
                self.error("invalid state: inferValueType -> invalid size specifier (This is an internal error and most likely a bug!)")
            if size.len() == 1:
                return Type(kind: Int64)
            let typ = size[1].toIntrinsic()
            if typ != nil:
                return typ
            else:
                self.error(&"invalid type specifier '{size[1]}' for int")
        of floatExpr:
            let node = LiteralExpr(node)
            let size = node.token.lexeme.split("'")
            if len(size) notin 1..2:
                self.error("invalid state: inferValueType -> invalid size specifier (This is an internal error and most likely a bug!)")
            if size.len() == 1 or size[1] == "f64":
                return Type(kind: Float64)
            let typ = size[1].toIntrinsic()
            if typ != nil:
                return typ
            else:
                self.error(&"invalid type specifier '{size[1]}' for float")
        of nilExpr:
            return Type(kind: Nil)
        of trueExpr:
            return Type(kind: Bool)
        of falseExpr:
            return Type(kind: Bool)
        of nanExpr:
            return Type(kind: Nan)
        of infExpr:
            return Type(kind: Inf)
        else:
            discard  # TODO


proc inferExprType(self: Compiler, node: ASTNode): Type =
    ## Infers the type of a given expression and
    ## returns it
    case node.kind:
        of identExpr:
            var node = IdentExpr(node)
            var name = self.resolve(node)
            return name.valueType
        of unaryExpr:
            return self.inferValueType(UnaryExpr(node).a)
        of binaryExpr:
            var node = BinaryExpr(node)
            var a = self.inferExprType(node.a)
            var b = self.inferExprType(node.b)
            if a == nil or b == nil:
                return nil
            return a
        of {intExpr, hexExpr, binExpr, octExpr, 
            strExpr, falseExpr, trueExpr, infExpr, 
            nanExpr, floatExpr, nilExpr, listExpr,
            dictExpr, setExpr, tupleExpr
            }:
            return self.inferValueType(node)
        else:
            discard  # Unreachable
    

proc inferDeclType(self: Compiler, node: Declaration): Type =
    ## Infers the type of a given declaration if it's
    ## not already defined and returns it
    case node.kind:
        of funDecl:
            var node = FunDecl(node)
            let resolved = self.resolve(node.name)
            if resolved != nil:
                return resolved.valueType
        of NodeKind.varDecl:
            var node = VarDecl(node)
            let resolved = self.resolve(node.name)
            if resolved != nil:
                return resolved.valueType
            else:
                return self.inferExprType(node.value)
        else:
            return  # Unreachable


proc typeToStr(self: Compiler, typ: Type): string =
    case typ.kind:
        of {Int8, UInt8, Int16, UInt16, Int32,
        UInt32, Int64, UInt64, Float32, Float64,
        Char, Byte, String, Nil, Nan, Bool, Inf}:
            return ($typ.kind).toLowerAscii()
        of Function:
            result = "function ("
            case typ.node.kind:
                of funDecl:
                    var node = FunDecl(typ.node)
                    for i, argument in node.arguments:
                        result &= &"{argument.name.token.lexeme}: {self.typeToStr(self.inferExprType(argument.name))}"
                        if i < node.arguments.len():
                            result &= ", "
                    result &= ")"
                of lambdaExpr:
                    var node = LambdaExpr(typ.node)
                    for i, argument in node.arguments:
                        result &= &"{argument.name.token.lexeme}: {argument.valueType}"
                        if i < node.arguments.len():
                            result &= ", "
                    result &= ")"
                else:
                    discard  # Unreachable
        of List, Tuple, Set:
            result &= &"{($typ.kind).toLowerAscii()}["
        of Dict:
            result &= &"{($typ.kind).toLowerAscii()}[]"
        else:
            discard

proc `==`(self, other: Type): bool =
    if system.`==`(self, nil):
        return system.`==`(other, nil)
    elif system.`==`(other, nil):
        return system.`==`(self, nil)
    if self.kind != other.kind:
        return false
    case self.kind:
        of {Int8, UInt8, Int16, UInt16, Int32,
        UInt32, Int64, UInt64, Float32, Float64,
        Char, Byte, String, Nil, Nan, Bool, Inf}:
            return true
        of Function:
            discard  # TODO
        of List, Tuple, Set:
            return self.memberType == other.memberType
        of Dict:
            return self.keyType == other.keyType and self.valueType == other.valueType
        else:
            discard


## End of utility functions

proc literal(self: Compiler, node: ASTNode) =
    ## Emits instructions for literals such
    ## as singletons, strings, numbers and
    ## collections
    case node.kind:
        of trueExpr:
            self.emitByte(OpCode.True)
        of falseExpr:
            self.emitByte(OpCode.False)
        of nilExpr:
            self.emitByte(OpCode.Nil)
        of infExpr:
            self.emitByte(OpCode.Inf)
        of nanExpr:
            self.emitByte(OpCode.Nan)
        of strExpr:
            self.emitConstant(LiteralExpr(node))
        of intExpr:
            var x: int
            var y = IntExpr(node)
            try:
                discard parseInt(y.literal.lexeme, x)
            except ValueError:
                self.error("integer value out of range")
            self.emitConstant(y)
        of hexExpr:
            var x: int
            var y = HexExpr(node)
            try:
                discard parseHex(y.literal.lexeme, x)
            except ValueError:
                self.error("integer value out of range")
            self.emitConstant(newIntExpr(Token(lexeme: $x, line: y.token.line,
                    pos: (start: y.token.pos.start, stop: y.token.pos.start +
                    len($x)))))
        of binExpr:
            var x: int
            var y = BinExpr(node)
            try:
                discard parseBin(y.literal.lexeme, x)
            except ValueError:
                self.error("integer value out of range")
            self.emitConstant(newIntExpr(Token(lexeme: $x, line: y.token.line,
                    pos: (start: y.token.pos.start, stop: y.token.pos.start +
                    len($x)))))
        of octExpr:
            var x: int
            var y = OctExpr(node)
            try:
                discard parseOct(y.literal.lexeme, x)
            except ValueError:
                self.error("integer value out of range")
            self.emitConstant(newIntExpr(Token(lexeme: $x, line: y.token.line,
                    pos: (start: y.token.pos.start, stop: y.token.pos.start +
                    len($x)))))
        of floatExpr:
            var x: float
            var y = FloatExpr(node)
            try:
                discard parseFloat(y.literal.lexeme, x)
            except ValueError:
                self.error("floating point value out of range")
            self.emitConstant(y)
        of listExpr:
            var y = ListExpr(node)
            if y.members.len() > 16777216:
                self.error("list literals can't have more than 16777216 elements")
            for member in y.members:
                self.expression(member)
            self.emitByte(BuildList)
            self.emitBytes(y.members.len().toTriple()) # 24-bit integer, meaning collection literals can have up to 2^24 elements
        of tupleExpr:
            var y = TupleExpr(node)
            if y.members.len() > 16777216:
                self.error("tuple literals can't have more than 16777216 elements")
            for member in y.members:
                self.expression(member)
            self.emitByte(BuildTuple)
            self.emitBytes(y.members.len().toTriple())
        of setExpr:
            var y = SetExpr(node)
            if y.members.len() > 16777216:
                self.error("set literals can't have more than 16777216 elements")
            for member in y.members:
                self.expression(member)
            self.emitByte(BuildSet)
            self.emitBytes(y.members.len().toTriple())
        of dictExpr:
            var y = DictExpr(node)
            if y.keys.len() > 16777216:
                self.error("dict literals can't have more than 16777216 elements")
            for (key, value) in zip(y.keys, y.values):
                self.expression(key)
                self.expression(value)
            self.emitByte(BuildDict)
            self.emitBytes(y.keys.len().toTriple())
        of awaitExpr:
            var y = AwaitExpr(node)
            self.expression(y.expression)
            self.emitByte(OpCode.Await)
        else:
            self.error(&"invalid AST node of kind {node.kind} at literal(): {node} (This is an internal error and most likely a bug!)")


proc unary(self: Compiler, node: UnaryExpr) =
    ## Compiles unary expressions such as decimal or
    ## bitwise negation
    self.expression(node.a)     # Pushes the operand onto the stack
    case node.operator.kind:
        of Minus:
            self.emitByte(NoOp)
        of Plus:
            self.emitByte(NoOp)
        of TokenType.LogicalNot:
            self.emitByte(NoOp)
        of Tilde:
            self.emitByte(NoOp)
        else:
            self.error(&"invalid AST node of kind {node.kind} at unary(): {node} (This is an internal error and most likely a bug!)")


proc binary(self: Compiler, node: BinaryExpr) =
    ## Compiles all binary expressions

    # These two lines prepare the stack by pushing the
    # opcode's operands onto it
    self.expression(node.a)
    self.expression(node.b)
    # TODO: Find implementation of
    # the given operator and call it
    case node.operator.kind:
        of Plus:
            # a + b
            self.emitByte(NoOp)
        of Minus:
            # a - b
            self.emitByte(NoOp)
        of Star:
            # a * b
            self.emitByte(NoOp)
        of DoubleStar:
            # a ** b
            self.emitByte(NoOp)
        of Percentage:
            # a % b
            self.emitByte(NoOp)
        of FloorDiv:
            # a // b
            self.emitByte(NoOp)
        of Slash:
            # a / b
            self.emitByte(NoOp)
        of Ampersand:
            # a & b
            self.emitByte(NoOp)
        of Caret:
            # a ^ b
            self.emitByte(NoOp)
        of Pipe:
            # a | b
            self.emitByte(NoOp)
        of Is:
            # a is b
            self.emitByte(NoOp)
        of IsNot:
            # a isnot b
            self.emitByte(NoOp)
        of Of:
            # a of b
            self.emitByte(NoOp)
        of As:
            # a as b
            self.emitByte(NoOp)
        of RightShift:
            # a >> b
            self.emitByte(NoOp)
        of LeftShift:
            # a << b
            self.emitByte(NoOp)
        of LessThan:
            # a < b
            self.emitByte(NoOp)
        of GreaterThan:
            # a > b
            self.emitByte(NoOp)
        of DoubleEqual:
            # a == b
            self.emitByte(NoOp)
        of LessOrEqual:
            # a <= b
            self.emitByte(NoOp)
        of GreaterOrEqual:
            # a >= b
            self.emitByte(NoOp)
        of LogicalAnd:
            # a and b
            self.expression(node.a)
            var jump: int
            if self.enableOptimizations:
                jump = self.emitJump(JumpIfFalseOrPop)
            else:
                jump = self.emitJump(JumpIfFalse)
                self.emitByte(Pop)
            self.expression(node.b)
            self.patchJump(jump)
        of LogicalOr:
            # a or b
            self.expression(node.a)
            let jump = self.emitJump(JumpIfTrue)
            self.expression(node.b)
            self.patchJump(jump)
        else:
            self.error(&"invalid AST node of kind {node.kind} at binary(): {node} (This is an internal error and most likely a bug!)")


proc declareName(self: Compiler, node: Declaration) =
    ## Statically declares a name into the current scope
    case node.kind:
        of NodeKind.varDecl:
            var node = VarDecl(node)
            # Creates a new Name entry so that self.identifier emits the proper stack offset
            if self.names.high() > 16777215:
                # If someone ever hits this limit in real-world scenarios, I swear I'll
                # slap myself 100 times with a sign saying "I'm dumb". Mark my words
                self.error("cannot declare more than 16777216 variables at a time")
            self.names.add(Name(depth: self.scopeDepth, 
                                name: node.name,
                                isPrivate: node.isPrivate,
                                owner: self.currentModule,
                                isConst: node.isConst,
                                valueType: Type(kind: self.inferExprType(node.value).kind, node: node),
                                codePos: self.chunk.code.len(),
                                isLet: node.isLet))
            self.emitByte(StoreVar)
            self.emitBytes(self.names.high().toTriple())
        of NodeKind.funDecl:
            var node = FunDecl(node)
            # Declares the function's name in the
            # current scope but no StoreVar is emitted
            # because a function's name is only useful
            # at compile time and has no runtime meaning
            # given that we implement functions with jumps
            self.names.add(Name(depth: self.scopeDepth,
                                isPrivate: node.isPrivate,
                                isConst: false,
                                owner: self.currentModule,
                                valueType: Type(kind: Function, node: node),
                                codePos: -1,
                                name: node.name,
                                isLet: false))
            for argument in node.arguments:
                if self.names.high() > 16777215:
                    self.error("cannot declare more than 16777216 variables at a time")
                self.names.add(Name(depth: self.scopeDepth + 1, 
                                    isPrivate: true, 
                                    owner: self.currentModule, 
                                    isConst: false, 
                                    name: argument.name, 
                                    valueType: self.inferExprType(argument.name),
                                    codePos: self.chunk.code.len(),
                                    isLet: false))
                self.emitByte(StoreVar)
                self.emitBytes(self.names.high().toTriple())
            # TODO: Default arguments and unpacking
        else:
            discard  # Unreachable
        

proc identifier(self: Compiler, node: IdentExpr) =
    ## Compiles access to identifiers
    let s = self.resolve(node)
    if s == nil:
        self.error(&"reference to undeclared name '{node.token.lexeme}'")
    elif s.isConst:
        # Constants are emitted as, you guessed it, LoadConstant instructions
        # no matter the scope depth. If optimizations are enabled, the compiler
        # will reuse the same constant every time it is referenced instead of
        # allocating a new one each time
        self.emitConstant(node)
    else:
        self.detectClosureVariable(s.name)
        let t = self.getStackPos(node)
        let index = t.pos
        # We don't check if index is -1 because if it 
        # were, self.resolve() would have returned nil
        if not t.closedOver:
            # Static name resolution, loads value at index in the stack. Very fast. Much wow.
            self.emitByte(LoadVar)
            self.emitBytes(index.toTriple())
        else:
            if self.closedOver.len() == 0:
                self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)")
            # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics.
            # This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both 
            # dynamic arrays at runtime anyway)
            self.emitByte(LoadHeap)
            self.emitBytes(self.closedOver.high().toTriple())


proc assignment(self: Compiler, node: ASTNode) =
    ## Compiles assignment expressions
    case node.kind:
        of assignExpr:
            var node = AssignExpr(node)
            let name = IdentExpr(node.name)
            let r = self.resolve(name)
            if r == nil:
                self.error(&"assignment to undeclared name '{node.name}'")
            elif r.isConst:
                self.error(&"cannot assign to '{node.name}'")
            elif r.isLet:
                self.error(&"cannot reassign '{node.name}'")
            self.expression(node.value)
            let t = self.getStackPos(name)
            let index = t.pos
            case node.token.kind:
                of InplaceAdd:
                    self.emitByte(NoOp)
                of InplaceSub:
                    self.emitByte(NoOp)
                of InplaceDiv:
                    self.emitByte(NoOp)
                of InplaceMul:
                    self.emitByte(NoOp)
                of InplacePow:
                    self.emitByte(NoOp)
                of InplaceFloorDiv:
                    self.emitByte(NoOp)
                of InplaceMod:
                    self.emitByte(NoOp)
                of InplaceAnd:
                    self.emitByte(NoOp)
                of InplaceXor:
                    self.emitByte(NoOp)
                of InplaceRightShift:
                    self.emitByte(NoOp)
                of InplaceLeftShift:
                    self.emitByte(NoOp)
                else:
                    discard # Unreachable
            # In-place operators just change
            # what values is set to a given
            # offset in a dynamic array, so we only
            # need to perform the operation as usual 
            # and then store it. We could combine
            # everything into a single opcode, but
            # that would require variants of each
            # one for regular stack variables as
            # well as closed-over ones
            if index != -1:
                if not t.closedOver:
                    self.emitByte(StoreVar)
                else:
                    self.emitByte(StoreHeap)
                self.emitBytes(index.toTriple())
            else:
                self.error(&"reference to undeclared name '{node.token.lexeme}'")
        of setItemExpr:
            discard
            # TODO
        else:
            self.error(&"invalid AST node of kind {node.kind} at assignment(): {node} (This is an internal error and most likely a bug)")


proc beginScope(self: Compiler) =
    ## Begins a new local scope by incrementing the current
    ## scope's depth
    inc(self.scopeDepth)


proc endScope(self: Compiler) =
    ## Ends the current local scope
    if self.scopeDepth < 0:
        self.error("cannot call endScope with scopeDepth < 0 (This is an internal error and most likely a bug)")
    var popped: int = 0
    for ident in reversed(self.names):
        if ident.depth > self.scopeDepth:
            inc(popped)
            if not self.enableOptimizations:
                # All variables with a scope depth larger than the current one
                # are now out of scope. Begone, you're now homeless!
                self.emitByte(Pop)
    if self.enableOptimizations and popped > 1:
        # If we're popping less than 65535 variables, then
        # we can emit a PopN instruction. This is true for
        # 99.99999% of the use cases of the language (who the
        # hell is going to use 65 THOUSAND local variables?), but
        # if you'll ever use more then JAPL will emit a PopN instruction
        # for the first 65 thousand and change local variables and then
        # emit another batch of plain ol' Pop instructions for the rest
        if popped <= uint16.high().int():
            self.emitByte(PopN)
            self.emitBytes(popped.toDouble())
        else:
            self.emitByte(PopN)
            self.emitBytes(uint16.high().int.toDouble())
            for i in countdown(self.names.high(), popped - uint16.high().int()):
                if self.names[i].depth > self.scopeDepth:
                    self.emitByte(Pop)
    elif popped == 1:
        # We only emit PopN if we're popping more than one value
        self.emitByte(Pop)
    for _ in countup(0, popped - 1):
        discard self.names.pop()
    dec(self.scopeDepth)


proc blockStmt(self: Compiler, node: BlockStmt) =
    ## Compiles block statements, which create a new
    ## local scope.
    self.beginScope()
    for decl in node.code:
        self.declaration(decl)
    self.endScope()


proc ifStmt(self: Compiler, node: IfStmt) =
    ## Compiles if/else statements for conditional
    ## execution of code
    self.expression(node.condition)
    var jumpCode: OpCode
    if self.enableOptimizations:
        jumpCode = JumpIfFalsePop
    else:
        jumpCode = JumpIfFalse
    let jump = self.emitJump(jumpCode)
    if not self.enableOptimizations:
        self.emitByte(Pop)
    self.statement(node.thenBranch)
    self.patchJump(jump)
    if node.elseBranch != nil:
        let jump = self.emitJump(JumpForwards)
        self.statement(node.elseBranch)
        self.patchJump(jump)


proc emitLoop(self: Compiler, begin: int) =
    ## Emits a JumpBackwards instruction with the correct
    ## jump offset
    var offset: int
    case OpCode(self.chunk.code[begin + 1]): # The jump instruction
        of LongJumpForwards, LongJumpBackwards, LongJumpIfFalse,
                LongJumpIfFalsePop, LongJumpIfTrue:
            offset = self.chunk.code.len() - begin + 4
        else:
            offset = self.chunk.code.len() - begin
    if offset > uint16.high().int:
        if offset > 16777215:
            self.error("cannot jump more than 16777215 bytecode instructions")
        self.emitByte(LongJumpBackwards)
        self.emitBytes(offset.toTriple())
    else:
        self.emitByte(JumpBackwards)
        self.emitBytes(offset.toDouble())


proc whileStmt(self: Compiler, node: WhileStmt) =
    ## Compiles C-style while loops
    let start = self.chunk.code.len()
    self.expression(node.condition)
    let jump = self.emitJump(JumpIfFalsePop)
    self.statement(node.body)
    self.patchJump(jump)
    self.emitLoop(start)


proc expression(self: Compiler, node: ASTNode) =
    ## Compiles all expressions
    if self.inferExprType(node) == nil:
        if node.kind != identExpr:
            # So we can raise a more appropriate
            # error in self.identifier()
            self.error("expression has no type")
    case node.kind:
        of getItemExpr:
            discard  # TODO
        # Note that for setItem and assign we don't convert
        # the node to its true type because that type information
        # would be lost in the call anyway. The differentiation
        # happens in self.assignment
        of setItemExpr, assignExpr:
            self.assignment(node)
        of identExpr:
            self.identifier(IdentExpr(node))
        of unaryExpr:
            # Unary expressions such as ~5 and -3
            self.unary(UnaryExpr(node))
        of groupingExpr:
            # Grouping expressions like (2 + 1)
            self.expression(GroupingExpr(node).expression)
        of binaryExpr:
            # Binary expressions such as 2 ^ 5 and 0.66 * 3.14
            self.binary(BinaryExpr(node))
        of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr,
           infExpr, nanExpr, floatExpr, nilExpr, tupleExpr, setExpr, listExpr, 
           dictExpr:
            # Since all of these AST nodes mostly share
            # the same overall structure, and the kind
            # discriminant is enough to tell one
            # from the other, why bother with
            # specialized cases when one is enough?
            self.literal(node)
        else:
            self.error(&"invalid AST node of kind {node.kind} at expression(): {node} (This is an internal error and most likely a bug)")


proc awaitStmt(self: Compiler, node: AwaitStmt) =
    ## Compiles await statements. An await statement
    ## is like an await expression, but parsed in the
    ## context of statements for usage outside expressions,
    ## meaning it can be used standalone. It's basically the
    ## same as an await expression followed by a semicolon.
    ## Await expressions are the only native construct to
    ## run coroutines from within an already asynchronous
    ## context (which should be orchestrated by an event loop).
    ## They block in the caller until the callee returns
    self.expression(node.expression)
    self.emitByte(OpCode.Await)


proc deferStmt(self: Compiler, node: DeferStmt) =
    ## Compiles defer statements. A defer statement
    ## is executed right before its containing function
    ## exits (either because of a return or an exception)
    let current = self.chunk.code.len
    self.expression(node.expression)
    for i in countup(current, self.chunk.code.high()):
        self.deferred.add(self.chunk.code[i])
        self.chunk.code.del(i)


proc returnStmt(self: Compiler, node: ReturnStmt) =
    ## Compiles return statements. An empty return
    ## implicitly returns nil
    let returnType = self.inferExprType(node.value)
    let typ = self.inferDeclType(self.currentFunction)
    if returnType == nil and self.currentFunction.returnType != nil:
        self.error(&"expected return value of type '{self.currentFunction.returnType.token.lexeme}', but expression has no type")
    elif self.currentFunction.returnType == nil:
        if node.value.kind != nilExpr:
            self.error("non-nil return value is not allowed in functions without an explicit return type")
    else:
        if returnType != typ:
            self.error(&"expected return value of type '{self.typeToStr(typ)}', got '{self.typeToStr(returnType)}' instead")
    self.expression(node.value)
    self.emitByte(OpCode.Return)


proc yieldStmt(self: Compiler, node: YieldStmt) =
    ## Compiles yield statements
    self.expression(node.expression)
    self.emitByte(OpCode.Yield)


proc raiseStmt(self: Compiler, node: RaiseStmt) =
    ## Compiles yield statements
    self.expression(node.exception)
    self.emitByte(OpCode.Raise)


proc continueStmt(self: Compiler, node: ContinueStmt) =
    ## Compiles continue statements. A continue statements
    ## jumps to the next iteration in a loop
    if self.currentLoop.start <= 65535:
        self.emitByte(Jump)
        self.emitBytes(self.currentLoop.start.toDouble())
    else:
        self.emitByte(LongJump)
        self.emitBytes(self.currentLoop.start.toTriple())


proc breakStmt(self: Compiler, node: BreakStmt) =
    ## Compiles break statements. A continue statement
    ## jumps to the next iteration in a loop

    # Emits dummy jump offset, this is
    # patched later
    discard self.emitJump(OpCode.Jump)
    self.currentLoop.breakPos.add(self.chunk.code.high() - 4)
    if self.currentLoop.depth > self.scopeDepth:
        # Breaking out of a loop closes its scope
        self.endScope()


proc patchBreaks(self: Compiler) =
    ## Patches "break" opcodes with
    ## actual jumps. This is needed
    ## because the size of code
    ## to skip is not known before
    ## the loop is fully compiled
    for brk in self.currentLoop.breakPos:
        self.chunk.code[brk] = JumpForwards.uint8()
        self.patchJump(brk)


proc assertStmt(self: Compiler, node: AssertStmt) =
    ## Compiles assert statements (raise
    ## AssertionError if the expression is falsey)
    self.expression(node.expression)
    self.emitByte(OpCode.Assert)


proc statement(self: Compiler, node: ASTNode) =
    ## Compiles all statements
    case node.kind:
        of exprStmt:
            var expression = ExprStmt(node).expression
            self.expression(expression)
            self.emitByte(Pop)  # Expression statements discard their value. Their main use case is side effects in function calls
        of NodeKind.ifStmt:
            self.ifStmt(IfStmt(node))
        of NodeKind.assertStmt:
            self.assertStmt(AssertStmt(node))
        of NodeKind.raiseStmt:
            self.raiseStmt(RaiseStmt(node))
        of NodeKind.breakStmt:
            self.breakStmt(BreakStmt(node))
        of NodeKind.continueStmt:
            self.continueStmt(ContinueStmt(node))
        of NodeKind.returnStmt:
            self.returnStmt(ReturnStmt(node))
        of NodeKind.importStmt:
            discard
        of NodeKind.whileStmt, NodeKind.forStmt:
            ## Our parser already desugars for loops to
            ## while loops!
            let loop = self.currentLoop
            self.currentLoop = Loop(start: self.chunk.code.len(),
                    depth: self.scopeDepth, breakPos: @[])
            self.whileStmt(WhileStmt(node))
            self.patchBreaks()
            self.currentLoop = loop
        of NodeKind.forEachStmt:
            discard
        of NodeKind.blockStmt:
            self.blockStmt(BlockStmt(node))
        of NodeKind.yieldStmt:
            self.yieldStmt(YieldStmt(node))
        of NodeKind.awaitStmt:
            self.awaitStmt(AwaitStmt(node))
        of NodeKind.deferStmt:
            self.deferStmt(DeferStmt(node))
        of NodeKind.tryStmt:
            discard
        else:
            self.expression(node)


proc varDecl(self: Compiler, node: VarDecl) =
    ## Compiles variable declarations
    let kind = node.valueType.toIntrinsic()
    let typ = self.inferExprType(node.value)
    if kind == nil and typ == nil:
        self.error(&"cannot determine the type of '{node.name.token.lexeme}'")
    elif typ != kind and kind != nil:
        self.error(&"expected value of type '{self.typeToStr(kind)}', but '{node.name.token.lexeme}' is of type '{self.typeToStr(typ)}'")
    self.expression(node.value)
    self.declareName(node)


proc funDecl(self: Compiler, node: FunDecl) =
    ## Compiles function declarations

    # We store the current function
    var function = self.currentFunction
    self.currentFunction = node
    # A function's code is just compiled linearly
    # and then jumped over
    let jmp = self.emitJump(JumpForwards)
    self.declareName(node)

    # Since the deferred array is a linear 
    # sequence of instructions and we want
    # to keep track to whose function's each
    # set of deferred instruction belongs,
    # we record the length of the deferred
    # array before compiling the function
    # and use this info later to compile
    # the try/finally block with the deferred
    # code
    var deferStart = self.deferred.len()

    self.blockStmt(BlockStmt(node.body))
    # Yup, we're done. That was easy, huh?
    # But after all functions are just named
    # scopes, and we compile them just like that:
    # we declare their name and arguments (before
    # their body so recursion works) and then just
    # handle them as a block statement (which takes
    # care of incrementing self.scopeDepth so locals
    # are resolved properly). There's a need for a bit
    # of boilerplate code to make closures work, but
    # that's about it
    self.emitBytes(OpCode.Nil, OpCode.Return)

    # Currently defer is not functional so we
    # just pop the instructions
    for i in countup(deferStart, self.deferred.len(), 1):
        self.deferred.delete(i)

    self.patchJump(jmp)
    # This makes us compile nested functions correctly
    self.currentFunction = function



proc declaration(self: Compiler, node: ASTNode) =
    ## Compiles all declarations
    case node.kind:
        of NodeKind.varDecl:
            self.varDecl(VarDecl(node))
        of NodeKind.funDecl:
            self.funDecl(FunDecl(node))
        else:
            self.statement(node)


proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk =
    ## Compiles a sequence of AST nodes into a chunk
    ## object
    self.chunk = newChunk()
    self.ast = ast
    self.file = file
    self.names = @[]
    self.scopeDepth = 0
    self.currentFunction = nil
    self.currentModule = self.file
    self.current = 0
    while not self.done():
        self.declaration(self.step())
    if self.ast.len() > 0:
        # *Technically* an empty program is a valid program
        self.endScope()
        self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope
    result = self.chunk
    if self.ast.len() > 0 and self.scopeDepth != -1:
        self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?")