nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Collections are no longer a compiler intrinsic. Added parser support for ptr/ref arguments

This commit is contained in:
Mattia Giambirtone 2022-05-04 14:01:38 +02:00
parent f189b0214e
commit a1c5430773
8 changed files with 69 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
src/frontend/compiler.nim
View File

@ -15,6 +15,7 @@ import meta/token
import meta/ast import meta/ast
import meta/errors import meta/errors
import meta/bytecode import meta/bytecode
import meta/typing
import ../config import ../config
import ../util/multibyte import ../util/multibyte
@ -22,7 +23,6 @@ import ../util/multibyte
import strformat import strformat
import algorithm import algorithm
import parseutils import parseutils
import sequtils
import strutils import strutils
@ -33,28 +33,6 @@ export multibyte
type type
TypeKind = enum
## An enumeration of compile-time
## types
Int8, UInt8, Int16, UInt16, Int32,
UInt32, Int64, UInt64, Float32, Float64,
Char, Byte, String, Function, CustomType,
Dict, List, Tuple, Set, Nil, Nan, Bool,
Inf
Type = ref object
## A wrapper around
## compile-time types
node: ASTNode
case kind: TypeKind:
of Function:
returnType: Type
of List, Tuple, Set:
memberType: Type
of Dict:
keyType: Type
valueType: Type
else:
discard
Name = ref object Name = ref object
## A compile-time wrapper around ## A compile-time wrapper around
## statically resolved names. ## statically resolved names.
@ -348,19 +326,13 @@ proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.sc
## than the given one and modifies the code emitted for it ## than the given one and modifies the code emitted for it
## to store it as a closure variable if it is. Does nothing if the name ## to store it as a closure variable if it is. Does nothing if the name
## hasn't been declared yet or is unreachable (for example if it's ## hasn't been declared yet or is unreachable (for example if it's
## declared as private in another module), if the name itself is a ## declared as private in another module). This function must be called
## global variable and if either the current or the outer scope are ## each time a name is referenced in order for closed-over variables
## the global (outermost) one. This function must be called each
## time a name is referenced in order for closed-over variables
## to be emitted properly, otherwise the runtime may behave ## to be emitted properly, otherwise the runtime may behave
## unpredictably or crash ## unpredictably or crash
if depth == 0 or depth - 1 == 0:
return
let entry = self.resolve(name) let entry = self.resolve(name)
if entry == nil: if entry == nil:
return return
if entry.depth == 0:
return
if entry.depth < depth: if entry.depth < depth:
# Ding! The given name is closed over: we need to # Ding! The given name is closed over: we need to
# change the StoreVar instruction that created this # change the StoreVar instruction that created this
@ -370,7 +342,7 @@ proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.sc
# whether or not this function is called # whether or not this function is called
self.closedOver.add(entry.name) self.closedOver.add(entry.name)
if self.closedOver.len() >= 16777216: if self.closedOver.len() >= 16777216:
self.error("too many consecutive closure-over variables (max is 16777216)") self.error("too many consecutive closed-over variables (max is 16777216)")
let idx = self.closedOver.high().toTriple() let idx = self.closedOver.high().toTriple()
self.chunk.code[entry.codePos] = StoreHeap.uint8 self.chunk.code[entry.codePos] = StoreHeap.uint8
self.chunk.code[entry.codePos + 1] = idx[0] self.chunk.code[entry.codePos + 1] = idx[0]
@ -378,70 +350,10 @@ proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.sc
self.chunk.code[entry.codePos + 3] = idx[2] self.chunk.code[entry.codePos + 3] = idx[2]
proc toIntrinsic(name: string): Type =
## Converts a string to an intrinsic
## type if it is valid and returns nil
## otherwise
if name in ["int", "int64", "i64"]:
return Type(kind: Int64)
elif name in ["uint64", "u64"]:
return Type(kind: UInt64)
elif name in ["int32", "i32"]:
return Type(kind: Int32)
elif name in ["uint32", "u32"]:
return Type(kind: UInt32)
elif name in ["int16", "i16"]:
return Type(kind: Int16)
elif name in ["uint16", "u16"]:
return Type(kind: UInt16)
elif name in ["int8", "i8"]:
return Type(kind: Int8)
elif name in ["uint8", "u8"]:
return Type(kind: UInt8)
elif name in ["f64", "float", "float64"]:
return Type(kind: Float64)
elif name in ["f32", "float32"]:
return Type(kind: Float32)
elif name == "byte":
return Type(kind: Byte)
elif name == "char":
return Type(kind: Char)
elif name == "nan":
return Type(kind: Nan)
elif name == "nil":
return Type(kind: Nil)
elif name == "inf":
return Type(kind: Inf)
elif name == "bool":
return Type(kind: Bool)
else:
return nil
proc toIntrinsic(typ: Expression): Type =
## Gets an expression's
## intrinsic type, if possible
if typ == nil:
return nil
case typ.kind:
of identExpr:
return typ.token.lexeme.toIntrinsic()
else:
discard
proc inferValueType(self: Compiler, node: ASTNode): Type = proc inferValueType(self: Compiler, node: ASTNode): Type =
## Infers the type of a given literal expression ## Infers the type of a given literal expression
case node.kind: case node.kind:
of listExpr:
return Type(kind: List, memberType: self.inferExprType(ListExpr(node).valueType))
of tupleExpr:
return Type(kind: Tuple, memberType: self.inferExprType(TupleExpr(node).valueType))
of setExpr:
return Type(kind: Set, memberType: self.inferExprType(SetExpr(node).valueType))
of dictExpr:
let node = DictExpr(node)
return Type(kind: Dict, keyType: self.inferExprType(node.valueType), valueType: self.inferExprType(node.valueType))
of intExpr, binExpr, octExpr, hexExpr: of intExpr, binExpr, octExpr, hexExpr:
let node = LiteralExpr(node) let node = LiteralExpr(node)
let size = node.token.lexeme.split("'") let size = node.token.lexeme.split("'")
@ -473,9 +385,9 @@ proc inferValueType(self: Compiler, node: ASTNode): Type =
of falseExpr: of falseExpr:
return Type(kind: Bool) return Type(kind: Bool)
of nanExpr: of nanExpr:
return Type(kind: Nan) return Type(kind: TypeKind.Nan)
of infExpr: of infExpr:
return Type(kind: Inf) return Type(kind: TypeKind.Inf)
else: else:
discard # TODO discard # TODO
@ -499,8 +411,7 @@ proc inferExprType(self: Compiler, node: ASTNode): Type =
return a return a
of {intExpr, hexExpr, binExpr, octExpr, of {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr, strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr, listExpr, nanExpr, floatExpr, nilExpr
dictExpr, setExpr, tupleExpr
}: }:
return self.inferValueType(node) return self.inferValueType(node)
else: else:
@ -528,10 +439,13 @@ proc inferDeclType(self: Compiler, node: Declaration): Type =
proc typeToStr(self: Compiler, typ: Type): string = proc typeToStr(self: Compiler, typ: Type): string =
## Returns the string representation of a
## type object
case typ.kind: case typ.kind:
of {Int8, UInt8, Int16, UInt16, Int32, of Int8, UInt8, Int16, UInt16, Int32,
UInt32, Int64, UInt64, Float32, Float64, UInt32, Int64, UInt64, Float32, Float64,
Char, Byte, String, Nil, Nan, Bool, Inf}: Char, Byte, String, Nil, TypeKind.Nan, Bool,
TypeKind.Inf:
return ($typ.kind).toLowerAscii() return ($typ.kind).toLowerAscii()
of Function: of Function:
result = "function (" result = "function ("
@ -552,35 +466,27 @@ proc typeToStr(self: Compiler, typ: Type): string =
result &= ")" result &= ")"
else: else:
discard # Unreachable discard # Unreachable
of List, Tuple, Set:
result &= &"{($typ.kind).toLowerAscii()}["
of Dict:
result &= &"{($typ.kind).toLowerAscii()}[]"
else: else:
discard discard
proc `==`(self, other: Type): bool =
if system.`==`(self, nil): proc toIntrinsic(self: Compiler, typ: Expression): Type =
return system.`==`(other, nil) ## Gets an expression's
elif system.`==`(other, nil): ## intrinsic type, if possible
return system.`==`(self, nil) if typ == nil:
if self.kind != other.kind: return nil
return false case typ.kind:
case self.kind: of trueExpr, falseExpr, intExpr, floatExpr:
of {Int8, UInt8, Int16, UInt16, Int32, return typ.token.lexeme.toIntrinsic()
UInt32, Int64, UInt64, Float32, Float64, of identExpr:
Char, Byte, String, Nil, Nan, Bool, Inf}: let inferred = self.inferExprType(typ)
return true if inferred != nil:
of Function: return
discard # TODO
of List, Tuple, Set:
return self.memberType == other.memberType
of Dict:
return self.keyType == other.keyType and self.valueType == other.valueType
else: else:
discard discard
## End of utility functions ## End of utility functions
proc literal(self: Compiler, node: ASTNode) = proc literal(self: Compiler, node: ASTNode) =
@ -646,39 +552,6 @@ proc literal(self: Compiler, node: ASTNode) =
except ValueError: except ValueError:
self.error("floating point value out of range") self.error("floating point value out of range")
self.emitConstant(y) self.emitConstant(y)
of listExpr:
var y = ListExpr(node)
if y.members.len() > 16777216:
self.error("list literals can't have more than 16777216 elements")
for member in y.members:
self.expression(member)
self.emitByte(BuildList)
self.emitBytes(y.members.len().toTriple()) # 24-bit integer, meaning collection literals can have up to 2^24 elements
of tupleExpr:
var y = TupleExpr(node)
if y.members.len() > 16777216:
self.error("tuple literals can't have more than 16777216 elements")
for member in y.members:
self.expression(member)
self.emitByte(BuildTuple)
self.emitBytes(y.members.len().toTriple())
of setExpr:
var y = SetExpr(node)
if y.members.len() > 16777216:
self.error("set literals can't have more than 16777216 elements")
for member in y.members:
self.expression(member)
self.emitByte(BuildSet)
self.emitBytes(y.members.len().toTriple())
of dictExpr:
var y = DictExpr(node)
if y.keys.len() > 16777216:
self.error("dict literals can't have more than 16777216 elements")
for (key, value) in zip(y.keys, y.values):
self.expression(key)
self.expression(value)
self.emitByte(BuildDict)
self.emitBytes(y.keys.len().toTriple())
of awaitExpr: of awaitExpr:
var y = AwaitExpr(node) var y = AwaitExpr(node)
self.expression(y.expression) self.expression(y.expression)
@ -1077,8 +950,7 @@ proc expression(self: Compiler, node: ASTNode) =
# Binary expressions such as 2 ^ 5 and 0.66 * 3.14 # Binary expressions such as 2 ^ 5 and 0.66 * 3.14
self.binary(BinaryExpr(node)) self.binary(BinaryExpr(node))
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr,
infExpr, nanExpr, floatExpr, nilExpr, tupleExpr, setExpr, listExpr, infExpr, nanExpr, floatExpr, nilExpr:
dictExpr:
# Since all of these AST nodes mostly share # Since all of these AST nodes mostly share
# the same overall structure, and the kind # the same overall structure, and the kind
# discriminant is enough to tell one # discriminant is enough to tell one
@ -1233,7 +1105,7 @@ proc statement(self: Compiler, node: ASTNode) =
proc varDecl(self: Compiler, node: VarDecl) = proc varDecl(self: Compiler, node: VarDecl) =
## Compiles variable declarations ## Compiles variable declarations
let kind = node.valueType.toIntrinsic() let kind = self.toIntrinsic(node.valueType)
let typ = self.inferExprType(node.value) let typ = self.inferExprType(node.value)
if kind == nil and typ == nil: if kind == nil and typ == nil:
self.error(&"cannot determine the type of '{node.name.token.lexeme}'") self.error(&"cannot determine the type of '{node.name.token.lexeme}'")

83
src/frontend/meta/ast.nim
View File

@ -18,7 +18,6 @@
import strformat import strformat
import strutils import strutils
import sequtils
import token import token
@ -65,10 +64,6 @@ type
# Primary expressions # Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4) groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
trueExpr, trueExpr,
listExpr,
tupleExpr,
dictExpr,
setExpr,
falseExpr, falseExpr,
strExpr, strExpr,
charExpr, charExpr,
@ -132,20 +127,6 @@ type
NanExpr* = ref object of LiteralExpr NanExpr* = ref object of LiteralExpr
InfExpr* = ref object of LiteralExpr InfExpr* = ref object of LiteralExpr
ListExpr* = ref object of LiteralExpr
members*: seq[Expression]
valueType*: IdentExpr
SetExpr* = ref object of ListExpr
TupleExpr* = ref object of ListExpr
DictExpr* = ref object of Expression
keys*: seq[Expression]
values*: seq[Expression]
keyType*: IdentExpr
valueType*: IdentExpr
IdentExpr* = ref object of LiteralExpr IdentExpr* = ref object of LiteralExpr
name*: Token name*: Token
@ -184,7 +165,7 @@ type
LambdaExpr* = ref object of Expression LambdaExpr* = ref object of Expression
body*: Statement body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]] arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression] defaults*: seq[Expression]
isGenerator*: bool isGenerator*: bool
isAsync*: bool isAsync*: bool
@ -263,7 +244,7 @@ type
FunDecl* = ref object of Declaration FunDecl* = ref object of Declaration
name*: IdentExpr name*: IdentExpr
body*: Statement body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]] arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression] defaults*: seq[Expression]
isAsync*: bool isAsync*: bool
isGenerator*: bool isGenerator*: bool
@ -276,21 +257,10 @@ proc isConst*(self: ASTNode): bool =
## AST node represents a value ## AST node represents a value
## of constant type. All integers, ## of constant type. All integers,
## strings and singletons count as ## strings and singletons count as
## constants, as well as collections ## constants
## comprised only of those types
case self.kind: case self.kind:
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr: of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr:
return true return true
of tupleExpr, setExpr, listExpr:
for item in ListExpr(self).members:
if not item.isConst():
return false
return true
of dictExpr:
for (key, value) in zip(DictExpr(self).keys, DictExpr(self).values):
if not key.isConst() or not value.isConst():
return false
return true
else: else:
return false return false
@ -299,8 +269,7 @@ proc isLiteral*(self: ASTNode): bool {.inline.} =
## Returns if the AST node represents a literal ## Returns if the AST node represents a literal
self.kind in {intExpr, hexExpr, binExpr, octExpr, self.kind in {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr, strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr, listExpr, nanExpr, floatExpr, nilExpr
dictExpr, setExpr, tupleExpr
} }
## AST node constructors ## AST node constructors
@ -373,7 +342,7 @@ proc newGroupingExpr*(expression: Expression, token: Token): GroupingExpr =
result.token = token result.token = token
proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]], defaults: seq[Expression], body: Statement, proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression], body: Statement,
isGenerator: bool, isAsync: bool, token: Token, returnType: Expression): LambdaExpr = isGenerator: bool, isAsync: bool, token: Token, returnType: Expression): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr) result = LambdaExpr(kind: lambdaExpr)
result.body = body result.body = body
@ -392,34 +361,6 @@ proc newGetItemExpr*(obj: Expression, name: IdentExpr, token: Token): GetItemExp
result.token = token result.token = token
proc newListExpr*(members: seq[Expression], token: Token): ListExpr =
result = ListExpr(kind: listExpr)
result.members = members
result.token = token
result.literal = result.token
proc newSetExpr*(members: seq[Expression], token: Token): SetExpr =
result = SetExpr(kind: setExpr)
result.members = members
result.token = token
result.literal = result.token
proc newTupleExpr*(members: seq[Expression], token: Token): TupleExpr =
result = TupleExpr(kind: tupleExpr)
result.members = members
result.token = token
result.literal = result.token
proc newDictExpr*(keys, values: seq[Expression], token: Token): DictExpr =
result = DictExpr(kind: dictExpr)
result.keys = keys
result.values = values
result.token = token
proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetItemExpr = proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetItemExpr =
result = SetItemExpr(kind: setItemExpr) result = SetItemExpr(kind: setItemExpr)
result.obj = obj result.obj = obj
@ -593,7 +534,7 @@ proc newVarDecl*(name: IdentExpr, value: Expression, isConst: bool = false,
result.pragmas = pragmas result.pragmas = pragmas
proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]], defaults: seq[Expression], proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression],
body: Statement, isAsync, isGenerator: bool, body: Statement, isAsync, isGenerator: bool,
isPrivate: bool, token: Token, pragmas: seq[Token], isPrivate: bool, token: Token, pragmas: seq[Token],
returnType: Expression): FunDecl = returnType: Expression): FunDecl =
@ -695,18 +636,6 @@ proc `$`*(self: ASTNode): string =
of funDecl: of funDecl:
var self = FunDecl(self) var self = FunDecl(self)
result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate})""" result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate})"""
of tupleExpr:
var self = TupleExpr(self)
result &= &"""Tuple([{self.members.join(", ")}])"""
of setExpr:
var self = SetExpr(self)
result &= &"""Set([{self.members.join(", ")}])"""
of listExpr:
var self = ListExpr(self)
result &= &"""List([{self.members.join(", ")}])"""
of dictExpr:
var self = DictExpr(self)
result &= &"""Dict(keys=[{self.keys.join(", ")}], values=[{self.values.join(", ")}])"""
of lambdaExpr: of lambdaExpr:
var self = LambdaExpr(self) var self = LambdaExpr(self)
result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync})""" result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync})"""

30
src/frontend/meta/bytecode.nim
View File

@ -14,6 +14,7 @@
## Low level bytecode implementation details ## Low level bytecode implementation details
import ast import ast
import typing
import ../../util/multibyte import ../../util/multibyte
import errors import errors
@ -27,9 +28,11 @@ export ast
type type
Chunk* = ref object Chunk* = ref object
## A piece of bytecode. ## A piece of bytecode.
## Consts represents the constants table the code is referring to. ## consts represents the constants table the code is referring to.
## Code is the linear sequence of compiled bytecode instructions. ## byteConsts represents the actual encoding of the constants table
## Lines maps bytecode instructions to line numbers using Run ## used when serializing to/from a bytecode stream.
## code is the linear sequence of compiled bytecode instructions.
## lines maps bytecode instructions to line numbers using Run
## Length Encoding. Instructions are encoded in groups whose structure ## Length Encoding. Instructions are encoded in groups whose structure
## follows the following schema: ## follows the following schema:
## - The first integer represents the line number ## - The first integer represents the line number
@ -43,12 +46,13 @@ type
## This is more efficient than using the naive approach, which would encode ## This is more efficient than using the naive approach, which would encode
## the same line number multiple times and waste considerable amounts of space. ## the same line number multiple times and waste considerable amounts of space.
consts*: seq[LiteralExpr] consts*: seq[LiteralExpr]
byteConsts*: seq[uint8]
code*: seq[uint8] code*: seq[uint8]
lines*: seq[int] lines*: seq[int]
reuseConsts*: bool reuseConsts*: bool
OpCode* {.pure.} = enum OpCode* {.pure.} = enum
## Enum of possible opcodes. ## Enum of Peon's bytecode opcodes
# Note: x represents the argument # Note: x represents the argument
# to unary opcodes, while a and b # to unary opcodes, while a and b
@ -60,7 +64,7 @@ type
# closure array. Some other opcodes (e.g. # closure array. Some other opcodes (e.g.
# jumps), take arguments in the form of 16 # jumps), take arguments in the form of 16
# or 24 bit numbers that are defined statically # or 24 bit numbers that are defined statically
# at compilation time into the bytecode. # at compilation time into the bytecode
# These push a constant onto the stack # These push a constant onto the stack
LoadInt64 = 0u8, LoadInt64 = 0u8,
@ -107,6 +111,7 @@ type
LongJumpForwards, LongJumpForwards,
LongJumpBackwards, LongJumpBackwards,
## Functions ## Functions
Call, # Calls a function
Return # Returns from the current function Return # Returns from the current function
## Exception handling ## Exception handling
Raise, # Raises exception x or re-raises active exception if x is nil Raise, # Raises exception x or re-raises active exception if x is nil
@ -116,13 +121,8 @@ type
Yield, # Yields control from a generator back to the caller Yield, # Yields control from a generator back to the caller
## Coroutines ## Coroutines
Await, # Calls an asynchronous function Await, # Calls an asynchronous function
## Collection literals
BuildList,
BuildDict,
BuildSet,
BuildTuple,
## Misc ## Misc
Assert, # Raises an AssertionFailed exception if the x is false Assert, # Raises an AssertionFailed exception if x is false
NoOp, # Just a no-op NoOp, # Just a no-op
@ -163,9 +163,6 @@ const jumpInstructions* = {JumpIfFalse, JumpIfFalsePop,
LongJumpForwards, LongJumpBackwards, LongJumpForwards, LongJumpBackwards,
JumpIfTrue, LongJumpIfTrue} JumpIfTrue, LongJumpIfTrue}
# Collection instructions push a built-in collection type onto the stack
const collectionInstructions* = {BuildList, BuildDict, BuildSet, BuildTuple}
proc newChunk*(reuseConsts: bool = true): Chunk = proc newChunk*(reuseConsts: bool = true): Chunk =
## Initializes a new, empty chunk ## Initializes a new, empty chunk
@ -235,7 +232,6 @@ proc findOrAddConstant(self: Chunk, constant: LiteralExpr): int =
if c.kind != constant.kind: if c.kind != constant.kind:
continue continue
if constant.isConst(): if constant.isConst():
var constant = constant
if c.literal.lexeme == constant.literal.lexeme: if c.literal.lexeme == constant.literal.lexeme:
# This wouldn't work for stuff like 2e3 and 2000.0, but those # This wouldn't work for stuff like 2e3 and 2000.0, but those
# forms are collapsed in the compiler before being written # forms are collapsed in the compiler before being written
@ -256,10 +252,10 @@ proc addConstant*(self: Chunk, constant: LiteralExpr): array[3, uint8] =
## Writes a constant to a chunk. Returns its index casted to a 3-byte ## Writes a constant to a chunk. Returns its index casted to a 3-byte
## sequence (array). Constant indexes are reused if a constant is used ## sequence (array). Constant indexes are reused if a constant is used
## more than once and self.reuseConsts equals true ## more than once and self.reuseConsts equals true
if self.consts.len() == 16777215: if self.consts.high() == 16777215:
# The constant index is a 24 bit unsigned integer, so that's as far # The constant index is a 24 bit unsigned integer, so that's as far
# as we can index into the constant table (the same applies # as we can index into the constant table (the same applies
# to our stack by the way). Not that anyone's ever gonna hit this # to our stack by the way). Not that anyone's ever gonna hit this
# limit in the real world, but you know, just in case # limit in the real world, but you know, just in case
raise newException(CompileError, "cannot encode more than 16777215 constants") raise newException(CompileError, "cannot encode more than 16777216 constants")
result = self.findOrAddConstant(constant).toTriple() result = self.findOrAddConstant(constant).toTriple()

2
src/frontend/meta/token.nim
View File

@ -39,7 +39,7 @@ type
Foreach, Yield, Of, Defer, Foreach, Yield, Of, Defer,
Try, Except, Finally, Type, Try, Except, Finally, Type,
Operator, Case, Enum, From, Operator, Case, Enum, From,
Emit, As Emit, As, Ptr, Ref
# Literal types # Literal types
Integer, Float, String, Identifier, Integer, Float, String, Identifier,

94
src/frontend/parser.nim
View File

@ -224,65 +224,8 @@ proc primary(self: Parser): Expression =
result = newIdentExpr(self.step()) result = newIdentExpr(self.step())
of LeftParen: of LeftParen:
let tok = self.step() let tok = self.step()
if self.match(RightParen): result = newGroupingExpr(self.expression(), tok)
# This yields an empty tuple self.expect(RightParen, "unterminated parenthesized expression")
result = newTupleExpr(@[], tok)
else:
result = self.expression()
if self.match(Comma):
var tupleObject = newTupleExpr(@[result], tok)
while not self.check(RightParen):
tupleObject.members.add(self.expression())
if not self.match(Comma):
break
result = tupleObject
self.expect(RightParen, "unterminated tuple literal")
else:
self.expect(RightParen, "unterminated parenthesized expression")
result = newGroupingExpr(result, tok)
of LeftBracket:
let tok = self.step()
if self.match(RightBracket):
# This yields an empty list
result = newListExpr(@[], tok)
else:
var listObject = newListExpr(@[], tok)
while not self.check(RightBracket):
listObject.members.add(self.expression())
if not self.match(Comma):
break
result = listObject
self.expect(RightBracket, "unterminated list literal")
of LeftBrace:
let tok = self.step()
if self.match(RightBrace):
# This yields an empty dictionary, not an empty set!
# For empty sets, there will be a builtin set() type
# that can be instantiated with no arguments
result = newDictExpr(@[], @[], tok)
else:
result = self.expression()
if self.match(Comma) or self.check(RightBrace):
var setObject = newSetExpr(@[result], tok)
while not self.check(RightBrace):
setObject.members.add(self.expression())
if not self.match(Comma):
break
result = setObject
self.expect(RightBrace, "unterminated set literal")
elif self.match(Colon):
var dictObject = newDictExpr(@[result], @[self.expression()], tok)
if self.match(RightBrace):
return dictObject
if self.match(Comma):
while not self.check(RightBrace):
dictObject.keys.add(self.expression())
self.expect(Colon)
dictObject.values.add(self.expression())
if not self.match(Comma):
break
self.expect(RightBrace, "unterminated dict literal")
result = dictObject
of Yield: of Yield:
let tok = self.step() let tok = self.step()
if self.currentFunction == nil: if self.currentFunction == nil:
@ -704,19 +647,6 @@ proc tryStmt(self: Parser): Statement =
elif BinaryExpr(excName).a.kind != identExpr: elif BinaryExpr(excName).a.kind != identExpr:
self.error("expecting exception name") self.error("expecting exception name")
excName = BinaryExpr(excName).a excName = BinaryExpr(excName).a
# Note how we don't use elif here: when the if above sets excName to As'
# first operand, that might be a tuple, which we unpack below
if excName.kind == tupleExpr:
# This allows to do except (a, b, c) as SomeError {...}
# TODO: Consider adding the ability to make exc a sequence
# instead of adding the same body with different exception
# types each time
handlerBody = self.statement()
for element in TupleExpr(excName).members:
if element.kind != identExpr:
self.error("expecting exception name")
handlers.add((body: handlerBody, exc: IdentExpr(element), name: asName))
continue
else: else:
excName = nil excName = nil
if self.match(Else): if self.match(Else):
@ -859,8 +789,8 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declarat
discard # Unreachable discard # Unreachable
proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]], proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]],
parameter: var tuple[name: IdentExpr, valueType: Expression, mutable: bool], parameter: var tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool],
defaults: var seq[Expression]) = defaults: var seq[Expression]) =
while not self.check(RightParen): while not self.check(RightParen):
if arguments.len > 255: if arguments.len > 255:
@ -871,6 +801,10 @@ proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr,
parameter.mutable = false parameter.mutable = false
if self.match(Var): if self.match(Var):
parameter.mutable = true parameter.mutable = true
elif self.match(Ptr):
parameter.isPtr = true
elif self.match(Ref):
parameter.isRef = true
parameter.valueType = self.expression() parameter.valueType = self.expression()
for i in countdown(arguments.high(), 0): for i in countdown(arguments.high(), 0):
if arguments[i].valueType != nil: if arguments[i].valueType != nil:
@ -898,7 +832,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
## Parses functions, coroutines, generators, anonymous functions and custom operators ## Parses functions, coroutines, generators, anonymous functions and custom operators
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingFunction = self.currentFunction var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
var returnType: Expression var returnType: Expression
if not isLambda and self.check(Identifier): if not isLambda and self.check(Identifier):
@ -941,12 +875,12 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# the type declaration for a function lacks # the type declaration for a function lacks
# the braces that would qualify it as an # the braces that would qualify it as an
# expression # expression
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator, returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine, isAsync=self.peek(-1).kind == Coroutine,
token=self.peek(-1), returnType=nil) token=self.peek(-1), returnType=nil)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool] var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen): if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(Colon): if self.match(Colon):
@ -955,17 +889,17 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
returnType = self.expression() returnType = self.expression()
if not self.match(LeftBrace): if not self.match(LeftBrace):
self.expect(LeftParen) self.expect(LeftParen)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool] var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(Colon): if self.match(Colon):
# Function's return type # Function's return type
if self.match([Function, Coroutine, Generator]): if self.match([Function, Coroutine, Generator]):
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator, returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine, isAsync=self.peek(-1).kind == Coroutine,
token=self.peek(-1), returnType=nil) token=self.peek(-1), returnType=nil)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool] var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen): if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(Colon): if self.match(Colon):

2
src/test.nim
View File

@ -244,6 +244,8 @@ proc fillSymbolTable(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("and", TokenType.LogicalAnd) tokenizer.symbols.addKeyword("and", TokenType.LogicalAnd)
tokenizer.symbols.addKeyword("or", TokenType.LogicalOr) tokenizer.symbols.addKeyword("or", TokenType.LogicalOr)
tokenizer.symbols.addKeyword("not", TokenType.LogicalNot) tokenizer.symbols.addKeyword("not", TokenType.LogicalNot)
tokenizer.symbols.addKeyword("ref", Ref)
tokenizer.symbols.addKeyword("ptr", Ptr)
# P.S.: There's no reason for the order of addition of # P.S.: There's no reason for the order of addition of
# symbols to be ascending in length (the symbol table uses # symbols to be ascending in length (the symbol table uses

31
src/util/debugger.nim
View File

@ -75,7 +75,7 @@ proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int
proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int = proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
## Debugs instructions that operate on a hardcoded value value on the stack using a 16-bit operand ## Debugs instructions that operate on a hardcoded value on the stack using a 16-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble() var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble()
printInstruction(instruction) printInstruction(instruction)
stdout.write(&", has argument ") stdout.write(&", has argument ")
@ -123,33 +123,6 @@ proc jumpInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
return offset + 3 return offset + 3
proc collectionInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
## Debugs instructions that push collection types on the stack
var elemCount = int([chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple())
printInstruction(instruction, true)
case instruction:
of BuildList, BuildTuple, BuildSet:
var elements: seq[ASTNode] = @[]
for n in countup(0, elemCount - 1):
elements.add(chunk.consts[n])
printDebug("Elements: ")
setForegroundColor(fgYellow)
stdout.write(&"""[{elements.join(", ")}]""")
setForegroundColor(fgGreen)
of BuildDict:
var elements: seq[tuple[key: ASTNode, value: ASTNode]] = @[]
for n in countup(0, (elemCount - 1) * 2, 2):
elements.add((key: chunk.consts[n], value: chunk.consts[n + 1]))
printDebug("Elements: ")
setForegroundColor(fgYellow)
stdout.write(&"""[{elements.join(", ")}]""")
setForegroundColor(fgGreen)
else:
discard # Unreachable
echo ""
return offset + 4
proc disassembleInstruction*(chunk: Chunk, offset: int): int = proc disassembleInstruction*(chunk: Chunk, offset: int): int =
## Takes one bytecode instruction and prints it ## Takes one bytecode instruction and prints it
setForegroundColor(fgGreen) setForegroundColor(fgGreen)
@ -175,8 +148,6 @@ proc disassembleInstruction*(chunk: Chunk, offset: int): int =
result = argumentDoubleInstruction(opcode, chunk, offset) result = argumentDoubleInstruction(opcode, chunk, offset)
of jumpInstructions: of jumpInstructions:
result = jumpInstruction(opcode, chunk, offset) result = jumpInstruction(opcode, chunk, offset)
of collectionInstructions:
result = collectionInstruction(opcode, chunk, offset)
else: else:
echo &"DEBUG - Unknown opcode {opcode} at index {offset}" echo &"DEBUG - Unknown opcode {opcode} at index {offset}"
result = offset + 1 result = offset + 1

7
src/util/serializer.nim
View File

@ -151,10 +151,9 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
proc readConstants(self: Serializer, stream: seq[byte]): int = proc readConstants(self: Serializer, stream: seq[byte]): int =
## Reads the constant table from the given stream and ## Reads the constant table from the given stream and
## adds each constant to the chunk object (note: most compile-time ## adds each constant to the chunk object.
## information such as the original token objects and line info is lost when ## Returns the number of bytes that were processed in
## serializing the data, so those fields are set to nil or some default ## the stream
## value). Returns the number of bytes that were processed in the stream
var stream = stream var stream = stream
var count: int = 0 var count: int = 0
while true: while true: