Massive improvements to import system (added export statement, initial work on namespaces, module names can now be paths, added module search paths)

This commit is contained in:
Mattia Giambirtone 2022-10-17 11:28:00 +02:00
parent c7893fb14b
commit d33a597f19
8 changed files with 354 additions and 489 deletions

View File

@ -28,6 +28,8 @@ const PeonBytecodeMarker* = "PEON_BYTECODE" # Magic value at the beginning of
const HeapGrowFactor* = 2 # The growth factor used by the GC to schedule the next collection
const FirstGC* = 1024 * 1024; # How many bytes to allocate before running the first GC
const enableVMChecks* {.booldefine.} = true; # Enables all types of compiler (nim-wise) checks in the VM
# List of paths where peon looks for modules, in order (empty path means current directory, which always takes precedence)
const lookupPaths*: seq[string] = @["", "src/peon/stdlib"]
when HeapGrowFactor <= 1:
{.fatal: "Heap growth factor must be > 1".}
const PeonVersion* = (major: 0, minor: 1, patch: 0)

View File

@ -18,6 +18,8 @@ import ../util/multibyte
import ../util/symbols
import lexer as l
import parser as p
import ../config
import std/tables
import std/strformat
@ -88,13 +90,13 @@ export bytecode
type
NameKind {.pure.} = enum
## A name enumeration type
None, Argument, Var, Function, Type
None, Module, Argument, Var, Function, CustomType, Enum
Name = ref object
## A compile-time wrapper around
## statically resolved names
# Name of the identifier
name: IdentExpr
# The name's identifier
ident: IdentExpr
# Type of the identifier (NOT of the value!)
kind: NameKind
# Owner of the identifier (module)
@ -124,18 +126,20 @@ type
# is needed because we compile declarations only
# if they're actually used
node: Declaration
# Is this name exported? (Only makes sense if isPrivate
# equals false)
exported: bool
Loop = object
## A "loop object" used
## by the compiler to emit
## appropriate jump offsets
## for continue and break
## statements
# Position in the bytecode where the loop starts
start: int
# Scope depth where the loop is located
depth: int
# Absolute jump offsets into our bytecode that we need to
# Jump offsets into our bytecode that we need to
# patch. Used for break statements
breakJumps: seq[int]
@ -162,8 +166,6 @@ type
scopeOwners: seq[tuple[owner: Name, depth: int]]
# The current function being compiled
currentFunction: Name
# Are optimizations turned on?
enableOptimizations: bool
# The current loop being compiled (used to
# keep track of where to jump)
currentLoop: Loop
@ -193,9 +195,10 @@ type
closedOver: seq[Name]
# Compiler procedures called by pragmas
compilerProcs: TableRef[string, proc (self: Compiler, pragma: Pragma, node: ASTNode, name: Name)]
# Stores line data
# Stores line data for error reporting
lines: seq[tuple[start, stop: int]]
# The source of the current module
# The source of the current module,
# used for error reporting
source: string
# Currently imported modules
modules: HashSet[string]
@ -203,6 +206,11 @@ type
jumps: seq[tuple[patched: bool, offset: int]]
# List of CFI start offsets into our CFI data
cfiOffsets: seq[tuple[start, stop: int, fn: Name]]
# We store these objects to compile modules
lexer: Lexer
parser: Parser
# Are we compiling the main module?
isMainModule: bool
CompileError* = ref object of PeonException
compiler*: Compiler
node*: ASTNode
@ -212,18 +220,19 @@ type
# Forward declarations
proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil,
terminateScope: bool = true, incremental: bool = false): Chunk
incremental: bool = false, isMainModule: bool = true): Chunk
proc expression(self: Compiler, node: Expression)
proc statement(self: Compiler, node: Statement)
proc declaration(self: Compiler, node: Declaration)
proc peek(self: Compiler, distance: int = 0): ASTNode
proc identifier(self: Compiler, node: IdentExpr)
proc varDecl(self: Compiler, node: VarDecl, name: Name)
proc specializeGeneric(self: Compiler, fn: Name, args: seq[Expression]): Name
proc specialize(self: Compiler, name: Name, args: seq[Expression]): Name
proc matchImpl(self: Compiler, name: string, kind: Type, node: ASTNode = nil): Name
proc infer(self: Compiler, node: LiteralExpr): Type
proc infer(self: Compiler, node: Expression): Type
proc findByName(self: Compiler, name: string): seq[Name]
proc findByModule(self: Compiler, name: string): seq[Name]
proc findByType(self: Compiler, name: string, kind: Type, depth: int = -1): seq[Name]
proc compare(self: Compiler, a, b: Type): bool
proc patchReturnAddress(self: Compiler, pos: int)
@ -232,12 +241,12 @@ proc handlePurePragma(self: Compiler, pragma: Pragma, node: ASTnode, name: Name)
proc dispatchPragmas(self: Compiler, node: ASTnode, name: Name)
proc funDecl(self: Compiler, node: FunDecl, name: Name)
proc typeDecl(self: Compiler, node: TypeDecl, name: Name)
proc compileModule(self: Compiler, filename: string)
proc compileModule(self: Compiler, moduleName: string)
proc generateCall(self: Compiler, fn: Name, args: seq[Expression], line: int)
# End of forward declarations
proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Compiler =
proc newCompiler*(replMode: bool = false): Compiler =
## Initializes a new Compiler object
new(result)
result.ast = @[]
@ -248,7 +257,6 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com
result.lines = @[]
result.jumps = @[]
result.currentFunction = nil
result.enableOptimizations = enableOptimizations
result.replMode = replMode
result.currentModule = ""
result.compilerProcs = newTable[string, proc (self: Compiler, pragma: Pragma, node: ASTNode, name: Name)]()
@ -256,6 +264,10 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com
result.compilerProcs["pure"] = handlePurePragma
result.source = ""
result.scopeOwners = @[]
result.lexer = newLexer()
result.lexer.fillSymbolTable()
result.parser = newParser()
result.isMainModule = false
## Public getters for nicer error formatting
@ -483,17 +495,19 @@ proc resolve(self: Compiler, name: IdentExpr,
## for the first time, calling this function will also
## cause its declaration to be compiled
for obj in reversed(self.names):
if obj.name.token.lexeme == name.token.lexeme:
if obj.isPrivate and obj.owner != self.currentModule:
continue # There may be a name in the current module that
# matches, so we skip this
if obj.ident.token.lexeme == name.token.lexeme:
if obj.owner != self.currentModule:
if obj.isPrivate:
continue
elif obj.exported:
return obj
if not obj.resolved:
obj.resolved = true
obj.codePos = self.chunk.code.len()
case obj.kind:
of NameKind.Var:
self.varDecl(VarDecl(obj.node), obj)
of NameKind.Type:
of NameKind.CustomType:
self.typeDecl(TypeDecl(obj.node), obj)
of NameKind.Function:
if not obj.valueType.isGeneric:
@ -512,17 +526,19 @@ proc resolve(self: Compiler, name: string,
## Version of resolve that takes strings instead
## of AST nodes
for obj in reversed(self.names):
if obj.name.token.lexeme == name:
if obj.isPrivate and obj.owner != self.currentModule:
continue # There may be a name in the current module that
# matches, so we skip this
if obj.ident.token.lexeme == name:
if obj.owner != self.currentModule:
if obj.isPrivate:
continue
elif obj.exported:
return obj
if not obj.resolved:
obj.resolved = true
obj.codePos = self.chunk.code.len()
case obj.kind:
of NameKind.Var:
self.varDecl(VarDecl(obj.node), obj)
of NameKind.Type:
of NameKind.CustomType:
self.typeDecl(TypeDecl(obj.node), obj)
of NameKind.Function:
if not obj.valueType.isGeneric:
@ -780,13 +796,13 @@ proc infer(self: Compiler, node: Expression): Type =
let impl = self.matchImpl(node.operator.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: @[("", self.infer(node.a))]), node)
result = impl.valueType.returnType
if result.kind == Generic:
result = self.specializeGeneric(impl, @[node.a]).valueType.returnType
result = self.specialize(impl, @[node.a]).valueType.returnType
of binaryExpr:
let node = BinaryExpr(node)
let impl = self.matchImpl(node.operator.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: @[("", self.infer(node.a)), ("", self.infer(node.b))]), node)
result = impl.valueType.returnType
if result.kind == Generic:
result = self.specializeGeneric(impl, @[node.a, node.b]).valueType.returnType
result = self.specialize(impl, @[node.a, node.b]).valueType.returnType
of {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr
@ -872,9 +888,18 @@ proc findByName(self: Compiler, name: string): seq[Name] =
## Looks for objects that have been already declared
## with the given name. Returns all objects that apply
for obj in reversed(self.names):
if obj.name.token.lexeme == name:
if obj.isPrivate and obj.owner != self.currentModule:
continue
if obj.ident.token.lexeme == name:
if obj.owner != self.currentModule:
if obj.isPrivate or not obj.exported:
continue
result.add(obj)
proc findByModule(self: Compiler, name: string): seq[Name] =
## Looks for objects that have been already declared AS
## public within the given module. Returns all objects that apply
for obj in reversed(self.names):
if obj.owner == name:
result.add(obj)
@ -911,7 +936,7 @@ proc matchImpl(self: Compiler, name: string, kind: Type, node: ASTNode = nil): N
msg &= "s"
msg &= ": "
for name in names:
msg &= &"\n - in module '{name.owner}' at line {name.name.token.line} of type '{self.typeToStr(name.valueType)}'"
msg &= &"\n - in module '{name.owner}' at line {name.ident.token.line} of type '{self.typeToStr(name.valueType)}'"
if name.valueType.kind != Function:
msg &= ", not a callable"
elif kind.args.len() != name.valueType.args.len():
@ -1060,6 +1085,8 @@ proc endScope(self: Compiler) =
self.error("cannot call endScope with scopeDepth < 0 (This is an internal error and most likely a bug)")
discard self.scopeOwners.pop()
dec(self.scopeDepth)
if not self.isMainModule:
return
var names: seq[Name] = @[]
var popCount = 0
for name in self.names:
@ -1145,7 +1172,7 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m
of binaryExpr:
let condition = BinaryExpr(condition)
case condition.operator.lexeme:
of "|", "or":
of "|":
self.unpackGenerics(condition.a, list)
self.unpackGenerics(condition.b, list)
else:
@ -1153,7 +1180,7 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m
of unaryExpr:
let condition = UnaryExpr(condition)
case condition.operator.lexeme:
of "~", "not":
of "~":
self.unpackGenerics(condition.a, list, accept=false)
else:
self.error("invalid type constraint in generic declaration", condition)
@ -1161,13 +1188,14 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m
self.error("invalid type constraint in generic declaration", condition)
proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name =
proc declareName(self: Compiler, node: ASTNode, mutable: bool = false): Name =
## Statically declares a name into the current scope.
## "Declaring" a name only means updating our internal
## list of identifiers so that further calls to resolve()
## correctly return them. There is no code to actually
## declare a variable at runtime: the value is already
## on the stack
var declaredName: string = ""
case node.kind:
of NodeKind.varDecl:
var node = VarDecl(node)
@ -1176,14 +1204,9 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name
# If someone ever hits this limit in real-world scenarios, I swear I'll
# slap myself 100 times with a sign saying "I'm dumb". Mark my words
self.error("cannot declare more than 16777215 variables at a time")
for name in self.findByName(node.name.token.lexeme):
if name.kind == NameKind.Var and name.depth == self.scopeDepth:
# Trying to redeclare a variable in the same scope is an error, but it's okay
# if it's something else, like a function argument (for example, if you want to copy a
# number to mutate it)
self.error(&"attempt to redeclare '{node.name.token.lexeme}', which was previously defined in '{name.owner}' at line {name.line}")
declaredName = node.name.token.lexeme
self.names.add(Name(depth: self.scopeDepth,
name: node.name,
ident: node.name,
isPrivate: node.isPrivate,
owner: self.currentModule,
isConst: node.isConst,
@ -1213,7 +1236,7 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name
owner: self.currentModule,
line: node.token.line,
valueType: Type(kind: Generic, name: gen.name.token.lexeme, mutable: false, cond: constraints),
name: gen.name,
ident: gen.name,
node: node))
generics.add(self.names[^1])
let fn = Name(depth: self.scopeDepth,
@ -1225,11 +1248,12 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name
args: @[],
fun: node,
children: @[]),
name: node.name,
ident: node.name,
node: node,
isLet: false,
line: node.token.line,
kind: NameKind.Function)
kind: NameKind.Function,
belongsTo: self.currentFunction)
self.names.add(fn)
var name: Name
for argument in node.arguments:
@ -1242,7 +1266,7 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name
isPrivate: true,
owner: self.currentModule,
isConst: false,
name: argument.name,
ident: argument.name,
valueType: nil,
codePos: 0,
isLet: false,
@ -1259,8 +1283,26 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name
if generics.len() > 0:
fn.valueType.isGeneric = true
return fn
of NodeKind.importStmt:
var node = ImportStmt(node)
var name = node.moduleName.token.lexeme.extractFilename().replace(".pn", "")
declaredName = name
for name in self.findByName(name):
if name.kind in [NameKind.Var, NameKind.Module] and name.depth == self.scopeDepth:
self.error(&"attempt to redeclare '{name}', which was previously defined in '{name.owner}' at line {name.line}")
self.names.add(Name(depth: self.scopeDepth,
owner: self.currentModule,
ident: newIdentExpr(Token(kind: Identifier, lexeme: name, line: node.moduleName.token.line)),
line: node.moduleName.token.line,
kind: NameKind.Module,
isPrivate: false
))
return self.names[^1]
else:
discard # TODO: Types, enums
for name in self.findByName(declaredName):
if (name.kind == NameKind.Var and name.depth == self.scopeDepth) or name.kind in [NameKind.Module, NameKind.CustomType, NameKind.Enum]:
self.error(&"attempt to redeclare '{name}', which was previously defined in '{name.owner}' at line {name.line}")
proc emitLoop(self: Compiler, begin: int, line: int) =
@ -1341,18 +1383,17 @@ proc patchReturnAddress(self: Compiler, pos: int) =
self.chunk.consts[pos + 7] = address[7]
proc terminateProgram(self: Compiler, pos: int, terminateScope: bool = true) =
proc terminateProgram(self: Compiler, pos: int) =
## Utility to terminate a peon program
if terminateScope:
self.endScope()
self.patchReturnAddress(pos + 3)
self.endScope()
self.emitByte(OpCode.Return, self.peek().token.line)
self.emitByte(0, self.peek().token.line) # Entry point has no return value (TODO: Add easter eggs, cuz why not)
self.patchReturnAddress(pos)
proc beginProgram(self: Compiler, incremental: bool = false): int =
## Utility to begin a peon program
## compiled. Returns the position of
proc beginProgram(self: Compiler): int =
## Utility to begin a peon program's
## bytecode. Returns the position of
## a dummy return address of the program's
## entry point to be patched by terminateProgram
@ -1366,11 +1407,7 @@ proc beginProgram(self: Compiler, incremental: bool = false): int =
# of where our program ends (which we don't know yet).
# To fix this, we emit dummy offsets and patch them
# later, once we know the boundaries of our hidden main()
var main: Name
if incremental:
main = self.names[0]
else:
main = Name(depth: 0,
var main = Name(depth: 0,
isPrivate: true,
isConst: false,
isLet: false,
@ -1379,19 +1416,19 @@ proc beginProgram(self: Compiler, incremental: bool = false): int =
returnType: nil,
args: @[],
),
codePos: 12, # Jump address is hardcoded
name: newIdentExpr(Token(lexeme: "", kind: Identifier)),
codePos: self.chunk.code.len() + 12,
ident: newIdentExpr(Token(lexeme: "", kind: Identifier)),
kind: NameKind.Function,
line: -1)
self.names.add(main)
self.scopeOwners.add((main, 0))
self.emitByte(LoadUInt64, 1)
self.emitBytes(self.chunk.writeConstant(main.codePos.toLong()), 1)
self.emitByte(LoadUInt64, 1)
self.emitBytes(self.chunk.writeConstant(0.toLong()), 1)
self.emitByte(Call, 1)
self.emitBytes(0.toTriple(), 1)
result = 5
self.names.add(main)
self.scopeOwners.add((main, 0))
self.emitByte(LoadUInt64, 1)
self.emitBytes(self.chunk.writeConstant(main.codePos.toLong()), 1)
self.emitByte(LoadUInt64, 1)
self.emitBytes(self.chunk.writeConstant(0.toLong()), 1)
result = self.chunk.consts.len() - 8
self.emitByte(Call, 1)
self.emitBytes(0.toTriple(), 1)
## End of utility functions
@ -1683,40 +1720,44 @@ proc generateCall(self: Compiler, fn: Name, args: seq[Expression], line: int) =
self.patchReturnAddress(pos)
proc specializeGeneric(self: Compiler, fn: Name, args: seq[Expression]): Name =
## Specializes a generic function by
proc specialize(self: Compiler, name: Name, args: seq[Expression]): Name =
## Specializes a generic type by
## instantiating a concrete version
## of it
var mapping: TableRef[string, Type] = newTable[string, Type]()
var kind: Type
result = deepCopy(fn)
# This first loop checks if a user tries to reassign a generic's
# name to a different type
for i, (name, typ) in result.valueType.args:
if typ.kind != Generic:
continue
kind = self.infer(args[i])
if typ.name in mapping and not self.compare(kind, mapping[typ.name]):
self.error(&"expected generic argument '{typ.name}' to be of type {self.typeToStr(mapping[typ.name])}, got {self.typeToStr(kind)} instead")
mapping[typ.name] = kind
result.valueType.args[i].kind = kind
for (argExpr, argName) in zip(args, result.valueType.args):
if self.names.high() > 16777215:
self.error("cannot declare more than 16777215 variables at a time")
self.names.add(Name(depth: self.scopeDepth + 1,
isPrivate: true,
owner: self.currentModule,
isConst: false,
name: newIdentExpr(Token(lexeme: argName.name)),
valueType: argName.kind,
codePos: 0,
isLet: false,
line: fn.line,
belongsTo: result,
kind: NameKind.Argument
))
if result.valueType.returnType.kind == Generic:
result.valueType.returnType = mapping[result.valueType.returnType.name]
result = deepCopy(name)
case name.kind:
of NameKind.Function:
# This first loop checks if a user tries to reassign a generic's
# name to a different type
for i, (name, typ) in result.valueType.args:
if typ.kind != Generic:
continue
kind = self.infer(args[i])
if typ.name in mapping and not self.compare(kind, mapping[typ.name]):
self.error(&"expected generic argument '{typ.name}' to be of type {self.typeToStr(mapping[typ.name])}, got {self.typeToStr(kind)} instead")
mapping[typ.name] = kind
result.valueType.args[i].kind = kind
for (argExpr, argName) in zip(args, result.valueType.args):
if self.names.high() > 16777215:
self.error("cannot declare more than 16777215 variables at a time")
self.names.add(Name(depth: self.scopeDepth + 1,
isPrivate: true,
owner: self.currentModule,
isConst: false,
ident: newIdentExpr(Token(lexeme: argName.name)),
valueType: argName.kind,
codePos: 0,
isLet: false,
line: name.line,
belongsTo: result,
kind: NameKind.Argument
))
if result.valueType.returnType.kind == Generic:
result.valueType.returnType = mapping[result.valueType.returnType.name]
else:
discard # TODO: Custom user-defined types
proc callExpr(self: Compiler, node: CallExpr): Name {.discardable.} =
@ -1743,7 +1784,7 @@ proc callExpr(self: Compiler, node: CallExpr): Name {.discardable.} =
# Calls like hi()
result = self.matchImpl(IdentExpr(node.callee).name.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: args), node)
if result.valueType.isGeneric:
result = self.specializeGeneric(result, argExpr)
result = self.specialize(result, argExpr)
# We can't instantiate a concrete version
# of a generic function without the types
# of its arguments, so we wait until the
@ -1910,16 +1951,40 @@ proc forEachStmt(self: Compiler, node: ForEachStmt) =
proc importStmt(self: Compiler, node: ImportStmt) =
## Imports a module at compile time
# TODO: This is obviously horrible. It's just a test
let filename = node.moduleName.token.lexeme & ".pn"
let filename = splitPath(node.moduleName.token.lexeme).tail
try:
self.compileModule(filename)
self.compileModule(node.moduleName.token.lexeme)
discard self.declareName(node)
except IOError:
self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""")
except OSError:
self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""")
proc exportStmt(self: Compiler, node: ExportStmt) =
## Exports a name at compile time to
## all modules importing us
var name = self.resolve(node.name)
if name.isNil():
self.error(&"reference to undefined name '{node.name.token.lexeme}'")
if name.isPrivate:
self.error("cannot export private names")
name.exported = true
case name.kind:
of NameKind.Module:
# We need to export everything
# this module defines!
for name in self.findByModule(name.ident.token.lexeme):
name.exported = true
of NameKind.Function:
for name in self.findByName(name.ident.token.lexeme):
if name.kind != NameKind.Function:
continue
name.exported = true
else:
discard
proc printRepl(self: Compiler, typ: Type, node: Expression) =
## Emits instruction to print
## peon types in REPL mode
@ -1985,6 +2050,8 @@ proc statement(self: Compiler, node: Statement) =
self.returnStmt(ReturnStmt(node))
of NodeKind.importStmt:
self.importStmt(ImportStmt(node))
of NodeKind.exportStmt:
self.exportStmt(ExportStmt(node))
of NodeKind.whileStmt:
# Note: Our parser already desugars
# for loops to while loops
@ -2063,8 +2130,8 @@ proc funDecl(self: Compiler, node: FunDecl, name: Name) =
self.chunk.cfi.add(0.toTriple()) # Patched it later
self.chunk.cfi.add(uint8(node.arguments.len()))
if not node.name.isNil():
self.chunk.cfi.add(name.name.token.lexeme.len().toDouble())
var s = name.name.token.lexeme
self.chunk.cfi.add(name.ident.token.lexeme.len().toDouble())
var s = name.ident.token.lexeme
if s.len() >= uint16.high().int:
s = node.name.token.lexeme[0..uint16.high()]
self.chunk.cfi.add(s.toBytes())
@ -2141,7 +2208,7 @@ proc declaration(self: Compiler, node: Declaration) =
proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil,
terminateScope: bool = true, incremental: bool = false): Chunk =
incremental: bool = false, isMainModule: bool = true): Chunk =
## Compiles a sequence of AST nodes into a chunk
## object
if chunk.isNil():
@ -2150,47 +2217,54 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tu
self.chunk = chunk
self.ast = ast
self.file = file
var terminateScope = terminateScope
if incremental:
terminateScope = false
self.scopeDepth = 0
self.currentFunction = nil
self.currentModule = self.file.extractFilename()
self.currentModule = self.file.extractFilename().replace(".pn", "")
self.current = 0
self.lines = lines
self.source = source
self.jumps = @[]
let pos = self.beginProgram(incremental)
if incremental and self.replMode:
discard self.chunk.code.pop()
discard self.chunk.code.pop()
self.isMainModule = isMainModule
if not incremental:
self.jumps = @[]
let pos = self.beginProgram()
while not self.done():
self.declaration(Declaration(self.step()))
self.terminateProgram(pos, terminateScope)
self.terminateProgram(pos)
# TODO: REPL is broken, we need a new way to make
# incremental compilation resume from where it stopped!
result = self.chunk
if incremental and not self.replMode:
discard self.chunk.code.pop()
discard self.chunk.code.pop()
proc compileModule(self: Compiler, filename: string) =
## Compiles an imported module into an existing chunk.
## A temporary compiler object is initialized internally
let path = joinPath(splitPath(self.file).head, filename)
proc compileModule(self: Compiler, moduleName: string) =
## Compiles an imported module into an existing chunk
## using the compiler's internal parser and lexer objects
var path = ""
for i, searchPath in lookupPaths:
path = joinPath(getCurrentDir(), joinPath(searchPath, moduleName))
if fileExists(path):
break
elif i == searchPath.high():
self.error(&"""could not import '{path}': module not found""")
if self.modules.contains(path):
return
var lexer = newLexer()
var parser = newParser()
var compiler = newCompiler()
lexer.fillSymbolTable()
let source = readFile(path)
let tokens = lexer.lex(source, filename)
let ast = parser.parse(tokens, filename, lexer.getLines(), source)
compiler.names.add(self.names[0])
discard compiler.compile(ast, filename, lexer.getLines(), source, chunk=self.chunk, incremental=true)
for name in compiler.names:
if name.owner in self.modules:
continue
self.names.add(name)
let current = self.current
let ast = self.ast
let file = self.file
let module = self.currentModule
let lines = self.lines
let src = self.source
self.isMainModule = false
discard self.compile(self.parser.parse(self.lexer.lex(source, path),
path, self.lexer.getLines(),
source, persist=true),
path, self.lexer.getLines(), source, chunk=self.chunk, incremental=true,
isMainModule=false)
self.scopeDepth = 0
self.current = current
self.ast = ast
self.file = file
self.currentModule = module
self.lines = lines
self.source = src
self.modules.incl(path)
self.closedOver &= compiler.closedOver

View File

@ -47,6 +47,7 @@ type
yieldStmt,
awaitStmt,
importStmt,
exportStmt,
deferStmt,
# An expression followed by a semicolon
exprStmt,
@ -198,6 +199,9 @@ type
ImportStmt* = ref object of Statement
moduleName*: IdentExpr
ExportStmt* = ref object of Statement
name*: IdentExpr
AssertStmt* = ref object of Statement
expression*: Expression
@ -510,6 +514,12 @@ proc newImportStmt*(moduleName: IdentExpr, token: Token): ImportStmt =
result.token = token
proc newExportStmt*(name: IdentExpr, token: Token): ExportStmt =
result = ExportStmt(kind: exportStmt)
result.name = name
result.token = token
proc newYieldStmt*(expression: Expression, token: Token): YieldStmt =
result = YieldStmt(kind: yieldStmt)
result.expression = expression

View File

@ -40,7 +40,8 @@ type
Raise, Assert, Await, Foreach,
Yield, Defer, Try, Except,
Finally, Type, Operator, Case,
Enum, From, Ptr, Ref, Object
Enum, From, Ptr, Ref, Object,
Export,
# Literal types
Integer, Float, String, Identifier,

View File

@ -24,6 +24,7 @@ import meta/ast
import meta/errors
import lexer as l
import ../util/symbols
import ../config
export token, ast, errors
@ -42,6 +43,7 @@ type
Or,
And,
Compare,
Bitwise,
Addition,
Multiplication,
Power,
@ -86,8 +88,7 @@ type
currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int
# TODO
scopes: seq[Declaration]
# Operator table
operators: OperatorTable
# The AST node
tree: seq[Declaration]
@ -95,6 +96,8 @@ type
lines: seq[tuple[start, stop: int]]
# The source of the current module
source: string
# Keeps track of imported modules
modules: seq[tuple[name: string, loaded: bool]]
ParseError* = ref object of PeonException
parser*: Parser
file*: string
@ -121,9 +124,9 @@ proc addOperator(self: OperatorTable, lexeme: string) =
var prec = Power
if lexeme.len() >= 2 and lexeme[^2..^1] in ["->", "~>", "=>"]:
prec = Arrow
elif lexeme in ["and", "&"]:
elif lexeme == "and":
prec = Precedence.And
elif lexeme in ["or", "|"]:
elif lexeme == "or":
prec = Precedence.Or
elif lexeme.endsWith("=") and lexeme[0] notin {'<', '>', '!', '?', '~', '='} or lexeme == "=":
prec = Assign
@ -131,8 +134,10 @@ proc addOperator(self: OperatorTable, lexeme: string) =
prec = Power
elif lexeme[0] in {'*', '%', '/', '\\'}:
prec = Multiplication
elif lexeme[0] in {'+', '-', '|', '~'}:
elif lexeme[0] in {'+', '-'}:
prec = Addition
elif lexeme in [">>", "<<", "|", "~", "&", "^"]:
prec = Bitwise
elif lexeme[0] in {'<', '>', '=', '!'}:
prec = Compare
self.tokens.add(lexeme)
@ -301,6 +306,7 @@ proc parseFunExpr(self: Parser): LambdaExpr
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration
proc declaration(self: Parser): Declaration
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration]
# End of forward declarations
@ -485,9 +491,20 @@ proc parseAdd(self: Parser): Expression =
result = newBinaryExpr(result, operator, right)
proc parseBitwise(self: Parser): Expression =
## Parses bitwise expressions
result = self.parseAdd()
var operator: Token
var right: Expression
while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Bitwise:
operator = self.step()
right = self.parseAdd()
result = newBinaryExpr(result, operator, right)
proc parseCmp(self: Parser): Expression =
## Parses comparison expressions
result = self.parseAdd()
result = self.parseBitwise()
var operator: Token
var right: Expression
while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Compare:
@ -553,7 +570,7 @@ proc assertStmt(self: Parser): Statement =
## fed into them is false
let tok = self.peek(-1)
var expression = self.expression()
endOfLine("missing statement terminator after 'assert'")
endOfLine("missing semicolon after 'assert'")
result = newAssertStmt(expression, tok)
@ -588,7 +605,7 @@ proc breakStmt(self: Parser): Statement =
let tok = self.peek(-1)
if self.currentLoop != Loop:
self.error("'break' cannot be used outside loops")
endOfLine("missing statement terminator after 'break'")
endOfLine("missing semicolon after 'break'")
result = newBreakStmt(tok)
@ -597,7 +614,7 @@ proc deferStmt(self: Parser): Statement =
let tok = self.peek(-1)
if self.currentFunction.isNil():
self.error("'defer' cannot be used outside functions")
endOfLine("missing statement terminator after 'defer'")
endOfLine("missing semicolon after 'defer'")
result = newDeferStmt(self.expression(), tok)
@ -606,7 +623,7 @@ proc continueStmt(self: Parser): Statement =
let tok = self.peek(-1)
if self.currentLoop != Loop:
self.error("'continue' cannot be used outside loops")
endOfLine("missing statement terminator after 'continue'")
endOfLine("missing semicolon after 'continue'")
result = newContinueStmt(tok)
@ -621,7 +638,7 @@ proc returnStmt(self: Parser): Statement =
# we need to check if there's an actual value
# to return or not
value = self.expression()
endOfLine("missing statement terminator after 'return'")
endOfLine("missing semicolon after 'return'")
result = newReturnStmt(value, tok)
case self.currentFunction.kind:
of NodeKind.funDecl:
@ -641,7 +658,7 @@ proc yieldStmt(self: Parser): Statement =
result = newYieldStmt(self.expression(), tok)
else:
result = newYieldStmt(newNilExpr(Token(lexeme: "nil")), tok)
endOfLine("missing statement terminator after 'yield'")
endOfLine("missing semicolon after 'yield'")
proc awaitStmt(self: Parser): Statement =
@ -651,7 +668,7 @@ proc awaitStmt(self: Parser): Statement =
self.error("'await' cannot be used outside functions")
if self.currentFunction.token.kind != Coroutine:
self.error("'await' can only be used inside coroutines")
endOfLine("missing statement terminator after 'await'")
endOfLine("missing semicolon after 'await'")
result = newAwaitStmt(self.expression(), tok)
@ -663,7 +680,7 @@ proc raiseStmt(self: Parser): Statement =
# Raise can be used on its own, in which
# case it re-raises the last active exception
exception = self.expression()
endOfLine("missing statement terminator after 'raise'")
endOfLine("missing semicolon after 'raise'")
result = newRaiseStmt(exception, tok)
@ -681,12 +698,13 @@ proc forEachStmt(self: Parser): Statement =
self.currentLoop = enclosingLoop
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration]
proc findOperators(self: Parser, tokens: seq[Token])
proc importStmt(self: Parser, fromStmt: bool = false): Statement =
## Parses import statements
## Parses import statements. This is a little
## convoluted because we need to pre-parse the
## module to import the operators from it
if self.scopeDepth > 0:
self.error("import statements are only allowed at the top level")
var tok: Token
@ -694,43 +712,71 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement =
tok = self.peek(-2)
else:
tok = self.peek(-1)
# TODO: New AST node
self.expect(Identifier, "expecting module name(s) after import statement")
endOfLine("missing statement terminator after 'import'")
result = newImportStmt(newIdentExpr(self.peek(-2), self.scopeDepth), tok)
var filename = ImportStmt(result).moduleName.token.lexeme & ".pn"
var moduleName = ""
while not self.check(Semicolon) and not self.done():
if self.match(".."):
if not self.check("/"):
self.error("expecting '/' after '..' in import statement")
moduleName &= "../"
elif self.match("/"):
self.expect(Identifier, "expecting identifier after '/' in import statement")
moduleName &= &"/{self.peek(-1).lexeme}"
elif self.match(Identifier):
moduleName &= self.peek(-1).lexeme
else:
break
endOfLine("missing semicolon after import statement")
moduleName &= ".pn"
result = newImportStmt(newIdentExpr(Token(kind: Identifier, lexeme: moduleName, line: self.peek(-1).line), self.scopeDepth), tok)
var lexer = newLexer()
lexer.fillSymbolTable()
let path = joinPath(splitPath(self.file).head, filename)
# TODO: This is obviously horrible. It's just a test
var path = ""
for i, searchPath in lookupPaths:
path = joinPath(getCurrentDir(), joinPath(searchPath, moduleName))
if fileExists(path):
break
elif i == searchPath.high():
self.error(&"""could not import '{path}': module not found""")
try:
self.findOperators(lexer.lex(readFile(path), filename))
var source = readFile(path)
var tree = self.tree
var current = self.current
var tokens = self.tokens
var src = self.source
var file = self.file
discard self.parse(lexer.lex(source, path), file=path, source=source, lines=lexer.getLines(), persist=true)
self.file = file
self.source = src
self.tree = tree
self.current = current
self.tokens = tokens
except IOError:
self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""")
self.error(&"""could not import '{path}': {getCurrentExceptionMsg()}""")
except OSError:
self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""")
self.error(&"""could not import '{path}': {getCurrentExceptionMsg()} [errno {osLastError()}]""")
proc tryStmt(self: Parser): Statement =
## Parses try/except/else/finally blocks
let tok = self.peek(-1)
var body = self.statement()
self.expect(LeftBrace, "expecting '{' after 'try'")
var body = self.blockStmt()
var handlers: seq[tuple[body: Statement, exc: IdentExpr]] = @[]
var finallyClause: Statement
var elseClause: Statement
var excName: Expression
var handlerBody: Statement
while self.match(Except):
excName = self.expression()
if excName.kind == identExpr:
handlerBody = self.statement()
handlers.add((body: handlerBody, exc: IdentExpr(excName)))
if self.match(LeftBrace):
handlers.add((body: self.blockStmt(), exc: newIdentExpr(self.peek(-1))))
else:
excName = nil
self.expect(Identifier, "expecting exception name after 'except'")
self.expect(LeftBrace, "expecting '{' after exception name")
handlers.add((body: self.blockStmt(), exc: nil))
if self.match(Else):
elseClause = self.statement()
self.expect(LeftBrace, "expecting '{' after 'else' name")
elseClause = self.blockStmt()
if self.match(Finally):
finallyClause = self.statement()
self.expect(LeftBrace, "expecting '{' after 'finally'")
finallyClause = self.blockStmt()
if handlers.len() == 0 and elseClause.isNil() and finallyClause.isNil():
self.error("expecting 'except', 'finally' or 'else' statement after 'try' block", tok)
for i, handler in handlers:
@ -751,65 +797,6 @@ proc whileStmt(self: Parser): Statement =
self.currentLoop = enclosingLoop
self.endScope()
#[
proc forStmt(self: Parser): Statement =
## Parses a C-style for loop
self.beginScope()
let tok = self.peek(-1)
var enclosingLoop = self.currentLoop
self.currentLoop = Loop
self.expect(LeftParen, "expecting '(' after 'for'")
var initializer: ASTNode = nil
var condition: Expression = nil
var increment: Expression = nil
if self.match(Semicolon):
discard
elif self.match(TokenType.Var):
initializer = self.varDecl()
if not VarDecl(initializer).isPrivate:
self.error("cannot declare public for loop initializer")
else:
initializer = self.expressionStatement()
if not self.check(Semicolon):
condition = self.expression()
self.expect(Semicolon, "expecting ';' after for loop condition")
if not self.check(RightParen):
increment = self.expression()
self.expect(RightParen, "unterminated for loop increment")
var body = self.statement()
if not increment.isNil():
# The increment runs after each iteration, so we
# inject it into the block as the last statement
body = newBlockStmt(@[Declaration(body), newExprStmt(increment,
increment.token)], tok)
if condition.isNil():
## An empty condition is functionally
## equivalent to "true"
condition = newTrueExpr(Token(lexeme: "true"))
# We can use a while loop, which in this case works just as well
body = newWhileStmt(condition, body, tok)
if not initializer.isNil():
# Nested blocks, so the initializer is
# only executed once
body = newBlockStmt(@[Declaration(initializer), Declaration(body)], tok)
# This desgugars the following code:
# for (var i = 0; i < 10; i += 1) {
# print(i);
# }
# To the semantically equivalent snippet
# below:
# {
# var i = 0;
# while (i < 10) {
# print(i);
# i += 1;
# }
# }
result = body
self.currentLoop = enclosingLoop
self.endScope()
]#
proc ifStmt(self: Parser): Statement =
## Parses if statements
@ -827,6 +814,17 @@ proc ifStmt(self: Parser): Statement =
result = newIfStmt(condition, thenBranch, elseBranch, tok)
proc exportStmt(self: Parser): Statement =
## Parses export statements
var exported: IdentExpr
let tok = self.peek(-1)
if not self.match(Identifier):
self.error("expecting identifier after 'export' in export statement")
exported = newIdentExpr(self.peek(-1))
endOfLine("missing semicolon after 'raise'")
result = newExportStmt(exported, tok)
template checkDecl(self: Parser, isPrivate: bool) =
## Handy utility template that avoids us from copy
## pasting the same checks to all declaration handlers
@ -973,6 +971,20 @@ proc parseFunExpr(self: Parser): LambdaExpr =
result.defaults = defaults
proc parseGenericConstraint(self: Parser): Expression =
## Recursivelt parses a generic constraint
## and returns it as an expression
result = self.expression() # First value is always an identifier of some sort
if not self.check(RightBracket):
case self.peek().lexeme:
of "|":
result = newBinaryExpr(result, self.step(), self.parseGenericConstraint())
of "~":
result = newUnaryExpr(self.step(), result)
else:
self.error("invalid type constraint in generic declaration")
proc parseGenerics(self: Parser, decl: Declaration) =
## Parses generics in declarations
var gen: tuple[name: IdentExpr, cond: Expression]
@ -980,7 +992,7 @@ proc parseGenerics(self: Parser, decl: Declaration) =
self.expect(Identifier, "expecting generic type name")
gen.name = newIdentExpr(self.peek(-1), self.scopeDepth)
self.expect(":", "expecting type constraint after generic name")
gen.cond = self.expression()
gen.cond = self.parseGenericConstraint()
decl.generics.add(gen)
if not self.match(Comma):
break
@ -1032,7 +1044,6 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
returnType=nil, depth=self.scopeDepth)
self.scopes.add(FunDecl(self.currentFunction))
if self.match(":"):
# Function has explicit return type
if self.match([Function, Coroutine, Generator]):
@ -1098,7 +1109,6 @@ proc expression(self: Parser): Expression =
result = self.parseArrow() # Highest-level expression
proc expressionStatement(self: Parser): Statement =
## Parses expression statements, which
## are expressions followed by a semicolon
@ -1131,6 +1141,9 @@ proc statement(self: Parser): Statement =
of Import:
discard self.step()
result = self.importStmt()
of Export:
discard self.step()
result = self.exportStmt()
of From:
# TODO
# from module import a [, b, c as d]
@ -1139,11 +1152,6 @@ proc statement(self: Parser): Statement =
of While:
discard self.step()
result = self.whileStmt()
#[
of For:
discard self.step()
result = self.forStmt()
]#
of Foreach:
discard self.step()
result = self.forEachStmt()
@ -1247,25 +1255,24 @@ proc findOperators(self: Parser, tokens: seq[Token]) =
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token)
self.operators.addOperator(tokens[i + 1].lexeme)
if i == tokens.high() and token.kind != EndOfFile:
# Since we're iterating this list anyway might as
# Since we're iterating this list anyway we might as
# well perform some extra checks
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token)
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration] =
## Parses a sequence of tokens into a sequence of AST nodes
self.tokens = tokens
self.file = file
self.current = 0
self.currentLoop = LoopContext.None
self.currentFunction = nil
self.scopeDepth = 0
if not persist:
self.operators = newOperatorTable()
self.tree = @[]
self.source = source
self.lines = lines
self.current = 0
self.scopeDepth = 0
self.currentLoop = LoopContext.None
self.currentFunction = nil
self.tree = @[]
if not persist:
self.operators = newOperatorTable()
self.findOperators(tokens)
while not self.done():
self.tree.add(self.declaration())

View File

@ -111,7 +111,7 @@ proc repl =
for node in tree:
styledEcho fgGreen, "\t", $node
echo ""
discard compiler.compile(tree, "stdin", tokenizer.getLines(), input, chunk=compiled, incremental=incremental, terminateScope=false)
discard compiler.compile(tree, "stdin", tokenizer.getLines(), input, chunk=compiled, incremental=incremental)
incremental = true
when debugCompiler:
styledEcho fgCyan, "Compilation step:\n"
@ -153,9 +153,8 @@ proc repl =
fgYellow, &"'{exc.file.extractFilename()}'", fgRed, ", line ", fgYellow, $exc.line, fgRed, " at ", fgYellow, &"'{exc.lexeme.escape()}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(exc.lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(exc.lexeme))
except ParseError:
echo getCurrentExceptionMsg()
input = ""
let exc = ParseError(getCurrentException())
let lexeme = exc.token.lexeme
@ -170,7 +169,7 @@ proc repl =
fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme))
except CompileError:
let exc = CompileError(getCurrentException())
let lexeme = exc.node.token.lexeme
@ -185,7 +184,7 @@ proc repl =
fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme))
except SerializationError:
let exc = SerializationError(getCurrentException())
stderr.styledWriteLine(fgRed, "A fatal error occurred while (de-)serializing", fgYellow, &"'{exc.file}'", fgGreen, ": ", getCurrentExceptionMsg())
@ -274,21 +273,17 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) =
vm.run(serialized.chunk)
except LexingError:
var exc = LexingError(getCurrentException())
if exc.lexeme == "":
exc.line -= 1
let relPos = exc.lexer.getRelPos(exc.line)
let line = exc.lexer.getSource().splitLines()[exc.line - 1].strip(chars={'\n'})
stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{exc.file}'", fgRed, ", module ",
fgYellow, &"'{exc.file.extractFilename()}'", fgRed, ", line ", fgYellow, $exc.line, fgRed, " at ", fgYellow, &"'{exc.lexeme.escape()}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(exc.lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(exc.lexeme))
except ParseError:
let exc = ParseError(getCurrentException())
let lexeme = exc.token.lexeme
var lineNo = exc.token.line
if exc.token.kind == EndOfFile:
lineNo -= 1
let relPos = exc.parser.getRelPos(lineNo)
let fn = parser.getCurrentFunction()
let line = exc.parser.getSource().splitLines()[lineNo - 1].strip(chars={'\n'})
@ -299,13 +294,11 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) =
fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme))
except CompileError:
let exc = CompileError(getCurrentException())
let lexeme = exc.node.token.lexeme
var lineNo = exc.node.token.line
if exc.node.token.kind == EndOfFile:
lineNo -= 1
let relPos = exc.compiler.getRelPos(lineNo)
let line = exc.compiler.getSource().splitLines()[lineNo - 1].strip(chars={'\n'})
var fn = exc.compiler.getCurrentFunction()
@ -316,7 +309,7 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) =
fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
styledEcho fgBlue, "Source line: " , fgDefault, line
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme))
except SerializationError:
let exc = SerializationError(getCurrentException())
stderr.styledWriteLine(fgRed, "A fatal error occurred while (de-)serializing", fgYellow, &"'{exc.file}'", fgGreen, ": ", getCurrentExceptionMsg())

View File

@ -3,8 +3,8 @@ import ../frontend/lexer
proc fillSymbolTable*(tokenizer: Lexer) =
## Initializes the Lexer's symbol
## table with the builtin symbols
## and keywords
## table with builtin symbols and
## keywords
# 1-byte symbols
tokenizer.symbols.addSymbol("{", LeftBrace)
@ -16,7 +16,6 @@ proc fillSymbolTable*(tokenizer: Lexer) =
tokenizer.symbols.addSymbol(".", Dot)
tokenizer.symbols.addSymbol(",", Comma)
tokenizer.symbols.addSymbol(";", Semicolon)
# tokenizer.symbols.addSymbol("\n", Semicolon) # TODO: Broken
# Keywords
tokenizer.symbols.addKeyword("type", TokenType.Type)
tokenizer.symbols.addKeyword("enum", Enum)
@ -46,10 +45,11 @@ proc fillSymbolTable*(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("yield", TokenType.Yield)
tokenizer.symbols.addKeyword("return", TokenType.Return)
tokenizer.symbols.addKeyword("object", Object)
tokenizer.symbols.addKeyword("export", Export)
# These are more like expressions with a reserved
# name that produce a value of a builtin type,
# but we don't need to care about that until
# we're in the parsing/ compilation steps so
# we're in the parsing/compilation steps so
# it's fine
tokenizer.symbols.addKeyword("nan", NotANumber)
tokenizer.symbols.addKeyword("inf", Infinity)
@ -59,5 +59,6 @@ proc fillSymbolTable*(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("ref", TokenType.Ref)
tokenizer.symbols.addKeyword("ptr", TokenType.Ptr)
for sym in [">", "<", "=", "~", "/", "+", "-", "_", "*", "?", "@", ":", "==", "!=",
">=", "<=", "+=", "-=", "/=", "*=", "**=", "!", "%", "&", "|", "^"]:
">=", "<=", "+=", "-=", "/=", "*=", "**=", "!", "%", "&", "|", "^",
">>", "<<"]:
tokenizer.symbols.addSymbol(sym, Symbol)

View File

@ -1,223 +0,0 @@
## The peon standard library
# Builtin arithmetic operators for Peon
# Note: Most of these do nothing on their own. All they do
# is serve as placeholders for emitting specific VM
# instructions. They're implemented this way because:
# - They tie into the existing type system nicely
# - It makes the implementation easier and more flexible
operator `+`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "Add", pure]
}
operator `+`(a, b: float): float {
#pragma[magic: "AddFloat64", pure]
}
operator `+`(a, b: float32): float32 {
#pragma[magic: "AddFloat32", pure]
}
operator `-`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "Subtract", pure]
}
operator `-`*(a, b: float64): float64 {
#pragma[magic: "SubtractFloat64", pure]
}
operator `-`*(a, b: float32): float32 {
#pragma[magic: "SubtractFloat32", pure]
}
operator `-`*[T: int | int32 | int16 | int8](a: T): T {
#pragma[magic: "Negate", pure]
}
operator `-`*(a: float64): float64 {
#pragma[magic: "NegateFloat64", pure]
}
operator `-`*(a: float32): float32 {
#pragma[magic: "NegateFloat32", pure]
}
operator `-`*(a: inf): inf {
#pragma[magic: "NegInf", pure]
}
operator `*`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "Multiply", pure]
}
operator `*`*(a, b: float64): float64 {
#pragma[magic: "MultiplyFloat64", pure]
}
operator `*`*(a, b: float32): float32 {
#pragma[magic: "MultiplyFloat32", pure]
}
operator `/`*[T: int | int32 | int16 | int8](a, b: T): T {
#pragma[magic: "SignedDivide", pure]
}
operator `/`*[T: uint64 | uint32 | uint16 | uint8](a, b: T): T {
#pragma[magic: "Divide", pure]
}
operator `/`*(a, b: float64): float64 {
#pragma[magic: "DivFloat64", pure]
}
operator `/`*(a, b: float32): float32 {
#pragma[magic: "DivFloat32", pure]
}
operator `**`*[T: int | int32 | int16 | int8](a, b: T): T {
#pragma[magic: "SignedPow", pure]
}
operator `**`*[T: uint64 | uint32 | uint16 | uint8](a, b: T): T {
#pragma[magic: "Pow", pure]
}
operator `%`*(a, b: int64): int64 {
#pragma[magic: "SignedMod", pure]
}
# Comparison operators
operator `>`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "GreaterThan", pure]
}
operator `<`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "LessThan", pure]
}
operator `==`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "Equal", pure]
}
operator `!=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "NotEqual", pure]
}
operator `>=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "GreaterOrEqual", pure]
}
operator `<=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool {
#pragma[magic: "LessOrEqual", pure]
}
operator `and`*(a, b: bool): bool {
#pragma[magic: "LogicalAnd", pure]
}
operator `or`*(a, b: bool): bool {
#pragma[magic: "LogicalOr", pure]
}
operator `not`*(a: bool): bool {
#pragma[magic: "LogicalNot", pure]
}
operator `&`*(a, b: int): bool {
#pragma[magic: "And", pure]
}
operator `|`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): int {
#pragma[magic: "Or", pure]
}
operator `~`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a: T): T {
#pragma[magic: "Not", pure]
}
operator `>>`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "RShift", pure]
}
operator `<<`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "LShift", pure]
}
operator `^`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T {
#pragma[magic: "Xor", pure]
}
# Assignment operators
operator `=`*[T: all](a: var T, b: T) { # TODO: This is just a placeholder right now
#pragma[magic: "GenericAssign"]
}
# Some useful builtins
fn clock*: float {
#pragma[magic: "SysClock64", pure]
}
fn print*(x: int) {
#pragma[magic: "PrintInt64"]
}
fn print*(x: uint64) {
#pragma[magic: "PrintUInt64"]
}
fn print*(x: float) {
#pragma[magic: "PrintFloat64"]
}
fn print*(x: string) {
#pragma[magic: "PrintString"]
}
fn print*(x: bool) {
#pragma[magic: "PrintBool"]
}