Initial work on multi-backend support

This commit is contained in:
Mattia Giambirtone 2022-12-15 11:48:36 +01:00
parent b912d58cad
commit dc393bbb34
20 changed files with 344 additions and 568 deletions

View File

@ -8,18 +8,17 @@ Peon is a simple, functional, async-first programming language with a focus on c
## Project structure
- `src/` -> Contains the entirety of peon's toolchain
- ~~`src/memory/` -> Contains peon's memory allocator and GC~~ GC is in the VM right now, sorry!
- `src/frontend/` -> Contains the tokenizer, parser and compiler
- `src/frontend/` -> Contains the tokenizer, parser and compiler targets
- `src/frontend/meta/` -> Contains shared error definitions, AST node and token
declarations as well as the bytecode used by the compiler
- `src/backend/` -> Contains the peon VM and type system
- `src/backend/` -> Contains the various backends supported by peon (bytecode, native)
- `src/util/` -> Contains utilities such as the bytecode debugger and serializer as well
as procedures to handle multi-byte sequences
- `src/config.nim` -> Contains compile-time configuration variables
- `src/main.nim` -> Ties up the whole toolchain together by tokenizing,
parsing, compiling, debugging, (de-)serializing and executing peon code
parsing, compiling, debugging, (de-)serializing and (optionally) executing peon code
- `docs/` -> Contains documentation for various components of peon (bytecode, syntax, etc.)
- `tests/` -> Contains tests (both in peon and Nim) for the toolchain
- `tests/` -> Contains peon tests for the toolchain
## What's peon?

View File

@ -1 +1,2 @@
--hints:off --warnings:off
--hints:off --warnings:off
path="$HOME/Desktop/peon/src"

View File

@ -1,219 +0,0 @@
## A simple, dynamically-growing stack implementation
import strformat
import ../config
type Stack*[T] = object
## A stack for use in the
## peon runtime environment
container: ptr UncheckedArray[T]
capacity*: int
length: int
proc newStack*[T]: ptr Stack[T] =
## Allocates a new, empty stack
## with a starting capacity of 8
result = cast[ptr Stack[T]](alloc(sizeof(Stack)))
result.capacity = 8
result.container = cast[ptr UncheckedArray[T]](alloc(sizeof(T) * 8))
result.length = 0
proc push*[T](self: ptr Stack[T], elem: T) =
## Pushes an object onto the stack
if self.capacity <= self.length:
self.capacity *= HeapGrowFactor
self.container = cast[ptr UncheckedArray[T]](realloc(self.container, self.capacity))
self.container[self.length] = elem
self.length += 1
proc pop*[T](self: ptr Stack[T], idx: int = -1): T =
## Pops an item off the stack. By default, the last
## element is popped, in which case the operation's
## time complexity is O(1). When an arbitrary element
## is popped, the complexity rises to O(k) where k
## is the number of elements that had to be shifted
## by 1 to avoid empty slots
var idx = idx
if self.length == 0:
raise newException(IndexDefect, "pop from empty Stack")
if idx == -1:
idx = self.length - 1
if idx notin 0..self.length - 1:
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
result = self.container[idx]
if idx != self.length - 1:
for i in countup(idx, self.length - 1):
self.container[i] = self.container[i + 1]
self.capacity -= 1
self.length -= 1
proc pop*[T](self: ptr Stack[T], idx: uint64): T =
## Pops an item off the stack
var idx = idx
if self.length == 0:
raise newException(IndexDefect, "pop from empty Stack")
if idx notin 0'u64..uint64(self.length - 1):
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
result = self.container[idx]
if idx != uint64(self.length - 1):
for i in countup(idx, uint64(self.length - 1)):
self.container[i] = self.container[i + 1]
self.capacity -= 1
self.length -= 1
proc `[]`*[T](self: ptr Stack[T], idx: int): T =
## Retrieves an item from the stack, in constant
## time
if self.length == 0:
raise newException(IndexDefect, &"Stack index out of bounds: : {idx} notin 0..{self.length - 1}")
if idx notin 0..self.length - 1:
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
result = self.container[idx]
proc `[]`*[T](self: ptr Stack[T], idx: uint64): T =
## Retrieves an item from the stack, in constant
## time
if self.length == 0:
raise newException(IndexDefect, &"Stack index out of bounds: : {idx} notin 0..{self.length - 1}")
if idx notin 0'u64..uint64(self.length - 1):
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
result = self.container[idx]
proc `[]`*[T](self: ptr Stack[T], idx: BackwardsIndex): T =
## Retrieves an item from the stack, in constant
## time
result = self[self.len() - int(idx)]
proc `[]`*[T](self: ptr Stack[T], slice: Hslice[int, int]): ptr Stack[T] =
## Retrieves a subset of the stack, in O(k) time where k is the size
## of the slice
if self.length == 0:
raise newException(IndexDefect, "Stack index out of bounds")
if slice.a notin 0..self.length - 1 or slice.b notin 0..self.length:
raise newException(IndexDefect, "Stack index out of bounds")
result = newStack()
for i in countup(slice.a, slice.b - 1):
result.push(self.container[i])
proc `[]=`*[T](self: ptr Stack[T], idx: int, obj: T) =
## Assigns an object to the given index, in constant
## time
if self.length == 0:
raise newException(IndexDefect, "Stack is empty")
if idx notin 0..self.length - 1:
raise newException(IndexDefect, "Stack index out of bounds")
self.container[idx] = obj
proc `[]=`*[T](self: ptr Stack[T], idx: uint64, obj: T) =
## Assigns an object to the given index, in constant
## time
if self.length == 0:
raise newException(IndexDefect, "Stack is empty")
if idx notin 0'u64..uint64(self.length - 1):
raise newException(IndexDefect, "Stack index out of bounds")
self.container[idx] = obj
proc delete*[T](self: ptr Stack[T], idx: int) =
## Deletes an object from the given index.
## uint64his method shares the time complexity
## of self.pop()
if self.length == 0:
raise newException(IndexDefect, "delete from empty Stack")
if idx notin 0..self.length - 1:
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
discard self.pop(idx)
proc delete*[T](self: ptr Stack[T], idx: uint64) =
## Deletes an object from the given index.
## uint64his method shares the time complexity
## of self.pop()
if self.length == 0:
raise newException(IndexDefect, "delete from empty Stack")
if idx notin 0'u64..uint64(self.length - 1):
raise newException(IndexDefect, &"Stack index out of bounds: {idx} notin 0..{self.length - 1}")
discard self.pop(idx)
proc contains*[T](self: ptr Stack[T], elem: T): bool =
## Returns true if the given object is present
## in the list, false otherwise. O(n) complexity
if self.length > 0:
for i in 0..self.length - 1:
if self[i] == elem:
return true
return false
proc high*[T](self: ptr Stack[T]): int =
## Returns the index of the last
## element in the list, in constant time
result = self.length - 1
proc len*[T](self: ptr Stack[T]): int =
## Returns the length of the list
## in constant time
result = self.length
iterator pairs*[T](self: ptr Stack[T]): tuple[key: int, val: T] =
## Implements pairwise iteration (similar to python's enumerate)
for i in countup(0, self.length - 1):
yield (key: i, val: self[i])
iterator items*[T](self: ptr Stack[T]): T =
## Implements iteration
for i in countup(0, self.length - 1):
yield self[i]
proc reversed*[T](self: ptr Stack[T], first: int = -1, last: int = 0): ptr Stack[T] =
## Returns a reversed version of the given stack, from first to last.
## First defaults to -1 (the end of the list) and last defaults to 0 (the
## beginning of the list)
var first = first
if first == -1:
first = self.length - 1
result = newStack()
for i in countdown(first, last):
result.push(self[i])
proc extend*[T](self: ptr Stack[T], other: seq[T]) =
## Iteratively calls self.push() with the elements
## from a nim sequence
for elem in other:
self.push(elem)
proc extend*[T](self: ptr Stack[T], other: ptr Stack[T]) =
## Iteratively calls self.push() with the elements
## from another Stack
for elem in other:
self.push(elem)
proc `$`*[T](self: ptr Stack[T]): string =
## Returns a string representation
## of self
result = "["
if self.length > 0:
for i in 0..self.length - 1:
result = result & $self.container[i]
if i < self.length - 1:
result = result & ", "
result = result & "]"

View File

@ -1,17 +0,0 @@
# Copyright 2022 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## The peon garbage collector and memory manager

View File

@ -31,8 +31,8 @@ import std/sets
import std/monotimes
import ../frontend/meta/bytecode
import ../util/multibyte
import ../frontend/compiler/targets/bytecode/opcodes
import ../frontend/compiler/targets/bytecode/util/multibyte
when debugVM or debugMem or debugGC:

View File

@ -0,0 +1,139 @@
# Copyright 2022 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2022 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import std/tables
import std/strformat
import std/algorithm
import std/parseutils
import std/strutils
import std/sequtils
import std/sets
import std/os
import std/terminal
import std/hashes
import errors
import config
import util/symbols
import frontend/parsing/token
import frontend/parsing/ast
import frontend/parsing/lexer as l
import frontend/parsing/parser as p
export ast, token, symbols, config, errors
type
TypeKind* = enum
## An enumeration of compile-time
## types
Int8, UInt8, Int16, UInt16, Int32,
UInt32, Int64, UInt64, Float32, Float64,
Char, Byte, String, Function, CustomType,
Nil, Nan, Bool, Inf, Typevar, Generic,
Reference, Pointer, Any, All, Union, Auto
Type* = ref object
## A wrapper around
## compile-time types
case kind*: TypeKind:
of Function:
isLambda*: bool
isGenerator*: bool
isCoroutine*: bool
isAuto*: bool
args*: seq[tuple[name: string, kind: Type, default: Expression]]
returnType*: Type
builtinOp*: string
fun*: Declaration
retJumps*: seq[int]
forwarded*: bool
location*: int
compiled*: bool
of CustomType:
fields*: TableRef[string, Type]
of Reference, Pointer:
value*: Type
of Generic:
# cond represents a type constraint. For
# example, fn foo[T*: int & ~uint](...) {...}
# would map to [(true, int), (false, uint)]
cond*: seq[tuple[match: bool, kind: Type]]
asUnion*: bool # If this is true, the constraint is treated like a type union
name*: string
of Union:
types*: seq[tuple[match: bool, kind: Type]]
else:
discard
WarningKind* {.pure.} = enum
## A warning enumeration type
UnreachableCode, UnusedName, ShadowOuterScope,
MutateOuterScope
CompileMode* {.pure.} = enum
## A compilation mode enumeration
Debug, Release
CompileError* = ref object of PeonException
node*: ASTNode
function*: Declaration
Compiler* = ref object {.inheritable.}
## A wrapper around the Peon compiler's state
# The output of our parser (AST)
ast*: seq[Declaration]
# The current AST node we're looking at
current*: int
# The current file being compiled (used only for
# error reporting)
file*: string
# The current scope depth. If > 0, we're
# in a local scope, otherwise it's global
depth*: int
# Are we in REPL mode? If so, Pop instructions
# for expression statements at the top level are
# swapped for a special instruction that prints
# the result of the expression once it is evaluated
replMode*: bool
# Stores line data for error reporting
lines*: seq[tuple[start, stop: int]]
# The source of the current module,
# used for error reporting
source*: string
# We store these objects to compile modules
lexer*: Lexer
parser*: Parser
# Are we compiling the main module?
isMainModule*: bool
# List of disabled warnings
disabledWarnings*: seq[WarningKind]
# Whether to show detailed info about type
# mismatches when we dispatch with match()
showMismatches*: bool
# Are we compiling in debug mode?
mode*: CompileMode

View File

@ -17,7 +17,8 @@
import std/strutils
import std/strformat
import ../../util/multibyte
import util/multibyte
type

View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import ../frontend/meta/bytecode
import ../opcodes
import multibyte

View File

@ -13,19 +13,18 @@
# limitations under the License.
## Implementation of the peon bytecode serializer
import ../frontend/meta/errors
import ../frontend/meta/bytecode
import ../frontend/compiler
import multibyte
import ../config
import std/strformat
import std/strutils
import std/times
import config
import errors
import multibyte
import frontend/parsing/ast
import frontend/compiler/targets/bytecode/opcodes
export ast
type

View File

@ -23,6 +23,7 @@ import std/strutils
import token
export token
type
NodeKind* = enum
## Enumeration of the AST

View File

@ -21,12 +21,11 @@ import std/strformat
import std/tables
import meta/token
import meta/errors
import token
import errors
export token
export errors
export token, errors
type

View File

@ -18,12 +18,13 @@ import std/strformat
import std/strutils
import std/os
import meta/token
import meta/ast
import meta/errors
import ast
import token
import errors
import config
import lexer as l
import ../util/symbols
import ../config
import util/symbols
export token, ast, errors
@ -172,8 +173,8 @@ proc getCurrent*(self: Parser): int {.inline.} = self.current
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
self.tokens.high() or
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
proc getSource*(self: Parser): string {.inline.} = self.source
proc getCurrentFunction*(self: Parser): Declaration {.inline.} = self.currentFunction
proc getSource*(self: Parser): string = self.source
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
template endOfLine(msg: string, tok: Token = nil) = self.expect(Semicolon, msg, tok)

View File

@ -15,12 +15,13 @@
## Peon's main executable
# Our stuff
import frontend/lexer as l
import frontend/parser as p
import frontend/compiler as c
import frontend/parsing/lexer as l
import frontend/parsing/parser as p
import frontend/compiler/targets/bytecode/target as b
import frontend/compiler/compiler as c
import backend/vm as v
import util/serializer as s
import util/debugger
import frontend/compiler/targets/bytecode/util/serializer as s
import frontend/compiler/targets/bytecode/util/debugger
import util/symbols
import util/fmterr
import config
@ -106,7 +107,7 @@ proc repl =
for node in tree:
styledEcho fgGreen, "\t", $node
echo ""
compiled = newCompiler(replMode=true).compile(tree, "stdin", tokenizer.getLines(), current)
compiled = newBytecodeCompiler(replMode=true).compile(tree, "stdin", tokenizer.getLines(), current)
when debugCompiler:
styledEcho fgCyan, "Compilation step:\n"
debugger.disassembleChunk(compiled, "stdin")
@ -146,7 +147,10 @@ proc repl =
except CompileError:
print(CompileError(getCurrentException()))
except SerializationError:
print(SerializationError(getCurrentException()))
var file = SerializationError(getCurrentException()).file
if file notin ["<string>", ""]:
file = relativePath(file, getCurrentDir())
stderr.styledWriteLine(fgRed, styleBright, "Error while (de-)serializing ", fgYellow, file, fgDefault, &": {getCurrentException().msg}")
quit(0)
@ -159,7 +163,7 @@ proc runFile(f: string, fromString: bool = false, dump: bool = true, breakpoints
serialized: Serialized
tokenizer = newLexer()
parser = newParser()
compiler = newCompiler()
compiler = newBytecodeCompiler()
debugger {.used.} = newDebugger()
serializer = newSerializer()
vm = newPeonVM()
@ -246,11 +250,17 @@ proc runFile(f: string, fromString: bool = false, dump: bool = true, breakpoints
except CompileError:
print(CompileError(getCurrentException()))
except SerializationError:
print(SerializationError(getCurrentException()))
var file = SerializationError(getCurrentException()).file
if file notin ["<string>", ""]:
file = relativePath(file, getCurrentDir())
stderr.styledWriteLine(fgRed, styleBright, "Error while (de-)serializing ", fgYellow, file, fgDefault, &": {getCurrentException().msg}")
except IOError:
print(IOError(getCurrentException()[]), f)
let exc = getCurrentException()
stderr.styledWriteLine(fgRed, styleBright, "Error while trying to read ", fgYellow, f, fgDefault, &": {exc.msg}")
except OSError:
print(OSError(getCurrentException()[]), f, osLastError())
let exc = getCurrentException()
stderr.styledWriteLine(fgRed, styleBright, "Error while trying to read ", fgYellow, f, fgDefault, &": {exc.msg} ({osErrorMsg(osLastError())})",
fgRed, "[errno ", fgYellow, $osLastError(), fgRed, "]")

View File

@ -13,10 +13,10 @@
# limitations under the License.
## Utilities to print formatted error messages to stderr
import ../frontend/compiler
import ../frontend/parser
import ../frontend/lexer
import ../util/serializer
import frontend/compiler/compiler
import frontend/parsing/parser
import frontend/parsing/lexer
import errors
import std/os
@ -72,30 +72,3 @@ proc print*(exc: LexingError) =
printError(file, exc.lexer.getSource().splitLines()[exc.line - 1].strip(chars={'\n'}),
exc.line, exc.pos, nil, exc.msg)
proc print*(exc: SerializationError) =
## Prints a formatted error message
## for serialization errors to stderr
var file = exc.file
if file notin ["<string>", ""]:
file = relativePath(exc.file, getCurrentDir())
stderr.styledWriteLine(fgRed, styleBright, "Error while (de-)serializing ", fgYellow, file, fgDefault, &": {exc.msg}")
proc print*(exc: IOError, file: string) =
## Prints a formatted error message
## for nim I/O errors to stderr
var file = file
if file notin ["<string>", ""]:
file = relativePath(file, getCurrentDir())
stderr.styledWriteLine(fgRed, styleBright, "Error while trying to read ", fgYellow, file, fgDefault, &": {exc.msg}")
proc print*(exc: OSError, file: string, errno: OSErrorCode) =
## Prints a formatted error message
## for nim OS errors to stderr
var file = file
if file notin ["<string>", ""]:
file = relativePath(file, getCurrentDir())
stderr.styledWriteLine(fgRed, styleBright, "Error while trying to read ", fgYellow, file, fgDefault, &": {exc.msg} ({osErrorMsg(errno)})",
fgRed, "[errno ", fgYellow, $errno, fgRed, "]")

View File

@ -1,4 +1,4 @@
import ../frontend/lexer
import ../frontend/parsing/lexer
proc fillSymbolTable*(tokenizer: Lexer) =