japl
/
JustAnotherJAPL
mirror of https://github.com/japl-lang/JustAnotherJAPL.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added basic (untested) support for static and dynamic variable declarations. Minor fixes to main.nim

This commit is contained in:
Nocturn9x 2021-11-17 19:05:03 +01:00
parent 77224d707c
commit 352af30bee
5 changed files with 237 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
src/backend/compiler.nim
View File

@ -20,6 +20,7 @@ import ../util/multibyte
import strformat
import algorithm
import parseutils
import sequtils
@ -30,9 +31,9 @@ export token
export multibyte
type
Name = ref object
## A wrapper around declared names.
type
Name = ref object of RootObj
## A compile-time wrapper around names.
## Depth indicates to which scope
## the variable belongs, zero meaning
## the global one. Note that all names
@ -41,11 +42,15 @@ type
## the compiler cannot resolve a name
## at compile-time it will error out even
## if everything would be fine at runtime
name: ASTNode
isStatic: bool
isPrivate: bool
name: IdentExpr
owner: string
depth: int
isPrivate: bool
StaticName = ref object of Name
## A wrapper around statically bound names.
isConst: bool
Compiler* = ref object
## A wrapper around the compiler's state
chunk: Chunk
@ -54,6 +59,11 @@ type
current: int
file: string
names: seq[Name]
# This differentiation is needed for a few things, namely:
# - To forbid assignment to constants
# - To error out on name resolution for a never-declared variable (static or dynamic)
# - To correctly predict where locals will be on the stack at runtime
staticNames: seq[StaticName]
scopeDepth: int
currentFunction: FunDecl
@ -65,6 +75,7 @@ proc initCompiler*(): Compiler =
result.current = 0
result.file = ""
result.names = @[]
result.staticNames = @[]
result.scopeDepth = 0
result.currentFunction = nil
@ -182,8 +193,10 @@ proc identifierConstant(self: Compiler, identifier: IdentExpr): array[3, uint8]
## Emits an identifier name as a string in the current chunk's constant
## table. This is used to load globals declared as dynamic that cannot
## be resolved statically by the compiler
result = self.makeConstant(identifier)
try:
result = self.makeConstant(identifier)
except CompileError:
self.error(getCurrentExceptionMsg())
## End of utility functions
@ -343,12 +356,167 @@ proc binary(self: Compiler, node: BinaryExpr) =
self.error(&"invalid AST node of kind {node.kind} at binary(): {node} (This is an internal error and most likely a bug)")
proc declareName(self: Compiler, node: ASTNode) =
## Compiles all name declarations (constants, static,
## and dynamic)
case node.kind:
of varDecl:
var node = VarDecl(node)
if not node.isStatic:
# This emits code for dynamically-resolved variables (i.e. just globals declared as dynamic)
self.emitByte(DeclareName)
self.emitBytes(self.identifierConstant(IdentExpr(node.name)))
self.names.add(Name(depth: self.scopeDepth, name: IdentExpr(node.name),
isPrivate: node.isPrivate, owner: node.owner))
elif node.isConst:
# Constants are emitted as, you guessed it, constant instructions
# no matter the scope depth. Also, name resolution specifiers do not
# apply to them (because what would it mean for a constant to be dynamic
# anyway?)
self.names.add(Name(depth: self.scopeDepth, name: IdentExpr(node.name),
isPrivate: node.isPrivate, owner: node.owner))
self.emitConstant(node.value)
else:
# Statically resolved variable here
if self.staticNames.high() > 16777215:
# If someone ever hits this limit in real-world scenarios, I swear I'll
# slap myself 100 times with a sign saying "I'm dumb". Mark my words
self.error("cannot declare more than 16777215 static variables at a time")
self.staticNames.add(StaticName(depth: self.scopeDepth, name: IdentExpr(node.name),
isPrivate: node.isPrivate, owner: node.owner, isConst: node.isConst))
else:
discard # TODO: Classes, functions
proc varDecl(self: Compiler, node: VarDecl) =
## Compiles variable declarations
self.expression(node.value)
self.declareName(node)
proc resolveDynamic(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): Name =
## Traverses self.names backwards and returns the
## first name object with the given name at the given
## depth. The default depth is the current one. Returns
## nil when the name can't be found. This helper function
## is only useful when detecting a few errors and edge
## cases
for obj in reversed(self.names):
if obj.name.token.lexeme == name.token.lexeme and obj.depth == depth:
return obj
return nil
proc resolveStatic(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): StaticName =
## Traverses self.staticNames backwards and returns the
## first name object with the given name at the given
## depth. The default depth is the current one. Returns
## nil when the name can't be found. This helper function
## is only useful when detecting a few errors and edge
## cases
for obj in reversed(self.staticNames):
if obj.name.token.lexeme == name.token.lexeme and obj.depth == depth:
return obj
return nil
proc getStaticIndex(self: Compiler, name: IdentExpr): int =
## Gets the predicted stack position of the given variable
## if it is static, returns -1 if it is to be bound dynamically
for i, variable in self.staticNames:
if name.name.lexeme == variable.name.name.lexeme:
return i
return -1
proc identifier(self: Compiler, node: IdentExpr) =
## Compiles access to identifiers
let r = self.resolveDynamic(node)
if r == nil and self.scopeDepth == 0:
# Usage of undeclared globals is easy to detect at the top level
self.error(&"reference to undeclared name '{r.name.name.lexeme}'")
let index = self.getStaticIndex(node)
if index != -1:
self.emitByte(LoadNameFast) # Scoping/static resolution
self.emitBytes(index.toTriple())
else:
self.emitByte(LoadName)
self.emitBytes(self.identifierConstant(node))
proc assignment(self: Compiler, node: ASTNode) =
## Compiles assignment expressions
case node.kind:
of assignExpr:
var node = AssignExpr(node)
var name = IdentExpr(node.name)
let r = self.resolveStatic(name)
if r != nil and r.isConst:
self.error("cannot assign to constant")
# Assignment only encompasses variable assignments
# so we can ensure the name is a constant
self.expression(node.value)
let index = self.getStaticIndex(name)
if index != -1:
self.emitByte(UpdateNameFast)
self.emitBytes(index.toTriple())
else:
self.emitByte(UpdateName)
self.emitBytes(self.makeConstant(name))
of setItemExpr:
var node = SetItemExpr(node)
# TODO
else:
self.error(&"invalid AST node of kind {node.kind} at assignment(): {node} (This is an internal error and most likely a bug)")
proc beginScope(self: Compiler) =
## Begins a new local scope
inc(self.scopeDepth)
proc endScope(self: Compiler) =
## Ends the current local scope
if self.scopeDepth <= 0:
self.error("cannot call endScope with scopeDepth <= 0 (This is an internal error and most likely a bug)")
for ident in reversed(self.staticNames):
if ident.depth > self.scopeDepth:
# All variables with a scope depth larger than the current one
# are now out of scope. Begone, you're now homeless!
self.emitByte(Pop)
discard self.staticNames.pop()
dec(self.scopeDepth)
proc blockStmt(self: Compiler, node: BlockStmt) =
## Compiles block statements, which create a new
## local scope.
self.beginScope()
while not self.done():
self.declaration(self.step())
self.endScope()
proc expression(self: Compiler, node: ASTNode) =
## Compiles all expressions
case node.kind:
of getItemExpr:
discard
# Note that for setItem and assign we don't convert
# the node to its true type because that type information
# would be lost in the call anyway. The differentiation
# happens in self.assignment
of setItemExpr:
self.assignment(node)
of assignExpr:
self.assignment(node)
of identExpr:
self.identifier(IdentExpr(node))
of unaryExpr:
# Unary expressions such as ~5 and -3
self.unary(UnaryExpr(node))
of groupingExpr:
self.expression(GroupingExpr(node).expression)
of binaryExpr:
# Binary expressions such as 2 ^ 5 and 0.66 * 3.14
self.binary(BinaryExpr(node))
@ -369,20 +537,6 @@ proc expression(self: Compiler, node: ASTNode) =
self.error(&"invalid AST node of kind {node.kind} at expression(): {node} (This is an internal error and most likely a bug)") # TODO
proc defineVariable(self: Compiler, index: array[3, uint8]) =
## Defines a variable
self.emitByte(DeclareName)
self.emitBytes(index)
proc varDecl(self: Compiler, node: VarDecl) =
## Compiles variable declarations
self.expression(node.value)
if not node.isStatic:
self.defineVariable(self.identifierConstant(IdentExpr(node.name)))
# TODO
proc statement(self: Compiler, node: ASTNode) =
## Compiles all statements
case node.kind:
@ -414,8 +568,8 @@ proc statement(self: Compiler, node: ASTNode) =
discard
of forEachStmt:
discard
of blockStmt:
discard
of NodeKind.blockStmt:
self.blockStmt(BlockStmt(node))
of yieldStmt:
discard
of awaitStmt:
@ -446,6 +600,7 @@ proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk =
self.ast = ast
self.file = file
self.names = @[]
self.staticNames = @[]
self.scopeDepth = 0
self.currentFunction = nil
self.current = 0

56
src/backend/meta/bytecode.nim
View File

@ -13,6 +13,7 @@
# limitations under the License.
import ast
import ../../util/multibyte
import errors
import strutils
@ -43,6 +44,7 @@ type
consts*: seq[ASTNode]
code*: seq[uint8]
lines*: seq[int]
reuseConsts*: bool
OpCode* {.pure.} = enum
## Enum of possible opcodes.
@ -114,7 +116,6 @@ type
# Name resolution/handling
LoadAttribute,
DeclareName,
DeclareNameFast,
LoadName,
LoadNameFast, # Compile-time optimization for statically resolved global variables
UpdateName,
@ -153,18 +154,16 @@ const simpleInstructions* = {Return, BinaryAdd, BinaryMultiply,
InPlaceFloorDiv, InPlaceMod, InPlaceMultiply,
InPlaceSubtract, BinaryFloorDiv, BinaryOf, Raise,
ReRaise, BeginTry, FinishTry, Yield, Await}
const constantInstructions* = {LoadConstant, DeclareName,
LoadName, UpdateName,
DeleteName}
const byteInstructions* = {UpdateNameFast, LoadNameFast,
DeleteNameFast, Call}
const constantInstructions* = {LoadConstant, DeclareName, LoadName, UpdateName, DeleteName,
UpdateNameFast, DeleteNameFast}
const byteInstructions* = {Call}
const jumpInstructions* = {JumpIfFalse, Jump}
const collectionInstructions* = {BuildList, BuildDict, BuildSet, BuildTuple}
proc newChunk*(): Chunk =
proc newChunk*(reuseConsts: bool = true): Chunk =
## Initializes a new, empty chunk
result = Chunk(consts: @[], code: @[], lines: @[])
result = Chunk(consts: @[], code: @[], lines: @[], reuseConsts: reuseConsts)
proc `$`*(self: Chunk): string = &"""Chunk(consts=[{self.consts.join(", ")}], code=[{self.code.join(", ")}], lines=[{self.lines.join(", ")}])"""
@ -218,8 +217,43 @@ proc getLine*(self: Chunk, idx: int): int =
raise newException(IndexDefect, "index out of range")
proc findOrAddConstant(self: Chunk, constant: ASTNode): int =
## Small optimization function that reuses the same constant
## if it's already been written before (only if self.reuseConstants
## equals true)
if self.reuseConsts:
for i, c in self.consts:
# We cannot use simple equality because the nodes likely have
# different token objects with different values
if c.kind != constant.kind:
continue
if constant.isConst():
var c = LiteralExpr(c)
var constant = LiteralExpr(constant)
if c.literal.lexeme == constant.literal.lexeme:
# This woldn't work for stuff like 2e3 and 2000.0, but those
# forms are collapsed in the compiler before being written
# to the constants table
return i
elif constant.kind == identExpr:
var c = IdentExpr(c)
var constant = IdentExpr(constant)
if c.name.lexeme == constant.name.lexeme:
return i
else:
continue
self.consts.add(constant)
result = self.consts.high()
proc addConstant*(self: Chunk, constant: ASTNode): array[3, uint8] =
## Writes a constant to a chunk. Returns its index casted to a 3-byte
## sequence (array)
self.consts.add(constant)
result = self.consts.high().toTriple()
## sequence (array). Constant indexes are reused if a constant is used
## more than once!
if self.consts.len() == 16777215:
# The constant index is a 24 bit unsigned integer, so that's as far
# as we can index into the constant table (the same applies
# to our stack by the way). Not that anyone's ever gonna hit this
# limit in the real world, but you know, just in case
raise newException(CompileError, "cannot encode more than 16777215 constants")
result = self.findOrAddConstant(constant).toTriple()

5
src/backend/serializer.nim
View File

@ -59,6 +59,8 @@ proc initSerializer*(): Serializer =
result.chunk = nil
## Basic routines and helpers to convert various objects from and to to their byte representation
proc toBytes(self: Serializer, s: string): seq[byte] =
for c in s:
result.add(byte(c))
@ -83,13 +85,14 @@ proc bytesToInt(self: Serializer, input: array[8, byte]): int =
proc extend[T](s: var seq[T], a: openarray[T]) =
## Extends s with the elements of a
for e in a:
s.add(e)
proc dumpBytes*(self: Serializer, chunk: Chunk, file, filename: string): seq[byte] =
## Dumps the given bytecode and file to a sequence of bytes and returns it.
## The file's content is needed to compute its SHA256 hash.
## The file argument must be the actual file's content and is needed to compute its SHA256 hash.
self.file = file
self.filename = filename
self.chunk = chunk

13
src/main.nim
View File

@ -39,13 +39,12 @@ proc main() =
var compiled: Chunk
var serialized: Serialized
var serializedRaw: seq[byte]
var lexer = initLexer()
var parser = initParser()
var optimizer = initOptimizer(foldConstants=false)
var compiler = initCompiler()
var serializer = initSerializer()
var hashMatches: bool
var compileDate: string
echo "NimVM REPL\n"
while true:
@ -77,7 +76,7 @@ proc main() =
optimized = optimizer.optimize(tree)
echo &"Optimization step (constant folding enabled {optimizer.foldConstants}):"
echo &"Optimization step (constant folding enabled: {optimizer.foldConstants}):"
for node in optimized.tree:
echo "\t", node
echo ""
@ -106,14 +105,12 @@ proc main() =
echo ""
serialized = serializer.loadBytes(serializedRaw)
hashMatches = if computeSHA256(source).toHex().toLowerAscii() == serialized.fileHash: true else: false
echo "Deserialization step:"
echo &"\t\t- File hash: {serialized.fileHash} (matches: {hashMatches})"
echo &"\t\t- File hash: {serialized.fileHash} (matches: {computeSHA256(source).toHex().toLowerAscii() == serialized.fileHash})"
echo &"\t\t- JAPL version: {serialized.japlVer.major}.{serialized.japlVer.minor}.{serialized.japlVer.patch} (commit {serialized.commitHash[0..8]} on branch {serialized.japlBranch})"
compileDate = fromUnix(serialized.compileDate).format("d/M/yyyy H:mm:ss")
echo &"\t\t- Compilation date & time: {compileDate}"
stdout.write("\t\t")
echo &"""- Compilation date & time: {fromUnix(serialized.compileDate).format("d/M/yyyy H:mm:ss")}"""
except:
raise
echo &"A Nim runtime exception occurred: {getCurrentExceptionMsg()}"
continue

4
src/util/debugger.nim
View File

@ -56,7 +56,9 @@ proc simpleInstruction(instruction: OpCode, offset: int): int =
proc byteInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
var slot = chunk.code[offset + 1]
printInstruction(instruction)
stdout.write(&", points to slot {slot}")
stdout.write(&", points to slot ")
setForegroundColor(fgYellow)
stdout.write(&"{slot}")
nl()
return offset + 2