From d33a597f193f196e20977090cc78419e5e600bfb Mon Sep 17 00:00:00 2001 From: Mattia Giambirtone Date: Mon, 17 Oct 2022 11:28:00 +0200 Subject: [PATCH] Massive improvements to import system (added export statement, initial work on namespaces, module names can now be paths, added module search paths) --- src/config.nim | 2 + src/frontend/compiler.nim | 342 ++++++++++++++++++++++-------------- src/frontend/meta/ast.nim | 10 ++ src/frontend/meta/token.nim | 3 +- src/frontend/parser.nim | 231 ++++++++++++------------ src/main.nim | 21 +-- src/util/symbols.nim | 11 +- tests/std.pn | 223 ----------------------- 8 files changed, 354 insertions(+), 489 deletions(-) delete mode 100644 tests/std.pn diff --git a/src/config.nim b/src/config.nim index e414d1a..cc9d333 100644 --- a/src/config.nim +++ b/src/config.nim @@ -28,6 +28,8 @@ const PeonBytecodeMarker* = "PEON_BYTECODE" # Magic value at the beginning of const HeapGrowFactor* = 2 # The growth factor used by the GC to schedule the next collection const FirstGC* = 1024 * 1024; # How many bytes to allocate before running the first GC const enableVMChecks* {.booldefine.} = true; # Enables all types of compiler (nim-wise) checks in the VM +# List of paths where peon looks for modules, in order (empty path means current directory, which always takes precedence) +const lookupPaths*: seq[string] = @["", "src/peon/stdlib"] when HeapGrowFactor <= 1: {.fatal: "Heap growth factor must be > 1".} const PeonVersion* = (major: 0, minor: 1, patch: 0) diff --git a/src/frontend/compiler.nim b/src/frontend/compiler.nim index bd338ee..c17c9f1 100644 --- a/src/frontend/compiler.nim +++ b/src/frontend/compiler.nim @@ -18,6 +18,8 @@ import ../util/multibyte import ../util/symbols import lexer as l import parser as p +import ../config + import std/tables import std/strformat @@ -88,13 +90,13 @@ export bytecode type NameKind {.pure.} = enum ## A name enumeration type - None, Argument, Var, Function, Type + None, Module, Argument, Var, Function, CustomType, Enum Name = ref object ## A compile-time wrapper around ## statically resolved names - # Name of the identifier - name: IdentExpr + # The name's identifier + ident: IdentExpr # Type of the identifier (NOT of the value!) kind: NameKind # Owner of the identifier (module) @@ -124,18 +126,20 @@ type # is needed because we compile declarations only # if they're actually used node: Declaration + # Is this name exported? (Only makes sense if isPrivate + # equals false) + exported: bool Loop = object ## A "loop object" used ## by the compiler to emit ## appropriate jump offsets ## for continue and break ## statements - # Position in the bytecode where the loop starts start: int # Scope depth where the loop is located depth: int - # Absolute jump offsets into our bytecode that we need to + # Jump offsets into our bytecode that we need to # patch. Used for break statements breakJumps: seq[int] @@ -162,8 +166,6 @@ type scopeOwners: seq[tuple[owner: Name, depth: int]] # The current function being compiled currentFunction: Name - # Are optimizations turned on? - enableOptimizations: bool # The current loop being compiled (used to # keep track of where to jump) currentLoop: Loop @@ -193,9 +195,10 @@ type closedOver: seq[Name] # Compiler procedures called by pragmas compilerProcs: TableRef[string, proc (self: Compiler, pragma: Pragma, node: ASTNode, name: Name)] - # Stores line data + # Stores line data for error reporting lines: seq[tuple[start, stop: int]] - # The source of the current module + # The source of the current module, + # used for error reporting source: string # Currently imported modules modules: HashSet[string] @@ -203,6 +206,11 @@ type jumps: seq[tuple[patched: bool, offset: int]] # List of CFI start offsets into our CFI data cfiOffsets: seq[tuple[start, stop: int, fn: Name]] + # We store these objects to compile modules + lexer: Lexer + parser: Parser + # Are we compiling the main module? + isMainModule: bool CompileError* = ref object of PeonException compiler*: Compiler node*: ASTNode @@ -212,18 +220,19 @@ type # Forward declarations proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, - terminateScope: bool = true, incremental: bool = false): Chunk + incremental: bool = false, isMainModule: bool = true): Chunk proc expression(self: Compiler, node: Expression) proc statement(self: Compiler, node: Statement) proc declaration(self: Compiler, node: Declaration) proc peek(self: Compiler, distance: int = 0): ASTNode proc identifier(self: Compiler, node: IdentExpr) proc varDecl(self: Compiler, node: VarDecl, name: Name) -proc specializeGeneric(self: Compiler, fn: Name, args: seq[Expression]): Name +proc specialize(self: Compiler, name: Name, args: seq[Expression]): Name proc matchImpl(self: Compiler, name: string, kind: Type, node: ASTNode = nil): Name proc infer(self: Compiler, node: LiteralExpr): Type proc infer(self: Compiler, node: Expression): Type proc findByName(self: Compiler, name: string): seq[Name] +proc findByModule(self: Compiler, name: string): seq[Name] proc findByType(self: Compiler, name: string, kind: Type, depth: int = -1): seq[Name] proc compare(self: Compiler, a, b: Type): bool proc patchReturnAddress(self: Compiler, pos: int) @@ -232,12 +241,12 @@ proc handlePurePragma(self: Compiler, pragma: Pragma, node: ASTnode, name: Name) proc dispatchPragmas(self: Compiler, node: ASTnode, name: Name) proc funDecl(self: Compiler, node: FunDecl, name: Name) proc typeDecl(self: Compiler, node: TypeDecl, name: Name) -proc compileModule(self: Compiler, filename: string) +proc compileModule(self: Compiler, moduleName: string) proc generateCall(self: Compiler, fn: Name, args: seq[Expression], line: int) # End of forward declarations -proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Compiler = +proc newCompiler*(replMode: bool = false): Compiler = ## Initializes a new Compiler object new(result) result.ast = @[] @@ -248,7 +257,6 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com result.lines = @[] result.jumps = @[] result.currentFunction = nil - result.enableOptimizations = enableOptimizations result.replMode = replMode result.currentModule = "" result.compilerProcs = newTable[string, proc (self: Compiler, pragma: Pragma, node: ASTNode, name: Name)]() @@ -256,6 +264,10 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com result.compilerProcs["pure"] = handlePurePragma result.source = "" result.scopeOwners = @[] + result.lexer = newLexer() + result.lexer.fillSymbolTable() + result.parser = newParser() + result.isMainModule = false ## Public getters for nicer error formatting @@ -483,17 +495,19 @@ proc resolve(self: Compiler, name: IdentExpr, ## for the first time, calling this function will also ## cause its declaration to be compiled for obj in reversed(self.names): - if obj.name.token.lexeme == name.token.lexeme: - if obj.isPrivate and obj.owner != self.currentModule: - continue # There may be a name in the current module that - # matches, so we skip this + if obj.ident.token.lexeme == name.token.lexeme: + if obj.owner != self.currentModule: + if obj.isPrivate: + continue + elif obj.exported: + return obj if not obj.resolved: obj.resolved = true obj.codePos = self.chunk.code.len() case obj.kind: of NameKind.Var: self.varDecl(VarDecl(obj.node), obj) - of NameKind.Type: + of NameKind.CustomType: self.typeDecl(TypeDecl(obj.node), obj) of NameKind.Function: if not obj.valueType.isGeneric: @@ -512,17 +526,19 @@ proc resolve(self: Compiler, name: string, ## Version of resolve that takes strings instead ## of AST nodes for obj in reversed(self.names): - if obj.name.token.lexeme == name: - if obj.isPrivate and obj.owner != self.currentModule: - continue # There may be a name in the current module that - # matches, so we skip this + if obj.ident.token.lexeme == name: + if obj.owner != self.currentModule: + if obj.isPrivate: + continue + elif obj.exported: + return obj if not obj.resolved: obj.resolved = true obj.codePos = self.chunk.code.len() case obj.kind: of NameKind.Var: self.varDecl(VarDecl(obj.node), obj) - of NameKind.Type: + of NameKind.CustomType: self.typeDecl(TypeDecl(obj.node), obj) of NameKind.Function: if not obj.valueType.isGeneric: @@ -780,13 +796,13 @@ proc infer(self: Compiler, node: Expression): Type = let impl = self.matchImpl(node.operator.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: @[("", self.infer(node.a))]), node) result = impl.valueType.returnType if result.kind == Generic: - result = self.specializeGeneric(impl, @[node.a]).valueType.returnType + result = self.specialize(impl, @[node.a]).valueType.returnType of binaryExpr: let node = BinaryExpr(node) let impl = self.matchImpl(node.operator.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: @[("", self.infer(node.a)), ("", self.infer(node.b))]), node) result = impl.valueType.returnType if result.kind == Generic: - result = self.specializeGeneric(impl, @[node.a, node.b]).valueType.returnType + result = self.specialize(impl, @[node.a, node.b]).valueType.returnType of {intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr @@ -872,9 +888,18 @@ proc findByName(self: Compiler, name: string): seq[Name] = ## Looks for objects that have been already declared ## with the given name. Returns all objects that apply for obj in reversed(self.names): - if obj.name.token.lexeme == name: - if obj.isPrivate and obj.owner != self.currentModule: - continue + if obj.ident.token.lexeme == name: + if obj.owner != self.currentModule: + if obj.isPrivate or not obj.exported: + continue + result.add(obj) + + +proc findByModule(self: Compiler, name: string): seq[Name] = + ## Looks for objects that have been already declared AS + ## public within the given module. Returns all objects that apply + for obj in reversed(self.names): + if obj.owner == name: result.add(obj) @@ -911,7 +936,7 @@ proc matchImpl(self: Compiler, name: string, kind: Type, node: ASTNode = nil): N msg &= "s" msg &= ": " for name in names: - msg &= &"\n - in module '{name.owner}' at line {name.name.token.line} of type '{self.typeToStr(name.valueType)}'" + msg &= &"\n - in module '{name.owner}' at line {name.ident.token.line} of type '{self.typeToStr(name.valueType)}'" if name.valueType.kind != Function: msg &= ", not a callable" elif kind.args.len() != name.valueType.args.len(): @@ -1060,6 +1085,8 @@ proc endScope(self: Compiler) = self.error("cannot call endScope with scopeDepth < 0 (This is an internal error and most likely a bug)") discard self.scopeOwners.pop() dec(self.scopeDepth) + if not self.isMainModule: + return var names: seq[Name] = @[] var popCount = 0 for name in self.names: @@ -1145,7 +1172,7 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m of binaryExpr: let condition = BinaryExpr(condition) case condition.operator.lexeme: - of "|", "or": + of "|": self.unpackGenerics(condition.a, list) self.unpackGenerics(condition.b, list) else: @@ -1153,7 +1180,7 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m of unaryExpr: let condition = UnaryExpr(condition) case condition.operator.lexeme: - of "~", "not": + of "~": self.unpackGenerics(condition.a, list, accept=false) else: self.error("invalid type constraint in generic declaration", condition) @@ -1161,13 +1188,14 @@ proc unpackGenerics(self: Compiler, condition: Expression, list: var seq[tuple[m self.error("invalid type constraint in generic declaration", condition) -proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name = +proc declareName(self: Compiler, node: ASTNode, mutable: bool = false): Name = ## Statically declares a name into the current scope. ## "Declaring" a name only means updating our internal ## list of identifiers so that further calls to resolve() ## correctly return them. There is no code to actually ## declare a variable at runtime: the value is already ## on the stack + var declaredName: string = "" case node.kind: of NodeKind.varDecl: var node = VarDecl(node) @@ -1176,14 +1204,9 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name # If someone ever hits this limit in real-world scenarios, I swear I'll # slap myself 100 times with a sign saying "I'm dumb". Mark my words self.error("cannot declare more than 16777215 variables at a time") - for name in self.findByName(node.name.token.lexeme): - if name.kind == NameKind.Var and name.depth == self.scopeDepth: - # Trying to redeclare a variable in the same scope is an error, but it's okay - # if it's something else, like a function argument (for example, if you want to copy a - # number to mutate it) - self.error(&"attempt to redeclare '{node.name.token.lexeme}', which was previously defined in '{name.owner}' at line {name.line}") + declaredName = node.name.token.lexeme self.names.add(Name(depth: self.scopeDepth, - name: node.name, + ident: node.name, isPrivate: node.isPrivate, owner: self.currentModule, isConst: node.isConst, @@ -1213,7 +1236,7 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name owner: self.currentModule, line: node.token.line, valueType: Type(kind: Generic, name: gen.name.token.lexeme, mutable: false, cond: constraints), - name: gen.name, + ident: gen.name, node: node)) generics.add(self.names[^1]) let fn = Name(depth: self.scopeDepth, @@ -1225,11 +1248,12 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name args: @[], fun: node, children: @[]), - name: node.name, + ident: node.name, node: node, isLet: false, line: node.token.line, - kind: NameKind.Function) + kind: NameKind.Function, + belongsTo: self.currentFunction) self.names.add(fn) var name: Name for argument in node.arguments: @@ -1242,7 +1266,7 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name isPrivate: true, owner: self.currentModule, isConst: false, - name: argument.name, + ident: argument.name, valueType: nil, codePos: 0, isLet: false, @@ -1259,8 +1283,26 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false): Name if generics.len() > 0: fn.valueType.isGeneric = true return fn + of NodeKind.importStmt: + var node = ImportStmt(node) + var name = node.moduleName.token.lexeme.extractFilename().replace(".pn", "") + declaredName = name + for name in self.findByName(name): + if name.kind in [NameKind.Var, NameKind.Module] and name.depth == self.scopeDepth: + self.error(&"attempt to redeclare '{name}', which was previously defined in '{name.owner}' at line {name.line}") + self.names.add(Name(depth: self.scopeDepth, + owner: self.currentModule, + ident: newIdentExpr(Token(kind: Identifier, lexeme: name, line: node.moduleName.token.line)), + line: node.moduleName.token.line, + kind: NameKind.Module, + isPrivate: false + )) + return self.names[^1] else: discard # TODO: Types, enums + for name in self.findByName(declaredName): + if (name.kind == NameKind.Var and name.depth == self.scopeDepth) or name.kind in [NameKind.Module, NameKind.CustomType, NameKind.Enum]: + self.error(&"attempt to redeclare '{name}', which was previously defined in '{name.owner}' at line {name.line}") proc emitLoop(self: Compiler, begin: int, line: int) = @@ -1341,18 +1383,17 @@ proc patchReturnAddress(self: Compiler, pos: int) = self.chunk.consts[pos + 7] = address[7] -proc terminateProgram(self: Compiler, pos: int, terminateScope: bool = true) = +proc terminateProgram(self: Compiler, pos: int) = ## Utility to terminate a peon program - if terminateScope: - self.endScope() - self.patchReturnAddress(pos + 3) + self.endScope() self.emitByte(OpCode.Return, self.peek().token.line) self.emitByte(0, self.peek().token.line) # Entry point has no return value (TODO: Add easter eggs, cuz why not) + self.patchReturnAddress(pos) -proc beginProgram(self: Compiler, incremental: bool = false): int = - ## Utility to begin a peon program - ## compiled. Returns the position of +proc beginProgram(self: Compiler): int = + ## Utility to begin a peon program's + ## bytecode. Returns the position of ## a dummy return address of the program's ## entry point to be patched by terminateProgram @@ -1366,11 +1407,7 @@ proc beginProgram(self: Compiler, incremental: bool = false): int = # of where our program ends (which we don't know yet). # To fix this, we emit dummy offsets and patch them # later, once we know the boundaries of our hidden main() - var main: Name - if incremental: - main = self.names[0] - else: - main = Name(depth: 0, + var main = Name(depth: 0, isPrivate: true, isConst: false, isLet: false, @@ -1379,19 +1416,19 @@ proc beginProgram(self: Compiler, incremental: bool = false): int = returnType: nil, args: @[], ), - codePos: 12, # Jump address is hardcoded - name: newIdentExpr(Token(lexeme: "", kind: Identifier)), + codePos: self.chunk.code.len() + 12, + ident: newIdentExpr(Token(lexeme: "", kind: Identifier)), kind: NameKind.Function, line: -1) - self.names.add(main) - self.scopeOwners.add((main, 0)) - self.emitByte(LoadUInt64, 1) - self.emitBytes(self.chunk.writeConstant(main.codePos.toLong()), 1) - self.emitByte(LoadUInt64, 1) - self.emitBytes(self.chunk.writeConstant(0.toLong()), 1) - self.emitByte(Call, 1) - self.emitBytes(0.toTriple(), 1) - result = 5 + self.names.add(main) + self.scopeOwners.add((main, 0)) + self.emitByte(LoadUInt64, 1) + self.emitBytes(self.chunk.writeConstant(main.codePos.toLong()), 1) + self.emitByte(LoadUInt64, 1) + self.emitBytes(self.chunk.writeConstant(0.toLong()), 1) + result = self.chunk.consts.len() - 8 + self.emitByte(Call, 1) + self.emitBytes(0.toTriple(), 1) ## End of utility functions @@ -1683,40 +1720,44 @@ proc generateCall(self: Compiler, fn: Name, args: seq[Expression], line: int) = self.patchReturnAddress(pos) -proc specializeGeneric(self: Compiler, fn: Name, args: seq[Expression]): Name = - ## Specializes a generic function by +proc specialize(self: Compiler, name: Name, args: seq[Expression]): Name = + ## Specializes a generic type by ## instantiating a concrete version ## of it var mapping: TableRef[string, Type] = newTable[string, Type]() var kind: Type - result = deepCopy(fn) - # This first loop checks if a user tries to reassign a generic's - # name to a different type - for i, (name, typ) in result.valueType.args: - if typ.kind != Generic: - continue - kind = self.infer(args[i]) - if typ.name in mapping and not self.compare(kind, mapping[typ.name]): - self.error(&"expected generic argument '{typ.name}' to be of type {self.typeToStr(mapping[typ.name])}, got {self.typeToStr(kind)} instead") - mapping[typ.name] = kind - result.valueType.args[i].kind = kind - for (argExpr, argName) in zip(args, result.valueType.args): - if self.names.high() > 16777215: - self.error("cannot declare more than 16777215 variables at a time") - self.names.add(Name(depth: self.scopeDepth + 1, - isPrivate: true, - owner: self.currentModule, - isConst: false, - name: newIdentExpr(Token(lexeme: argName.name)), - valueType: argName.kind, - codePos: 0, - isLet: false, - line: fn.line, - belongsTo: result, - kind: NameKind.Argument - )) - if result.valueType.returnType.kind == Generic: - result.valueType.returnType = mapping[result.valueType.returnType.name] + result = deepCopy(name) + case name.kind: + of NameKind.Function: + # This first loop checks if a user tries to reassign a generic's + # name to a different type + for i, (name, typ) in result.valueType.args: + if typ.kind != Generic: + continue + kind = self.infer(args[i]) + if typ.name in mapping and not self.compare(kind, mapping[typ.name]): + self.error(&"expected generic argument '{typ.name}' to be of type {self.typeToStr(mapping[typ.name])}, got {self.typeToStr(kind)} instead") + mapping[typ.name] = kind + result.valueType.args[i].kind = kind + for (argExpr, argName) in zip(args, result.valueType.args): + if self.names.high() > 16777215: + self.error("cannot declare more than 16777215 variables at a time") + self.names.add(Name(depth: self.scopeDepth + 1, + isPrivate: true, + owner: self.currentModule, + isConst: false, + ident: newIdentExpr(Token(lexeme: argName.name)), + valueType: argName.kind, + codePos: 0, + isLet: false, + line: name.line, + belongsTo: result, + kind: NameKind.Argument + )) + if result.valueType.returnType.kind == Generic: + result.valueType.returnType = mapping[result.valueType.returnType.name] + else: + discard # TODO: Custom user-defined types proc callExpr(self: Compiler, node: CallExpr): Name {.discardable.} = @@ -1743,7 +1784,7 @@ proc callExpr(self: Compiler, node: CallExpr): Name {.discardable.} = # Calls like hi() result = self.matchImpl(IdentExpr(node.callee).name.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: args), node) if result.valueType.isGeneric: - result = self.specializeGeneric(result, argExpr) + result = self.specialize(result, argExpr) # We can't instantiate a concrete version # of a generic function without the types # of its arguments, so we wait until the @@ -1910,16 +1951,40 @@ proc forEachStmt(self: Compiler, node: ForEachStmt) = proc importStmt(self: Compiler, node: ImportStmt) = ## Imports a module at compile time - # TODO: This is obviously horrible. It's just a test - let filename = node.moduleName.token.lexeme & ".pn" + let filename = splitPath(node.moduleName.token.lexeme).tail try: - self.compileModule(filename) + self.compileModule(node.moduleName.token.lexeme) + discard self.declareName(node) except IOError: self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""") except OSError: self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""") +proc exportStmt(self: Compiler, node: ExportStmt) = + ## Exports a name at compile time to + ## all modules importing us + var name = self.resolve(node.name) + if name.isNil(): + self.error(&"reference to undefined name '{node.name.token.lexeme}'") + if name.isPrivate: + self.error("cannot export private names") + name.exported = true + case name.kind: + of NameKind.Module: + # We need to export everything + # this module defines! + for name in self.findByModule(name.ident.token.lexeme): + name.exported = true + of NameKind.Function: + for name in self.findByName(name.ident.token.lexeme): + if name.kind != NameKind.Function: + continue + name.exported = true + else: + discard + + proc printRepl(self: Compiler, typ: Type, node: Expression) = ## Emits instruction to print ## peon types in REPL mode @@ -1985,6 +2050,8 @@ proc statement(self: Compiler, node: Statement) = self.returnStmt(ReturnStmt(node)) of NodeKind.importStmt: self.importStmt(ImportStmt(node)) + of NodeKind.exportStmt: + self.exportStmt(ExportStmt(node)) of NodeKind.whileStmt: # Note: Our parser already desugars # for loops to while loops @@ -2063,8 +2130,8 @@ proc funDecl(self: Compiler, node: FunDecl, name: Name) = self.chunk.cfi.add(0.toTriple()) # Patched it later self.chunk.cfi.add(uint8(node.arguments.len())) if not node.name.isNil(): - self.chunk.cfi.add(name.name.token.lexeme.len().toDouble()) - var s = name.name.token.lexeme + self.chunk.cfi.add(name.ident.token.lexeme.len().toDouble()) + var s = name.ident.token.lexeme if s.len() >= uint16.high().int: s = node.name.token.lexeme[0..uint16.high()] self.chunk.cfi.add(s.toBytes()) @@ -2141,7 +2208,7 @@ proc declaration(self: Compiler, node: Declaration) = proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, - terminateScope: bool = true, incremental: bool = false): Chunk = + incremental: bool = false, isMainModule: bool = true): Chunk = ## Compiles a sequence of AST nodes into a chunk ## object if chunk.isNil(): @@ -2150,47 +2217,54 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tu self.chunk = chunk self.ast = ast self.file = file - var terminateScope = terminateScope - if incremental: - terminateScope = false self.scopeDepth = 0 self.currentFunction = nil - self.currentModule = self.file.extractFilename() + self.currentModule = self.file.extractFilename().replace(".pn", "") self.current = 0 self.lines = lines self.source = source - self.jumps = @[] - let pos = self.beginProgram(incremental) - if incremental and self.replMode: - discard self.chunk.code.pop() - discard self.chunk.code.pop() + self.isMainModule = isMainModule + if not incremental: + self.jumps = @[] + let pos = self.beginProgram() while not self.done(): self.declaration(Declaration(self.step())) - self.terminateProgram(pos, terminateScope) + self.terminateProgram(pos) + # TODO: REPL is broken, we need a new way to make + # incremental compilation resume from where it stopped! result = self.chunk - if incremental and not self.replMode: - discard self.chunk.code.pop() - discard self.chunk.code.pop() -proc compileModule(self: Compiler, filename: string) = - ## Compiles an imported module into an existing chunk. - ## A temporary compiler object is initialized internally - let path = joinPath(splitPath(self.file).head, filename) +proc compileModule(self: Compiler, moduleName: string) = + ## Compiles an imported module into an existing chunk + ## using the compiler's internal parser and lexer objects + var path = "" + for i, searchPath in lookupPaths: + path = joinPath(getCurrentDir(), joinPath(searchPath, moduleName)) + if fileExists(path): + break + elif i == searchPath.high(): + self.error(&"""could not import '{path}': module not found""") if self.modules.contains(path): return - var lexer = newLexer() - var parser = newParser() - var compiler = newCompiler() - lexer.fillSymbolTable() let source = readFile(path) - let tokens = lexer.lex(source, filename) - let ast = parser.parse(tokens, filename, lexer.getLines(), source) - compiler.names.add(self.names[0]) - discard compiler.compile(ast, filename, lexer.getLines(), source, chunk=self.chunk, incremental=true) - for name in compiler.names: - if name.owner in self.modules: - continue - self.names.add(name) + let current = self.current + let ast = self.ast + let file = self.file + let module = self.currentModule + let lines = self.lines + let src = self.source + self.isMainModule = false + discard self.compile(self.parser.parse(self.lexer.lex(source, path), + path, self.lexer.getLines(), + source, persist=true), + path, self.lexer.getLines(), source, chunk=self.chunk, incremental=true, + isMainModule=false) + self.scopeDepth = 0 + self.current = current + self.ast = ast + self.file = file + self.currentModule = module + self.lines = lines + self.source = src self.modules.incl(path) - self.closedOver &= compiler.closedOver diff --git a/src/frontend/meta/ast.nim b/src/frontend/meta/ast.nim index 9f0f276..8578e7f 100644 --- a/src/frontend/meta/ast.nim +++ b/src/frontend/meta/ast.nim @@ -47,6 +47,7 @@ type yieldStmt, awaitStmt, importStmt, + exportStmt, deferStmt, # An expression followed by a semicolon exprStmt, @@ -198,6 +199,9 @@ type ImportStmt* = ref object of Statement moduleName*: IdentExpr + ExportStmt* = ref object of Statement + name*: IdentExpr + AssertStmt* = ref object of Statement expression*: Expression @@ -510,6 +514,12 @@ proc newImportStmt*(moduleName: IdentExpr, token: Token): ImportStmt = result.token = token +proc newExportStmt*(name: IdentExpr, token: Token): ExportStmt = + result = ExportStmt(kind: exportStmt) + result.name = name + result.token = token + + proc newYieldStmt*(expression: Expression, token: Token): YieldStmt = result = YieldStmt(kind: yieldStmt) result.expression = expression diff --git a/src/frontend/meta/token.nim b/src/frontend/meta/token.nim index d84e32d..28ad9a5 100644 --- a/src/frontend/meta/token.nim +++ b/src/frontend/meta/token.nim @@ -40,7 +40,8 @@ type Raise, Assert, Await, Foreach, Yield, Defer, Try, Except, Finally, Type, Operator, Case, - Enum, From, Ptr, Ref, Object + Enum, From, Ptr, Ref, Object, + Export, # Literal types Integer, Float, String, Identifier, diff --git a/src/frontend/parser.nim b/src/frontend/parser.nim index 3cce51c..5f01dae 100644 --- a/src/frontend/parser.nim +++ b/src/frontend/parser.nim @@ -24,6 +24,7 @@ import meta/ast import meta/errors import lexer as l import ../util/symbols +import ../config export token, ast, errors @@ -42,6 +43,7 @@ type Or, And, Compare, + Bitwise, Addition, Multiplication, Power, @@ -86,8 +88,7 @@ type currentFunction: Declaration # Stores the current scope depth (0 = global, > 0 local) scopeDepth: int - # TODO - scopes: seq[Declaration] + # Operator table operators: OperatorTable # The AST node tree: seq[Declaration] @@ -95,6 +96,8 @@ type lines: seq[tuple[start, stop: int]] # The source of the current module source: string + # Keeps track of imported modules + modules: seq[tuple[name: string, loaded: bool]] ParseError* = ref object of PeonException parser*: Parser file*: string @@ -121,9 +124,9 @@ proc addOperator(self: OperatorTable, lexeme: string) = var prec = Power if lexeme.len() >= 2 and lexeme[^2..^1] in ["->", "~>", "=>"]: prec = Arrow - elif lexeme in ["and", "&"]: + elif lexeme == "and": prec = Precedence.And - elif lexeme in ["or", "|"]: + elif lexeme == "or": prec = Precedence.Or elif lexeme.endsWith("=") and lexeme[0] notin {'<', '>', '!', '?', '~', '='} or lexeme == "=": prec = Assign @@ -131,8 +134,10 @@ proc addOperator(self: OperatorTable, lexeme: string) = prec = Power elif lexeme[0] in {'*', '%', '/', '\\'}: prec = Multiplication - elif lexeme[0] in {'+', '-', '|', '~'}: + elif lexeme[0] in {'+', '-'}: prec = Addition + elif lexeme in [">>", "<<", "|", "~", "&", "^"]: + prec = Bitwise elif lexeme[0] in {'<', '>', '=', '!'}: prec = Compare self.tokens.add(lexeme) @@ -301,6 +306,7 @@ proc parseFunExpr(self: Parser): LambdaExpr proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration proc declaration(self: Parser): Declaration +proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration] # End of forward declarations @@ -485,9 +491,20 @@ proc parseAdd(self: Parser): Expression = result = newBinaryExpr(result, operator, right) +proc parseBitwise(self: Parser): Expression = + ## Parses bitwise expressions + result = self.parseAdd() + var operator: Token + var right: Expression + while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Bitwise: + operator = self.step() + right = self.parseAdd() + result = newBinaryExpr(result, operator, right) + + proc parseCmp(self: Parser): Expression = ## Parses comparison expressions - result = self.parseAdd() + result = self.parseBitwise() var operator: Token var right: Expression while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Compare: @@ -553,7 +570,7 @@ proc assertStmt(self: Parser): Statement = ## fed into them is false let tok = self.peek(-1) var expression = self.expression() - endOfLine("missing statement terminator after 'assert'") + endOfLine("missing semicolon after 'assert'") result = newAssertStmt(expression, tok) @@ -588,7 +605,7 @@ proc breakStmt(self: Parser): Statement = let tok = self.peek(-1) if self.currentLoop != Loop: self.error("'break' cannot be used outside loops") - endOfLine("missing statement terminator after 'break'") + endOfLine("missing semicolon after 'break'") result = newBreakStmt(tok) @@ -597,7 +614,7 @@ proc deferStmt(self: Parser): Statement = let tok = self.peek(-1) if self.currentFunction.isNil(): self.error("'defer' cannot be used outside functions") - endOfLine("missing statement terminator after 'defer'") + endOfLine("missing semicolon after 'defer'") result = newDeferStmt(self.expression(), tok) @@ -606,7 +623,7 @@ proc continueStmt(self: Parser): Statement = let tok = self.peek(-1) if self.currentLoop != Loop: self.error("'continue' cannot be used outside loops") - endOfLine("missing statement terminator after 'continue'") + endOfLine("missing semicolon after 'continue'") result = newContinueStmt(tok) @@ -621,7 +638,7 @@ proc returnStmt(self: Parser): Statement = # we need to check if there's an actual value # to return or not value = self.expression() - endOfLine("missing statement terminator after 'return'") + endOfLine("missing semicolon after 'return'") result = newReturnStmt(value, tok) case self.currentFunction.kind: of NodeKind.funDecl: @@ -641,7 +658,7 @@ proc yieldStmt(self: Parser): Statement = result = newYieldStmt(self.expression(), tok) else: result = newYieldStmt(newNilExpr(Token(lexeme: "nil")), tok) - endOfLine("missing statement terminator after 'yield'") + endOfLine("missing semicolon after 'yield'") proc awaitStmt(self: Parser): Statement = @@ -651,7 +668,7 @@ proc awaitStmt(self: Parser): Statement = self.error("'await' cannot be used outside functions") if self.currentFunction.token.kind != Coroutine: self.error("'await' can only be used inside coroutines") - endOfLine("missing statement terminator after 'await'") + endOfLine("missing semicolon after 'await'") result = newAwaitStmt(self.expression(), tok) @@ -663,7 +680,7 @@ proc raiseStmt(self: Parser): Statement = # Raise can be used on its own, in which # case it re-raises the last active exception exception = self.expression() - endOfLine("missing statement terminator after 'raise'") + endOfLine("missing semicolon after 'raise'") result = newRaiseStmt(exception, tok) @@ -681,12 +698,13 @@ proc forEachStmt(self: Parser): Statement = self.currentLoop = enclosingLoop -proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration] proc findOperators(self: Parser, tokens: seq[Token]) proc importStmt(self: Parser, fromStmt: bool = false): Statement = - ## Parses import statements + ## Parses import statements. This is a little + ## convoluted because we need to pre-parse the + ## module to import the operators from it if self.scopeDepth > 0: self.error("import statements are only allowed at the top level") var tok: Token @@ -694,43 +712,71 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement = tok = self.peek(-2) else: tok = self.peek(-1) - # TODO: New AST node - self.expect(Identifier, "expecting module name(s) after import statement") - endOfLine("missing statement terminator after 'import'") - result = newImportStmt(newIdentExpr(self.peek(-2), self.scopeDepth), tok) - var filename = ImportStmt(result).moduleName.token.lexeme & ".pn" + var moduleName = "" + while not self.check(Semicolon) and not self.done(): + if self.match(".."): + if not self.check("/"): + self.error("expecting '/' after '..' in import statement") + moduleName &= "../" + elif self.match("/"): + self.expect(Identifier, "expecting identifier after '/' in import statement") + moduleName &= &"/{self.peek(-1).lexeme}" + elif self.match(Identifier): + moduleName &= self.peek(-1).lexeme + else: + break + endOfLine("missing semicolon after import statement") + moduleName &= ".pn" + result = newImportStmt(newIdentExpr(Token(kind: Identifier, lexeme: moduleName, line: self.peek(-1).line), self.scopeDepth), tok) var lexer = newLexer() lexer.fillSymbolTable() - let path = joinPath(splitPath(self.file).head, filename) - # TODO: This is obviously horrible. It's just a test + var path = "" + for i, searchPath in lookupPaths: + path = joinPath(getCurrentDir(), joinPath(searchPath, moduleName)) + if fileExists(path): + break + elif i == searchPath.high(): + self.error(&"""could not import '{path}': module not found""") try: - self.findOperators(lexer.lex(readFile(path), filename)) + var source = readFile(path) + var tree = self.tree + var current = self.current + var tokens = self.tokens + var src = self.source + var file = self.file + discard self.parse(lexer.lex(source, path), file=path, source=source, lines=lexer.getLines(), persist=true) + self.file = file + self.source = src + self.tree = tree + self.current = current + self.tokens = tokens except IOError: - self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""") + self.error(&"""could not import '{path}': {getCurrentExceptionMsg()}""") except OSError: - self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""") + self.error(&"""could not import '{path}': {getCurrentExceptionMsg()} [errno {osLastError()}]""") proc tryStmt(self: Parser): Statement = ## Parses try/except/else/finally blocks let tok = self.peek(-1) - var body = self.statement() + self.expect(LeftBrace, "expecting '{' after 'try'") + var body = self.blockStmt() var handlers: seq[tuple[body: Statement, exc: IdentExpr]] = @[] var finallyClause: Statement var elseClause: Statement - var excName: Expression - var handlerBody: Statement while self.match(Except): - excName = self.expression() - if excName.kind == identExpr: - handlerBody = self.statement() - handlers.add((body: handlerBody, exc: IdentExpr(excName))) + if self.match(LeftBrace): + handlers.add((body: self.blockStmt(), exc: newIdentExpr(self.peek(-1)))) else: - excName = nil + self.expect(Identifier, "expecting exception name after 'except'") + self.expect(LeftBrace, "expecting '{' after exception name") + handlers.add((body: self.blockStmt(), exc: nil)) if self.match(Else): - elseClause = self.statement() + self.expect(LeftBrace, "expecting '{' after 'else' name") + elseClause = self.blockStmt() if self.match(Finally): - finallyClause = self.statement() + self.expect(LeftBrace, "expecting '{' after 'finally'") + finallyClause = self.blockStmt() if handlers.len() == 0 and elseClause.isNil() and finallyClause.isNil(): self.error("expecting 'except', 'finally' or 'else' statement after 'try' block", tok) for i, handler in handlers: @@ -751,65 +797,6 @@ proc whileStmt(self: Parser): Statement = self.currentLoop = enclosingLoop self.endScope() -#[ -proc forStmt(self: Parser): Statement = - ## Parses a C-style for loop - self.beginScope() - let tok = self.peek(-1) - var enclosingLoop = self.currentLoop - self.currentLoop = Loop - self.expect(LeftParen, "expecting '(' after 'for'") - var initializer: ASTNode = nil - var condition: Expression = nil - var increment: Expression = nil - if self.match(Semicolon): - discard - elif self.match(TokenType.Var): - initializer = self.varDecl() - if not VarDecl(initializer).isPrivate: - self.error("cannot declare public for loop initializer") - else: - initializer = self.expressionStatement() - if not self.check(Semicolon): - condition = self.expression() - self.expect(Semicolon, "expecting ';' after for loop condition") - if not self.check(RightParen): - increment = self.expression() - self.expect(RightParen, "unterminated for loop increment") - var body = self.statement() - if not increment.isNil(): - # The increment runs after each iteration, so we - # inject it into the block as the last statement - body = newBlockStmt(@[Declaration(body), newExprStmt(increment, - increment.token)], tok) - if condition.isNil(): - ## An empty condition is functionally - ## equivalent to "true" - condition = newTrueExpr(Token(lexeme: "true")) - # We can use a while loop, which in this case works just as well - body = newWhileStmt(condition, body, tok) - if not initializer.isNil(): - # Nested blocks, so the initializer is - # only executed once - body = newBlockStmt(@[Declaration(initializer), Declaration(body)], tok) - # This desgugars the following code: - # for (var i = 0; i < 10; i += 1) { - # print(i); - # } - # To the semantically equivalent snippet - # below: - # { - # var i = 0; - # while (i < 10) { - # print(i); - # i += 1; - # } - # } - result = body - self.currentLoop = enclosingLoop - self.endScope() -]# - proc ifStmt(self: Parser): Statement = ## Parses if statements @@ -827,6 +814,17 @@ proc ifStmt(self: Parser): Statement = result = newIfStmt(condition, thenBranch, elseBranch, tok) +proc exportStmt(self: Parser): Statement = + ## Parses export statements + var exported: IdentExpr + let tok = self.peek(-1) + if not self.match(Identifier): + self.error("expecting identifier after 'export' in export statement") + exported = newIdentExpr(self.peek(-1)) + endOfLine("missing semicolon after 'raise'") + result = newExportStmt(exported, tok) + + template checkDecl(self: Parser, isPrivate: bool) = ## Handy utility template that avoids us from copy ## pasting the same checks to all declaration handlers @@ -973,6 +971,20 @@ proc parseFunExpr(self: Parser): LambdaExpr = result.defaults = defaults +proc parseGenericConstraint(self: Parser): Expression = + ## Recursivelt parses a generic constraint + ## and returns it as an expression + result = self.expression() # First value is always an identifier of some sort + if not self.check(RightBracket): + case self.peek().lexeme: + of "|": + result = newBinaryExpr(result, self.step(), self.parseGenericConstraint()) + of "~": + result = newUnaryExpr(self.step(), result) + else: + self.error("invalid type constraint in generic declaration") + + proc parseGenerics(self: Parser, decl: Declaration) = ## Parses generics in declarations var gen: tuple[name: IdentExpr, cond: Expression] @@ -980,7 +992,7 @@ proc parseGenerics(self: Parser, decl: Declaration) = self.expect(Identifier, "expecting generic type name") gen.name = newIdentExpr(self.peek(-1), self.scopeDepth) self.expect(":", "expecting type constraint after generic name") - gen.cond = self.expression() + gen.cond = self.parseGenericConstraint() decl.generics.add(gen) if not self.match(Comma): break @@ -1032,7 +1044,6 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, elif isLambda: self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok, returnType=nil, depth=self.scopeDepth) - self.scopes.add(FunDecl(self.currentFunction)) if self.match(":"): # Function has explicit return type if self.match([Function, Coroutine, Generator]): @@ -1098,7 +1109,6 @@ proc expression(self: Parser): Expression = result = self.parseArrow() # Highest-level expression - proc expressionStatement(self: Parser): Statement = ## Parses expression statements, which ## are expressions followed by a semicolon @@ -1131,6 +1141,9 @@ proc statement(self: Parser): Statement = of Import: discard self.step() result = self.importStmt() + of Export: + discard self.step() + result = self.exportStmt() of From: # TODO # from module import a [, b, c as d] @@ -1139,11 +1152,6 @@ proc statement(self: Parser): Statement = of While: discard self.step() result = self.whileStmt() - #[ - of For: - discard self.step() - result = self.forStmt() - ]# of Foreach: discard self.step() result = self.forEachStmt() @@ -1247,25 +1255,24 @@ proc findOperators(self: Parser, tokens: seq[Token]) = self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token) self.operators.addOperator(tokens[i + 1].lexeme) if i == tokens.high() and token.kind != EndOfFile: - # Since we're iterating this list anyway might as + # Since we're iterating this list anyway we might as # well perform some extra checks self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token) - proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration] = ## Parses a sequence of tokens into a sequence of AST nodes self.tokens = tokens self.file = file - self.current = 0 - self.currentLoop = LoopContext.None - self.currentFunction = nil - self.scopeDepth = 0 - if not persist: - self.operators = newOperatorTable() - self.tree = @[] self.source = source self.lines = lines + self.current = 0 + self.scopeDepth = 0 + self.currentLoop = LoopContext.None + self.currentFunction = nil + self.tree = @[] + if not persist: + self.operators = newOperatorTable() self.findOperators(tokens) while not self.done(): self.tree.add(self.declaration()) diff --git a/src/main.nim b/src/main.nim index 7ab5bc0..e90ed8d 100644 --- a/src/main.nim +++ b/src/main.nim @@ -111,7 +111,7 @@ proc repl = for node in tree: styledEcho fgGreen, "\t", $node echo "" - discard compiler.compile(tree, "stdin", tokenizer.getLines(), input, chunk=compiled, incremental=incremental, terminateScope=false) + discard compiler.compile(tree, "stdin", tokenizer.getLines(), input, chunk=compiled, incremental=incremental) incremental = true when debugCompiler: styledEcho fgCyan, "Compilation step:\n" @@ -153,9 +153,8 @@ proc repl = fgYellow, &"'{exc.file.extractFilename()}'", fgRed, ", line ", fgYellow, $exc.line, fgRed, " at ", fgYellow, &"'{exc.lexeme.escape()}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(exc.lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(exc.lexeme)) except ParseError: - echo getCurrentExceptionMsg() input = "" let exc = ParseError(getCurrentException()) let lexeme = exc.token.lexeme @@ -170,7 +169,7 @@ proc repl = fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme)) except CompileError: let exc = CompileError(getCurrentException()) let lexeme = exc.node.token.lexeme @@ -185,7 +184,7 @@ proc repl = fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme)) except SerializationError: let exc = SerializationError(getCurrentException()) stderr.styledWriteLine(fgRed, "A fatal error occurred while (de-)serializing", fgYellow, &"'{exc.file}'", fgGreen, ": ", getCurrentExceptionMsg()) @@ -274,21 +273,17 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = vm.run(serialized.chunk) except LexingError: var exc = LexingError(getCurrentException()) - if exc.lexeme == "": - exc.line -= 1 let relPos = exc.lexer.getRelPos(exc.line) let line = exc.lexer.getSource().splitLines()[exc.line - 1].strip(chars={'\n'}) stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{exc.file}'", fgRed, ", module ", fgYellow, &"'{exc.file.extractFilename()}'", fgRed, ", line ", fgYellow, $exc.line, fgRed, " at ", fgYellow, &"'{exc.lexeme.escape()}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(exc.lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(exc.lexeme)) except ParseError: let exc = ParseError(getCurrentException()) let lexeme = exc.token.lexeme var lineNo = exc.token.line - if exc.token.kind == EndOfFile: - lineNo -= 1 let relPos = exc.parser.getRelPos(lineNo) let fn = parser.getCurrentFunction() let line = exc.parser.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) @@ -299,13 +294,11 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme)) except CompileError: let exc = CompileError(getCurrentException()) let lexeme = exc.node.token.lexeme var lineNo = exc.node.token.line - if exc.node.token.kind == EndOfFile: - lineNo -= 1 let relPos = exc.compiler.getRelPos(lineNo) let line = exc.compiler.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) var fn = exc.compiler.getCurrentFunction() @@ -316,7 +309,7 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line - styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) + styledEcho fgCyan, " ".repeat(len("Source line: ") + line.find(lexeme)) & "^".repeat(relPos.stop - relPos.start - line.find(lexeme)) except SerializationError: let exc = SerializationError(getCurrentException()) stderr.styledWriteLine(fgRed, "A fatal error occurred while (de-)serializing", fgYellow, &"'{exc.file}'", fgGreen, ": ", getCurrentExceptionMsg()) diff --git a/src/util/symbols.nim b/src/util/symbols.nim index 68a4423..05038ac 100644 --- a/src/util/symbols.nim +++ b/src/util/symbols.nim @@ -3,8 +3,8 @@ import ../frontend/lexer proc fillSymbolTable*(tokenizer: Lexer) = ## Initializes the Lexer's symbol - ## table with the builtin symbols - ## and keywords + ## table with builtin symbols and + ## keywords # 1-byte symbols tokenizer.symbols.addSymbol("{", LeftBrace) @@ -16,7 +16,6 @@ proc fillSymbolTable*(tokenizer: Lexer) = tokenizer.symbols.addSymbol(".", Dot) tokenizer.symbols.addSymbol(",", Comma) tokenizer.symbols.addSymbol(";", Semicolon) - # tokenizer.symbols.addSymbol("\n", Semicolon) # TODO: Broken # Keywords tokenizer.symbols.addKeyword("type", TokenType.Type) tokenizer.symbols.addKeyword("enum", Enum) @@ -46,10 +45,11 @@ proc fillSymbolTable*(tokenizer: Lexer) = tokenizer.symbols.addKeyword("yield", TokenType.Yield) tokenizer.symbols.addKeyword("return", TokenType.Return) tokenizer.symbols.addKeyword("object", Object) + tokenizer.symbols.addKeyword("export", Export) # These are more like expressions with a reserved # name that produce a value of a builtin type, # but we don't need to care about that until - # we're in the parsing/ compilation steps so + # we're in the parsing/compilation steps so # it's fine tokenizer.symbols.addKeyword("nan", NotANumber) tokenizer.symbols.addKeyword("inf", Infinity) @@ -59,5 +59,6 @@ proc fillSymbolTable*(tokenizer: Lexer) = tokenizer.symbols.addKeyword("ref", TokenType.Ref) tokenizer.symbols.addKeyword("ptr", TokenType.Ptr) for sym in [">", "<", "=", "~", "/", "+", "-", "_", "*", "?", "@", ":", "==", "!=", - ">=", "<=", "+=", "-=", "/=", "*=", "**=", "!", "%", "&", "|", "^"]: + ">=", "<=", "+=", "-=", "/=", "*=", "**=", "!", "%", "&", "|", "^", + ">>", "<<"]: tokenizer.symbols.addSymbol(sym, Symbol) \ No newline at end of file diff --git a/tests/std.pn b/tests/std.pn deleted file mode 100644 index cae8bb1..0000000 --- a/tests/std.pn +++ /dev/null @@ -1,223 +0,0 @@ -## The peon standard library - -# Builtin arithmetic operators for Peon -# Note: Most of these do nothing on their own. All they do -# is serve as placeholders for emitting specific VM -# instructions. They're implemented this way because: -# - They tie into the existing type system nicely -# - It makes the implementation easier and more flexible - - -operator `+`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "Add", pure] -} - - -operator `+`(a, b: float): float { - #pragma[magic: "AddFloat64", pure] -} - - -operator `+`(a, b: float32): float32 { - #pragma[magic: "AddFloat32", pure] -} - - -operator `-`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "Subtract", pure] -} - - -operator `-`*(a, b: float64): float64 { - #pragma[magic: "SubtractFloat64", pure] -} - - -operator `-`*(a, b: float32): float32 { - #pragma[magic: "SubtractFloat32", pure] -} - - -operator `-`*[T: int | int32 | int16 | int8](a: T): T { - #pragma[magic: "Negate", pure] -} - - -operator `-`*(a: float64): float64 { - #pragma[magic: "NegateFloat64", pure] -} - - -operator `-`*(a: float32): float32 { - #pragma[magic: "NegateFloat32", pure] -} - - -operator `-`*(a: inf): inf { - #pragma[magic: "NegInf", pure] -} - - -operator `*`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "Multiply", pure] -} - - -operator `*`*(a, b: float64): float64 { - #pragma[magic: "MultiplyFloat64", pure] -} - - -operator `*`*(a, b: float32): float32 { - #pragma[magic: "MultiplyFloat32", pure] -} - - -operator `/`*[T: int | int32 | int16 | int8](a, b: T): T { - #pragma[magic: "SignedDivide", pure] -} - - -operator `/`*[T: uint64 | uint32 | uint16 | uint8](a, b: T): T { - #pragma[magic: "Divide", pure] -} - - -operator `/`*(a, b: float64): float64 { - #pragma[magic: "DivFloat64", pure] -} - - -operator `/`*(a, b: float32): float32 { - #pragma[magic: "DivFloat32", pure] -} - - -operator `**`*[T: int | int32 | int16 | int8](a, b: T): T { - #pragma[magic: "SignedPow", pure] -} - - -operator `**`*[T: uint64 | uint32 | uint16 | uint8](a, b: T): T { - #pragma[magic: "Pow", pure] -} - - -operator `%`*(a, b: int64): int64 { - #pragma[magic: "SignedMod", pure] -} - -# Comparison operators - -operator `>`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "GreaterThan", pure] -} - - -operator `<`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "LessThan", pure] -} - - -operator `==`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "Equal", pure] - -} - -operator `!=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "NotEqual", pure] -} - - -operator `>=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "GreaterOrEqual", pure] -} - - -operator `<=`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8 | float | float32](a, b: T): bool { - #pragma[magic: "LessOrEqual", pure] -} - - -operator `and`*(a, b: bool): bool { - #pragma[magic: "LogicalAnd", pure] -} - - -operator `or`*(a, b: bool): bool { - #pragma[magic: "LogicalOr", pure] -} - - -operator `not`*(a: bool): bool { - #pragma[magic: "LogicalNot", pure] -} - - -operator `&`*(a, b: int): bool { - #pragma[magic: "And", pure] -} - - -operator `|`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): int { - #pragma[magic: "Or", pure] -} - - -operator `~`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a: T): T { - #pragma[magic: "Not", pure] -} - - -operator `>>`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "RShift", pure] -} - - -operator `<<`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "LShift", pure] -} - - -operator `^`*[T: int | uint64 | int32 | uint32 | int16 | uint16 | int8 | uint8](a, b: T): T { - #pragma[magic: "Xor", pure] -} - - -# Assignment operators - -operator `=`*[T: all](a: var T, b: T) { # TODO: This is just a placeholder right now - #pragma[magic: "GenericAssign"] -} - - -# Some useful builtins - -fn clock*: float { - #pragma[magic: "SysClock64", pure] -} - - -fn print*(x: int) { - #pragma[magic: "PrintInt64"] -} - - -fn print*(x: uint64) { - #pragma[magic: "PrintUInt64"] -} - - -fn print*(x: float) { - #pragma[magic: "PrintFloat64"] -} - - -fn print*(x: string) { - #pragma[magic: "PrintString"] -} - - -fn print*(x: bool) { - #pragma[magic: "PrintBool"] -} \ No newline at end of file