Started the nim lexer

JAPL/meta/tokenobject.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from .tokentype import TokenType
 
+
 @dataclass
 class Token(object):
     """The representation of a JAPL token"""

JAPL/meta/tokentype.py

@@ -1,5 +1,6 @@
 from enum import Enum, auto
 
+
 class TokenType(Enum):
     """
     An enumeration for all JAPL types

nimjapl/lexer.nim

@@ -0,0 +1,85 @@
+import tables
+import meta/tokentype
+import meta/tokenobject
+import meta/exceptions
+import strformat
+import system
+
+
+const TOKENS = to_table({
+              "(": TokenType.LP, ")": TokenType.RP,
+              "{": TokenType.LB, "}": TokenType.RB,
+              ".": TokenType.DOT, ",": TokenType.COMMA,
+              "-": TokenType.MINUS, "+": TokenType.PLUS,
+              ";": TokenType.SEMICOLON, "*": TokenType.STAR,
+              ">": TokenType.GT, "<": TokenType.LT,
+              "=": TokenType.EQ, "!": TokenType.NEG,
+              "/": TokenType.SLASH, "%": TokenType.MOD})
+
+const RESERVED = to_table({
+                "or": TokenType.OR, "and": TokenType.AND,
+                "class": TokenType.CLASS, "fun": TokenType.FUN,
+                "if": TokenType.IF, "else": TokenType.ELSE,
+                "for": TokenType.FOR, "while": TokenType.WHILE,
+                "var": TokenType.VAR, "nil": TokenType.NIL,
+                "true": TokenType.TRUE, "false": TokenType.FALSE,
+                "return": TokenType.RETURN,
+                "this": TokenType.THIS, "super": TokenType.SUPER,
+                "del": TokenType.DEL, "break": TokenType.BREAK})
+
+
+type Lexer* = object
+  source: string
+  tokens: seq[Token]
+  line: int
+  start: int
+  current: int
+
+
+func initLexer*(source: string): Lexer =
+  result = Lexer(source: source, tokens: @[], line: 1, start: 0, current: 0)
+
+
+proc step*(self: var Lexer): char =
+  result = self.source[self.current]
+  self.current = self.current + 1
+
+
+proc done*(self: Lexer): bool =
+    result = self.current >= self.source.len
+
+
+proc peek*(self: Lexer): string =
+    if self.done():
+        result = ""
+    else:
+        result = &"{self.source[self.current]}"
+
+
+proc peekNext*(self: Lexer): string =
+    if self.current + 1 >= self.source.len:
+        result = ""
+    else:
+        result = &"{self.source[self.current + 1]}"
+
+
+proc createToken*[T](self: var Lexer, tokenType: TokenType, literal: T): Token =
+    result = Token(kind: tokenType,
+                   lexeme: self.source[self.start..self.current],
+                   literal: literal
+                   )
+
+
+proc parseString*(self: var Lexer, delimiter: string) =
+    while self.peek() != delimiter and not self.done():
+        if self.peek() == "\n":
+            self.line = self.line + 1
+        discard self.step()
+    if self.done():
+        raise newException(ParseError, &"Unterminated string literal at {self.line}")
+    discard self.step()
+    let value = self.source[self.start + 1..self.current - 1]  # Get the value between quotes
+    let token = self.createToken[string](STR, value)
+
+var lexer = initLexer("print")
+echo lexer.createToken[string](STR, "ehlo", STR)

nimjapl/meta/classtype.nim

@@ -0,0 +1,5 @@
+# Class types enumeration
+
+type
+    ClassType* = enum
+        CLASS, NONE

nimjapl/meta/exceptions.nim

@@ -0,0 +1,5 @@
+# Errors for JAPL
+
+type
+    ParseError* = object of CatchableError
+    JAPLError* = object of CatchableError

nimjapl/meta/expression.nim

@@ -0,0 +1,5 @@
+import tokentype
+import tokenobject
+
+
+        

nimjapl/meta/tokenobject.nim

@@ -0,0 +1,9 @@
+import tokentype
+# Token object
+
+type
+  Token*[T] = ref object of RootOBJ
+    kind*: TokenType
+    lexeme*: string
+    literal*: T
+    line*: int

nimjapl/meta/tokentype.nim

@@ -0,0 +1,14 @@
+# Token types enumeration
+
+type
+  TokenType* = enum
+    PLUS, MINUS, SLASH, STAR,
+    NEG, NE, EQ, DEQ, LT, GE,
+    LE, MOD, POW, GT, LP, RP,
+    LB, RB, COMMA, DOT, ID,
+    INT, FLOAT, BOOL, STR,
+    SEMICOLON, AND, CLASS,
+    ELSE, FOR, FUN, FALSE,
+    IF, NIL, RETURN, SUPER,
+    THIS, OR, TRUE, VAR,
+    WHILE, DEL, BREAK, EOF