More lexer bug fixes and tests. Minor changes to error reporting. Added intrinsic aliases

src/frontend/compiler/typechecker.nim

@@ -294,25 +294,25 @@ proc toIntrinsic(name: string): Type =
             return Type(kind: Any, intrinsic: true)
         of "auto":
             return Type(kind: Auto, intrinsic: true)
-        of "int64":
+        of "int64", "i64":
             return Type(kind: Integer, size: LongLong, signed: true, intrinsic: true)
-        of "uint64":
+        of "uint64", "u64":
             return Type(kind: Integer, size: LongLong, signed: false, intrinsic: true)
-        of "int32":
+        of "int32", "i32":
             return Type(kind: Integer, size: Long, signed: true, intrinsic: true)
-        of "uint32":
+        of "uint32", "u32":
             return Type(kind: Integer, size: Long, signed: false, intrinsic: true)
-        of "int16":
+        of "int16", "i16":
             return Type(kind: Integer, size: Short, signed: true, intrinsic: true)
-        of "uint16":
+        of "uint16", "u16":
             return Type(kind: Integer, size: Short, signed: false, intrinsic: true)
-        of "int8":
+        of "int8", "i8":
             return Type(kind: Integer, size: Tiny, signed: true, intrinsic: true)
-        of "uint8":
+        of "uint8", "u8":
             return Type(kind: Integer, size: Tiny, signed: false, intrinsic: true)
-        of "float", "float64":
+        of "float", "float64", "f64":
             return Type(kind: Float, width: Full, intrinsic: true)
-        of "float32":
+        of "float32", "f32":
             return Type(kind: Float, width: Half, intrinsic: true)
         of "byte":
             return Type(kind: Byte, intrinsic: true)
@@ -352,14 +352,18 @@ proc infer(self: TypeChecker, node: LiteralExpr): TypedExpr =
             let size = node.token.lexeme.split("'")
             if size.len() == 1:
                 return newTypedExpr(node, "int64".toIntrinsic())
-            result = newTypedExpr(node, size[1].toIntrinsic())
-            if result.isNil():
-                self.error(&"invalid type specifier '{size[1]}' for int", node)
+            let typ = size[1].toIntrinsic()
+            if typ.isNil() or typ.kind != TypeKind.Integer:
+                self.error(&"invalid type specifier '{size[1]}' for integer", node)
+            result = newTypedExpr(node, typ)
         of floatExpr:
             let size = node.token.lexeme.split("'")
             if size.len() == 1:
                 return newTypedExpr(node, "float".toIntrinsic())
-            result = newTypedExpr(node, size[1].toIntrinsic())
+            let typ = size[1].toIntrinsic()
+            if typ.isNil() or typ.kind != TypeKind.Float:
+                self.error(&"invalid type specifier '{size[1]}' for float", node)
+            result = newTypedExpr(node, typ)
             if result.isNil():
                 self.error(&"invalid type specifier '{size[1]}' for float", node)
         else:

src/frontend/parsing/lexer.nim

@@ -454,7 +454,7 @@ proc parseString(self: Lexer, delimiter: string, mode: StringParseMode = Default
 
 proc parseBinary(self: Lexer) =
     ## Parses binary numbers
-    while self.peek().isDigit():
+    while self.peek().isDigit() and not self.done():
         if not self.check(["0", "1"]):
             self.error(&"invalid digit '{self.peek()}' in binary literal")
         discard self.step()
@@ -462,7 +462,7 @@ proc parseBinary(self: Lexer) =
 
 proc parseOctal(self: Lexer) =
     ## Parses octal numbers
-    while self.peek().isDigit():
+    while self.peek().isDigit() and not self.done():
         if self.peek() notin "0".."7":
             self.error(&"invalid digit '{self.peek()}' in octal literal")
         discard self.step()
@@ -470,7 +470,7 @@ proc parseOctal(self: Lexer) =
 
 proc parseHex(self: Lexer) =
     ## Parses hexadecimal numbers
-    while self.peek().isAlphaNumeric():
+    while self.peek().isAlphaNumeric() and not self.done():
         if not self.peek().isDigit() and self.peek().toLowerAscii() notin "a".."f":
             self.error(&"invalid hexadecimal literal")
         discard self.step()
@@ -516,7 +516,7 @@ proc parseNumber(self: Lexer) =
             elif self.check("."):
                 # TODO: Is there a better way?
                 discard self.step()
-                if not isDigit(self.peek()):
+                if not isDigit(self.peek()) or self.done():
                     self.error("invalid float number literal")
                 kind = Float
                 while isDigit(self.peek()) and not self.done():

src/util/fmterr.nim

@@ -39,8 +39,11 @@ proc formatError*(outFile = stderr, file, line: string, lineNo: int, pos: tuple[
         # Print the line where the error occurred and underline the exact node that caused
         # the error. Might be inaccurate, but definitely better than nothing
         outFile.styledWrite(fgRed, styleBright, "Source line: ", resetStyle, fgDefault, line[0..<pos.start])
-        outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..<pos.stop])
-        outFile.styledWriteLine(fgDefault, line[pos.stop..^1])
+        outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..pos.stop])
+        if pos.stop + 1 <= line.high():
+            outFile.styledWriteLine(fgDefault, line[pos.stop + 1..^1])
+        else:
+            outFile.styledWriteLine(fgDefault, "")
 
 
 proc print*(exc: TypeCheckError, includeSource = true) =

src/util/testing.nim

@@ -123,7 +123,7 @@ proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
         for (token, kind) in zip(tokens, test.tokens):
             if token.kind != kind:
                 test.status = Failed
-                test.reason = &"Token type mismatch at #{i}: expected {token.kind}, got {kind}"
+                test.reason = &"Token type mismatch at #{i}: expected {kind}, got {token.kind}"
                 return
             inc(i)
     except LexingError:

tests/tokenize.nim

@@ -21,6 +21,15 @@ when isMainModule:
             testTokenizeSucceeds("stroppedSingleUnicode", "`🌎` `😂` `👩‍👩‍👦‍👦`", @[TokenType.Identifier, TokenType.Identifier, TokenType.Identifier, TokenType.EndOfFile]),
             testTokenizeSucceeds("stroppedMultiUnicode", "`🌎🌎` `😂😂` `👩‍👩‍👦‍👦👩‍👩‍👦‍👦`", @[TokenType.Identifier, TokenType.Identifier, TokenType.Identifier, TokenType.EndOfFile]),
             testTokenizeSucceeds("stringWithEscapes", """ "\n\t\r\e\f" """, @[TokenType.String, TokenType.EndOfFile]),
+            testTokenizeSucceeds("allIntegers", "1 0x1 0o1 0b1", @[TokenType.Integer, TokenType.Hex, TokenType.Octal, TokenType.Binary, TokenType.EndOfFile]),
+            testTokenizeSucceeds("sizedNumbers", "1'u8 0x1'i8 0o1'i64 0b1'u32 2.0'f32 1e5'f64 1E5'f32 1.5e4'f64 1.5E4'f32", 
+                                 @[TokenType.Integer, TokenType.Hex, TokenType.Octal, TokenType.Binary, 
+                                   TokenType.Float, TokenType.Float, TokenType.Float, TokenType.Float, TokenType.Float, 
+                                   TokenType.EndOfFile]),
+            testTokenizeSucceeds("allFloats", "1.0 1e5 1E5 1.5e4 1.5E4", @[TokenType.Float, TokenType.Float, TokenType.Float, 
+                                                                           TokenType.Float, TokenType.Float, TokenType.EndOfFile]),
+            testTokenizeFails("invalidFloatEndsWithDot", "2.", "invalid float number literal",  line=1, location=(0, 1)),
+            testTokenizeFails("invalidFloatSpuriousChats", "2.f", "invalid float number literal",  line=1, location=(0, 1)),
             testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", line=1, location=(0, 0)),
             testTokenizeFails("emptyChar", "''", "character literal cannot be of length zero",  line=1, location=(0, 1)),
             testTokenizeFails("charTooLong", "'ab'", "invalid character literal (length must be one!)",  line=1, location=(0, 3)),