Add test cases for edge cases

dilanSachi · Jan 23, 2025 · 364080b · 364080b
1 parent 6dee6c4
commit 364080b
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 4 deletions.
diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
@@ -1,4 +1,4 @@
-name: PR Build
+name: Build Status
 
 on: [pull_request, push]
 

diff --git a/src/main/java/fi/helsinki/compiler/tokenizer/Tokenizer.java b/src/main/java/fi/helsinki/compiler/tokenizer/Tokenizer.java
@@ -11,7 +11,7 @@ public class Tokenizer {
 
     private static final Pattern patternWhitespace = Pattern.compile("^ +");
     private static final Pattern patternTab = Pattern.compile("^\t+");
-    private static final Pattern patternComment = Pattern.compile("^((\\/\\/|##).*?\\n)");
+    private static final Pattern patternComment = Pattern.compile("^((\\/\\/|##).*?(\\n|$))");
     private static final Pattern patternIdentifier = Pattern.compile("^(((?!var(?![a-zA-Z]))(?<![a-zA-Z])[a-z|A-Z|_]+[a-z|A-Z|_|0-9]*)|((var)[a-z|A-Z|_|0-9]+))");
     private static final Pattern patternOperator = Pattern.compile("^(==| or | and |=|!=|<=|>|>=|<|\\+|-|\\*|\\/|%)");
     private static final Pattern patternKeyword = Pattern.compile("^(if|while|function|var|do|then|else)( |\n|$)");
@@ -135,7 +135,8 @@ public List<Token> tokenize(String sourceCode, String filename) throws TokenizeE
             if (matcher.hitEnd()) {
                 return tokens;
             }
-            throw new TokenizeException("Found invalid token");
+            throw new TokenizeException("Found invalid token: '" + sourceCode.substring(0, 1)
+                    + "' at line: " + line + " and column: " + column);
         }
     }
 }
diff --git a/src/test/java/fi/helsinki/compiler/tokenizer/TokenizerTests.java b/src/test/java/fi/helsinki/compiler/tokenizer/TokenizerTests.java
@@ -1,5 +1,6 @@
 package fi.helsinki.compiler.tokenizer;
 
+import fi.helsinki.compiler.exceptions.TokenizeException;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
@@ -21,6 +22,52 @@ public void testBasicKeywords() {
         assertTokens(tokens, expectedTokens);
     }
 
+    @Test
+    public void testSimpleStatement() {
+        Tokenizer testTokenizer = new Tokenizer();
+        List<Token> tokens = testTokenizer.tokenize("if a <= bee then print_int(123)", "Testfile.dl");
+        Token[] expectedTokens = new Token[]{
+                new Token("if", TokenType.KEYWORD, new TokenLocation("Testfile.dl", 0, 0)),
+                new Token("a", TokenType.IDENTIFIER, new TokenLocation("Testfile.dl", 0, 3)),
+                new Token("<=", TokenType.OPERATOR, new TokenLocation("Testfile.dl", 0, 5)),
+                new Token("bee", TokenType.IDENTIFIER, new TokenLocation("Testfile.dl", 0, 8)),
+                new Token("then", TokenType.KEYWORD, new TokenLocation("Testfile.dl", 0, 12)),
+                new Token("print_int", TokenType.IDENTIFIER, new TokenLocation("Testfile.dl", 0, 17)),
+                new Token("(", TokenType.PUNCTUATION, new TokenLocation("Testfile.dl", 0, 26)),
+                new Token("123", TokenType.INTEGER_LITERAL, new TokenLocation("Testfile.dl", 0, 27)),
+                new Token(")", TokenType.PUNCTUATION, new TokenLocation("Testfile.dl", 0, 30))};
+        assertTokens(tokens, expectedTokens);
+    }
+
+    @Test
+    public void testStringLiteralWithDifferentCharacters() {
+        Tokenizer testTokenizer = new Tokenizer();
+        List<Token> tokens = testTokenizer.tokenize(
+                "var s : String = \"This is a str|ing 093!@#$%^&*()-_+=][}{`~.>,</?\";", "Testfile.dl");
+        Token[] expectedTokens = new Token[]{
+                new Token("var", TokenType.KEYWORD, new TokenLocation("Testfile.dl", 0, 0)),
+                new Token("s", TokenType.IDENTIFIER, new TokenLocation("Testfile.dl", 0, 4)),
+                new Token(":", TokenType.PUNCTUATION, new TokenLocation("Testfile.dl", 0, 6)),
+                new Token("String", TokenType.IDENTIFIER, new TokenLocation("Testfile.dl", 0, 8)),
+                new Token("=", TokenType.OPERATOR, new TokenLocation("Testfile.dl", 0, 15)),
+                new Token("\"This is a str|ing 093!@#$%^&*()-_+=][}{`~.>,</?\"", TokenType.STRING_LITERAL,
+                        new TokenLocation("Testfile.dl", 0, 17)),
+                new Token(";", TokenType.PUNCTUATION, new TokenLocation("Testfile.dl", 0, 66))};
+        assertTokens(tokens, expectedTokens);
+    }
+
+    @Test
+    public void testInvalidSimpleStatement() {
+        Tokenizer testTokenizer = new Tokenizer();
+        try {
+        testTokenizer.tokenize("if a <= bee @then print_int(123)", "Testfile.dl");
+        } catch (TokenizeException e) {
+            assertEquals(e.getMessage(), "Found invalid token: '@' at line: 0 and column: 12");
+            return;
+        }
+        fail("Expected a TokenizeException");
+    }
+
     @Test
     public void testSimpleProgram() {
         String sourceCode = "var n: Int = read_int();\n" +
@@ -33,7 +80,8 @@ public void testSimpleProgram() {
                 "        n = 3*n + 1;\n" +
                 "    }\n" +
                 "    print_int(n);\n" +
-                "}";
+                "}" +
+                "## End comment";
         Tokenizer testTokenizer = new Tokenizer();
         List<Token> tokens = testTokenizer.tokenize(sourceCode, "Testfile.dl");
         Token[] expectedTokens = new Token[]{