Merge branch 'main' into docker-more-assignments

README.md

@@ -12,6 +12,7 @@
   * [Code coverage](#code-coverage)
   * [Mutation coverage](#mutation-coverage)
   * [Meta tests](#meta-tests)
+  * [Penalty meta tests](#penalty-meta-tests)
   * [Code checks](#code-checks)
   * [Penalty code checks](#penalty-code-checks)
   * [Success message](#success-message)
@@ -285,6 +286,39 @@ Meta test: returns wrong elements (weight: 1) FAILED
 Meta test: compares only elements in the same index (weight: 3) FAILED
 Meta test: background service does not work (weight: 2) PASSED
 ```
+#### Penalty meta tests
+
+Penalty meta tests work in a similar way to regular meta tests. However, they are not considered a grading component. Instead, if a penalty meta test passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade.
+
+Penalty meta tests must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty meta tests where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead.
+
+Penalty meta tests can be defined as follows:
+
+```java
+@Override
+public List<MetaTest> penaltyMetaTests() {
+        return List.of(
+            MetaTest.insertAt(2, "returns empty list when list1 is null", 29,
+                "if (list1 == null) return result;"
+            ),
+            MetaTest.withStringReplacement("returns wrong elements",
+                "if (hashSet.contains(e))",
+                "if (!hashSet.contains(e))"
+            ),
+            MetaTest.withLineReplacement(3, "compares only elements in the same index", 29, 41,
+                    """
+                    for (int i = 0; i < list1.size() && i < list2.size(); i++) {
+                        if (list1.get(i).equals(list2.get(i))) {
+                            result.add(list1.get(i));
+                        }
+                    }
+                    """
+            )
+        );
+}
+```
+
+If this method is not overridden, penalty meta tests are disabled.
 
 #### Code checks
 

andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java

@@ -45,6 +45,10 @@ public List<MetaTest> metaTests() {
         return Collections.emptyList();
     }
 
+    public List<MetaTest> penaltyMetaTests() {
+        return Collections.emptyList();
+    }
+
     public List<String> listOfMutants() {
         return DEFAULTS;
     }

andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java

@@ -16,13 +16,17 @@ public class SecureExamRunConfiguration extends RunConfiguration {
     private final List<String> listOfMutants;
     private final int numberOfMutationsToConsider;
     private final ExternalProcess externalProcess;
+    private final boolean skipJacoco;
+    private final boolean skipPitest;
 
     public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) {
         this.classesUnderTest = runConfigurationToClone.classesUnderTest();
         this.listOfMutants = runConfigurationToClone.listOfMutants();
         this.numberOfMutationsToConsider = runConfigurationToClone.numberOfMutationsToConsider();
         this.externalProcess = runConfigurationToClone.externalProcess();
         this.successMessage = runConfigurationToClone.successMessage();
+        this.skipJacoco = runConfigurationToClone.skipJacoco();
+        this.skipPitest = runConfigurationToClone.skipPitest();
     }
 
     public Mode mode() {
@@ -60,5 +64,13 @@ public String successMessage() {
         return successMessage;
     }
 
+    @Override
+    public boolean skipJacoco() {
+        return this.skipJacoco;
+    }
 
+    @Override
+    public boolean skipPitest() {
+        return this.skipPitest;
+    }
 }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java

@@ -129,6 +129,7 @@ public static List<ExecutionStep> fullMode() {
                 new RunPenaltyCodeChecksStep(),
                 new RunCodeChecksStep(),
                 new KillExternalProcessStep(),
+                new RunPenaltyMetaTestsStep(),
                 new RunMetaTestsStep()
         );
     }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java

@@ -0,0 +1,62 @@
+package nl.tudelft.cse1110.andy.execution.step;
+
+import nl.tudelft.cse1110.andy.config.DirectoryConfiguration;
+import nl.tudelft.cse1110.andy.config.MetaTest;
+import nl.tudelft.cse1110.andy.config.RunConfiguration;
+import nl.tudelft.cse1110.andy.execution.Context.Context;
+import nl.tudelft.cse1110.andy.execution.ExecutionStep;
+import nl.tudelft.cse1110.andy.result.MetaTestResult;
+import nl.tudelft.cse1110.andy.result.ResultBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RunPenaltyMetaTestsStep implements ExecutionStep {
+
+    @Override
+    public void execute(Context ctx, ResultBuilder result) {
+        DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration();
+        RunConfiguration runCfg = ctx.getRunConfiguration();
+
+        int score = 0;
+        int totalWeight = 0;
+
+        ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+        try {
+
+            List<MetaTest> metaTests = runCfg.penaltyMetaTests();
+            List<MetaTestResult> metaTestResults = new ArrayList<>();
+
+            for (MetaTest metaTest : metaTests) {
+
+                boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg);
+
+                if (passesTheMetaTest) {
+                    score += metaTest.getWeight();
+                }
+
+                metaTestResults.add(new MetaTestResult(metaTest.getName(), metaTest.getWeight(), passesTheMetaTest));
+
+                totalWeight += metaTest.getWeight();
+            }
+
+            result.logPenaltyMetaTests(score, totalWeight, metaTestResults);
+        } catch (Exception ex) {
+            result.genericFailure(this, ex);
+        } finally {
+            /* restore the class loader to the one before meta tests */
+            Thread.currentThread().setContextClassLoader(currentClassLoader);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof RunPenaltyMetaTestsStep;
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+}

andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java

@@ -86,15 +86,16 @@ public GradeValues setPenalty(int penalty) {
         return this;
     }
 
-    public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) {
+    public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) {
         GradeValues grades = new GradeValues();
         grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches());
         grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks());
         grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants());
         grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests());
 
         // penalty is equal to the sum of the weights of all failed penalty code checks
-        grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum());
+        grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()
+                + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum());
 
         return grades;
     }

andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java

@@ -13,21 +13,23 @@ public class Result {
     private final CodeChecksResult penaltyCodeChecks;
     private final CoverageResult coverage;
     private final MetaTestsResult metaTests;
+    private final MetaTestsResult penaltyMetaTests;
     private final int penalty;
     private final int finalGrade;
     private final GenericFailure genericFailure;
     private final double timeInSeconds;
     private final GradeWeight weights;
     private final String successMessage;
 
-    public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
+    public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
         this.compilation = compilation;
         this.tests = tests;
         this.mutationTesting = mutationTesting;
         this.codeChecks = codeChecks;
         this.penaltyCodeChecks = penaltyCodeChecks;
         this.coverage = coverage;
         this.metaTests = metaTests;
+        this.penaltyMetaTests = penaltyMetaTests;
         this.penalty = penalty;
         this.finalGrade = finalGrade;
         this.genericFailure = genericFailure;
@@ -40,7 +42,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest
     }
 
     public Result(CompilationResult compilation, double timeInSeconds) {
-        this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
+        this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
     }
 
     public CompilationResult getCompilation() {
@@ -70,6 +72,9 @@ public CoverageResult getCoverage() {
     public MetaTestsResult getMetaTests() {
         return metaTests;
     }
+    public MetaTestsResult getPenaltyMetaTests() {
+        return penaltyMetaTests;
+    }
 
     public int getPenalty() {
         return penalty;

andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java

@@ -49,6 +49,7 @@ public class ResultBuilder {
     private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty();
     private CoverageResult coverageResults = CoverageResult.empty();
     private MetaTestsResult metaTestResults = MetaTestsResult.empty();
+    private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty();
 
     public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) {
         this.ctx = ctx;
@@ -244,6 +245,10 @@ public void logMetaTests(int score, int totalTests, List<MetaTestResult> metaTes
         this.metaTestResults.addResults(score, totalTests, metaTestResults);
     }
 
+    public void logPenaltyMetaTests(int score, int totalTests, List<MetaTestResult> metaTestResults) {
+        this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults);
+    }
+
     /*
      * Generic failures
      */
@@ -271,7 +276,7 @@ public Result build() {
         if(!compilation.successful()) {
             return new Result(compilation, timeInSeconds);
         } else {
-            GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults);
+            GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults);
             GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights());
             String successMessage = ctx.getRunConfiguration().successMessage();
 
@@ -287,6 +292,7 @@ public Result build() {
                     penaltyCodeCheckResults,
                     coverageResults,
                     metaTestResults,
+                    penaltyMetaTestResults,
                     penalty,
                     finalGrade,
                     genericFailureObject,

andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java

@@ -65,7 +65,7 @@ private void writeStdOutFile(Context ctx, Result result) {
             printCoverageResults(result.getCoverage());
             printMutationTestingResults(result.getMutationTesting());
             printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks());
-            printMetaTestResults(ctx, result.getMetaTests());
+            printMetaTestResults(ctx, result.getMetaTests(), result.getPenaltyMetaTests());
             printFinalGrade(ctx, result);
             printModeAndTimeToRun(ctx, result.getTimeInSeconds());
         }
@@ -244,8 +244,8 @@ private void printModeAndTimeToRun(Context ctx, double timeInSeconds) {
             l(String.format("\nAndy took %.1f seconds to assess your solution.", timeInSeconds));
     }
 
-    private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
-        if (!metaTests.wasExecuted() || metaTests.hasNoMetaTests())
+    private void printMetaTestResults(Context ctx, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests) {
+        if ((!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) && (!penaltyMetaTests.wasExecuted() || penaltyMetaTests.hasNoMetaTests()))
             return;
 
         boolean allHints = modeActionSelector(ctx).shouldShowFullHints();
@@ -255,7 +255,8 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
             return;
 
         l("\n--- Meta tests");
-        l(String.format("%d/%d passed", metaTests.getPassedMetaTests(), metaTests.getTotalTests()));
+        l(String.format("%d/%d passed", metaTests.getPassedMetaTests() + penaltyMetaTests.getPassedMetaTests(),
+                metaTests.getTotalTests() + penaltyMetaTests.getTotalTests()));
 
         if (allHints) {
             for (MetaTestResult metaTestResult : metaTests.getMetaTestResults()) {
@@ -265,6 +266,13 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
                     l(String.format("Meta test: %s (weight: %d) FAILED", metaTestResult.getName(), metaTestResult.getWeight()));
                 }
             }
+            for (MetaTestResult penaltyResult : penaltyMetaTests.getMetaTestResults()) {
+                if (penaltyResult.succeeded()) {
+                    l(String.format("Meta test: %s (penalty: %d) PASSED", penaltyResult.getName(), penaltyResult.getWeight()));
+                } else {
+                    l(String.format("Meta test: %s (penalty: %d) FAILED", penaltyResult.getName(), penaltyResult.getWeight()));
+                }
+            }
         }
 
     }

andy/src/test/java/integration/LibraryMetaTestsTest.java

@@ -88,4 +88,46 @@ void metaTestInternalFailure() {
                 .isEqualTo(RunMetaTestsStep.class.getSimpleName());
     }
 
+    @Test
+    void allPenaltyMetaTestsPassing() {
+        Result result = run("NumberUtilsAddLibrary",
+                "NumberUtilsAddOfficialSolution", "NumberUtilsAddWithPenaltyMetaTestsConfiguration");
+
+        assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3);
+        assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(3);
+        assertThat(result.getPenaltyMetaTests())
+                .has(passedMetaTest("DoesNotApplyCarryAtAll"))
+                .has(passedMetaTest("DoesNotApplyLastCarry"))
+                .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"));
+    }
+
+    @Test
+    void somePenaltyMetaTestFailing() {
+        Result result = run("NumberUtilsAddLibrary",
+                "NumberUtilsAddAllTestsPass", "NumberUtilsAddWithPenaltyMetaTestsConfiguration");
+
+        assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3);
+        assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(1);
+
+        assertThat(result.getPenaltyMetaTests())
+                .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"))
+                .has(failedMetaTest("DoesNotApplyCarryAtAll"))
+                .has(failedMetaTest("DoesNotApplyLastCarry"));
+    }
+
+    @Test
+    void somePenaltyMetaTestFailingWithWeights() {
+        Result result = run("NumberUtilsAddLibrary",
+                "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests");
+
+        assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(7);
+        assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(2);
+
+        assertThat(result.getPenaltyMetaTests())
+                .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"))
+                .has(failedMetaTest("AppliesMultipleCarriesWrongly"))
+                .has(failedMetaTest("DoesNotApplyCarryAtAll"))
+                .has(failedMetaTest("DoesNotApplyLastCarry"));
+    }
+
 }