From fec659f4f3cea8ff7e25804c9c6058a6debb379c Mon Sep 17 00:00:00 2001 From: Francesco Hamar Date: Wed, 1 May 2024 18:32:56 +0200 Subject: [PATCH 1/2] Added Meta test that subtracts points #209 (#266) * remove upstream * introducing penalty metachecks * Added penalty meta test exectution step * added penalty meta tests to final output writer * added logging for penalty metatest * fixed equals in exec step * suitable Result constructor in testutils * Penalty MetaTests integration Tests _1 * Update README with penalty meta tests description * More tests on PenaltyMetaTests * Increase consistency in writing the result * improve conditions about displaying metatests and codechecks results * Standard result writer tests * penalty meta tests in Weblab result writer * Remove unnecessary import * Fix logic bug * Revert "Increase consistency in writing the result" This reverts commit bc24b9caa04903051f87adcde422a7a6837b456e. * Revert output format changes --------- Co-authored-by: puctom Co-authored-by: Martin Mladenov --- README.md | 34 ++++++ .../cse1110/andy/config/RunConfiguration.java | 4 + .../execution/mode/ModeActionSelector.java | 1 + .../step/RunPenaltyMetaTestsStep.java | 62 +++++++++++ .../cse1110/andy/grade/GradeValues.java | 5 +- .../tudelft/cse1110/andy/result/Result.java | 9 +- .../cse1110/andy/result/ResultBuilder.java | 8 +- .../writer/standard/StandardResultWriter.java | 16 ++- .../integration/LibraryMetaTestsTest.java | 42 +++++++ .../java/integration/ModesAndActionsTest.java | 56 ++++++++++ .../java/testutils/ResultTestDataBuilder.java | 12 +- .../StandardResultTestAssertions.java | 10 +- .../standard/StandardResultWriterTest.java | 68 +++++++++++- ...urationWithWeightWithPenaltyMetaTests.java | 100 +++++++++++++++++ ...sAddWithPenaltyMetaTestsConfiguration.java | 105 ++++++++++++++++++ ...oftWhereConfigPenaltyMetaTestsFailing.java | 90 +++++++++++++++ ...ftWhereConfigPenaltyMetaTestsFailing2.java | 102 +++++++++++++++++ .../writer/weblab/WebLabResultWriter.java | 1 + 18 files changed, 710 insertions(+), 15 deletions(-) create mode 100644 andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java diff --git a/README.md b/README.md index ed87b9f5e..cb6f297c5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ * [Code coverage](#code-coverage) * [Mutation coverage](#mutation-coverage) * [Meta tests](#meta-tests) + * [Penalty meta tests](#penalty-meta-tests) * [Code checks](#code-checks) * [Penalty code checks](#penalty-code-checks) * [Success message](#success-message) @@ -285,6 +286,39 @@ Meta test: returns wrong elements (weight: 1) FAILED Meta test: compares only elements in the same index (weight: 3) FAILED Meta test: background service does not work (weight: 2) PASSED ``` +#### Penalty meta tests + +Penalty meta tests work in a similar way to regular meta tests. However, they are not considered a grading component. Instead, if a penalty meta test passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade. + +Penalty meta tests must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty meta tests where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead. + +Penalty meta tests can be defined as follows: + +```java +@Override +public List penaltyMetaTests() { + return List.of( + MetaTest.insertAt(2, "returns empty list when list1 is null", 29, + "if (list1 == null) return result;" + ), + MetaTest.withStringReplacement("returns wrong elements", + "if (hashSet.contains(e))", + "if (!hashSet.contains(e))" + ), + MetaTest.withLineReplacement(3, "compares only elements in the same index", 29, 41, + """ + for (int i = 0; i < list1.size() && i < list2.size(); i++) { + if (list1.get(i).equals(list2.get(i))) { + result.add(list1.get(i)); + } + } + """ + ) + ); +} +``` + +If this method is not overridden, penalty meta tests are disabled. #### Code checks diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java index 25af1fe04..bfc67e918 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java @@ -45,6 +45,10 @@ public List metaTests() { return Collections.emptyList(); } + public List penaltyMetaTests() { + return Collections.emptyList(); + } + public List listOfMutants() { return DEFAULTS; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java index c2532c474..1fe2877d5 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java @@ -129,6 +129,7 @@ public static List fullMode() { new RunPenaltyCodeChecksStep(), new RunCodeChecksStep(), new KillExternalProcessStep(), + new RunPenaltyMetaTestsStep(), new RunMetaTestsStep() ); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java new file mode 100644 index 000000000..6041fe20b --- /dev/null +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java @@ -0,0 +1,62 @@ +package nl.tudelft.cse1110.andy.execution.step; + +import nl.tudelft.cse1110.andy.config.DirectoryConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.execution.ExecutionStep; +import nl.tudelft.cse1110.andy.result.MetaTestResult; +import nl.tudelft.cse1110.andy.result.ResultBuilder; + +import java.util.ArrayList; +import java.util.List; + +public class RunPenaltyMetaTestsStep implements ExecutionStep { + + @Override + public void execute(Context ctx, ResultBuilder result) { + DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); + RunConfiguration runCfg = ctx.getRunConfiguration(); + + int score = 0; + int totalWeight = 0; + + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + + try { + + List metaTests = runCfg.penaltyMetaTests(); + List metaTestResults = new ArrayList<>(); + + for (MetaTest metaTest : metaTests) { + + boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg); + + if (passesTheMetaTest) { + score += metaTest.getWeight(); + } + + metaTestResults.add(new MetaTestResult(metaTest.getName(), metaTest.getWeight(), passesTheMetaTest)); + + totalWeight += metaTest.getWeight(); + } + + result.logPenaltyMetaTests(score, totalWeight, metaTestResults); + } catch (Exception ex) { + result.genericFailure(this, ex); + } finally { + /* restore the class loader to the one before meta tests */ + Thread.currentThread().setContextClassLoader(currentClassLoader); + } + } + + @Override + public boolean equals(Object other) { + return other instanceof RunPenaltyMetaTestsStep; + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java index 71dc41291..11918ed57 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java @@ -86,7 +86,7 @@ public GradeValues setPenalty(int penalty) { return this; } - public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) { + public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) { GradeValues grades = new GradeValues(); grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches()); grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); @@ -94,7 +94,8 @@ public static GradeValues fromResults(CoverageResult coverageResults, CodeChecks grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); // penalty is equal to the sum of the weights of all failed penalty code checks - grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()); + grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum() + + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum()); return grades; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java index 27356df70..18d0bf374 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java @@ -13,6 +13,7 @@ public class Result { private final CodeChecksResult penaltyCodeChecks; private final CoverageResult coverage; private final MetaTestsResult metaTests; + private final MetaTestsResult penaltyMetaTests; private final int penalty; private final int finalGrade; private final GenericFailure genericFailure; @@ -20,7 +21,7 @@ public class Result { private final GradeWeight weights; private final String successMessage; - public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { + public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { this.compilation = compilation; this.tests = tests; this.mutationTesting = mutationTesting; @@ -28,6 +29,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest this.penaltyCodeChecks = penaltyCodeChecks; this.coverage = coverage; this.metaTests = metaTests; + this.penaltyMetaTests = penaltyMetaTests; this.penalty = penalty; this.finalGrade = finalGrade; this.genericFailure = genericFailure; @@ -40,7 +42,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest } public Result(CompilationResult compilation, double timeInSeconds) { - this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null); + this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null); } public CompilationResult getCompilation() { @@ -70,6 +72,9 @@ public CoverageResult getCoverage() { public MetaTestsResult getMetaTests() { return metaTests; } + public MetaTestsResult getPenaltyMetaTests() { + return penaltyMetaTests; + } public int getPenalty() { return penalty; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 86ebe84cf..6b127a2aa 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -49,6 +49,7 @@ public class ResultBuilder { private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); + private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty(); public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) { this.ctx = ctx; @@ -244,6 +245,10 @@ public void logMetaTests(int score, int totalTests, List metaTes this.metaTestResults.addResults(score, totalTests, metaTestResults); } + public void logPenaltyMetaTests(int score, int totalTests, List metaTestResults) { + this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults); + } + /* * Generic failures */ @@ -271,7 +276,7 @@ public Result build() { if(!compilation.successful()) { return new Result(compilation, timeInSeconds); } else { - GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults); + GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults); GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights()); String successMessage = ctx.getRunConfiguration().successMessage(); @@ -287,6 +292,7 @@ public Result build() { penaltyCodeCheckResults, coverageResults, metaTestResults, + penaltyMetaTestResults, penalty, finalGrade, genericFailureObject, diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index 811654428..bfedbf37a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -65,7 +65,7 @@ private void writeStdOutFile(Context ctx, Result result) { printCoverageResults(result.getCoverage()); printMutationTestingResults(result.getMutationTesting()); printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks()); - printMetaTestResults(ctx, result.getMetaTests()); + printMetaTestResults(ctx, result.getMetaTests(), result.getPenaltyMetaTests()); printFinalGrade(ctx, result); printModeAndTimeToRun(ctx, result.getTimeInSeconds()); } @@ -244,8 +244,8 @@ private void printModeAndTimeToRun(Context ctx, double timeInSeconds) { l(String.format("\nAndy took %.1f seconds to assess your solution.", timeInSeconds)); } - private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { - if (!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) + private void printMetaTestResults(Context ctx, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests) { + if ((!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) && (!penaltyMetaTests.wasExecuted() || penaltyMetaTests.hasNoMetaTests())) return; boolean allHints = modeActionSelector(ctx).shouldShowFullHints(); @@ -255,7 +255,8 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { return; l("\n--- Meta tests"); - l(String.format("%d/%d passed", metaTests.getPassedMetaTests(), metaTests.getTotalTests())); + l(String.format("%d/%d passed", metaTests.getPassedMetaTests() + penaltyMetaTests.getPassedMetaTests(), + metaTests.getTotalTests() + penaltyMetaTests.getTotalTests())); if (allHints) { for (MetaTestResult metaTestResult : metaTests.getMetaTestResults()) { @@ -265,6 +266,13 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { l(String.format("Meta test: %s (weight: %d) FAILED", metaTestResult.getName(), metaTestResult.getWeight())); } } + for (MetaTestResult penaltyResult : penaltyMetaTests.getMetaTestResults()) { + if (penaltyResult.succeeded()) { + l(String.format("Meta test: %s (penalty: %d) PASSED", penaltyResult.getName(), penaltyResult.getWeight())); + } else { + l(String.format("Meta test: %s (penalty: %d) FAILED", penaltyResult.getName(), penaltyResult.getWeight())); + } + } } } diff --git a/andy/src/test/java/integration/LibraryMetaTestsTest.java b/andy/src/test/java/integration/LibraryMetaTestsTest.java index 3199ca88f..9faba1b6b 100644 --- a/andy/src/test/java/integration/LibraryMetaTestsTest.java +++ b/andy/src/test/java/integration/LibraryMetaTestsTest.java @@ -88,4 +88,46 @@ void metaTestInternalFailure() { .isEqualTo(RunMetaTestsStep.class.getSimpleName()); } + @Test + void allPenaltyMetaTestsPassing() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddOfficialSolution", "NumberUtilsAddWithPenaltyMetaTestsConfiguration"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotApplyCarryAtAll")) + .has(passedMetaTest("DoesNotApplyLastCarry")) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")); + } + + @Test + void somePenaltyMetaTestFailing() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddAllTestsPass", "NumberUtilsAddWithPenaltyMetaTestsConfiguration"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(1); + + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")) + .has(failedMetaTest("DoesNotApplyCarryAtAll")) + .has(failedMetaTest("DoesNotApplyLastCarry")); + } + + @Test + void somePenaltyMetaTestFailingWithWeights() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(7); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(2); + + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")) + .has(failedMetaTest("AppliesMultipleCarriesWrongly")) + .has(failedMetaTest("DoesNotApplyCarryAtAll")) + .has(failedMetaTest("DoesNotApplyLastCarry")); + } + } \ No newline at end of file diff --git a/andy/src/test/java/integration/ModesAndActionsTest.java b/andy/src/test/java/integration/ModesAndActionsTest.java index 4a1c66ad6..38a262cd9 100644 --- a/andy/src/test/java/integration/ModesAndActionsTest.java +++ b/andy/src/test/java/integration/ModesAndActionsTest.java @@ -31,6 +31,8 @@ void practiceModeRunsEverything() { .has(codeCheck("getTripById should be set up", true, 1)); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(91); } @@ -49,10 +51,33 @@ void noPenaltyCodeChecksDefined() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(91); } + @Test + void failingPenaltyMetaTests() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigPenaltyMetaTestsFailing"); + + assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); + assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); + assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); + assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(20); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(0); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); + assertThat(result.getFinalGrade()).isEqualTo(91 - 20); + } + @Test void failingPenaltyCodeChecks() { Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing"); @@ -67,6 +92,8 @@ void failingPenaltyCodeChecks() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks()).isEqualTo(10); assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks()).isEqualTo(10 + 5 + 3); assertThat(result) @@ -76,6 +103,29 @@ void failingPenaltyCodeChecks() { assertThat(result.getFinalGrade()).isEqualTo(91 - (5 + 3)); } + @Test + void failingPenaltyMetaTestsOverrideGradeTo0() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigPenaltyMetaTestsFailing2"); + + assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); + assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); + assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); + assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(220); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(0); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId2")); + assertThat(result.getFinalGrade()).isEqualTo(0); + } + + @Test void failingPenaltyCodeChecksOverrideGradeTo0() { Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2"); @@ -90,6 +140,8 @@ void failingPenaltyCodeChecksOverrideGradeTo0() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks(false)).isEqualTo(1); assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks(false)).isEqualTo(3); assertThat(result) @@ -111,6 +163,7 @@ void runOnlyTests() { assertThat(result.getMutationTesting().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isFalse(); assertThat(result.getFinalGrade()).isEqualTo(0); } @@ -130,6 +183,7 @@ void runOnlyTestsAndCoverageToolsDuringExam() { assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); assertThat(result.getMetaTests().wasExecuted()).isFalse(); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); @@ -157,6 +211,8 @@ void gradingModeShouldRunEverything(Action action) { .has(penaltyCodeCheck("getTripById should not be set up penalty", false, 5)); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(86); } diff --git a/andy/src/test/java/testutils/ResultTestDataBuilder.java b/andy/src/test/java/testutils/ResultTestDataBuilder.java index 1289b7c3c..a1c67e057 100644 --- a/andy/src/test/java/testutils/ResultTestDataBuilder.java +++ b/andy/src/test/java/testutils/ResultTestDataBuilder.java @@ -20,6 +20,7 @@ public class ResultTestDataBuilder { private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); + private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty(); private int penalty = 0; private int finalGrade = 0; private double timeInSeconds = 10; @@ -95,6 +96,15 @@ public ResultTestDataBuilder withMetaTestResults(List list) { return this; } + public ResultTestDataBuilder withPenaltyMetaTestResults(List list) { + penaltyMetaTestResults = MetaTestsResult.build( + (int) list.stream().filter(MetaTestResult::succeeded).count(), + list.size(), + list + ); + return this; + } + public ResultTestDataBuilder withWeights(float branchCoverageWeight, float mutationCoverageWeight, float metaTestsWeight, float codeChecksWeight) { weights = new GradeWeight(branchCoverageWeight, mutationCoverageWeight, metaTestsWeight, codeChecksWeight); return this; @@ -113,6 +123,6 @@ public ResultTestDataBuilder withSuccessMessage(String successMessage) { public Result build() { GenericFailure genericFailure = GenericFailure.build(genericFailureMessage, genericFailureStepName, genericFailureExceptionMessage, genericFailureExternalProcessExitCode, genericFailureExternalProcessErrorMessages); - return new Result(compilation, testResults, mutationResults, codeCheckResults, penaltyCodeCheckResults, coverageResults, metaTestResults, penalty, finalGrade, genericFailure, timeInSeconds, weights, successMessage); + return new Result(compilation, testResults, mutationResults, codeCheckResults, penaltyCodeCheckResults, coverageResults, metaTestResults, penaltyMetaTestResults, penalty, finalGrade, genericFailure, timeInSeconds, weights, successMessage); } } diff --git a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java index d72664d9e..2605a11ac 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java @@ -171,6 +171,14 @@ public static Condition metaTestPassing(String metaTestName) { return containsRegex("Meta test: " + metaTestName + " \\(.*\\) PASSED"); } + public static Condition penaltyMetaTestFailing(String metaTestName) { + return containsRegex("Meta test: " + metaTestName + " \\(penalty: .*\\) FAILED"); + } + + public static Condition penaltyMetaTestPassing(String metaTestName) { + return containsRegex("Meta test: " + metaTestName + " \\(penalty: .*\\) PASSED"); + } + public static Condition finalGrade(int score) { return new Condition<>() { @Override @@ -327,7 +335,7 @@ public static Condition noCodeChecks() { return not(codeChecks()); } - public static Condition noPenaltyCodeChecks() { + public static Condition noPenaltyCodeChecksOrMetaTests() { return not(containsRegex(".*\\(penalty: \\d+\\)")); } diff --git a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java index 60d506479..6f965efd8 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java @@ -284,7 +284,7 @@ void testPrintFinalGrade() { .has(mutationScore(5, 6)) .has(noMetaTests()) .has(noCodeChecks()) - .has(noPenaltyCodeChecks()) + .has(noPenaltyCodeChecksOrMetaTests()) .has(noPenalty()) .has(not(zeroScoreExplanation())) .doesNotContain("test success message"); @@ -570,6 +570,11 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b new MetaTestResult("e", 3, false), new MetaTestResult("f", 1, true) )) + .withPenaltyMetaTestResults(List.of( + new MetaTestResult("d1", 16, true), + new MetaTestResult("e1", 32, false), + new MetaTestResult("f1", 64, true) + )) .withPenaltyCodeCheckResults(List.of( new CodeCheckResult("a1", 1, true), new CodeCheckResult("b1", 1, true), @@ -581,7 +586,6 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b writer.write(ctx, result); String output = generatedResult(); - assertThat(output) .has(versionInformation(versionInformation)) .has(finalGradeOnScreen(34)) @@ -595,8 +599,8 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(fullGradeDescriptionDisplayed("Meta tests", 2, 3, 0.25)) .has(mutationScore(5, 6)) .has(scoreOfCodeChecks(7, 8)) - .has(metaTestsPassing(2)) - .has(metaTests(3)) + .has(metaTestsPassing(4)) + .has(metaTests(6)) .has(noPenalty()) .has(not(zeroScoreExplanation())); @@ -604,6 +608,9 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(metaTestDisplayed("d", true, fullHints)) .has(metaTestDisplayed("e", false, fullHints)) .has(metaTestDisplayed("f", true, fullHints)) + .has(penaltyMetaTestDisplayed("d1", true, fullHints)) + .has(penaltyMetaTestDisplayed("e1", false, fullHints)) + .has(penaltyMetaTestDisplayed("f1", true, fullHints)) .has(codeCheckDisplayed("a", true, 1, fullHints)) .has(codeCheckDisplayed("b", true, 2, fullHints)) .has(codeCheckDisplayed("c", false, 1, fullHints)) @@ -664,6 +671,55 @@ void testPrintFinalGradeWithPenaltyCodeChecksFailing(boolean fullHints, boolean .has(penaltyCodeCheckDisplayed("d1", true, 1, fullHints)); } + @ParameterizedTest + @CsvSource({ + "true,false", + "false,true", + "true,true" + }) + void testPrintFinalGradeWithPenaltyMetaTestsFailing(boolean fullHints, boolean partialHints) { + ModeActionSelector modeActionSelector = mock(ModeActionSelector.class); + when(modeActionSelector.shouldCalculateAndShowGrades()).thenReturn(true); + when(modeActionSelector.shouldGenerateAnalytics()).thenReturn(false); + when(modeActionSelector.shouldShowFullHints()).thenReturn(fullHints); + when(modeActionSelector.shouldShowPartialHints()).thenReturn(partialHints); + when(modeActionSelector.getMode()).thenReturn(Mode.PRACTICE); + + when(ctx.getModeActionSelector()).thenReturn(modeActionSelector); + + Result result = new ResultTestDataBuilder() + .withCoverageResult(CoverageResult.build( + 4, 7, 5, 8, 1, 2, + new CoverageLineByLine(List.of(), List.of(), List.of()))) + .withMutationTestingResults(5, 6) + .withPenaltyMetaTestResults(List.of( + new MetaTestResult("a1", 8, false), + new MetaTestResult("b1", 16, false), + new MetaTestResult("c1", 32, true), + new MetaTestResult("d1", 64, true) + )) + .withPenalty(24) + .build(); + + writer.write(ctx, result); + + String output = generatedResult(); + + assertThat(output) + .has(versionInformation(versionInformation)) + .has(compilationSuccess()) + .has(fullGradeDescriptionDisplayed("Meta tests", 0, 0, 0.25)) + .has(mutationScore(5, 6)) + .has(penalty(24)) + .has(not(zeroScoreExplanation())); + + assertThat(output) + .has(penaltyMetaTestDisplayed("a1", false, fullHints)) + .has(penaltyMetaTestDisplayed("b1", false, fullHints)) + .has(penaltyMetaTestDisplayed("c1", true, fullHints)) + .has(penaltyMetaTestDisplayed("d1", true, fullHints)); + } + @Test void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayedButNoCodeChecksOrTests() { ModeActionSelector modeActionSelector = mock(ModeActionSelector.class); @@ -865,5 +921,9 @@ protected Condition metaTestDisplayed(String description, boolea var metaTestCondition = pass ? metaTestPassing(description) : metaTestFailing(description); return shownInOutput ? metaTestCondition : not(metaTestCondition); } + protected Condition penaltyMetaTestDisplayed(String description, boolean pass, boolean shownInOutput) { + var penaltyMetaTestCondition = pass ? penaltyMetaTestPassing(description) : penaltyMetaTestFailing(description); + return shownInOutput ? penaltyMetaTestCondition : not(penaltyMetaTestCondition); + } } diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java new file mode 100644 index 000000000..a8c3afd20 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java @@ -0,0 +1,100 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.9f); + put("meta", 0.0f); + put("codechecks", 0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withStringReplacement(3, "AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """), + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("DoesNotApplyLastCarry", + """ + // add leftover carry + result.addFirst(carry); + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + return result; + """, + """ + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withLineReplacement(2, "DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java new file mode 100644 index 000000000..228399839 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java @@ -0,0 +1,105 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.7f); + put("meta", 0.2f); + put("codechecks", 0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """) + ); + } + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("DoesNotApplyLastCarry", + """ + // add leftover carry + result.addFirst(carry); + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + return result; + """, + """ + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withLineReplacement("DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java new file mode 100644 index 000000000..80ca5b051 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java @@ -0,0 +1,90 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.Comparison; +import nl.tudelft.cse1110.andy.codechecker.checks.MockClass; +import nl.tudelft.cse1110.andy.codechecker.checks.MockitoWhen; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.25f); + put("codechecks", 0.25f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SoftWhere"); + } + + @Override + public CheckScript checkScript() { + return new CheckScript(List.of( + new SingleCheck("Trip Repository should be mocked", new MockClass("TripRepository")), + new SingleCheck( "Trip should not be mocked", true, new MockClass("Trip")), + new SingleCheck( "getTripById should be set up", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + MetaTest.withStringReplacement("BoundaryCheck", + """ + if (capacityLeft(trip) < people.size()) + return false; + """, + """ + if (capacityLeft(trip) <= people.size()) + return false; + """), + MetaTest.withStringReplacement("DoesNotCheckCapacity", + """ + if (capacityLeft(trip) < people.size()) + return false; + """,""), + MetaTest.withLineReplacement("DoesNotCheckSave", 154, 154, "") + ); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement(20,"DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """) + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java new file mode 100644 index 000000000..150d4e4ab --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java @@ -0,0 +1,102 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.Comparison; +import nl.tudelft.cse1110.andy.codechecker.checks.MockClass; +import nl.tudelft.cse1110.andy.codechecker.checks.MockitoWhen; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.25f); + put("codechecks", 0.25f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SoftWhere"); + } + + @Override + public CheckScript checkScript() { + return new CheckScript(List.of( + new SingleCheck("Trip Repository should be mocked", new MockClass("TripRepository")), + new SingleCheck( "Trip should not be mocked", true, new MockClass("Trip")), + new SingleCheck( "getTripById should be set up", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + MetaTest.withStringReplacement("BoundaryCheck", + """ + if (capacityLeft(trip) < people.size()) + return false; + """, + """ + if (capacityLeft(trip) <= people.size()) + return false; + """), + MetaTest.withStringReplacement("DoesNotCheckCapacity", + """ + if (capacityLeft(trip) < people.size()) + return false; + """,""), + MetaTest.withLineReplacement("DoesNotCheckSave", 154, 154, "") + ); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement(20,"DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + + MetaTest.withLineReplacement(200,"DoesNotCheckInvalidTripId2", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """) + ); + } + +} \ No newline at end of file diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index 047fce61b..b3e49f973 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -102,6 +102,7 @@ private static void appendMetaScoreElements(Result result, Document doc, Element result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getPenaltyCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); + result.getPenaltyMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); testSuitesElement.appendChild(metaElement); } From 5a09ef3e62a7f2a72ea34eb6484ce093958bba75 Mon Sep 17 00:00:00 2001 From: Alexandru Gabriel Cojocaru <54269070+alexcojocaru2002@users.noreply.github.com> Date: Wed, 1 May 2024 22:36:59 +0200 Subject: [PATCH 2/2] SkipPitest / SkipJacoco (#267) Co-authored-by: Martin Mladenov <30376060+martinmladenov@users.noreply.github.com> --- .../andy/config/SecureExamRunConfiguration.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java index 75af9acc6..ff7681e39 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java @@ -16,6 +16,8 @@ public class SecureExamRunConfiguration extends RunConfiguration { private final List listOfMutants; private final int numberOfMutationsToConsider; private final ExternalProcess externalProcess; + private final boolean skipJacoco; + private final boolean skipPitest; public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) { this.classesUnderTest = runConfigurationToClone.classesUnderTest(); @@ -23,6 +25,8 @@ public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) { this.numberOfMutationsToConsider = runConfigurationToClone.numberOfMutationsToConsider(); this.externalProcess = runConfigurationToClone.externalProcess(); this.successMessage = runConfigurationToClone.successMessage(); + this.skipJacoco = runConfigurationToClone.skipJacoco(); + this.skipPitest = runConfigurationToClone.skipPitest(); } public Mode mode() { @@ -60,5 +64,13 @@ public String successMessage() { return successMessage; } + @Override + public boolean skipJacoco() { + return this.skipJacoco; + } + @Override + public boolean skipPitest() { + return this.skipPitest; + } }