diff --git a/README.md b/README.md index edbc321c..bb8029be 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ * [Mutation coverage](#mutation-coverage) * [Meta tests](#meta-tests) * [Code checks](#code-checks) - * [Required code checks](#required-code-checks) + * [Penalty code checks](#penalty-code-checks) * [Success message](#success-message) * [External process](#external-process) * [Team](#team) @@ -370,25 +370,29 @@ should have properties: FAIL (weight: 2) either make use of Arbitraries or JQWik IntRange-like annotations: FAIL (weight: 8) ``` -#### Required code checks +#### Penalty code checks -Required code checks work in a similar way to regular code checks. However, they are not considered a grading component. Instead, if one or more required code checks fail, the final grade is overridden to 0, irrespective of the score of any of the grading components. +Penalty code checks work in a similar way to regular code checks. However, they are not considered a grading component. Instead, if a penalty code check passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade. -They can be defined as follows: +Penalty code checks must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty code checks where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead. + +Penalty code checks can be defined as follows: ```java @Override -public CheckScript requiredCheckScript() { +public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked", + // this check deducts 10 points if it fails + new SingleCheck(10, "Trip Repository should be mocked", new MockClass("TripRepository")), - new SingleCheck("Reservation Repository should be mocked", + // this check overrides the grade to 0 if it fails + new SingleCheck(100, "Reservation Repository should be mocked", new MockClass("ReservationRepository")) )); } ``` -If this method is not overridden, required code checks are disabled. +If this method is not overridden, penalty code checks are disabled. #### Success message diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java index bdebf1f6..25af1fe0 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java @@ -37,7 +37,7 @@ public CheckScript checkScript() { return new CheckScript(Collections.emptyList()); } - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(Collections.emptyList()); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java index 633029ca..914c1b9a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java @@ -125,7 +125,7 @@ public static List fullMode() { new RunJUnitTestsStep(), new CollectCoverageInformationStep(), new RunPitestStep(), - new RunRequiredCodeChecksStep(), + new RunPenaltyCodeChecksStep(), new RunCodeChecksStep(), new KillExternalProcessStep(), new RunMetaTestsStep() diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunRequiredCodeChecksStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyCodeChecksStep.java similarity index 80% rename from andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunRequiredCodeChecksStep.java rename to andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyCodeChecksStep.java index 5aea39aa..4558c73c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunRequiredCodeChecksStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyCodeChecksStep.java @@ -11,24 +11,24 @@ import static nl.tudelft.cse1110.andy.utils.FilesUtils.findSolution; -public class RunRequiredCodeChecksStep implements ExecutionStep { +public class RunPenaltyCodeChecksStep implements ExecutionStep { @Override public void execute(Context ctx, ResultBuilder result) { DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); RunConfiguration runCfg = ctx.getRunConfiguration(); - CheckScript script = runCfg.requiredCheckScript(); + CheckScript script = runCfg.penaltyCheckScript(); try { script.runChecks(findSolution(dirCfg.getWorkingDir())); } catch (FileNotFoundException e) { throw new RuntimeException(e); } - result.logRequiredCodeChecks(script); + result.logPenaltyCodeChecks(script); } @Override public boolean equals(Object other) { - return other instanceof RunRequiredCodeChecksStep; + return other instanceof RunPenaltyCodeChecksStep; } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java index 8c262b0b..8b3bc18f 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java @@ -26,6 +26,18 @@ public int calculateFinalGrade(GradeValues gradeValues, GradeWeight weights) { if(finalGrade < 0 || finalGrade > 100) throw new RuntimeException("Invalid grade calculation"); + if(gradeValues.getPenalty() < 0){ + throw new RuntimeException("Negative penalty: " + gradeValues.getPenalty()); + } + + // Apply penalty + finalGrade -= gradeValues.getPenalty(); + + // Grade should not go below 0 + // The total penalty can be more than 100: for example, if there are two failing code checks + // which both have a penalty of 100, the total penalty will be 200, and the final grade should be 0. + if(finalGrade < 0) finalGrade = 0; + // Grades between 99.5 and 100 should be rounded down to 99 instead of up if (finalGrade == 100 && hasIncompleteComponents(gradeValues, weights)) { finalGrade = 99; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java index a3406515..71dc4129 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java @@ -19,6 +19,8 @@ public class GradeValues { private int checksPassed; private int totalChecks; + private int penalty; + public int getCoveredBranches() { return coveredBranches; } @@ -75,14 +77,25 @@ public void setCheckGrade(int checksPassed, int totalChecks) { this.totalChecks = totalChecks; } - public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults) { + public int getPenalty() { + return penalty; + } + + public GradeValues setPenalty(int penalty) { + this.penalty = penalty; + return this; + } + + public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) { GradeValues grades = new GradeValues(); grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches()); grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants()); grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); + // penalty is equal to the sum of the weights of all failed penalty code checks + grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()); + return grades; } - } \ No newline at end of file diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/CodeChecksResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/CodeChecksResult.java index 6d5d7da7..db09e7de 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/CodeChecksResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/CodeChecksResult.java @@ -21,11 +21,21 @@ public static CodeChecksResult build(List checkResults) { } public int getNumberOfPassedChecks() { - return checkResults.stream().mapToInt(check -> check.passed() ? check.getWeight() : 0).sum(); + return getNumberOfPassedChecks(true); + } + + public int getNumberOfPassedChecks(boolean includeWeight) { + return checkResults.stream().mapToInt(check -> check.passed() ? + (includeWeight ? check.getWeight() : 1) : + 0).sum(); } public int getTotalNumberOfChecks() { - return checkResults.stream().mapToInt(c -> c.getWeight()).sum(); + return getTotalNumberOfChecks(true); + } + + public int getTotalNumberOfChecks(boolean includeWeight) { + return checkResults.stream().mapToInt(c -> (includeWeight ? c.getWeight() : 1)).sum(); } public List getCheckResults() { @@ -40,10 +50,6 @@ public boolean wasExecuted() { return wasExecuted; } - public boolean allChecksPass() { - return getNumberOfPassedChecks() == getTotalNumberOfChecks(); - } - @Override public String toString() { return "CodeChecksResult{" + diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java index 16e839a2..27356df7 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java @@ -10,23 +10,25 @@ public class Result { private final UnitTestsResult tests; private final MutationTestingResult mutationTesting; private final CodeChecksResult codeChecks; - private final CodeChecksResult requiredCodeChecks; + private final CodeChecksResult penaltyCodeChecks; private final CoverageResult coverage; private final MetaTestsResult metaTests; + private final int penalty; private final int finalGrade; private final GenericFailure genericFailure; private final double timeInSeconds; private final GradeWeight weights; private final String successMessage; - public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult requiredCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { + public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { this.compilation = compilation; this.tests = tests; this.mutationTesting = mutationTesting; this.codeChecks = codeChecks; - this.requiredCodeChecks = requiredCodeChecks; + this.penaltyCodeChecks = penaltyCodeChecks; this.coverage = coverage; this.metaTests = metaTests; + this.penalty = penalty; this.finalGrade = finalGrade; this.genericFailure = genericFailure; this.timeInSeconds = timeInSeconds; @@ -38,7 +40,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest } public Result(CompilationResult compilation, double timeInSeconds) { - this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, GenericFailure.noFailure(), timeInSeconds, null, null); + this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null); } public CompilationResult getCompilation() { @@ -53,8 +55,8 @@ public MutationTestingResult getMutationTesting() { return mutationTesting; } - public CodeChecksResult getRequiredCodeChecks() { - return requiredCodeChecks; + public CodeChecksResult getPenaltyCodeChecks() { + return penaltyCodeChecks; } public CodeChecksResult getCodeChecks() { @@ -69,6 +71,10 @@ public MetaTestsResult getMetaTests() { return metaTests; } + public int getPenalty() { + return penalty; + } + public int getFinalGrade() { return finalGrade; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index e4f1578e..ee1825b8 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -46,7 +46,7 @@ public class ResultBuilder { private UnitTestsResult testResults = UnitTestsResult.empty(); private MutationTestingResult mutationResults = MutationTestingResult.empty(); private CodeChecksResult codeCheckResults = CodeChecksResult.empty(); - private CodeChecksResult requiredCodeCheckResults = CodeChecksResult.empty(); + private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); @@ -168,10 +168,10 @@ public void logCodeChecks(CheckScript script) { } /* - * Required code checks + * Penalty code checks (subtract points from the final score if they fail) */ - public void logRequiredCodeChecks(CheckScript script) { - this.requiredCodeCheckResults = buildCodeChecksResult(script); + public void logPenaltyCodeChecks(CheckScript script) { + this.penaltyCodeCheckResults = buildCodeChecksResult(script); } private static CodeChecksResult buildCodeChecksResult(CheckScript script) { @@ -270,21 +270,23 @@ public Result build() { if(!compilation.successful()) { return new Result(compilation, timeInSeconds); } else { - GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults); + GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults); GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights()); String successMessage = ctx.getRunConfiguration().successMessage(); this.checkExternalProcessExit(); - int finalGrade = calculateFinalGrade(grades, weights); + final int finalGrade = calculateFinalGrade(grades, weights); + final int penalty = grades.getPenalty(); return new Result(compilation, testResults, mutationResults, codeCheckResults, - requiredCodeCheckResults, + penaltyCodeCheckResults, coverageResults, metaTestResults, + penalty, finalGrade, genericFailureObject, timeInSeconds, @@ -332,9 +334,8 @@ public boolean hasFailed() { boolean compilationFailed = compilation!=null && !compilation.successful(); boolean unitTestsFailed = testResults != null && testResults.didNotGoWell(); boolean hasGenericFailure = genericFailureObject.hasFailure(); - boolean requiredCodeChecksFailed = !requiredCodeCheckResults.allChecksPass(); - return compilationFailed || unitTestsFailed || hasGenericFailure || requiredCodeChecksFailed; + return compilationFailed || unitTestsFailed || hasGenericFailure; } public UnitTestsResult getTestResults() { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index 5b45ae1c..72cf9544 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -64,8 +64,7 @@ private void writeStdOutFile(Context ctx, Result result) { printTestResults(result.getTests()); printCoverageResults(result.getCoverage()); printMutationTestingResults(result.getMutationTesting()); - printRequiredCodeCheckResults(ctx, result.getRequiredCodeChecks()); - printCodeCheckResults(ctx, result.getCodeChecks()); + printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks()); printMetaTestResults(ctx, result.getMetaTests()); printFinalGrade(ctx, result); printModeAndTimeToRun(ctx, result.getTimeInSeconds()); @@ -176,6 +175,11 @@ private void printFinalGrade(Context ctx, Result result) { printGradeCalculationDetails("Mutation coverage", result.getMutationTesting().getKilledMutants(), result.getMutationTesting().getTotalNumberOfMutants(), result.getWeights().getMutationCoverageWeight()); printGradeCalculationDetails("Code checks", result.getCodeChecks().getNumberOfPassedChecks(), result.getCodeChecks().getTotalNumberOfChecks(), result.getWeights().getCodeChecksWeight()); printGradeCalculationDetails("Meta tests", result.getMetaTests().getPassedMetaTests(), result.getMetaTests().getTotalTests(), result.getWeights().getMetaTestsWeight()); + } + + // print penalty + if(result.getPenalty() != 0){ + l(String.format("Penalty: %d", result.getPenalty())); l(""); } @@ -287,8 +291,8 @@ private void printCoverageResults(CoverageResult coverage) { } - private void printCodeCheckResults(Context ctx, CodeChecksResult codeChecks) { - if(!codeChecks.wasExecuted()) + private void printCodeCheckResults(Context ctx, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks) { + if(!codeChecks.wasExecuted() && !penaltyCodeChecks.wasExecuted()) return; boolean allHints = modeActionSelector(ctx).shouldShowFullHints(); @@ -297,36 +301,16 @@ private void printCodeCheckResults(Context ctx, CodeChecksResult codeChecks) { if(!allHints && !onlyResult) return; - if(codeChecks.hasChecks()) { + if(codeChecks.hasChecks() || penaltyCodeChecks.hasChecks()) { l("\n--- Code checks"); - printCodeCheckOutput(codeChecks, allHints); - } - - } - - private void printRequiredCodeCheckResults(Context ctx, CodeChecksResult codeChecks) { - if(!codeChecks.wasExecuted()) - return; - - boolean allHints = modeActionSelector(ctx).shouldShowFullHints(); - boolean onlyResult = modeActionSelector(ctx).shouldShowPartialHints(); - - if(!allHints && !onlyResult) - return; - - if(codeChecks.hasChecks()) { - l("\n--- Required code checks"); - printCodeCheckOutput(codeChecks, allHints); - - if(!codeChecks.allChecksPass()){ - l("\nSome required code checks failed. Stopping the assessment."); - } + printCodeCheckOutput(codeChecks, penaltyCodeChecks, allHints); } } - private void printCodeCheckOutput(CodeChecksResult codeChecks, boolean allHints) { - l(String.format("%d/%d passed", codeChecks.getNumberOfPassedChecks(), codeChecks.getTotalNumberOfChecks())); + private void printCodeCheckOutput(CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, boolean allHints) { + l(String.format("%d/%d passed", codeChecks.getNumberOfPassedChecks() + penaltyCodeChecks.getNumberOfPassedChecks(false), + codeChecks.getTotalNumberOfChecks() + penaltyCodeChecks.getTotalNumberOfChecks(false))); if(allHints) { for (CodeCheckResult result : codeChecks.getCheckResults()) { @@ -335,6 +319,12 @@ private void printCodeCheckOutput(CodeChecksResult codeChecks, boolean allHints) result.passed() ? "PASS" : "FAIL", result.getWeight())); } + for (CodeCheckResult result : penaltyCodeChecks.getCheckResults()) { + l(String.format("%s: %s (penalty: %d)", + result.getDescription(), + result.passed() ? "PASS" : "FAIL", + result.getWeight())); + } } } diff --git a/andy/src/test/java/integration/CodeChecksTest.java b/andy/src/test/java/integration/CodeChecksTest.java index 0ee6d83a..39d285d1 100644 --- a/andy/src/test/java/integration/CodeChecksTest.java +++ b/andy/src/test/java/integration/CodeChecksTest.java @@ -32,30 +32,30 @@ void someChecksFail() { } @Test - void requiredCodeChecksPass() { - Result result = run( "SoftWhereLibrary", "SoftWhereTests", "SoftWhereConfigWithRequiredCodeChecksPassingConfiguration"); + void penaltyCodeChecksPass() { + Result result = run( "SoftWhereLibrary", "SoftWhereTests", "SoftWhereConfigWithPenaltyCodeChecksPassingConfiguration"); assertThat(result) .has(checksScore(2,5)) - .has(requiredCodeCheck("Trip Repository should be mocked required", true, 1)) - .has(requiredCodeCheck("getTripById should be set up required", true, 1)) + .has(penaltyCodeCheck("Trip Repository should be mocked penalty", true, 1)) + .has(penaltyCodeCheck("getTripById should be set up penalty", true, 1)) .has(codeCheck("Trip Repository should be mocked", true, 1)) .has(codeCheck("Trip should be mocked", false, 3)) // this check makes no sense, just for the check to fail .has(codeCheck("getTripById should be set up", true, 1)); } @Test - void requiredCodeChecksFail() { - Result result = run( "SoftWhereLibrary", "SoftWhereTests", "SoftWhereConfigWithRequiredCodeChecksFailingConfiguration"); + void penaltyCodeChecksFail() { + Result result = run( "SoftWhereLibrary", "SoftWhereTests", "SoftWhereConfigWithPenaltyCodeChecksFailingConfiguration"); assertThat(result) - .has(checksScore(0,0)) - .has(requiredCodeCheck("Trip Repository should be mocked required", true, 1)) - .has(requiredCodeCheck("getTripById should be set up required", true, 1)) - .has(requiredCodeCheck("Trip should be mocked required", false, 1)) - .has(not(codeCheck("Trip Repository should be mocked", true, 1))) - .has(not(codeCheck("Trip should be mocked", false, 3))) - .has(not(codeCheck("getTripById should be set up", true, 1))); + .has(checksScore(2,5)) + .has(penaltyCodeCheck("Trip Repository should be mocked penalty", true, 1)) + .has(penaltyCodeCheck("getTripById should be set up req", true, 200)) + .has(penaltyCodeCheck("Trip should be mocked required", false, 100)) + .has(codeCheck("Trip Repository should be mocked", true, 1)) + .has(codeCheck("Trip should be mocked", false, 3)) + .has(codeCheck("getTripById should be set up", true, 1)); } @Test @@ -89,12 +89,12 @@ public boolean matches(Result value) { }; } - public static Condition requiredCodeCheck(String name, boolean pass, int weight) { + public static Condition penaltyCodeCheck(String name, boolean pass, int penalty) { return new Condition<>() { @Override public boolean matches(Result value) { - return value.getRequiredCodeChecks().getCheckResults().stream().anyMatch(cc -> - cc.getWeight() == weight && + return value.getPenaltyCodeChecks().getCheckResults().stream().anyMatch(cc -> + cc.getWeight() == penalty && cc.passed() == pass && cc.getDescription().equals(name)); diff --git a/andy/src/test/java/integration/ModesAndActionsTest.java b/andy/src/test/java/integration/ModesAndActionsTest.java index 7a14bc66..4a1c66ad 100644 --- a/andy/src/test/java/integration/ModesAndActionsTest.java +++ b/andy/src/test/java/integration/ModesAndActionsTest.java @@ -8,7 +8,7 @@ import static integration.BaseMetaTestsTest.failedMetaTest; import static integration.CodeChecksTest.codeCheck; -import static integration.CodeChecksTest.requiredCodeCheck; +import static integration.CodeChecksTest.penaltyCodeCheck; import static org.assertj.core.api.Assertions.assertThat; public class ModesAndActionsTest extends IntegrationTestBase { @@ -23,6 +23,8 @@ void practiceModeRunsEverything() { assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks()).isEqualTo(200); + assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks()).isEqualTo(200); assertThat(result) .has(codeCheck("Trip Repository should be mocked", true, 1)) .has(codeCheck("Trip should not be mocked", true, 1)) @@ -34,8 +36,8 @@ void practiceModeRunsEverything() { } @Test - void noRequiredChecksDefined() { - Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksWithoutRequiredChecks"); + void noPenaltyCodeChecksDefined() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksWithoutPenaltyCodeChecks"); assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); @@ -43,8 +45,8 @@ void noRequiredChecksDefined() { assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); - assertThat(result.getRequiredCodeChecks().hasChecks()).isFalse(); - assertThat(result.getRequiredCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); @@ -52,20 +54,48 @@ void noRequiredChecksDefined() { } @Test - void failingRequiredCodeChecks() { - Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksRequiredCodeCheckFailing"); + void failingPenaltyCodeChecks() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing"); assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); - assertThat(result.getCodeChecks().wasExecuted()).isFalse(); - assertThat(result.getMetaTests().wasExecuted()).isFalse(); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isTrue(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks()).isEqualTo(10); + assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks()).isEqualTo(10 + 5 + 3); + assertThat(result) + .has(penaltyCodeCheck("Trip Repository should not be mocked penalty", false, 5)) + .has(penaltyCodeCheck("Trip should be mocked penalty", false, 3)) + .has(penaltyCodeCheck("getTripById should be set up penalty", true, 10)); + assertThat(result.getFinalGrade()).isEqualTo(91 - (5 + 3)); + } + + @Test + void failingPenaltyCodeChecksOverrideGradeTo0() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2"); + + assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); + assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); + assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); + assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isTrue(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks(false)).isEqualTo(1); + assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks(false)).isEqualTo(3); assertThat(result) - .has(requiredCodeCheck("Trip Repository should be mocked required", true, 1)) - .has(requiredCodeCheck("Trip should be mocked required", false, 1)) - .has(requiredCodeCheck("getTripById should be set up required", true, 1)); - assertThat(result.getRequiredCodeChecks().allChecksPass()).isFalse(); + .has(penaltyCodeCheck("Trip Repository should not be mocked penalty", false, 100)) + .has(penaltyCodeCheck("Trip should be mocked penalty", false, 30)) + .has(penaltyCodeCheck("getTripById should be set up penalty", true, 10)); assertThat(result.getFinalGrade()).isEqualTo(0); } @@ -80,7 +110,7 @@ void runOnlyTests() { assertThat(result.getMetaTests().wasExecuted()).isFalse(); assertThat(result.getMutationTesting().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); - assertThat(result.getRequiredCodeChecks().wasExecuted()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); assertThat(result.getFinalGrade()).isEqualTo(0); } @@ -101,7 +131,7 @@ void runOnlyTestsAndCoverageToolsDuringExam() { assertThat(result.getMetaTests().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); - assertThat(result.getRequiredCodeChecks().wasExecuted()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); assertThat(result.getFinalGrade()).isEqualTo(0); } @@ -117,16 +147,18 @@ void gradingModeShouldRunEverything(Action action) { assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks()).isEqualTo(100); + assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks()).isEqualTo(105); assertThat(result) .has(codeCheck("Trip Repository should be mocked", true, 1)) .has(codeCheck("Trip should not be mocked", true, 1)) .has(codeCheck("getTripById should be set up", true, 1)) - .has(requiredCodeCheck("Trip Repository should be mocked required", true, 1)) - .has(requiredCodeCheck("getTripById should be set up required", true, 1)); + .has(penaltyCodeCheck("Trip Repository should be mocked required", true, 100)) + .has(penaltyCodeCheck("getTripById should not be set up penalty", false, 5)); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); - assertThat(result.getFinalGrade()).isEqualTo(91); + assertThat(result.getFinalGrade()).isEqualTo(86); } } diff --git a/andy/src/test/java/testutils/ResultTestDataBuilder.java b/andy/src/test/java/testutils/ResultTestDataBuilder.java index 9bbff182..1289b7c3 100644 --- a/andy/src/test/java/testutils/ResultTestDataBuilder.java +++ b/andy/src/test/java/testutils/ResultTestDataBuilder.java @@ -17,9 +17,10 @@ public class ResultTestDataBuilder { private UnitTestsResult testResults = UnitTestsResult.empty(); private MutationTestingResult mutationResults = MutationTestingResult.empty(); private CodeChecksResult codeCheckResults = CodeChecksResult.empty(); - private CodeChecksResult requiredCodeCheckResults = CodeChecksResult.empty(); + private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); + private int penalty = 0; private int finalGrade = 0; private double timeInSeconds = 10; private GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); @@ -60,6 +61,11 @@ public ResultTestDataBuilder withCoverageResult(CoverageResult coverageResult) { return this; } + public ResultTestDataBuilder withPenalty(int penalty) { + this.penalty = penalty; + return this; + } + public ResultTestDataBuilder withGrade(int grade) { finalGrade = grade; return this; @@ -75,8 +81,8 @@ public ResultTestDataBuilder withCodeCheckResults(List list) { return this; } - public ResultTestDataBuilder withRequiredCodeCheckResults(List list) { - requiredCodeCheckResults = CodeChecksResult.build(list); + public ResultTestDataBuilder withPenaltyCodeCheckResults(List list) { + penaltyCodeCheckResults = CodeChecksResult.build(list); return this; } @@ -107,6 +113,6 @@ public ResultTestDataBuilder withSuccessMessage(String successMessage) { public Result build() { GenericFailure genericFailure = GenericFailure.build(genericFailureMessage, genericFailureStepName, genericFailureExceptionMessage, genericFailureExternalProcessExitCode, genericFailureExternalProcessErrorMessages); - return new Result(compilation, testResults, mutationResults, codeCheckResults, requiredCodeCheckResults, coverageResults, metaTestResults, finalGrade, genericFailure, timeInSeconds, weights, successMessage); + return new Result(compilation, testResults, mutationResults, codeCheckResults, penaltyCodeCheckResults, coverageResults, metaTestResults, penalty, finalGrade, genericFailure, timeInSeconds, weights, successMessage); } } diff --git a/andy/src/test/java/unit/grade/GradeCalculatorTest.java b/andy/src/test/java/unit/grade/GradeCalculatorTest.java index 786dc1a5..d7e8ea5e 100644 --- a/andy/src/test/java/unit/grade/GradeCalculatorTest.java +++ b/andy/src/test/java/unit/grade/GradeCalculatorTest.java @@ -6,7 +6,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.util.stream.Stream; @@ -189,4 +191,106 @@ private static Stream zeroWeightAndZeroTotal() { of(0, 0, 0, 1, 0f, 0f, 0f, 1f) ); } + + @Test + void fullWithZeroPenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(25, 25); + + grades.setPenalty(0); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(100); + } + + @Test + void fullWithSomePenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(25, 25); + + grades.setPenalty(15); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(85); + } + + @Test + void fullWithMaxPenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(25, 25); + + grades.setPenalty(100); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(0); + } + + @Test + void notFullWithZeroPenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(10, 25); + + grades.setPenalty(0); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(85); + } + + @Test + void notFullWithSomePenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(10, 25); + + grades.setPenalty(10); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(75); + } + + @Test + void notFullWithMaxPenalty() { + GradeWeight weights = new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(25, 25); + grades.setMutationGrade(25, 25); + grades.setMetaGrade(25, 25); + grades.setCheckGrade(10, 25); + + grades.setPenalty(100); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(0); + } } diff --git a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java index 27eb1333..d72664d9 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java @@ -183,6 +183,22 @@ public boolean matches(String value) { }; } + public static Condition penalty(int penalty) { + return new Condition<>() { + @Override + public boolean matches(String value) { + // the message in the output string is correct + boolean messageIsCorrect = value.contains("Penalty: " + penalty); + + return messageIsCorrect; + } + }; + } + + public static Condition noPenalty() { + return not(containsString("Penalty: ")); + } + public static Condition finalGradeInXml(String reportDir, int score) { return new Condition<>() { @Override @@ -268,6 +284,15 @@ public static Condition codeCheck(String description, boolean pass, int return containsString(expectedCheck); } + public static Condition penaltyCodeCheck(String description, boolean pass, int weight) { + String expectedCheck = String.format("%s: %s (penalty: %d)", + description, + pass ? "PASS" : "FAIL", + weight); + + return containsString(expectedCheck); + } + public static Condition totalTimeItTookToExecute() { return containsRegex("took \\d+(.\\d)? seconds"); } @@ -294,16 +319,16 @@ public static Condition noMetaTests() { } - public static Condition noCodeChecks() { - return not(containsString("--- Code checks")); + public static Condition codeChecks() { + return containsString("--- Code checks"); } - public static Condition noRequiredCodeChecks() { - return not(containsString("--- Required code checks")); + public static Condition noCodeChecks() { + return not(codeChecks()); } - public static Condition requiredCodeChecksFailed() { - return containsString("Some required code checks failed. Stopping the assessment."); + public static Condition noPenaltyCodeChecks() { + return not(containsRegex(".*\\(penalty: \\d+\\)")); } public static Condition noFinalGrade() { diff --git a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java index 7e6835f8..c9a34ff5 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java @@ -284,7 +284,8 @@ void testPrintFinalGrade() { .has(mutationScore(5, 6)) .has(noMetaTests()) .has(noCodeChecks()) - .has(noRequiredCodeChecks()) + .has(noPenaltyCodeChecks()) + .has(noPenalty()) .has(not(zeroScoreExplanation())) .doesNotContain("test success message"); @@ -569,11 +570,11 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b new MetaTestResult("e", 3, false), new MetaTestResult("f", 1, true) )) - .withRequiredCodeCheckResults(List.of( + .withPenaltyCodeCheckResults(List.of( new CodeCheckResult("a1", 1, true), new CodeCheckResult("b1", 1, true), - new CodeCheckResult("c1", 1, true), - new CodeCheckResult("d1", 1, true) + new CodeCheckResult("c1", 100, true), + new CodeCheckResult("d1", 5, true) )) .build(); @@ -593,10 +594,10 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(fullGradeDescriptionDisplayed("Code checks", 3, 4, 0.25)) .has(fullGradeDescriptionDisplayed("Meta tests", 2, 3, 0.25)) .has(mutationScore(5, 6)) - .has(scoreOfCodeChecks(3, 4)) + .has(scoreOfCodeChecks(7, 8)) .has(metaTestsPassing(2)) .has(metaTests(3)) - .has(not(requiredCodeChecksFailed())) + .has(noPenalty()) .has(not(zeroScoreExplanation())); assertThat(output) @@ -606,10 +607,10 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(codeCheckDisplayed("a", true, 1, fullHints)) .has(codeCheckDisplayed("b", true, 2, fullHints)) .has(codeCheckDisplayed("c", false, 1, fullHints)) - .has(codeCheckDisplayed("a1", true, 1, fullHints)) - .has(codeCheckDisplayed("b1", true, 1, fullHints)) - .has(codeCheckDisplayed("c1", true, 1, fullHints)) - .has(codeCheckDisplayed("d1", true, 1, fullHints)); + .has(penaltyCodeCheckDisplayed("a1", true, 1, fullHints)) + .has(penaltyCodeCheckDisplayed("b1", true, 1, fullHints)) + .has(penaltyCodeCheckDisplayed("c1", true, 100, fullHints)) + .has(penaltyCodeCheckDisplayed("d1", true, 5, fullHints)); } @ParameterizedTest @@ -618,7 +619,7 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b "false,true", "true,true" }) - void testPrintFinalGradeWithRequiredCodeChecksFailing(boolean fullHints, boolean partialHints) { + void testPrintFinalGradeWithPenaltyCodeChecksFailing(boolean fullHints, boolean partialHints) { ModeActionSelector modeActionSelector = mock(ModeActionSelector.class); when(modeActionSelector.shouldCalculateAndShowGrades()).thenReturn(true); when(modeActionSelector.shouldGenerateAnalytics()).thenReturn(false); @@ -633,12 +634,13 @@ void testPrintFinalGradeWithRequiredCodeChecksFailing(boolean fullHints, boolean 4, 7, 5, 8, 1, 2, new CoverageLineByLine(List.of(), List.of(), List.of()))) .withMutationTestingResults(5, 6) - .withRequiredCodeCheckResults(List.of( - new CodeCheckResult("a1", 1, true), - new CodeCheckResult("b1", 1, false), - new CodeCheckResult("c1", 1, true), + .withPenaltyCodeCheckResults(List.of( + new CodeCheckResult("a1", 1, false), + new CodeCheckResult("b1", 100, false), + new CodeCheckResult("c1", 50, true), new CodeCheckResult("d1", 1, true) )) + .withPenalty(101) .build(); writer.write(ctx, result); @@ -650,15 +652,16 @@ void testPrintFinalGradeWithRequiredCodeChecksFailing(boolean fullHints, boolean .has(compilationSuccess()) .has(fullGradeDescriptionDisplayed("Code checks", 0, 0, 0.25)) .has(mutationScore(5, 6)) - .has(requiredCodeChecksFailed()) - .has(noCodeChecks()) + .has(penalty(101)) + .has(codeChecks()) + .has(scoreOfCodeChecks(2, 4)) .has(not(zeroScoreExplanation())); assertThat(output) - .has(codeCheckDisplayed("a1", true, 1, fullHints)) - .has(codeCheckDisplayed("b1", false, 1, fullHints)) - .has(codeCheckDisplayed("c1", true, 1, fullHints)) - .has(codeCheckDisplayed("d1", true, 1, fullHints)); + .has(penaltyCodeCheckDisplayed("a1", false, 1, fullHints)) + .has(penaltyCodeCheckDisplayed("b1", false, 100, fullHints)) + .has(penaltyCodeCheckDisplayed("c1", true, 50, fullHints)) + .has(penaltyCodeCheckDisplayed("d1", true, 1, fullHints)); } @Test @@ -853,6 +856,11 @@ protected Condition codeCheckDisplayed(String description, boole return shownInOutput ? codeCheckCondition : not(codeCheckCondition); } + protected Condition penaltyCodeCheckDisplayed(String description, boolean pass, int weight, boolean shownInOutput) { + var codeCheckCondition = penaltyCodeCheck(description, pass, weight); + return shownInOutput ? codeCheckCondition : not(codeCheckCondition); + } + protected Condition metaTestDisplayed(String description, boolean pass, boolean shownInOutput) { var metaTestCondition = pass ? metaTestPassing(description) : metaTestFailing(description); return shownInOutput ? metaTestCondition : not(metaTestCondition); diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecks.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecks.java index 523e8208..627f98ea 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecks.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecks.java @@ -39,10 +39,10 @@ public CheckScript checkScript() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) + new SingleCheck(100, "Trip Repository should be mocked required", new MockClass("TripRepository")), + new SingleCheck( 100, "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksExam.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksExam.java index 8a98f23f..b07f1ed8 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksExam.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksExam.java @@ -40,10 +40,10 @@ public CheckScript checkScript() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck( "Trip should be mocked required", new MockClass("Trip")), + new SingleCheck(100, "Trip should be mocked required", new MockClass("Trip")), new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksGrading.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksGrading.java index 8cc87d5b..79c6342c 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksGrading.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksGrading.java @@ -72,10 +72,10 @@ public List metaTests() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) + new SingleCheck(100, "Trip Repository should be mocked required", new MockClass("TripRepository")), + new SingleCheck( 5, "getTripById should not be set up penalty", true, new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksRequiredCodeCheckFailing.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing.java similarity index 87% rename from andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksRequiredCodeCheckFailing.java rename to andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing.java index 392b916e..e142cdea 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksRequiredCodeCheckFailing.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing.java @@ -39,11 +39,11 @@ public CheckScript checkScript() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck( "Trip should be mocked required", new MockClass("Trip")), - new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) + new SingleCheck(5, "Trip Repository should not be mocked penalty", true, new MockClass("TripRepository")), + new SingleCheck(3, "Trip should be mocked penalty", new MockClass("Trip")), + new SingleCheck( 10, "getTripById should be set up penalty", new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2.java new file mode 100644 index 00000000..7392f94a --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2.java @@ -0,0 +1,82 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.Comparison; +import nl.tudelft.cse1110.andy.codechecker.checks.MockClass; +import nl.tudelft.cse1110.andy.codechecker.checks.MockitoWhen; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.25f); + put("codechecks", 0.25f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SoftWhere"); + } + + @Override + public CheckScript checkScript() { + return new CheckScript(List.of( + new SingleCheck("Trip Repository should be mocked", new MockClass("TripRepository")), + new SingleCheck( "Trip should not be mocked", true, new MockClass("Trip")), + new SingleCheck( "getTripById should be set up", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public CheckScript penaltyCheckScript() { + return new CheckScript(List.of( + new SingleCheck(100, "Trip Repository should not be mocked penalty", true, new MockClass("TripRepository")), + new SingleCheck(30, "Trip should be mocked penalty", new MockClass("Trip")), + new SingleCheck( 10, "getTripById should be set up penalty", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("BoundaryCheck", + """ + if (capacityLeft(trip) < people.size()) + return false; + """, + """ + if (capacityLeft(trip) <= people.size()) + return false; + """), + MetaTest.withStringReplacement("DoesNotCheckCapacity", + """ + if (capacityLeft(trip) < people.size()) + return false; + """,""), + MetaTest.withLineReplacement("DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + MetaTest.withLineReplacement("DoesNotCheckSave", 154, 154, "") + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksWithoutRequiredChecks.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksWithoutPenaltyCodeChecks.java similarity index 100% rename from andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksWithoutRequiredChecks.java rename to andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigMetaAndCodeChecksWithoutPenaltyCodeChecks.java diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksFailingConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksFailingConfiguration.java similarity index 81% rename from andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksFailingConfiguration.java rename to andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksFailingConfiguration.java index 60dc417c..d88d2141 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksFailingConfiguration.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksFailingConfiguration.java @@ -38,11 +38,11 @@ public CheckScript checkScript() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck(1, "Trip should be mocked required", new MockClass("Trip")), - new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) + new SingleCheck("Trip Repository should be mocked penalty", new MockClass("TripRepository")), + new SingleCheck(100, "Trip should be mocked required", new MockClass("Trip")), + new SingleCheck(200, "getTripById should be set up req", new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksPassingConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksPassingConfiguration.java similarity index 86% rename from andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksPassingConfiguration.java rename to andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksPassingConfiguration.java index 0241b8e9..763e9e67 100644 --- a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithRequiredCodeChecksPassingConfiguration.java +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigWithPenaltyCodeChecksPassingConfiguration.java @@ -38,10 +38,10 @@ public CheckScript checkScript() { } @Override - public CheckScript requiredCheckScript() { + public CheckScript penaltyCheckScript() { return new CheckScript(List.of( - new SingleCheck("Trip Repository should be mocked required", new MockClass("TripRepository")), - new SingleCheck( "getTripById should be set up required", new MockitoWhen("getTripById", Comparison.GTE, 1)) + new SingleCheck("Trip Repository should be mocked penalty", new MockClass("TripRepository")), + new SingleCheck( "getTripById should be set up penalty", new MockitoWhen("getTripById", Comparison.GTE, 1)) )); } diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index d40b96b3..0c853056 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -100,7 +100,7 @@ private static void appendMetaScoreElements(Result result, Document doc, Element appendMetaScore(doc, metaElement, "Meta tests", result.getMetaTests().getPassedMetaTests()); result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); - result.getRequiredCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); + result.getPenaltyCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); testSuitesElement.appendChild(metaElement);