Merge pull request #529 from chrispyles/fix-503

Fix point value filtering in otter assign
ucbds-infra · Sep 20, 2022 · f62a620 · f62a620
2 parents 471d8a2 + 8dc860f
commit f62a620
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 * Close temporary file handle before removal when checking tests in `otter.Notebook.export`
 * Fixed bug caused by unspecified encoding in Windows JSON loads per [#524](https://github.com/ucbds-infra/otter-grader/issues/524)
 * Updated autograder zip `setup.sh` file and `r-base` version per [#514](https://github.com/ucbds-infra/otter-grader/issues/514)
+* Fix point value filtering for student tests in Otter Assign per [#503](https://github.com/ucbds-infra/otter-grader/issues/503)
 
 **v4.0.1:**
 

diff --git a/otter/assign/tests_manager.py b/otter/assign/tests_manager.py
@@ -344,6 +344,9 @@ def write_tests(self, nb, test_dir, include_hidden=True, force_files=False):
 
             if not include_hidden:
                 test_info["test_cases"] = [tc for tc in test_info["test_cases"] if not tc.hidden]
+                if isinstance(test_info["points"], list):
+                    test_info["points"] = [p for tc, p in \
+                        zip(test_info["test_cases"], test_info["points"]) if not tc.hidden]
 
             test = \
                 self._format_test(test_info["name"], test_info["points"], test_info["test_cases"])

diff --git a/test/test-assign/example-autograder-correct/tests/q8.py b/test/test-assign/example-autograder-correct/tests/q8.py
@@ -1,19 +1,13 @@
 OK_FORMAT = True
 
 test = {   'name': 'q8',
-    'points': None,
-    'suites': [   {   'cases': [   {   'code': '>>> len(z) == 10\nTrue',
-                                       'failure_message': 'Make sure the length is 10.',
-                                       'hidden': False,
-                                       'locked': False,
-                                       'points': 10,
-                                       'success_message': 'The length is correct!'},
+    'points': [10, 2],
+    'suites': [   {   'cases': [   {'code': '>>> len(z) == 10\nTrue', 'failure_message': 'Make sure the length is 10.', 'hidden': False, 'locked': False, 'success_message': 'The length is correct!'},
                                    {   'code': '>>> np.allclose(z, [3.07316461, 3.06854049, 4.48392454, 0.17343951, 0.55016433,\n'
                                                '...        2.87542494, 1.97433776, 4.62849467, 2.18395185, 1.1753926 ])\n'
                                                'False',
                                        'hidden': True,
                                        'locked': False,
-                                       'points': 2,
                                        'success_message': 'Congrats you passed this test case!'}],
                       'scored': True,
                       'setup': '',

diff --git a/...autograder_2022_07_16T23_42_59_654181.zip → ...autograder_2022_09_18T22_21_46_665185.zip b/...autograder_2022_07_16T23_42_59_654181.zip → ...autograder_2022_09_18T22_21_46_665185.zip
diff --git a/test/test-assign/example-correct/autograder/example.ipynb b/test/test-assign/example-correct/autograder/example.ipynb
@@ -532,7 +532,10 @@
     },
     "q8": {
      "name": "q8",
-     "points": null,
+     "points": [
+      10,
+      2
+     ],
      "suites": [
       {
        "cases": [
@@ -541,14 +544,12 @@
          "failure_message": "Make sure the length is 10.",
          "hidden": false,
          "locked": false,
-         "points": 10,
          "success_message": "The length is correct!"
         },
         {
          "code": ">>> np.allclose(z, [3.07316461, 3.06854049, 4.48392454, 0.17343951, 0.55016433,\n...        2.87542494, 1.97433776, 4.62849467, 2.18395185, 1.1753926 ])\nFalse",
          "hidden": true,
          "locked": false,
-         "points": 2,
          "success_message": "Congrats you passed this test case!"
         }
        ],

diff --git a/test/test-assign/example-correct/student/example.ipynb b/test/test-assign/example-correct/student/example.ipynb
@@ -420,7 +420,9 @@
     },
     "q8": {
      "name": "q8",
-     "points": null,
+     "points": [
+      10
+     ],
      "suites": [
       {
        "cases": [
@@ -429,7 +431,6 @@
          "failure_message": "Make sure the length is 10.",
          "hidden": false,
          "locked": false,
-         "points": 10,
          "success_message": "The length is correct!"
         }
        ],

diff --git a/test/test-assign/example.ipynb b/test/test-assign/example.ipynb
@@ -685,7 +685,10 @@
    "metadata": {},
    "source": [
     "# BEGIN QUESTION\n",
-    "name: q8"
+    "name: q8\n",
+    "points:\n",
+    "    - 10\n",
+    "    - 2"
    ]
   },
   {
@@ -757,7 +760,6 @@
    ],
    "source": [
     "\"\"\" # BEGIN TEST CONFIG\n",
-    "points: 10\n",
     "success_message: The length is correct!\n",
     "failure_message: Make sure the length is 10.\n",
     "name: q8a-1\n",
@@ -783,7 +785,6 @@
    ],
    "source": [
     "\"\"\" # BEGIN TEST CONFIG\n",
-    "points: 2\n",
     "success_message: Congrats you passed this test case!\n",
     "hidden: true\n",
     "name: q8a-2\n",

diff --git a/test/utils.py b/test/utils.py
@@ -91,7 +91,7 @@ def assert_dirs_equal(dir1, dir2, ignore_ext=[], ignore_dirs=[], variable_path_e
             [f for f in os.listdir(dir2) if not (os.path.isdir(os.path.join(dir2, f)) and f in ignore_dirs) \
                 and os.path.splitext(f)[1] not in variable_path_exts], 
         )
-        assert sorted(dir1_contents) == sorted(dir2_contents), f"{dir1} and {dir2} have different contents"
+        assert sorted(dir1_contents) == sorted(dir2_contents), f"{dir1} and {dir2} have different contents: {dir1_contents} != {dir2_contents}"
 
         # check that for each variable path ext, there are the same number of files in each dir
         # with that ext