Query and display 'best' evaluation for any given submission

- Make the leaderboard show Submissions, rather than Evaluations - Simplify leaderboard to avoid REST API calls. No pagination will be done for now. - Add "ordering" and "ordering_priority" to EVALUATION_DISPLAY_CONFIG to determine the ordering of evaluations when scoring submission versions - Add datatables for dynamic sorting of both submissions & versions - Display the 'best' evaluation for any given submission in submission list Ref #177
2i2c-org · Nov 2, 2024 · c8f8121 · c8f8121
1 parent 96b2ec9
commit c8f8121
Show file tree

Hide file tree

Showing 8 changed files with 196 additions and 93 deletions.
diff --git a/frx_challenges/frx_challenges/settings.py b/frx_challenges/frx_challenges/settings.py
@@ -262,8 +262,19 @@
 }
 
 EVALUATION_DISPLAY_CONFIG = [
-    {"result_key": "chars", "display_name": "Characters"},
-    {"result_key": "lines", "display_name": "Lines"},
+    {
+        "result_key": "chars",
+        "display_name": "Characters",
+        # ordering options are "smaller_is_better" and "bigger_is_better"
+        "ordering": "smaller_is_better",
+        "ordering_priority": 2
+    },
+    {
+        "result_key": "lines",
+        "display_name": "Lines",
+        "ordering": "bigger_is_better",
+        "ordering_priority": 1
+    }
 ]
 
 django_yamlconf.load()

diff --git a/frx_challenges/web/models.py b/frx_challenges/web/models.py
@@ -6,7 +6,8 @@
 from django.contrib.auth.models import User
 from django.db import models, transaction
 from django_jsonform.models.fields import JSONField
-
+from django.db.models.fields.json import KT
+from typing import Optional, List
 # Create your models here.
 
 
@@ -40,6 +41,47 @@ class Submission(models.Model):
         blank=True, null=True, schema=settings.SITE_SUBMISSION_FORM_SCHEMA
     )
 
+    @property
+    def best_version(self) -> Version:
+        """
+        Return the 'best' evaluated version for this Submission
+        """
+        # Construct a query that returns evaluations that:
+        # 1. Belong to a version that belong to this submission
+        # 2. Have been succesfully evaluated
+        # 3. Have results that contain all the keys we use for ordering
+        # 4. Ordered by the ordering criteria expressed by EVALUATION_DISPLAY_CONFIG
+
+
+        # Sort the display_config by ordering_priority (smaller numbers go first)
+        # This primarily is used for tiebreaking, since we only want the 'best'
+        sorted_display_config = sorted(settings.EVALUATION_DISPLAY_CONFIG, key=lambda dc: -dc["ordering_priority"])
+
+        # Ordering criteria for querying our results
+        ordering_criteria = []
+        # List of keys that *must* be present in a result for it to count
+        result_must_have_keys = []
+
+        for dc in sorted_display_config:
+            result_must_have_keys.append(dc['result_key'])
+            k = KT(f"result__{dc['result_key']}")
+            if dc["ordering"] == "smaller_is_better":
+                k = k.asc()
+            elif dc["ordering"] == "bigger_is_better":
+                k = k.desc()
+            else:
+                raise ValueError(f"Invalid ordering {dc['ordering']} found for result_key {dc['result_key']}")
+            ordering_criteria.append(k)
+
+        best_evaluation = Evaluation.objects.filter(
+            version__submission=self,
+            status=Evaluation.Status.EVALUATED,
+            result__has_keys=result_must_have_keys
+        ).order_by(*ordering_criteria).first()
+
+        if best_evaluation:
+            return best_evaluation.version
+
 
 class Version(models.Model):
     """
@@ -52,7 +94,7 @@ class Status(models.TextChoices):
         UPLOADED = "UPLOADED"
         CLEARED = "CLEARED"
 
-    submission = models.ForeignKey(Submission, on_delete=models.CASCADE)
+    submission = models.ForeignKey(Submission, on_delete=models.CASCADE, related_name="versions")
     date_created = models.DateTimeField(auto_now=True)
     # FIXME: Cascade is probably not quite right?
     user = models.ForeignKey(User, on_delete=models.CASCADE)

diff --git a/frx_challenges/web/templates/results.html b/frx_challenges/web/templates/results.html
@@ -25,65 +25,60 @@ <h1>Explanatory Headline</h1>
     <div class="container py-2">
         <div class="row py-2">
             <div class="col py-2">
-                <table class="table" id="results">
-                    <thead>
-                    </thead>
-                    <tbody>
-                    </tbody>
-                </table>
-            </div>
+              <table id="results" class="table table-striped table-sm">
+                <thead>
+                    <tr>
+                        <th scope="col">ID</th>
+                        <th scope="col">Name</th>
+                        <th scope="col">Description</th>
+                        <th scope="col">Date created</th>
+                        {% for dc in evaluation_display_config %}
+                        <th scope="col">{{ dc.display_name }}</th>
+                        {% endfor %}
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for result in results %}
+                        <tr>
+                            <td>{{ result.submission.id }}</td>
+                            <td>
+                                <a href='{% url "submissions-detail" result.submission.id %}'>{{ result.submission.name }}</a>
+                            </td>
+                            <td>{{ result.submission.description }}</td>
+                            <td>{{ result.submission.date_created|date:"c" }}</td>
+                            {% for r in result.best_version.latest_evaluation.ordered_results %}
+                            <td scope="col">
+                                {% if r %}
+                                {{ r }}
+                                {% endif %}
+                            </td>
+                            {% endfor %}
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+          </div>
         </div>
     </div>
     <script>
-    dayjs.extend(window.dayjs_plugin_relativeTime);
-    async function main() {
-      const resp = await fetch("/results");
-      const respData = await resp.json();
-
-      const displayConfig = respData["display_config"];
-      const evaluations = respData["results"];
-
-      const columns = [
-        { data: "username", title: "username" },
-        { data: "status", title: "status" },
-        {
-          data: "last_updated",
-          name: "last_updated",
-          title: "last updated",
-          type: "date",
-          render: (data) => {
-            return dayjs(data).fromNow();
-          },
-        },
-      ].concat(
-        displayConfig.map((dc) => {
-          return {
-            title: dc.display_name,
-            data: (row) => {
-              if (row.result && dc.result_key in row.result) {
-                return row.result[dc.result_key];
-              } else {
-                return "";
-              }
-            },
-          };
-        }),
-      );
-
-      let table = new DataTable("#results", {
-        data: evaluations,
-        columns: columns,
-        order: {
-          name: "last_updated",
-          dir: "desc",
-        },
-      });
-
-      setInterval(() => {
-        table.ajax.reload();
-      }, 600 * 1000);
-    }
+      async function main() {
+          dayjs.extend(window.dayjs_plugin_relativeTime);
+          const resultsTable = new DataTable("#results", {
+              order: [
+                  // Apply reverse chronological ordering by default
+                  [3, "desc"]
+              ],
+              columnDefs: [
+                  {
+                      targets: 3,
+                      render: (data) => {
+                          return dayjs(data).fromNow();
+                      }
+                  }
+              ]
+          });
+      }
 
-    main();
-    </script>
+      main();
+      </script>
 {% endblock body %}
diff --git a/frx_challenges/web/templates/submission/detail.html b/frx_challenges/web/templates/submission/detail.html
@@ -1,11 +1,13 @@
 {% extends "page.html" %}
 {% block head %}
     <script src="https://code.jquery.com/jquery-3.7.1.min.js"
-            integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo="
-            crossorigin="anonymous"></script>
+        integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo="
+        crossorigin="anonymous"></script>
     <link rel="stylesheet"
-          href="https://cdn.datatables.net/2.0.8/css/dataTables.dataTables.css" />
+        href="https://cdn.datatables.net/2.0.8/css/dataTables.dataTables.css" />
     <script src="https://cdn.datatables.net/2.0.8/js/dataTables.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/dayjs@1/dayjs.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/dayjs@1/plugin/relativeTime.js"></script>
 {% endblock head %}
 {% block body %}
     <div class="container py-2">
@@ -39,7 +41,7 @@ <h1>{{ submission.name }}</h1>
                             <tr>
                                 <td>{{ v.id }}</td>
                                 <td>{{ v.filename }}</td>
-                                <td>{{ v.date_created|date:"M j Y" }}</td>
+                                <td>{{ v.date_created|date:"c" }}</td>
                                 <td>{{ v.latest_evaluation.status }}</td>
                                 {% for r in v.latest_evaluation.ordered_results %}
                                     <td scope="col">
@@ -61,7 +63,21 @@ <h1>{{ submission.name }}</h1>
         </div>
         <script>
         async function main() {
-            const resultsTable = new DataTable("#results");
+            dayjs.extend(window.dayjs_plugin_relativeTime);
+            const resultsTable = new DataTable("#results", {
+                order: [
+                    // Apply reverse chronological ordering by default
+                    [2, "desc"]
+                ],
+                columnDefs: [
+                    {
+                        targets: 2,
+                        render: (data) => {
+                            return dayjs(data).fromNow();
+                        }
+                    }
+                ]
+            });
         }
 
         main();

diff --git a/frx_challenges/web/templates/submission/list.html b/frx_challenges/web/templates/submission/list.html
@@ -33,13 +33,16 @@ <h1>My Submissions</h1>
                 </div>
             </div>
             <div class="row py-2">
-                <table class="table table-striped table-sm">
+                <table id="results" class="table table-striped table-sm">
                     <thead>
                         <tr>
                             <th scope="col">ID</th>
                             <th scope="col">Name</th>
                             <th scope="col">Description</th>
                             <th scope="col">Date created</th>
+                            {% for dc in evaluation_display_config %}
+                            <th scope="col">{{ dc.display_name }}</th>
+                            {% endfor %}
                         </tr>
                     </thead>
                     <tbody>
@@ -50,7 +53,14 @@ <h1>My Submissions</h1>
                                     <a href={{ sub.id }}>{{ sub.name }}</a>
                                 </td>
                                 <td>{{ sub.description }}</td>
-                                <td>{{ sub.date_created|date:"M j Y" }}</td>
+                                <td>{{ sub.date_created|date:"c" }}</td>
+                                {% for r in sub.best_version.latest_evaluation.ordered_results %}
+                                <td scope="col">
+                                    {% if r %}
+                                    {{ r }}
+                                    {% endif %}
+                                </td>
+                                {% endfor %}
                             </tr>
                         {% endfor %}
                     </tbody>
@@ -64,4 +74,26 @@ <h1>My Submissions</h1>
             </div>
         {% endif %}
     </div>
+
+    <script>
+    async function main() {
+        dayjs.extend(window.dayjs_plugin_relativeTime);
+        const resultsTable = new DataTable("#results", {
+            order: [
+                // Apply reverse chronological ordering by default
+                [3, "desc"]
+            ],
+            columnDefs: [
+                {
+                    targets: 3,
+                    render: (data) => {
+                        return dayjs(data).fromNow();
+                    }
+                }
+            ]
+        });
+    }
+
+    main();
+    </script>
 {% endblock body %}
diff --git a/frx_challenges/web/urls.py b/frx_challenges/web/urls.py
@@ -4,7 +4,6 @@
 
 urlpatterns = [
     path("upload/<int:id>", default.upload, name="upload"),
-    path("results", default.results, name="results"),
     path("teams/list", teams.list, name="teams-list"),
     path("teams/create", teams.create, name="teams-create"),
     path("teams/<int:id>", teams.view, name="teams-view"),

diff --git a/frx_challenges/web/views/default.py b/frx_challenges/web/views/default.py
@@ -54,32 +54,40 @@ def upload(request: HttpRequest, id: int) -> HttpResponse:
     )
 
 
-def results(request: HttpRequest) -> HttpResponse:
-    evaluations = Evaluation.objects.all()
-
-    evaluations_resp = {
-        "display_config": settings.EVALUATION_DISPLAY_CONFIG,
-        "results": [],
-    }
-
-    for ev in evaluations:
-        evaluations_resp["results"].append(
-            {
-                "evaluation_id": ev.id,
-                "username": ev.version.user.username,
-                "status": ev.status,
-                "last_updated": ev.last_updated.isoformat(),
-                "result": ev.result,
-            }
-        )
-
-    return JsonResponse(evaluations_resp)
-
-
 def leaderboard(request: HttpRequest) -> HttpResponse:
     if settings.CHALLENGE_STATE != "RUNNING":
         return HttpResponse(
             "Challenge hasn't started, so leaderboard is not available", status=400
         )
 
-    return render(request, "results.html")
+    sorted_display_config = sorted(settings.EVALUATION_DISPLAY_CONFIG, key=lambda dc: -dc["ordering_priority"])
+    results = []
+    all_submissions = Submission.objects.all()
+    for sub in all_submissions:
+        bv = sub.best_version
+        if not bv:
+            # Only display submissions with at least one 'best version'
+            continue
+        results.append({
+            "submission": sub,
+            "best_version": bv
+        })
+
+    def sort_key_func(r):
+        bv: Version = r["best_version"]
+        sort_key = []
+        for dc in sorted_display_config:
+            if dc["ordering"] == "smaller_is_better":
+                sort_key.append(-bv.latest_evaluation.result[dc["result_key"]])
+            elif dc["ordering"] == "bigger_is_better":
+                sort_key.append(bv.latest_evaluation.result[dc["result_key"]])
+            else:
+                raise ValueError(f"Invalid ordering {dc['ordering']} found for result_key {dc['result_key']}")
+
+        return sort_key
+
+    results = sorted(
+        results,
+        key=sort_key_func
+    )
+    return render(request, "results.html", {"results":results})
diff --git a/frx_challenges/web/views/submissions.py b/frx_challenges/web/views/submissions.py
@@ -60,7 +60,7 @@ def detail(request: HttpRequest, id: int) -> HttpResponse:
         submission = queryset.get(id=id)
     except Submission.DoesNotExist:
         raise Http404("Submission does not exist")
-    versions = submission.version_set.all()
+    versions = submission.versions.all()
     return render(
         request,
         "submission/detail.html",