Merge branch 'release/3.13.0'

.github/workflows/auto-merge-dependency-updates.yaml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/[email protected].4
+        uses: dependabot/[email protected].5
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve

.github/workflows/crowdin-actions.yaml

@@ -19,12 +19,12 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Upload or update source files to Crowdin
-      uses: crowdin/[email protected].0
+      uses: crowdin/[email protected].2
       with:
         upload_sources: true
 
     - name: Download German translations
-      uses: crowdin/[email protected].0
+      uses: crowdin/[email protected].2
       with:
         upload_sources: false
         download_translations: true
@@ -42,7 +42,7 @@ jobs:
         config: crowdin.yaml
 
     - name: Download Spanish translations
-      uses: crowdin/[email protected].0
+      uses: crowdin/[email protected].2
       with:
         upload_sources: false
         download_translations: true

.github/workflows/test-and-deploy.yaml

@@ -19,6 +19,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Create Docker network
         run: docker network create uccser-development-stack
+        # Required for the node service
+      - name: Set DOCKER_UID variable
+        run: echo "DOCKER_UID=$(echo $UID)" >> $GITHUB_ENV
       - name: Start systems
         run: docker compose -f docker-compose.local.yml up -d
       - name: Run Django system check
@@ -32,6 +35,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Create Docker network
         run: docker network create uccser-development-stack
+        # Required for the node service
+      - name: Set DOCKER_UID variable
+        run: echo "DOCKER_UID=$(echo $UID)" >> $GITHUB_ENV
       - name: Start systems
         run: docker compose -f docker-compose.local.yml up -d
       - name: Create static files
@@ -125,6 +131,9 @@ jobs:
 
       - name: Create Docker network
         run: docker network create uccser-development-stack
+        # Required for the node service
+      - name: Set DOCKER_UID variable
+        run: echo "DOCKER_UID=$(echo $UID)" >> $GITHUB_ENV
 
       - name: Start system
         run: docker compose -f docker-compose.local.yml up -d
@@ -174,6 +183,10 @@ jobs:
       - name: Create Docker network
         run: docker network create uccser-development-stack
 
+        # Required for the node service
+      - name: Set DOCKER_UID variable
+        run: echo "DOCKER_UID=$(echo $UID)" >> $GITHUB_ENV
+
       - name: Start system
         run: docker compose -f docker-compose.local.yml up -d
 

csfieldguide/chapters/content/en/coding-compression/sections/shannons-experiment.md

@@ -0,0 +1,62 @@
+# Shannon's experiment
+
+It turns out that there are limits to how small we can compress a file, and to explore this we’re going to look at multi-million dollar frauds, and a fun game that exposes the limits of compression.
+
+Every so often someone claims to have invented an amazing {glossary-link term="lossless"}lossless{glossary-link end} compression method that can compress *any* file, including compressed files. If that was true, it would mean that you could use the method to compress files down to just a few {glossary-link term="byte"}bytes{glossary-link end}. Any file could be downloaded in a fraction of a second, and computers could store billions of huge video files. It would revolutionise computing! But is there a limit on how small a file can be compressed?
+
+{panel type="curiosity"}
+
+# Fraud in data compression
+
+A number of fake systems have been produced that claim to compress any file as small as you want.
+They have even been demonstrated! But they all turn out to be a fake system - the common trick is to have a “compression” program that actually just hides the file being compressed somewhere else on a computer, and replaces it with a tiny file. The decompression program copies the hidden file back. It’s a very simple program to write, and looks very impressive because files are replaced with tiny ones, and then reproduced exactly.
+
+You can find a few examples of these if you search for “Pixelon”, “Adam’s platform”, “Near Zero”, or “Madison Priest” (add the terms “compression” and “fraud” if you are searching for these, as there are other legitimate organisations with similar names). Several of these organisations have taken millions of dollars from investors who didn’t understand the limits of compression, and all ended up failing.
+
+{panel end}
+
+## How small can we compress a file?
+
+With {glossary-link term="lossy"}lossy{glossary-link end} compression, there isn’t a limit to how small you can compress a file, since it’s just a matter of giving up quality to make the file smaller. You could compress a 10-megapixel photo down to just one pixel (perhaps the average colour of the whole photo). It wouldn’t be much use to anyone, but technically it’s a lossy version of the original photo.
+
+But with lossless compression, the original file needs to be able to be restored to exactly its original form.
+
+We’ve seen that compression works by taking advantage of patterns in the data being compressed. 
+In the 1950s an interesting experiment was developed by a scientist called Claude Shannon, in which he asked humans to predict English text, and he measured how good the compression would be using their ability to make predictions. 
+The idea is that if a computer was as good at English as a human, then that might be near the limit of what is possible.
+
+Shannon’s game is easy to play.
+Just click on the letter that you think is coming up next in the sentence (you’ll need to start by guessing the first letter). The number of guesses you make give an indication of how predictable the letter is. 
+These guesses are used to estimate how small the data could be compressed -- you can see this estimate by clicking on the “Show statistics” button.
+The “bits per character” is the estimate of how many bits would be needed on average to represent each character. 
+Plain English text is often stored in 7 or 8 bits for each character (using Unicode or ASCII), and you should find that using your predictions the experiment can do better than that, usually around 2 bits per character.
+That’s equivalent to compressing a normal file (8 bits per character) to a quarter of its size.
+
+Try it here:
+
+{interactive slug="shannon-experiment" type="whole-page" alt="Shannon's experiment"}
+
+But it’s very hard to get smaller than 1 bit per character (one eighth of the normal size). 
+Shannon found that this seems to be a limit for how much we can compress English text.
+And this is one reason that we should be suspicious of any system that claims to compress English text to much smaller than one eighth of its original size.
+
+{panel type="teacher-note"}
+
+# Creating your own experiment
+
+This interactive contains an option to create your own experiment. 
+For example, this could be used to tailor the sentence set to use words that are more familiar to your students.
+
+Additionally we have support for multiple different languages and sentence sets. 
+Currently, we have a sentence set for Te Reo Māori, and the original sentences used by Shannon in 1951. 
+
+If you would like to use another language with a different set of characters and/or accents, this also works!
+When creating a custom sentence, any characters that aren't already on the keyboard get added automatically.
+
+Lastly, you could also considering using a pattern that is easily guessed once they realise what is happening, such as "AAAAAAAAAAAAAAAA", "ABABABABABAB", or "blah blah blah blah blah blah blah blah blah blah".
+These have very close to zero information content as they are very predictable.
+At the other extreme, a (fake) passowrd such as "P6dQKg#S58dw66p" could be used to explore how hard it is to guess random characters.
+
+{panel end}
+
+{comment - could add more about Shannon, model at sender and received, movie about him}

csfieldguide/chapters/content/structure/coding-compression/sections/sections.yaml

@@ -19,8 +19,11 @@ general-purpose:
 audio-compression:
   section-number: 7
 
-the-whole-story:
+shannons-experiment:
   section-number: 8
 
-further-reading:
+the-whole-story:
   section-number: 9
+
+further-reading:
+  section-number: 10

csfieldguide/chapters/management/commands/_ChapterSectionHeadingsLoader.py

@@ -39,13 +39,16 @@ def load(self):
 
         """
         for language, content in self.content_translations.items():
+            i = 0
             if content.heading_tree:
-                for (i, heading_node) in enumerate(content.heading_tree):
+                for heading_node in content.heading_tree:
                     self.chapter_section.headings.update_or_create(
-                        slug=heading_node.title_slug,
+                        number=i,
+                        language=language,
                         defaults={
                             'name': heading_node.title,
-                            'language': language,
-                            'number': i,
+                            'slug': heading_node.title_slug
                         }
                     )
+                    i += 1
+            self.chapter_section.headings.filter(number__gte=i, language=language).delete()

csfieldguide/chapters/management/commands/_ChapterSectionsLoader.py

@@ -33,10 +33,9 @@ def load(self):
                 field.
         """
         chapter_sections_structure = self.load_yaml_file(self.structure_file_path)
-        section_numbers = []
+        next_section_number = 1
 
         for (section_slug, section_structure) in chapter_sections_structure.items():
-
             if section_structure is None:
                 raise MissingRequiredFieldError(
                     self.structure_file_path,
@@ -57,8 +56,15 @@ def load(self):
                     "section-number - value '{}' is invalid".format(section_number),
                     "section-number must be an integer value."
                 )
+            if section_number != next_section_number:
+                raise InvalidYAMLValueError(
+                    self.structure_file_path,
+                    "section-number - value '{}' is invalid".format(section_number),
+                    "section-numbers must be in sequential order. The next expected number was '{}'."
+                        .format(next_section_number)
+                )
 
-            section_numbers.append(section_number)
+            next_section_number += 1
 
             chapter_section_translations = self.get_blank_translation_dictionary()
 
@@ -69,9 +75,9 @@ def load(self):
                 chapter_section_translations[language]["name"] = content.title
 
             chapter_section, created = self.chapter.chapter_sections.update_or_create(
-                slug=section_slug,
+                number=section_number,
                 defaults={
-                    'number': section_number,
+                    'slug': section_slug,
                     'languages': list(content_translations.keys()),
                 }
             )
@@ -101,11 +107,4 @@ def load(self):
                 structure_filename=self.structure_file_path,
             ).load()
 
-        # assumes first section number is always 1
-        for counter, section_number in enumerate(section_numbers, 1):
-            if section_number != counter:
-                raise InvalidYAMLValueError(
-                    self.structure_file_path,
-                    "section-number - value '{}' is invalid".format(section_number),
-                    "section-numbers must be in sequential order. The next expected number was '{}'.".format(counter)
-                )
+        self.chapter.chapter_sections.filter(number__gte=next_section_number).delete()

csfieldguide/chapters/management/commands/_ChaptersLoader.py

@@ -69,9 +69,9 @@ def load(self):
 
         # Create or update chapter object and save to the db
         chapter, created = Chapter.objects.update_or_create(
-            slug=self.chapter_slug,
+            number=self.chapter_number,
             defaults={
-                'number': self.chapter_number,
+                'slug': self.chapter_slug,
                 'icon': chapter_icon,
                 'video': video
             }

csfieldguide/chapters/management/commands/loadchapters.py

@@ -3,6 +3,7 @@
 import os.path
 from django.core.management.base import BaseCommand
 from django.conf import settings
+from django.db import transaction
 from utils.BaseLoader import BaseLoader
 from utils.LoaderFactory import LoaderFactory
 from utils.errors.MissingRequiredFieldError import MissingRequiredFieldError
@@ -19,6 +20,7 @@ class Command(BaseCommand):
 
     help = "Converts Markdown files listed in structure file and stores"
 
+    @transaction.atomic
     def handle(self, *args, **options):
         """Automatically called when the loadchapters command is given."""
         factory = LoaderFactory()
@@ -51,6 +53,7 @@ def handle(self, *args, **options):
                 "Application Structure"
             )
         else:
+            next_chapter_number = 1
             for chapter_slug in chapters:
                 chapter_structure_file = "{}.yaml".format(chapter_slug)
 
@@ -67,6 +70,16 @@ def handle(self, *args, **options):
                         "chapter-number - value '{}' is invalid".format(chapter_number),
                         "chapter-number must be an integer value."
                     )
+                if chapter_number != next_chapter_number:
+                    raise InvalidYAMLValueError(
+                        structure_file_path,
+                        "chapter-number - value '{}' is invalid".format(chapter_number),
+                        ("chapter-numbers must be in sequential order. The next expected number was '{}'."
+                            .format(next_chapter_number))
+                    )
+
+                next_chapter_number += 1
+
                 factory.create_chapter_loader(
                     base_path=base_path,
                     content_path=chapter_slug,

csfieldguide/chapters/migrations/0037_chapter_slug_deferred.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-24 02:49
+
+from django.db import migrations, models
+import django.db.models.constraints
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chapters', '0036_alter_chaptersection_options'),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name='chapter',
+            constraint=models.UniqueConstraint(deferrable=django.db.models.constraints.Deferrable['DEFERRED'], fields=('slug',), name='slug_deferred'),
+        ),
+    ]

csfieldguide/chapters/migrations/0038_alter_chapter_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-24 03:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chapters', '0037_chapter_slug_deferred'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='chapter',
+            name='slug',
+            field=models.SlugField(),
+        ),
+    ]

csfieldguide/chapters/models.py

@@ -2,7 +2,6 @@
 
 from django.db import models
 from interactives.models import Interactive
-from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 from utils.TranslatableModel import TranslatableModel
 from django.urls import reverse
@@ -54,7 +53,7 @@ class Chapter(TranslatableModel):
     """Model for chapter in database."""
 
     #  Auto-incrementing 'id' field is automatically set by Django
-    slug = models.SlugField(unique=True)
+    slug = models.SlugField()  # This is set unique in the Meta child class
     name = models.CharField(max_length=100, default="")
     number = models.SmallIntegerField(unique=True)
     introduction = models.TextField(default="")
@@ -104,6 +103,10 @@ class Meta:
         verbose_name = _("chapter")
         verbose_name_plural = _("chapters")
 
+        constraints = [
+            models.UniqueConstraint(fields=["slug"], deferrable=models.Deferrable.DEFERRED, name="slug_deferred")
+        ]
+
 
 class ChapterSection(TranslatableModel):
     """Model for each section in a chapter in database."""
@@ -128,24 +131,6 @@ def __str__(self):
         """
         return self.name
 
-    def clean(self):
-        """Use to check for unique section numbers.
-
-        Raises:
-            ValidationError: when the section being added uses
-                an existing section number for this chapter.
-        """
-        # get all sections with same section number and chapter as new section being added
-        sections = ChapterSection.objects.filter(number=self.number, chapter=self.chapter)
-        # if already exists section with same number in same chapter, then throw error!
-        if len(sections) > 1:
-            raise ValidationError(('Section number must be unique per chapter.'))
-
-    def save(self, *args, **kwargs):
-        """Override save method to validate unique section numbers."""
-        super(ChapterSection, self).save(*args, **kwargs)
-        self.clean()
-
     class Meta:
         """Set consistent ordering of chapter sections."""
 

csfieldguide/config/__init__.py

@@ -1,3 +1,3 @@
 """Module for Django system configuration."""
 
-__version__ = "3.12.6"
+__version__ = "3.13.0"