From f3f5e451955fcf44a206e0417b8ad1732f991f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 20 Jun 2024 00:14:15 +0200 Subject: [PATCH 01/23] fix: nightly version suffix --- tutor/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/__about__.py b/tutor/__about__.py index 8c37f09911..db3c8ea853 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -10,7 +10,7 @@ # the nightly branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "" +__version_suffix__ = "nightly" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed From 30fa1eac7e8603f57bf3a4d9666a861eb64bcfc8 Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Tue, 25 Jun 2024 18:46:47 +0500 Subject: [PATCH 02/23] chore: ensure plugins are installed from nightly branches in nightly --- requirements/plugins.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 2c4be2fe12..907eb9f610 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,14 +1,14 @@ -# change version ranges when upgrading from redwood -tutor-android>=18.0.0,<19.0.0 -tutor-cairn>=18.0.0,<19.0.0 -tutor-credentials>=18.0.0,<19.0.0 -tutor-discovery>=18.0.0,<19.0.0 -tutor-ecommerce>=18.0.0,<19.0.0 -tutor-forum>=18.0.0,<19.0.0 -tutor-indigo>=18.0.0,<19.0.0 -tutor-jupyter>=18.0.0,<19.0.0 -tutor-mfe>=18.0.0,<19.0.0 -tutor-minio>=18.0.0,<19.0.0 -tutor-notes>=18.0.0,<19.0.0 -tutor-webui>=18.0.0,<19.0.0 -tutor-xqueue>=18.0.0,<19.0.0 +# # For Tutor Nightly, we install plugins from their nightly branches instead of from PyPI +tutor-android@git+https://github.com/overhangio/tutor-android@nightly +tutor-cairn@git+https://github.com/overhangio/tutor-cairn@nightly +tutor-credentials@git+https://github.com/overhangio/tutor-credentials@nightly +tutor-discovery@git+https://github.com/overhangio/tutor-discovery@nightly +tutor-ecommerce@git+https://github.com/overhangio/tutor-ecommerce@nightly +tutor-forum@git+https://github.com/overhangio/tutor-forum@nightly +tutor-indigo@git+https://github.com/overhangio/tutor-indigo@nightly +tutor-jupyter@git+https://github.com/overhangio/tutor-jupyter@nightly +tutor-mfe@git+https://github.com/overhangio/tutor-mfe@nightly +tutor-minio@git+https://github.com/overhangio/tutor-minio@nightly +tutor-notes@git+https://github.com/overhangio/tutor-notes@nightly +tutor-webui@git+https://github.com/overhangio/tutor-webui@nightly +tutor-xqueue@git+https://github.com/overhangio/tutor-xqueue@nightly From 0f2229b181c584c760442be6f063ea77f120a41e Mon Sep 17 00:00:00 2001 From: Danyal Faheem <138459282+Danyal-Faheem@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:10:10 +0500 Subject: [PATCH 03/23] fix: upgrade MySQL from 5.7 to 8.1 first and then to 8.4 This is required when upgrading from Tutor v15 to v18 directly MySQL does not allow direct upgrades from v5.7 to v8.4 --- ..._danyal.faheem_mysql_upgrade_5_7_to_8_4.md | 2 ++ tutor/commands/upgrade/common.py | 33 ++++++++++++++++++- tutor/commands/upgrade/compose.py | 22 ++++++++++++- tutor/commands/upgrade/k8s.py | 15 +++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md diff --git a/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md b/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md new file mode 100644 index 0000000000..01275c273d --- /dev/null +++ b/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md @@ -0,0 +1,2 @@ +- [Bugfix] Do not directly upgrade MySQL from v5.7 to v8.4 when upgrading from quince as MySQL does not allow that. First, upgrade to v8.1 and then to v8.4. (by @Danyal-Faheem) + This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security. \ No newline at end of file diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index e641a77001..783be6435d 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -1,10 +1,13 @@ from __future__ import annotations +from typing import Optional + import click +from packaging import version from tutor import config as tutor_config from tutor import fmt, plugins -from tutor.types import Config +from tutor.types import Config, get_typed def upgrade_from_lilac(config: Config) -> None: @@ -60,6 +63,34 @@ def get_mongo_upgrade_parameters( return mongo_version, admin_command +def get_intermediate_mysql_upgrade(config: Config) -> Optional[str]: + """ + Checks if a MySQL upgrade is needed based on the Tutor version and MySQL setup. + + This method ensures that MySQL is running and determines if the upgrade + process should proceed based on the Tutor version. It is intended for upgrades + from Tutor version 15 to version 18 or later. Manual upgrade steps are not + required for versions 16 or 17. + + Returns: + Optional[str]: The docker image of MySQL to upgrade to or None if not applicable + """ + if not get_typed(config, "RUN_MYSQL", bool): + fmt.echo_info( + "You are not running MySQL (RUN_MYSQL=false). It is your " + "responsibility to upgrade your MySQL instance to v8.4. There is " + "nothing left to do to upgrade from Olive." + ) + return None + image_tag = get_typed(config, "DOCKER_IMAGE_MYSQL", str).split(":")[-1] + # If latest image, we assign a constant value to invalidate the condition + # as we know that the latest image will always be greater than 8.1.0 + target_version = ( + version.Version("8.1.1") if image_tag == "latest" else version.parse(image_tag) + ) + return "docker.io/mysql:8.1.0" if target_version > version.parse("8.1.0") else None + + PALM_RENAME_ORA2_FOLDER_COMMAND = """ if stat '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' 2> /dev/null; then echo "Renaming ora2 folder..." diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index df54c71d8b..aedbd2159f 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -5,7 +5,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor import fmt -from tutor.commands import compose +from tutor.commands import compose, jobs from tutor.types import Config from . import common as common_upgrade @@ -158,6 +158,26 @@ def upgrade_from_olive(context: click.Context, config: Config) -> None: upgrade_mongodb(context, config, "4.2.17", "4.2") upgrade_mongodb(context, config, "4.4.22", "4.4") + intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade( + config + ) + if not intermediate_mysql_docker_image: + return + + click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}")) + + # Revert the MySQL image to build the data dictionary on v8.1 + mysql_docker_image = config["DOCKER_IMAGE_MYSQL"] + config["DOCKER_IMAGE_MYSQL"] = intermediate_mysql_docker_image + tutor_env.save(context.obj.root, config) + # Run the init command to make sure MySQL is ready for connections + context.invoke(jobs.initialise, limit="mysql") + + # Change the image back to v8.4 + config["DOCKER_IMAGE_MYSQL"] = mysql_docker_image + tutor_env.save(context.obj.root, config) + context.invoke(compose.stop, services=["mysql"]) + def upgrade_from_quince(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Quince")) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index db18835704..6be26096ea 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -165,6 +165,21 @@ def upgrade_from_olive(context: Context, config: Config) -> None: upgrade_mongodb(config, "4.2.17", "4.2") upgrade_mongodb(config, "4.4.22", "4.4") + intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade( + config + ) + if not intermediate_mysql_docker_image: + return + + message = f"""Automatic release upgrade is unsupported in Kubernetes. If you are upgrading from Olive or an earlier release to Redwood, you will have to first upgrade MySQL to v8.1 and then to v8.4. To upgrade, run the following commands: + + tutor k8s stop + tutor config save --set DOCKER_IMAGE_MYSQL=docker.io/mysql:8.1.0 + tutor k8s start + tutor config save --unset DOCKER_IMAGE_MYSQL + """ + fmt.echo_info(message) + def upgrade_from_quince(config: Config) -> None: click.echo(fmt.title("Upgrading from Quince")) From 6843f142fdcef43a54a4bae11df39e3ac8aa09f9 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 2 Oct 2024 12:50:16 -0400 Subject: [PATCH 04/23] feat: create volume for private LMS/CMS media assets (#1124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content Libraries will store asset files like images, transcripts, and PDFs using Learning Core–which in turn uses django-storages and expects some backing file store. These files should NOT be directly accessible via the browser, both because of access policies and the fact that the filenames will not be meaningful by themselves and must be translated by app logic. For details see: * https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0015-serving-static-assets.rst * https://github.com/openedx/openedx-learning/blob/main/openedx_learning/apps/authoring/contents/models.py * https://github.com/openedx/openedx-learning/blob/main/openedx_learning/apps/authoring/components/models.py The existing /data/openedx-media -> /openedx/media mount is publicly accessible by browser, and so is not appropriate for this purpose. This commit creates a parallel /data/openedx-media-private -> /openedx/media-private volume mapping instead. This commit also creates configuration needed for openedx-learning to point to this new directory. This commit does NOT currently add support for this in k8s, as that will require changes to the minio plugin. --- changelog.d/20240926_151223_dave_private_media.md | 1 + .../apps/openedx/settings/partials/common_all.py | 9 +++++++++ tutor/templates/local/docker-compose.yml | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 changelog.d/20240926_151223_dave_private_media.md diff --git a/changelog.d/20240926_151223_dave_private_media.md b/changelog.d/20240926_151223_dave_private_media.md new file mode 100644 index 0000000000..d7d3d16581 --- /dev/null +++ b/changelog.d/20240926_151223_dave_private_media.md @@ -0,0 +1 @@ +- [Feature] Create a new /data/openedx-media-private volume to store media files that are not publicly accessible via the browser, and create configuration that Learning Core (openedx-learning) will use to access this storage space. Learning Core will use a sub-directory of /data/openedx-media-private to store Content Library file uploads, but other apps can create their own sub-directories for their own use. (by @ormsbee) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index c317f4c06f..105ac5383c 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -246,5 +246,14 @@ "user": None, } +OPENEDX_LEARNING = { + 'MEDIA': { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": "/openedx/media-private/openedx-learning", + } + } +} + {{ patch("openedx-common-settings") }} ######## End of settings common to LMS and CMS diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 63c2d3c7d3..eddc29cb81 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -15,6 +15,7 @@ services: - ../../data/lms:/mounts/lms - ../../data/cms:/mounts/cms - ../../data/openedx-media:/mounts/openedx + - ../../data/openedx-media-private:/mounts/openedx-private {% if RUN_MONGODB %}- ../../data/mongodb:/mounts/mongodb{% endif %} {% if RUN_MYSQL %}- ../../data/mysql:/mounts/mysql{% endif %} {% if RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} @@ -112,6 +113,7 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/uwsgi.ini:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + - ../../data/openedx-media-private:/openedx/media-private {%- for mount in iter_mounts(MOUNTS, "openedx", "lms") %} - {{ mount }} {%- endfor %} @@ -138,6 +140,7 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/uwsgi.ini:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + - ../../data/openedx-media-private:/openedx/media-private {%- for mount in iter_mounts(MOUNTS, "openedx", "cms") %} - {{ mount }} {%- endfor %} @@ -166,6 +169,7 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + - ../../data/openedx-media-private:/openedx/media-private {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %} - {{ mount }} {%- endfor %} @@ -185,6 +189,7 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + - ../../data/openedx-media-private:/openedx/media-private {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %} - {{ mount }} {%- endfor %} From a49c5a5709ec4e716fc0c26104640c3c4fe3aff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 10 Oct 2024 13:21:04 +0200 Subject: [PATCH 05/23] feat: migrate from *.local.edly.io to *.local.openedx.io This makes it more clear for everyone that Tutor is an Open edX distribution. Close #1120 --- changelog.d/20241010_131903_regis_local_openedx_io.md | 1 + docs/dev.rst | 6 +++--- docs/tutorials/theming.rst | 4 ++-- tutor/interactive.py | 5 +++-- tutor/templates/local/docker-compose.prod.yml | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20241010_131903_regis_local_openedx_io.md diff --git a/changelog.d/20241010_131903_regis_local_openedx_io.md b/changelog.d/20241010_131903_regis_local_openedx_io.md new file mode 100644 index 0000000000..60b1316ff6 --- /dev/null +++ b/changelog.d/20241010_131903_regis_local_openedx_io.md @@ -0,0 +1 @@ +- πŸ’₯[Feature] Migrate from local.edly.io to local.openedx.io. (by @regisb) diff --git a/docs/dev.rst b/docs/dev.rst index 4d6c2249e2..5095b29c6e 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -28,7 +28,7 @@ This will perform several tasks. It will: * build the "openedx-dev" Docker image, which is based on the "openedx" production image but is `specialized for developer usage`_ (eventually with your fork), * stop any existing locally-running Tutor containers, * disable HTTPS, -* set ``LMS_HOST`` to `local.edly.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), +* set ``LMS_HOST`` to `local.openedx.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), * prompt for a platform details (with suitable defaults), * start LMS, CMS, supporting services, and any plugged-in services, * ensure databases are created and migrated, and @@ -42,8 +42,8 @@ Additionally, when a local clone of edx-platform is bind-mounted, it will: Once setup is complete, the platform will be running in the background: -* LMS will be accessible at `http://local.edly.io:8000 `_. -* CMS will be accessible at `http://studio.local.edly.io:8001 `_. +* LMS will be accessible at `http://local.openedx.io:8000 `_. +* CMS will be accessible at `http://studio.local.openedx.io:8001 `_. * Plugged-in services should be accessible at their documented URLs. Now, use the ``tutor dev ...`` command-line interface to manage the development environment. Some common commands are described below. diff --git a/docs/tutorials/theming.rst b/docs/tutorials/theming.rst index 6c98384612..7993a2b7f2 100644 --- a/docs/tutorials/theming.rst +++ b/docs/tutorials/theming.rst @@ -46,7 +46,7 @@ Then, run a local webserver:: tutor dev start lms -The LMS can then be accessed at http://local.edly.io:8000. You will then have to :ref:`enable that theme `:: +The LMS can then be accessed at http://local.openedx.io:8000. You will then have to :ref:`enable that theme `:: tutor dev do settheme mythemename @@ -54,4 +54,4 @@ Watch the themes folders for changes (in a different terminal):: tutor dev run watchthemes -Make changes to some of the files inside the theme directory: the theme assets should be automatically recompiled and visible at http://local.edly.io:8000. +Make changes to some of the files inside the theme directory: the theme assets should be automatically recompiled and visible at http://local.openedx.io:8000. diff --git a/tutor/interactive.py b/tutor/interactive.py index 5e059a005f..34a47ef489 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -28,6 +28,7 @@ def ask_questions( defaults = tutor_config.get_defaults() if run_for_prod is None: run_for_prod = not config.get("LMS_HOST") in [ + "local.openedx.io", "local.edly.io", "local.overhang.io", ] @@ -41,8 +42,8 @@ def ask_questions( ) if not run_for_prod: dev_values: Config = { - "LMS_HOST": "local.edly.io", - "CMS_HOST": "studio.local.edly.io", + "LMS_HOST": "local.openedx.io", + "CMS_HOST": "studio.local.openedx.io", "ENABLE_HTTPS": False, } fmt.echo_info( diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index 1e5606cbf7..63de8fb7df 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -19,7 +19,7 @@ services: networks: default: # These aliases are for internal communication between containers when running locally - # with *.local.edly.io hostnames. + # with *.local.openedx.io hostnames. aliases: - "{{ LMS_HOST }}" {{ patch("local-docker-compose-caddy-aliases")|indent(10) }} From 7e68baa89f5efad979321c60b65718af862d2684 Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali <40599381+DawoudSheraz@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:47:45 +0500 Subject: [PATCH 06/23] feat: update Open edX image to use ubuntu 22.04 as base OS (#1117) * feat: update Open edX image to use ubuntu 22.04 as base OS --- .github/workflows/release.yml | 2 +- Makefile | 2 +- .../20241010_122752_dawoud.sheraz_ubuntu_upgrade.md | 2 ++ tutor/templates/build/openedx/Dockerfile | 10 +++++++--- 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d23908ac6..4d55506ea2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: # It's important that we build the tutor binaries with the *oldest* possible # OS releases and Python version. See these docs for more information: # https://pyinstaller.org/en/stable/usage.html#making-gnu-linux-apps-forward-compatible - - os: ubuntu-20.04 + - os: ubuntu-22.04 locale: C.UTF-8 # https://endoflife.date/macos - os: macos-12 diff --git a/Makefile b/Makefile index b1755a5dc0..a92d0ba396 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ bootstrap-dev-plugins: bootstrap-dev ## Install dev requirements and all support pip install -r requirements/plugins.txt pull-base-images: # Manually pull base images - docker image pull docker.io/ubuntu:20.04 + docker image pull docker.io/ubuntu:22.04 ci-info: ## Print info about environment python --version diff --git a/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md b/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md new file mode 100644 index 0000000000..9a1adf3c98 --- /dev/null +++ b/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md @@ -0,0 +1,2 @@ +- πŸ’₯[Feature] Update Open edX Image to use Ubuntu 22.04 as base OS. (by @dawoudsheraz) + - Adds xmlsec related dependencies to fix xmlsec import issues during translations build diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 621b463422..9ab4234a86 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,9 +1,10 @@ # syntax=docker/dockerfile:1 ###### Minimal image with base system requirements for most stages -FROM docker.io/ubuntu:20.04 AS minimal +FROM docker.io/ubuntu:22.04 AS minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive + RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && \ @@ -19,7 +20,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev python-openssl git + xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git # Install pyenv # https://www.python.org/downloads/ @@ -78,7 +79,10 @@ ENV XDG_CACHE_HOME=/openedx/.cache RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update \ - && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev + && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev \ + # Install xmlsec dependencies + libxml2-dev \ + libxmlsec1-openssl # Install the right version of pip/setuptools RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ From 75aab6e233f1a24e7a70ab2715149b8ad526a978 Mon Sep 17 00:00:00 2001 From: Danyal Faheem <138459282+Danyal-Faheem@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:09:13 +0500 Subject: [PATCH 07/23] fix: run MySQL 8.1 as a separate container during upgrade from Olive to Redwood (#1140) * ix: run MySQL 8.1 as a separate container during upgrade from Olive to Redwood We do this because MySQL 8.1 does not have the --mysql-native-password option We have this option turned on for backward compatibility --- ...eem_run_mysql_8_1_as_separate_container.md | 1 + tutor/commands/upgrade/compose.py | 50 +++++- tutor/commands/upgrade/k8s.py | 145 ++++++++++++++++-- 3 files changed, 179 insertions(+), 17 deletions(-) create mode 100644 changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md diff --git a/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md b/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md new file mode 100644 index 0000000000..a4b5403c75 --- /dev/null +++ b/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md @@ -0,0 +1 @@ +- [Bugfix] Run MySQL 8.1 as a separate container during upgrade from Olive to Redwood as it crashed otherwise due to the `--mysql-native-password` option not being present. (by @Danyal-Faheem) \ No newline at end of file diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index aedbd2159f..4f8f7773d8 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -4,6 +4,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env +from tutor import hooks from tutor import fmt from tutor.commands import compose, jobs from tutor.types import Config @@ -166,16 +167,53 @@ def upgrade_from_olive(context: click.Context, config: Config) -> None: click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}")) - # Revert the MySQL image to build the data dictionary on v8.1 - mysql_docker_image = config["DOCKER_IMAGE_MYSQL"] - config["DOCKER_IMAGE_MYSQL"] = intermediate_mysql_docker_image + # We start up a mysql-8.1 container to build data dictionary to preserve + # the upgrade order of 5.7 -> 8.1 -> 8.4 + # Use the mysql-8.1 context so that we can clear these filters later on + with hooks.Contexts.app("mysql-8.1").enter(): + hooks.Filters.ENV_PATCHES.add_items( + [ + ( + "local-docker-compose-services", + """ +mysql-8.1: + extends: mysql + image: docker.io/mysql:8.1.0 + command: > + mysqld + --character-set-server=utf8mb3 + --collation-server=utf8mb3_general_ci + --binlog-expire-logs-seconds=259200 + """, + ), + ( + "local-docker-compose-jobs-services", + """ +mysql-8.1-job: + image: docker.io/mysql:8.1.0 + depends_on: {{ [("mysql-8.1", RUN_MYSQL)]|list_if }} + """, + ), + ] + ) + hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-8.1")) + + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("mysql-8.1", tutor_env.read_core_template_file("jobs", "init", "mysql.sh")) + ) + tutor_env.save(context.obj.root, config) + # Run the init command to make sure MySQL is ready for connections - context.invoke(jobs.initialise, limit="mysql") + context.invoke(jobs.initialise, limit="mysql-8.1") + context.invoke(compose.stop, services=["mysql-8.1"]) - # Change the image back to v8.4 - config["DOCKER_IMAGE_MYSQL"] = mysql_docker_image + # Clear the filters added for mysql-8.1 as we don't need them anymore + hooks.clear_all(context="app:mysql-8.1") + + # Save environment and run init for mysql 8.4 to make sure MySQL is ready tutor_env.save(context.obj.root, config) + context.invoke(jobs.initialise, limit="mysql") context.invoke(compose.stop, services=["mysql"]) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 6be26096ea..e547bf4b80 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -2,7 +2,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt +from tutor import fmt, hooks from tutor.commands import k8s from tutor.commands.context import Context from tutor.types import Config @@ -39,7 +39,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None: running_release = "olive" if running_release == "olive": - upgrade_from_olive(context.obj, config) + upgrade_from_olive(context, config) running_release = "palm" if running_release == "palm": @@ -148,11 +148,11 @@ def upgrade_from_maple(context: Context, config: Config) -> None: ) -def upgrade_from_olive(context: Context, config: Config) -> None: +def upgrade_from_olive(context: click.Context, config: Config) -> None: # Note that we need to exec because the ora2 folder is not bind-mounted in the job # services. k8s.kubectl_apply( - context.root, + context.obj.root, "--selector", "app.kubernetes.io/name=lms", ) @@ -171,14 +171,137 @@ def upgrade_from_olive(context: Context, config: Config) -> None: if not intermediate_mysql_docker_image: return - message = f"""Automatic release upgrade is unsupported in Kubernetes. If you are upgrading from Olive or an earlier release to Redwood, you will have to first upgrade MySQL to v8.1 and then to v8.4. To upgrade, run the following commands: + click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}")) + + # We start up a mysql-8.1 container to build data dictionary to preserve + # the upgrade order of 5.7 -> 8.1 -> 8.4 + # Use the mysql-8.1 context so that we can clear these filters later on + with hooks.Contexts.app("mysql-8.1").enter(): + hooks.Filters.ENV_PATCHES.add_items( + [ + ( + "k8s-deployments", + """ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql-81 + labels: + app.kubernetes.io/name: mysql-81 +spec: + selector: + matchLabels: + app.kubernetes.io/name: mysql-81 + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: mysql-81 + spec: + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + fsGroupChangePolicy: "OnRootMismatch" + containers: + - name: mysql-81 + image: docker.io/mysql:8.1.0 + args: + - "mysqld" + - "--character-set-server=utf8mb3" + - "--collation-server=utf8mb3_general_ci" + - "--binlog-expire-logs-seconds=259200" + env: + - name: MYSQL_ROOT_PASSWORD + value: "{{ MYSQL_ROOT_PASSWORD }}" + ports: + - containerPort: 3306 + volumeMounts: + - mountPath: /var/lib/mysql + name: data + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: data + persistentVolumeClaim: + claimName: mysql + """, + ), + ( + "k8s-jobs", + """ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mysql-81-job + labels: + app.kubernetes.io/component: job +spec: + template: + spec: + restartPolicy: Never + containers: + - name: mysql-81 + image: docker.io/mysql:8.1.0 + """, + ), + ] + ) + hooks.Filters.ENV_PATCHES.add_item( + ( + "k8s-services", + """ +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql-81 + labels: + app.kubernetes.io/name: mysql-81 +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + selector: + app.kubernetes.io/name: mysql-81 + """, + ) + ) + hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-81")) - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MYSQL=docker.io/mysql:8.1.0 - tutor k8s start - tutor config save --unset DOCKER_IMAGE_MYSQL - """ - fmt.echo_info(message) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("mysql-81", tutor_env.read_core_template_file("jobs", "init", "mysql.sh")) + ) + + tutor_env.save(context.obj.root, config) + + # Run the init command to make sure MySQL is ready for connections + k8s.kubectl_apply( + context.obj.root, + "--selector", + "app.kubernetes.io/name=mysql-81", + ) + k8s.wait_for_deployment_ready(config, "mysql-81") + context.invoke(k8s.do.commands["init"], limit="mysql-8.1") + context.invoke(k8s.stop, names=["mysql-81"]) + + # Clear the filters added for mysql-8.1 as we don't need them anymore + hooks.clear_all(context="app:mysql-8.1") + + # Save environment and run init for mysql 8.4 to make sure MySQL is ready + tutor_env.save(context.obj.root, config) + k8s.kubectl_apply( + context.obj.root, + "--selector", + "app.kubernetes.io/name=mysql", + ) + k8s.wait_for_deployment_ready(config, "mysql") + context.invoke(k8s.do.commands["init"], limit="mysql") + context.invoke(k8s.stop, names=["mysql"]) def upgrade_from_quince(config: Config) -> None: From d4e6c6c6e55363c59f84808c022cb3052cc88b44 Mon Sep 17 00:00:00 2001 From: Danyal Faheem <138459282+Danyal-Faheem@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:21:43 +0500 Subject: [PATCH 08/23] feat: add do command to change MySQL charset to utf8mb4 (#1079) --- ...131_danyal.faheem_upgrade_mysql_utf8mb4.md | 7 + docs/local.rst | 27 ++++ docs/tutorials/datamigration.rst | 2 + tests/commands/test_jobs.py | 75 ++++++++++ tutor/commands/jobs.py | 106 ++++++++++++++ tutor/commands/jobs_utils.py | 133 ++++++++++++++++++ tutor/commands/upgrade/compose.py | 4 + 7 files changed, 354 insertions(+) create mode 100644 changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md create mode 100644 tutor/commands/jobs_utils.py diff --git a/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md b/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md new file mode 100644 index 0000000000..33d9d84c1b --- /dev/null +++ b/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md @@ -0,0 +1,7 @@ +- πŸ’₯[Feature] Upgrade default charset and collation of mysql to utf8mb4 and utf8mb4_unicode_ci respectively (by @Danyal-Faheem) + - Add do command to upgrade the charset and collation of tables in mysql. + - The command will perform the following upgrades: + - Upgrade all `utf8mb3` charset to `utf8mb4` + - Upgrade collation `utf8mb3_general_ci` to `utf8mb4_unicode_ci` + - Upgrade collation `utf8mb3_bin` to `utf8mb4_bin` + - Upgrade collation `utf8mb3_*` to `utf8mb4_*` \ No newline at end of file diff --git a/docs/local.rst b/docs/local.rst index 7f3ebcb307..997c61ffa2 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -141,6 +141,33 @@ The default Open edX theme is rather bland, so Tutor makes it easy to switch to Out of the box, only the default "open-edx" theme is available. We also developed `Indigo, a beautiful, customizable theme `__ which is easy to install with Tutor. +Changing the mysql charset and collation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This command has been tested only for users upgrading from Quince. While it is expected to work for users on earlier releases, please use it with caution as it has not been tested with those versions. + +The database's charset and collation might not support specific characters or emojis. Tutor will function normally without this change unless specific characters are used in the instance. + +.. warning:: This change is potentially irreversible. It is recommended to make a backup of the MySQL database. See the :ref:`database dump instructions ` to create a DB dump. + +To change the charset and collation of all the tables in the openedx database, run:: + + tutor local do convert-mysql-utf8mb4-charset + +Alternatively, to change the charset and collation of certain tables or to exclude certain tables, the ``--include`` or ``--exclude`` options can be used. These options take comma separated names of tables/apps with no space in-between. To upgrade the ``courseware_studentmodule`` and ``courseware_studentmodulehistory`` tables, run:: + + tutor local do convert-mysql-utf8mb4-charset --include=courseware_studentmodule,courseware_studentmodulehistory + +Tutor performs pattern matching from the start of the table name so just the name of the app is enough to include/exclude all the tables under that app. To upgrade all the tables in the database except the ones under the student and wiki apps, run:: + + tutor local do convert-mysql-utf8mb4-charset --exclude=student,wiki + +In the above command, all the tables whose name starts with either student or wiki will be excluded from the upgrade process. + +By default, only the tables in the openedx database are changed. For upgrading tables in any additional databases used by plugins, the ``--database`` option can be used to upgrade them. To upgrade all the tables in the discovery database, run:: + + tutor local do convert-mysql-utf8mb4-charset --database=discovery + Running arbitrary ``manage.py`` commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/tutorials/datamigration.rst b/docs/tutorials/datamigration.rst index e7923a6463..63e4506568 100644 --- a/docs/tutorials/datamigration.rst +++ b/docs/tutorials/datamigration.rst @@ -22,6 +22,8 @@ With Tutor, all data are stored in a single folder. This means that it's extreme tutor local start -d +.. _database_dumps: + Making database dumps --------------------- diff --git a/tests/commands/test_jobs.py b/tests/commands/test_jobs.py index 2ab388773b..d519629c83 100644 --- a/tests/commands/test_jobs.py +++ b/tests/commands/test_jobs.py @@ -90,3 +90,78 @@ def test_set_theme(self) -> None: self.assertIn("lms-job", dc_args) self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1]) self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1]) + + def test_convert_mysql_utf8mb4_charset_all_tables(self) -> None: + with temporary_root() as root: + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root( + root, + [ + "local", + "do", + "convert-mysql-utf8mb4-charset", + "--non-interactive", + ], + ) + dc_args, _dc_kwargs = mock_docker_compose.call_args + + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("lms-job", dc_args) + self.assertIn("utf8mb4", dc_args[-1]) + self.assertIn("openedx", dc_args[-1]) + self.assertIn("utf8mb4_unicode_ci", dc_args[-1]) + self.assertNotIn("regexp", dc_args[-1]) + + def test_convert_mysql_utf8mb4_charset_include_tables(self) -> None: + with temporary_root() as root: + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root( + root, + [ + "local", + "do", + "convert-mysql-utf8mb4-charset", + "--include=courseware_studentmodule,xblock", + ], + ) + dc_args, _dc_kwargs = mock_docker_compose.call_args + + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("lms-job", dc_args) + self.assertIn("openedx", dc_args[-1]) + self.assertIn("utf8mb4", dc_args[-1]) + self.assertIn("utf8mb4_unicode_ci", dc_args[-1]) + self.assertIn("regexp", dc_args[-1]) + self.assertIn("courseware_studentmodule", dc_args[-1]) + self.assertIn("xblock", dc_args[-1]) + + def test_convert_mysql_utf8mb4_charset_exclude_tables(self) -> None: + with temporary_root() as root: + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root( + root, + [ + "local", + "do", + "convert-mysql-utf8mb4-charset", + "--database=discovery", + "--exclude=course,auth", + ], + ) + dc_args, _dc_kwargs = mock_docker_compose.call_args + + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("lms-job", dc_args) + self.assertIn("utf8mb4", dc_args[-1]) + self.assertIn("utf8mb4_unicode_ci", dc_args[-1]) + self.assertIn("discovery", dc_args[-1]) + self.assertIn("regexp", dc_args[-1]) + self.assertIn("NOT", dc_args[-1]) + self.assertIn("course", dc_args[-1]) + self.assertIn("auth", dc_args[-1]) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 7510a83b31..21fbbe5c43 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -13,6 +13,8 @@ from tutor import config as tutor_config from tutor import env, fmt, hooks +from tutor.commands.context import Context +from tutor.commands.jobs_utils import get_mysql_change_charset_query from tutor.hooks import priorities @@ -315,6 +317,109 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: yield ("lms", command) +@click.command( + short_help="Convert the charset and collation of mysql to utf8mb4.", + help=( + "Convert the charset and collation of mysql to utf8mb4. You can either upgrade all tables, specify only certain tables to upgrade or specify certain tables to exclude from the upgrade process" + ), + context_settings={"ignore_unknown_options": True}, +) +@click.option( + "--include", + is_flag=False, + nargs=1, + help="Apps/Tables to include in the upgrade process. Requires comma-seperated values with no space in-between.", +) +@click.option( + "--exclude", + is_flag=False, + nargs=1, + help="Apps/Tables to exclude from the upgrade process. Requires comma-seperated values with no space in-between.", +) +@click.option( + "--database", + is_flag=False, + nargs=1, + default="{{ OPENEDX_MYSQL_DATABASE }}", + show_default=True, + required=True, + type=str, + help="The database of which the tables are to be upgraded", +) +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.pass_obj +def convert_mysql_utf8mb4_charset( + context: Context, + include: str, + exclude: str, + database: str, + non_interactive: bool, +) -> t.Iterable[tuple[str, str]]: + """ + Do command to upgrade the charset and collation of tables in MySQL + + Can specify whether to upgrade all tables, or include certain tables/apps or to exclude certain tables/apps + """ + + config = tutor_config.load(context.root) + + if not config["RUN_MYSQL"]: + fmt.echo_info( + f"You are not running MySQL (RUN_MYSQL=false). It is your " + f"responsibility to upgrade the charset and collation of your MySQL instance." + ) + return + + # Prompt user for confirmation of upgrading all tables + if not include and not exclude and not non_interactive: + upgrade_all_tables = click.confirm( + "Are you sure you want to upgrade all tables? This process is potentially irreversible and may take a long time.", + prompt_suffix=" ", + ) + if not upgrade_all_tables: + return + + charset_to_upgrade_from = "utf8mb3" + charset = "utf8mb4" + collation = "utf8mb4_unicode_ci" + + query_to_append = "" + if include or exclude: + + def generate_query_to_append(tables: list[str], exclude: bool = False) -> str: + include = "NOT" if exclude else "" + table_names = f"^{tables[0]}" + for i in range(1, len(tables)): + table_names += f"|^{tables[i]}" + # We use regexp for pattern matching the names from the start of the tablename + query_to_append = f"AND table_name {include} regexp '{table_names}' " + return query_to_append + + query_to_append += ( + generate_query_to_append(include.split(",")) if include else "" + ) + query_to_append += ( + generate_query_to_append(exclude.split(","), exclude=True) + if exclude + else "" + ) + click.echo( + fmt.title( + f"Updating charset and collation of tables in the {database} database to {charset} and {collation} respectively." + ) + ) + query = get_mysql_change_charset_query( + database, charset, collation, query_to_append, charset_to_upgrade_from + ) + + mysql_command = ( + "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --skip-column-names --silent " + + shlex.join([f"--database={database}", "-e", query]) + ) + yield ("lms", mysql_command) + fmt.echo_info("MySQL charset and collation successfully upgraded") + + def add_job_commands(do_command_group: click.Group) -> None: """ This is meant to be called with the `local/dev/k8s do` group commands, to add the @@ -390,6 +495,7 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None: hooks.Filters.CLI_DO_COMMANDS.add_items( [ + convert_mysql_utf8mb4_charset, createuser, importdemocourse, importdemolibraries, diff --git a/tutor/commands/jobs_utils.py b/tutor/commands/jobs_utils.py new file mode 100644 index 0000000000..e0d7dbbb0d --- /dev/null +++ b/tutor/commands/jobs_utils.py @@ -0,0 +1,133 @@ +""" +This module provides utility methods for tutor `do` commands + +Methods: +- `get_mysql_change_charset_query`: Generates MySQL queries to upgrade the charset and collation of columns, tables, and databases. +""" + + +def get_mysql_change_charset_query( + database: str, + charset: str, + collation: str, + query_to_append: str, + charset_to_upgrade_from: str, +) -> str: + """ + Helper function to generate the mysql query to upgrade the charset and collation of columns, tables, and databases + + Utilized in the `tutor local do convert-mysql-utf8mb4-charset` command + """ + return f""" + + DROP PROCEDURE IF EXISTS UpdateColumns; + DELIMITER $$ + + CREATE PROCEDURE UpdateColumns() + BEGIN + + DECLARE done_columns_loop INT DEFAULT FALSE; + DECLARE _table_name VARCHAR(255); + DECLARE _table_name_copy VARCHAR(255) DEFAULT ""; + DECLARE _column_name VARCHAR(255); + DECLARE _column_type VARCHAR(255); + DECLARE _collation_name VARCHAR(255); + + # We explicitly upgrade the utf8mb3_general_ci collations to utf8mb4_unicode_ci + # The other collations are upgraded from utf8mb3_* to utf8mb4_* + # For any other collation, we leave it as it is + DECLARE columns_cur CURSOR FOR + SELECT + TABLE_NAME, + COLUMN_NAME, + COLUMN_TYPE, + CASE + WHEN COLLATION_NAME LIKE CONCAT('{charset_to_upgrade_from}', '_general_ci') THEN 'utf8mb4_unicode_ci' + WHEN COLLATION_NAME LIKE CONCAT('{charset_to_upgrade_from}', '_%') THEN CONCAT('{charset}', SUBSTRING_INDEX(COLLATION_NAME, '{charset_to_upgrade_from}', -1)) + ELSE COLLATION_NAME + END AS COLLATION_NAME + FROM + INFORMATION_SCHEMA.COLUMNS + WHERE + TABLE_SCHEMA = '{database}' + AND COLLATION_NAME IS NOT NULL {query_to_append}; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done_columns_loop = TRUE; + OPEN columns_cur; + columns_loop: LOOP + FETCH columns_cur INTO _table_name, _column_name, _column_type, _collation_name; + + IF done_columns_loop THEN + LEAVE columns_loop; + END IF; + + # First, upgrade the default charset and collation of the table + If _table_name <> _table_name_copy THEN + select _table_name; + SET FOREIGN_KEY_CHECKS = 0; + SET @stmt = CONCAT('ALTER TABLE `', _table_name, '` CONVERT TO CHARACTER SET {charset} COLLATE {collation};'); + PREPARE query FROM @stmt; + EXECUTE query; + DEALLOCATE PREPARE query; + SET FOREIGN_KEY_CHECKS = 1; + SET _table_name_copy = _table_name; + END IF; + + # Then, upgrade the default charset and collation of each column + # This sequence of table -> column is necessary to preserve column defaults + SET FOREIGN_KEY_CHECKS = 0; + SET @statement = CONCAT('ALTER TABLE `', _table_name, '` MODIFY `', _column_name, '` ', _column_type,' CHARACTER SET {charset} COLLATE ', _collation_name, ';'); + PREPARE query FROM @statement; + EXECUTE query; + DEALLOCATE PREPARE query; + SET FOREIGN_KEY_CHECKS = 1; + + END LOOP; + CLOSE columns_cur; + + END$$ + + DELIMITER ; + + DROP PROCEDURE IF EXISTS UpdateTables; + DELIMITER $$ + + CREATE PROCEDURE UpdateTables() + # To upgrade the default character set and collation of any tables that were skipped from the previous procedure + BEGIN + + DECLARE done INT DEFAULT FALSE; + DECLARE table_name_ VARCHAR(255); + DECLARE cur CURSOR FOR + SELECT table_name FROM information_schema.tables + WHERE table_schema = '{database}' AND table_type = "BASE TABLE" AND table_collation not like 'utf8mb4_%' {query_to_append}; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + OPEN cur; + tables_loop: LOOP + FETCH cur INTO table_name_; + + IF done THEN + LEAVE tables_loop; + END IF; + + select table_name_; + + SET FOREIGN_KEY_CHECKS = 0; + SET @stmt = CONCAT('ALTER TABLE `', table_name_, '` CONVERT TO CHARACTER SET {charset} COLLATE {collation};'); + PREPARE query FROM @stmt; + EXECUTE query; + DEALLOCATE PREPARE query; + + SET FOREIGN_KEY_CHECKS = 1; + + END LOOP; + CLOSE cur; + + END$$ + DELIMITER ; + + use {database}; + ALTER DATABASE {database} CHARACTER SET {charset} COLLATE {collation}; + CALL UpdateColumns(); + CALL UpdateTables(); + """ diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index 4f8f7773d8..8b60baf12d 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -222,6 +222,10 @@ def upgrade_from_quince(context: click.Context, config: Config) -> None: upgrade_mongodb(context, config, "5.0.26", "5.0") upgrade_mongodb(context, config, "6.0.14", "6.0") upgrade_mongodb(context, config, "7.0.7", "7.0") + fmt.echo_alert( + """It is recommended to upgrade your character set and collation of the MySQL database after upgrading to Redwood. +You can use the convert-mysql-utf8mb4-charset do job to upgrade the collation and character set. You can find more details regarding the command at https://docs.tutor.edly.io/local.html#changing-the-mysql-charset-and-collation""" + ) def upgrade_mongodb( From 437e2cd07edbf71c94ff921d0e921716b912084b Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 15 Nov 2024 02:32:42 -0500 Subject: [PATCH 09/23] fix: Don't build uwsgi with xml support When uwsgi runs with xml support, it throws the following error on startup with the latest version of edx-platform: ``` xmlsec.InternalError: (-1, 'lxml & xmlsec libxml2 library version mismatch') ``` See https://github.com/xmlsec/python-xmlsec/issues/320 for more details on this. Essentially, the uWSGI wheel is built against a different version of libxml2 and it causes this version mismach error when trying to load xmlsec. The xml support in uWSGI is only needed for runing with xml configuration files, which tutor does not do. Following the guidance of the above issue, I updated the `openedx` Dockerfile to no longer compile with any xml support as a part of the build. The time to build uWSGI was only slightly more than building from cache so I'm not concerned about major slowdowns in the build time for un-cached builds. --- .../20241113_131448_feanil_fix_uwsgi_xml_issues_master.md | 1 + tutor/templates/build/openedx/Dockerfile | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md diff --git a/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md b/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md new file mode 100644 index 0000000000..318c809dce --- /dev/null +++ b/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md @@ -0,0 +1 @@ +- [Bugfix] Don't build uwsgi with XML support (by @feanil) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 9ab4234a86..8723642cc5 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -98,8 +98,10 @@ RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target pip install -r /openedx/edx-platform/requirements/edx/base.txt # Install extra requirements +# We don't need xml configuration support in uwsgi so don't install it. RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install \ + UWSGI_PROFILE_OVERRIDE="xml=no" \ + pip install --no-cache-dir --compile \ # Use redis as a django cache https://pypi.org/project/django-redis/ django-redis==5.4.0 \ # uwsgi server https://pypi.org/project/uWSGI/ From ba225a14bb1d3199d1eda789a31238a3285a44cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 15 Nov 2024 08:39:21 +0100 Subject: [PATCH 10/23] chore: split uwsgi/django-redis installation for efficiency --- tutor/templates/build/openedx/Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 8723642cc5..a7e01e9d4c 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -98,14 +98,18 @@ RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target pip install -r /openedx/edx-platform/requirements/edx/base.txt # Install extra requirements -# We don't need xml configuration support in uwsgi so don't install it. RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - UWSGI_PROFILE_OVERRIDE="xml=no" \ - pip install --no-cache-dir --compile \ + pip install \ # Use redis as a django cache https://pypi.org/project/django-redis/ - django-redis==5.4.0 \ - # uwsgi server https://pypi.org/project/uWSGI/ - uwsgi==2.0.24 + django-redis==5.4.0 + +# uwsgi server https://pypi.org/project/uWSGI/ +# We don't need xml configuration support in uwsgi so don't install it, as it causes +# uwsgi to crash +# https://github.com/xmlsec/python-xmlsec/issues/320 +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + UWSGI_PROFILE_OVERRIDE="xml=no" \ + pip install --no-cache-dir --compile uwsgi==2.0.24 {{ patch("openedx-dockerfile-post-python-requirements") }} From 202e9dd8a8e9e25ec43ec712c987c62720d8dc91 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Wed, 27 Nov 2024 01:08:34 -0500 Subject: [PATCH 11/23] feat!: Change version suffix from "nightly" to "main" --- tutor/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/__about__.py b/tutor/__about__.py index 84510f6009..6fbd7d44cd 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -10,7 +10,7 @@ # the main branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "nightly" +__version_suffix__ = "main" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed From 31160250ee54a0d62de2630154ff9b7da58e5ecd Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Wed, 27 Nov 2024 01:09:04 -0500 Subject: [PATCH 12/23] feat!: Load plugins from main branches rather than nightly branches --- requirements/plugins.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 907eb9f610..63a7f32e7a 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,14 +1,14 @@ -# # For Tutor Nightly, we install plugins from their nightly branches instead of from PyPI -tutor-android@git+https://github.com/overhangio/tutor-android@nightly -tutor-cairn@git+https://github.com/overhangio/tutor-cairn@nightly -tutor-credentials@git+https://github.com/overhangio/tutor-credentials@nightly -tutor-discovery@git+https://github.com/overhangio/tutor-discovery@nightly -tutor-ecommerce@git+https://github.com/overhangio/tutor-ecommerce@nightly -tutor-forum@git+https://github.com/overhangio/tutor-forum@nightly -tutor-indigo@git+https://github.com/overhangio/tutor-indigo@nightly -tutor-jupyter@git+https://github.com/overhangio/tutor-jupyter@nightly -tutor-mfe@git+https://github.com/overhangio/tutor-mfe@nightly -tutor-minio@git+https://github.com/overhangio/tutor-minio@nightly -tutor-notes@git+https://github.com/overhangio/tutor-notes@nightly -tutor-webui@git+https://github.com/overhangio/tutor-webui@nightly -tutor-xqueue@git+https://github.com/overhangio/tutor-xqueue@nightly +# # For Tutor Main, we install plugins from their main branches instead of from PyPI +tutor-android@git+https://github.com/overhangio/tutor-android@main +tutor-cairn@git+https://github.com/overhangio/tutor-cairn@main +tutor-credentials@git+https://github.com/overhangio/tutor-credentials@main +tutor-discovery@git+https://github.com/overhangio/tutor-discovery@main +tutor-ecommerce@git+https://github.com/overhangio/tutor-ecommerce@main +tutor-forum@git+https://github.com/overhangio/tutor-forum@main +tutor-indigo@git+https://github.com/overhangio/tutor-indigo@main +tutor-jupyter@git+https://github.com/overhangio/tutor-jupyter@main +tutor-mfe@git+https://github.com/overhangio/tutor-mfe@main +tutor-minio@git+https://github.com/overhangio/tutor-minio@main +tutor-notes@git+https://github.com/overhangio/tutor-notes@main +tutor-webui@git+https://github.com/overhangio/tutor-webui@main +tutor-xqueue@git+https://github.com/overhangio/tutor-xqueue@main From 067fb4ab738c4e336f08354d16e6eeeadff0b660 Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Thu, 17 Oct 2024 12:50:34 +0500 Subject: [PATCH 13/23] feat: upgrade to sumac --- changelog.d/20241017_124915_dawoud.sheraz_sumac.md | 1 + docs/configuration.rst | 6 +++--- docs/dev.rst | 2 +- docs/install.rst | 6 ++++-- docs/reference/indexes.rst | 10 +++++----- docs/tutorials/edx-platform.rst | 2 +- tests/test_env.py | 2 +- tutor/__about__.py | 4 ++-- tutor/commands/upgrade/__init__.py | 3 ++- tutor/commands/upgrade/compose.py | 3 +++ tutor/commands/upgrade/k8s.py | 3 +++ tutor/env.py | 1 + tutor/templates/build/openedx/revisions.yml | 2 +- tutor/templates/config/defaults.yml | 2 +- 14 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 changelog.d/20241017_124915_dawoud.sheraz_sumac.md diff --git a/changelog.d/20241017_124915_dawoud.sheraz_sumac.md b/changelog.d/20241017_124915_dawoud.sheraz_sumac.md new file mode 100644 index 0000000000..c9412e1e8b --- /dev/null +++ b/changelog.d/20241017_124915_dawoud.sheraz_sumac.md @@ -0,0 +1 @@ +- πŸ’₯[Feature] Upgrade to Sumac. (by @dawoudsheraz) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5c183f7871..4b3ceedaef 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -132,7 +132,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/redwood.3"``, or ``master`` in :ref:`Tutor Main
`) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/sumac.master"``, or ``master`` in :ref:`Tutor Main
`) This defines the default version that will be pulled from all Open edX git repositories. @@ -378,9 +378,9 @@ Note that your edx-platform version must be a fork of the latest release **tag** If you don't create your fork from this tag, you *will* have important compatibility issues with other services. In particular: -- Do not try to run a fork from an older (pre-Redwood) version of edx-platform: this will simply not work. +- Do not try to run a fork from an older (pre-Sumac) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. -- Do not try to run a fork from the open-release/redwood.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/redwood.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/redwood.master branch. +- Do not try to run a fork from the open-release/sumac.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/sumac.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/sumac.master branch. .. _i18n: diff --git a/docs/dev.rst b/docs/dev.rst index 19fd1fd92d..a5ca41d9e0 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -113,7 +113,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by - The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +- The edx-platform `development requirements `__ are installed. If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: diff --git a/docs/install.rst b/docs/install.rst index 59ff280422..df3c5337ae 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -123,11 +123,11 @@ Major Open edX releases are published twice a year, in June and December, by the 4. Test the new release in a sandboxed environment. 5. If you are running edx-platform, or some other repository from a custom branch, then you should rebase (and test) your changes on top of the latest release tag (see :ref:`edx_platform_fork`). -The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Quince to Redwood and rebuild some Docker images, run:: +The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Redwood to Sumac and rebuild some Docker images, run:: tutor config save tutor images build all # list the images that should be rebuilt here - tutor local upgrade --from=quince + tutor local upgrade --from=redwood tutor local launch @@ -157,6 +157,8 @@ Instructions for installing the appropriate Tutor version for older Open edX rel +-------------------+---------------+--------------------------------------------+ | Redwood | v18 | pip install 'tutor[full]>=18.0.0,<19.0.0' | +-------------------+---------------+--------------------------------------------+ +| Sumac | v19 | pip install 'tutor[full]>=19.0.0,<20.0.0' | ++-------------------+---------------+--------------------------------------------+ .. _autocomplete: diff --git a/docs/reference/indexes.rst b/docs/reference/indexes.rst index 45fff4b3c0..bed46dc3d6 100644 --- a/docs/reference/indexes.rst +++ b/docs/reference/indexes.rst @@ -7,10 +7,10 @@ Plugin indexes are a great way to have your plugins discovered by other users. P Index file paths ================ -A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Redwood" release: +A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Sumac" release: -- https://overhang.io/tutor/main/redwood/plugins.yml -- ``/path/to/your/local/index/redwood/plugins.yml`` +- https://overhang.io/tutor/main/sumac/plugins.yml +- ``/path/to/your/local/index/sumac/plugins.yml`` To add either indexes, run the ``tutor plugins index add`` command without the suffix. For instance:: @@ -106,9 +106,9 @@ Manage plugins in development Plugin developers and maintainers often want to install local versions of their plugins. They usually achieve this with ``pip install -e /path/to/tutor-plugin``. We can improve that workflow by creating an index for local plugins:: # Create the plugin index directory - mkdir -p ~/localindex/redwood/ + mkdir -p ~/localindex/sumac/ # Edit the index - vim ~/localindex/redwood/plugins.yml + vim ~/localindex/sumac/plugins.yml Add the following to the index:: diff --git a/docs/tutorials/edx-platform.rst b/docs/tutorials/edx-platform.rst index a5cd084816..08c973b7c5 100644 --- a/docs/tutorials/edx-platform.rst +++ b/docs/tutorials/edx-platform.rst @@ -69,7 +69,7 @@ Quite often, developers don't want to work on edx-platform directly, but on a de cd /my/workspace/edx-ora2 git clone https://github.com/openedx/edx-ora2 . -Then, check out the right version of the package. This is the version that is indicated in the `edx-platform/requirements/edx/base.txt `__. Be careful that the version that is currently in use in your version of edx-platform is **not necessarily the head of the master branch**:: +Then, check out the right version of the package. This is the version that is indicated in the `edx-platform/requirements/edx/base.txt `__. Be careful that the version that is currently in use in your version of edx-platform is **not necessarily the head of the master branch**:: git checkout diff --git a/tests/test_env.py b/tests/test_env.py index e2ba7bbe1d..bd0a3805fe 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -260,7 +260,7 @@ def test_current_version_in_latest_env(self) -> None: ) as f: f.write(__version__) self.assertEqual(__version__, env.current_version(root)) - self.assertEqual("redwood", env.get_env_release(root)) + self.assertEqual("sumac", env.get_env_release(root)) self.assertIsNone(env.should_upgrade_from_release(root)) self.assertTrue(env.is_up_to_date(root)) diff --git a/tutor/__about__.py b/tutor/__about__.py index 363d1d42cf..a56ced9cb8 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "18.2.2" +__version__ = "19.0.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and @@ -10,7 +10,7 @@ # the main branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "main" +__version_suffix__ = "" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed diff --git a/tutor/commands/upgrade/__init__.py b/tutor/commands/upgrade/__init__.py index 44592344a5..789384e609 100644 --- a/tutor/commands/upgrade/__init__.py +++ b/tutor/commands/upgrade/__init__.py @@ -1,4 +1,4 @@ -# Note: don't forget to change this when we upgrade from redwood +# Note: don't forget to change this when we upgrade from sumac OPENEDX_RELEASE_NAMES = [ "ironwood", "juniper", @@ -10,4 +10,5 @@ "palm", "quince", "redwood", + "sumac", ] diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index 8b60baf12d..315560e317 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -51,6 +51,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None: upgrade_from_quince(context, config) running_release = "redwood" + if running_release == "redwood": + running_release = "sumac" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index e547bf4b80..75ed767e1d 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -49,6 +49,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None: upgrade_from_quince(config) running_release = "redwood" + if running_release == "redwood": + running_release = "sumac" + def upgrade_from_ironwood(config: Config) -> None: upgrade_mongodb(config, "3.4.24", "3.4") diff --git a/tutor/env.py b/tutor/env.py index 2e71387abd..9bfdc97bd1 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -474,6 +474,7 @@ def get_release(version: str) -> str: "16": "palm", "17": "quince", "18": "redwood", + "19": "sumac", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index d30d0cbc0e..0ddb2fc922 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: redwood +EDX_PLATFORM_REVISION: sumac diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 76fd71d515..ae3fccddc9 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -61,7 +61,7 @@ OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" # the common version will be automatically set to "master" in the main branch -OPENEDX_COMMON_VERSION: "open-release/redwood.3" +OPENEDX_COMMON_VERSION: "open-release/sumac.master" OPENEDX_EXTRA_PIP_REQUIREMENTS: [] MYSQL_HOST: "mysql" MYSQL_PORT: 3306 From 921dcd8e748507261241c91e0e93800496ba15c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 17 Oct 2024 12:54:38 +0200 Subject: [PATCH 14/23] feat: replace Elasticsearch by Meilisearch With this change, we get rid of Elasticsearch across all of Tutor. Instead, we run Meilisearch, which is much more lightweight in terms of memory usage. Obviously, this is a (very) breaking change. Indexing commands will be run during init, such that search should work as before. After the edx-search PR is merged and the dependency is upgraded in edx-platform, we should remove the manual `RUN pip install ...` command. --- .../20241017_125209_regis_meilisearch.md | 1 + docs/configuration.rst | 24 ++++++++------ docs/tutorials/scale.rst | 2 +- tutor/commands/images.py | 2 +- tutor/commands/jobs.py | 12 ++++++- tutor/commands/k8s.py | 2 +- tutor/config.py | 1 - tutor/env.py | 2 ++ tutor/plugins/openedx.py | 12 +++++++ tutor/templates/apps/caddy/Caddyfile | 6 ++++ .../templates/apps/openedx/config/cms.env.yml | 1 - .../templates/apps/openedx/config/lms.env.yml | 3 -- .../apps/openedx/settings/cms/development.py | 6 ++-- .../apps/openedx/settings/lms/development.py | 2 ++ .../openedx/settings/partials/common_all.py | 14 ++++---- .../openedx/settings/partials/common_cms.py | 3 ++ .../openedx/settings/partials/common_lms.py | 5 +++ tutor/templates/apps/permissions/setowners.sh | 2 +- tutor/templates/config/base.yml | 3 ++ tutor/templates/config/defaults.yml | 13 ++++---- tutor/templates/dev/docker-compose.yml | 19 +++++------ tutor/templates/jobs/init/cms.sh | 8 +++++ tutor/templates/jobs/init/lms.sh | 3 ++ tutor/templates/jobs/init/meilisearch.sh | 18 +++++++++++ tutor/templates/k8s/deployments.yml | 32 +++++++------------ tutor/templates/k8s/services.yml | 10 +++--- tutor/templates/k8s/volumes.yml | 10 +++--- tutor/templates/local/docker-compose.jobs.yml | 2 +- tutor/templates/local/docker-compose.yml | 25 ++++++--------- tutor/utils.py | 21 ++++++++++++ 30 files changed, 174 insertions(+), 90 deletions(-) create mode 100644 changelog.d/20241017_125209_regis_meilisearch.md create mode 100644 tutor/templates/jobs/init/meilisearch.sh diff --git a/changelog.d/20241017_125209_regis_meilisearch.md b/changelog.d/20241017_125209_regis_meilisearch.md new file mode 100644 index 0000000000..8144dda24b --- /dev/null +++ b/changelog.d/20241017_125209_regis_meilisearch.md @@ -0,0 +1 @@ +- πŸ’₯[Feature] Replace Elasticsearch by Meilisearch. Elasticsearch was both a source of complexity and high resource usage. With this change, we no longer run Elasticsearch to perform common search queries across Open edX. This includes: course discovery, courseware search and studio search. Instead, we index all these documents in a Meilisearch instance, which is much more lightweight in terms of memory consumption. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 4b3ceedaef..99a1ddc11e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -40,7 +40,7 @@ With an up-to-date environment, Tutor is ready to launch an Open edX platform an Individual service activation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``RUN_ELASTICSEARCH`` (default: ``true``) +- ``RUN_MEILISEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) - ``RUN_REDIS`` (default: ``true``) @@ -71,9 +71,9 @@ This configuration parameter defines the name of the Docker image to run the dev This configuration parameter defines which Caddy Docker image to use. -- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.17.9"``) +- ``DOCKER_IMAGE_MEILISEARCH`` (default: ``"docker.io/getmeili/meilisearch:v1.8.4"``) -This configuration parameter defines which Elasticsearch Docker image to use. +This configuration parameter defines which Meilisearch Docker image to use. - ``DOCKER_IMAGE_MONGODB`` (default: ``"docker.io/mongo:7.0.7"``) @@ -228,13 +228,19 @@ By default, a running Open edX platform deployed with Tutor includes all necessa .. note:: When configuring an external MySQL database, please make sure it is using version 8.4. -Elasticsearch -************* +Meilisearch +*********** + +- ``MEILISEARCH_URL`` (default: ``"http://meilisearch:7700"``): internal URL used for backend-to-backend communication. +- ``MEILISEARCH_PUBLIC_URL`` (default: ``"{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://meilisearch.{{ LMS_HOST }}"``): external URL from which the frontend will access the Meilisearch instance. +- ``MEILISEARCH_INDEX_PREFIX`` (default: ``"tutor_"``) +- ``MEILISEARCH_MASTER_KEY`` (default: ``"{{ 24|random_string }}"``) +- ``MEILISEARCH_API_KEY_UID`` (default: ``"{{ 4|uuid }}"``): UID used to sign the API key. +- ``MEILISEARCH_API_KEY`` (default: ``"{{ MEILISEARCH_MASTER_KEY|uid_master_hash(MEILISEARCH_API_KEY_UID) }}"``) + +To reset the Meilisearch API key, make sure to unset both the API key and it's UID: -- ``ELASTICSEARCH_SCHEME`` (default: ``"http"``) -- ``ELASTICSEARCH_HOST`` (default: ``"elasticsearch"``) -- ``ELASTICSEARCH_PORT`` (default: ``9200``) -- ``ELASTICSEARCH_HEAP_SIZE`` (default: ``"1g"``) + tutor config save --unset MEILISEARCH_API_KEY_UID MEILISEARCH_API_KEY MongoDB ******* diff --git a/docs/tutorials/scale.rst b/docs/tutorials/scale.rst index 4cecdb3ef2..ea54331f66 100644 --- a/docs/tutorials/scale.rst +++ b/docs/tutorials/scale.rst @@ -37,11 +37,11 @@ Offloading data storage Aside from web workers, the most resource-intensive services are in the data persistence layer. They are, by decreasing resource usage: -- `Elasticsearch `__: indexing of course contents and forum topics, mostly for search. Elasticsearch is never a source of truth in Open edX, and the data can thus be trashed and re-created safely. - `MySQL `__: structured, consistent data storage which is the default destination of all data. - `MongoDB `__: structured storage of course data. - `Redis `__: caching and asynchronous task management. - `MinIO `__: S3-like object storage for user-uploaded files, which is enabled by the `tutor-minio `__ plugin. It is possible to replace MinIO by direct filesystem storage (the default), but scaling will then become much more difficult down the road. +- `Meilisearch `__: indexing of course contents and forum topics, mostly for search. Meilisearch is never a source of truth in Open edX, and the data can thus be trashed and re-created safely. When attempting to scale a single-server deployment, we recommend starting by offloading some of these stateful data storage components, in the same order of priority. There are multiple benefits: diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 2a7f53255e..a06054ada5 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -63,7 +63,7 @@ def _add_images_to_pull( """ vendor_images = [ ("caddy", "DOCKER_IMAGE_CADDY"), - ("elasticsearch", "DOCKER_IMAGE_ELASTICSEARCH"), + ("meilisearch", "DOCKER_IMAGE_MEILISEARCH"), ("mongodb", "DOCKER_IMAGE_MONGODB"), ("mysql", "DOCKER_IMAGE_MYSQL"), ("redis", "DOCKER_IMAGE_REDIS"), diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 21fbbe5c43..75bb6aa643 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -44,6 +44,10 @@ def _add_core_init_tasks() -> None: hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) + with hooks.Contexts.app("meilisearch").enter(): + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("lms", env.read_core_template_file("jobs", "init", "meilisearch.sh")) + ) with hooks.Contexts.app("lms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ( @@ -176,7 +180,13 @@ def importdemocourse( python ./manage.py cms import ../data "$course_root" # Re-index courses -./manage.py cms reindex_course --all --setup""" +# TODO this is no longer compatible with meilisearch indexing. That is, until this PR is merged: +# https://github.com/openedx/edx-platform/pull/35743 +# Anyway, it doesn't make much sense to reindex *all* courses after a single one has +# been created. Thus we should # rely on course authors to press the "reindex" button in +# the studio after the course has # been imported. +#./manage.py cms reindex_course --all --setup +""" yield ("cms", template) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 5064366c0c..17b0c285bc 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -390,7 +390,7 @@ def _start_base_deployments(_job_name: str, *_args: Any, **_kwargs: Any) -> None """ config = tutor_config.load(context.root) wait_for_deployment_ready(config, "caddy") - for name in ["elasticsearch", "mysql", "mongodb"]: + for name in ["meilisearch", "mysql", "mongodb"]: if tutor_config.is_service_activated(config, name): wait_for_deployment_ready(config, name) diff --git a/tutor/config.py b/tutor/config.py index 531056792c..5d29c17f40 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -235,7 +235,6 @@ def upgrade_obsolete(config: Config) -> None: for name in [ "ACTIVATE_LMS", "ACTIVATE_CMS", - "ACTIVATE_ELASTICSEARCH", "ACTIVATE_MONGODB", "ACTIVATE_MYSQL", "ACTIVATE_REDIS", diff --git a/tutor/env.py b/tutor/env.py index 9bfdc97bd1..e41abd3b5b 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -56,6 +56,8 @@ def _prepare_environment() -> None: ("reverse_host", utils.reverse_host), ("rsa_import_key", utils.rsa_import_key), ("rsa_private_key", utils.rsa_private_key), + ("uuid", utils.uuid), + ("uid_master_hash", utils.uid_master_hash), ], ) # Template variables diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py index 5ed9e0b6df..3666963e78 100644 --- a/tutor/plugins/openedx.py +++ b/tutor/plugins/openedx.py @@ -1,5 +1,6 @@ from __future__ import annotations + import os import re import typing as t @@ -61,6 +62,17 @@ def _edx_platform_public_hosts( return hosts +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _meilisearch_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts.append("{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}:7700") + else: + hosts.append("{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}") + return hosts + + @hooks.Filters.IMAGES_BUILD_MOUNTS.add() def _mount_edx_platform_build( volumes: list[tuple[str, str]], path: str diff --git a/tutor/templates/apps/caddy/Caddyfile b/tutor/templates/apps/caddy/Caddyfile index 96a6d9891b..77eb152ec4 100644 --- a/tutor/templates/apps/caddy/Caddyfile +++ b/tutor/templates/apps/caddy/Caddyfile @@ -82,4 +82,10 @@ } } +{% if RUN_MEILISEARCH %} +{{ MEILISEARCH_PUBLIC_URL.split("://")[1] }}{$default_site_port} { + import proxy "meilisearch:7700" +} +{% endif %} + {{ patch("caddyfile") }} diff --git a/tutor/templates/apps/openedx/config/cms.env.yml b/tutor/templates/apps/openedx/config/cms.env.yml index 53e99b0c2d..8eb7b462d8 100644 --- a/tutor/templates/apps/openedx/config/cms.env.yml +++ b/tutor/templates/apps/openedx/config/cms.env.yml @@ -9,7 +9,6 @@ FEATURES: {{ patch("cms-env-features")|indent(2) }} CERTIFICATES_HTML_VIEW: true PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" - ENABLE_COURSEWARE_INDEX: true ENABLE_CSMH_EXTENDED: false ENABLE_LEARNER_RECORDS: false ENABLE_LIBRARY_INDEX: true diff --git a/tutor/templates/apps/openedx/config/lms.env.yml b/tutor/templates/apps/openedx/config/lms.env.yml index 565820c81b..bc9927f23c 100644 --- a/tutor/templates/apps/openedx/config/lms.env.yml +++ b/tutor/templates/apps/openedx/config/lms.env.yml @@ -9,10 +9,7 @@ FEATURES: {{ patch("lms-env-features")|indent(2) }} CERTIFICATES_HTML_VIEW: true PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" - ENABLE_COURSE_DISCOVERY: true - ENABLE_COURSEWARE_SEARCH: true ENABLE_CSMH_EXTENDED: false - ENABLE_DASHBOARD_SEARCH: true ENABLE_COMBINED_LOGIN_REGISTRATION: true ENABLE_GRADE_DOWNLOADS: true ENABLE_LEARNER_RECORDS: false diff --git a/tutor/templates/apps/openedx/settings/cms/development.py b/tutor/templates/apps/openedx/settings/cms/development.py index af7af18e12..ee4c3b3b5b 100644 --- a/tutor/templates/apps/openedx/settings/cms/development.py +++ b/tutor/templates/apps/openedx/settings/cms/development.py @@ -2,20 +2,22 @@ import os from cms.envs.devstack import * +{% include "apps/openedx/settings/partials/common_cms.py" %} + LMS_BASE = "{{ LMS_HOST }}:8000" LMS_ROOT_URL = "http://" + LMS_BASE CMS_BASE = "{{ CMS_HOST }}:8001" CMS_ROOT_URL = "http://" + CMS_BASE +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}:7700" + # Authentication SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO_DEV }}" SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = LMS_ROOT_URL FEATURES["PREVIEW_LMS_BASE"] = "{{ PREVIEW_LMS_HOST }}:8000" -{% include "apps/openedx/settings/partials/common_cms.py" %} - # Setup correct webpack configuration file for development WEBPACK_CONFIG_PATH = "webpack.dev.config.js" diff --git a/tutor/templates/apps/openedx/settings/lms/development.py b/tutor/templates/apps/openedx/settings/lms/development.py index ed0c277366..8e06352e1d 100644 --- a/tutor/templates/apps/openedx/settings/lms/development.py +++ b/tutor/templates/apps/openedx/settings/lms/development.py @@ -15,6 +15,8 @@ CMS_ROOT_URL = "http://{}".format(CMS_BASE) LOGIN_REDIRECT_WHITELIST.append(CMS_BASE) +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}:7700" + # Session cookie SESSION_COOKIE_DOMAIN = "{{ LMS_HOST }}" SESSION_COOKIE_SECURE = False diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 105ac5383c..f156b0e423 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -34,12 +34,14 @@ # Behave like memcache when it comes to connection errors DJANGO_REDIS_IGNORE_EXCEPTIONS = True -# Elasticsearch connection parameters -ELASTIC_SEARCH_CONFIG = [{ - {% if ELASTICSEARCH_SCHEME == "https" %}"use_ssl": True,{% endif %} - "host": "{{ ELASTICSEARCH_HOST }}", - "port": {{ ELASTICSEARCH_PORT }}, -}] +# Meilisearch connection parameters +MEILISEARCH_ENABLED = True +MEILISEARCH_URL = "{{ MEILISEARCH_URL }}" +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}" +MEILISEARCH_INDEX_PREFIX = "{{ MEILISEARCH_INDEX_PREFIX }}" +MEILISEARCH_API_KEY = "{{ MEILISEARCH_API_KEY }}" +MEILISEARCH_MASTER_KEY = "{{ MEILISEARCH_MASTER_KEY }}" +SEARCH_ENGINE = "search.meilisearch.MeilisearchEngine" # Common cache config CACHES = { diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index c5dde04400..e1263100d5 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -20,6 +20,9 @@ FRONTEND_LOGIN_URL = LMS_ROOT_URL + '/login' FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register' +# Enable "reindex" button +FEATURES["ENABLE_COURSEWARE_INDEX"] = True + # Create folders if necessary for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 1a06d613c2..befd6d6d2f 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -37,6 +37,11 @@ "LOCATION": "staticfiles_lms", } +# Enable search features +FEATURES["ENABLE_COURSE_DISCOVERY"] = True +FEATURES["ENABLE_COURSEWARE_SEARCH"] = True +FEATURES["ENABLE_DASHBOARD_SEARCH"] = True + # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): diff --git a/tutor/templates/apps/permissions/setowners.sh b/tutor/templates/apps/permissions/setowners.sh index d4044f9067..6ec6e969f9 100644 --- a/tutor/templates/apps/permissions/setowners.sh +++ b/tutor/templates/apps/permissions/setowners.sh @@ -1,6 +1,6 @@ #! /bin/sh setowner $OPENEDX_USER_ID /mounts/lms /mounts/cms /mounts/openedx -{% if RUN_ELASTICSEARCH %}setowner 1000 /mounts/elasticsearch{% endif %} +{% if RUN_MEILISEARCH %}setowner 1000 /mounts/meilisearch{% endif %} {% if RUN_MONGODB %}setowner 999 /mounts/mongodb{% endif %} {% if RUN_MYSQL %}setowner 999 /mounts/mysql{% endif %} {% if RUN_REDIS %}setowner 1000 /mounts/redis{% endif %} diff --git a/tutor/templates/config/base.yml b/tutor/templates/config/base.yml index b1d6a14afa..61e4459a8d 100644 --- a/tutor/templates/config/base.yml +++ b/tutor/templates/config/base.yml @@ -2,6 +2,9 @@ CMS_OAUTH2_SECRET: "{{ 24|random_string }}" ID: "{{ 24|random_string }}" JWT_RSA_PRIVATE_KEY: "{{ 2048|rsa_private_key }}" +MEILISEARCH_MASTER_KEY: "{{ 24|random_string }}" +MEILISEARCH_API_KEY_UID: "{{ 4|uuid }}" +MEILISEARCH_API_KEY: "{{ MEILISEARCH_MASTER_KEY|uid_master_hash(MEILISEARCH_API_KEY_UID) }}" MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" OPENEDX_SECRET_KEY: "{{ 24|random_string }}" diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index ae3fccddc9..9a97a5a974 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -16,8 +16,8 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" # https://hub.docker.com/_/caddy/tags DOCKER_IMAGE_CADDY: "docker.io/caddy:2.7.4" -# https://hub.docker.com/_/elasticsearch/tags -DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.13" +# https://hub.docker.com/r/getmeili/meilisearch/tags +DOCKER_IMAGE_MEILISEARCH: "docker.io/getmeili/meilisearch:v1.8.4" # https://hub.docker.com/_/mongo/tags DOCKER_IMAGE_MONGODB: "docker.io/mongo:7.0.7" # https://hub.docker.com/_/mysql/tags @@ -29,10 +29,6 @@ DOCKER_IMAGE_REDIS: "docker.io/redis:7.2.4" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" -ELASTICSEARCH_HOST: "elasticsearch" -ELASTICSEARCH_PORT: 9200 -ELASTICSEARCH_SCHEME: "http" -ELASTICSEARCH_HEAP_SIZE: 1g ENABLE_HTTPS: false ENABLE_WEB_PROXY: true JWT_COMMON_AUDIENCE: "openedx" @@ -42,6 +38,9 @@ K8S_NAMESPACE: "openedx" LANGUAGE_CODE: "en" LMS_HOST: "www.myopenedx.com" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" +MEILISEARCH_URL: "http://meilisearch:7700" +MEILISEARCH_PUBLIC_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://meilisearch.{{ LMS_HOST }}" +MEILISEARCH_INDEX_PREFIX: "tutor_" MONGODB_AUTH_MECHANISM: "" MONGODB_AUTH_SOURCE: "admin" MONGODB_HOST: "mongodb" @@ -73,7 +72,7 @@ REDIS_HOST: "redis" REDIS_PORT: 6379 REDIS_USERNAME: "" REDIS_PASSWORD: "" -RUN_ELASTICSEARCH: true +RUN_MEILISEARCH: true RUN_MONGODB: true RUN_MYSQL: true RUN_REDIS: true diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index acbaa56925..5ef5387d39 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -32,19 +32,20 @@ services: ports: - "8001:8000" + {% if RUN_MEILISEARCH -%} + meilisearch: + ports: + - "127.0.0.1:7700:7700" + networks: + default: + aliases: + - "{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}" + {%- endif %} + # Additional service for watching theme changes watchthemes: <<: *openedx-service command: npm run watch-sass restart: unless-stopped - {% if RUN_ELASTICSEARCH and is_docker_rootless() %} - elasticsearch: - ulimits: - memlock: - # Fixes error setting rlimits for ready process in rootless docker - soft: 0 # zero means "unset" in the memlock context - hard: 0 - {% endif %} - {{ patch("local-docker-compose-dev-services")|indent(2) }} diff --git a/tutor/templates/jobs/init/cms.sh b/tutor/templates/jobs/init/cms.sh index 1d420a48ab..36ba45fcd0 100644 --- a/tutor/templates/jobs/init/cms.sh +++ b/tutor/templates/jobs/init/cms.sh @@ -16,3 +16,11 @@ fi # Create waffle switches to enable some features, if they have not been explicitly defined before # Copy-paste of units in Studio (highly requested new feature, but defaults to off in Quince) (./manage.py cms waffle_flag --list | grep contentstore.enable_copy_paste_units) || ./manage.py lms waffle_flag --create contentstore.enable_copy_paste_units --everyone + +# Re-index studio and courseware content +# Note that this might be slow for large courses until this issue is resolved: +# https://github.com/openedx/modular-learning/issues/235 +# But this is a necessary step to make sure the indexes are created and properly +# configured. +./manage.py cms reindex_studio --experimental +./manage.py cms reindex_course --active diff --git a/tutor/templates/jobs/init/lms.sh b/tutor/templates/jobs/init/lms.sh index 88c94625c5..93d179c127 100644 --- a/tutor/templates/jobs/init/lms.sh +++ b/tutor/templates/jobs/init/lms.sh @@ -10,6 +10,9 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE" ./manage.py lms migrate +# Create meilisearch indexes +./manage.py lms shell -c "import search.meilisearch; search.meilisearch.create_indexes()" + # Create oauth2 apps for CMS SSO # https://github.com/openedx/edx-platform/blob/master/docs/guides/studio_oauth.rst ./manage.py lms manage_user cms cms@openedx --unusable-password diff --git a/tutor/templates/jobs/init/meilisearch.sh b/tutor/templates/jobs/init/meilisearch.sh new file mode 100644 index 0000000000..2adb79c4e8 --- /dev/null +++ b/tutor/templates/jobs/init/meilisearch.sh @@ -0,0 +1,18 @@ +# Get or create Meilisearch API key +python -c " +import meilisearch +client = meilisearch.Client('{{ MEILISEARCH_URL }}', '{{ MEILISEARCH_MASTER_KEY }}') +try: + client.get_key('{{ MEILISEARCH_API_KEY_UID }}') + print('Key already exists') +except meilisearch.errors.MeilisearchApiError: + print('Key does not exist: creating...') + client.create_key({ + 'name': 'Open edX backend API key', + 'uid': '{{ MEILISEARCH_API_KEY_UID }}', + 'actions': ['*'], + 'indexes': ['{{ MEILISEARCH_INDEX_PREFIX }}*'], + 'expiresAt': None, + 'description': 'Use it for backend API calls -- Created by Tutor', + }) +" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index f4ba005b53..9df23055d8 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -278,24 +278,24 @@ spec: - name: config configMap: name: openedx-config -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: apps/v1 kind: Deployment metadata: - name: elasticsearch + name: meilisearch labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: selector: matchLabels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch strategy: type: Recreate template: metadata: labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: securityContext: runAsUser: 1000 @@ -303,30 +303,22 @@ spec: fsGroup: 1000 fsGroupChangePolicy: "OnRootMismatch" containers: - - name: elasticsearch - image: {{ DOCKER_IMAGE_ELASTICSEARCH }} + - name: meilisearch + image: {{ DOCKER_IMAGE_MEILISEARCH }} env: - - name: cluster.name - value: "openedx" - - name: bootstrap.memory_lock - value: "true" - - name: discovery.type - value: "single-node" - - name: ES_JAVA_OPTS - value: "-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" - - name: TAKE_FILE_OWNERSHIP - value: "1" + - name: MEILI_MASTER_KEY + value: "{{ MEILISEARCH_MASTER_KEY }}" ports: - - containerPort: 9200 + - containerPort: 7700 securityContext: allowPrivilegeEscalation: false volumeMounts: - - mountPath: /usr/share/elasticsearch/data + - mountPath: /meili_data name: data volumes: - name: data persistentVolumeClaim: - claimName: elasticsearch + claimName: meilisearch {% endif %} {% if RUN_MONGODB %} --- diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index c34d2255d8..6cfb554350 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -67,21 +67,21 @@ spec: protocol: TCP selector: app.kubernetes.io/name: lms -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: v1 kind: Service metadata: - name: elasticsearch + name: meilisearch labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: type: ClusterIP ports: - - port: 9200 + - port: 7700 protocol: TCP selector: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch {% endif %} {% if RUN_MONGODB %} --- diff --git a/tutor/templates/k8s/volumes.yml b/tutor/templates/k8s/volumes.yml index ffb4b66486..aa31555ba0 100644 --- a/tutor/templates/k8s/volumes.yml +++ b/tutor/templates/k8s/volumes.yml @@ -14,21 +14,21 @@ spec: requests: storage: 1Gi {% endif %} -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: elasticsearch + name: meilisearch labels: app.kubernetes.io/component: volume - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: accessModes: - ReadWriteOnce resources: requests: - storage: 2Gi + storage: 5Gi {% endif %} {% if RUN_MONGODB %} --- @@ -78,4 +78,4 @@ spec: requests: storage: 1Gi {% endif %} -{{ patch("k8s-volumes") }} \ No newline at end of file +{{ patch("k8s-volumes") }} diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 6155cd688a..1e463a02b1 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -43,6 +43,6 @@ services: {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-job") %} - {{ mount }} {%- endfor %} - depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("elasticsearch", RUN_ELASTICSEARCH), ("redis", RUN_REDIS)]|list_if }} + depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("meilisearch", RUN_MEILISEARCH), ("redis", RUN_REDIS)]|list_if }} {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index da33b653ff..3c5a44d9b9 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -18,7 +18,7 @@ services: - ../../data/openedx-media-private:/mounts/openedx-private {% if RUN_MONGODB %}- ../../data/mongodb:/mounts/mongodb{% endif %} {% if RUN_MYSQL %}- ../../data/mysql:/mounts/mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- ../../data/meilisearch:/mounts/meilisearch{% endif %} {% if RUN_REDIS %}- ../../data/redis:/mounts/redis{% endif %} {{ patch("local-docker-compose-permissions-volumes")|indent(6) }} @@ -54,22 +54,15 @@ services: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" {%- endif %} - {% if RUN_ELASTICSEARCH -%} - elasticsearch: - image: {{ DOCKER_IMAGE_ELASTICSEARCH }} + {% if RUN_MEILISEARCH -%} + meilisearch: + image: {{ DOCKER_IMAGE_MEILISEARCH }} environment: - - cluster.name=openedx - - bootstrap.memory_lock=true - - discovery.type=single-node - - "ES_JAVA_OPTS=-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" - ulimits: - memlock: - soft: -1 - hard: -1 + MEILI_MASTER_KEY: "{{ MEILISEARCH_MASTER_KEY }}" + volumes: + - ../../data/meilisearch:/meili_data restart: unless-stopped user: "1000:1000" - volumes: - - ../../data/elasticsearch:/usr/share/elasticsearch/data depends_on: - permissions {%- endif %} @@ -120,7 +113,7 @@ services: depends_on: - permissions {% if RUN_MYSQL %}- mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- meilisearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} @@ -148,7 +141,7 @@ services: - permissions - lms {% if RUN_MYSQL %}- mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- meilisearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} diff --git a/tutor/utils.py b/tutor/utils.py index cdb082f38a..344f7bcfc0 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -1,4 +1,6 @@ import base64 +import hashlib +import hmac import json import os import random @@ -13,6 +15,7 @@ from typing import List, Tuple from urllib.error import URLError from urllib.request import urlopen +import uuid as uuid_module import click from Crypto.Protocol.KDF import bcrypt, bcrypt_check @@ -110,6 +113,24 @@ def rsa_private_key(bits: int = 2048) -> str: return key.export_key().decode() +def uuid(size: int) -> str: + """ + Return a random uuid string with a given size. + """ + fn = getattr(uuid_module, f"uuid{size}") + return str(fn()) + + +def uid_master_hash(master_key: str, uid: str) -> str: + """ + Hash a key UID and master key to generate an API key + + This is used specifically for meilisearch. + Source: https://www.meilisearch.com/docs/reference/api/keys#key + """ + return hmac.new(master_key.encode(), uid.encode(), hashlib.sha256).hexdigest() + + def rsa_import_key(key: str) -> RsaKey: """ Import PEM-formatted RSA key and return the corresponding object. From e9e584f15b693380f4ef763d8544ae52be250770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 14 Nov 2024 13:51:38 +0100 Subject: [PATCH 15/23] fix: disable MySQL backend in forum on upgrade In Sumac, it was decided that we would not automatically migrating data from mongodb to mysql. Instead, we encourage users to do so manually, in the tutor-forum README. --- ...91908_regis_disable_forum_mysql_on_upgrade.md | 1 + tutor/commands/upgrade/common.py | 16 ++++++++++++++++ tutor/commands/upgrade/compose.py | 1 + tutor/commands/upgrade/k8s.py | 1 + 4 files changed, 19 insertions(+) create mode 100644 changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md diff --git a/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md b/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md new file mode 100644 index 0000000000..2decb0be93 --- /dev/null +++ b/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md @@ -0,0 +1 @@ +- [Feature] With the new forum v2 application, users have a choice to use MongoDB or MySQL as a storage backend, course per course. New users will automatically start using MySQL, while existing users will be responsible for migrating their data themselves (before the Teak release). Upgrade instructions are available here: https://github.com/overhangio/tutor-forum/#installation. (by @regisb) diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index 783be6435d..ba3797e124 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -46,6 +46,22 @@ def upgrade_from_nutmeg(context: click.Context, config: Config) -> None: ) +def upgrade_from_redwood(context: click.Context, config: Config) -> None: + # Prevent switching to the MySQL storage backend in forum v2 + if plugins.is_loaded("forum"): + fmt.echo_alert( + "Your platform is going to be configured to store forum data in MongoDB. " + "You are STRONGLY ENCOURAGED to migrate your forum data to MySQL as soon as possible. " + "To do so, refer to the tutor-forum plugin documentation: https://github.com/overhangio/tutor-forum/#installation" + ) + context.obj.job_runner(config).run_task( + "lms", + """ +(./manage.py lms waffle_flag --list | grep forum_v2.enable_mysql_backend) || ./manage.py lms waffle_flag --create --deactivate forum_v2.enable_mysql_backend +""", + ) + + def get_mongo_upgrade_parameters( docker_version: str, compatibility_version: str ) -> tuple[int, dict[str, int | str]]: diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index 315560e317..cdb3304aa8 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -52,6 +52,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None: running_release = "redwood" if running_release == "redwood": + common_upgrade.upgrade_from_redwood(context, config) running_release = "sumac" diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 75ed767e1d..93e3d845e5 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -50,6 +50,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None: running_release = "redwood" if running_release == "redwood": + common_upgrade.upgrade_from_redwood(context, config) running_release = "sumac" From 99c296f27395cc12baa7110bf17360545b688bbc Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Fri, 22 Nov 2024 12:09:53 +0500 Subject: [PATCH 16/23] chore: move mysql charset upgrade alert to upgrade_from_redwood --- tutor/commands/upgrade/common.py | 5 +++++ tutor/commands/upgrade/compose.py | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index ba3797e124..7a54995f80 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -61,6 +61,11 @@ def upgrade_from_redwood(context: click.Context, config: Config) -> None: """, ) + fmt.echo_alert( + """It is recommended to upgrade your character set and collation of the MySQL database after upgrading to Sumac. +You can use the convert-mysql-utf8mb4-charset do job to upgrade the collation and character set. You can find more details regarding the command at https://docs.tutor.edly.io/local.html#changing-the-mysql-charset-and-collation""" + ) + def get_mongo_upgrade_parameters( docker_version: str, compatibility_version: str diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index cdb3304aa8..534d9f1ad8 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -226,10 +226,6 @@ def upgrade_from_quince(context: click.Context, config: Config) -> None: upgrade_mongodb(context, config, "5.0.26", "5.0") upgrade_mongodb(context, config, "6.0.14", "6.0") upgrade_mongodb(context, config, "7.0.7", "7.0") - fmt.echo_alert( - """It is recommended to upgrade your character set and collation of the MySQL database after upgrading to Redwood. -You can use the convert-mysql-utf8mb4-charset do job to upgrade the collation and character set. You can find more details regarding the command at https://docs.tutor.edly.io/local.html#changing-the-mysql-charset-and-collation""" - ) def upgrade_mongodb( From a383649839dffd5e7b6fa3af16c63f9348f9fe92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 21 Nov 2024 11:11:50 +0100 Subject: [PATCH 17/23] feat: get rid of `is_docker_rootless` template filter This filter was used only by Elasticsearch, so we no longer need to include it in Tutor. --- ...121_110954_regis_remove_docker_rootless.md | 1 + tests/test_utils.py | 19 ------------------- tutor/env.py | 1 - tutor/utils.py | 14 -------------- 4 files changed, 1 insertion(+), 34 deletions(-) create mode 100644 changelog.d/20241121_110954_regis_remove_docker_rootless.md diff --git a/changelog.d/20241121_110954_regis_remove_docker_rootless.md b/changelog.d/20241121_110954_regis_remove_docker_rootless.md new file mode 100644 index 0000000000..c5e5b8486e --- /dev/null +++ b/changelog.d/20241121_110954_regis_remove_docker_rootless.md @@ -0,0 +1 @@ +- πŸ’₯[Improvement] Get rid of the `is_docker_rootless` template filter, which was used only by Elasticsearch. (by @regisb) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5d5a6428ee..d259da50e8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -242,25 +242,6 @@ def test_is_http(self) -> None: self.assertFalse(utils.is_http("home/user/")) self.assertFalse(utils.is_http("http-home/user/")) - @patch("subprocess.run") - def test_is_docker_rootless(self, mock_run: MagicMock) -> None: - # Mock rootless `docker info` output - utils.is_docker_rootless.cache_clear() - mock_run.return_value.stdout = "some prefix\n rootless foo bar".encode("utf-8") - self.assertTrue(utils.is_docker_rootless()) - - # Mock regular `docker info` output - utils.is_docker_rootless.cache_clear() - mock_run.return_value.stdout = "some prefix, regular docker".encode("utf-8") - self.assertFalse(utils.is_docker_rootless()) - - @patch("subprocess.run") - def test_is_docker_rootless_podman(self, mock_run: MagicMock) -> None: - """Test the `is_docker_rootless` when podman is used or any other error with `docker info`""" - utils.is_docker_rootless.cache_clear() - mock_run.side_effect = subprocess.CalledProcessError(1, "docker info") - self.assertFalse(utils.is_docker_rootless()) - def test_format_table(self) -> None: rows: List[Tuple[str, ...]] = [ ("a", "xyz", "value 1"), diff --git a/tutor/env.py b/tutor/env.py index e41abd3b5b..4d00118ca1 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -72,7 +72,6 @@ def _prepare_environment() -> None: ("TUTOR_VERSION_MAJOR", int(__version__.split(".")[0])), ("TUTOR_VERSION_MINOR", int(__version__.split(".")[1])), ("TUTOR_BRANCH_IS_MAIN", __version_suffix__ == "main"), - ("is_docker_rootless", utils.is_docker_rootless), ], ) diff --git a/tutor/utils.py b/tutor/utils.py index 344f7bcfc0..45d4abecd5 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -194,20 +194,6 @@ def docker(*command: str) -> int: return execute("docker", *command) -@lru_cache(maxsize=None) -def is_docker_rootless() -> bool: - """ - A helper function to determine if Docker is running in rootless mode. - - - https://docs.docker.com/engine/security/rootless/ - """ - try: - results = subprocess.run(["docker", "info"], capture_output=True, check=True) - return "rootless" in results.stdout.decode() - except subprocess.CalledProcessError: - return False - - def docker_compose(*command: str) -> int: return execute("docker", "compose", *command) From ddc1201e464e8495759bc58405416e1a61812c32 Mon Sep 17 00:00:00 2001 From: Zia Fazal Date: Wed, 4 Dec 2024 11:24:56 +0500 Subject: [PATCH 18/23] chore: version bump of scorm xblock --- tutor/templates/build/openedx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index ebc715727f..15e63cb33d 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -114,7 +114,7 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ {{ patch("openedx-dockerfile-post-python-requirements") }} # Install scorm xblock -RUN pip install "openedx-scorm-xblock>=18.0.0,<19.0.0" +RUN pip install "openedx-scorm-xblock>=19.0.0,<20.0.0" {% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %} RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ From 6e1c62f5fb7ca8f715e9932bd350834dfe8448d4 Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Fri, 6 Dec 2024 19:12:22 +0500 Subject: [PATCH 19/23] chore: update plugin version ranges in plugins.txt --- requirements/plugins.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 63a7f32e7a..d215778139 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,14 +1,14 @@ -# # For Tutor Main, we install plugins from their main branches instead of from PyPI -tutor-android@git+https://github.com/overhangio/tutor-android@main -tutor-cairn@git+https://github.com/overhangio/tutor-cairn@main -tutor-credentials@git+https://github.com/overhangio/tutor-credentials@main -tutor-discovery@git+https://github.com/overhangio/tutor-discovery@main -tutor-ecommerce@git+https://github.com/overhangio/tutor-ecommerce@main -tutor-forum@git+https://github.com/overhangio/tutor-forum@main -tutor-indigo@git+https://github.com/overhangio/tutor-indigo@main -tutor-jupyter@git+https://github.com/overhangio/tutor-jupyter@main -tutor-mfe@git+https://github.com/overhangio/tutor-mfe@main -tutor-minio@git+https://github.com/overhangio/tutor-minio@main -tutor-notes@git+https://github.com/overhangio/tutor-notes@main -tutor-webui@git+https://github.com/overhangio/tutor-webui@main -tutor-xqueue@git+https://github.com/overhangio/tutor-xqueue@main +# change version ranges when upgrading from sumac +tutor-android>=19.0.0,<20.0.0 +tutor-cairn>=19.0.0,<20.0.0 +tutor-credentials>=19.0.0,<20.0.0 +tutor-discovery>=19.0.0,<20.0.0 +tutor-ecommerce>=19.0.0,<20.0.0 +tutor-forum>=19.0.0,<20.0.0 +tutor-indigo>=19.0.0,<20.0.0 +tutor-jupyter>=19.0.0,<20.0.0 +tutor-mfe>=19.0.0,<20.0.0 +tutor-minio>=19.0.0,<20.0.0 +tutor-notes>=19.0.0,<20.0.0 +tutor-webui>=19.0.0,<20.0.0 +tutor-xqueue>=19.0.0,<20.0.0 \ No newline at end of file From 07b46e4d752d6f1b8f53a6cec3719c7589040ee4 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Mon, 9 Dec 2024 10:33:58 -0300 Subject: [PATCH 20/23] feat: Force-enable navigation sidebar in upgrade to Sumac When upgrading to Sumac, forcefully enable the Learning MFE's navigation sidebar. --- ..._arbrandes_force_enable_navigation_sidebar_in_upgrade.md | 1 + tutor/commands/upgrade/common.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md diff --git a/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md b/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md new file mode 100644 index 0000000000..7a98871cf4 --- /dev/null +++ b/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md @@ -0,0 +1 @@ +[Improvement] Forcefully enable the Learning MFE's navigation sidebar when upgrading to Sumac. (by @arbrandes) diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index 7a54995f80..2c4d47be2e 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -47,6 +47,12 @@ def upgrade_from_nutmeg(context: click.Context, config: Config) -> None: def upgrade_from_redwood(context: click.Context, config: Config) -> None: + # Forcefully enable the learning MFE's navigation sidebar. + if plugins.is_loaded("mfe"): + context.obj.job_runner(config).run_task( + "lms", "./manage.py lms waffle_flag --create --everyone courseware.enable_navigation_sidebar" + ) + # Prevent switching to the MySQL storage backend in forum v2 if plugins.is_loaded("forum"): fmt.echo_alert( From 59abd50d1b26a36983545f590d038b55e8c7e30a Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Fri, 6 Dec 2024 12:25:14 -0800 Subject: [PATCH 21/23] feat: avoid slow reindex of studio content during init --- tutor/templates/jobs/init/cms.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tutor/templates/jobs/init/cms.sh b/tutor/templates/jobs/init/cms.sh index 36ba45fcd0..6fe572977b 100644 --- a/tutor/templates/jobs/init/cms.sh +++ b/tutor/templates/jobs/init/cms.sh @@ -17,10 +17,10 @@ fi # Copy-paste of units in Studio (highly requested new feature, but defaults to off in Quince) (./manage.py cms waffle_flag --list | grep contentstore.enable_copy_paste_units) || ./manage.py lms waffle_flag --create contentstore.enable_copy_paste_units --everyone -# Re-index studio and courseware content -# Note that this might be slow for large courses until this issue is resolved: -# https://github.com/openedx/modular-learning/issues/235 -# But this is a necessary step to make sure the indexes are created and properly -# configured. -./manage.py cms reindex_studio --experimental +# Create the index for studio and courseware content. Because we specify --init, +# this will not populate the index (potentially slow) nor replace any existing +# index (resulting in broken features until it is complete). If either of those +# are necessary, it will print instructions on what command to run to do so. +./manage.py cms reindex_studio --experimental --init +# Create the courseware content index ./manage.py cms reindex_course --active From 33d2bc2c71e3cd30545417afb18ba2bd989a19fd Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Wed, 11 Dec 2024 17:30:51 +0500 Subject: [PATCH 22/23] chore: add changelog for v19, small doc update --- CHANGELOG.md | 30 +++++++++++++++++++ ...131_danyal.faheem_upgrade_mysql_utf8mb4.md | 7 ----- .../20240926_151223_dave_private_media.md | 1 - ...010_122752_dawoud.sheraz_ubuntu_upgrade.md | 2 -- .../20241010_131903_regis_local_openedx_io.md | 1 - .../20241017_124915_dawoud.sheraz_sumac.md | 1 - .../20241017_125209_regis_meilisearch.md | 1 - ...1448_feanil_fix_uwsgi_xml_issues_master.md | 1 - ...08_regis_disable_forum_mysql_on_upgrade.md | 1 - ...121_110954_regis_remove_docker_rootless.md | 1 - ...ce_enable_navigation_sidebar_in_upgrade.md | 1 - docs/configuration.rst | 2 +- tutor/commands/upgrade/common.py | 3 +- 13 files changed, 33 insertions(+), 19 deletions(-) delete mode 100644 changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md delete mode 100644 changelog.d/20240926_151223_dave_private_media.md delete mode 100644 changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md delete mode 100644 changelog.d/20241010_131903_regis_local_openedx_io.md delete mode 100644 changelog.d/20241017_124915_dawoud.sheraz_sumac.md delete mode 100644 changelog.d/20241017_125209_regis_meilisearch.md delete mode 100644 changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md delete mode 100644 changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md delete mode 100644 changelog.d/20241121_110954_regis_remove_docker_rootless.md delete mode 100644 changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b73b1bdef..3bd4b06c15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,36 @@ instructions, because git commits are used to generate release notes: + +## v19.0.0 (2024-12-11) + +- πŸ’₯[Feature] Upgrade default charset and collation of mysql to utf8mb4 and utf8mb4_unicode_ci respectively (by @Danyal-Faheem) + - Add do command to upgrade the charset and collation of tables in mysql. + - The command will perform the following upgrades: + - Upgrade all `utf8mb3` charset to `utf8mb4` + - Upgrade collation `utf8mb3_general_ci` to `utf8mb4_unicode_ci` + - Upgrade collation `utf8mb3_bin` to `utf8mb4_bin` + - Upgrade collation `utf8mb3_*` to `utf8mb4_*` + +- [Feature] Create a new /data/openedx-media-private volume to store media files that are not publicly accessible via the browser, and create configuration that Learning Core (openedx-learning) will use to access this storage space. Learning Core will use a sub-directory of /data/openedx-media-private to store Content Library file uploads, but other apps can create their own sub-directories for their own use. (by @ormsbee) + +- πŸ’₯[Feature] Update Open edX Image to use Ubuntu 22.04 as base OS. (by @dawoudsheraz) + - Adds xmlsec related dependencies to fix xmlsec import issues during translations build + +- πŸ’₯[Feature] Migrate from local.edly.io to local.openedx.io. (by @regisb) + +- πŸ’₯[Feature] Upgrade to Sumac. (by @dawoudsheraz) + +- πŸ’₯[Feature] Replace Elasticsearch by Meilisearch. Elasticsearch was both a source of complexity and high resource usage. With this change, we no longer run Elasticsearch to perform common search queries across Open edX. This includes: course discovery, courseware search and studio search. Instead, we index all these documents in a Meilisearch instance, which is much more lightweight in terms of memory consumption. (by @regisb) + +- [Bugfix] Don't build uwsgi with XML support (by @feanil) + +- [Feature] With the new forum v2 application, users have a choice to use MongoDB or MySQL as a storage backend, course per course. New users will automatically start using MySQL, while existing users will be responsible for migrating their data themselves (before the Teak release). Upgrade instructions are available here: https://github.com/overhangio/tutor-forum/#installation. (by @regisb) + +- πŸ’₯[Improvement] Get rid of the `is_docker_rootless` template filter, which was used only by Elasticsearch. (by @regisb) + +[Improvement] Forcefully enable the Learning MFE's navigation sidebar when upgrading to Sumac. (by @arbrandes) + ## v18.2.2 (2024-12-10) diff --git a/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md b/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md deleted file mode 100644 index 33d9d84c1b..0000000000 --- a/changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md +++ /dev/null @@ -1,7 +0,0 @@ -- πŸ’₯[Feature] Upgrade default charset and collation of mysql to utf8mb4 and utf8mb4_unicode_ci respectively (by @Danyal-Faheem) - - Add do command to upgrade the charset and collation of tables in mysql. - - The command will perform the following upgrades: - - Upgrade all `utf8mb3` charset to `utf8mb4` - - Upgrade collation `utf8mb3_general_ci` to `utf8mb4_unicode_ci` - - Upgrade collation `utf8mb3_bin` to `utf8mb4_bin` - - Upgrade collation `utf8mb3_*` to `utf8mb4_*` \ No newline at end of file diff --git a/changelog.d/20240926_151223_dave_private_media.md b/changelog.d/20240926_151223_dave_private_media.md deleted file mode 100644 index d7d3d16581..0000000000 --- a/changelog.d/20240926_151223_dave_private_media.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] Create a new /data/openedx-media-private volume to store media files that are not publicly accessible via the browser, and create configuration that Learning Core (openedx-learning) will use to access this storage space. Learning Core will use a sub-directory of /data/openedx-media-private to store Content Library file uploads, but other apps can create their own sub-directories for their own use. (by @ormsbee) diff --git a/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md b/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md deleted file mode 100644 index 9a1adf3c98..0000000000 --- a/changelog.d/20241010_122752_dawoud.sheraz_ubuntu_upgrade.md +++ /dev/null @@ -1,2 +0,0 @@ -- πŸ’₯[Feature] Update Open edX Image to use Ubuntu 22.04 as base OS. (by @dawoudsheraz) - - Adds xmlsec related dependencies to fix xmlsec import issues during translations build diff --git a/changelog.d/20241010_131903_regis_local_openedx_io.md b/changelog.d/20241010_131903_regis_local_openedx_io.md deleted file mode 100644 index 60b1316ff6..0000000000 --- a/changelog.d/20241010_131903_regis_local_openedx_io.md +++ /dev/null @@ -1 +0,0 @@ -- πŸ’₯[Feature] Migrate from local.edly.io to local.openedx.io. (by @regisb) diff --git a/changelog.d/20241017_124915_dawoud.sheraz_sumac.md b/changelog.d/20241017_124915_dawoud.sheraz_sumac.md deleted file mode 100644 index c9412e1e8b..0000000000 --- a/changelog.d/20241017_124915_dawoud.sheraz_sumac.md +++ /dev/null @@ -1 +0,0 @@ -- πŸ’₯[Feature] Upgrade to Sumac. (by @dawoudsheraz) diff --git a/changelog.d/20241017_125209_regis_meilisearch.md b/changelog.d/20241017_125209_regis_meilisearch.md deleted file mode 100644 index 8144dda24b..0000000000 --- a/changelog.d/20241017_125209_regis_meilisearch.md +++ /dev/null @@ -1 +0,0 @@ -- πŸ’₯[Feature] Replace Elasticsearch by Meilisearch. Elasticsearch was both a source of complexity and high resource usage. With this change, we no longer run Elasticsearch to perform common search queries across Open edX. This includes: course discovery, courseware search and studio search. Instead, we index all these documents in a Meilisearch instance, which is much more lightweight in terms of memory consumption. (by @regisb) diff --git a/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md b/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md deleted file mode 100644 index 318c809dce..0000000000 --- a/changelog.d/20241113_131448_feanil_fix_uwsgi_xml_issues_master.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Don't build uwsgi with XML support (by @feanil) diff --git a/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md b/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md deleted file mode 100644 index 2decb0be93..0000000000 --- a/changelog.d/20241119_091908_regis_disable_forum_mysql_on_upgrade.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] With the new forum v2 application, users have a choice to use MongoDB or MySQL as a storage backend, course per course. New users will automatically start using MySQL, while existing users will be responsible for migrating their data themselves (before the Teak release). Upgrade instructions are available here: https://github.com/overhangio/tutor-forum/#installation. (by @regisb) diff --git a/changelog.d/20241121_110954_regis_remove_docker_rootless.md b/changelog.d/20241121_110954_regis_remove_docker_rootless.md deleted file mode 100644 index c5e5b8486e..0000000000 --- a/changelog.d/20241121_110954_regis_remove_docker_rootless.md +++ /dev/null @@ -1 +0,0 @@ -- πŸ’₯[Improvement] Get rid of the `is_docker_rootless` template filter, which was used only by Elasticsearch. (by @regisb) diff --git a/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md b/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md deleted file mode 100644 index 7a98871cf4..0000000000 --- a/changelog.d/20241209_103522_arbrandes_force_enable_navigation_sidebar_in_upgrade.md +++ /dev/null @@ -1 +0,0 @@ -[Improvement] Forcefully enable the Learning MFE's navigation sidebar when upgrading to Sumac. (by @arbrandes) diff --git a/docs/configuration.rst b/docs/configuration.rst index 99a1ddc11e..711fde2734 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -132,7 +132,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/sumac.master"``, or ``master`` in :ref:`Tutor Main
`) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/sumac.1"``, or ``master`` in :ref:`Tutor Main
`) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index 2c4d47be2e..f711270bd2 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -50,7 +50,8 @@ def upgrade_from_redwood(context: click.Context, config: Config) -> None: # Forcefully enable the learning MFE's navigation sidebar. if plugins.is_loaded("mfe"): context.obj.job_runner(config).run_task( - "lms", "./manage.py lms waffle_flag --create --everyone courseware.enable_navigation_sidebar" + "lms", + "./manage.py lms waffle_flag --create --everyone courseware.enable_navigation_sidebar", ) # Prevent switching to the MySQL storage backend in forum v2 From 2310fe67cbf98772b2c5bcb9d62b0a66b12ab8ed Mon Sep 17 00:00:00 2001 From: Syed Muhammad Dawoud Sheraz Ali Date: Mon, 16 Dec 2024 21:11:46 +0500 Subject: [PATCH 23/23] chore: update OPENEDX_COMMON_VERSION to sumac.1 tag --- tutor/templates/config/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 9a97a5a974..aa8bd9ab9a 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -60,7 +60,7 @@ OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" # the common version will be automatically set to "master" in the main branch -OPENEDX_COMMON_VERSION: "open-release/sumac.master" +OPENEDX_COMMON_VERSION: "open-release/sumac.1" OPENEDX_EXTRA_PIP_REQUIREMENTS: [] MYSQL_HOST: "mysql" MYSQL_PORT: 3306