Merge pull request openedx#32239 from openedx/feanil/migration_check

Migrate migration checks to `minimal.yml` config.

.github/workflows/migrations-check.yml

@@ -1,4 +1,4 @@
-name: Migrations check on MySql 57
+name: Check Django Migrations
 
 on:
   workflow_dispatch:
@@ -15,8 +15,54 @@ jobs:
       matrix:
         os: [ ubuntu-20.04 ]
         python-version: [ 3.8 ]
-
+        mongo-version: ["4"]
+        mysql-version: ["5.7", "8"]
+    services:
+      mongo:
+        image: mongo:${{ matrix.mongo-version }}
+        ports:
+          - 27017:27017
+        # Note: Calling mongo here only works with mongo 4, in newer versions of mongo
+        # we'll have to use `mongosh`
+        options: >-
+          --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
+      mysql:
+        image: mysql:${{ matrix.mysql-version }}
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_DATABASE: "edxapp"
+          MYSQL_USER: "edxapp001"
+          MYSQL_PASSWORD: "password"
+          MYSQL_RANDOM_ROOT_PASSWORD: true
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
     steps:
+    - name: Setup mongodb user
+      run: |
+        mongosh edxapp --eval '
+          db.createUser(
+            {
+              user: "edxapp",
+              pwd:  "password",
+              roles: [
+                   { role: "readWrite", db: "edxapp" },
+              ]
+            }
+          );
+        '
+
+    - name: Verify mongo and mysql db credentials
+      run: |
+        mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp
+        mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp
+
     - name: Checkout repo
       uses: actions/checkout@v2
 
@@ -28,7 +74,7 @@ jobs:
     - name: Install system Packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y libxmlsec1-dev
+        make ubuntu-requirements
 
     - name: Get pip cache dir
       id: pip-cache-dir
@@ -43,51 +89,17 @@ jobs:
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
         restore-keys: ${{ runner.os }}-pip-
 
-    - name: Install MySQL 5.7
-      run: |
-        mkdir ${RUNNER_WORKSPACE}/mysql_packages && cd ${RUNNER_WORKSPACE}/mysql_packages
-        sudo apt-get remove --purge *mysql* -y
-        sudo rm -rvf /etc/init.d/mysql* /etc/mysql* /var/lib/mysql*
-        sudo apt-get install libaio1 libnuma1 libtinfo5 -y
-        wget https://cdn.mysql.com/archives/mysql-5.7/mysql-server_5.7.35-1ubuntu18.04_amd64.deb-bundle.tar
-        tar -xf mysql-server_5.7*.tar
-        sudo DEBIAN_FRONTEND=noninteractive dpkg --install mysql-common_5.7*.deb libmysqlclient*.deb mysql-server_5.7*.deb mysql-client_5.7*.deb mysql-community-server_5.7*.deb mysql-community-client_5.7*.deb
-
-    - name: Ubuntu and sql Versions
-      run: |
-        lsb_release -a
-        mysql -V
-
     - name: Install Python dependencies
       run: |
         make dev-requirements
 
-    - name: Initiate Services
-      run: |
-        sudo systemctl start mongod
-        sudo /etc/init.d/mysql start
-
-    - name: Reset mysql password
-      run: |
-        cat <<EOF | sudo mysql -u root
-          UPDATE mysql.user SET plugin = 'mysql_native_password' WHERE user = 'root';
-          UPDATE mysql.user SET authentication_string = null WHERE user = 'root';
-          FLUSH PRIVILEGES;
-        EOF
-    
-    - name: Install mysqlclient-dev binary
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libmysqlclient-dev
-
     - name: Run Tests
       env:
-        LMS_CFG: lms/envs/bok_choy.yml
+        LMS_CFG: lms/envs/minimal.yml
+        # This is from the LMS dir on purpose since we don't need anything different for the CMS yet.
+        STUDIO_CFG: lms/envs/minimal.yml
       run: |
-        echo "CREATE DATABASE IF NOT EXISTS edxtest;" | sudo mysql -u root
-        echo "CREATE DATABASE IF NOT EXISTS student_module_history_test;" | sudo mysql -u root
-
         echo "Running the LMS migrations."
-        ./manage.py lms --settings bok_choy migrate
+        ./manage.py lms migrate
         echo "Running the CMS migrations."
-        ./manage.py cms --settings bok_choy migrate
+        ./manage.py cms migrate

cms/envs/common.py

@@ -1052,7 +1052,7 @@
         'ATOMIC_REQUESTS': True,
         'CONN_MAX_AGE': 0,
         'ENGINE': 'django.db.backends.mysql',
-        'HOST': 'localhost',
+        'HOST': '127.0.0.1',
         'NAME': 'edxapp',
         'OPTIONS': {},
         'PASSWORD': 'password',
@@ -1062,7 +1062,7 @@
     'read_replica': {
         'CONN_MAX_AGE': 0,
         'ENGINE': 'django.db.backends.mysql',
-        'HOST': 'localhost',
+        'HOST': '127.0.0.1',
         'NAME': 'edxapp',
         'OPTIONS': {},
         'PASSWORD': 'password',
@@ -1072,7 +1072,7 @@
     'student_module_history': {
         'CONN_MAX_AGE': 0,
         'ENGINE': 'django.db.backends.mysql',
-        'HOST': 'localhost',
+        'HOST': '127.0.0.1',
         'NAME': 'edxapp_csmh',
         'OPTIONS': {},
         'PASSWORD': 'password',

lms/envs/common.py

@@ -1613,7 +1613,7 @@ def _make_mako_template_dirs(settings):
                     'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
                     'OPTIONS': {
                         'default_class': 'xmodule.hidden_block.HiddenBlock',
-                        'fs_root': DATA_DIR,
+                        'fs_root': lambda settings: settings.DATA_DIR,
                         'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string',
                     }
                 },
@@ -1623,7 +1623,7 @@ def _make_mako_template_dirs(settings):
                     'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
                     'OPTIONS': {
                         'default_class': 'xmodule.hidden_block.HiddenBlock',
-                        'fs_root': DATA_DIR,
+                        'fs_root': lambda settings: settings.DATA_DIR,
                         'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string',
                     }
                 }
@@ -1632,6 +1632,7 @@ def _make_mako_template_dirs(settings):
     }
 }
 
+
 DATABASES = {
     # edxapp's edxapp-migrate scripts and the edxapp_migrate play
     # will ensure that any DB not named read_replica will be migrated

lms/envs/minimal.yml

@@ -26,6 +26,10 @@ REST_FRAMEWORK: {}
 # off-machine storage to more easily scale the system.
 MEDIA_ROOT: "/tmp/edx-platform/media/"
 
+# DATA_DIR is overridden twice in ./lms/envs/common.py  override it here temporarily until that's cleaned up and we
+# can default to something dev friendly and have overrides in production.py for production friendly settings.
+DATA_DIR: "/tmp/edx-platform/data_dir"
+
 # For just the CMS
 LMS_ROOT_URL: "http://localhost"
 LMS_INTERNAL_ROOT_URL: "http://localhost"

lms/envs/production.py

@@ -490,6 +490,18 @@ def get_env_setting(setting):
 # Get the MODULESTORE from auth.json, but if it doesn't exist,
 # use the one from common.py
 MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE))
+
+# After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the
+# fs_root entry to derived collection so that if it's a callable it can be resolved.  We need to do this because the
+# `derived_collection_entry` takes an exact index value but the config file might have overidden the number of stores
+# and so we can't be sure that the 2 we define in common.py will be there when we try to derive settings.  This could
+# lead to execptions being thrown when the `derive_settings` call later in this file tries to update settings.  We call
+# the derived_collection_entry function here to ensure that we update the fs_root for any callables that remain after
+# we've updated the MODULESTORE setting from our config file.
+for idx, store in enumerate(MODULESTORE['default']['OPTIONS']['stores']):
+    if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]:
+        derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root')
+
 MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {})
 
 EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '')  # django default is ''