test(bdd): make test more robust

Ensure install is properly completed
Ensure upgrade is properly completed
Compare against v-next and not v-next since there's no current way
of knowing which version is the current helm chart.
This may get addressed if we start building the images and the
chart as v-next from the outset.

Signed-off-by: Tiago Castro <[email protected]>

scripts/helm/install.sh

@@ -7,22 +7,23 @@ repo_add() {
   local -r preferred_name=$2
 
   local repo
-  if [ "$(helm repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then
-    repo=$(helm repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name")
+  if [ -z "$DRY_RUN" ] && [ "$($HELM repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then
+    repo=$($HELM repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name")
   else
-    helm repo add "$preferred_name" "$url" > /dev/null
+    $HELM repo add "$preferred_name" "$url" > /dev/null
     repo=$preferred_name
   fi
 
-  helm repo update > /dev/null || true
+  $HELM repo update > /dev/null || true
 
   echo "$repo"
 }
 
 
 TIMEOUT="5m"
 WAIT=
-DRY_RUN=""
+DRY_RUN=
+HELM_DRY_RUN=""
 SCRIPT_DIR="$(dirname "$0")"
 CHART_DIR="$SCRIPT_DIR"/../../chart
 CHART_SOURCE=$CHART_DIR
@@ -34,6 +35,8 @@ HOSTED=
 VERSION=
 REGISTRY=
 DEFAULT_REGISTRY="https://openebs.github.io/mayastor-extensions"
+HELM="helm"
+KUBECTL="kubectl"
 
 help() {
   cat <<EOF
@@ -43,7 +46,8 @@ Options:
   -h, --help                     Display this text.
   --timeout  <timeout>           How long to wait for helm to complete install (Default: $TIMEOUT).
   --wait                         Wait for helm to complete install.
-  --dry-run                      Install helm with --dry-run.
+  --dry-run                      Don't run any commands, output them only.
+  --helm-dry-run                 Install helm with --dry-run.
   --dep-update                   Run helm dependency update.
   --fail-if-installed            Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace.
   --hosted-chart                 Install a hosted chart instead of the local chart.
@@ -79,8 +83,13 @@ while [ "$#" -gt 0 ]; do
     --wait)
       WAIT="yes"
       shift;;
+    --helm-dry-run)
+      HELM_DRY_RUN=" --dry-run"
+      shift;;
     --dry-run)
-      DRY_RUN=" --dry-run"
+      DRY_RUN="yes"
+      HELM="echo $HELM"
+      KUBECTL="echo $KUBECTL"
       shift;;
     --dep-update)
       DEP_UPDATE="y"
@@ -136,7 +145,7 @@ if [ -n "$HOSTED" ]; then
   DEP_UPDATE_ARG=
 fi
 
-if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
+if [ -z "$DRY_RUN" ] && [ "$($HELM ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
   already_exists_log="Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
   if [ -n "$FAIL_IF_INSTALLED" ]; then
     die "ERROR: $already_exists_log" 1
@@ -145,12 +154,12 @@ if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEAS
 else
   echo "Installing Mayastor Chart"
   set -x
-  helm install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
+  $HELM install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
        --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \
        --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \
        --set="eventing.enabled=false" \
-       $DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG
+       $HELM_DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG
   set +x
 fi
 
-kubectl get pods -n "$K8S_NAMESPACE" -o wide
+$KUBECTL get pods -n "$K8S_NAMESPACE" -o wide

tests/bdd/README.md

@@ -0,0 +1,58 @@
+# BDD Tests
+
+The BDD tests are written in Python and make use of the pytest-bdd library.
+
+The feature files in the `features` directory define the behaviour expected of mayastor. These behaviours are
+described using the [Gherkin](https://cucumber.io/docs/gherkin/) syntax.
+
+The feature files can be used to auto-generate the test file. For example
+running `pytest-bdd generate upgrade.feature > test_upgrade.py`
+generates the `test_upgrade.py` test file from the `upgrade.feature` file.
+When updating the feature file, you can also get some helpe updating the python code.
+Example: `pytest --generate-missing --feature upgrade.feature test_upgrade.py`
+
+**:warning: Note: Running pytest-bdd generate will overwrite any existing files with the same name**
+
+## Running the Tests by entering the python virtual environment
+
+Before running any tests run the `setup.sh` script. This sets up the necessary environment to run the tests:
+
+```bash
+# NOTE: you should be inside the nix-shell to begin
+source ./setup.sh
+```
+
+To run all the tests:
+
+```bash
+pytest .
+```
+
+To run individual test files:
+
+```bash
+pytest features/test_upgrade.py
+```
+
+To run an individual test within a test file use the `-k` option followed by the test name:
+
+```bash
+pytest features/test_upgrade.py -k test_upgrade_to_vnext
+```
+
+## Running the Tests
+
+The script in `../../scripts/python/test.sh` can be used to run the tests without entering the venv.
+This script will implicitly enter and exit the venv during test execution.
+
+To run all the tests:
+
+```bash
+../../scripts/python/test.sh
+```
+
+Arguments will be passed directly to pytest. Example running individual tests:
+
+```bash
+../../scripts/python/test.sh features/test_upgrade.py -k test_upgrade_to_vnext
+```

tests/bdd/common/environment.py

@@ -4,7 +4,7 @@
 logger = logging.getLogger(__name__)
 
 
-def get_env(variable: str, warn=True):
+def get_env(variable: str, warn=False):
     value = os.getenv(variable)
     if value is None:
         if warn:

tests/bdd/common/kubectl_mayastor.py

@@ -32,15 +32,14 @@ def kubectl_mayastor(args: list[str]):
             check=True,
             text=True,
         )
-        logger.info(f"kubectl-mayastor command succeeded")
-        logger.error(f"Error Output: {result.stderr}\nOut Output: {result.stdout}")
+        logger.debug(f"Error Output: {result.stderr}\nOut Output: {result.stdout}")
         return result.stdout.strip()
 
     except subprocess.CalledProcessError as e:
-        logger.error(f"Error: command '{command}' failed with exit code {e.returncode}")
-        logger.error(f"Error Output: {e.stderr}\nOut Output: {e.stdout}")
+        logger.error(
+            f"Error: command '{command}' failed with exit code {e.returncode}\nError Output: {e.stderr}\nOut Output: {e.stdout}")
         raise e
 
     except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}")
+        logger.error(f"An unexpected error occurred whilst running kubectl-mayastor: {e}")
         raise e

tests/bdd/features/test_upgrade.py

@@ -1,5 +1,5 @@
 """Upgrade feature tests."""
-
+import json
 import logging
 
 import pytest
@@ -17,15 +17,15 @@
 helm = HelmReleaseClient()
 
 
-@scenario("upgrade.feature", "upgrade command is issued")
-def test_upgrade_command_is_issued():
-    """upgrade command is issued."""
+@scenario("upgrade.feature", "Upgrading to the local chart as v-next")
+def test_upgrade_to_vnext():
+    """Upgrading to the local chart as v-next."""
 
 
-@given("an installed mayastor helm chart")
-def an_installed_mayastor_helm_chart():
-    """an installed mayastor helm chart."""
-    helm.install_mayastor(ChartSource.HOSTED, latest_chart_so_far())
+@given("the latest mayastor helm chart is installed")
+def the_latest_mayastor_is_installed(latest_chart_version):
+    """the latest mayastor helm chart is installed."""
+    helm.install_mayastor(ChartSource.HOSTED, latest_chart_version)
 
 
 @when("a kubectl mayastor upgrade command is issued")
@@ -34,8 +34,14 @@ def a_kubectl_mayastor_upgrade_command_is_issued():
     kubectl_mayastor(["upgrade"])
 
 
-@then("the installed chart should be upgraded to the kubectl mayastor plugin's version")
-def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version():
+@then("all io-engine nodes shall be listed by kubectl-mayastor")
+def all_io_engine_nodes_shall_be_listed(latest_chart_version):
+    """all io-engine nodes shall be listed by kubectl-mayastor."""
+    wait_rest_nodes_version(latest_chart_version)
+
+
+@then("eventually the installed chart should be upgraded to the kubectl mayastor plugin's version")
+def eventually_the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version(latest_chart_version):
     """the installed chart should be upgraded to the kubectl mayastor plugin's version."""
 
     upgrade_target_version = get_env("UPGRADE_TARGET_VERSION")
@@ -59,8 +65,7 @@ def helm_upgrade_succeeded():
             logger.info("Checking if helm upgrade succeeded...")
         metadata = helm.get_metadata_mayastor()
         if log:
-            logger.debug(f"helm get metadata output={metadata}")
-            logger.debug(f"upgrade_target_version={upgrade_target_version}")
+            logger.info(f"helm get metadata output={metadata}")
         if metadata:
             assert metadata["version"] == upgrade_target_version
             return
@@ -70,7 +75,7 @@ def helm_upgrade_succeeded():
         stop_max_attempt_number=600,
         wait_fixed=2000,
     )
-    def data_plane_upgrade_succeeded(target_tag):
+    def data_plane_upgrade_succeeded(not_target_tag):
         log = log_it()
         if log:
             logger.info("Checking if data-plane upgrade succeeded...")
@@ -93,7 +98,9 @@ def data_plane_upgrade_succeeded(target_tag):
         for pod in io_engines:
             for i, container in enumerate(pod.spec.containers):
                 if container.name == "io-engine":
-                    if not container.image.endswith(f":{target_tag}"):
+                    # Not straightforward to know which version to expect here, so let's check that
+                    # the version is not the latest instead?
+                    if container.image.endswith(f":v{not_target_tag.strip('v')}"):
                         all_done = False
                     if log:
                         logger.info(
@@ -102,8 +109,50 @@ def data_plane_upgrade_succeeded(target_tag):
                     break
         assert all_done is True
 
+        nodes = client.CoreV1Api().list_node(
+            label_selector="openebs.io/engine=mayastor"
+        )
+
+        assert len(nodes.items) == len(io_engines)
+
     pytest.attempts = 0
     helm_upgrade_succeeded()
     pytest.attempts = 0
-    # todo: should be release-$v on release branches
-    data_plane_upgrade_succeeded("develop")
+
+    data_plane_upgrade_succeeded(latest_chart_version)
+
+    # Not straightforward to know which version to expect here, so let's check that
+    # the version is not the latest instead?
+    wait_rest_nodes_version(latest_chart_version, match=False)
+
+
+@pytest.fixture(scope="module")
+def latest_chart_version():
+    yield latest_chart_so_far()
+
+
+@retry(
+    stop_max_attempt_number=60,
+    wait_fixed=1000,
+)
+def wait_rest_nodes_version(version, match=True):
+    config.load_kube_config()
+    nodes = client.CoreV1Api().list_node(
+        label_selector="openebs.io/engine=mayastor"
+    )
+    k8s_nodes = len(nodes.items)
+
+    rest_nodes = json.loads(kubectl_mayastor(["get", "nodes", "-o=json"]))
+    rest_io_engines = len(rest_nodes)
+
+    assert k8s_nodes == rest_io_engines, f"Found {k8s_nodes} k8s nodes with the io-engine label, but only {rest_io_engines} nodes from kubectl-mayastor"
+
+    assert all(node["spec"]["version"] == node["state"]["version"] for node in rest_nodes)
+
+    version_stripped = version.strip("v")
+    if match:
+        all_on_version = all(node["spec"]["version"].strip("v") == version_stripped for node in rest_nodes)
+        assert all_on_version, f"Not all nodes on the version v{version_stripped}"
+    else:
+        all_not_on_version = all(node["spec"]["version"].strip("v") != version_stripped for node in rest_nodes)
+        assert all_not_on_version, f"Some of the nodes are still on the version v{version_stripped}"

tests/bdd/features/upgrade.feature

@@ -1,8 +1,9 @@
 Feature: Upgrade
 
   Background:
-    Given an installed mayastor helm chart
+    Given the latest mayastor helm chart is installed
+    Then all io-engine nodes shall be listed by kubectl-mayastor
 
-  Scenario: upgrade command is issued
+  Scenario: Upgrading to the local chart as v-next
     When a kubectl mayastor upgrade command is issued
-    Then the installed chart should be upgraded to the kubectl mayastor plugin's version
+    Then eventually the installed chart should be upgraded to the kubectl mayastor plugin's version