Merge pull request #224 from gboutry/feat/risk-detection

Use manifest from detected risk

sunbeam-python/sunbeam/commands/prepare_node.py

@@ -86,22 +86,25 @@
 # Check the snap channel and deduce risk level from it
 snap_output=$(snap list openstack --unicode=never --color=never | grep openstack)
 track=$(awk -v col=4 '{{print $col}}' <<<"$snap_output")
-risk=stable
 
 # if never installed from the store, the channel is "-"
 if [[ $track =~ "edge" ]] || [[ $track == "-" ]]; then
     risk="edge"
-fi
-
-if [[ $track =~ "candidate" ]]; then
+elif [[ $track =~ "beta" ]]; then
+    risk="beta"
+elif [[ $track =~ "candidate" ]]; then
     risk="candidate"
+else
+    risk="stable"
 fi
 
 if [[ $risk != "stable" ]]; then
     echo "You're deploying from $risk channel," \
         " to test $risk charms, you must provide the $risk manifest."
-    echo "Example: sunbeam cluster bootstrap " \
-        "--manifest /snap/openstack/current/etc/manifests/$risk.yml"
+    sudo snap set openstack deployment.risk=$risk
+    echo "Snap has been automatically configured to deploy from" \
+        "$risk channel."
+    echo "Override by passing a custom manifest with -m/--manifest."
 fi
 """
 

sunbeam-python/sunbeam/commands/refresh.py

@@ -75,7 +75,7 @@ def refresh(
     # Validate manifest file
     manifest = None
     if clear_manifest:
-        run_plan([AddManifestStep(client)], console)
+        run_plan([AddManifestStep(client, clear=True)], console)
     elif manifest_path:
         manifest = deployment.get_manifest(manifest_path)
         run_plan([AddManifestStep(client, manifest_path)], console)

sunbeam-python/sunbeam/hooks.py

@@ -27,6 +27,7 @@
     "daemon.group": "snap_daemon",
     "daemon.debug": False,
     "k8s.provider": "microk8s",
+    "deployment.risk": "stable",
 }
 
 OPTION_KEYS = set(k.split(".")[0] for k in DEFAULT_CONFIG.keys())

sunbeam-python/sunbeam/jobs/common.py

@@ -27,6 +27,7 @@
 from juju.client.client import FullStatus
 from rich.console import Console
 from rich.status import Status
+from snaphelpers import Snap, UnknownConfigKey
 
 from sunbeam.clusterd.client import Client
 
@@ -422,3 +423,30 @@ class SunbeamException(Exception):
     """Base exception for sunbeam."""
 
     pass
+
+
+class RiskLevel(str, enum.Enum):
+    STABLE = "stable"
+    CANDIDATE = "candidate"
+    BETA = "beta"
+    EDGE = "edge"
+
+
+def infer_risk(snap: Snap) -> RiskLevel:
+    """Compute risk level from environment."""
+    try:
+        risk = snap.config.get("deployment.risk")
+    except UnknownConfigKey:
+        return RiskLevel.STABLE
+
+    match risk:
+        case "candidate":
+            return RiskLevel.CANDIDATE
+        # Beta and edge are considered the same for now
+        case "beta":
+            LOG.debug("Beta channel detected, using edge instead.")
+            return RiskLevel.EDGE
+        case "edge":
+            return RiskLevel.EDGE
+        case _:
+            return RiskLevel.STABLE

sunbeam-python/sunbeam/jobs/deployment.py

@@ -31,9 +31,14 @@
     ConfigItemNotFoundException,
 )
 from sunbeam.commands.terraform import TerraformHelper
-from sunbeam.jobs.common import _get_default_no_proxy_settings, read_config
+from sunbeam.jobs.common import (
+    RiskLevel,
+    _get_default_no_proxy_settings,
+    infer_risk,
+    read_config,
+)
 from sunbeam.jobs.juju import JujuAccount, JujuController
-from sunbeam.jobs.manifest import Manifest
+from sunbeam.jobs.manifest import Manifest, embedded_manifest_path
 from sunbeam.versions import MANIFEST_ATTRIBUTES_TFVAR_MAP, TERRAFORM_DIR_NAMES
 
 LOG = logging.getLogger(__name__)
@@ -175,18 +180,27 @@ def get_manifest(self, manifest_file: pathlib.Path | None = None) -> Manifest:
             except ClusterServiceUnavailableException:
                 LOG.debug(
                     "Failed to get manifest from clusterd, might not be bootstrapped,"
-                    " consider empty manifest from database."
+                    " consider default manifest."
                 )
             except ConfigItemNotFoundException:
                 LOG.debug(
-                    "No manifest found in clusterd, consider empty"
+                    "No manifest found in clusterd, consider default"
                     " manifest from database."
                 )
             except ValueError:
                 LOG.debug(
                     "Failed to get clusterd client, might no be bootstrapped,"
                     " consider empty manifest from database."
                 )
+            if override_manifest is None:
+                # Only get manifest from embedded if manifest not present in clusterd
+                snap = Snap()
+                risk = infer_risk(snap)
+                if risk != RiskLevel.STABLE:
+                    manifest_file = embedded_manifest_path(snap, risk)
+                    LOG.debug(f"Risk {risk.value} detected, loading {manifest_file}...")
+                    override_manifest = Manifest.from_file(manifest_file)
+                    LOG.debug("Manifest loaded from embedded manifest.")
 
         if override_manifest is not None:
             override_manifest.validate_against_default(manifest)

sunbeam-python/sunbeam/jobs/manifest.py

@@ -29,13 +29,24 @@
     ClusterServiceUnavailableException,
     ManifestItemNotFoundException,
 )
-from sunbeam.jobs.common import BaseStep, Result, ResultType, Status
+from sunbeam.jobs.common import (
+    BaseStep,
+    Result,
+    ResultType,
+    RiskLevel,
+    Status,
+    infer_risk,
+)
 from sunbeam.versions import MANIFEST_CHARM_VERSIONS, TERRAFORM_DIR_NAMES
 
 LOG = logging.getLogger(__name__)
 EMPTY_MANIFEST = {"charms": {}, "terraform": {}}
 
 
+def embedded_manifest_path(snap: Snap, risk: str) -> Path:
+    return snap.paths.snap / "etc" / "manifests" / f"{risk}.yml"
+
+
 class JujuManifest(pydantic.BaseModel):
     # Setting Field alias not supported in pydantic 1.10.0
     # Old version of pydantic is used due to dependencies
@@ -195,29 +206,54 @@ def validate_against_default(self, default_manifest: "Manifest") -> None:
 
 
 class AddManifestStep(BaseStep):
-    """Add Manifest file to cluster database"""
-
-    def __init__(self, client: Client, manifest: Optional[Path] = None):
+    """Add Manifest file to cluster database.
+
+    This step writes the manifest file to cluster database if:
+    - The user provides a manifest file.
+    - The user clears the manifest.
+    - The risk level is not stable.
+    Any other reason will be skipped.
+    """
+
+    def __init__(
+        self,
+        client: Client,
+        manifest_file: Path | None = None,
+        clear: bool = False,
+    ):
         super().__init__("Write Manifest to database", "Writing Manifest to database")
-        # Write EMPTY_MANIFEST if manifest not provided
-        self.manifest = manifest
         self.client = client
+        self.manifest_file = manifest_file
+        self.clear = clear
         self.manifest_content = None
+        self.snap = Snap()
 
     def is_skip(self, status: Optional[Status] = None) -> Result:
         """Skip if the user provided manifest and the latest from db are same."""
+        risk = infer_risk(self.snap)
         try:
-            if self.manifest:
-                with self.manifest.open("r") as file:
+            if self.manifest_file:
+                with self.manifest_file.open("r") as file:
                     self.manifest_content = yaml.safe_load(file)
-            else:
+            elif self.clear:
                 self.manifest_content = EMPTY_MANIFEST
+            elif risk != RiskLevel.STABLE:
+                self.manifest_content = yaml.safe_load(
+                    embedded_manifest_path(self.snap, risk).read_bytes()
+                )
+            else:
+                # No manifest to update
+                return Result(ResultType.SKIPPED)
+        except (yaml.YAMLError, IOError) as e:
+            LOG.debug("Failed to load manifest", exc_info=True)
+            return Result(ResultType.FAILED, str(e))
 
+        try:
             latest_manifest = self.client.cluster.get_latest_manifest()
         except ManifestItemNotFoundException:
             return Result(ResultType.COMPLETED)
-        except (ClusterServiceUnavailableException, yaml.YAMLError, IOError) as e:
-            LOG.debug(e)
+        except ClusterServiceUnavailableException as e:
+            LOG.debug("Failed to fetch latest manifest from clusterd", exc_info=True)
             return Result(ResultType.FAILED, str(e))
 
         if yaml.safe_load(latest_manifest.get("data", {})) == self.manifest_content:

sunbeam-python/sunbeam/provider/local/commands.py

@@ -276,8 +276,7 @@ def bootstrap(
     plan.append(JujuLoginStep(deployment.juju_account))
     # bootstrapped node is always machine 0 in controller model
     plan.append(ClusterInitStep(client, roles_to_str_list(roles), 0))
-    if manifest_path:
-        plan.append(AddManifestStep(client, manifest_path))
+    plan.append(AddManifestStep(client, manifest_path))
     plan.append(
         PromptForProxyStep(
             deployment, accept_defaults=accept_defaults, deployment_preseed=preseed

sunbeam-python/sunbeam/provider/maas/commands.py

@@ -367,10 +367,9 @@ def bootstrap(
         sys.exit(1)
 
     client = deployment.get_client()
-    if manifest_path:
-        plan3 = []
-        plan3.append(AddManifestStep(client))
-        run_plan(plan3, console)
+    plan3 = []
+    plan3.append(AddManifestStep(client, manifest_path))
+    run_plan(plan3, console)
 
     if proxy_from_user and isinstance(proxy_from_user, dict):
         LOG.debug(f"Writing proxy information to clusterdb: {proxy_from_user}")

sunbeam-python/tests/unit/sunbeam/conftest.py

@@ -12,30 +12,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 import pytest
 from snaphelpers import Snap, SnapConfig, SnapServices
 
 
 @pytest.fixture
-def snap_env():
+def snap_env(tmp_path: Path, mocker):
     """Environment variables defined in the snap.
 
     This is primarily used to setup the snaphelpers bit.
     """
-    yield {
-        "SNAP": "/snap/mysnap/2",
-        "SNAP_COMMON": "/var/snap/mysnap/common",
-        "SNAP_DATA": "/var/snap/mysnap/2",
+    snap_name = "sunbeam-test"
+    real_home = tmp_path / "home/ubuntu"
+    snap_user_common = real_home / f"snap/{snap_name}/common"
+    snap_user_data = real_home / f"snap/{snap_name}/2"
+    snap_path = tmp_path / f"snap/2/{snap_name}"
+    snap_common = tmp_path / f"var/snap/{snap_name}/common"
+    snap_data = tmp_path / f"var/snap/{snap_name}/2"
+    env = {
+        "SNAP": str(snap_path),
+        "SNAP_COMMON": str(snap_common),
+        "SNAP_DATA": str(snap_data),
+        "SNAP_USER_COMMON": str(snap_user_common),
+        "SNAP_USER_DATA": str(snap_user_data),
+        "SNAP_REAL_HOME": str(real_home),
         "SNAP_INSTANCE_NAME": "",
-        "SNAP_NAME": "mysnap",
+        "SNAP_NAME": snap_name,
         "SNAP_REVISION": "2",
-        "SNAP_USER_COMMON": "/var/snap/mysnap/usercommon",
-        "SNAP_USER_DATA": "",
         "SNAP_VERSION": "1.2.3",
-        "SNAP_REAL_HOME": "/home/ubuntu",
     }
+    mocker.patch("os.environ", env)
+    yield env
 
 
 @pytest.fixture

sunbeam-python/tests/unit/sunbeam/jobs/test_checks.py

@@ -16,7 +16,6 @@
 import base64
 import json
 import os
-from pathlib import PosixPath
 from unittest.mock import Mock
 
 from sunbeam.jobs import checks
@@ -44,7 +43,7 @@ def test_run_missing_interface(self, mocker, snap):
         result = check.run()
 
         assert result is False
-        assert "sudo snap connect mysnap:ssh-keys" in check.message
+        assert f"sudo snap connect {snap.name}:ssh-keys" in check.message
 
 
 class TestDaemonGroupCheck:
@@ -80,7 +79,7 @@ def test_run(self, mocker, snap):
         result = check.run()
 
         assert result is True
-        os.path.exists.assert_called_with(PosixPath("/home/ubuntu/.local/share"))
+        os.path.exists.assert_called_with(snap.paths.real_home / ".local/share")
 
     def test_run_missing(self, mocker, snap):
         mocker.patch.object(checks, "Snap", return_value=snap)
@@ -92,7 +91,7 @@ def test_run_missing(self, mocker, snap):
 
         assert result is False
         assert "directory not detected" in check.message
-        os.path.exists.assert_called_with(PosixPath("/home/ubuntu/.local/share"))
+        os.path.exists.assert_called_with(snap.paths.real_home / ".local/share")
 
 
 class TestVerifyFQDNCheck:

sunbeam-python/tests/unit/sunbeam/jobs/test_deployment.py

@@ -51,7 +51,11 @@
 
 
 @pytest.fixture()
-def deployment():
+def deployment(mocker, snap):
+    mocker.patch.object(manifest_mod, "Snap", return_value=snap)
+    mocker.patch.object(deployment_mod, "Snap", return_value=snap)
+    snap_config = {"deployment.risk": "stable"}
+    snap.config.get.side_effect = snap_config.__getitem__
     with patch("sunbeam.jobs.deployment.Deployment") as p:
         dep = p(name="", url="", type="")
         dep.get_manifest.side_effect = functools.partial(Deployment.get_manifest, dep)
@@ -68,16 +72,14 @@ def deployment():
 
 class TestDeployment:
 
-    def test_get_default_manifest(self, mocker, snap, deployment: Deployment):
-        mocker.patch.object(manifest_mod, "Snap", return_value=snap)
+    def test_get_default_manifest(self, deployment: Deployment):
         manifest = deployment.get_manifest()
 
         # Assert core charms / plans are present
         assert set(manifest.software.charms.keys()) >= MANIFEST_CHARM_VERSIONS.keys()
         assert set(manifest.software.terraform.keys()) >= TERRAFORM_DIR_NAMES.keys()
 
-    def test_load_on_default(self, mocker, snap, deployment: Deployment, tmpdir):
-        mocker.patch.object(manifest_mod, "Snap", return_value=snap)
+    def test_load_on_default(self, deployment: Deployment, tmpdir):
         manifest_file = tmpdir.mkdir("manifests").join("test_manifest.yaml")
         manifest_file.write(test_manifest)
         manifest_obj = deployment.get_manifest(manifest_file)
@@ -94,8 +96,7 @@ def test_load_on_default(self, mocker, snap, deployment: Deployment, tmpdir):
         assert nova_manifest.revision is None
         assert nova_manifest.config is None
 
-    def test_load_latest_from_clusterdb(self, mocker, snap, deployment: Deployment):
-        mocker.patch.object(manifest_mod, "Snap", return_value=snap)
+    def test_load_latest_from_clusterdb(self, deployment: Deployment):
         client = Mock()
         client.cluster.get_latest_manifest.return_value = {"data": test_manifest}
         deployment.get_client.side_effect = None