Added tests

doc/changes/changes_1.1.0.md

@@ -12,9 +12,9 @@ Version: 1.1.0
 
 ## Features
 
-* 223: Added support to add docker image tag "latest"
-* 204: Updated developer guide
-* 177: Disabled core dumps
+* #223: Added support to add docker image tag "latest"
+* #204: Updated developer guide
+* #177: Disabled core dumps
 
 ## Security
 
@@ -26,7 +26,7 @@ n/a
 
 ## Documentation
 
-* 204: Updated developer guide
+* #204: Updated developer guide
 * #219: Described Virtual Box setup in user guide
 
 ## Refactoring
@@ -36,3 +36,4 @@ n/a
 * #221: Changed wording in the main configuration notebook, as suggested by PM.
 * #66: Used a non-root user to run Jupyter in the Docker Image ai-lab
 * #149: Split AWS tests
+* #252: Added tests for access to Docker socket

exasol/ds/sandbox/cli/commands/create_docker_image.py

@@ -8,15 +8,13 @@
     DEFAULT_ORG_AND_REPOSITORY,
     DssDockerImage,
     DockerRegistry,
+    USER_ENV,
+    PASSWORD_ENV,
 )
 from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS
 from exasol.ds.sandbox.lib.logging import set_log_level
 
 
-USER_ENV = "DOCKER_REGISTRY_USER"
-PASSWORD_ENV = "DOCKER_REGISTRY_PASSWORD"
-
-
 @cli.command()
 @add_options([
     click.option(

exasol/ds/sandbox/lib/dss_docker/__init__.py

@@ -1,2 +1,9 @@
 from .create_image import DssDockerImage, DEFAULT_ORG_AND_REPOSITORY
 from .push_image import DockerRegistry
+
+# Names of environment variables for user and password to access docker
+# services.  This is especially required for rate limits docker hub sometimes
+# applies for tests running on AWS CodeBuild.
+
+USER_ENV = "DOCKER_REGISTRY_USER"
+PASSWORD_ENV = "DOCKER_REGISTRY_PASSWORD"

test/docker/container.py

@@ -1,8 +1,9 @@
 import re
 
 import docker
-from datetime import timedelta
+from datetime import datetime, timedelta
 
+from contextlib import contextmanager
 from re import Pattern
 from tenacity import Retrying
 from tenacity.wait import wait_fixed
@@ -12,23 +13,32 @@
 from docker.models.images import Image
 
 
-def sanitize_test_name(test_name: str):
-    test_name = re.sub('[^0-9a-zA-Z]+', '_', test_name)
+def sanitize_container_name(test_name: str):
+    test_name = re.sub('[^0-9a-zA-Z-]+', '_', test_name)
     test_name = re.sub('_+', '_', test_name)
     return test_name
 
 
-def container(request, base_name: str, image: Union[Image, str], start: bool = True, **kwargs) \
-        -> Generator[Container, None, None]:
+def timestamp() -> str:
+    return f'{datetime.now().timestamp():.0f}'
+
+
+def container(
+        request,
+        image: Union[Image, str],
+        suffix: str = None,
+        start: bool = True,
+        **kwargs,
+) -> Generator[Container, None, None]:
     """
     Create a Docker container based on the specified Docker image.
     """
+    if suffix is not None:
+        suffix = f"_{suffix}"
+    image_name = image.id if hasattr(image, "id") else image
+    container_name = sanitize_container_name(f"{image_name}_{request.node.name}{suffix}")
     client = docker.from_env()
-    base_container_name = base_name.replace("-", "_")
-    test_name = sanitize_test_name(str(request.node.name))
-    container_name = f"{base_container_name}_{test_name}"
     try:
-        image_name = image.id if hasattr(image, "id") else image
         container = client.containers.create(
             image=image_name,
             name=container_name,
@@ -43,6 +53,11 @@ def container(request, base_name: str, image: Union[Image, str], start: bool = T
         client.close()
 
 
+@contextmanager
+def container_context(request, image_name: str, **kwargs):
+    yield from container(request, image_name, **kwargs)
+
+
 def wait_for(
         container: Container,
         log_message: Union[str, Pattern],
@@ -72,3 +87,13 @@ def wait_for_socket_access(container: Container):
         container,
         f"entrypoint.py: Enabled access to {DOCKER_SOCKET_CONTAINER}",
     )
+
+
+def assert_exec_run(container: Container, command: str, **kwargs) -> str:
+    """
+    Execute command in container and verify success.
+    """
+    exit_code, output = container.exec_run(command, **kwargs)
+    output = output.decode("utf-8").strip()
+    assert exit_code == 0, output
+    return output

test/docker/dss_docker_image.py

@@ -3,10 +3,26 @@
 import pytest
 import stat
 
-from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+from exasol.ds.sandbox.lib.dss_docker import (
+    DssDockerImage,
+    USER_ENV,
+    PASSWORD_ENV,
+)
 
 
 def pytest_addoption(parser):
+    def add_options_for_docker_image(base_name: str, test_group: str = None):
+        if test_group is None:
+            test_group = f"tests with {base_name}"
+        parser.addoption(
+            f"--docker-image-{base_name}", default=None,
+            help="Name and version of existing Docker image to use for {test_group}",
+        )
+        parser.addoption(
+            f"--keep-docker-image-{base_name}", action="store_true", default=False,
+            help=f"Keep the created Docker image for {test_group} for subsequent inspection or reuse.",
+        )
+
     parser.addoption(
         "--dss-docker-image", default=None,
         help="Name and version of existing Docker image to use for tests",
@@ -15,20 +31,26 @@ def pytest_addoption(parser):
         "--keep-dss-docker-image", action="store_true", default=False,
         help="Keep the created dss docker image for inspection or reuse."
     )
-    parser.addoption(
-        "--docker-image-notebook-test", default=None,
-        help="Name and version of existing Docker image for Notebook testing to use for tests",
-    )
-    parser.addoption(
-        "--keep-docker-image-notebook-test", action="store_true", default=False,
-        help="Keep the created notebook-test docker image for inspection or reuse.",
-    )
+    add_options_for_docker_image("notebook-test", "Notebook testing")
+    add_options_for_docker_image("ai-lab-with-additional-group")
     parser.addoption(
         "--docker-registry", default=None, metavar="HOST:PORT",
         help="Docker registry for pushing Docker images to",
     )
 
 
+@pytest.fixture(scope="session")
+def docker_auth():
+    username = os.environ.get(USER_ENV, None)
+    password = os.environ.get(PASSWORD_ENV, None)
+    if not (username and password):
+        return None
+    return {
+        "username": username,
+        "password": password,
+    }
+
+
 @pytest.fixture(scope="session")
 def dss_docker_image(request):
     """

test/docker/image.py

@@ -1,13 +1,29 @@
 import json
+import logging
 import re
+
 from datetime import datetime
 from typing import List, Dict, Any, Tuple, Optional
+from dataclasses import dataclass
 
 import docker
 from docker.errors import BuildError
 from docker.models.images import Image
 
 
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DockerImageSpec:
+    repository: str
+    tag: str
+
+    @property
+    def name(self) -> str:
+        return f"{self.repository}:{self.tag}"
+
+
 def format_build_log(build_log: List[Dict[str, Any]]):
     def format_entry(entry: Dict[str, Any]):
         if "stream" in entry:
@@ -24,11 +40,25 @@ def __init__(self, reason, build_log: List[Dict[str, Any]]):
         super().__init__(f"{reason}\n\n{format_build_log(build_log)}", build_log)
 
 
+def pull(
+    spec: DockerImageSpec,
+    auth_config: Optional[Dict[str, str]] = None,
+):
+    client = docker.from_env()
+    if not client.images.list(spec.name):
+        _logger.debug(f"Pulling Docker image {spec.name}")
+        client.images.pull(
+            spec.repository,
+            spec.tag,
+            auth_config=auth_config,
+        )
+
+
 def image(request, name: str, print_log=False, **kwargs) -> Image:
     """
     Create a Docker image.
     The function supports a pair of pytest cli options with a suffix derived from parameter ``name``:
-    Option `--docker-image-(suffix)` specifies the name of an existing image to be used 
+    Option `--docker-image-(suffix)` specifies the name of an existing image to be used
     instead of creating a new one.
     Option `--keep-docker-image-(suffix)` skips removing the image after test execution.
     """
@@ -37,12 +67,14 @@ def image(request, name: str, print_log=False, **kwargs) -> Image:
     keep_image = request.config.getoption(f"--keep-docker-image-{base_command_line}")
     client = docker.from_env()
     if image_tag:
-        return client.images.get(image_tag)
+        yield client.images.get(image_tag)
+        return
     timestamp = f'{datetime.now().timestamp():.0f}'
     image_name = name.replace("-", "_")
     image_tag = f"{image_name}:{timestamp}"
     try:
-        log_generator = client.api.build(tag=image_tag, **kwargs)
+        # rm=True removes intermediate containers after building
+        log_generator = client.api.build(tag=image_tag, rm=True, **kwargs)
         image_id, log, error = analyze_build_log(log_generator)
         if image_id is None:
             raise BuildErrorWithLog(error, log)

test/integration/docker_socket_and_groups.py

@@ -0,0 +1,89 @@
+from datetime import datetime
+from inspect import cleandoc
+from pathlib import Path
+from contextlib import contextmanager
+from test.docker.in_memory_build_context import InMemoryBuildContext
+from test.docker.image import image
+from test.docker.container import (
+    container_context,
+    DOCKER_SOCKET_CONTAINER,
+    wait_for_socket_access,
+    assert_exec_run,
+)
+
+
+def numeric_gid(group_entry: str) -> int:
+    """group_entry is "ubuntu:x:971:", for example"""
+    return int(group_entry.split(':')[2])
+
+
+class SocketInspector:
+    def __init__(self, request, image_name: str, socket_on_host: Path):
+        self.request = request
+        self.image_name = image_name
+        self.socket_on_host = socket_on_host
+        self._container = None
+        self._context = None
+
+    def __enter__(self):
+        self._context = container_context(
+            self.request,
+            image_name=self.image_name,
+            volumes={ self.socket_on_host: {
+                'bind': DOCKER_SOCKET_CONTAINER,
+                'mode': 'rw', }, },
+        )
+        self._container = self._context.__enter__()
+        wait_for_socket_access(self._container)
+        return self
+
+    def __exit__(self, exc_type, exc, exc_tb):
+        self._container = None
+        self._context.__exit__(exc_type, exc, exc_tb)
+
+    def run(self, command: str, **kwargs) -> str:
+        return assert_exec_run(self._container, command, **kwargs)
+
+    def get_gid(self, group_name: str) -> int:
+        output = self.run(f"getent group {group_name}")
+        return numeric_gid(output)
+
+    def assert_jupyter_member_of(self, group_name: str):
+        output = self.run(f"getent group {group_name}")
+        members = output.split(":")[3].split(",")
+        assert "jupyter" in members
+
+    def assert_write_to_socket(self):
+        signal = f"Is there anybody out there {datetime.now()}?"
+        self.run(
+            f'bash -c "echo {signal} > {DOCKER_SOCKET_CONTAINER}"',
+            user="jupyter")
+        assert signal == self.socket_on_host.read_text().strip()
+
+
+class GroupChanger:
+    def __init__(self, context_provider):
+        self._context_provider = context_provider
+
+    def chgrp(self, gid: int, path_on_host: Path):
+        path_in_container = "/mounted"
+        with self._context_provider(path_on_host, path_in_container) as container:
+            assert_exec_run(container, f"chgrp {gid} {path_in_container}")
+
+
+@contextmanager
+def dss_image_with_added_group(request, base_image, gid, group_name):
+    dockerfile_content = cleandoc(
+        f"""
+        FROM {base_image}
+        RUN sudo groupadd --gid {gid} {group_name}
+        """
+    )
+    with InMemoryBuildContext() as context:
+        context.add_string_to_file(name="Dockerfile", string=dockerfile_content)
+    yield from image(
+        request,
+        name=f"ai_lab_with_additional_group",
+        fileobj=context.fileobj,
+        custom_context=True,
+    )