Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor docker runner, updated logging #17542

Open
wants to merge 2 commits into
base: develop2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion conan/api/conan_api.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import sys
from typing import Optional

from conan.api.output import init_colorama
from conan.api.subapi.cache import CacheAPI
Expand Down Expand Up @@ -26,7 +27,7 @@


class ConanAPI:
def __init__(self, cache_folder=None):
def __init__(self, cache_folder: Optional[str]=None):

version = sys.version_info
if version.major == 2 or version.minor < 6:
Expand Down
7 changes: 5 additions & 2 deletions conan/api/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def rewrite_line(self, line):
self.stream.flush()
self._color = tmp_color

def _write_message(self, msg, fg=None, bg=None):
def _write_message(self, msg, fg=None, bg=None, newline=True):
if isinstance(msg, dict):
# For traces we can receive a dict already, we try to transform then into more natural
# text
Expand All @@ -206,8 +206,11 @@ def _write_message(self, msg, fg=None, bg=None):
else:
ret += "{}".format(msg)

if newline:
ret = "%s\n" % ret

with self.lock:
self.stream.write("{}\n".format(ret))
self.stream.write(ret)
self.stream.flush()

def trace(self, msg):
Expand Down
237 changes: 123 additions & 114 deletions conan/internal/runner/docker.py
Original file line number Diff line number Diff line change
@@ -1,68 +1,24 @@
from collections import namedtuple
from argparse import Namespace
import os
import sys
import json
import platform
import shutil
from typing import Optional, NamedTuple
import yaml
from conan.api.conan_api import ConanAPI
from conan.api.model import ListPattern
from conan.api.output import Color, ConanOutput
from conan.api.conan_api import ConfigAPI
from conan.cli import make_abs_path
from conan.internal.runner import RunnerException
from conan.errors import ConanException
from conans.model.profile import Profile
from conans.model.version import Version


def config_parser(file_path):
Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from'])
Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes', 'network'])
Conf = namedtuple('Conf', ['image', 'build', 'run'])
if file_path:
def _instans_or_error(value, obj):
if value and (not isinstance(value, obj)):
raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}")
return value
with open(file_path, 'r') as f:
runnerfile = yaml.safe_load(f)
return Conf(
image=_instans_or_error(runnerfile.get('image'), str),
build=Build(
dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str),
build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str),
build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict),
cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list),
),
run=Run(
name=_instans_or_error(runnerfile.get('run', {}).get('name'), str),
environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict),
user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str),
privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool),
cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list),
security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list),
volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict),
network=_instans_or_error(runnerfile.get('run', {}).get('network'), str),
)
)
else:
return Conf(
image=None,
build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None),
run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None,
security_opt=None, volumes=None, network=None)
)


def _docker_info(msg, error=False):
fg=Color.BRIGHT_MAGENTA
if error:
fg=Color.BRIGHT_RED
ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg)
ConanOutput().status(f'| {msg} |', fg=fg)
ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg)
from pathlib import Path


class DockerRunner:
def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args):
def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, build_profile: Profile, args: Namespace, raw_args: list[str]):
import docker
import docker.api.build
try:
Expand All @@ -89,13 +45,16 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
self.conan_api = conan_api
self.build_profile = build_profile
self.args = args
self.abs_host_path = make_abs_path(args.path)
abs_path = make_abs_path(args.path)
if abs_path is None:
raise ConanException("Could not determine absolute path")
self.abs_host_path = Path(abs_path)
if args.format:
raise ConanException("format argument is forbidden if running in a docker runner")

# Container config
# https://containers.dev/implementors/json_reference/
self.configfile = config_parser(host_profile.runner.get('configfile'))
self.configfile = self.config_parser(host_profile.runner.get('configfile'))
self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile
self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context
self.image = host_profile.runner.get('image') or self.configfile.image
Expand All @@ -108,79 +67,89 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
self.container = None

# Runner config>
self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner')
self.abs_runner_home_path = self.abs_host_path / '.conanrunner'
self.docker_user_name = self.configfile.run.user or 'root'
self.abs_docker_path = os.path.join(f'/{self.docker_user_name}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/")

# Update conan command and some paths to run inside the container
raw_args[raw_args.index(args.path)] = self.abs_docker_path
self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json'])
self.logger = DockerOutput(self.name)

def run(self):
def run(self) -> None:
"""
run conan inside a Docker continer
Run conan inside a Docker container
"""
if self.dockerfile:
_docker_info(f'Building the Docker image: {self.image}')
self.build_image()
volumes, environment = self.create_runner_environment()
error = False
try:
if self.docker_client.containers.list(all=True, filters={'name': self.name}):
_docker_info('Starting the docker container')
self.container = self.docker_client.containers.get(self.name)
self.container.start()
else:
if self.configfile.run.environment:
environment.update(self.configfile.run.environment)
if self.configfile.run.volumes:
volumes.update(self.configfile.run.volumes)
_docker_info('Creating the docker container')
self.container = self.docker_client.containers.run(
self.image,
"/bin/bash -c 'while true; do sleep 30; done;'",
name=self.name,
volumes=volumes,
environment=environment,
user=self.configfile.run.user,
privileged=self.configfile.run.privileged,
cap_add=self.configfile.run.cap_add,
security_opt=self.configfile.run.security_opt,
detach=True,
auto_remove=False,
network=self.configfile.run.network)
_docker_info(f'Container {self.name} running')
except Exception as e:
raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"'
f'\n\n{str(e)}')
self.build_image()
self.start_container()
try:
self.init_container()
self.run_command(self.command)
self.update_local_cache()
except ConanException as e:
error = True
raise e
except RunnerException as e:
error = True
raise ConanException(f'"{e.command}" inside docker fail'
f'\n\nLast command output: {str(e.stdout_log)}')
finally:
if self.container:
error_prefix = 'ERROR: ' if error else ''
_docker_info(f'{error_prefix}Stopping container', error)
error = sys.exc_info()[0] is not None # Check if error has been raised
log = self.logger.error if error else self.logger.status
log('Stopping container')
self.container.stop()
if self.remove:
_docker_info(f'{error_prefix}Removing container', error)
log('Removing container')
self.container.remove()

def build_image(self):
Build = NamedTuple('Build', [('dockerfile', Optional[str]), ('build_context', Optional[str]), ('build_args', Optional[dict]), ('cache_from', Optional[list])])
Run = NamedTuple('Run', [('name', Optional[str]), ('environment', Optional[dict]), ('user', Optional[str]), ('privileged', Optional[bool]), ('cap_add', Optional[list]), ('security_opt', Optional[list]), ('volumes', Optional[dict]), ('network', Optional[str])])
Conf = NamedTuple('Conf', [('image', Optional[str]), ('build', Build), ('run', Run)])

@staticmethod
def config_parser(file_path: str) -> Conf:
if file_path:
def _instans_or_error(value, obj):
if value and (not isinstance(value, obj)):
raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}")
return value
with open(file_path, 'r') as f:
runnerfile = yaml.safe_load(f)
return DockerRunner.Conf(
image=_instans_or_error(runnerfile.get('image'), str),
build=DockerRunner.Build(
dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str),
build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str),
build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict),
cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list),
),
run=DockerRunner.Run(
name=_instans_or_error(runnerfile.get('run', {}).get('name'), str),
environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict),
user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str),
privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool),
cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list),
security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list),
volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict),
network=_instans_or_error(runnerfile.get('run', {}).get('network'), str),
)
)
else:
return DockerRunner.Conf(
image=None,
build=DockerRunner.Build(dockerfile=None, build_context=None, build_args=None, cache_from=None),
run=DockerRunner.Run(name=None, environment=None, user=None, privileged=None, cap_add=None,
security_opt=None, volumes=None, network=None)
)

def build_image(self) -> None:
if not self.dockerfile:
return
self.logger.status(f'Building the Docker image: {self.image}')
dockerfile_file_path = self.dockerfile
if os.path.isdir(self.dockerfile):
dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile')
with open(dockerfile_file_path) as f:
build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path)
ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'")
ConanOutput().highlight(f"Docker build context: '{build_path}'\n")
self.logger.highlight(f"Dockerfile path: '{dockerfile_file_path}'")
self.logger.highlight(f"Docker build context: '{build_path}'\n")
docker_build_logs = self.docker_api.build(
path=build_path,
dockerfile=f.read(),
Expand All @@ -189,16 +158,47 @@ def build_image(self):
cache_from=self.configfile.build.cache_from,
)
for chunk in docker_build_logs:
for line in chunk.decode("utf-8").split('\r\n'):
if line:
stream = json.loads(line).get('stream')
if stream:
ConanOutput().status(stream.strip())
for line in chunk.decode("utf-8").split('\r\n'):
if line:
stream = json.loads(line).get('stream')
if stream:
ConanOutput().status(stream.strip())

def start_container(self) -> None:
volumes, environment = self.create_runner_environment()
try:
if self.docker_client.containers.list(all=True, filters={'name': self.name}):
self.logger.status('Starting the docker container', fg=Color.BRIGHT_MAGENTA)
self.container = self.docker_client.containers.get(self.name)
self.container.start()
else:
if self.configfile.run.environment:
environment.update(self.configfile.run.environment)
if self.configfile.run.volumes:
volumes.update(self.configfile.run.volumes)
self.logger.status('Creating the docker container', fg=Color.BRIGHT_MAGENTA)
self.container = self.docker_client.containers.run(
self.image,
"/bin/bash -c 'while true; do sleep 30; done;'",
name=self.name,
volumes=volumes,
environment=environment,
user=self.configfile.run.user,
privileged=self.configfile.run.privileged,
cap_add=self.configfile.run.cap_add,
security_opt=self.configfile.run.security_opt,
detach=True,
auto_remove=False,
network=self.configfile.run.network)
self.logger.status(f'Container {self.name} running', fg=Color.BRIGHT_MAGENTA)
except Exception as e:
raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"'
f'\n\n{str(e)}')

def run_command(self, command, workdir=None, log=True):
def run_command(self, command: str, workdir: Optional[str] = None, log: bool = True) -> tuple[str, str]:
workdir = workdir or self.abs_docker_path
if log:
_docker_info(f'Running in container: "{command}"')
self.logger.status(f'Running in container: "{command}"')
exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", workdir=workdir, tty=True)
exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,)
stderr_log, stdout_log = '', ''
Expand All @@ -224,7 +224,7 @@ def run_command(self, command, workdir=None, log=True):
raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log)
return stdout_log, stderr_log

def create_runner_environment(self):
def create_runner_environment(self) -> tuple[dict, dict]:
shutil.rmtree(self.abs_runner_home_path, ignore_errors=True)
volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}}
environment = {'CONAN_RUNNER_ENVIRONMENT': '1'}
Expand All @@ -246,17 +246,16 @@ def create_runner_environment(self):

if self.cache == 'copy':
tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz')
_docker_info(f'Save host cache in: {tgz_path}')
self.logger.status(f'Save host cache in: {tgz_path}')
self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path)
return volumes, environment

def init_container(self):
def init_container(self) -> None:
min_conan_version = '2.1'
stdout, _ = self.run_command('conan --version', log=True)
docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color
if Version(docker_conan_version) <= Version(min_conan_version):
ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED)
raise ConanException( f'conan version inside the container must be greater than {min_conan_version}')
raise ConanException(f'conan version inside the container must be greater than {min_conan_version}')
if self.cache != 'shared':
self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False)
self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False)
Expand All @@ -266,10 +265,20 @@ def init_container(self):
if self.cache in ['copy']:
self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"')

def update_local_cache(self):
def update_local_cache(self) -> None:
if self.cache != 'shared':
self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False)
self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz')
tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz')
_docker_info(f'Restore host cache from: {tgz_path}')
package_list = self.conan_api.cache.restore(tgz_path)
self.logger.status(f'Restore host cache from: {tgz_path}')
self.conan_api.cache.restore(tgz_path)

class DockerOutput(ConanOutput):
def __init__(self, image: str):
super().__init__()
self.image = image

def _write_message(self, msg, fg=None, bg=None, newline=True):
super()._write_message(f"===> Docker Runner ({self.image}): ", Color.BLACK,
Color.BRIGHT_YELLOW, newline=False)
super()._write_message(msg, fg, bg, newline)
Loading