From b5a6ae9ace156ef85ee61d8c409fdcc391b4a834 Mon Sep 17 00:00:00 2001 From: Nadav Goldin Date: Mon, 29 May 2017 21:02:09 +0300 Subject: [PATCH] Move sysprep to Jinja 2 templates Shifting the sysprep logic to Jinja2 templates, and calling 'sysprep' with the generated file. This allows splitting the logic to different distros, and re-use templates where applicable. Signed-off-by: Nadav Goldin --- automation/check-patch.packages | 1 + automation/check-patch.packages.el7 | 1 + lago.spec.in | 6 + lago/providers/libvirt/vm.py | 48 +++---- lago/sysprep.py | 199 +++++++++------------------- lago/templates/sysprep-base.j2 | 12 ++ lago/templates/sysprep-debian.j2 | 6 + lago/templates/sysprep-el6.j2 | 5 + lago/templates/sysprep-el7.j2 | 10 ++ lago/templates/sysprep-fc24.j2 | 9 ++ lago/templates/sysprep-fc25.j2 | 1 + lago/templates/sysprep-fc26.j2 | 1 + lago/templates/sysprep-macros.j2 | 39 ++++++ requirements.txt | 1 + test-requires.txt | 6 +- tests/unit/lago/test_sysprep.py | 73 ++++++++++ 16 files changed, 244 insertions(+), 174 deletions(-) create mode 100644 lago/templates/sysprep-base.j2 create mode 100644 lago/templates/sysprep-debian.j2 create mode 100644 lago/templates/sysprep-el6.j2 create mode 100644 lago/templates/sysprep-el7.j2 create mode 100644 lago/templates/sysprep-fc24.j2 create mode 100644 lago/templates/sysprep-fc25.j2 create mode 100644 lago/templates/sysprep-fc26.j2 create mode 100644 lago/templates/sysprep-macros.j2 create mode 100644 tests/unit/lago/test_sysprep.py diff --git a/automation/check-patch.packages b/automation/check-patch.packages index eaaa820b..93ac125b 100644 --- a/automation/check-patch.packages +++ b/automation/check-patch.packages @@ -6,6 +6,7 @@ git grubby libffi-devel libvirt-devel +libguestfs-devel libxslt-devel openssl-devel pandoc diff --git a/automation/check-patch.packages.el7 b/automation/check-patch.packages.el7 index 3d018e3b..ecfac610 100644 --- a/automation/check-patch.packages.el7 +++ b/automation/check-patch.packages.el7 @@ -3,6 +3,7 @@ bats git libffi-devel libvirt-devel +libguestfs-devel libxslt-devel openssl-devel pandoc diff --git a/lago.spec.in b/lago.spec.in index 3313cc8b..b12abd7d 100644 --- a/lago.spec.in +++ b/lago.spec.in @@ -64,7 +64,9 @@ BuildRequires: python-wrapt %if 0%{?fedora} >= 24 BuildRequires: python2-configparser BuildRequires: python2-paramiko >= 2.1.1 +BuildRequires: python2-jinja2 %else +BuildRequires: python-jinja2 BuildRequires: python-configparser BuildRequires: python2-paramiko %endif @@ -98,9 +100,11 @@ Requires: python-enum Requires: pyxdg Requires: python-wrapt %if 0%{?fedora} >= 24 +Requires: python2-jinja2 Requires: python2-configparser Requires: python2-paramiko >= 2.1.1 %else +Requires: python-jinja2 Requires: python-configparser Requires: python2-paramiko %endif @@ -121,9 +125,11 @@ Requires: sudo %doc AUTHORS COPYING README.rst %{python2_sitelib}/%{name}/*.py* %{python2_sitelib}/%{name}/plugins/*.py* +%{python2_sitelib}/%{name}/templates/*.j2 %{python2_sitelib}/%{name}/providers/*.py* %{python2_sitelib}/%{name}/providers/libvirt/*.py* %{python2_sitelib}/%{name}/providers/libvirt/templates/*.xml + %{python2_sitelib}/%{name}-%{version}-py*.egg-info %{_bindir}/lagocli %{_bindir}/lago diff --git a/lago/providers/libvirt/vm.py b/lago/providers/libvirt/vm.py index 89694707..6a1286e1 100644 --- a/lago/providers/libvirt/vm.py +++ b/lago/providers/libvirt/vm.py @@ -187,38 +187,22 @@ def bootstrap(self): if self.vm._spec['disks'][0]['type'] != 'empty' and self.vm._spec[ 'disks' ][0]['format'] != 'iso': - sysprep_cmd = [ - sysprep.set_hostname(self.vm.name()), - sysprep.delete_file(KDUMP_SERVICE), - sysprep.delete_file(POSTFIX_SERVICE), - sysprep.set_root_password(self.vm.root_password()), - sysprep.add_ssh_key( - self.vm.virt_env.prefix.paths.ssh_id_rsa_pub(), - ), - sysprep.set_iscsi_initiator_name(self.vm.iscsi_name()) - ] - - if self.vm.distro() in ('fc24', 'fc25', 'debian', 'el7'): - path = '/boot/grub2/grub.cfg' - if self.vm.distro() == 'debian': - path = '/boot/grub/grub.cfg' - sysprep_cmd.append( - sysprep.edit(path, 's/set timeout=5/set timeout=0/s') - ) - - # In fc25 NetworkManager configures the interfaces successfuly - # on boot. - if self.vm.distro() not in ('fc25', 'fc26'): - ifaces = [ - ('eth{0}'.format(idx), utils.ipv4_to_mac(nic['ip'])) - for idx, nic in enumerate(self.vm.spec['nics']) - ] - sysprep_cmd.extend( - sysprep. - config_net_ifaces_dhcp(self.vm.distro(), ifaces) - ) - - sysprep.sysprep(self.vm._spec['disks'][0]['path'], sysprep_cmd) + root_disk = self.vm._spec['disks'][0]['path'] + mappings = { + 'eth{0}'.format(idx): utils.ipv4_to_mac(nic['ip']) + for idx, nic in enumerate(self.vm.spec['nics']) + } + public_ssh_key = self.vm.virt_env.prefix.paths.ssh_id_rsa_pub() + + sysprep.sysprep( + disk=root_disk, + mappings=mappings, + distro=self.vm.distro(), + root_password=self.vm.root_password(), + public_key=public_ssh_key, + iscsi_name=self.vm.iscsi_name(), + hostname=self.vm.name(), + ) def state(self): """ diff --git a/lago/sysprep.py b/lago/sysprep.py index 6b7e4e46..ef2eedbe 100644 --- a/lago/sysprep.py +++ b/lago/sysprep.py @@ -1,5 +1,5 @@ # -# Copyright 2015 Red Hat, Inc. +# Copyright 2015-2017 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,164 +20,83 @@ import os import utils -from textwrap import dedent -import tempfile import logging -LOGGER = logging.getLogger(__name__) -_DOT_SSH = '/root/.ssh' -_AUTHORIZED_KEYS = os.path.join(_DOT_SSH, 'authorized_keys') -_SELINUX_CONF_DIR = '/etc/selinux' -_SELINUX_CONF_PATH = os.path.join(_SELINUX_CONF_DIR, 'config') -_ISCSI_DIR = '/etc/iscsi' - - -def set_hostname(hostname): - return ('--hostname', hostname) - - -def set_root_password(password): - return ('--root-password', 'password:%s' % password) - - -def _write_file(path, content): - return ('--write', '%s:%s' % (path, content)) - +import tempfile +from jinja2 import Environment, PackageLoader +import textwrap +import sys -def _upload_file(local_path, remote_path): - return ('--upload', '%s:%s' % (remote_path, local_path)) +LOGGER = logging.getLogger(__name__) +try: + import guestfs +except ImportError: + LOGGER.debug('guestfs not available, ignoring') -def set_iscsi_initiator_name(name): - return ('--mkdir', _ISCSI_DIR, '--chmod', - '0755:%s' % _ISCSI_DIR, ) + _write_file( - os.path.join(_ISCSI_DIR, 'initiatorname.iscsi'), - 'InitiatorName=%s' % name, - ) # noqa: E126 +def _guestfs_version(default={'major': 1L, 'minor': 20L}): + if 'guestfs' in sys.modules: + g = guestfs.GuestFS(python_return_dict=True) + guestfs_ver = g.version() + g.close() + else: + guestfs_ver = default -def add_ssh_key(key, with_restorecon_fix=False): - extra_options = ('--mkdir', _DOT_SSH, '--chmod', '0700:%s' % - _DOT_SSH, ) + _upload_file(_AUTHORIZED_KEYS, key) - if (not os.stat(key).st_uid == 0 or not os.stat(key).st_gid == 0): - extra_options += ( - '--run-command', 'chown root.root %s' % _AUTHORIZED_KEYS, - ) - if with_restorecon_fix: - # Fix for fc23 not relabeling on boot - # https://bugzilla.redhat.com/1049656 - extra_options += ('--firstboot-command', 'restorecon -R /root/.ssh', ) - return extra_options - - -def set_selinux_mode(mode): - return ( - '--mkdir', _SELINUX_CONF_DIR, '--chmod', '0755:%s' % _SELINUX_CONF_DIR, - ) + _write_file( - _SELINUX_CONF_PATH, - ('SELINUX=%s\n' - 'SELINUXTYPE=targeted\n') % mode, - ) + return guestfs_ver -def _config_net_interface(path, iface, type, bootproto, onboot, hwaddr): - iface_path = os.path.join(path, 'ifcfg-{0}'.format(iface)) - cfg = dedent( - """ - HWADDR="{hwaddr}" - BOOTPROTO="{bootproto}" - ONBOOT="{onboot}" - TYPE="{type}" - NAME="{iface}" - """.format( - hwaddr=hwaddr, - bootproto=bootproto, - onboot=onboot, - type=type, - iface=iface - ) - ).lstrip() - with tempfile.NamedTemporaryFile(delete=False) as ifcfg_file: - ifcfg_file.write(cfg) - LOGGER.debug('generated %s for %s:\n%s', ifcfg_file.name, iface_path, cfg) - return ('--mkdir', path, '--chmod', '0755:{0}'.format(path) - ) + _upload_file(iface_path, ifcfg_file.name) - - -def config_net_iface_debian(name, mac): - iface = dedent( - """ - auto {name} - iface {name} inet6 auto - iface {name} inet dhcp - hwaddress ether {mac} - """.format(name=name, mac=mac) +def _render_template(distro, loader, **kwargs): + env = Environment( + loader=loader, + trim_blocks=True, + lstrip_blocks=True, ) - return ( - _write_file( - os.path.join( - '/etc/network/interfaces.d', 'ifcfg-{0}.cfg'.format(name) - ), iface - ) + env.filters['dedent'] = textwrap.dedent + template_name = 'sysprep-{0}.j2'.format(distro) + template = env.select_template([template_name, 'sysprep-base.j2']) + sysprep_content = template.render(guestfs_ver=_guestfs_version(), **kwargs) + with tempfile.NamedTemporaryFile(delete=False) as sysprep_file: + sysprep_file.write('# {0}\n'.format(template.name)) + sysprep_file.write(sysprep_content) + + LOGGER.debug( + ('Generated sysprep template ' + 'at {0}:\n{1}').format(sysprep_file.name, sysprep_content) ) + return sysprep_file.name -def config_net_iface_loop_debian(): - loop_device = dedent( - """ - auto lo - iface lo inet loopback - - source /etc/network/interfaces.d/*.cfg +def sysprep(disk, distro, loader=None, backend='direct', **kwargs): + """ + Run virt-sysprep on the ``disk``, commands are built from the distro + specific template and arguments passed in ``kwargs``. If no template is + available it will default to ``sysprep-base.j2``. + + Args: + disk(str): path to disk + distro(str): distro to render template for + loader(jinja2.BaseLoader): Jinja2 template loader, if not passed, + will search Lago's package. + backend(str): libguestfs backend to use + **kwargs(dict): environment variables for Jinja2 template + + Returns: + None + + Raises: + RuntimeError: On virt-sysprep none 0 exit code. """ - ) - return (_write_file('/etc/network/interfaces', loop_device)) - - -def config_net_ifaces_dhcp(distro, mapping): - if distro == 'debian': - cmd = [config_net_iface_loop_debian()] - cmd.extend( - [config_net_iface_debian(name, mac) for name, mac in mapping] - ) - else: - cmd = [config_net_iface_dhcp(name, mac) for name, mac in mapping] - - return cmd - - -def config_net_iface_dhcp( - iface, hwaddr, path='/etc/sysconfig/network-scripts' -): - return _config_net_interface( - path=path, - iface=iface, - type='Ethernet', - bootproto='dhcp', - onboot='yes', - hwaddr=hwaddr, - ) - - -def edit(filename, expression): - editstr = '%s:""%s""' % (filename, expression) - return ('--edit', editstr, ) - - -def delete_file(filename): - return ('--delete', filename) - -def update(): - return ('--update', '--network', ) + if loader is None: + loader = PackageLoader('lago', 'templates') + sysprep_file = _render_template(distro, loader=loader, **kwargs) + cmd = ['virt-sysprep', '-a', disk] + cmd.extend(['--commands-from-file', sysprep_file]) -def sysprep(disk, mods, backend='direct'): - cmd = ['virt-sysprep', '-a', disk, '--selinux-relabel'] env = os.environ.copy() if 'LIBGUESTFS_BACKEND' not in env: env['LIBGUESTFS_BACKEND'] = backend - for mod in mods: - cmd.extend(mod) ret = utils.run_command(cmd, env=env) if ret: diff --git a/lago/templates/sysprep-base.j2 b/lago/templates/sysprep-base.j2 new file mode 100644 index 00000000..f4399238 --- /dev/null +++ b/lago/templates/sysprep-base.j2 @@ -0,0 +1,12 @@ +selinux-relabel +hostname {{ hostname }} +root-password password:{{ root_password }} +{% if guestfs_ver['major'] >= 1 and guestfs_ver['minor'] >= 34 %} +ssh-inject root:file:{{public_key}} +{% else %} +mkdir /root/.ssh +chmod 0700:/root/.ssh +upload {{ public_key }}:/root/.ssh/authorized_keys +chmod 0600:/root/.ssh/authorized_keys +run-command chown root:root /root/.ssh/authorized_keys +{% endif -%} diff --git a/lago/templates/sysprep-debian.j2 b/lago/templates/sysprep-debian.j2 new file mode 100644 index 00000000..e0ed8ac0 --- /dev/null +++ b/lago/templates/sysprep-debian.j2 @@ -0,0 +1,6 @@ +{% import 'sysprep-macros.j2' as macros %} +{% include 'sysprep-base.j2' %} + +{{ macros.network_devices_debian(mappings=mappings) }} + +edit /boot/grub/grub.cfg:s/set timeout=5/set timeout=0/g diff --git a/lago/templates/sysprep-el6.j2 b/lago/templates/sysprep-el6.j2 new file mode 100644 index 00000000..e70758c1 --- /dev/null +++ b/lago/templates/sysprep-el6.j2 @@ -0,0 +1,5 @@ +{% import 'sysprep-macros.j2' as macros %} +{% include 'sysprep-base.j2' %} + +{{ macros.iscsi(name=iscsi_name, hostname=hostname) }} +{{ macros.network_devices_el(mappings=mappings) }} diff --git a/lago/templates/sysprep-el7.j2 b/lago/templates/sysprep-el7.j2 new file mode 100644 index 00000000..d870db3a --- /dev/null +++ b/lago/templates/sysprep-el7.j2 @@ -0,0 +1,10 @@ +{% import 'sysprep-macros.j2' as macros %} +{% include 'sysprep-base.j2' %} + +{{ macros.iscsi(name=iscsi_name, hostname=hostname) }} +{{ macros.network_devices_el(mappings=mappings) }} + +edit /boot/grub2/grub.cfg:s/set timeout=5/set timeout=0/g + +delete /etc/systemd/system/multi-user.target.wants/kdump.service +delete /etc/systemd/system/multi-user.target.wants/postfix.service diff --git a/lago/templates/sysprep-fc24.j2 b/lago/templates/sysprep-fc24.j2 new file mode 100644 index 00000000..f674f136 --- /dev/null +++ b/lago/templates/sysprep-fc24.j2 @@ -0,0 +1,9 @@ +{% import 'sysprep-macros.j2' as macros %} +{% include 'sysprep-base.j2' %} + +{{ macros.iscsi(name=iscsi_name, hostname=hostname) }} + +edit /boot/grub2/grub.cfg:s/set timeout=5/set timeout=0/g + +delete /etc/systemd/system/multi-user.target.wants/kdump.service +delete /etc/systemd/system/multi-user.target.wants/postfix.service diff --git a/lago/templates/sysprep-fc25.j2 b/lago/templates/sysprep-fc25.j2 new file mode 100644 index 00000000..32352506 --- /dev/null +++ b/lago/templates/sysprep-fc25.j2 @@ -0,0 +1 @@ +{% include 'sysprep-fc24.j2' %} diff --git a/lago/templates/sysprep-fc26.j2 b/lago/templates/sysprep-fc26.j2 new file mode 100644 index 00000000..32352506 --- /dev/null +++ b/lago/templates/sysprep-fc26.j2 @@ -0,0 +1 @@ +{% include 'sysprep-fc24.j2' %} diff --git a/lago/templates/sysprep-macros.j2 b/lago/templates/sysprep-macros.j2 new file mode 100644 index 00000000..73b89fd6 --- /dev/null +++ b/lago/templates/sysprep-macros.j2 @@ -0,0 +1,39 @@ +{% macro iscsi(name, hostname, path='/etc/iscsi') %} +mkdir /etc/iscsi +chmod 0755:/etc/iscsi +write /etc/iscsi/initiatorname.iscsi:InitiatorName={{ name }}:{{ hostname }} +{% endmacro %} + +{% macro network_devices_el(mappings) %} +mkdir /etc/sysconfig/network-scripts +chmod 0755:/etc/sysconfig/network-scripts +{% filter dedent %} +{% for iface, mac in mappings.viewitems() %} + write /etc/sysconfig/network-scripts/ifcfg-{{iface}}:HWADDR="{{mac}}" \ + BOOTPROTO="dhcp" \ + ONBOOT="yes" \ + TYPE="Ethernet" \ + NAME="{{iface}}" \ + +{% endfor %} +{% endfilter %} +{% endmacro %} + +{% macro network_devices_debian(mappings) %} +mkdir /etc/network/interfaces.d +chmod 0755:/etc/network/interfaces.d + +write /etc/network/interfaces:auto lo \ +iface lo inet loopback \ +source /etc/network/interfaces.d/*.cfg \ + +{% filter dedent %} +{% for iface, mac in mappings.viewitems() %} + write /etc/network/interfaces.d/ifcfg-{{ iface }}.cfg:auto {{ iface }} \ + iface {{ iface }} inet6 auto \ + iface {{ iface }} inet dhcp \ + hwaddress ether {{ mac }} \ + +{% endfor %} +{% endfilter %} +{% endmacro %} diff --git a/requirements.txt b/requirements.txt index f990ccef..710066d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ enum pyxdg configparser wrapt +Jinja2 diff --git a/test-requires.txt b/test-requires.txt index b116af09..b4497eaf 100644 --- a/test-requires.txt +++ b/test-requires.txt @@ -1,7 +1,5 @@ # https://github.com/pypa/setuptools/issues/1042 six -# not provided by pip -#python-libguestfs enum dulwich flake8 @@ -25,3 +23,7 @@ pytest-cov pytest-timeout pytest-catchlog wrapt +Jinja2 +# guestfs is not available on PyPI, however it can be installed +# with a direct link. +http://download.libguestfs.org/python/guestfs-1.36.4.tar.gz diff --git a/tests/unit/lago/test_sysprep.py b/tests/unit/lago/test_sysprep.py new file mode 100644 index 00000000..1da93d81 --- /dev/null +++ b/tests/unit/lago/test_sysprep.py @@ -0,0 +1,73 @@ +import pytest +from pytest import fixture +from lago import sysprep +import jinja2 +import os +import mock +import sys + + +class TemplateFactory(object): + def __init__(self, dst): + self.dst = dst + self.loader = jinja2.FileSystemLoader(dst) + + def add(self, name, content): + path = os.path.join(self.dst, name) + with open(path, 'w+') as template: + template.write(content) + + def add_base(self, content): + self.add(content=content, name='sysprep-base.j2') + + +@fixture +def factory(tmpdir): + return TemplateFactory(dst=str(tmpdir)) + + +class TestSysprep(object): + @pytest.mark.parametrize( + 'distro,templates,expected', [ + ('distro_a', ['base'], 'base'), + ('distro_a', ['base', 'distro_a'], 'distro_a'), + ('distro_b', ['base', 'distro_a'], 'base') + ] + ) + def test_render_template_loads_expected( + self, factory, distro, templates, expected + ): + for template in templates: + factory.add('sysprep-{0}.j2'.format(template), 'empty template') + filename = sysprep._render_template( + distro=distro, loader=factory.loader + ) + + with open(filename, 'r') as generated: + lines = [line.strip() for line in generated.readlines()] + assert lines[0].strip() == '# sysprep-{0}.j2'.format(expected) + assert lines[1].strip() == 'empty template' + + def test_dedent_filter(self, factory): + template = '{{ var|dedent }}' + factory.add_base(template) + filename = sysprep._render_template( + distro='base', loader=factory.loader, var='\tremove-indent' + ) + + with open(filename, 'r') as generated: + lines = generated.readlines() + assert lines[0].strip() == '# sysprep-base.j2' + assert lines[1] == 'remove-indent' + + def test_guestfs_version_failed_import(self): + with mock.patch.dict('sys.modules'): + del sys.modules['guestfs'] + assert sysprep._guestfs_version(default={'no': 'no'}) == { + 'no': 'no' + } + assert 'guestfs' in sys.modules + + def test_guestfs_version_good_import(self): + assert 'guestfs' in sys.modules + assert sysprep._guestfs_version(default={'no': 'no'}) != {'no': 'no'}