From ae88ae1d6659d339083da34ef8192e1b33b8c5c8 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 13 Mar 2024 14:39:22 +0000 Subject: [PATCH] testo --- .../workflows/overcloud-host-image-build.yml | 113 +++++++---- terraform/host-image-builder/README.rst | 191 ++++++++++++++++++ terraform/host-image-builder/outputs.tf | 15 ++ terraform/host-image-builder/provider.tf | 14 ++ .../templates/userdata.cfg.tpl | 21 ++ terraform/host-image-builder/vm.tf | 98 +++++++++ 6 files changed, 415 insertions(+), 37 deletions(-) create mode 100644 terraform/host-image-builder/README.rst create mode 100644 terraform/host-image-builder/outputs.tf create mode 100644 terraform/host-image-builder/provider.tf create mode 100644 terraform/host-image-builder/templates/userdata.cfg.tpl create mode 100644 terraform/host-image-builder/vm.tf diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index b36d3d9ca0..9a0298c2df 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -39,18 +39,26 @@ jobs: overcloud-host-image-build: name: Build overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: arc-skc-host-image-builder-runner + runs-on: arc-skc-aio-runner # TODO: switch back to host image builder permissions: {} steps: - - name: Install package dependencies - run: | - sudo apt update - sudo apt install -y debootstrap gdisk zstd build-essential git unzip nodejs python3-wheel python3-pip python3-venv rpm dnf + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - path: src/kayobe-config + submodules: true + + - name: Make sure dockerd is running and test Docker + run: | + docker ps + + - name: Output host image builder tag + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT - name: Determine OpenStack release id: openstack_release @@ -68,47 +76,78 @@ jobs: run: | echo "${{ steps.host_image_tag.outputs.host_image_tag }}" - - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v4 - with: - repository: stackhpc/kayobe - ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} - path: src/kayobe + - name: Install terraform + uses: hashicorp/setup-terraform@v2 - - name: Install Kayobe - run: | - mkdir -p venvs && - pushd venvs && - python3 -m venv kayobe && - source kayobe/bin/activate && - pip install -U pip && - pip install ../src/kayobe - - - name: Bootstrap the control host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe control host bootstrap --skip-tags bootstrap + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/terraform/host-image-builder - - name: Create bifrost_httpboot Docker volume - run: | - sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/terraform/host-image-builder - name: Generate clouds.yaml run: | cat << EOF > clouds.yaml ${{ secrets.CLOUDS_YAML }} EOF + working-directory: ${{ github.workspace }}/terraform/host-image-builder - - name: Install OpenStack client + - name: Generate terraform.tfvars run: | - source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "${{ env.SSH_USERNAME }}" + host_image_builder_vm_interface = "${{ env.VM_INTERFACE }}" + host_image_builder_vm_name = "${{ env.VM_NAME }}" + host_image_builder_vm_image = "${{ env.VM_IMAGE }}" + host_image_builder_vm_flavor = "${{ env.VM_FLAVOR }}" + host_image_builder_vm_network = "${{ env.VM_NETWORK }}" + host_image_builder_vm_subnet = "${{ env.VM_SUBNET }}" + host_image_builder_vm_tags = ${{ env.VM_TAGS }} + EOF + working-directory: ${{ github.workspace }}/terraform/host-image-builder + env: + SSH_USERNAME: "cloud-user" + VM_NAME: "skc-ci-host-image-builder" + VM_IMAGE: "overcloud-rocky-9-yoga-20240124T094316" + VM_FLAVOR: "en1.medium" + VM_NETWORK: "stackhpc-ci" + VM_SUBNET: "stackhpc-ci" + VM_INTERFACE: "ens3" + VM_TAGS: '["skc-ci-aio", "alex-testo"]' + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - - name: Testo + - name: Terraform Apply run: | - mkdir -p /tmp - mkdir -p /dev + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: fail + run: exit 1 - name: Build a CentOS Stream 8 overcloud host image id: build_centos_stream_8 diff --git a/terraform/host-image-builder/README.rst b/terraform/host-image-builder/README.rst new file mode 100644 index 0000000000..908b7af151 --- /dev/null +++ b/terraform/host-image-builder/README.rst @@ -0,0 +1,191 @@ +========================== +Terraform All in one (aio) +========================== + +This Terraform configuration deploys a single VM on an OpenStack cloud, to be +used as an all-in-one Kayobe test environment. + +This configuration is used in the GitHub Actions all-in-one.yml workflow for CI +testing. + +Usage +===== + +These instructions show how to use this Terraform configuration manually. They +assume you are running an Ubuntu host that will be used to run Terraform. The +machine should have network access to the VM that will be created by this +configuration. + +Install Terraform: + +.. code-block:: console + + wget -qO - terraform.gpg https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/terraform-archive-keyring.gpg + sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/terraform-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/terraform.list + sudo apt update + sudo apt install docker.io terraform + +Clone and initialise the Kayobe config: + +.. code-block:: console + + git clone https://github.com/stackhpc/stackhpc-kayobe-config + cd stackhpc-kayobe-config + git submodule init + git submodule update + +Change to the terraform/aio directory: + +.. code-block:: console + + cd terraform/aio + +Initialise Terraform: + +.. code-block:: console + + terraform init + +Generate an SSH keypair: + +.. code-block:: console + + ssh-keygen -f id_rsa -N '' + +Create an OpenStack clouds.yaml file with your credentials to access an +OpenStack cloud. Alternatively, download one from Horizon. + +.. code-block:: console + + cat << EOF > clouds.yaml + --- + clouds: + sms-lab: + auth: + auth_url: https://api.sms-lab.cloud:5000 + username: + project_name: + domain_name: default + interface: public + EOF + +Export environment variables to use the correct cloud and provide a password: + +.. code-block:: console + + export OS_CLOUD=sms-lab + read -p OS_PASSWORD -s OS_PASSWORD + export OS_PASSWORD + +Generate Terraform variables: + +.. code-block:: console + + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + aio_vm_name = "kayobe-aio" + aio_vm_image = "overcloud-centos-8-stream-yoga-20230525T095243" + aio_vm_flavor = "general.v1.medium" + aio_vm_network = "stackhpc-ipv4-geneve" + aio_vm_subnet = "stackhpc-ipv4-geneve-subnet" + EOF + +Generate a plan: + +.. code-block:: console + + terraform plan + +Apply the changes: + +.. code-block:: console + + terraform apply -auto-approve + +Write Terraform outputs to a Kayobe config file: + +.. code-block:: console + + terraform output -json > ../../etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml + +Change to the repository root: + +.. code-block:: console + + cd ../../ + +Write Terraform network config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml + + admin_oc_net_name: admin + admin_cidr: "{{ access_cidr.value }}" + admin_allocation_pool_start: 0.0.0.0 + admin_allocation_pool_end: 0.0.0.0 + admin_gateway: "{{ access_gw.value }}" + admin_bootproto: dhcp + admin_ips: + controller0: "{{ access_ip_v4.value }}" + EOF + +Write Terraform network interface config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces + admin_interface: "{{ access_interface.value }}" + EOF + +Build a Kayobe image: + +.. code-block:: console + + sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . + +Use the ci-aio environment: + +.. code-block:: console + + export KAYOBE_ENVIRONMENT=ci-aio + +Set the Kayobe Vault password env var: + +.. code-block:: console + + read -p KAYOBE_VAULT_PASSWORD -s KAYOBE_VAULT_PASSWORD + export KAYOBE_VAULT_PASSWORD + +Set the Kayobe SSH private key env var: + +.. code-block:: console + + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat terraform/aio/id_rsa) + +Host configure: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh + +Service deploy: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh + +Configure aio resources: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + +Run Tempest: + +.. code-block:: console + + mkdir -p tempest-artifacts + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + +Tempest results are in tempest-artifacts. diff --git a/terraform/host-image-builder/outputs.tf b/terraform/host-image-builder/outputs.tf new file mode 100644 index 0000000000..9b01cfd864 --- /dev/null +++ b/terraform/host-image-builder/outputs.tf @@ -0,0 +1,15 @@ +output "access_ip_v4" { + value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 +} + +output "access_cidr" { + value = data.openstack_networking_subnet_v2.network.cidr +} + +output "access_gw" { + value = data.openstack_networking_subnet_v2.network.gateway_ip +} + +output "access_interface" { + value = var.host_image_builder_interface +} diff --git a/terraform/host-image-builder/provider.tf b/terraform/host-image-builder/provider.tf new file mode 100644 index 0000000000..0e7cbdb051 --- /dev/null +++ b/terraform/host-image-builder/provider.tf @@ -0,0 +1,14 @@ +#provider "openstack" { +# use environment variables +#} + +terraform { + required_version = ">= 0.14" + backend "local" { + } + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + } + } +} diff --git a/terraform/host-image-builder/templates/userdata.cfg.tpl b/terraform/host-image-builder/templates/userdata.cfg.tpl new file mode 100644 index 0000000000..aebd192296 --- /dev/null +++ b/terraform/host-image-builder/templates/userdata.cfg.tpl @@ -0,0 +1,21 @@ +#cloud-config +# Don't automatically mount ephemeral disk +mounts: + - [/dev/vdb, null] +# WORKAROUND: internal DNS missing from SMS lab. +runcmd: + - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts' + - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts' +# Configure SSH keys here, to avoid creating an ephemeral keypair. +# This means only the instance needs to be cleaned up if the destroy fails. +ssh_authorized_keys: + - ${ssh_public_key} + +write_files: + # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409 + - content: | + #!/bin/bash + docker exec openvswitch_vswitchd ovs-vsctl "$@" + owner: root:root + path: /usr/bin/ovs-vsctl + permissions: '0755' diff --git a/terraform/host-image-builder/vm.tf b/terraform/host-image-builder/vm.tf new file mode 100644 index 0000000000..4a040814a6 --- /dev/null +++ b/terraform/host-image-builder/vm.tf @@ -0,0 +1,98 @@ +variable "ssh_public_key" { + type = string +} + +variable "ssh_username" { + type = string +} + +variable "host_image_builder_name" { + type = string + default = "kayobe-host-image-builder" +} + +variable "host_image_builder_image" { + type = string + default = "CentOS-stream8" +} + +variable "host_image_builder_interface" { + type = string + default = "ens3" +} + +variable "host_image_builder_flavor" { + type = string +} + +variable "host_image_builder_network" { + type = string +} + +variable "host_image_builder_subnet" { + type = string +} + +variable "host_image_builder_volume_size" { + type = number + default = 35 +} + +variable "host_image_builder_tags" { + type = list(string) + default = [] +} + +locals { + image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.host_image_builder_image)) > 0 +} + +data "openstack_images_image_v2" "image" { + name = var.host_image_builder_image + most_recent = true + count = local.image_is_uuid ? 0 : 1 +} + +data "openstack_networking_subnet_v2" "network" { + name = var.host_image_builder_subnet +} + +resource "openstack_compute_instance_v2" "kayobe-host-image-builder" { + name = var.host_image_builder_name + flavor_name = var.host_image_builder_flavor + config_drive = true + user_data = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)}) + network { + name = var.host_image_builder_network + } + + block_device { + uuid = local.image_is_uuid ? var.host_image_builder_image: data.openstack_images_image_v2.image[0].id + source_type = "image" + volume_size = var.host_image_builder_volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + tags = var.host_image_builder_tags +} + +# Wait for the instance to be accessible via SSH before progressing. +resource "null_resource" "kayobe-host-image-builder" { + provisioner "remote-exec" { + connection { + host = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 + user = var.ssh_username + private_key = file("id_rsa") + # Terraform will run the start script from /tmp by default. For the + # current images, /tmp is noexec, so the path must be changed + script_path = "/home/${var.ssh_username}/start.sh" + } + + inline = [ + "#!/bin/sh", + "echo 'connected!'" + ] + } +}