Bug/remote hosts (#752)

- Adds a Remote Server container for testing Aegir's remote server feature.
- Add devshop_remote play to playbook for configuring SSH server, SSH authorized key.
- Add service arg to `robo shell` command: `robo sh aegir devshop_remote`
- Make sites/X folder writable after using "drush site:install"
- Sync entire platform folder on verify instead of individual folders (Fixes missing private/temp directory)
- Fix chdir warning if folder doesn't exist.
- Fix broken HTTPS vhost config, add platform apache config.
- Let sites that get stuck "disabled" to show the enable task
- Fix devshop-ansible-playbook, run roles/devshop.server/play.yml file.
- Fixed occaissonally failing tests.
- Ensure hosting-tasks --force passes the force option to hosting-task so it won't skip tasks if there is another one present.

.github/workflows/devshop.yml

@@ -114,11 +114,15 @@ jobs:
       - name: "Launch devshop/server:latest container"
         working-directory: docker
         run: |
-          docker-compose up --detach devshop.server
+          docker-compose up --detach
           sleep 3
           docker-compose logs
           docker-compose exec -T devshop.server /usr/share/devshop/scripts/devshop-logo "GitHub Actions Container populated with code. Running SHA ${GITHUB_PR_SHA}"
 
+      - name: "Git Status"
+        working-directory: docker
+        run: git status
+
       - name: "Review Container Environment"
         working-directory: docker
         run: |
@@ -135,6 +139,7 @@ jobs:
         # Since the container is launched with a new DEVSHOP_DOCKER_COMMAND_RUN, run the default.
         run: |
           docker-compose exec -T devshop.server bash -c '${DEFAULT_DEVSHOP_DOCKER_COMMAND_RUN}'
+          docker-compose exec -T devshop.remote bash -c '${DEFAULT_DEVSHOP_DOCKER_COMMAND_RUN}'
 
       - name: "Test DevShop"
         working-directory: docker

.github/workflows/docker.yml

@@ -8,9 +8,10 @@ on:
     branches:
       # This workflow will only run on pull requests for branch test/docker/actions.
       - test/docker/actions
-  push:
-    branches:
-      - 1.x
+# @TODO: All builds are failing. Uncomment once this is fixed.
+#  push:
+#    branches:
+#      - 1.x
   schedule:
     - cron: "10 1 * * 0"
 

.gitignore

@@ -16,6 +16,8 @@ src/DevShop/Control/web
 .vagrant
 devmaster_id_rsa.pub
 
+vars.local.yml
+
 # Ignore bin dir, except for devshop CLI.
 # @TODO: Move CLI to a component, so we can ignore the entire folder.
 bin/*

RoboFile.php

@@ -577,7 +577,7 @@ public function up($docker_command = '', $opts = [
         $test_command = "/usr/share/devshop/tests/devshop-tests-upgrade.sh";
       }
       else {
-        $cmd[] = "docker compose up --detach devshop.server";
+        $cmd[] = "docker compose up --detach";
         if (!$opts['no-follow']) {
           $cmd[] = "docker-compose logs -f";
         }
@@ -622,8 +622,14 @@ public function up($docker_command = '', $opts = [
       // Process $extra vars into JSON for ENV var.
       $env_run['ANSIBLE_EXTRA_VARS'] = json_encode($extra_vars);
 
+      // Include an extra local vars file.
+      // Load Public SSH key from user to pass to devshop.remote authorized keys.
+      $vars_file_local = "aegir_user_authorized_keys: " . file_get_contents(getenv('HOME') . "/.ssh/id_rsa.pub");
+
+      file_put_contents('vars.local.yml', $vars_file_local);
+
       // Add vars.development.yml as final command line option.
-      $env_run['ANSIBLE_PLAYBOOK_COMMAND_OPTIONS'] = '--extra-vars=@/usr/share/devshop/vars.development.yml';
+      $env_run['ANSIBLE_PLAYBOOK_COMMAND_OPTIONS'] = '--extra-vars=@/usr/share/devshop/vars.development.yml --extra-vars=@/usr/share/devshop/vars.local.yml';
 
       // Override the DEVSHOP_DOCKER_COMMAND_RUN if specified.
       if (!empty($docker_command)) {
@@ -786,13 +792,13 @@ public function restart() {
   /**
    * Enter a bash shell in the devmaster container.
    */
-  public function shell($user = 'aegir') {
+  public function shell($user = 'aegir', $service = 'devshop.server') {
 
     if ($user) {
-        $process = new \Symfony\Component\Process\Process("docker-compose exec --user $user devshop.server bash");
+        $process = new \Symfony\Component\Process\Process("docker-compose exec --user $user $service bash");
     }
     else {
-        $process = new \Symfony\Component\Process\Process("docker-compose exec devshop.server bash");
+        $process = new \Symfony\Component\Process\Process("docker-compose exec $service bash");
     }
     $process->setTty(TRUE);
     $process->setTimeout(NULL);

docker/Dockerfile.remote

@@ -0,0 +1,18 @@
+#
+# DevShop Remote Server container
+#
+FROM devshop/server:latest
+
+LABEL maintainer="Jon Pugh"
+
+RUN echo "Building container from Dockerfile.remote, from $DEVSHOP_CONTAINER_FROM ..."
+RUN apt-get install openssh-server -y
+RUN service mysql start && mysql -e "GRANT ALL PRIVILEGES ON *.* TO root@'%' IDENTIFIED BY 'root' WITH GRANT OPTION;"
+RUN rm -rf ${DEVSHOP_ANSIBLE_PATH}/hosts && ln -s ${DEVSHOP_PATH}/roles/devshop.server/inventory.remote ${DEVSHOP_ANSIBLE_PATH}/hosts
+
+#ENV DEVSHOP_DOCKER_COMMAND_RUN ""
+
+ENTRYPOINT /usr/share/devshop/docker/bin/docker-systemd-entrypoint
+#CMD ""
+
+RUN devshop-logo 'DevShop Remote Server container build finished.'

docker/docker-compose.override.yml

@@ -10,5 +10,11 @@ services:
       - ../:/usr/share/devshop
       - ../aegir-home:/var/aegir
       - $HOME/.ssh:/var/aegir/.ssh
+    environment:
+      - XDEBUG_CONFIG="remote_host=172.17.0.1 idekey=PHPSTORM"
+
+  devshop.remote:
+    volumes:
+      - ../:/usr/share/devshop
     environment:
       - XDEBUG_CONFIG="remote_host=172.17.0.1 idekey=PHPSTORM"

docker/docker-compose.yml

@@ -4,21 +4,22 @@ version: '3'
 services:
   devshop.server:
     image: ${DEVSHOP_DOCKER_IMAGE:-devshop/server:php8.1}
-    build:
-      context: ".."
-      dockerfile: "docker/Dockerfile.${OS:-ubuntu2004}"
-      args:
-        # see .env files for defaults.
-        - DEVSHOP_CONTAINER_FROM
-        - DEVSHOP_DOCKER_COMMAND_BUILD_ARG
-        - ANSIBLE_EXTRA_VARS_ARG
-        - ANSIBLE_TAGS_ARG
-        - ANSIBLE_PLAYBOOK_COMMAND_OPTIONS_ARG
+# Builds Break. We must upgrade to ubuntu 20
+#    build:
+#      context: ".."
+#      dockerfile: "docker/Dockerfile.${OS:-ubuntu2004}"
+#      args:
+#        # see .env files for defaults.
+#        - DEVSHOP_CONTAINER_FROM
+#        - DEVSHOP_DOCKER_COMMAND_BUILD_ARG
+#        - ANSIBLE_EXTRA_VARS_ARG
+#        - ANSIBLE_TAGS_ARG
+#        - ANSIBLE_PLAYBOOK_COMMAND_OPTIONS_ARG
     hostname: devshop.local.computer
     ports:
       - "80:80"
       - "443:443"
-      - "2222:22"
+#      - "2222:22"
       - "33066:3306"
     privileged: true
     entrypoint: /usr/share/devshop/docker/bin/docker-systemd-entrypoint
@@ -43,3 +44,28 @@ services:
       - DEVSHOP_TESTS_ARTIFACTS_PATH
       # Setting this ensures the docker containers have this value, regardless if the container is an older image.
       - DEVSHOP_ANSIBLE_GROUP_NAME=devshop_server
+
+  devshop.remote:
+    image: devshop/remote:latest
+    build:
+      context: ".."
+      dockerfile: "docker/Dockerfile.remote"
+    hostname: remote.local.computer
+    ports:
+      - "8080:80"
+      - "4433:443"
+      - "2222:22"
+#      - "33066:3306"
+    privileged: true
+    volumes:
+      - /var/lib/mysql
+      - ../:/usr/share/devshop
+    command:
+      - "devshop-ansible-playbook"
+    entrypoint: /usr/share/devshop/docker/bin/docker-systemd-entrypoint
+    environment:
+      DEVSHOP_DOCKER_COMMAND_RUN: "echo 'Welcome to devshop.remote container.'"
+      DEVSHOP_ANSIBLE_GROUP_NAME: devshop_remote
+      ANSIBLE_EXTRA_VARS:
+      ANSIBLE_PLAYBOOK_COMMAND_OPTIONS:
+

roles/devshop.server/inventory.remote

@@ -0,0 +1,10 @@
+---
+# DevShop Ansible Static Inventory File
+# -------------------------------------
+
+devshop_remote:
+  hosts:
+    devshop.remote:
+
+  vars:
+    ansible_connection: local

roles/devshop.server/play.yml

@@ -129,3 +129,60 @@
     # Always run everything. Tags inside the roles will limit tasks run.
     - role: opendevshop.devmaster
       tags: [always]
+
+- hosts: devshop_remote
+  become: true
+
+  # These vars override everything except command line variables. Set variables per host in the inventory or vars files in /etc/ansible.
+  # See https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html#variable-precedence-where-should-i-put-a-variable
+  vars:
+    devshop_server_role: 'devshop.server'
+    aegir_user_name: aegir
+
+    # Set these in local inventory.
+    aegir_remote_database_user: remote_root
+    aegir_remote_database_password: password
+    aegir_user_authorized_keys: ""
+
+  vars_files:
+    - ../../vars.yml
+
+  tasks:
+      - name: "devshop.remote"
+        debug:
+          msg:
+            - "DevShop Remote Server Playbook begin..."
+        tags: [ always ]
+
+      - name: Install OpenSSH Server
+        apt:
+          pkg:
+            - "openssh-server"
+          state: present
+          update_cache: true
+        tags: [ always ]
+        when: ansible_os_family == "Debian"
+
+      - name: Ensure MySQL is started and enabled on boot.
+        service: "name=mysql state=started enabled=true"
+        tags: [ runtime ]
+
+      - name: Grant mysql user access from any server.
+        mysql_user:
+          name: "{{ aegir_remote_database_user }}"
+          host: "%"
+          password: "{{ aegir_remote_database_password }}"
+          priv: "*.*:ALL,GRANT"
+        tags: [ runtime ]
+
+      # @TODO Add authorized_keys
+      - name: Ensure authorized_keys for DevShop Server user account is present.
+        authorized_key:
+          user: "{{ aegir_user_name }}"
+          key: "{{ aegir_user_authorized_keys }}"
+          manage_dir: yes
+        tags: [ runtime ]
+        when:
+          - aegir_user_authorized_keys | length > 0
+
+      # @TODO Add authorized_keys

roles/opendevshop.devmaster/tasks/config-devmaster.yml

@@ -1,9 +1,12 @@
 ---
-- name: Clear drush caches
-  command: "{{ drush_executable_path }} cc drush"
-  become: true
-  become_user: "{{ aegir_user_name }}"
-  become_method: "{{ ansible_become_method_aegir }}"
+# Trying to solve https://github.com/opendevshop/devshop/actions/runs/5546470029/jobs/10126861378?pr=752#step:12:2470
+# Error:   localhost failed | msg: non-zero return code | stderr: Command hosting-tasks needs the following extension(s) enabled to        [error]
+#run: hosting.
+#- name: Clear drush caches
+#  command: "{{ drush_executable_path }} cc drush"
+#  become: true
+#  become_user: "{{ aegir_user_name }}"
+#  become_method: "{{ ansible_become_method_aegir }}"
 
 - name: DevShop Control | Save GitHub Token
   become: true
@@ -93,8 +96,16 @@
 #  become_user: "{{ aegir_user_name }}"
 #  become_method: "{{ ansible_become_method_aegir }}"
 #
-#- name: Clear drush caches
-#  command: "{{ drush_executable_path }} cc drush"
-#  become: true
-#  become_user: "{{ aegir_user_name }}"
-#  become_method: "{{ ansible_become_method_aegir }}"
+# Trying to solve https://github.com/opendevshop/devshop/actions/runs/5546571307/jobs/10127031820?pr=752#step:13:165
+#
+  #Error: nable to load https_apache driver for the http service: Expecting       [error]
+  #      class Provision_Service_http_https_apache
+  #Warning: efined array key "http" server.php:125                              [warning]
+  #Error: rush command terminated abnormally due to an unrecoverable error.       [error]
+  #      Error: Uncaught Error: Call to a member function setContext() on null
+  #      in
+- name: Clear drush caches
+  command: "{{ drush_executable_path }} cc drush"
+  become: true
+  become_user: "{{ aegir_user_name }}"
+  become_method: "{{ ansible_become_method_aegir }}"

roles/opendevshop.users/defaults/main.yml

@@ -8,7 +8,7 @@ aegir_user_force_set_uid: false
 
 aegir_user_name: aegir
 aegir_user_home: /var/aegir
-aegir_user_authorized_keys: ReplaceAtRuntime
+aegir_user_authorized_keys: ""
 aegir_logs_path: /var/log/aegir
 
 # Populated in prepare-user.yml

scripts/devshop-ansible-playbook

@@ -22,7 +22,7 @@ if [ -f "${DEVSHOP_PATH}/${ANSIBLE_PLAYBOOK}" ]; then
   ANSIBLE_PLAYBOOK="${DEVSHOP_PATH}/${ANSIBLE_PLAYBOOK}"
 fi
 
-ANSIBLE_PLAYBOOK=${ANSIBLE_PLAYBOOK:-"/etc/ansible/play.yml"}
+ANSIBLE_PLAYBOOK=${ANSIBLE_PLAYBOOK:-"$DEVSHOP_PATH/roles/devshop.server/play.yml"}
 ANSIBLE_CONFIG=${ANSIBLE_CONFIG:-""}
 ANSIBLE_TAGS=${ANSIBLE_TAGS:-""}
 ANSIBLE_SKIP_TAGS=${ANSIBLE_SKIP_TAGS:-""}
@@ -46,6 +46,7 @@ fi
 
 ON_FAIL=${ON_FAIL:-"systemctl status --no-pager"}
 
+cd $DEVSHOP_PATH
 devshop-logo "Checking Ansible Inventory for group '$DEVSHOP_ANSIBLE_GROUP_NAME' ..."
 echo "> Contents of /etc/ansible/hosts:"
 cat /etc/ansible/hosts

src/DevShop/Console/Application.php

@@ -150,7 +150,6 @@ public function __construct($version, $release_date)
           $this->devmaster_version = '0.5 or earlier';
         }
 
-        $this->devmaster_platform = $aliases['hostmaster']['platform'];
         $this->devmaster_root = $aliases['hostmaster']['root'];
         $this->devmaster_uri = $aliases['hostmaster']['uri'];
       }

src/DevShop/Control/scripts/composer/ScriptHandler.php

@@ -87,10 +87,10 @@ public static function createRequiredFiles(Event $event) {
       }
     }
 
-    // @TODO: Detect non-provision composer installs and continue through the "standard" settings.php file setup.
-    // This will probably have to be done to allow alternative hosts for "devshop control" sites.
-    $event->getIO()->warning("Skipping settings.php setup. Provision will configure the files.");
-    return;
+//    // @TODO: Detect non-provision composer installs and continue through the "standard" settings.php file setup.
+//    // This will probably have to be done to allow alternative hosts for "devshop control" sites.
+//    $event->getIO()->warning("Skipping settings.php setup. Provision will configure the files.");
+//    return;
 
     // Prepare the settings file for installation
     if (!$fs->exists($drupalRoot . '/sites/default/settings.php') && $fs->exists($drupalRoot . '/sites/default/default.settings.php')) {

src/DevShop/Control/tests/features/letsencrypt.feature

@@ -13,6 +13,7 @@ Feature: DevShop Servers have LetsEncrypt enabled out of the box.
     Then I select the radio button "LetsEncrypt"
     Then I select the radio button "Staging"
     And I press "Save"
+    When I run drush "cache-clear drush"
     When I run drush "hosting-tasks --force --fork=0 --strict=0"
 
     Then I click "Edit"