Finish automated test result saving to github action artifacts (#593)

* Move test-assets folder to /var/aegir/test-assets inside the containers. [ch-17]
* Fix last page output printing
* Create the test-assets folder in the robo prepare:source command.
* Catch and report problems with the test assets path: does not exist, or is not writable.
* Only write output to an alternate directory if there is a path string.
* Rename test-assets directory to test-artifacts.
* Revert accidentally committed line.
* Default DEVSHOP_TESTS_ASSETS_PATH to blank and the http-accessible path will be used. CI will override when needed.
* Include the docker-compose.tests.yml file so that the test artifacts folder is there, and add the env var to that docker-compose file.
* Typo! :(
* Remove unneeded test.yml github workflow file
* Create the test-artifacts folder before running robo test.
* Use mkdir -p to ensure command passes if dir already exists.
* Make each output html file different so multiple tests can save their own file.
* Wrong path on test volumes for test assets.
* Make test-artifacts 777 for full writability, at least for now.
* Save docker logs separately from test errors to create two artifacts.

.github/workflows/build.yml

@@ -121,17 +121,25 @@ jobs:
 
     - name: Test DevShop
       run: |
-        bin/robo test
+        bin/robo test --compose-file=docker-compose.yml:docker-compose.tests.yml
 
-    - name: Save Assets
+    - name: Save Docker Logs as Artifacts
       if: always()
       run: |
-        docker ps -a > ./.github/test-assets/build-docker-ps.log && \
-        $DEBUG_COMMAND > ./.github/test-assets/build-debug-command.log && \
-        docker-compose logs > ./.github/test-assets/build-docker-compose.log
+        docker ps -a >        artifact-docker-ps.log && \
+        $DEBUG_COMMAND >      artifact-debug-command-output.log && \
+        docker-compose logs > artifact-docker-compose.log
 
-    - uses: actions/upload-artifact@v1
+    - name: Upload Test Artifacts
+      uses: actions/upload-artifact@v2
       if: always()
       with:
-        name: test-assets
-        path: ./.github/test-assets
+        name: TestFailures
+        path: ./.github/test-artifacts
+
+    - name: Upload Log Artifacts
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: DockerLogs
+        path: artifact-*

Dockerfile

@@ -220,7 +220,9 @@ RUN mkdir -p /var/log/aegir/ && \
     touch /var/log/aegir/hostmaster.access.log
 
 ENV DEVSHOP_ENTRYPOINT_LOG_FILES="/var/log/aegir/*"
-ENV DEVSHOP_TESTS_ASSETS_PATH="${DEVSHOP_PATH}/.github/test-assets"
+
+# Keep this blank and the http-accessible path will be used. CI will override.
+ENV DEVSHOP_TESTS_ASSETS_PATH=""
 
 # Set devshop_install_phase runtime here, since the Dockerfile is ALWAYS buildtime.
 ENV ANSIBLE_BUILD_COMMAND="devshop-ansible-playbook \
@@ -235,7 +237,6 @@ RUN \
 
 # Cleanup unwanted systemd files. See bin/docker-systemd-prepare and https://github.com/geerlingguy/docker-ubuntu1804-ansible/pull/12
 RUN bash $DEVSHOP_PATH/bin/docker-systemd-prepare
-RUN chmod 766 $DEVSHOP_TESTS_ASSETS_PATH
 
 # Remove devmaster dir if desired so that devshop code is reinstalled.
 ARG DEVSHOP_REMOVE_DEVMASTER_ARG=0

Dockerfile.base

@@ -215,7 +215,6 @@ ENV DEVSHOP_USER_UID ${DEVSHOP_USER_UID_ARG:-1000}
 RUN mkdir -p /var/log/devshop/
 
 ENV DEVSHOP_ENTRYPOINT_LOG_FILES="/var/log/devshop/*"
-ENV DEVSHOP_TESTS_ASSETS_PATH="${DEVSHOP_PATH}/.github/test-assets"
 
 # Set devshop_install_phase runtime here, since the Dockerfile is ALWAYS buildtime.
 ENV ANSIBLE_BUILD_COMMAND="devshop-ansible-playbook \
@@ -231,8 +230,6 @@ RUN \
 # Prepare systemd to run inside a container.
 RUN docker-systemd-prepare
 
-RUN chmod 766 $DEVSHOP_TESTS_ASSETS_PATH
-
 # Pre-build Information
 RUN \
   devshop-logo "Ansible Playbook Build Environment" && \

RoboFile.php

@@ -242,6 +242,9 @@ public function prepareSourcecode($opts = [
       $this->taskExecStack()
         ->exec("mkdir -p {$this->devshop_root_path}/aegir-home/.drush/commands")
         ->run();
+      $this->taskExecStack()
+        ->exec("mkdir -p {$this->devshop_root_path}/aegir-home/test-artifacts")
+        ->run();
     }
 
     // Clone all git repositories.
@@ -946,22 +949,34 @@ public function test($user = 'aegir', $opts = array(
   )) {
     $is_tty = !empty($_SERVER['XDG_SESSION_TYPE']) && $_SERVER['XDG_SESSION_TYPE'] == 'tty';
     $no_tty = !$is_tty? '-T': '';
-    $command = "docker-compose exec $no_tty --user $user devshop /usr/share/devshop/tests/devshop-tests.sh";
-    $provision_io = new \DevShop\Component\PowerProcess\PowerProcessStyle($this->input, $this->output);
-    $process = new \DevShop\Component\PowerProcess\PowerProcess($command, $provision_io);
 
-    $process->setTty(!empty($_SERVER['XDG_SESSION_TYPE']) && $_SERVER['XDG_SESSION_TYPE'] == 'tty');
+    // If running in CI, create the test-artifacts directory and ensure ownership first.
+    // @TODO: Move logic to a special CI container.
+    if (!empty($_SERVER['CI'])) {
+      $commands[] = "docker-compose exec $no_tty devshop mkdir -p /var/aegir/test-artifacts";
+      $commands[] = "docker-compose exec $no_tty devshop chown aegir:aegir /var/aegir/test-artifacts -R";
+      $commands[] = "docker-compose exec $no_tty devshop chmod 777 /var/aegir/test-artifacts -R";
+    }
 
-    $process->setEnv([
-      'COMPOSE_FILE' => $opts['compose-file'],
-    ]);
-    $process->setTimeout(NULL);
-    $process->disableOutput();
+    $commands[] = "docker-compose exec $no_tty --user $user devshop /usr/share/devshop/tests/devshop-tests.sh";
+    $provision_io = new \DevShop\Component\PowerProcess\PowerProcessStyle($this->input, $this->output);
+    foreach ($commands as $command) {
+      $process = new \DevShop\Component\PowerProcess\PowerProcess($command, $provision_io);
 
+      $process->setTty(!empty($_SERVER['XDG_SESSION_TYPE']) && $_SERVER['XDG_SESSION_TYPE'] == 'tty');
 
+      $process->setEnv([
+        'COMPOSE_FILE' => $opts['compose-file'],
+      ]);
+      $process->setTimeout(NULL);
+      $process->disableOutput();
     // @TODO: Figure out why PowerProcess::mustRun() fails so miserably: https://github.com/opendevshop/devshop/pull/541/checks?check_run_id=518074346#step:7:45
     // $process->mustRun();
-    $process->run();
+      $process->run();
+      if (!$process->isSuccessful()) {
+        return $process->getExitCode();
+      }
+    }
     return $process->getExitCode();
   }
 

devmaster/tests/features/bootstrap/FeatureContext.php

@@ -55,20 +55,34 @@ public function logAfterFailedStep($event)
             $base_url = $this->getMinkParameter('base_url');
             $drush_config = $this->drupalContext->getDrupalParameter('drush');
             $alias = $drush_config['alias'];
+            $url_to_path = preg_replace("/[^a-z0-9\.]/","", strtolower($this->getSession()->getCurrentUrl()));
 
             // If environment variable is set, save assets to that.
-            if (isset($_SERVER['DEVSHOP_TESTS_ASSETS_PATH']) && is_writable($_SERVER['DEVSHOP_TESTS_ASSETS_PATH'])) {
-              $files_path = $_SERVER['DEVSHOP_TESTS_ASSETS_PATH'];
-              $logged_string = "$files_path";
+            if (!empty(($_SERVER['DEVSHOP_TESTS_ASSETS_PATH']))) {
+              if (is_writable($_SERVER['DEVSHOP_TESTS_ASSETS_PATH'])) {
+                $files_path = $_SERVER['DEVSHOP_TESTS_ASSETS_PATH'];
+                $output_file_name = "output-{$url_to_path}.html";
+                $output_file_path = "{$files_path}/{$output_file_name}";
+                $output_notification_string = $output_file_path;
+              }
+              elseif (!file_exists($_SERVER['DEVSHOP_TESTS_ASSETS_PATH'])) {
+                throw new \Exception("DEVSHOP_TESTS_ASSETS_PATH was set, but the directory does not exist. Change the environment variable or create the directory: " . $_SERVER['DEVSHOP_TESTS_ASSETS_PATH']);
+              }
+              else {
+                throw new \Exception("DEVSHOP_TESTS_ASSETS_PATH was set, but is not writable: Change the environment variable or create the directory: " . $_SERVER['DEVSHOP_TESTS_ASSETS_PATH']);
+              }
             }
             // If not, load the public writable files folder for devshop, so the asset can be served over HTTP.
             else {
               // Lookup file_directory_path
               $cmd = "drush @$alias vget file_public_path --format=string";
               $files_path = trim(shell_exec($cmd));
-              $logged_string = "$base_url/$files_path/output.html";
+              $output_file_name = "output-{$url_to_path}.html";
+              $output_file_path = "{$files_path}/{$output_file_name}";
+              $output_notification_string = "{$base_url}/{$files_path}/{$output_file_name}}";
             }
 
+
             // Check for various problems.
             if (empty($files_path)) {
                 throw new \Exception("Unable to load files_public_path from devmaster: Results of command '$cmd' was empty.'");
@@ -80,8 +94,6 @@ public function logAfterFailedStep($event)
                 throw new \Exception("Assets path '$files_path' is not writable by the testing scipt and user.");
             }
 
-            $output_path = $files_path .'/output.html';
-
             // Print Current URL and Last reponse after any step failure.
             echo "Step Failed. \n";
             echo "Site: $alias \n";
@@ -90,22 +102,20 @@ public function logAfterFailedStep($event)
             if (!file_exists($files_path)) {
                 mkdir($files_path);
             }
-            $wrote = file_put_contents($output_path, $this->getSession()->getPage()->getContent());
+            $wrote = file_put_contents($output_file_path, $this->getSession()->getPage()->getContent());
 
+            if ($wrote) {
+                echo "Last Page Output Saved to: $output_notification_string \n";
+            }
+            else {
+                throw new \Exception("Something failed when writing output to $output_file_path ... \n");
+            }
 
             if (isset($_SERVER['CI'])) {
                 echo "\nLasts Response:\n";
                 $this->minkContext->printLastResponse();
             }
 
-            if ($wrote) {
-                echo "Last Page Output Saved to: $output_path \n";
-                echo "Last Page Output: $logged_string \n";
-            }
-            else {
-                throw new \Exception("Something failed when writing output to $files_path ... \n");
-            }
-
             echo "\nWatchdog Errors:\n";
             $this->drushContext->assertDrushCommand('wd-show');
             $this->drushContext->printLastDrushOutput();

docker-compose.tests.yml

@@ -14,6 +14,7 @@ services:
     environment:
     - GITHUB_TOKEN
     - CI=CI
+    - DEVSHOP_TESTS_ASSETS_PATH=/var/aegir/test-artifacts
 
     volumes:
-      - ./.github/.test-assets:/usr/share/devshop/.github/test-assets
+      - ./.github/test-artifacts:/var/aegir/test-artifacts

docker-compose.volumes.yml

@@ -15,3 +15,4 @@ services:
       - /var/lib/mysql
       - /var/logs/aegir
       - $HOME/.ssh:/var/aegir/.ssh
+      - ./.github/.test-artifacts:/var/aegir/test-artifacts

docker-compose.yml

@@ -82,10 +82,13 @@ services:
       # To allow the devmaster container to launch other docker containers, we need the host's path to aegir home directory.
       - HOST_AEGIR_HOME=/home/jon/Projects/devshop/aegir-home
 
-
       # Add additional files you would like to output in the container logs here, separated by a space.
       # - DEVSHOP_ENTRYPOINT_LOG_FILES="/var/log/aegir/*"
-      # - DEVSHOP_TESTS_ASSETS_PATH="/usr/share/devshop/.github/test-assets"
+
+      # For development, test output will be saved to the sites "files" folder so you can click the results.
+      # If you want to save test error output to a different folder change this variable.
+      # For CI, set this to the folder that will be saved as artifacts.
+      - DEVSHOP_TESTS_ASSETS_PATH=
 
     privileged: true
     volumes: