[WIP] CI: Use Azure DevOps and Internal BlueQuartz CI Cluster

.azure/azure-pipelines.yml

@@ -1,10 +1,27 @@
+#==================================================================================================
+# Building against a branch *other* than `develop` on simplnx. There are a few items that will
+# need to be changed.
+#
+# [1] Under `resources->repository->name` you will need to put your fork.
+#     You will also need to add another attribute to the `repository` for the branch/ref from github
+# [2] Under the `jobs->job->variables` section you will need to adjust the "simplnx_origin_name"
+#     variable to match your fork.
+#==================================================================================================
 
 resources:
   repositories:
   - repository: simplnx
     type: github
     endpoint: BlueQuartzSoftware
     name: BlueQuartzSoftware/simplnx
+  - repository: FileStore
+    type: github
+    endpoint: BlueQuartzSoftware
+    name: BlueQuartzSoftware/FileStore
+  - repository: SimplnxReview
+    type: github
+    endpoint: BlueQuartzSoftware
+    name: BlueQuartzSoftware/SimplnxReview
 
 trigger:
   - develop
@@ -13,75 +30,162 @@ jobs:
 - job:
   strategy:
     matrix:
-      macOS:
-        imageName: Darwin
-      Windows:
+      # macOS-OoC:
+      #   imageName: Darwin
+      #   preset_name: ci-macos-x64
+      #   home_dir: /Users/buildbot
+      Windows-OoC:
         imageName: Windows_NT
-      Linux:
+        preset_name: ci-windows-v143-ooc
+        home_dir: C:/Users/buildbot
+      Linux-OoC:
         imageName: Linux
+        preset_name: ci-linux-x64-ooc
+        home_dir: /home/buildbot
+        python_dir: /opt/local/anaconda3/envs/dream3d/bin
 
   pool:
     name: BlueQuartz Self Hosted
     demands:
       - Agent.OS -equals $(imageName)
-      - BQ.COMPLEX
- 
+      - BQ.PROXMOX
+
   workspace:
     clean: all
 
   timeoutInMinutes: 120
 
   variables:
-    simplnx_source_dir: $(Build.Repository.LocalPath)
-    build_type: Release
+    simplnx_source_dir: $(Build.Repository.LocalPath)/simplnx
+    filestore_source_dir: $(Build.Repository.LocalPath)/FileStore
+    simplnx_review_source_dir: $(Build.Repository.LocalPath)/SimplnxReview
     dream3d_data_dir: $(Agent.WorkFolder)/DREAM3D_Data
+    model_type: Experimental
+    build_type: Release
+    simplnx_origin_name: bluequartzsoftware
+    simplnx_branch_name: develop
 
   steps:
   - checkout: self
     submodules: true
+  - checkout: FileStore
+    submodules: true
+  - checkout: SimplnxReview
+    submodules: true
 
-  - script: |
-      echo System.PullRequest.SourceBranch=$(System.PullRequest.SourceBranch)
-      echo System.PullRequest.PullRequestNumber=$(System.PullRequest.PullRequestNumber)
+  - powershell: |
+      try {
+        echo System.PullRequest.SourceBranch=$(System.PullRequest.SourceBranch)
+        echo System.PullRequest.PullRequestNumber=$(System.PullRequest.PullRequestNumber)
+      } catch {
+      }
       echo Build.SourceBranchName=$(Build.SourceBranchName)
       echo Build.Repository.Name=$(Build.Repository.Name)
       echo Build.Repository.Uri=$(Build.Repository.Uri)
+      echo Build.Repository.LocalPath=$(Build.Repository.LocalPath)
+      echo Build.BuildNumber=$(Build.BuildNumber)
       echo Agent.WorkFolder=$(Agent.WorkFolder)
       echo Agent.OS=$(Agent.OS)
-      echo Build.BuildNumber=$(Build.BuildNumber)
+      echo preset_name=$(preset_name)
     displayName: 'Dump Azure Variables'
 
+#==================================================================================================
+# Change from the default https checkout to a git remote with ssh
+# The build bots have appropriate ssh keys to make this work
+# This also allows CDash to get the real git commit that we are on for DREAM3DNX
+#==================================================================================================
+  - powershell: |
+      cd $(Build.Repository.LocalPath)/simplnx
+      git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa"
+      git remote rename origin azure
+      git remote add origin ssh://[email protected]/$(simplnx_origin_name)/simplnx
+      git fetch origin
+    displayName: Update Simplnx Repo
+
+  - powershell: |
+      cd $(Build.Repository.LocalPath)/FileStore
+      git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa"
+      git remote rename origin azure
+      git remote add origin ssh://[email protected]/$(simplnx_origin_name)/FileStore
+      git fetch origin
+    displayName: Update FileStore Repo
+
   - powershell: |
-      $DLL_PATH = Get-Content $(Agent.WorkFolder)/NX_DLL_PATH.txt -Raw
-      Write-Host "##vso[task.prependpath]$DLL_PATH"
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
-    displayName: Appending to PATH
+      cd $(Build.Repository.LocalPath)/SimplnxReview
+      git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa"
+      git remote rename origin azure
+      git remote add origin ssh://[email protected]/$(simplnx_origin_name)/SimplnxReview
+      git fetch origin
+    displayName: Update SimplnxReview Repo
 
-  - script: |
-      cmake -S $(simplnx_source_dir) -B $(Build.BinariesDirectory) -G Ninja -DCMAKE_BUILD_TYPE:STRING=$(build_type) -DDREAM3D_DATA_DIR=$(dream3d_data_dir) -C $(Agent.WorkFolder)/NX.cmake -DBUILDNAME:STRING="simplnx-$(Agent.MachineName)-$(Agent.OS)-PR$(System.PullRequest.PullRequestNumber)"
+#==================================================================================================
+# CDash Section:
+#    In this section we are going to call `ctest` 3 times in order to separate out the output from
+# each CDash stage: Configure,Update :: Build :: Testing
+# This makes the output easier to parse through for the testing.
+#==================================================================================================
 
-    displayName: Configure SIMPLNX
+  - powershell: |
+      $UpdatedPath = '$(python_dir)' + [IO.Path]::PathSeparator + [System.Environment]::GetEnvironmentVariable('PATH', [System.EnvironmentVariableTarget]::Process)
+      [System.Environment]::SetEnvironmentVariable('PATH', $UpdatedPath, [System.EnvironmentVariableTarget]::Process)
+      cd $(Build.BinariesDirectory)
+      ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_configure.cmake
+    displayName: CDash Update/Configure
+
+  - powershell: |
+      cd $(Build.BinariesDirectory)
+      ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_build.cmake
+    displayName: CDash Build
 
-  - script: |
-      cmake --build $(Build.BinariesDirectory) --config $(build_type) --target all
-    displayName: Build SIMPLNX
+  - powershell: |
+      cd $(Build.BinariesDirectory)
+      ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_test.cmake
+    displayName: CDash Test
 
-  - script: |
+#==================================================================================================
+# CPack Section
+#==================================================================================================
+  - powershell: |
       cd $(Build.BinariesDirectory)
       cpack -C $(build_type) --verbose
-    continueOnError: true
     displayName: Packing SIMPLNX
 
-  - script: |
-      cd $(Build.BinariesDirectory)
-      ctest -C $(build_type) -D Experimental --timeout 7200 -DCTEST_SITE:STRING=$(Agent.MachineName).bluequartz.net -Ddashboard_source_name:STRING=DREAM3D
-    continueOnError: true
-    displayName: Testing SIMPLNX
-
+#==================================================================================================
+# Publish Test Results
+#==================================================================================================
   - task: PublishTestResults@2
     inputs:
       testResultsFormat: CTest
       testResultsFiles: $(Build.BinariesDirectory)/Testing/*/Test.xml
       testRunTitle: CTest_$(Agent.Name)_$(Agent.OS)
-      failTaskOnFailedTests: false
-    displayName: Publish Test Results
+      failTaskOnFailedTests: true
+    displayName: Publish Commercial Test Results
+
+  # - script: |
+  #     cmake -S $(simplnx_source_dir) -B $(Build.BinariesDirectory)-Free -G Ninja -DDREAM3D_ENABLE_DOCUMENTATION=OFF -DDREAM3D_ENABLE_COMMERCIAL_BUILD=OFF -DCMAKE_BUILD_TYPE:STRING=$(build_type) -DDREAM3D_DATA_DIR=$(dream3d_data_dir) -C $(Agent.WorkFolder)/NX.cmake -DSIMPLNX_EXTRA_PLUGINS="Core;ITKImageProcessing;OrientationAnalysis" -DCTEST_BUILD_NAME:STRING="NX-Free-$(Agent.MachineName)-$(Agent.OS)-$(Build.SourceBranchName)_$(Build.BuildNumber)"
+  #   displayName: Configure Free DREAM3D
+
+  # - script: |
+  #     cmake --build $(Build.BinariesDirectory)-Free --config $(build_type) --target all
+  #   displayName: Build Free DREAM3D
+
+  # - script: |
+  #     cd $(Build.BinariesDirectory)-Free
+  #     cpack -C $(build_type) --verbose
+  #   continueOnError: false
+  #   displayName: Packing Free DREAM3D
+
+  # - script: |
+  #     cd $(Build.BinariesDirectory)-Free
+  #     ctest -C $(build_type) -D Experimental --timeout 7200 -DCTEST_SITE:STRING=$(Agent.MachineName).bluequartz.net -Ddashboard_source_name:STRING=DREAM3D
+  #   continueOnError: true
+  #   displayName: Testing Free DREAM3D
+
+  # - task: PublishTestResults@2
+  #   inputs:
+  #     testResultsFormat: CTest
+  #     testResultsFiles: $(Build.BinariesDirectory)-Free/Testing/*/Test.xml
+  #     testRunTitle: CTest_$(Agent.Name)_$(Agent.OS)
+  #     failTaskOnFailedTests: true
+  #   displayName: Publish Free Test Results
+

.azure/azure_ci_build.cmake

@@ -0,0 +1,39 @@
+
+#==================================================================================================
+# Define these variables
+#==================================================================================================
+# get_filename_component(HOST_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" ABSOLUTE)
+## This is the Fully Qualified host name of your system
+set(CTEST_SITE "$ENV{AGENT_NAME}.bluequartz.net")
+## The name for this build. Combine the build type, compiler, and OS into a single string
+set(CTEST_BUILD_NAME "$ENV{PRESET_NAME}-$ENV{BUILD_BUILDNUMBER}-PR$ENV{SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}")
+## The type of build we are going to do "Release | Debug"
+#set(CTEST_CONFIGURATION_TYPE Release)
+## The type of generator we are going to use "Make | Ninja | NMake | JOM"
+set(CTEST_CMAKE_GENERATOR "Ninja")
+
+#==================================================================================================
+# Append to the existing CTEST TAG
+#==================================================================================================
+ctest_start(Experimental ${CTEST_SOURCE_DIR} ${CTEST_BINARY_DIRECTORY} APPEND)
+
+#==================================================================================================
+# Build the project
+#==================================================================================================
+ctest_build(BUILD ${CTEST_BINARY_DIRECTORY}
+            CONFIGURATION ${CTEST_CONFIGURATION_TYPE}
+            NUMBER_ERRORS ctest_build_errors
+            NUMBER_WARNINGS ctest_build_warnings
+            RETURN_VALUE ctest_build_result
+            CAPTURE_CMAKE_ERROR ctest_cmake_result
+            )
+ctest_submit(PARTS Build)
+
+if("${ctest_build_result}" EQUAL -1 OR "${ctest_cmake_result}" EQUAL -1)
+  message(FATAL_ERROR "CONFIGURE ERRORS: Go to https://my.cdash.org/index.php?project=DREAM3D for more information.
+                  Site: ${CTEST_SITE}
+                  Build Name: ${CTEST_BUILD_NAME}
+                  CMake returned the following error code during build: ${ctest_cmake_result}
+                  ctest_build() returned the following error code during build: ${ctest_build_result}")
+endif()
+

.azure/azure_ci_configure.cmake

@@ -0,0 +1,79 @@
+
+#==================================================================================================
+# Define these variables
+#==================================================================================================
+# get_filename_component(HOST_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" ABSOLUTE)
+## This is the Fully Qualified host name of your system
+set(CTEST_SITE "$ENV{AGENT_NAME}.bluequartz.net")
+## The name for this build. Combine the build type, compiler, and OS into a single string
+set(CTEST_BUILD_NAME "$ENV{PRESET_NAME}-$ENV{BUILD_BUILDNUMBER}-PR$ENV{SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}")
+## The type of build we are going to do "Release | Debug"
+#set(CTEST_CONFIGURATION_TYPE Release)
+## The type of generator we are going to use "Make | Ninja | NMake | JOM"
+set(CTEST_CMAKE_GENERATOR "Ninja")
+
+#==================================================================================================
+# Look for a GIT command-line client.
+#==================================================================================================
+if(NOT DEFINED CTEST_GIT_COMMAND)
+  find_program(CTEST_GIT_COMMAND NAMES git git.exe git.cmd)
+endif()
+
+if(NOT EXISTS ${CTEST_GIT_COMMAND})
+  message(FATAL_ERROR "CONFIGURE ERRORS: Go to https://my.cdash.org/index.php?project=DREAM3D for more information.
+                  Site: ${CTEST_SITE}
+                  Build Name: ${CTEST_BUILD_NAME}
+                  No Git Found.")
+endif()
+
+
+#==================================================================================================
+# Configure the project
+#==================================================================================================
+file(WRITE ${CTEST_BINARY_DIRECTORY}/CMakeCache.txt "
+      SITE:STRING=${CTEST_SITE}
+      BUILDNAME:STRING=${CTEST_BUILD_NAME}
+      CTEST_USE_LAUNCHERS:BOOL=${CTEST_USE_LAUNCHERS}
+      DART_TESTING_TIMEOUT:STRING=${CTEST_TEST_TIMEOUT}
+      CMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE}
+      DART_TESTING_TIMEOUT:STRING=1500"
+)
+
+#==================================================================================================
+# Start CTest the project. This will create the TAG that is APPENEDED to for the other 2 steps in
+# the process.
+#==================================================================================================
+ctest_start(Experimental ${CTEST_SOURCE_DIR} ${CTEST_BINARY_DIRECTORY})
+
+#==================================================================================================
+# Update Git Hashes for the project
+#==================================================================================================
+set(REPO_NAMES "s")
+foreach(p ${REPO_NAMES})
+  ctest_update(SOURCE ${CTEST_DASHBOARD_ROOT}/${p} 
+              RETURN_VALUE ctest_update_result
+              CAPTURE_CMAKE_ERROR ctest_cmake_result)        
+endforeach(p ${REPO_NAMES})
+ctest_submit(PARTS Update)
+
+
+#==================================================================================================
+# Configure the project
+#==================================================================================================
+ctest_configure(  BUILD ${CTEST_BINARY_DIRECTORY}
+                  SOURCE ${CTEST_SOURCE_DIR}
+                  OPTIONS "--preset ${CMAKE_PRESET_NAME}"
+                  RETURN_VALUE ctest_configure_result
+                  CAPTURE_CMAKE_ERROR ctest_cmake_result)
+
+ctest_submit(PARTS Configure Notes)
+message(STATUS "ctest_configure_result: ${ctest_configure_result}")
+message(STATUS "    ctest_cmake_restult: ${ctest_cmake_result}")
+
+if("${ctest_cmake_result}" EQUAL -1 OR "${ctest_configure_result}" EQUAL -1)
+  message(FATAL_ERROR "CONFIGURE ERRORS: Go to https://my.cdash.org/index.php?project=DREAM3D for more information.
+                  Site: ${CTEST_SITE}
+                  Build Name: ${CTEST_BUILD_NAME}
+                  CMake returned the following error code during build: ${ctest_cmake_result}
+                  ctest_configure() returned the following error code during build: ${ctest_configure_result}")
+endif()

.azure/azure_ci_test.cmake

@@ -0,0 +1,26 @@
+
+#==================================================================================================
+# Define these variables
+#==================================================================================================
+# get_filename_component(HOST_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" ABSOLUTE)
+## This is the Fully Qualified host name of your system
+set(CTEST_SITE "$ENV{AGENT_NAME}.bluequartz.net")
+## The name for this build. Combine the build type, compiler, and OS into a single string
+set(CTEST_BUILD_NAME "$ENV{PRESET_NAME}-$ENV{BUILD_BUILDNUMBER}-PR$ENV{SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}")
+## The type of build we are going to do "Release | Debug"
+#set(CTEST_CONFIGURATION_TYPE Release)
+## The type of generator we are going to use "Make | Ninja | NMake | JOM"
+set(CTEST_CMAKE_GENERATOR "Ninja")
+
+
+#==================================================================================================
+# Append to the existing CTEST TAG
+#==================================================================================================
+ctest_start(Experimental ${CTEST_SOURCE_DIR} ${CTEST_BINARY_DIRECTORY} APPEND)
+
+
+#==================================================================================================
+# Run the unit tests
+#==================================================================================================
+ctest_test(${CTEST_TEST_ARGS} APPEND)
+ctest_submit(PARTS Test)