Skip to content

Commit

Permalink
contrib/intel/jenkins: Introduce new CI Tool for running tests
Browse files Browse the repository at this point in the history
New CI Tool for running tests is an Intel internal project. It will
be referred to as CI in our Jenkinsfile.

Signed-off-by: Zach Dworkin <[email protected]>
  • Loading branch information
zachdworkin committed Apr 3, 2024
1 parent 7d48b95 commit a358161
Showing 1 changed file with 72 additions and 28 deletions.
100 changes: 72 additions & 28 deletions contrib/intel/jenkins/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ def run_middleware(providers, stage_name, test, hw, partition, node_num,
}
}

def run_ci(stage_name, config_name) {
sh """source ${CI_LOCATION}/${env.CI_MODULE}/venv/bin/activate;\
python run.py \
--output=${env.LOG_DIR}/${stage_name} \
--job=${config_name}
"""
}

def gather_logs(cluster, key, dest, source) {
def address = "${env.USER}@${cluster}"

Expand Down Expand Up @@ -159,9 +167,22 @@ def checkout_ci_resources() {
"""
}

def checkout_ci() {
sh """
if [[ ! -d ${env.WORKSPACE}/ci ]]; then
mkdir ${env.WORKSPACE}/ci
else
rm -rf ${env.WORKSPACE}/ci && mkdir ${env.WORKSPACE}/ci
fi
git clone --recurse-submodules ${env.CI} ${env.WORKSPACE}/ci
"""
}

def checkout_external_resources() {
checkout_ci_resources()
checkout_upstream()
checkout_ci()
}

def generate_diff(def branch_name, def output_loc) {
Expand Down Expand Up @@ -240,6 +261,10 @@ def build(item, mode=null, hw=null, additional_args=null) {
run_python(PYTHON_VERSION, cmd)
}

def build_ci() {
sh "${CI_LOCATION}/${env.CI_MODULE}/bootstrap.sh"
}

def check_target() {
echo "CHANGE_TARGET = ${env.CHANGE_TARGET}"
if (changeRequest()) {
Expand Down Expand Up @@ -322,6 +347,7 @@ pipeline {
CUSTOM_WORKSPACE="${CB_HOME}/workspace/${JOB_NAME}/${env.BUILD_NUMBER}"
DELETE_LOCATION="${env.CUSTOM_WORKSPACE}/middlewares"
RUN_LOCATION="${env.CUSTOM_WORKSPACE}/ci_resources/legacy_pipeline_scripts/"
CI_LOCATION="${env.CUSTOM_WORKSPACE}/ci"
LOG_DIR = "${env.CUSTOM_WORKSPACE}/log_dir"
}
stages {
Expand Down Expand Up @@ -394,7 +420,17 @@ pipeline {
stage ('parallel-builds') {
when { equals expected: true, actual: DO_RUN }
parallel {
stage ('build-ci') {
steps {
script {
build_ci()
}
}
}
stage ('build-water') {
environment {
build_type = "water"
}
steps {
script {
slurm_build(BUILD_MODES, "water", "water", "water")
Expand All @@ -412,6 +448,9 @@ pipeline {
}
}
stage ('build-grass') {
environment {
build_type = "grass"
}
steps {
script {
slurm_build(BUILD_MODES, "grass", "grass", "grass")
Expand All @@ -429,6 +468,9 @@ pipeline {
}
}
stage ('build-electric') {
environment {
build_type = "electric"
}
steps {
script {
slurm_build(BUILD_MODES, "electric", "electric", "electric")
Expand Down Expand Up @@ -505,48 +547,50 @@ pipeline {
stage('parallel-tests') {
when { equals expected: true, actual: DO_RUN }
parallel {
stage('MPI_verbs-rxm_IMB') {
stage ('CI_MPI_verbs-rxm_IMB') {
environment {
build_type = "water"
}
steps {
script {
dir (RUN_LOCATION) {
def providers = [["verbs", "rxm"]]
for (def mpi in ["impi"]) {
for (imb_grp = 1; imb_grp < 4; imb_grp++) {
run_middleware(providers, "MPI", "IMB", "water",
"squirtle,totodile", "2", "${mpi}",
"${imb_grp}")
}
}
dir (CI_LOCATION) {
run_ci("CI_MPI_verbs-rxm_IMB", "pr_imb_water.json")
}
}
}
}
stage('MPI_verbs-rxm_OSU') {
stage ('CI_MPI_verbs-rxm_OSU') {
environment {
build_type = "water"
}
steps {
script {
dir (RUN_LOCATION) {
def providers = [["verbs", "rxm"]]
for (def mpi in ["impi", "mpich"]) {
run_middleware(providers, "MPI", "osu", "water",
"squirtle,totodile", "2", "${mpi}")
}
dir (CI_LOCATION) {
run_ci("CI_MPI_verbs-rxm_OSU", "pr_osu_water.json")
}
}
}
}
stage('MPI_tcp') {
stage ('CI_MPI_tcp_IMB') {
environment {
build_type = "grass"
}
steps {
script {
dir (RUN_LOCATION) {
def providers = [["tcp", null]]
for (imb_grp = 1; imb_grp < 4; imb_grp++) {
run_middleware(providers, "MPI", "IMB", "grass",
"bulbasaur", "2", "impi", "${imb_grp}")
}
for (def mpi in ["impi", "mpich"]) {
run_middleware(providers, "MPI", "osu", "grass", "bulbasaur",
"2", "${mpi}")
}
dir (CI_LOCATION) {
run_ci("CI_MPI_tcp_IMB", "pr_imb_grass.json")
}
}
}
}
stage ('CI_MPI_tcp_OSU') {
environment {
build_type = "grass"
}
steps {
script {
dir (CI_LOCATION) {
run_ci("CI_MPI_tcp_OSU", "pr_osu_grass.json")
}
}
}
Expand Down

0 comments on commit a358161

Please sign in to comment.