diff --git a/Jenkinsfile b/Jenkinsfile index 93e3ea1..815ec78 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,234 +99,233 @@ pipeline { cleanWs() } } - stage('Initialize') { - steps { - // Start preparing environment. - parallel( - "Report": { - sh 'env > env.txt' - sh 'echo $BRANCH_NAME > branch.txt' - sh 'echo "$BRANCH_NAME"' - sh 'cat env.txt' - sh 'cat branch.txt' - sh 'echo $START_DAY > dow.txt' - sh 'echo "$START_DAY"' - }, - "Reset base": { - // Get a mount point ready - sh 'mkdir -p $WORKSPACE/mnt || true' - // Ninja in our file credentials from Jenkins. - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - // Try and ssh fuse skyhook onto our local system. - sh 'sshfs -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -o idmap=user skyhook@skyhook.berkeleybop.org:/home/skyhook $WORKSPACE/mnt/' - } - // Remove anything we might have left around from - // times past. - sh 'rm -r -f $WORKSPACE/mnt/$BRANCH_NAME || true' - // Rebuild directory structure. - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/bin || true' - // WARNING/BUG: needed for arachne to run at - // this point. - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/lib || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/ttl || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/blazegraph || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/annotations || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/pages || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/solr || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/panther || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/metadata || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/annotations || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/ontology || true' - sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/reports || true' - // Tag the top to let the world know I was at least - // here. - sh 'echo "Runtime summary." > $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'date >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Release notes: https://github.com/geneontology/go-site/tree/master/releases" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Branch: $BRANCH_NAME" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Start day: $START_DAY" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Start date: $START_DATE" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // stage('Initialize') { + // steps { + // // Start preparing environment. + // parallel( + // "Report": { + // sh 'env > env.txt' + // sh 'echo $BRANCH_NAME > branch.txt' + // sh 'echo "$BRANCH_NAME"' + // sh 'cat env.txt' + // sh 'cat branch.txt' + // sh 'echo $START_DAY > dow.txt' + // sh 'echo "$START_DAY"' + // }, + // "Reset base": { + // // Get a mount point ready + // sh 'mkdir -p $WORKSPACE/mnt || true' + // // Ninja in our file credentials from Jenkins. + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // // Try and ssh fuse skyhook onto our local system. + // sh 'sshfs -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -o idmap=user skyhook@skyhook.berkeleybop.org:/home/skyhook $WORKSPACE/mnt/' + // } + // // Remove anything we might have left around from + // // times past. + // sh 'rm -r -f $WORKSPACE/mnt/$BRANCH_NAME || true' + // // Rebuild directory structure. + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/bin || true' + // // WARNING/BUG: needed for arachne to run at + // // this point. + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/lib || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/ttl || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/blazegraph || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/annotations || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/pages || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/solr || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/panther || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/metadata || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/annotations || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/ontology || true' + // sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/reports || true' + // // Tag the top to let the world know I was at least + // // here. + // sh 'echo "Runtime summary." > $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'date >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Release notes: https://github.com/geneontology/go-site/tree/master/releases" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Branch: $BRANCH_NAME" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Start day: $START_DAY" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Start date: $START_DATE" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Official release date: metadata/release-date.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Official Zenodo archive DOI: metadata/release-archive-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "Official Zenodo archive DOI: metadata/release-reference-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - sh 'echo "TODO: Note software versions." >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' - // TODO: This should be wrapped in exception - // handling. In fact, this whole thing should be. - sh 'fusermount -u $WORKSPACE/mnt/ || true' - } - ) - } - } - // Build owltools and get it into the shared filesystem. - stage('Ready production software') { - steps { - // Legacy: build 'owltools-build' - dir('./owltools') { - // Remember that git lays out into CWD. - git 'https://github.com/owlcollab/owltools.git' - sh 'mvn -f OWLTools-Parent/pom.xml -U clean install -DskipTests -Dmaven.javadoc.skip=true -Dsource.skip=true' - // Attempt to rsync produced into bin/. - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Runner/target/owltools skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Oort/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-NCBI/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' - sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Oort/reporting/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' - sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Runner/contrib/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' - } - } - } - } - // We need some metadata to make the current NEO build work. - stage('Produce metadata') { - steps { - // Create a relative working directory and setup our - // data environment. - dir('./go-site') { - git 'https://github.com/geneontology/go-site.git' + // sh 'echo "Official release date: metadata/release-date.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Official Zenodo archive DOI: metadata/release-archive-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "Official Zenodo archive DOI: metadata/release-reference-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // sh 'echo "TODO: Note software versions." >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt' + // // TODO: This should be wrapped in exception + // // handling. In fact, this whole thing should be. + // sh 'fusermount -u $WORKSPACE/mnt/ || true' + // } + // ) + // } + // } + // // Build owltools and get it into the shared filesystem. + // stage('Ready production software') { + // steps { + // // Legacy: build 'owltools-build' + // dir('./owltools') { + // // Remember that git lays out into CWD. + // git 'https://github.com/owlcollab/owltools.git' + // sh 'mvn -f OWLTools-Parent/pom.xml -U clean install -DskipTests -Dmaven.javadoc.skip=true -Dsource.skip=true' + // // Attempt to rsync produced into bin/. + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Runner/target/owltools skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Oort/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-NCBI/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' + // sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Oort/reporting/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' + // sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Runner/contrib/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/' + // } + // } + // } + // } + // // We need some metadata to make the current NEO build work. + // stage('Produce metadata') { + // steps { + // // Create a relative working directory and setup our + // // data environment. + // dir('./go-site') { + // git 'https://github.com/geneontology/go-site.git' - sh './scripts/combine-datasets-metadata.py metadata/datasets/*.yaml > metadata/datasets.json' + // sh './scripts/combine-datasets-metadata.py metadata/datasets/*.yaml > metadata/datasets.json' - // Deploy to S3 location for pickup in next stage. - withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) { - sh 's3cmd -c $S3CMD_JSON --acl-public --reduced-redundancy --mime-type=application/json put metadata/datasets.json s3://go-build/metadata/' - } - } - } - } - // See https://github.com/geneontology/go-ontology for details - // on the ontology release pipeline. This ticket runs - // daily(TODO?) and creates all the files normally included in - // a release, and deploys to S3. - stage('Produce NEO') { - steps { - // Create a relative working directory and setup our - // data environment. - dir('./neo') { - git 'https://github.com/geneontology/neo.git' + // // Deploy to S3 location for pickup in next stage. + // withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) { + // sh 's3cmd -c $S3CMD_JSON --acl-public --reduced-redundancy --mime-type=application/json put metadata/datasets.json s3://go-build/metadata/' + // } + // } + // } + // } + // // See https://github.com/geneontology/go-ontology for details + // // on the ontology release pipeline. This ticket runs + // // daily(TODO?) and creates all the files normally included in + // // a release, and deploys to S3. + // stage('Produce NEO') { + // steps { + // // Create a relative working directory and setup our + // // data environment. + // dir('./neo') { + // git 'https://github.com/geneontology/neo.git' - // Default namespace. - sh 'OBO=http://purl.obolibrary.org/obo' + // // Default namespace. + // sh 'OBO=http://purl.obolibrary.org/obo' - // Make all software products available in bin/. - sh 'mkdir -p bin/' - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/* ./bin/' - } - sh 'chmod +x bin/*' + // // Make all software products available in bin/. + // sh 'mkdir -p bin/' + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/* ./bin/' + // } + // sh 'chmod +x bin/*' - // - withEnv(['PATH+EXTRA=:bin:./bin', 'JAVA_OPTS=-Xmx128G', 'OWLTOOLS_MEMORY=128G', 'BGMEM=128G']){ - retry(3){ - sh 'make clean all' - } - } + // // + // withEnv(['PATH+EXTRA=:bin:./bin', 'JAVA_OPTS=-Xmx128G', 'OWLTOOLS_MEMORY=128G', 'BGMEM=128G']){ + // retry(3){ + // sh 'make clean all' + // } + // } - // Make sure that we copy any files there, - // including the core dump of produced. - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo.obo neo.owl skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/' - } + // // Make sure that we copy any files there, + // // including the core dump of produced. + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo.obo neo.owl skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/' + // } - // WARNING/BUG: This occurs "early" as we need NEO - // in the proper location before building GO (for - // testing, solr loading, etc.). Once we have - // ubiquitous ontology catalogs, publishing can - // occur properly at the end and after testing. - // See commented out section below. - // - // Deploy to S3 location for pickup by PURL via CF. - withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) { - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/' - } - } - } - } - // Produce the standard GO package. - stage('Produce GO') { - agent { - docker { - image 'obolibrary/odkfull:v1.2.22' - // Reset Jenkins Docker agent default to original - // root. - args '-u root:root' - } - } - steps { - // Create a relative working directory and setup our - // data environment. - dir('./go-ontology') { - git 'https://github.com/geneontology/go-ontology.git' - - // Default namespace. - sh 'env' + // // WARNING/BUG: This occurs "early" as we need NEO + // // in the proper location before building GO (for + // // testing, solr loading, etc.). Once we have + // // ubiquitous ontology catalogs, publishing can + // // occur properly at the end and after testing. + // // See commented out section below. + // // + // // Deploy to S3 location for pickup by PURL via CF. + // withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) { + // sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/' + // sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/' + // } + // } + // } + // } + // // Produce the standard GO package. + // stage('Produce GO') { + // agent { + // docker { + // image 'obolibrary/odkfull:v1.2.22' + // // Reset Jenkins Docker agent default to original + // // root. + // args '-u root:root' + // } + // } + // steps { + // // Create a relative working directory and setup our + // // data environment. + // dir('./go-ontology') { + // git 'https://github.com/geneontology/go-ontology.git' - dir('./src/ontology') { - retry(3){ - sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all' - } - retry(3){ - sh 'make prepare_release' - } - } + // // Default namespace. + // sh 'env' - // Make sure that we copy any files there, - // including the core dump of produced. - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - //sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology' - sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -r target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/' - } + // dir('./src/ontology') { + // retry(3){ + // sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all' + // } + // retry(3){ + // sh 'make prepare_release' + // } + // } - // Now that the files are safely away onto skyhook for - // debugging, test for the core dump. - script { - if( WE_ARE_BEING_SAFE_P == 'TRUE' ){ + // // Make sure that we copy any files there, + // // including the core dump of produced. + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // //sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology' + // sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -r target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/' + // } - def found_core_dump_p = fileExists 'target/core_dump.owl' - if( found_core_dump_p ){ - error 'ROBOT core dump detected--bailing out.' - } - } - } - } - } - } - //.. - stage('Produce derivatives (GOlr)') { - agent { - docker { - image 'geneontology/golr-autoindex-ontology:0aeeb57b6e20a4b41d677a8ae934fdf9ecd4b0cd_2019-01-24T124316' - // Reset Jenkins Docker agent default to original - // root. - args '-u root:root --mount type=tmpfs,destination=/srv/solr/data' - } - } - steps { - /// - /// Produce Solr index. - /// + // // Now that the files are safely away onto skyhook for + // // debugging, test for the core dump. + // script { + // if( WE_ARE_BEING_SAFE_P == 'TRUE' ){ - // sh 'ls /srv' - // sh 'ls /tmp' + // def found_core_dump_p = fileExists 'target/core_dump.owl' + // if( found_core_dump_p ){ + // error 'ROBOT core dump detected--bailing out.' + // } + // } + // } + // } + // } + // } + // //.. + // stage('Produce derivatives (GOlr)') { + // agent { + // docker { + // image 'geneontology/golr-autoindex-ontology:0aeeb57b6e20a4b41d677a8ae934fdf9ecd4b0cd_2019-01-24T124316' + // // Reset Jenkins Docker agent default to original + // // root. + // args '-u root:root --mount type=tmpfs,destination=/srv/solr/data' + // } + // } + // steps { + // /// + // /// Produce Solr index. + // /// - // Build index into tmpfs. - sh 'bash /tmp/run-indexer.sh' + // // sh 'ls /srv' + // // sh 'ls /tmp' - // Copy tmpfs Solr contents onto skyhook. - sh 'tar --use-compress-program=pigz -cvf /tmp/golr-index-contents.tgz -C /srv/solr/data/index .' - withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { - // Copy over index. - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr-index-contents.tgz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/' - // Copy over log. - sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr_timestamp.log skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/' - } - } - } + // // Build index into tmpfs. + // sh 'bash /tmp/run-indexer.sh' + // // Copy tmpfs Solr contents onto skyhook. + // sh 'tar --use-compress-program=pigz -cvf /tmp/golr-index-contents.tgz -C /srv/solr/data/index .' + // withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) { + // // Copy over index. + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr-index-contents.tgz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/' + // // Copy over log. + // sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr_timestamp.log skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/' + // } + // } + // } stage('Produce derivatives (Blazegraph)') { agent { docker { @@ -350,16 +349,19 @@ pipeline { // BUG/TODO: This will need to point inward at some point. sh 'curl -L -o /tmp/reacto.owl http://snapshot.geneontology.org/ontology/extensions/reacto.owl' // DEBUG: confirm for the moment. - sh 'ls -AlF /tmp/neo.owl' - sh 'ls -AlF /tmp/reacto.owl' + sh 'ls -AlF /tmp/*' // WARNING: Having trouble getting the journal to the // right location. Theoretically, if the pipeline // choked at the wrong time, a hard-to-erase file // could be left on the system. See "rm" above. sh 'java -cp /tmp/blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org /tmp/blazegraph.properties /tmp/go-lego.owl' + sh 'ls -AlF /tmp/*' sh 'cp blazegraph.jnl /tmp/blazegraph-go-lego.jnl' + sh 'ls -AlF /tmp/*' sh 'java -cp /tmp/blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org /tmp/blazegraph.properties /tmp/reacto.owl' + sh 'ls -AlF /tmp/*' sh 'java -cp /tmp/blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org /tmp/blazegraph.properties /tmp/neo.owl' + sh 'ls -AlF /tmp/*' sh 'mv blazegraph.jnl /tmp/blazegraph-go-lego-reacto-neo.jnl' sh 'pigz /tmp/blazegraph-go-lego.jnl' sh 'pigz /tmp/blazegraph-go-lego-reacto-neo.jnl'