Updated data dump for tblDocExternalMetadata and tblDocAnalystReports. #398
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| type: | |
| - create | |
| - synchronize | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-pr-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| prNum: ${{ github.event.pull_request.number }} | |
| jobs: | |
| contexts: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - | |
| name: github | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| run: echo "$GITHUB_CONTEXT" | |
| solr-check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| change-detected: ${{ steps.check-dumps.outputs.change-detected }} | |
| last-update: ${{ steps.check-dumps.outputs.last-update }} | |
| cacher: ${{ steps.cache-commit-refs.outputs.cacher }} | |
| env: | |
| THIS: ${{ github.event.pull_request.head.ref }} | |
| GH_TOKEN: ${{ github.token }} | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| HEAD_REF: ${{ github.event.pull_request.head.ref }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: true | |
| steps: | |
| - | |
| name: Login to REVSYS Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: registry.revsys.com | |
| username: github | |
| password: ${{ secrets.REVSYS_DOCKER_REGISTRY_PASSWORD }} | |
| - | |
| name: install just | |
| run: | | |
| docker run -v /home/runner/.local/bin:/dist registry.revsys.com/just:0.9.4 || just --version | |
| - | |
| name: Checkout code | |
| uses: gladiatr72/checkout-action@v3-no-noise | |
| with: | |
| ref: ${{ env.THIS }} | |
| fetch-depth: 0 | |
| - | |
| name: Restore Cache | |
| run: | | |
| docker run --rm --user $UID -v $PWD/.cache:/mnt $( just tag )-cacher || true | |
| - | |
| name: check for new dump | |
| id: check-dumps | |
| run: | | |
| baseBranch=${{ env.BASE_REF }} | |
| headRef=${{ env.HEAD_REF }} | |
| prSpan="${baseBranch}..HEAD" # ${headRef} | |
| git fetch origin ${baseBranch}:${baseBranch} --depth 1 # this provides an anchor for the next two commands | |
| lastUpdate=$( git log -1 --format='%H' ${prSpan} -- dumps/nuremberg_prod_dump_latest.sqlite3.zip ) | |
| [[ -n "${lastUpdate}" ]] && \ | |
| echo "last-update=${lastUpdate}" >> $GITHUB_OUTPUT && | |
| echo "change-detected=true" >> $GITHUB_OUTPUT || | |
| echo "change-detected=false" >> $GITHUB_OUTPUT | |
| # extract file linked from the symlink | |
| DUMP_LINK_TGT=$( find dumps -inum $( stat -t -L dumps/*latest.sqlite3.zip | cut -d" " -f8 ) ) | |
| echo "## slr-check" | |
| [[ -n ${lastUpdate} ]] && | |
| echo " * last-update: ${lastUpdate}" >> $GITHUB_STEP_SUMMARY && | |
| [[ ! -f .cache/${lastUpdate} ]] && | |
| echo " * found new data: \`${DUMP_LINK_TGT}\`" >> $GITHUB_STEP_SUMMARY && | |
| echo ${lastUpdate} > .cache/${lastUpdate} || | |
| echo " * no new data" >> $GITHUB_STEP_SUMMARY | |
| - | |
| name: Save Cache | |
| id: cache-commit-refs | |
| run: | | |
| just push cacher | |
| echo "cacher=$( just tag )-cacher" >> $GITHUB_OUTPUT | |
| solr-refs: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - solr-check | |
| env: | |
| cacher: ${{ needs.solr-check.outputs.cacher }} | |
| steps: | |
| - | |
| name: Login to REVSYS Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: registry.revsys.com | |
| username: github | |
| password: ${{ secrets.REVSYS_DOCKER_REGISTRY_PASSWORD }} | |
| - | |
| name: display cached data commit refs | |
| run: | | |
| docker run --rm -v $PWD/.cache:/mnt ${{ env.cacher }} && | |
| cat <<CDCR >> $GITHUB_STEP_SUMMARY || true | |
| cache contents | |
| ---- | |
| CDCR | |
| cd .cache && find . -type f >> $GITHUB_STEP_SUMMARY | |
| exit 0 | |
| run: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: true | |
| needs: | |
| - solr-check | |
| env: | |
| THIS: ${{ format('refs/heads/{0}', github.event.pull_request.head.ref) }} | |
| buildSolr: ${{ needs.solr-check.outputs.change-detected }} | |
| last-update: ${{ needs.solr-check.outputs.last-update }} | |
| steps: | |
| - | |
| name: Login to REVSYS Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: registry.revsys.com | |
| username: github | |
| password: ${{ secrets.REVSYS_DOCKER_REGISTRY_PASSWORD }} | |
| - | |
| name: wut | |
| run: | | |
| cat <<wutEOF >> $GITHUB_STEP_SUMMARY | |
| * solr-check outputs | |
| * last-update: ${{env.last-update}} | |
| * buildSolr: ${{env.buildSolr}} | |
| wutEOF | |
| echo last-update=${{env.last-update}} | |
| echo buildSolr=${{env.buildSolr}} | |
| echo $output >> $GITHUB_STEP_SUMMARY | |
| - | |
| name: install just | |
| run: | | |
| docker run -v /home/runner/.local/bin:/dist registry.revsys.com/just:0.9.4 || just --version | |
| - | |
| name: update docker-compose | |
| run: | | |
| mkdir -pv $HOME/.docker/cli-plugins | |
| curl -Ls -o $HOME/.docker/cli-plugins/docker-compose \ | |
| https://github.com/docker/compose/releases/download/v2.13.0/docker-compose-linux-x86_64 | |
| chmod 755 $HOME/.docker/cli-plugins/docker-compose | |
| ln -sfv $HOME/.docker/cli-plugins/docker-compose $( which docker-compose ) | |
| - | |
| name: Checkout code | |
| uses: gladiatr72/checkout-action@v3-no-noise | |
| with: | |
| ref: ${{ env.THIS }} | |
| # env: | |
| # GITHUB_TRACE: 1 | |
| # GIT_CURL_VERBOSE: 1 | |
| - | |
| name: Restore Cache | |
| run: | | |
| docker run --user $UID --rm -v $PWD/.cache:/mnt $( just tag )-cacher || true | |
| - | |
| name: pull/rebuild solr image | |
| if: ${{ env.buildSolr == 'false' }} | |
| run: | | |
| #!/bin/bash | |
| cat <<SOLRTOP_${{github.run_number}} >> $GITHUB_STEP_SUMMARY | |
| ---- | |
| SOLRTOP_${{github.run_number}} | |
| set -o xtrace | |
| PRSOLRVTAG=$( just tag )-solr # v0.0.0-prNum-solr | |
| TGT_TAG=$( prNum= just tag )-solr # v0.0.0-solr | |
| echo "### pull/rebuild _Solr_ for test run" >> $GITHUB_STEP_SUMMARY # built on previous commit to PR branch | |
| ( | |
| docker manifest inspect ${PRSOLRVTAG} >& /dev/null && | |
| docker pull --quiet ${PRSOLRVTAG} && | |
| docker tag ${PRSOLRVTAG} ${TGT_TAG} && | |
| echo " * using \`${PRSOLRVTAG}\` (exists)" >> $GITHUB_STEP_SUMMARY | |
| ) || \ | |
| ( | |
| docker manifest inspect ${TGT_TAG} >& /dev/null && # use $( just tag )-solr if no PR-specific data changes | |
| docker pull --quiet ${TGT_TAG} && | |
| echo " * using ${TGT_TAG} (exists, no updates)" >> $GITHUB_STEP_SUMMARY | |
| ) || \ | |
| ( | |
| echo " * building \`${PRSOLRVTAG}\` (new data)" # last effort builds & pushes a PR-specific solr image | |
| NO_CACHE_TO=1 just regen-solr-image && | |
| docker tag ${PRSOLRVTAG} ${TGT_TAG} && | |
| docker push ${PRSOLRVTAG} && | |
| echo ${PRSOLRVTAG} > .cache/${{ env.last-update }} | |
| ) | |
| - | |
| name: build solr loader | |
| if: ${{ env.buildSolr == 'true' }} | |
| run: | | |
| export prNum= && | |
| NO_CACHE_TO=1 just build release --load '' | |
| echo " * using \`$( just tag )\` as Solr loader.." >> $GITHUB_STEP_SUMMARY | |
| - | |
| name: build solr image (new data) | |
| if: ${{ env.buildSolr == 'true' }} | |
| run: | | |
| set -o xtrace | |
| set -o verbose | |
| NO_CACHE_TO=1 just regen-solr-image && | |
| PRSOLRVTAG=$( just tag )-solr | |
| echo ${PRSOLRVTAG} > .cache/${{ env.last-update }} && | |
| echo " * re/built & persisted PR SOLR image (\`${PRSOLRVTAG}\`)" >> $GITHUB_STEP_SUMMARY | |
| # - | |
| # name: run tests | |
| # run: | | |
| # /bin/echo -en "\n----\n" >> $GITHUB_STEP_SUMMARY && | |
| # egrep -q '\[[[:blank:]]*notest[[:blank:]]*\]' <( git log -1 --format=%B ) && | |
| # cat <<RUNTESTS >> $GITHUB_STEP_SUMMARY && exit 0 | |
| # ---- | |
| # **NOTICE**: skipping tests (\`[notest]\` in msg body) | |
| # ----- | |
| # RUNTESTS | |
| # just test | |
| - | |
| name: saving cache | |
| run: just push cacher |