all.yml

name: Compile, Test, and Deploy

on:
  pull_request: {}
  push:
    branches:
      - main
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  lint-java:
    runs-on: 'ubuntu-latest'
    continue-on-error: true
    name: Lint Java
    defaults:
      run:
        working-directory: java
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 11
        uses: actions/setup-java@v3
        with:
          java-version: '11'
          distribution: 'corretto'
      - name: Check Formatting
        run: mvn --batch-mode com.spotify.fmt:fmt-maven-plugin:check

  lint-cpp:
    runs-on: 'ubuntu-latest'
    continue-on-error: true
    name: Lint C++
    steps:
      - uses: actions/checkout@v3
      - name: Check C++ Formatting
        uses: jidicula/clang-format-action@v4.4.1
        with:
          clang-format-version: 14
          fallback-style: LLVM

  run-java-tests:
    continue-on-error: true
    name: Test with Java ${{ matrix.java-version }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    # Use an old Linux container to get us an older (but much more compatible) glibc:
    container: ${{ matrix.container }}
    defaults:
      run:
        working-directory: java
    strategy:
      matrix:
        include:
          - { java-version: 11, os: "ubuntu-latest", container: "quay.io/pypa/manylinux2014_x86_64" }
          - { java-version: 11, os: "windows-latest", container: "" }
          - { java-version: 11, os: "macos-12", container: "" }
          - { java-version: 17, os: "ubuntu-latest", container: "quay.io/pypa/manylinux2014_x86_64" }
          - { java-version: 17, os: "windows-latest", container: "" }
          - { java-version: 17, os: "macos-12", container: "" }
    steps:
      - uses: actions/checkout@v3
      - name: Upgrade GPG and NodeJS
        if: runner.os == 'Linux'
        run: apt-get update && apt-get upgrade -y gnupg nodejs
      - name: Set up JDK ${{ matrix.java-version }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.java-version }}
          distribution: 'corretto'
          gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
          gpg-passphrase: MAVEN_GPG_PASSPHRASE
      - uses: ilammy/msvc-dev-cmd@v1
        if: runner.os == 'Windows'
      - name: Compile
        run: make
      - name: Run Tests
        run: mvn --batch-mode test

  run-python-tests:
    runs-on: ${{ matrix.os }}
    continue-on-error: true
    defaults:
      run:
        working-directory: python
    strategy:
      matrix:
        python-version:
#           - '3.7'
          - '3.8'
#           - '3.9'
#           - '3.10'
#           - '3.11'
        # TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed
        os: ['ubuntu-latest', windows-latest, macos-12]
    name: Test with Python ${{ matrix.python-version }} on ${{ matrix.os }}
    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      - name: Install Linux dependencies
        if: runner.os == 'Linux'
        run: sudo apt-get update && sudo apt-get install -y pkg-config 
      - name: Install test dependencies
        env:
          # on macOS and with Python 3.10: building NumPy from source fails without these options:
          NPY_BLAS_ORDER: ""
          NPY_LAPACK_ORDER: ""
        run: |
          python -m pip install --upgrade pip
          pip install wheel
          pip install -r dev-requirements.txt
      - name: Build voyager locally
        run: python setup.py develop
      - name: Run tox tests
        run: tox

  run-python-tests-with-address-sanitizer:
    defaults:
      run:
        working-directory: python
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version:
#           - '3.7'
          - '3.8'
#           - '3.9'
#           - '3.10'
#           - '3.11'
        os: ['ubuntu-latest']
    name: Test with Python ${{ matrix.python-version }} + Address Sanitizer
    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      - name: Install Linux dependencies
        if: runner.os == 'Linux'
        run: sudo apt-get update && sudo apt-get install -y pkg-config 
      - name: Install test dependencies
        run: |
          python -m pip install --upgrade pip
          pip install wheel
          pip install -r dev-requirements.txt
      - name: Build voyager locally
        env:
          DEBUG: "0"
          USE_ASAN: "1"
          CC: clang
          CXX: clang++
        run: python setup.py develop
      - name: Run tox tests with ASan loaded
        # pytest can exit before all Python objects have been destroyed,
        # so we tell ASan to ignore leaks.
        run: |
          ASAN_OPTIONS=detect_leaks=0 \
          LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \
          tox

  build-python-wheels:
    needs: [run-python-tests, run-python-tests-with-address-sanitizer]
    runs-on: ${{ matrix.os }}
    continue-on-error: true
    if: true # (github.event_name == 'release' && github.event.action == 'published') || contains(github.event.pull_request.labels.*.name, 'Also Test Wheels')
    strategy:
      matrix:
        include:
        - { os: macos-12, build: cp37-macosx_x86_64 }
        - { os: macos-12, build: cp38-macosx_x86_64 }
        - { os: macos-12, build: cp39-macosx_x86_64 }
        - { os: macos-12, build: cp310-macosx_x86_64 }
        - { os: macos-12, build: cp311-macosx_x86_64 }
        - { os: macos-12, build: cp312-macosx_x86_64 }
        - { os: macos-12, build: cp38-macosx_universal2 }
        - { os: macos-12, build: cp39-macosx_universal2 }
        - { os: macos-12, build: cp310-macosx_universal2 }
        - { os: macos-12, build: cp311-macosx_universal2 }
        - { os: macos-12, build: cp312-macosx_universal2 }
        - { os: macos-12, build: cp38-macosx_arm64 }
        - { os: macos-12, build: cp39-macosx_arm64 }
        - { os: macos-12, build: cp310-macosx_arm64 }
        - { os: macos-12, build: cp311-macosx_arm64 }
        - { os: macos-12, build: cp312-macosx_arm64 }
        # - { os: macos-12, build: pp37-macosx_x86_64 }
        # - { os: macos-12, build: pp38-macosx_x86_64 }
        # - { os: macos-12, build: pp39-macosx_x86_64 }
        - { os: windows-latest, build: cp37-win_amd64 }
        - { os: windows-latest, build: cp38-win_amd64 }
        - { os: windows-latest, build: cp39-win_amd64 }
        - { os: windows-latest, build: cp310-win_amd64 }
        - { os: windows-latest, build: cp311-win_amd64 }
        - { os: windows-latest, build: cp312-win_amd64 }
        # - { os: windows-latest, build: pp37-win_amd64 }
        # - { os: windows-latest, build: pp38-win_amd64 }
        # - { os: windows-latest, build: pp39-win_amd64 }
        - { os: 'ubuntu-latest', build: cp37-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp37-manylinux_aarch64 }
        - { os: 'ubuntu-latest', build: cp38-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp38-manylinux_aarch64 }
        - { os: 'ubuntu-latest', build: cp39-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp39-manylinux_aarch64 }
        - { os: 'ubuntu-latest', build: cp310-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp310-manylinux_aarch64 }
        - { os: 'ubuntu-latest', build: cp311-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp311-manylinux_aarch64 }
        - { os: 'ubuntu-latest', build: cp312-manylinux_x86_64 }
        - { os: 'ubuntu-latest', build: cp312-manylinux_aarch64 }
        # - { os: 'ubuntu-latest', build: pp37-manylinux_x86_64 }
        # - { os: 'ubuntu-latest', build: pp37-manylinux_aarch64 }
        # - { os: 'ubuntu-latest', build: pp38-manylinux_x86_64 }
        # - { os: 'ubuntu-latest', build: pp38-manylinux_aarch64 }
        # - { os: 'ubuntu-latest', build: pp39-manylinux_x86_64 }
        # - { os: 'ubuntu-latest', build: pp39-manylinux_aarch64 }
    name: Build wheel for ${{ matrix.build }}
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      # Used to host cibuildwheel, so version doesn't really matter
      - uses: actions/setup-python@v3
        with:
          python-version: "3.10"
      - name: Install cibuildwheel
        run: python -m pip install 'cibuildwheel>=2.11.0'
      - name: Copy README into place
        run: cp README.md python/
      - name: Set up QEMU for aarch64 on Linux
        if: runner.os == 'Linux'
        uses: docker/setup-qemu-action@v1
        with:
          platforms: all
      - name: Build wheels
        run: cp -r cpp python/cpp && cd python && python -m cibuildwheel --output-dir ../wheelhouse
        env:
          CIBW_TEST_REQUIRES: -r dev-requirements.txt
          CIBW_TEST_COMMAND: "pytest {project}/tests -x"
          CIBW_BUILD: ${{ matrix.build }}
          CIBW_ARCHS: auto64 # Only support building 64-bit wheels. It's 2022!
          CIBW_ARCHS_LINUX: auto64 aarch64 # Useful for building linux images with Apple Silicon
          CIBW_ARCHS_MACOS: x86_64 universal2 arm64 # Support Apple Silicon
          CIBW_ARCHS_WINDOWS: AMD64
          # on macOS and with Python 3.10: building NumPy from source fails without these options:
          CIBW_ENVIRONMENT: NPY_BLAS_ORDER="" NPY_LAPACK_ORDER=""
          CIBW_REPAIR_WHEEL_COMMAND_LINUX: pip install auditwheel-symbols && (auditwheel repair -w {dest_dir} {wheel} || auditwheel-symbols --manylinux 2010 {wheel})
          # The manylinux container doesn't have a new enough glibc version,
          # so we can't run post-wheel-build tests there.
          # Also testing any pypy versions fails, as TensorFlow isn't pypy compatible.
          CIBW_TEST_SKIP: "*manylinux* *pp* *-macosx_universal2:arm64"
          # Use the minimum macOS deployment target that has C++17 support:
          MACOSX_DEPLOYMENT_TARGET: "10.13"
      - uses: actions/upload-artifact@v3
        with:
          if-no-files-found: error
          name: python-wheels
          path: ./wheelhouse/*.whl

  build-java-jars:
    continue-on-error: false
    name: Build JAR with on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    # Use an old Linux container to get us an older (but much more compatible) glibc:
    container: ${{ matrix.container }}
    defaults:
      run:
        working-directory: java
    strategy:
      matrix:
        include:
          - { java-version: 11, os: "ubuntu-latest", container: "quay.io/pypa/manylinux2014_x86_64" }
          - { java-version: 11, os: "windows-latest", container: "" }
          - { java-version: 11, os: "macos-12", container: "" }
    steps:
      - uses: actions/checkout@v3
      - name: Upgrade GPG and NodeJS
        if: runner.os == 'Linux'
        run: apt-get update && apt-get upgrade -y gnupg nodejs
      - name: Set up JDK ${{ matrix.java-version }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.java-version }}
          distribution: 'corretto'
          gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
          gpg-passphrase: MAVEN_GPG_PASSPHRASE
      - name: Set up QEMU for aarch64 on Linux
        if: runner.os == 'Linux'
        uses: docker/setup-qemu-action@v1
        with:
          platforms: all
      - uses: ilammy/msvc-dev-cmd@v1
        if: runner.os == 'Windows'
      - name: Compile
        run: make
      - name: Package
        run: mvn --batch-mode package
        env:
          MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }}
      - uses: actions/upload-artifact@v3
        with:
          if-no-files-found: error
          name: java-${{ matrix.os }}
          path: java/target/voyager-*.jar

  combine-java-jars:
    name: Combine JARs
    runs-on: ubuntu-latest
    needs: build-java-jars
    steps:
      - uses: actions/download-artifact@v3
        with:
          path: artifacts
      - name: Unpack and Repack JARs
        shell: bash
        working-directory: artifacts
        run: |
          rm -fv java-*/*-sources.jar
          rm -fv java-*/*-javadoc.jar
          for artifact in java-*; do unzip -o $artifact/*.jar -d combined-jar; done
          # Quick test that the JARs were built for the correct architectures
          echo "Checking linux-x64/libvoyager.so for x86 architecture..."
          file combined-jar/linux-x64/libvoyager.so | grep x86
          echo "Checking mac-x64/libvoyager.dylib for x86 architecture..."
          file combined-jar/mac-x64/libvoyager.dylib | grep x86
          echo "Checking linux-aarch64/libvoyager.so for aarch64 architecture..."
          file combined-jar/linux-aarch64/libvoyager.so | grep aarch64
          echo "Checking mac-aarch64/libvoyager.dylib for arm64 architecture..."
          file combined-jar/mac-aarch64/libvoyager.dylib | grep arm64

          # Check the Linux JARs to ensure a sufficiently old required GLIBC and GLIBCXX version:
          echo "Checking linux-aarch64/libvoyager.so for GLIBC 2.27..."
          objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 2\.27
          echo "Checking linux-aarch64/libvoyager.so for GLIBCXX 3.4.22..."
          objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 3\.4\.22
          echo "Checking linux-x64/libvoyager.so for GLIBC 2.29..."
          objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 2\.29
          echo "Checking linux-x64/libvoyager.so for GLIBCXX 3.4.22..."
          objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 3\.4\.22

          zip -r $(ls java-* | grep jar | tail -n 1) combined-jar/*
      - uses: actions/upload-artifact@v3
        with:
          if-no-files-found: error
          name: java-all-platforms
          path: artifacts/*.jar

  upload-pypi:
    defaults:
      run:
        working-directory: python
    needs: [build-python-wheels, run-python-tests-with-address-sanitizer, run-python-tests]
    runs-on: 'ubuntu-latest'
    name: "Upload wheels to PyPI"
    if: github.event_name == 'release' && github.event.action == 'published'
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: python-wheels
          path: dist
      - uses: pypa/gh-action-pypi-publish@v1.4.2
        with:
          user: __token__
          password: ${{ secrets.PYPI_DEPLOY_TOKEN }}

  upload-maven:
    name: Upload to Maven Central
    runs-on: ubuntu-latest
    needs: combine-java-jars
    if: github.event_name == 'release' && github.event.action == 'published'
    defaults:
      run:
        working-directory: java
    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: java-all-platforms
          path: combined-jar
      - name: Set up JDK 11
        uses: actions/setup-java@v3
        with:
          java-version: 11
          distribution: 'corretto'
          server-id: ossrh
          server-username: MAVEN_USERNAME
          server-password: MAVEN_PASSWORD
          gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
          gpg-passphrase: MAVEN_GPG_PASSPHRASE
      - name: Deploy
        # Note: this will re-run the build even though we already have the JAR file ready to deploy
        run: |
          # Copy the native artifacts built for each platform into the appropriate place:
          unzip ../combined-jar/*.jar -d ../
          mkdir -p target/classes/
          cp -r ../combined-jar/linux* target/classes/
          cp -r ../combined-jar/mac* target/classes/
          cp -r ../combined-jar/win* target/classes/
          mvn --batch-mode deploy -Dmaven.test.skip=true
        env:
          MAVEN_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
          MAVEN_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
          MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }}