Skip to content

Commit

Permalink
CI: revamp installer to support ARM
Browse files Browse the repository at this point in the history
  • Loading branch information
@mratsim
mratsim committed Jan 25, 2025
1 parent 4a1b5bc commit 1e170dc
Show file tree
Hide file tree
Showing 2 changed files with 437 additions and 81 deletions.
165 changes: 84 additions & 81 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ permissions:
contents: read # list commits, change to write to create a release

jobs:
build:
test:
strategy:
fail-fast: false
max-parallel: 20
Expand All @@ -17,65 +17,74 @@ jobs:
# version-1-6 and version-2-0 have issues with templates in typedef operating on a generic for Fp12 tower in https://github.com/mratsim/constantine/pull/485
# Hence we only test and officially sypport 2.2.0, though 99% of Constantine should work on older compilers

# For ARM support on Linux we need to set a specific builder
# For ARM support on Linux we need to set a specific host
# either ubuntu-22.04-arm
# or ubuntu-24.04-arm
# We might as well test on 22.04 for ARM

nim_version: [version-2-2]
rust_toolchain: [stable] # [beta, nightly]
go_toolchain: [stable]

target:
# We only need to test 32-bit for targets like RISC-V that don't have ASM.
- os: linux
cpu: i386
BACKEND: NO_ASM

# We only need to test 32-bit for targets like RISC-V that don't have ASM.
# - os: linux
# cpu: i386
# BACKEND: ASM
ctt_backend: NO_ASM
host: ubuntu-latest

- os: linux
cpu: amd64
BACKEND: NO_ASM
ctt_backend: NO_ASM
host: ubuntu-latest
- os: linux
cpu: amd64
BACKEND: ASM
- os: linux
cpu: arm64
builder: ubuntu-22.04-arm
BACKEND: NO_ASM
ctt_backend: ASM
host: ubuntu-latest

# ASM code for ARM Linux is not available yet
- os: linux
cpu: arm64
builder: ubuntu-22.04-arm
BACKEND: ASM
ctt_backend: NO_ASM
host: ubuntu-22.04-arm

- os: windows
cpu: amd64
BACKEND: NO_ASM
ctt_backend: NO_ASM
host: windows-latest
- os: windows
cpu: amd64
BACKEND: ASM

# MacOS Github agents are now using ARM and we need Nim ARM nightlies: https://github.com/mratsim/constantine/issues/372
# - os: macos
# cpu: amd64
# BACKEND: NO_ASM
# - os: macos
# cpu: amd64
# BACKEND: ASM
ctt_backend: ASM
host: windows-latest

- os: macos
cpu: arm64
ctt_backend: NO_ASM
host: macos-latest
- os: macos
cpu: arm64
ctt_backend: ASM
host: macos-latest

include:
- target:
os: linux
builder: ubuntu-latest
# - target:
# os: macos
# builder: macos-latest
nim_channel: nightly
- target:
os: linux
nim_channel: nightly
- target:
os: macos
# MacOS Github agents are now using ARM and there are no ARM nightlies:
# https://github.com/mratsim/constantine/issues/372
# So we build from source
nim_channel: source
- target:
os: windows
builder: windows-latest
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.BACKEND }} (${{ matrix.nim_version }})'
runs-on: ${{ matrix.builder }}
nim_channel: nightly

name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.ctt_backend }} (${{ matrix.nim_version }})'
runs-on: ${{ matrix.target.host }}
steps:
- name: Get branch name
shell: bash
Expand Down Expand Up @@ -149,27 +158,43 @@ jobs:
echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
- name: Restore Nim from cache
if: matrix.nim_version != 'devel'
- name: Get latest Nim commit hash & install dir
id: versions
shell: bash
run: |
getHash() {
git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
}
nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}')
csourcesHash=$(getHash nim-lang/csources)
echo "::set-output name=nim::$nimHash"
echo "::set-output name=csources::$csourcesHash"
echo "::set-output name=installdir::nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.nim_version }}-$nimHash"
# Nightlies are already on Github
- name: Restore Nim from cache (except nightlies)
if: matrix.nim_channel != 'nightly'
id: nim-compiler-cache
uses: actions/cache@v4
with:
path: '${{ github.workspace }}/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}'
key: 'nim-${{ matrix.target.cpu }}-${{ matrix.nim_version }}'
path: '${{ github.workspace }}/${{ steps.versions.outputs.installdir }}'
key: '${{ steps.versions.outputs.installdir }}'

- name: Setup Nim
if: steps.nim-compiler-cache.outputs.cache-hit != 'true'
uses: alaviss/[email protected]
with:
path: 'nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}'
version: ${{ matrix.nim_version }}
architecture: ${{ matrix.target.cpu }}
add-to-path: false

- name: Path to cached Nim
shell: bash
run: |
echo '${{ github.workspace }}'"/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
bash constantine/.github/workflows/install_nim.sh \
--nim-channel ${{ matrix.nim_channel }} \
--nim-version ${{ matrix.nim_version }} \
--os ${{ matrix.target.os }} \
--arch ${{ matrix.target.cpu }} \
--install-dir ${{ steps.versions.outputs.installdir }}
- name: Path to Nim
shell: bash
run: |
echo '${{ github.workspace }}'/"${{ steps.versions.outputs.installdir }}"/bin >> $GITHUB_PATH
echo '${{ github.workspace }}'"/.nimble/bin" >> $GITHUB_PATH
- name: Install test dependencies (Linux amd64)
Expand Down Expand Up @@ -211,29 +236,6 @@ jobs:
if: runner.os == 'macOS'
run: |
brew install gmp
- name: Install Clang with Intel Assembly support (macOS)
if: runner.os == 'macOS'
# Apple Clang does not support Intel syntax due to missing
# commit: https://github.com/llvm/llvm-project/commit/ae98182cf7341181e4aa815c372a072dec82779f
# Revision: https://reviews.llvm.org/D113707
# Apple upstream: FB12137688

# Furthermore, Apple Clang can delete symbols when building a static library
# in particular the hasAdxImpl bool for CPU runtime detection.

# run: |
# mkdir -p external/bin
# cat << EOF > external/bin/clang
# #!/bin/bash
# exec $(brew --prefix llvm@15)/bin/clang "\$@"
# EOF
# cat << EOF > external/bin/clang++
# #!/bin/bash
# exec $(brew --prefix llvm@15)/bin/clang++ "\$@"
# EOF
# chmod 755 external/bin/{clang,clang++}
# echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
run: echo "$(brew --prefix llvm@15)/bin" >> $GITHUB_PATH
- name: Setup MSYS2 (Windows)
if: runner.os == 'Windows'
Expand All @@ -257,6 +259,7 @@ jobs:
# gcc is an alias to Apple Clang on MacOS
run: |
nim -v
nimble -v
gcc -v
clang -v
go version
Expand Down Expand Up @@ -295,15 +298,15 @@ jobs:
go mod download -modfile=go_test.mod
- name: Run Constantine as C library tests (UNIX with Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'
if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM'
shell: bash
run: |
cd constantine
nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
- name: Run Constantine as C library tests (UNIX no Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM'
if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM'
shell: bash
run: |
cd constantine
Expand All @@ -313,7 +316,7 @@ jobs:
- name: Run Constantine as C library tests (Windows with Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM'
if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM'
shell: msys2 {0}
run: |
cd constantine
Expand All @@ -323,7 +326,7 @@ jobs:
- name: Run Constantine as C library tests (Windows no Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM'
if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM'
shell: msys2 {0}
run: |
cd constantine
Expand All @@ -340,14 +343,14 @@ jobs:
go test -modfile=../go_test.mod
- name: Run Constantine as Rust library tests (with Assembly)
if: matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386'
if: matrix.target.ctt_backend == 'ASM' && matrix.target.cpu != 'i386'
shell: bash
# We need to deactivate the assembly (used by default for benches)
run: |
cd constantine
cargo test -- --nocapture
- name: Run Constantine as Rust library tests (NO Assembly)
if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386'
if: matrix.target.ctt_backend == 'NO_ASM' && matrix.target.cpu != 'i386'
shell: bash
# We need to deactivate the assembly (used by default for benches)
run: |
Expand All @@ -357,7 +360,7 @@ jobs:
- name: Compile Constantine Zkalc benchmark (no assembly)
# Skip 32-bit as that would need clang-multilib or -m32
# Skip Windows as clang throws fatal error LNK1107
if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows'
if: matrix.target.ctt_backend == 'NO_ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows'
shell: bash
run: |
cd constantine
Expand All @@ -366,20 +369,20 @@ jobs:
- name: Compile Constantine Zkalc benchmark (with assembly)
# Skip 32-bit as that would need clang-multilib or -m32
# Skip Windows as clang throws fatal error LNK1107
if: matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows'
if: matrix.target.ctt_backend == 'ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows'
shell: bash
run: |
cd constantine
nimble make_zkalc
- name: Run Constantine in-depth tests (Unix - with GMP, with Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'
if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM'
shell: bash
run: |
cd constantine
nimble test_parallel --verbose
- name: Run Constantine in-depth tests (Unix - with GMP, no Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM'
if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM'
shell: bash
run: |
cd constantine
Expand All @@ -390,7 +393,7 @@ jobs:
# but nim-gmp cannot find GMP on Windows CI
# Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118
# And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason)
if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM'
if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM'
shell: msys2 {0}
run: |
cd constantine
Expand All @@ -403,7 +406,7 @@ jobs:
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
# Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118
if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM'
if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM'
shell: msys2 {0}
run: |
cd constantine
Expand Down
Loading

0 comments on commit 1e170dc

Please sign in to comment.