Built an int64 container

.github/workflows/build.yml

@@ -26,7 +26,7 @@ jobs:
     # Run on our self-hosted machines
     runs-on: self-hosted
     container:
-      image: firedrakeproject/firedrake-env:latest
+      image: firedrakeproject/firedrake-test:latest
     strategy:
       # Don't immediately kill real if complex fails and vice versa.
       fail-fast: false
@@ -61,6 +61,7 @@ jobs:
           # shellcheck disable=SC2086
           ./firedrake/scripts/firedrake-install \
             $COMPLEX \
+            --petsc-int-type=int64 \
             --honour-petsc-dir \
             --mpicc="$MPICH_DIR"/mpicc \
             --mpicxx="$MPICH_DIR"/mpicxx \
@@ -150,3 +151,12 @@ jobs:
       tag: ${{ needs.docker_tag.outputs.tag }}
       status: ${{ needs.build.result }}
     secrets: inherit
+  #~ docker_testing:
+    #~ # Strictly for testing!
+    #~ uses: ./.github/workflows/docker_reuse.yml
+    #~ with:
+      #~ target: firedrake-test
+      #~ tag: latest
+      #~ dockerfile: docker/Dockerfile.testing
+    #~ secrets: inherit
+

docker/Dockerfile.testing

@@ -0,0 +1,168 @@
+# DockerFile for an environment into which firedrake can be installed.
+
+FROM ubuntu:24.04
+
+# Update and install required packages for Firedrake
+USER root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update \
+    && apt-get -y dist-upgrade \
+    && DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata \
+    && apt-get -y install curl vim docker.io \
+                 openssh-client build-essential autoconf automake \
+                 cmake gfortran git libopenblas-serial-dev \
+                 libtool python3-dev python3-pip python3-tk python3-venv \
+                 python3-requests zlib1g-dev libboost-dev sudo gmsh \
+                 bison flex ninja-build \
+                 libocct-ocaf-dev libocct-data-exchange-dev \
+                 swig graphviz \
+                 libcurl4-openssl-dev libxml2-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Use a more sane locale
+ENV LC_ALL C.UTF-8
+
+# Change the `ubuntu` user to `firedrake`
+# and ensure that we do not run as root on self-hosted systems
+RUN usermod -d /home/firedrake -m ubuntu && \
+    usermod -l firedrake ubuntu && \
+    groupmod -n firedrake ubuntu && \
+    usermod -aG sudo firedrake && \
+    echo "firedrake:docker" | chpasswd && \
+    echo "firedrake ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
+    ldconfig
+
+USER firedrake
+WORKDIR /home/firedrake
+
+# Fetch PETSc, SLEPc and eigen
+RUN git clone https://github.com/firedrakeproject/petsc.git
+RUN git clone https://github.com/firedrakeproject/slepc.git
+
+# Build MPICH manually because we don't want PETSc to build it twice
+RUN bash -c 'cd petsc; \
+    ./configure \
+        --with-64-bit-indices=1 \
+        --COPTFLAGS=-O3 -march=native -mtune=native \
+        --CXXOPTFLAGS=-O3 -march=native -mtune=native \
+        --FOPTFLAGS=-O3 -march=native -mtune=native \
+        --with-c2html=0 \
+        --with-debugging=0 \
+        --with-fortran-bindings=0 \
+        --with-make-np=12 \
+        --with-shared-libraries=1 \
+        --with-zlib \
+        --download-fftw \
+        --download-hdf5 \
+        --download-hwloc \
+        --download-hypre \
+        --download-metis \
+        --download-mumps \
+        --download-mpich \
+        --download-mpich-device=ch3:sock \
+        --download-netcdf \
+        --download-pastix \
+        --download-pnetcdf \
+        --download-ptscotch \
+        --download-scalapack \
+        --download-suitesparse \
+        --download-superlu_dist \
+        PETSC_ARCH=packages; \
+        mv packages/include/petscconf.h packages/include/old_petscconf.nope;'
+# Don't run make here, we only want MPICH and HWLOC
+# It is also necessary to move `petscconf.h` so packages isn't treated like a working PETSc
+
+# Build default Firedrake PETSc
+RUN bash -c 'export PACKAGES=/home/firedrake/petsc/packages; \
+    cd petsc; \
+    ./configure \
+        --with-64-bit-indices=1 \
+        --COPTFLAGS=-O3 -march=native -mtune=native \
+        --CXXOPTFLAGS=-O3 -march=native -mtune=native \
+        --FOPTFLAGS=-O3 -march=native -mtune=native \
+        --with-c2html=0 \
+        --with-debugging=0 \
+        --with-fortran-bindings=0 \
+        --with-make-np=12 \
+        --with-shared-libraries=1 \
+        --with-bison \
+        --with-flex \
+        --with-zlib \
+        --with-fftw-dir=$PACKAGES \
+        --with-hdf5-dir=$PACKAGES \
+        --with-hwloc-dir=$PACKAGES \
+        --with-hypre-dir=$PACKAGES \
+        --with-metis-dir=$PACKAGES \
+        --with-mpi-dir=$PACKAGES \
+        --with-mumps-dir=$PACKAGES \
+        --with-netcdf-dir=$PACKAGES \
+        --with-pastix-dir=$PACKAGES \
+        --with-pnetcdf-dir=$PACKAGES \
+        --with-ptscotch-dir=$PACKAGES \
+        --with-scalapack-dir=$PACKAGES \
+        --with-suitesparse-dir=$PACKAGES \
+        --with-superlu_dist-dir=$PACKAGES \
+        PETSC_ARCH=default; \
+    make PETSC_DIR=/home/firedrake/petsc PETSC_ARCH=default all;'
+
+# Build default Firedrake SLEPc
+RUN bash -c 'export PETSC_DIR=/home/firedrake/petsc; \
+    export PETSC_ARCH=default; \
+    cd slepc; \
+    ./configure; \
+    make SLEPC_DIR=/home/firedrake/slepc PETSC_DIR=/home/firedrake/petsc PETSC_ARCH=default;'
+
+# Additionally build complex PETSc for Firedrake
+RUN bash -c 'export PACKAGES=/home/firedrake/petsc/packages; \
+    cd petsc; \
+    ./configure \
+        --with-64-bit-indices=1 \
+        --COPTFLAGS=-O3 -march=native -mtune=native \
+        --CXXOPTFLAGS=-O3 -march=native -mtune=native \
+        --FOPTFLAGS=-O3 -march=native -mtune=native \
+        --with-c2html=0 \
+        --with-debugging=0 \
+        --with-fortran-bindings=0 \
+        --with-make-np=12 \
+        --with-scalar-type=complex \
+        --with-shared-libraries=1 \
+        --with-bison \
+        --with-flex \
+        --with-zlib \
+        --with-fftw-dir=$PACKAGES \
+        --with-hdf5-dir=$PACKAGES \
+        --with-hwloc-dir=$PACKAGES \
+        --with-metis-dir=$PACKAGES \
+        --with-mpi-dir=$PACKAGES \
+        --with-mumps-dir=$PACKAGES \
+        --with-netcdf-dir=$PACKAGES \
+        --with-pastix-dir=$PACKAGES \
+        --with-pnetcdf-dir=$PACKAGES \
+        --with-ptscotch-dir=$PACKAGES \
+        --with-scalapack-dir=$PACKAGES \
+        --with-suitesparse-dir=$PACKAGES \
+        --with-superlu_dist-dir=$PACKAGES \
+        PETSC_ARCH=complex; \
+    make PETSC_DIR=/home/firedrake/petsc PETSC_ARCH=complex all;'
+
+# Build complex Firedrake SLEPc
+RUN bash -c 'export PETSC_DIR=/home/firedrake/petsc; \
+    export PETSC_ARCH=complex; \
+    cd slepc; \
+    ./configure; \
+    make SLEPC_DIR=/home/firedrake/slepc PETSC_DIR=/home/firedrake/petsc PETSC_ARCH=complex;'
+
+# Clean up unnecessary files
+RUN rm -rf /home/firedrake/petsc/**/externalpackages \
+    && rm -rf /home/firedrake/petsc/src/docs \
+    && rm -f /home/firedrake/petsc/src/**/tutorials/output/* \
+    && rm -f /home/firedrake/petsc/src/**/tests/output/*
+
+# Set some useful environment variables
+ENV PETSC_DIR /home/firedrake/petsc
+ENV SLEPC_DIR /home/firedrake/slepc
+ENV MPICH_DIR /home/firedrake/petsc/packages/bin
+ENV HDF5_DIR /home/firedrake/petsc/packages
+ENV HDF5_MPI ON
+ENV OMP_NUM_THREADS 1
+ENV OPENBLAS_NUM_THREADS 1

tests/regression/test_star_pc.py

@@ -128,7 +128,9 @@ def test_star_equivalence(problem_type, backend):
         a = inner(grad(z), grad(v))*dx + inner(p, q)*dx
 
         bcs = DirichletBC(V.sub(0), Constant((1., 0.)), "on_boundary")
-        nsp = MixedVectorSpaceBasis(V, [V.sub(0), VectorSpaceBasis(constant=True)])
+        nsp = MixedVectorSpaceBasis(
+            V, [V.sub(0), VectorSpaceBasis(constant=True, comm=COMM_WORLD)]
+        )
 
         star_params = {"mat_type": "aij",
                        "snes_type": "ksponly",
@@ -299,7 +301,9 @@ def test_vanka_equivalence(problem_type):
         a = inner(grad(z), grad(v))*dx - inner(p, div(v))*dx - inner(div(z), q)*dx
 
         bcs = DirichletBC(V.sub(0), Constant((1., 0.)), "on_boundary")
-        nsp = MixedVectorSpaceBasis(V, [V.sub(0), VectorSpaceBasis(constant=True)])
+        nsp = MixedVectorSpaceBasis(
+            V, [V.sub(0), VectorSpaceBasis(constant=True, comm=COMM_WORLD)]
+        )
 
         vanka_params = {"mat_type": "aij",
                         "snes_type": "ksponly",

tinyasm/matinvert.cpp

@@ -1,10 +1,11 @@
 #include <petsc.h>
 #include <petscblaslapack.h>
 
-
-PetscErrorCode mymatinvert(PetscInt* n, PetscScalar* mat, PetscInt* piv, PetscInt* info, PetscScalar* work) {
-    PetscCallBLAS("LAPACKgetrf",LAPACKgetrf_(n,n,mat,n,piv,info));
-    PetscCallBLAS("LAPACKgetri",LAPACKgetri_(n,mat, n, piv,work,n,info));
-    return 0;
+PetscErrorCode mymatinvert(PetscBLASInt* n, PetscScalar* mat, PetscBLASInt* piv, PetscBLASInt* info, PetscScalar* work) {
+    PetscCallBLAS("LAPACKgetrf", LAPACKgetrf_(n, n, mat, n, piv, info));
+    PetscCheck(!(*info), PETSC_COMM_SELF, PETSC_ERR_LIB, "TinyASM error calling ?getrf in mymatinvert");
+    PetscCallBLAS("LAPACKgetri", LAPACKgetri_(n, mat, n, piv, work, n, info));
+    PetscCheck(!(*info), PETSC_COMM_SELF, PETSC_ERR_LIB, "TinyASM error calling ?getri in mymatinvert");
+    return PETSC_SUCCESS;
 }
 

tinyasm/tinyasm.cpp

@@ -26,7 +26,7 @@ namespace py = pybind11;
 
 PetscLogEvent PC_tinyasm_SetASMLocalSubdomains, PC_tinyasm_apply, PC_tinyasm_setup;
 
-PetscErrorCode mymatinvert(PetscInt* n, PetscScalar* mat, PetscInt* piv, PetscInt* info, PetscScalar* work);
+PetscErrorCode mymatinvert(PetscBLASInt* n, PetscScalar* mat, PetscBLASInt* piv, PetscBLASInt* info, PetscScalar* work);
 
 class BlockJacobi {
     public:
@@ -41,15 +41,15 @@ class BlockJacobi {
         Mat *localmats;
 
         vector<IS> dofis;
-        vector<PetscInt> piv;
+        vector<PetscBLASInt> piv;
         vector<PetscScalar> fwork;
 
         BlockJacobi(vector<vector<PetscInt>> _dofsPerBlock, vector<vector<PetscInt>> _globalDofsPerBlock, int localSize, PetscSF _sf)
             : dofsPerBlock(_dofsPerBlock), globalDofsPerBlock(_globalDofsPerBlock), sf(_sf) {
 
                 int numBlocks = dofsPerBlock.size();
                 PetscInt dof;
-                int biggestBlock = 0;
+                PetscInt biggestBlock = 0;
                 for(int p=0; p<numBlocks; p++) {
                     dof = dofsPerBlock[p].size();
                     biggestBlock = max(biggestBlock, dof);
@@ -58,7 +58,7 @@ class BlockJacobi {
                 workb = vector<PetscScalar>(biggestBlock, 0);
                 localb = vector<PetscScalar>(localSize, 0);
                 localx = vector<PetscScalar>(localSize, 0);
-                piv = vector<PetscInt>(biggestBlock, 0.);
+                piv = vector<PetscBLASInt>(biggestBlock, 0.);
                 iota(piv.begin(), piv.end(), 1);
                 fwork = vector<PetscScalar>(biggestBlock, 0.);
                 localmats_aij = NULL;
@@ -91,14 +91,14 @@ class BlockJacobi {
         }
 
         PetscInt updateValuesPerBlock(Mat P) {
-            PetscInt ierr, dof;
+            PetscInt ierr;
+            PetscBLASInt dof, info;
             int numBlocks = dofsPerBlock.size();
             ierr = MatCreateSubMatrices(P, numBlocks, dofis.data(), dofis.data(), localmats_aij ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX, &localmats_aij);CHKERRQ(ierr);
-            PetscInt info;
             PetscScalar *vv;
             for(int p=0; p<numBlocks; p++) {
                 ierr = MatConvert(localmats_aij[p], MATDENSE, localmats[p] ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX,&localmats[p]);CHKERRQ(ierr);
-                PetscInt dof = dofsPerBlock[p].size();
+                PetscCall(PetscBLASIntCast(dofsPerBlock[p].size(), &dof));
                 ierr = MatDenseGetArrayWrite(localmats[p],&vv);CHKERRQ(ierr);
                 if (dof) mymatinvert(&dof, vv, piv.data(), &info, fwork.data());
                 ierr = MatDenseRestoreArrayWrite(localmats[p],&vv);CHKERRQ(ierr);
@@ -108,9 +108,9 @@ class BlockJacobi {
 
 
         PetscInt solve(const PetscScalar* __restrict b, PetscScalar* __restrict x) {
-            PetscInt dof, ierr;
+            PetscInt ierr;
             PetscScalar dOne = 1.0;
-            PetscInt one = 1;
+            PetscBLASInt dof, one = 1;
             PetscScalar dZero = 0.0;
 
             const PetscScalar *matvalues;
@@ -324,9 +324,9 @@ PetscErrorCode PCView_TinyASM(PC pc, PetscViewer viewer) {
         PetscInt biggestblock = *std::max_element(blocksizes.begin(), blocksizes.end());
         PetscScalar avgblock = std::accumulate(blocksizes.begin(), blocksizes.end(), 0.)/nblocks;
         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
-        ierr = PetscViewerASCIIPrintf(viewer, "TinyASM (block Jacobi) preconditioner with %d blocks\n", nblocks);CHKERRQ(ierr);
+        ierr = PetscViewerASCIIPrintf(viewer, "TinyASM (block Jacobi) preconditioner with %" PetscInt_FMT " blocks\n", nblocks);CHKERRQ(ierr);
         ierr = PetscViewerASCIIPrintf(viewer, "Average block size %f \n", avgblock);CHKERRQ(ierr);
-        ierr = PetscViewerASCIIPrintf(viewer, "Largest block size %d \n", biggestblock);CHKERRQ(ierr);
+        ierr = PetscViewerASCIIPrintf(viewer, "Largest block size %" PetscInt_FMT " \n", biggestblock);CHKERRQ(ierr);
         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
     }
     return 0;
@@ -428,6 +428,6 @@ PYBIND11_MODULE(_tinyasm, m) {
               auto blockjacobi = new BlockJacobi(dofsPerBlock, globalDofsPerBlock, localsize, newsf);
               pc->data = (void*)blockjacobi;
               ierr = PetscLogEventEnd(PC_tinyasm_SetASMLocalSubdomains, pc, 0, 0, 0);CHKERRQ(ierr);
-              return ierr;
+              return (int) ierr;
           });
 }