Skip to content

Commit

Permalink
Flattening directory structure and adding getpackages4cache.sh script
Browse files Browse the repository at this point in the history
  • Loading branch information
@bobrobey
bobrobey committed Oct 17, 2024
1 parent 56f2168 commit fd96ac2
Show file tree
Hide file tree
Showing 43 changed files with 395 additions and 237 deletions.
38 changes: 19 additions & 19 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,61 +384,61 @@ The `main_install.sh` script calls a series of other scripts to install the soft

```bash
// install linux software such as cmake and system compilers
rocm/sources/scripts/baseospackages_setup.sh
rocm/scripts/baseospackages_setup.sh

// install lmod and create the modulepath
rocm/sources/scripts/lmod_setup.sh
rocm/scripts/lmod_setup.sh

// install ROCm and create ROCm module
rocm/sources/scripts/rocm_setup.sh
rocm/scripts/rocm_setup.sh

// install OpenMPI and create OpenMPI module
comm/sources/scripts/openmpi_setup.sh
comm/scripts/openmpi_setup.sh

// install MVAPICH and create MVAPICH module
comm/sources/scripts/mvapich_setup.sh
comm/scripts/mvapich_setup.sh

// install MPI4PY and create MPI4PY module
comm/sources/scripts/mpi4py_setup.sh
comm/scripts/mpi4py_setup.sh

// install Miniconda3 and create Miniconda3 module
tools/sources/scripts/miniconda3_setup.sh
tools/scripts/miniconda3_setup.sh

// install AMD Research Omnitrace and create module
tools/sources/scripts/omnitrace_setup.sh
tools/scripts/omnitrace_setup.sh

// install Grafana (needed for Omniperf)
tools/sources/scripts/grafana_setup.sh
tools/scripts/grafana_setup.sh

// install AMD Research Omniperf and create module
tools/sources/scripts/omniperf_setup.sh
tools/scripts/omniperf_setup.sh

// install HPCToolkit and create HPCToolkit module
tools/sources/scripts/hpctoolkit_setup.sh
tools/scripts/hpctoolkit_setup.sh

// install TAU and create TAU module
tools/sources/scripts/tau_setup.sh
tools/scripts/tau_setup.sh

// install clang/14 clang/15 gcc/11 gcc/12 gcc/13 and create modules
extras/sources/scripts/compiler_setup.sh
extras/scripts/compiler_setup.sh

// install liblapack and libopenblas
extras/sources/scripts/apps_setup_basic.sh
extras/scripts/apps_setup_basic.sh

// install CuPy and create CuPy module
extras/sources/scripts/cupy_setup.sh
extras/scripts/cupy_setup.sh

// install JAX and create JAX module
extras/sources/scripts/jax_setup.sh
extras/scripts/jax_setup.sh

// install PyTorch and create PyTorch module
extras/sources/scripts/pytorch_setup.sh
extras/scripts/pytorch_setup.sh

// install additional libs and apps such as valgrind, boost, parmetis, openssl, etc.
extras/sources/scripts/apps_setup.sh
extras/scripts/apps_setup.sh

// install Kokkos
extras/sources/scripts/kokkos_setup.sh
extras/scripts/kokkos_setup.sh

```

Expand Down
2 changes: 1 addition & 1 deletion bare_system/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ WORKDIR /tmp
COPY bare_system/bootstrap_os.sh /tmp/bootstrap_os.sh
RUN /tmp/bootstrap_os.sh && rm /tmp/bootstrap_os.sh

COPY rocm/sources/scripts/lmod_setup.sh /tmp/lmod_setup.sh
COPY rocm/scripts/lmod_setup.sh /tmp/lmod_setup.sh

RUN /tmp/lmod_setup.sh && \
rm /tmp/lmod_setup.sh
Expand Down
48 changes: 24 additions & 24 deletions bare_system/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,139 +51,139 @@ all_packages: all_packages.timestamp

# From base rocm layer
baseospackages.timestamp:
rocm/sources/scripts/baseospackages_setup.sh
rocm/scripts/baseospackages_setup.sh
touch baseospackages.timestamp

rocm.timestamp: baseospackages.timestamp # lmod.timestamp
rocm/sources/scripts/rocm_setup.sh \
rocm/scripts/rocm_setup.sh \
--rocm-version ${ROCM_VERSION}
touch rocm.timestamp

rocm_omnitrace.timestamp: baseospackages.timestamp rocm.timestamp
rocm/sources/scripts/rocm_omnitrace_setup.sh \
rocm/scripts/rocm_omnitrace_setup.sh \
--rocm-version ${ROCM_VERSION}
touch rocm_omnitrace.timestamp

rocm_omniperf.timestamp: baseospackages.timestamp rocm.timestamp
rocm/sources/scripts/rocm_omniperf_setup.sh \
rocm/scripts/rocm_omniperf_setup.sh \
--rocm-version ${ROCM_VERSION}
touch rocm_omniperf.timestamp

slurm.timestamp:
rocm/sources/scripts/slurm_setup.sh
rocm/scripts/slurm_setup.sh
touch slurm.timestamp

# From Comm layer
openmpi.timestamp: rocm.timestamp
comm/sources/scripts/openmpi_setup.sh \
comm/scripts/openmpi_setup.sh \
--rocm-version ${ROCM_VERSION} \
--amdgpu-gfxmodel ${AMDGPU_GFXMODEL}
touch openmpi.timestamp

mvapich.timestamp: rocm.timestamp
comm/sources/scripts/mvapich_setup.sh \
comm/scripts/mvapich_setup.sh \
--rocm-version ${ROCM_VERSION}
touch mvapich.timestamp

mpi4py.timestamp: rocm.timestamp
comm/sources/scripts/mpi4py_setup.sh \
comm/scripts/mpi4py_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-mpi4py ${BUILD_MPI4PY}
touch mpi4py.timestamp

# From Tools layer

omnitrace.timestamp: openmpi.timestamp
tools/sources/scripts/omnitrace_setup.sh \
tools/scripts/omnitrace_setup.sh \
--rocm-version ${ROCM_VERSION} \
--amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \
--omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE}
touch omnitrace.timestamp

grafana.timestamp:
tools/sources/scripts/grafana_setup.sh
tools/scripts/grafana_setup.sh

omniperf.timestamp: rocm.timestamp
tools/sources/scripts/omniperf_setup.sh \
tools/scripts/omniperf_setup.sh \
--rocm-version ${ROCM_VERSION}
touch omniperf.timestamp

tau.timestamp: rocm.timestamp
tools/sources/scripts/tau_setup.sh \
tools/scripts/tau_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-tau ${BUILD_TAU}
touch tau.timestamp

scorep.timestamp: rocm.timestamp
tools/sources/scripts/scorep_setup.sh \
tools/scripts/scorep_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-scorep ${BUILD_SCOREP}
touch scorep.timestamp

hpctoolkit.timestamp: rocm.timestamp
tools/sources/scripts/hpctoolkit_setup.sh \
tools/scripts/hpctoolkit_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-hpctoolkit ${BUILD_HPCTOOLKIT}
touch hpctoolkit.timestamp

# From Training layer

compiler_setup.sh:
extras/sources/scripts/compiler_setup.sh
extras/scripts/compiler_setup.sh
touch compiler_setup.timestamp

amd_gcc.timestamp: rocm.timestamp
extras/sources/scripts/amd_gcc_setup.sh \
extras/scripts/amd_gcc_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-gcc-latest ${BUILD_GCC_LATEST}
touch amd_gcc.timestamp

aomp.timestamp: rocm.timestamp
extras/sources/scripts/aomp_setup.sh \
extras/scripts/aomp_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-aomp-latest ${BUILD_AOMP_LATEST}
touch aomp.timestamp

miniconda3.timestamp:
extras/sources/scripts/miniconda3_setup.sh \
extras/scripts/miniconda3_setup.sh \
--rocm-version ${ROCM_VERSION} \
--python-versions ${PYTHON_VERSIONS}
touch miniconda3.timestamp

kokkos.timestamp:
extras/sources/scripts/kokkos_setup.sh \
extras/scripts/kokkos_setup.sh \
--rocm-version ${ROCM_VERSION} \
--build-kokkos ${BUILD_KOKKOS}
touch kokkos.timestamp

cupy.timestamp: rocm.timestamp
extras/sources/scripts/cupy_setup.sh \
extras/scripts/cupy_setup.sh \
--rocm-version ${ROCM_VERSION} \
--amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \
--build-cupy ${BUILD_CUPY}
touch cupy.timestamp

pytorch.timestamp: rocm.timestamp
extras/sources/scripts/pytorch_setup.sh \
extras/scripts/pytorch_setup.sh \
--rocm-version ${ROCM_VERSION} \
--amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \
--build-pytorch ${BUILD_PYTORCH}
touch pytorch.timestamp

jax.timestamp: rocm.timestamp
extras/sources/scripts/jax_setup.sh \
extras/scripts/jax_setup.sh \
--rocm-version ${ROCM_VERSION} \
--amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \
--build-jax ${BUILD_JAX}
touch jax.timestamp

x11vnc.timestamp:
extras/sources/scripts/x11vnc_setup.sh \
extras/scripts/x11vnc_setup.sh \
--build-x11vnc ${BUILD_X11VNC}
touch x11vnc.timestamp

apps_setup.timestamp:
extras/sources/scripts/apps_setup.sh
extras/scripts/apps_setup.sh
touch apps_basic.timestamp


Expand Down
46 changes: 23 additions & 23 deletions bare_system/main_setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,53 +99,53 @@ if [ "${USE_MAKEFILE}" == 1 ]; then
fi


rocm/sources/scripts/baseospackages_setup.sh
rocm/scripts/baseospackages_setup.sh

rocm/sources/scripts/lmod_setup.sh
rocm/scripts/lmod_setup.sh

source ~/.bashrc

rocm/sources/scripts/rocm_setup.sh --rocm-version ${ROCM_VERSION}
rocm/scripts/rocm_setup.sh --rocm-version ${ROCM_VERSION}

rocm/sources/scripts/rocm_omnitrace_setup.sh --rocm-version ${ROCM_VERSION}
rocm/scripts/rocm_omnitrace_setup.sh --rocm-version ${ROCM_VERSION}

rocm/sources/scripts/rocm_omniperf_setup.sh --rocm-version ${ROCM_VERSION}
rocm/scripts/rocm_omniperf_setup.sh --rocm-version ${ROCM_VERSION}

comm/sources/scripts/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL}
comm/scripts/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL}

comm/sources/scripts/mpi4py_setup.sh --rocm-version ${ROCM_VERSION} --build-mpi4py ${BUILD_MPI4PY}
comm/scripts/mpi4py_setup.sh --rocm-version ${ROCM_VERSION} --build-mpi4py ${BUILD_MPI4PY}

comm/sources/scripts/mvapich_setup.sh --rocm-version ${ROCM_VERSION}
comm/scripts/mvapich_setup.sh --rocm-version ${ROCM_VERSION}

tools/sources/scripts/miniconda3_setup.sh --rocm-version ${ROCM_VERSION} --python-versions ${PYTHON_VERSIONS}
tools/scripts/miniconda3_setup.sh --rocm-version ${ROCM_VERSION} --python-versions ${PYTHON_VERSIONS}

tools/sources/scripts/omnitrace_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE}
tools/scripts/omnitrace_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE}

tools/sources/scripts/grafana_setup.sh
tools/scripts/grafana_setup.sh

tools/sources/scripts/omniperf_setup.sh --rocm-version ${ROCM_VERSION}
tools/scripts/omniperf_setup.sh --rocm-version ${ROCM_VERSION}

tools/sources/scripts/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit ${BUILD_HPCTOOLKIT}
tools/scripts/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit ${BUILD_HPCTOOLKIT}

tools/sources/scripts/scorep_setup.sh --rocm-version ${ROCM_VERSION} --build-scorep ${BUILD_SCOREP}
tools/scripts/scorep_setup.sh --rocm-version ${ROCM_VERSION} --build-scorep ${BUILD_SCOREP}

tools/sources/scripts/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU}
tools/scripts/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU}

extras/sources/scripts/compiler_setup.sh
extras/scripts/compiler_setup.sh

extras/sources/scripts/apps_setup_basic.sh
extras/scripts/apps_setup_basic.sh

extras/sources/scripts/cupy_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-cupy ${BUILD_CUPY}
extras/scripts/cupy_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-cupy ${BUILD_CUPY}

extras/sources/scripts/jax_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-jax ${BUILD_JAX}
extras/scripts/jax_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-jax ${BUILD_JAX}

extras/sources/scripts/pytorch_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-pytorch ${BUILD_PYTORCH}
extras/scripts/pytorch_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-pytorch ${BUILD_PYTORCH}

extras/sources/scripts/apps_setup.sh
extras/scripts/apps_setup.sh

extras/sources/scripts/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_KOKKOS}
extras/scripts/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_KOKKOS}

extras/sources/scripts/x11vnc_setup.sh --build-x11vnc ${BUILD_X11VNC}
extras/scripts/x11vnc_setup.sh --build-x11vnc ${BUILD_X11VNC}

#If ROCm should be installed in a different location
if [ "${ROCM_INSTALLPATH}" != "/opt/" ]; then
Expand Down
Loading

0 comments on commit fd96ac2

Please sign in to comment.