diff --git a/README.md b/README.md index 2686b30..0e88381 100644 --- a/README.md +++ b/README.md @@ -384,61 +384,61 @@ The `main_install.sh` script calls a series of other scripts to install the soft ```bash // install linux software such as cmake and system compilers -rocm/sources/scripts/baseospackages_setup.sh +rocm/scripts/baseospackages_setup.sh // install lmod and create the modulepath -rocm/sources/scripts/lmod_setup.sh +rocm/scripts/lmod_setup.sh // install ROCm and create ROCm module -rocm/sources/scripts/rocm_setup.sh +rocm/scripts/rocm_setup.sh // install OpenMPI and create OpenMPI module -comm/sources/scripts/openmpi_setup.sh +comm/scripts/openmpi_setup.sh // install MVAPICH and create MVAPICH module -comm/sources/scripts/mvapich_setup.sh +comm/scripts/mvapich_setup.sh // install MPI4PY and create MPI4PY module -comm/sources/scripts/mpi4py_setup.sh +comm/scripts/mpi4py_setup.sh // install Miniconda3 and create Miniconda3 module -tools/sources/scripts/miniconda3_setup.sh +tools/scripts/miniconda3_setup.sh // install AMD Research Omnitrace and create module -tools/sources/scripts/omnitrace_setup.sh +tools/scripts/omnitrace_setup.sh // install Grafana (needed for Omniperf) -tools/sources/scripts/grafana_setup.sh +tools/scripts/grafana_setup.sh // install AMD Research Omniperf and create module -tools/sources/scripts/omniperf_setup.sh +tools/scripts/omniperf_setup.sh // install HPCToolkit and create HPCToolkit module -tools/sources/scripts/hpctoolkit_setup.sh +tools/scripts/hpctoolkit_setup.sh // install TAU and create TAU module -tools/sources/scripts/tau_setup.sh +tools/scripts/tau_setup.sh // install clang/14 clang/15 gcc/11 gcc/12 gcc/13 and create modules -extras/sources/scripts/compiler_setup.sh +extras/scripts/compiler_setup.sh // install liblapack and libopenblas -extras/sources/scripts/apps_setup_basic.sh +extras/scripts/apps_setup_basic.sh // install CuPy and create CuPy module -extras/sources/scripts/cupy_setup.sh +extras/scripts/cupy_setup.sh // install JAX and create JAX module -extras/sources/scripts/jax_setup.sh +extras/scripts/jax_setup.sh // install PyTorch and create PyTorch module -extras/sources/scripts/pytorch_setup.sh +extras/scripts/pytorch_setup.sh // install additional libs and apps such as valgrind, boost, parmetis, openssl, etc. -extras/sources/scripts/apps_setup.sh +extras/scripts/apps_setup.sh // install Kokkos -extras/sources/scripts/kokkos_setup.sh +extras/scripts/kokkos_setup.sh ``` diff --git a/bare_system/Dockerfile b/bare_system/Dockerfile index 7a623cc..53c0086 100644 --- a/bare_system/Dockerfile +++ b/bare_system/Dockerfile @@ -28,7 +28,7 @@ WORKDIR /tmp COPY bare_system/bootstrap_os.sh /tmp/bootstrap_os.sh RUN /tmp/bootstrap_os.sh && rm /tmp/bootstrap_os.sh -COPY rocm/sources/scripts/lmod_setup.sh /tmp/lmod_setup.sh +COPY rocm/scripts/lmod_setup.sh /tmp/lmod_setup.sh RUN /tmp/lmod_setup.sh && \ rm /tmp/lmod_setup.sh diff --git a/bare_system/Makefile b/bare_system/Makefile index 2506812..c924878 100644 --- a/bare_system/Makefile +++ b/bare_system/Makefile @@ -51,42 +51,42 @@ all_packages: all_packages.timestamp # From base rocm layer baseospackages.timestamp: - rocm/sources/scripts/baseospackages_setup.sh + rocm/scripts/baseospackages_setup.sh touch baseospackages.timestamp rocm.timestamp: baseospackages.timestamp # lmod.timestamp - rocm/sources/scripts/rocm_setup.sh \ + rocm/scripts/rocm_setup.sh \ --rocm-version ${ROCM_VERSION} touch rocm.timestamp rocm_omnitrace.timestamp: baseospackages.timestamp rocm.timestamp - rocm/sources/scripts/rocm_omnitrace_setup.sh \ + rocm/scripts/rocm_omnitrace_setup.sh \ --rocm-version ${ROCM_VERSION} touch rocm_omnitrace.timestamp rocm_omniperf.timestamp: baseospackages.timestamp rocm.timestamp - rocm/sources/scripts/rocm_omniperf_setup.sh \ + rocm/scripts/rocm_omniperf_setup.sh \ --rocm-version ${ROCM_VERSION} touch rocm_omniperf.timestamp slurm.timestamp: - rocm/sources/scripts/slurm_setup.sh + rocm/scripts/slurm_setup.sh touch slurm.timestamp # From Comm layer openmpi.timestamp: rocm.timestamp - comm/sources/scripts/openmpi_setup.sh \ + comm/scripts/openmpi_setup.sh \ --rocm-version ${ROCM_VERSION} \ --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} touch openmpi.timestamp mvapich.timestamp: rocm.timestamp - comm/sources/scripts/mvapich_setup.sh \ + comm/scripts/mvapich_setup.sh \ --rocm-version ${ROCM_VERSION} touch mvapich.timestamp mpi4py.timestamp: rocm.timestamp - comm/sources/scripts/mpi4py_setup.sh \ + comm/scripts/mpi4py_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-mpi4py ${BUILD_MPI4PY} touch mpi4py.timestamp @@ -94,34 +94,34 @@ mpi4py.timestamp: rocm.timestamp # From Tools layer omnitrace.timestamp: openmpi.timestamp - tools/sources/scripts/omnitrace_setup.sh \ + tools/scripts/omnitrace_setup.sh \ --rocm-version ${ROCM_VERSION} \ --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \ --omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE} touch omnitrace.timestamp grafana.timestamp: - tools/sources/scripts/grafana_setup.sh + tools/scripts/grafana_setup.sh omniperf.timestamp: rocm.timestamp - tools/sources/scripts/omniperf_setup.sh \ + tools/scripts/omniperf_setup.sh \ --rocm-version ${ROCM_VERSION} touch omniperf.timestamp tau.timestamp: rocm.timestamp - tools/sources/scripts/tau_setup.sh \ + tools/scripts/tau_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-tau ${BUILD_TAU} touch tau.timestamp scorep.timestamp: rocm.timestamp - tools/sources/scripts/scorep_setup.sh \ + tools/scripts/scorep_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-scorep ${BUILD_SCOREP} touch scorep.timestamp hpctoolkit.timestamp: rocm.timestamp - tools/sources/scripts/hpctoolkit_setup.sh \ + tools/scripts/hpctoolkit_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-hpctoolkit ${BUILD_HPCTOOLKIT} touch hpctoolkit.timestamp @@ -129,61 +129,61 @@ hpctoolkit.timestamp: rocm.timestamp # From Training layer compiler_setup.sh: - extras/sources/scripts/compiler_setup.sh + extras/scripts/compiler_setup.sh touch compiler_setup.timestamp amd_gcc.timestamp: rocm.timestamp - extras/sources/scripts/amd_gcc_setup.sh \ + extras/scripts/amd_gcc_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-gcc-latest ${BUILD_GCC_LATEST} touch amd_gcc.timestamp aomp.timestamp: rocm.timestamp - extras/sources/scripts/aomp_setup.sh \ + extras/scripts/aomp_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-aomp-latest ${BUILD_AOMP_LATEST} touch aomp.timestamp miniconda3.timestamp: - extras/sources/scripts/miniconda3_setup.sh \ + extras/scripts/miniconda3_setup.sh \ --rocm-version ${ROCM_VERSION} \ --python-versions ${PYTHON_VERSIONS} touch miniconda3.timestamp kokkos.timestamp: - extras/sources/scripts/kokkos_setup.sh \ + extras/scripts/kokkos_setup.sh \ --rocm-version ${ROCM_VERSION} \ --build-kokkos ${BUILD_KOKKOS} touch kokkos.timestamp cupy.timestamp: rocm.timestamp - extras/sources/scripts/cupy_setup.sh \ + extras/scripts/cupy_setup.sh \ --rocm-version ${ROCM_VERSION} \ --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \ --build-cupy ${BUILD_CUPY} touch cupy.timestamp pytorch.timestamp: rocm.timestamp - extras/sources/scripts/pytorch_setup.sh \ + extras/scripts/pytorch_setup.sh \ --rocm-version ${ROCM_VERSION} \ --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \ --build-pytorch ${BUILD_PYTORCH} touch pytorch.timestamp jax.timestamp: rocm.timestamp - extras/sources/scripts/jax_setup.sh \ + extras/scripts/jax_setup.sh \ --rocm-version ${ROCM_VERSION} \ --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} \ --build-jax ${BUILD_JAX} touch jax.timestamp x11vnc.timestamp: - extras/sources/scripts/x11vnc_setup.sh \ + extras/scripts/x11vnc_setup.sh \ --build-x11vnc ${BUILD_X11VNC} touch x11vnc.timestamp apps_setup.timestamp: - extras/sources/scripts/apps_setup.sh + extras/scripts/apps_setup.sh touch apps_basic.timestamp diff --git a/bare_system/main_setup.sh b/bare_system/main_setup.sh index 85c84eb..3e96efd 100755 --- a/bare_system/main_setup.sh +++ b/bare_system/main_setup.sh @@ -99,53 +99,53 @@ if [ "${USE_MAKEFILE}" == 1 ]; then fi -rocm/sources/scripts/baseospackages_setup.sh +rocm/scripts/baseospackages_setup.sh -rocm/sources/scripts/lmod_setup.sh +rocm/scripts/lmod_setup.sh source ~/.bashrc -rocm/sources/scripts/rocm_setup.sh --rocm-version ${ROCM_VERSION} +rocm/scripts/rocm_setup.sh --rocm-version ${ROCM_VERSION} -rocm/sources/scripts/rocm_omnitrace_setup.sh --rocm-version ${ROCM_VERSION} +rocm/scripts/rocm_omnitrace_setup.sh --rocm-version ${ROCM_VERSION} -rocm/sources/scripts/rocm_omniperf_setup.sh --rocm-version ${ROCM_VERSION} +rocm/scripts/rocm_omniperf_setup.sh --rocm-version ${ROCM_VERSION} -comm/sources/scripts/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} +comm/scripts/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} -comm/sources/scripts/mpi4py_setup.sh --rocm-version ${ROCM_VERSION} --build-mpi4py ${BUILD_MPI4PY} +comm/scripts/mpi4py_setup.sh --rocm-version ${ROCM_VERSION} --build-mpi4py ${BUILD_MPI4PY} -comm/sources/scripts/mvapich_setup.sh --rocm-version ${ROCM_VERSION} +comm/scripts/mvapich_setup.sh --rocm-version ${ROCM_VERSION} -tools/sources/scripts/miniconda3_setup.sh --rocm-version ${ROCM_VERSION} --python-versions ${PYTHON_VERSIONS} +tools/scripts/miniconda3_setup.sh --rocm-version ${ROCM_VERSION} --python-versions ${PYTHON_VERSIONS} -tools/sources/scripts/omnitrace_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE} +tools/scripts/omnitrace_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --omnitrace-build-from-source ${OMNITRACE_BUILD_FROM_SOURCE} -tools/sources/scripts/grafana_setup.sh +tools/scripts/grafana_setup.sh -tools/sources/scripts/omniperf_setup.sh --rocm-version ${ROCM_VERSION} +tools/scripts/omniperf_setup.sh --rocm-version ${ROCM_VERSION} -tools/sources/scripts/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit ${BUILD_HPCTOOLKIT} +tools/scripts/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit ${BUILD_HPCTOOLKIT} -tools/sources/scripts/scorep_setup.sh --rocm-version ${ROCM_VERSION} --build-scorep ${BUILD_SCOREP} +tools/scripts/scorep_setup.sh --rocm-version ${ROCM_VERSION} --build-scorep ${BUILD_SCOREP} -tools/sources/scripts/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU} +tools/scripts/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU} -extras/sources/scripts/compiler_setup.sh +extras/scripts/compiler_setup.sh -extras/sources/scripts/apps_setup_basic.sh +extras/scripts/apps_setup_basic.sh -extras/sources/scripts/cupy_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-cupy ${BUILD_CUPY} +extras/scripts/cupy_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-cupy ${BUILD_CUPY} -extras/sources/scripts/jax_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-jax ${BUILD_JAX} +extras/scripts/jax_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-jax ${BUILD_JAX} -extras/sources/scripts/pytorch_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-pytorch ${BUILD_PYTORCH} +extras/scripts/pytorch_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} --build-pytorch ${BUILD_PYTORCH} -extras/sources/scripts/apps_setup.sh +extras/scripts/apps_setup.sh -extras/sources/scripts/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_KOKKOS} +extras/scripts/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_KOKKOS} -extras/sources/scripts/x11vnc_setup.sh --build-x11vnc ${BUILD_X11VNC} +extras/scripts/x11vnc_setup.sh --build-x11vnc ${BUILD_X11VNC} #If ROCm should be installed in a different location if [ "${ROCM_INSTALLPATH}" != "/opt/" ]; then diff --git a/build-docker.sh b/build-docker.sh index cc23e3d..7ae9edd 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -129,7 +129,6 @@ usage() print_default_option push -- "Push the image to Dockerhub" "do not push" } - n=0 while [[ $# -gt 0 ]] do @@ -203,6 +202,11 @@ do RETRY=${1} reset-last ;; + "--build-options") + shift + BUILD_OPTIONS=${1} + reset-last + ;; "--build-aomp-latest") BUILD_AOMP_LATEST="1" reset-last @@ -215,10 +219,6 @@ do BUILD_GCC_LATEST="1" reset-last ;; - "--build-og-latest") - BUILD_OG_LATEST="1" - reset-last - ;; "--build-clacc-latest") BUILD_CLACC_LATEST="1" reset-last @@ -300,6 +300,138 @@ do shift done +if [ "${BUILD_OPTIONS}" != "" ]; then + echo "Requesting additional \"${BUILD_OPTIONS}\" build options" + for i in ${BUILD_OPTIONS//;/ } + do + case "$i" in + # optional communication packages + "mpi4py") + echo "Setting MPI4PY build" + BUILD_MPI4PY=1 + ;; + # optional tool packages + "grafana") + echo "Setting grafana build" + BUILD_GRAFANA=1 + ;; + "hpctoolkit") + echo "Setting hpctoolkit build" + BUILD_HPCTOOLKIT=1 + ;; + "omnitrace_research") + echo "Setting omnitrace research build" + BUILD_OMNITRACE_RESEARCH=1 + ;; + "omniperf_research") + echo "Setting omniperf research build" + BUILD_OMNIPERF_RESEARCH=1 + ;; + "scorep") + echo "Setting scorep build" + BUILD_SCOREP=1 + ;; + "tau") + echo "Setting TAU build" + BUILD_TAU=1 + ;; + # optional compiler packages + "aomp_latest") + echo "Setting AOMP_LATEST build" + BUILD_AOMP_LATEST=1 + ;; + "clacc_latest") + echo "Setting CLACC_LATEST build" + BUILD_CLACC_LATEST=1 + ;; + "flang-new") + echo "Setting FLANGNEW build" + BUILD_FLANGNEW=1 + ;; + "gcc_latest") + echo "Setting GCC_LATEST build" + BUILD_GCC_LATEST=1 + ;; + "llvm_latest") + echo "Setting LLVM_LATEST build" + BUILD_LLVM_LATEST=1 + ;; + # optional AI packages + "cupy") + echo "Setting CUPY build" + BUILD_CUPY=1 + ;; + "jax") + echo "Setting JAX build" + BUILD_JAX=1 + ;; + "pytorch") + echo "Setting PYTORCH build" + BUILD_PYTORCH=1 + ;; + # optional languages/frameworks + "kokkos") + echo "Setting KOKKOS build" + BUILD_KOKKOS=1 + ;; + # optional graphics interfaces + "x11vnc") + echo "Setting X11VNC build" + BUILD_X11VNC=1 + ;; + # All latest recommended + "all-latest") + echo "Setting all latest build" + BUILD_AOMP_LATEST="1" + #BUILD_LLVM_LATEST="1" + BUILD_GCC_LATEST="1" + #BUILD_OG_LATEST="1" + #BUILD_CLACC_LATEST="1" + BUILD_PYTORCH="1" + BUILD_CUPY="1" + BUILD_JAX="1" + BUILD_KOKKOS="1" + BUILD_TAU="1" + BUILD_SCOREP="1" + BUILD_MPI4PY="1" + BUILD_HPCTOOLKIT="1" + BUILD_X11VNC="1" + BUILD_FLANGNEW="1" + BUILD_X11VNC=1 + ;; + *) + echo "Unsupported build option request \"$i\"" + echo "Valid options are:" + echo " # optional communication packages" + echo " mpi4py" + echo " # optional tool packages" + echo " grafana" + echo " hpctoolkit" + echo " omnitrace_research" + echo " omniperf_research" + echo " scorep" + echo " tau" + echo " # optional compiler packages" + echo " aomp_latest" + echo " clacc_latest" + echo " flang-new" + echo " gcc_latest" + echo " llvm_latest" + echo " # optional AI packages" + echo " cupy" + echo " jax" + echo " pytorch" + echo " # optional languages/frameworks" + echo " kokkos" + echo " # optional graphics interfaces" + echo " x11vnc" + echo " # All latest recommended" + echo " all-latest" + ;; + esac + done +fi + if [ "x${ADMIN_PASSWORD}" == "x" ] ; then echo "A password for the admin user is required" echo " --admin-password " @@ -310,16 +442,6 @@ fi AMDGPU_GFXMODEL_FIRST=`echo ${AMDGPU_GFXMODEL} | cut -f1 -d';'` AMDGPU_GFXMODEL_STRING=`echo ${AMDGPU_GFXMODEL} | sed -e 's/;/_/g'` -ROCM_DOCKER_OPTS="${PULL} -f rocm/Dockerfile ${NO_CACHE}" - -COMM_DOCKER_OPTS="-f comm/Dockerfile ${NO_CACHE} --build-arg DOCKER_USER=${DOCKER_USER}" - -TOOLS_DOCKER_OPTS="-f tools/Dockerfile ${NO_CACHE} --build-arg DOCKER_USER=${DOCKER_USER} --build-arg OMNITRACE_BUILD_FROM_SOURCE=\"${OMNITRACE_BUILD_FROM_SOURCE}\" --build-arg BUILD_HPCTOOLKIT=${BUILD_HPCTOOLKIT} --build-arg BUILD_TAU=${BUILD_TAU} --build-arg BUILD_SCOREP=${BUILD_SCOREP} " - -EXTRAS_DOCKER_OPTS="${NO_CACHE} --build-arg DOCKER_USER=${DOCKER_USER} --build-arg BUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') --build-arg OG_BUILD_DATE=$(date -u +'%y-%m-%d') --build-arg BUILD_VERSION=1.1 --build-arg DISTRO=${DISTRO} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" --build-arg ADMIN_USERNAME=${ADMIN_USERNAME} --build-arg ADMIN_PASSWORD=${ADMIN_PASSWORD}" - -EXTRAS_DOCKER_OPTS="${EXTRAS_DOCKER_OPTS} -f extras/Dockerfile" - ADD_OPTIONS="" PODMAN_DETECT=`docker |& grep "Emulate Docker CLI using podman" | wc -l` if [[ "${PODMAN_DETECT}" -ge "1" ]]; then @@ -334,33 +456,41 @@ do USE_CACHED_APPS=1 fi - GENERAL_DOCKER_OPTS="${ADD_OPTIONS} --build-arg DISTRO=${DISTRO} --build-arg DISTRO_VERSION=${DISTRO_VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION}" + GENERAL_DOCKER_OPTS="${ADD_OPTIONS} ${OUTPUT_VERBOSITY} ${NO_CACHE}" + GENERAL_DOCKER_OPTS="${GENERAL_DOCKER_OPTS} --build-arg DISTRO=${DISTRO}" + GENERAL_DOCKER_OPTS="${GENERAL_DOCKER_OPTS} --build-arg DISTRO_VERSION=${DISTRO_VERSION}" + GENERAL_DOCKER_OPTS="${GENERAL_DOCKER_OPTS} --build-arg ROCM_VERSION=${ROCM_VERSION}" + GENERAL_DOCKER_OPTS="${GENERAL_DOCKER_OPTS} --build-arg USE_CACHED_APPS=${USE_CACHED_APPS}" + GENERAL_DOCKER_OPTS="${GENERAL_DOCKER_OPTS} --build-arg DOCKER_USER=${DOCKER_USER}" # Building rocm docker - verbose-build docker build ${OUTPUT_VERBOSITY} ${GENERAL_DOCKER_OPTS} ${ROCM_DOCKER_OPTS} \ + verbose-build docker build ${GENERAL_DOCKER_OPTS} ${PULL} \ --build-arg AMDGPU_GFXMODEL=\"${AMDGPU_GFXMODEL}\" \ --build-arg AMDGPU_GFXMODEL_STRING=\"${AMDGPU_GFXMODEL_STRING}\" \ - --build-arg USE_CACHED_APPS=${USE_CACHED_APPS} \ --tag ${DOCKER_USER}/rocm:release-base-${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION} \ - . + -f rocm/Dockerfile . # Building comm docker - verbose-build docker build ${OUTPUT_VERBOSITY} ${GENERAL_DOCKER_OPTS} ${COMM_DOCKER_OPTS} \ + verbose-build docker build ${GENERAL_DOCKER_OPTS} \ --build-arg AMDGPU_GFXMODEL=\"${AMDGPU_GFXMODEL}\" \ --build-arg AMDGPU_GFXMODEL_STRING=\"${AMDGPU_GFXMODEL_STRING}\" \ - --build-arg USE_CACHED_APPS=${USE_CACHED_APPS} \ + --build-arg BUILD_MPI4PY=${BUILD_MPI4PY} \ -t ${DOCKER_USER}/comm:release-base-${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION} \ - . + -f comm/Dockerfile . # Building tools docker - verbose-build docker build ${OUTPUT_VERBOSITY} ${GENERAL_DOCKER_OPTS} ${TOOLS_DOCKER_OPTS} \ + verbose-build docker build ${GENERAL_DOCKER_OPTS} \ --build-arg AMDGPU_GFXMODEL=\"${AMDGPU_GFXMODEL}\" \ --build-arg INSTALL_GRAFANA="${INSTALL_GRAFANA}" \ + --build-arg OMNITRACE_BUILD_FROM_SOURCE=\"${OMNITRACE_BUILD_FROM_SOURCE}\" \ + --build-arg BUILD_HPCTOOLKIT=${BUILD_HPCTOOLKIT} \ + --build-arg BUILD_TAU=${BUILD_TAU} \ + --build-arg BUILD_SCOREP=${BUILD_SCOREP} \ -t ${DOCKER_USER}/tools:release-base-${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION} \ - . + -f tools/Dockerfile . # Building extrasdocker - verbose-build docker build ${OUTPUT_VERBOSITY} ${GENERAL_DOCKER_OPTS} ${EXTRAS_DOCKER_OPTS} \ + verbose-build docker build ${GENERAL_DOCKER_OPTS} \ --build-arg AMDGPU_GFXMODEL=\"${AMDGPU_GFXMODEL}\" \ --build-arg BUILD_GCC_LATEST=${BUILD_GCC_LATEST} \ --build-arg BUILD_AOMP_LATEST=${BUILD_AOMP_LATEST} \ @@ -371,16 +501,18 @@ do --build-arg BUILD_CUPY=${BUILD_CUPY} \ --build-arg BUILD_JAX=${BUILD_JAX} \ --build-arg BUILD_KOKKOS=${BUILD_KOKKOS} \ - --build-arg BUILD_TAU=${BUILD_TAU} \ - --build-arg BUILD_SCOREP=${BUILD_SCOREP} \ - --build-arg BUILD_MPI4PY=${BUILD_MPI4PY} \ - --build-arg BUILD_HPCTOOLKIT=${BUILD_HPCTOOLKIT} \ --build-arg BUILD_X11VNC=${BUILD_X11VNC} \ --build-arg BUILD_FLANGNEW=${BUILD_FLANGNEW} \ - --build-arg USE_CACHED_APPS=${USE_CACHED_APPS} \ + --build-arg BUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg OG_BUILD_DATE=$(date -u +'%y-%m-%d') \ + --build-arg BUILD_VERSION=1.1 \ + --build-arg DISTRO=${DISTRO} \ + --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" \ + --build-arg ADMIN_USERNAME=${ADMIN_USERNAME} \ + --build-arg ADMIN_PASSWORD=${ADMIN_PASSWORD} \ -t ${DOCKER_USER}/training:release-base-${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION} \ -t training \ - . + -f extras/Dockerfile . if [ "${PUSH}" -ne 0 ]; then docker push ${CONTAINER} diff --git a/comm/Dockerfile b/comm/Dockerfile index bb0a6e5..293e1f2 100644 --- a/comm/Dockerfile +++ b/comm/Dockerfile @@ -31,7 +31,7 @@ ENV TZ "US/Chicago" # install OpenMPI, UCC, and UCX # -COPY comm/sources/scripts/openmpi_setup.sh /tmp/openmpi_setup.sh +COPY comm/scripts/openmpi_setup.sh /tmp/openmpi_setup.sh RUN /tmp/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && \ rm /tmp/openmpi_setup.sh @@ -40,7 +40,7 @@ RUN /tmp/openmpi_setup.sh --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AM # Install mvapich # -COPY comm/sources/scripts/mvapich_setup.sh /tmp/mvapich_setup.sh +COPY comm/scripts/mvapich_setup.sh /tmp/mvapich_setup.sh RUN /tmp/mvapich_setup.sh --rocm-version ${ROCM_VERSION} && \ rm /tmp/mvapich_setup.sh @@ -49,7 +49,7 @@ RUN /tmp/mvapich_setup.sh --rocm-version ${ROCM_VERSION} && \ # install MPI4PY # -COPY comm/sources/scripts/mpi4py_setup.sh /tmp/mpi4py_setup.sh +COPY comm/scripts/mpi4py_setup.sh /tmp/mpi4py_setup.sh RUN /tmp/mpi4py_setup.sh --rocm-version ${ROCM_VERSION} --build-mpi4py 1 && \ rm /tmp/mpi4py_setup.sh diff --git a/comm/sources/scripts/mpi4py_setup.sh b/comm/scripts/mpi4py_setup.sh similarity index 100% rename from comm/sources/scripts/mpi4py_setup.sh rename to comm/scripts/mpi4py_setup.sh diff --git a/comm/sources/scripts/mvapich_setup.sh b/comm/scripts/mvapich_setup.sh similarity index 100% rename from comm/sources/scripts/mvapich_setup.sh rename to comm/scripts/mvapich_setup.sh diff --git a/comm/sources/scripts/openmpi_setup.sh b/comm/scripts/openmpi_setup.sh similarity index 100% rename from comm/sources/scripts/openmpi_setup.sh rename to comm/scripts/openmpi_setup.sh diff --git a/comm/sources/scripts/rocm_separate_rccl.sh b/comm/scripts/rocm_separate_rccl.sh similarity index 100% rename from comm/sources/scripts/rocm_separate_rccl.sh rename to comm/scripts/rocm_separate_rccl.sh diff --git a/extras/Dockerfile b/extras/Dockerfile index 8692a53..6fa0f15 100644 --- a/extras/Dockerfile +++ b/extras/Dockerfile @@ -69,7 +69,7 @@ EXPOSE 22 # # Install miniconda # -COPY extras/sources/scripts/miniconda3_setup.sh /tmp/miniconda3_setup.sh +COPY extras/scripts/miniconda3_setup.sh /tmp/miniconda3_setup.sh RUN /tmp/miniconda3_setup.sh \ --rocm-version ${ROCM_VERSION} \ @@ -79,7 +79,7 @@ RUN /tmp/miniconda3_setup.sh \ # # Install our desired compilers # -COPY extras/sources/scripts/compiler_setup.sh /tmp/compiler_setup.sh +COPY extras/scripts/compiler_setup.sh /tmp/compiler_setup.sh RUN /tmp/compiler_setup.sh && \ rm /tmp/compiler_setup.sh @@ -89,40 +89,36 @@ RUN /tmp/compiler_setup.sh && \ #RUN pip3 install CppHeaderParser # Build the latest GCC for AMD compiler if requested -COPY extras/sources/scripts/amd_gcc_setup.sh /tmp/amd_gcc_setup.sh +COPY extras/scripts/amd_gcc_setup.sh /tmp/amd_gcc_setup.sh RUN /tmp/amd_gcc_setup.sh --build-gcc-latest ${BUILD_GCC_LATEST} --rocm-version ${ROCM_VERSION} && \ rm /tmp/amd_gcc_setup.sh # install aomp -COPY extras/sources/scripts/aomp_setup.sh /tmp/aomp_setup.sh +COPY extras/scripts/aomp_setup.sh /tmp/aomp_setup.sh RUN /tmp/aomp_setup.sh --build-aomp-latest ${BUILD_AOMP_LATEST} --rocm-version ${ROCM_VERSION} && rm /tmp/aomp_setup.sh RUN echo "At end of aomp install" && cd /app && ls -l && cd /tmp && ls -l && cd && ls -l # install flang-new -COPY extras/sources/scripts/flang-new_setup.sh /tmp/flang-new_setup.sh +COPY extras/scripts/flang-new_setup.sh /tmp/flang-new_setup.sh RUN /tmp/flang-new_setup.sh --build-flang-new ${BUILD_FLANGNEW} --rocm-version ${ROCM_VERSION} && rm /tmp/flang-new_setup.sh # install latest llvm -COPY extras/sources/scripts/llvm-build.sh /tmp/llvm-build.sh +COPY extras/scripts/llvm-build.sh /tmp/llvm-build.sh RUN /tmp/llvm-build.sh --build-llvm-latest ${BUILD_LLVM_LATEST} --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && rm /tmp/llvm-build.sh # Grab any cache files of compiler builds # Build the clacc_clang compiler -COPY extras/sources/scripts/clacc_setup.sh /tmp/clacc_setup.sh +COPY extras/scripts/clacc_setup.sh /tmp/clacc_setup.sh RUN /tmp/clacc_setup.sh --build-clacc-latest ${BUILD_CLACC_LATEST} --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && rm /tmp/clacc_setup.sh -# Install the OpenMP GCC compiler latest drop -COPY extras/sources/scripts/og_setup.sh /tmp/og_setup.sh -RUN /tmp/og_setup.sh --build-og-latest ${BUILD_OG_LATEST} --rocm-version ${ROCM_VERSION} && rm /tmp/og_setup.sh - WORKDIR /tmp # # Install kokkos # -COPY extras/sources/scripts/kokkos_setup.sh /tmp/kokkos_setup.sh +COPY extras/scripts/kokkos_setup.sh /tmp/kokkos_setup.sh RUN /tmp/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_KOKKOS} && rm /tmp/kokkos_setup.sh @@ -130,7 +126,7 @@ RUN /tmp/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_K # Moved into scripts that need these # Install any additional apps or libs that are needed # -#COPY extras/sources/scripts/apps_setup_basic.sh /tmp/apps_setup_basic.sh +#COPY extras/scripts/apps_setup_basic.sh /tmp/apps_setup_basic.sh # #RUN chmod u+x /tmp/apps_setup_basic.sh && \ # /tmp/apps_setup_basic.sh; rm /tmp/apps_setup_basic.sh @@ -138,7 +134,7 @@ RUN /tmp/kokkos_setup.sh --rocm-version ${ROCM_VERSION} --build-kokkos ${BUILD_K # # Install cupy # -COPY extras/sources/scripts/cupy_setup.sh /tmp/cupy_setup.sh +COPY extras/scripts/cupy_setup.sh /tmp/cupy_setup.sh RUN /tmp/cupy_setup.sh --build-cupy ${BUILD_CUPY} --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && rm /tmp/cupy_setup.sh @@ -147,7 +143,7 @@ WORKDIR /app # # Install jax # -COPY extras/sources/scripts/jax_setup.sh /tmp/jax_setup.sh +COPY extras/scripts/jax_setup.sh /tmp/jax_setup.sh RUN /tmp/jax_setup.sh --build-jax ${BUILD_JAX} --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && rm /tmp/jax_setup.sh @@ -156,8 +152,8 @@ WORKDIR /app # Install pytorch # -COPY extras/sources/scripts/pytorch_setup.sh /tmp/pytorch_setup.sh -COPY extras/sources/scripts/pytorch_build_triton_wheel_py.patch /tmp/pytorch_build_triton_wheel_py.patch +COPY extras/scripts/pytorch_setup.sh /tmp/pytorch_setup.sh +COPY extras/scripts/pytorch_build_triton_wheel_py.patch /tmp/pytorch_build_triton_wheel_py.patch RUN /tmp/pytorch_setup.sh --build-pytorch ${BUILD_PYTORCH} --rocm-version ${ROCM_VERSION} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} |& tee /app/pytorch_build.out && rm /tmp/pytorch_setup.sh @@ -172,7 +168,7 @@ ADD extras/image/usr /usr ADD extras/image/home /users/default EXPOSE 5900-5920 EXPOSE 6080 -COPY extras/sources/scripts/x11vnc_setup.sh /tmp/x11vnc_setup.sh +COPY extras/scripts/x11vnc_setup.sh /tmp/x11vnc_setup.sh RUN chmod u+x /tmp/x11vnc_setup.sh && \ /tmp/x11vnc_setup.sh --build-x11vnc ${BUILD_X11VNC} ; rm /tmp/x11vnc_setup.sh @@ -180,7 +176,7 @@ RUN chmod u+x /tmp/x11vnc_setup.sh && \ # # Install any additional apps or libs that are needed # -COPY extras/sources/scripts/apps_setup.sh /tmp/apps_setup.sh +COPY extras/scripts/apps_setup.sh /tmp/apps_setup.sh RUN chmod u+x /tmp/apps_setup.sh && \ /tmp/apps_setup.sh; rm /tmp/apps_setup.sh @@ -213,7 +209,7 @@ RUN usermod -a -G audio,video,render,renderalt ${ADMIN_USERNAME} \ # add some users -#COPY ./sources/scripts/add_users.sh /tmp/add_users.sh +#COPY ./scripts/add_users.sh /tmp/add_users.sh #RUN chmod u+x /tmp/add_users.sh \ # && /tmp/add_users.sh \ # && rm -f /tmp/add_users.sh diff --git a/extras/image/home/.config/lxsession/LXDE/desktop.conf b/extras/image/home/.config/lxsession/LXDE/desktop.conf index 96c7a94..4ed420a 100644 --- a/extras/image/home/.config/lxsession/LXDE/desktop.conf +++ b/extras/image/home/.config/lxsession/LXDE/desktop.conf @@ -1,7 +1,7 @@ [Session] window_manager=openbox-lxde disable_autostart=no -polkit/command=pkexec +polkit/command=lxpolkit clipboard/command=lxclipboard xsettings_manager/command=build-in proxy_manager/command=build-in diff --git a/extras/sources/scripts/amd_gcc_setup.sh b/extras/scripts/amd_gcc_setup.sh similarity index 100% rename from extras/sources/scripts/amd_gcc_setup.sh rename to extras/scripts/amd_gcc_setup.sh diff --git a/extras/sources/scripts/aomp_setup.sh b/extras/scripts/aomp_setup.sh similarity index 100% rename from extras/sources/scripts/aomp_setup.sh rename to extras/scripts/aomp_setup.sh diff --git a/extras/sources/scripts/apps_setup.sh b/extras/scripts/apps_setup.sh similarity index 100% rename from extras/sources/scripts/apps_setup.sh rename to extras/scripts/apps_setup.sh diff --git a/extras/sources/scripts/apps_setup_basic.sh b/extras/scripts/apps_setup_basic.sh similarity index 100% rename from extras/sources/scripts/apps_setup_basic.sh rename to extras/scripts/apps_setup_basic.sh diff --git a/extras/sources/scripts/clacc_setup.sh b/extras/scripts/clacc_setup.sh similarity index 100% rename from extras/sources/scripts/clacc_setup.sh rename to extras/scripts/clacc_setup.sh diff --git a/extras/sources/scripts/compiler_setup.sh b/extras/scripts/compiler_setup.sh similarity index 100% rename from extras/sources/scripts/compiler_setup.sh rename to extras/scripts/compiler_setup.sh diff --git a/extras/sources/scripts/cupy_setup.sh b/extras/scripts/cupy_setup.sh similarity index 100% rename from extras/sources/scripts/cupy_setup.sh rename to extras/scripts/cupy_setup.sh diff --git a/extras/sources/scripts/flang-new_setup.sh b/extras/scripts/flang-new_setup.sh similarity index 100% rename from extras/sources/scripts/flang-new_setup.sh rename to extras/scripts/flang-new_setup.sh diff --git a/extras/sources/scripts/jax_setup.sh b/extras/scripts/jax_setup.sh similarity index 100% rename from extras/sources/scripts/jax_setup.sh rename to extras/scripts/jax_setup.sh diff --git a/extras/sources/scripts/kokkos_setup.sh b/extras/scripts/kokkos_setup.sh similarity index 100% rename from extras/sources/scripts/kokkos_setup.sh rename to extras/scripts/kokkos_setup.sh diff --git a/extras/sources/scripts/llvm-build.sh b/extras/scripts/llvm-build.sh similarity index 100% rename from extras/sources/scripts/llvm-build.sh rename to extras/scripts/llvm-build.sh diff --git a/extras/sources/scripts/miniconda3_setup.sh b/extras/scripts/miniconda3_setup.sh similarity index 100% rename from extras/sources/scripts/miniconda3_setup.sh rename to extras/scripts/miniconda3_setup.sh diff --git a/extras/sources/scripts/pytorch_build_triton_wheel_py.patch b/extras/scripts/pytorch_build_triton_wheel_py.patch similarity index 100% rename from extras/sources/scripts/pytorch_build_triton_wheel_py.patch rename to extras/scripts/pytorch_build_triton_wheel_py.patch diff --git a/extras/sources/scripts/pytorch_setup.sh b/extras/scripts/pytorch_setup.sh similarity index 100% rename from extras/sources/scripts/pytorch_setup.sh rename to extras/scripts/pytorch_setup.sh diff --git a/extras/sources/scripts/x11vnc_setup.sh b/extras/scripts/x11vnc_setup.sh similarity index 93% rename from extras/sources/scripts/x11vnc_setup.sh rename to extras/scripts/x11vnc_setup.sh index 14c9f02..61dc9e3 100755 --- a/extras/sources/scripts/x11vnc_setup.sh +++ b/extras/scripts/x11vnc_setup.sh @@ -69,7 +69,14 @@ ${SUDO} ${DEB_FRONTEND} apt-get install -y xserver-xorg-video-dummy \ libopengl0 mesa-utils libglu1-mesa libgl1-mesa-dri libjpeg8 libjpeg62 \ xauth xdg-utils \ x11vnc \ - novnc + novnc \ + dbus-x11 +echo "Starting dbus" +${SUDO} service dbus status +${SUDO} service dbus start +echo "Checking dbus status" +${SUDO} service dbus status + #curl -O https://bootstrap.pypa.io/get-pip.py && \ # python3 get-pip.py && \ # pip3 install --no-cache-dir \ diff --git a/extras/sources/scripts/og_setup.sh b/extras/sources/scripts/og_setup.sh deleted file mode 100755 index 264a253..0000000 --- a/extras/sources/scripts/og_setup.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -DISTRO=`cat /etc/os-release | grep '^NAME' | sed -e 's/NAME="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' ` -DISTRO_VERSION=`cat /etc/os-release | grep '^VERSION_ID' | sed -e 's/VERSION_ID="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' ` -SUDO="sudo" - -if [ -f /.singularity.d/Singularity ]; then - SUDO="" -fi - -n=0 -while [[ $# -gt 0 ]] -do - case "${1}" in - "--rocm-version") - shift - ROCM_VERSION=${1} - ;; - "--build-og-latest") - shift - BUILD_OG_LATEST=${1} - ;; - *) - last ${1} - ;; - esac - n=$((${n} + 1)) - shift -done - -echo "" -echo "===================================" -echo "Starting OG Latest Install with" -echo "BUILD_OG_LATEST: $BUILD_OG_LATEST" -echo "ROCM_VERSION: $ROCM_VERSION" -echo "===================================" -echo "" - - -if [ "${BUILD_OG_LATEST}" = "1" ] ; then - if [ -f /opt/rocmplus-${ROCM_VERSION}/og13.tgz ]; then - echo "" - echo "============================" - echo " Installing Cached OG Latest" - echo "============================" - echo "" - - #install the cached version - cd /opt/rocmplus-${ROCM_VERSION} - tar -xzf og13.tgz - chown -R root:root /opt/rocmplus-${ROCM_VERSION}/og13-* - rm /opt/rocmplus-${ROCM_VERSION}/og13.tgz - - cd /etc/lmod/modules/ROCmPlus-LatestCompilers - tar -xzf /opt/rocmplus-${ROCM_VERSION}/og13module.tgz - chown -R root:root /etc/lmod/modules/ROCmPlus-LatestCompilers/og* - rm /opt/rocmplus-${ROCM_VERSION}/og13module.tgz - else - echo "" - echo "============================" - echo " Building OG Latest" - echo "============================" - echo "" - - - # Install the OpenMP GCC compiler latest drop - export OG_INSTALL_DIR=/opt/rocmplus-${ROCM_VERSION}/og13-SCRIPT_OG_BUILD_DATE - export OGDIR=/opt/rocmplus-${ROCM_VERSION}/og_build - cd /opt/rocmplus-${ROCM_VERSION} - git clone --depth 1 https://github.com/ROCm-Developer-Tools/og - cd og - bin/build_og13.sh - cd /opt/rocmplus-${ROCM_VERSION} - rm -rf og og_build - - # Only install module when building OG gcc development compiler - # For cached version, use a cached module file - export MODULE_PATH=/etc/lmod/modules/ROCmPlus-LatestCompilers/og - - ${SUDO} mkdir -p ${MODULE_PATH} - - # The - option suppresses tabs - cat <<-EOF | ${SUDO} tee ${MODULE_PATH}/gcc-develop-SCRIPT_OG_BUILD_DATE.lua - whatis("GCC Development Version SCRIPT_OG_BUILD_DATE compiler") - - local base = "/opt/rocmplus-${ROCM_VERSION}/og13-SCRIPT_OG_BUILD_DATE" - - setenv("CC", pathJoin(base, "bin/gcc")) - setenv("CXX", pathJoin(base, "bin/g++")) - setenv("F77", pathJoin(base, "bin/gfortran")) - setenv("F90", pathJoin(base, "bin/gfortran")) - setenv("FC", pathJoin(base,"bin/gfortran")) - append_path("INCLUDE_PATH", pathJoin(base, "include")) - prepend_path("PATH", pathJoin(base, "bin")) - prepend_path("LD_LIBRARY_PATH", pathJoin(base, "lib")) - prepend_path("LD_LIBRARY_PATH", pathJoin(base, "lib64")) - load("rocm/${ROCM_VERSION}") - family("compiler") -EOF - - fi -fi diff --git a/getpackages4cache.sh b/getpackages4cache.sh new file mode 100755 index 0000000..ae7cc0a --- /dev/null +++ b/getpackages4cache.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +DISTRO=`cat /etc/os-release | grep '^NAME' | sed -e 's/NAME="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' ` +DISTRO_VERSION=`cat /etc/os-release | grep '^VERSION_ID' | sed -e 's/VERSION_ID="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' ` + +SUDO="sudo" +if [ -f /.singularity.d/Singularity ]; then + SUDO="" +fi + +usage() +{ + echo "Usage:" + echo " --admin-user [ ADMIN_USER ] default autodetected" + echo " --amdgpu-gfxmodel-string [ AMDGPU_GFXMODEL_STRING ]" + echo " --identity-file [ IDENTITY_FILE ]" + echo " --port-number [ PORT_NUMBER ]" + echo " --rocm-version [ ROCM_VERSION ] default $ROCM_VERSION" + echo " --help: this usage information" + exit 1 +} + +send-error() +{ + usage + echo -e "\nError: ${@}" + exit 1 +} + +reset-last() +{ + last() { send-error "Unsupported argument :: ${1}"; } +} + +n=0 +while [[ $# -gt 0 ]] +do + case "${1}" in + "--admin-user") + shift + ADMIN_USER=${1} + reset-last + ;; + "--amdgpu-gfxmodel-string") + shift + AMDGPU_GFXMODEL_STRING=${1} + reset-last + ;; + "--distro") + shift + DISTRO=${1} + reset-last + ;; + "--distro-version") + shift + DISTRO_VERSION=${1} + reset-last + ;; + "--identity-file") + shift + IDENTITY_FILE=${1} + reset-last + ;; + "--port-number") + shift + PORT_NUMBER=${1} + reset-last + ;; + "--rocm-version") + shift + ROCM_VERSION=${1} + reset-last + ;; + "--*") + send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" + ;; + *) + last ${1} + ;; + esac + n=$((${n} + 1)) + shift +done + +if [ "${IDENTITY_FILE}" != "" ]; then + IDENTITY_FILE="-i ${IDENTITY_FILE}" +fi + +ssh-copy-id ${IDENTITY_FILE} -p ${PORT_NUMBER} -o UpdateHostKeys=yes ${ADMIN_USER}@localhost + +PACKAGE_LIST=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "ls /opt/rocmplus-${ROCM_VERSION}"` +# Remove trailing line break +PACKAGE_LIST=${PACKAGE_LIST%?} +echo "local package.list" +echo ":${PACKAGE_LIST}:" + +app=rocm-${ROCM_VERSION} +if [ ! -f "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" ]; then + PACKAGE_SIZE=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "du -sk /opt/${app} | cut -f1 | tr -d $'\n'" ` + echo "Package size is ${PACKAGE_SIZE}" + if [ "${PACKAGE_SIZE}" -gt "100" ]; then + echo "Retrieving ${app}" + ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "cd /opt && tar -czpf /users/${ADMIN_USER}/${app}.tgz ${app}" + rsync -avz -e "ssh -p ${PORT_NUMBER} ${IDENTITY_FILE}" ${ADMIN_USER}@localhost:${app}.tgz HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz + fi +fi +if [ `du -sk "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" | cut -f1` -eq 0 ] ; then + rm "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" +fi + +for app in ${PACKAGE_LIST} +do + if [ ! -f "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" ]; then + PACKAGE_SIZE=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "du -sk /opt/rocmplus-${ROCM_VERSION}/${app} | cut -f1 | tr -d $'\n'" ` + echo "Package size is ${PACKAGE_SIZE}" + if [ "${PACKAGE_SIZE}" -gt "100" ]; then + echo "Retrieving ${app}" + ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "cd /opt/rocmplus-${ROCM_VERSION} && tar -czf /users/${ADMIN_USER}/${app}.tgz ${app}" + rsync -avz -e "ssh -p ${PORT_NUMBER} ${IDENTITY_FILE}" ${ADMIN_USER}@localhost:${app}.tgz HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz + fi + fi + if [ `du -sk "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" | cut -f1` -eq 0 ] ; then + rm "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" + fi +done diff --git a/rocm/Dockerfile b/rocm/Dockerfile index 571d8a5..78b1645 100644 --- a/rocm/Dockerfile +++ b/rocm/Dockerfile @@ -27,12 +27,12 @@ ARG AMDGPU_GFXMODEL_FIRST=gfx90a ARG AMDGPU_GFXMODEL_STRING=gfx90a ENV PATH ${HOME}/.local/bin:${PATH} -COPY rocm/sources/scripts/baseospackages_setup.sh /tmp/baseospackages_setup.sh +COPY rocm/scripts/baseospackages_setup.sh /tmp/baseospackages_setup.sh RUN /tmp/baseospackages_setup.sh && \ rm /tmp/baseospackages_setup.sh -COPY rocm/sources/scripts/lmod_setup.sh /tmp/lmod_setup.sh +COPY rocm/scripts/lmod_setup.sh /tmp/lmod_setup.sh RUN /tmp/lmod_setup.sh && \ rm /tmp/lmod_setup.sh @@ -41,7 +41,7 @@ RUN /tmp/lmod_setup.sh && \ ADD CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING} \ /CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING} -COPY rocm/sources/scripts/rocm_setup.sh /tmp/rocm_setup.sh +COPY rocm/scripts/rocm_setup.sh /tmp/rocm_setup.sh RUN /tmp/rocm_setup.sh --amdgpu-gfxmodel-string ${AMDGPU_GFXMODEL_STRING} --rocm-version ${ROCM_VERSION} && \ rm /tmp/rocm_setup.sh @@ -49,12 +49,12 @@ RUN /tmp/rocm_setup.sh --amdgpu-gfxmodel-string ${AMDGPU_GFXMODEL_STRING} --roc COPY rocm/sources/lua/bash.bashrc /tmp/bash.bashrc RUN cat /tmp/bash.bashrc >> /etc/bash.bashrc && rm -f /tmp/bash.bashrc -COPY rocm/sources/scripts/rocm_omnitrace_setup.sh /tmp/rocm_omnitrace_setup.sh +COPY rocm/scripts/rocm_omnitrace_setup.sh /tmp/rocm_omnitrace_setup.sh RUN /tmp/rocm_omnitrace_setup.sh --rocm-version ${ROCM_VERSION} && \ rm /tmp/rocm_omnitrace_setup.sh -COPY rocm/sources/scripts/rocm_omniperf_setup.sh /tmp/rocm_omniperf_setup.sh +COPY rocm/scripts/rocm_omniperf_setup.sh /tmp/rocm_omniperf_setup.sh RUN /tmp/rocm_omniperf_setup.sh --rocm-version ${ROCM_VERSION} && \ rm /tmp/rocm_omniperf_setup.sh @@ -75,7 +75,7 @@ RUN groupadd renderalt -g 110 # Install Slurm # -COPY rocm/sources/scripts/slurm_setup.sh /tmp/slurm_setup.sh +COPY rocm/scripts/slurm_setup.sh /tmp/slurm_setup.sh COPY rocm/sources/slurm/slurm_${AMDGPU_GFXMODEL_FIRST}.conf /tmp/slurm.conf COPY rocm/sources/slurm/gres_${AMDGPU_GFXMODEL_FIRST}.conf /tmp/gres.conf diff --git a/rocm/sources/scripts/baseospackages_setup.sh b/rocm/scripts/baseospackages_setup.sh similarity index 100% rename from rocm/sources/scripts/baseospackages_setup.sh rename to rocm/scripts/baseospackages_setup.sh diff --git a/rocm/sources/scripts/lmod_setup.sh b/rocm/scripts/lmod_setup.sh similarity index 100% rename from rocm/sources/scripts/lmod_setup.sh rename to rocm/scripts/lmod_setup.sh diff --git a/rocm/sources/scripts/rocm_omniperf_setup.sh b/rocm/scripts/rocm_omniperf_setup.sh similarity index 100% rename from rocm/sources/scripts/rocm_omniperf_setup.sh rename to rocm/scripts/rocm_omniperf_setup.sh diff --git a/rocm/sources/scripts/rocm_omnitrace_setup.sh b/rocm/scripts/rocm_omnitrace_setup.sh similarity index 100% rename from rocm/sources/scripts/rocm_omnitrace_setup.sh rename to rocm/scripts/rocm_omnitrace_setup.sh diff --git a/rocm/sources/scripts/rocm_setup.sh b/rocm/scripts/rocm_setup.sh similarity index 100% rename from rocm/sources/scripts/rocm_setup.sh rename to rocm/scripts/rocm_setup.sh diff --git a/rocm/sources/scripts/slurm_setup.sh b/rocm/scripts/slurm_setup.sh similarity index 100% rename from rocm/sources/scripts/slurm_setup.sh rename to rocm/scripts/slurm_setup.sh diff --git a/tools/Dockerfile b/tools/Dockerfile index 54b0654..190e43f 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -22,7 +22,7 @@ ARG BUILD_SCOREP LABEL maintainer="bob.robey@amd.com" -COPY tools/sources/scripts/omnitrace_setup.sh /tmp/omnitrace_setup.sh +COPY tools/scripts/omnitrace_setup.sh /tmp/omnitrace_setup.sh RUN /tmp/omnitrace_setup.sh \ --rocm-version ${ROCM_VERSION} \ @@ -48,7 +48,7 @@ RUN chmod 1777 /tmp # Install Grafana # -COPY tools/sources/scripts/grafana_setup.sh /tmp/grafana_setup.sh +COPY tools/scripts/grafana_setup.sh /tmp/grafana_setup.sh RUN /tmp/grafana_setup.sh --install-grafana ${INSTALL_GRAFANA} && \ rm /tmp/grafana_setup.sh @@ -59,7 +59,7 @@ ENV PATH=$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH # Install Omniperf # -COPY tools/sources/scripts/omniperf_setup.sh /tmp/omniperf_setup.sh +COPY tools/scripts/omniperf_setup.sh /tmp/omniperf_setup.sh RUN /tmp/omniperf_setup.sh --rocm-version ${ROCM_VERSION} && \ rm /tmp/omniperf_setup.sh @@ -77,7 +77,7 @@ RUN chmod a+x /docker-entrypoint.sh # # Install HPCToolkit # -COPY tools/sources/scripts/hpctoolkit_setup.sh /tmp/hpctoolkit_setup.sh +COPY tools/scripts/hpctoolkit_setup.sh /tmp/hpctoolkit_setup.sh RUN /tmp/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit ${BUILD_HPCTOOLKIT} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && \ rm /tmp/hpctoolkit_setup.sh @@ -86,7 +86,7 @@ RUN /tmp/hpctoolkit_setup.sh --rocm-version ${ROCM_VERSION} --build-hpctoolkit # # Install TAU # -COPY tools/sources/scripts/tau_setup.sh /tmp/tau_setup.sh +COPY tools/scripts/tau_setup.sh /tmp/tau_setup.sh RUN /tmp/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && \ rm /tmp/tau_setup.sh @@ -95,7 +95,7 @@ RUN /tmp/tau_setup.sh --rocm-version ${ROCM_VERSION} --build-tau ${BUILD_TAU} - # # Install Score-P # -COPY tools/sources/scripts/scorep_setup.sh /tmp/scorep_setup.sh +COPY tools/scripts/scorep_setup.sh /tmp/scorep_setup.sh RUN /tmp/scorep_setup.sh --rocm-version ${ROCM_VERSION} --build-scorep ${BUILD_SCOREP} --amdgpu-gfxmodel ${AMDGPU_GFXMODEL} && \ rm /tmp/scorep_setup.sh diff --git a/tools/sources/scripts/grafana_setup.sh b/tools/scripts/grafana_setup.sh similarity index 100% rename from tools/sources/scripts/grafana_setup.sh rename to tools/scripts/grafana_setup.sh diff --git a/tools/sources/scripts/hpctoolkit_setup.sh b/tools/scripts/hpctoolkit_setup.sh similarity index 100% rename from tools/sources/scripts/hpctoolkit_setup.sh rename to tools/scripts/hpctoolkit_setup.sh diff --git a/tools/sources/scripts/omniperf_setup.sh b/tools/scripts/omniperf_setup.sh similarity index 100% rename from tools/sources/scripts/omniperf_setup.sh rename to tools/scripts/omniperf_setup.sh diff --git a/tools/sources/scripts/omnitrace_setup.sh b/tools/scripts/omnitrace_setup.sh similarity index 100% rename from tools/sources/scripts/omnitrace_setup.sh rename to tools/scripts/omnitrace_setup.sh diff --git a/tools/sources/scripts/scorep_setup.sh b/tools/scripts/scorep_setup.sh similarity index 100% rename from tools/sources/scripts/scorep_setup.sh rename to tools/scripts/scorep_setup.sh diff --git a/tools/sources/scripts/tau_setup.sh b/tools/scripts/tau_setup.sh similarity index 100% rename from tools/sources/scripts/tau_setup.sh rename to tools/scripts/tau_setup.sh