From 96e3c2d605414aa4c856b49069088219852c9107 Mon Sep 17 00:00:00 2001 From: Fabian Bernhard <10245680+fabern@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:48:29 +0200 Subject: [PATCH] Fix use on SLURM by replacing parallel::detectCores() Replaced parallel::detectCores() with length(parallelly::availableWorkers()). Since the former does not respect the allocations of queueing systems such as SLURM, it was not ready to be used on systems such as UBELIX that use e.g. SLURM (see https://stackoverflow.com/a/75923399) --- R/map2tidy.R | 4 ++-- man/map2tidy.Rd | 2 +- vignettes/parallel_computation.Rmd | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/map2tidy.R b/R/map2tidy.R index 405494a..145ce8a 100644 --- a/R/map2tidy.R +++ b/R/map2tidy.R @@ -24,7 +24,7 @@ #' @param fileprefix A character string specifying the file name prefix. #' @param ncores Number of cores for parallel execution (distributing #' extraction of longitude slices). When set to \code{"all"}, the number of -#' cores for parallelisation is determined by \code{parallel::detectCores()}. +#' cores for parallelisation is determined by \code{length(parallelly::availableWorkers())}. #' Defaults to \code{1} (no parallelisation). #' @param fgetdate A function to derive the date used for the time dimension #' based on the file name. @@ -114,7 +114,7 @@ map2tidy <- function( if (ncores=="all"){ - ncores <- parallel::detectCores() - 1 + ncores <- length(parallelly::availableWorkers()) - 1 } ncores <- min(ncores, nrow(ilon_arg)) # No need to have more cores if we only produce 1 file diff --git a/man/map2tidy.Rd b/man/map2tidy.Rd index c434455..a51f04c 100644 --- a/man/map2tidy.Rd +++ b/man/map2tidy.Rd @@ -47,7 +47,7 @@ frames are written using the \code{save} statement. If omitted (defaults to \item{ncores}{Number of cores for parallel execution (distributing extraction of longitude slices). When set to \code{"all"}, the number of -cores for parallelisation is determined by \code{parallel::detectCores()}. +cores for parallelisation is determined by \code{length(parallelly::availableWorkers())}. Defaults to \code{1} (no parallelisation).} \item{fgetdate}{A function to derive the date used for the time dimension diff --git a/vignettes/parallel_computation.Rmd b/vignettes/parallel_computation.Rmd index 96a6ff6..79a3255 100644 --- a/vignettes/parallel_computation.Rmd +++ b/vignettes/parallel_computation.Rmd @@ -127,7 +127,7 @@ print("getting data for longitude indices:") list_of_LON_str <- gsub(".*(LON_[0-9.+-]*).*.rds","\\1", res_tidy$data) # extract LONGITUDES from res_tidy # get all available cores -ncores <- 3 # parallel::detectCores() +ncores <- 3 # length(parallelly::availableWorkers()) # set up the cluster, sending required objects to each core cl <- multidplyr::new_cluster(ncores) %>%