Merge Pull Request #10476 from trilinos/Trilinos/master_merge_2022042…

…8_005818

Automatically Merged using Trilinos Master Merge AutoTester
PR Title: Trilinos Master Merge PR Generator: Auto PR created to promote from master_merge_20220428_005818 branch to master
PR Author: trilinos-autotester

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.crs.cmake

@@ -89,7 +89,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
   # Select test disables
-  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
 
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS
       "-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE}"

cmake/ctest/drivers/ascicgpu031/TrilinosCTestDriverCore.ascicgpu031.gcc-cuda.sake.cmake

@@ -89,7 +89,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
 
   # Select test disables
-  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing")
+  set (KokkosCore_UnitTest_CudaTimingBased_MPI_1_DISABLE ON CACHE BOOL "Not to be run in nightly testing" FORCE)
 
 
   SET(EXTRA_SYSTEM_CONFIGURE_OPTIONS

cmake/ctest/drivers/ascicgpu031/ctest_linux_nightly_mpi_debug_nouvm_ascicgpu031_crs.cmake

@@ -69,7 +69,7 @@ SET(BUILD_NAME_DETAILS NOUVM_NODEPRECATED_CRS)
 
 SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
-SET(Trilinos_TRACK Experimental)    # Set the CDash track
+SET(Trilinos_TRACK Nightly)    # Set the CDash track
 SET(CTEST_TEST_TIMEOUT 900)
 
 SET(Trilinos_PACKAGES Amesos2 Belos Tpetra Ifpack2 MueLu Xpetra Zoltan2)

cmake/ctest/drivers/ascicgpu031/ctest_linux_nightly_mpi_debug_uvm_deprecated_ascicgpu031_crs.cmake

@@ -69,7 +69,7 @@ SET(BUILD_NAME_DETAILS UVM_DEPRECATED_CRS)
 
 SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
-SET(Trilinos_TRACK Experimental)    # Set the CDash track
+SET(Trilinos_TRACK Nightly)    # Set the CDash track
 SET(CTEST_TEST_TIMEOUT 900)
 
 SET(Trilinos_PACKAGES Amesos2 Belos Tpetra Ifpack2 MueLu Xpetra Zoltan2)

cmake/ctest/drivers/ascicgpu031/ctest_linux_nightly_mpi_debug_uvm_nodeprecated_ascicgpu031_crs.cmake

@@ -69,7 +69,7 @@ SET(BUILD_NAME_DETAILS UVM_NODEPRECATED_CRS)
 
 SET(CTEST_PARALLEL_LEVEL 8)
 SET(CTEST_TEST_TYPE Nightly)
-SET(Trilinos_TRACK Experimental)    # Set the CDash track
+SET(Trilinos_TRACK Nightly)    # Set the CDash track
 SET(CTEST_TEST_TIMEOUT 900)
 
 SET(Trilinos_PACKAGES Amesos2 Belos Tpetra Ifpack2 MueLu Xpetra Zoltan2)

cmake/ctest/drivers/lightsaber/ctest_linux_experimental_mpi_release_float_lightsaber.cmake

@@ -74,8 +74,21 @@ SET(CTEST_TEST_TIMEOUT 14400) # twice the default value, for valgrind
 SET(CTEST_DO_MEMORY_TESTING FALSE)
 SET(Trilinos_PACKAGES MueLu Tpetra)
 
+# Disable the Epetra stack
+set (Trilinos_ENABLE_Amesos OFF CACHE BOOL "We do not want Amesos" FORCE)
+set (Trilinos_ENABLE_AztecOO OFF CACHE BOOL "We do not want AztecOO" FORCE)
+set (Trilinos_ENABLE_Epetra OFF CACHE BOOL "We do not want Epetra" FORCE)
+set (Trilinos_ENABLE_EpetraExt OFF CACHE BOOL "We do not want EpetraExt" FORCE)
+set (Trilinos_ENABLE_Ifpack OFF CACHE BOOL "We do not want Ifpack" FORCE)
+set (Trilinos_ENABLE_ML OFF CACHE BOOL "We do not want ML" FORCE)
+set (Trilinos_ENABLE_Teko OFF CACHE BOOL "We do not want Teko" FORCE)
+set (Trilinos_ENABLE_Zoltan OFF CACHE BOOL "We do not want Zoltan" FORCE)
+
+
 
 SET(EXTRA_CONFIGURE_OPTIONS
+  "-DTrilinos_ENABLE_Epetra=OFF"
+  "-DTrilinos_ENABLE_EpetraExt=OFF"
   "-DTrilinos_ENABLE_COMPLEX:BOOL=OFF"
   "-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION=ON"
   "-DTrilinos_ENABLE_DEPENDENCY_UNIT_TESTS=OFF"

cmake/std/PullRequestLinuxDriver.sh

@@ -34,26 +34,12 @@ function bootstrap_modules() {
             exit -1
         fi
     else
-        source /projects/sems/modulefiles/utils/sems-archive-modules-init.sh
-        module unload sems-archive-git
-        module unload sems-archive-python
-        module load sems-archive-git/2.10.1
-
-#        module load sems-python/3.5.2      # Currently not on cloud nodes
-#        #pip3 install --user configparser
-#        get_python_packages pip3
-#        export PYTHON_EXE=python3
-
-#         envvar_set_or_create     PYTHONHOME /projects/sierra/linux_rh7/install/Python/3.6.3
-         envvar_set_or_create     PYTHONHOME /projects/sierra/linux_rh7/install/Python/3.6.10
-#        #envvar_set_or_create     PYTHONPATH ${HOME}/.local/lib/python3.6/site-packages
-#        #envvar_append_or_create  PYTHONPATH ${PYTHONHOME:?}/lib/python3.6/site-packages
-#        unset PYTHONHOME
-         unset PYTHONPATH
-         envvar_prepend_or_create PATH       ${PYTHONHOME:?}/bin
-         envvar_set_or_create     PYTHON_EXE ${PYTHONHOME:?}/bin/python3
-         #export PYTHONHOME=/projects/sierra/linux_rh7/install/Python/3.6.3
-         #export PYTHONPATH=${HOME}/.local/lib/python3.6/site-packages:${PYTHONHOME:?}/lib/python3.6/site-packages
+	source /projects/sems/modulefiles/utils/sems-modules-init.sh
+	module unload sems-git
+	module unload sems-python
+	module load sems-git/2.29.0
+	module load sems-python/3.8.6
+	export PYTHON_EXE=python3
     fi
 
     module list

packages/PyTrilinos/cmake/PyTrilinos_config.h.in

@@ -116,14 +116,22 @@
 /* Define if want to build with Tpetra enabled */
 #cmakedefine HAVE_PYTRILINOS_TPETRA
 
+/* Define if want to build with Tpetra long long support enabled */
+#cmakedefine HAVE_PYTRILINOS_TPETRA_INST_INT_LONG_LONG
+
 /* Define if want to build with TriUtils enabled */
 #cmakedefine HAVE_PYTRILINOS_TRIUTILS
 
 /**********************************************************************/
 
 /* PyTrilinos ordinal types */
 
+#ifdef HAVE_PYTRILINOS_TPETRA_INST_INT_LONG_LONG
 #define PYTRILINOS_GLOBAL_ORD long long
+#else
+#define PYTRILINOS_GLOBAL_ORD int
+#endif
+
 #define PYTRILINOS_LOCAL_ORD  int
 
 /**********************************************************************/

packages/PyTrilinos/src/CMakeLists.txt

@@ -80,6 +80,12 @@ IF(PyTrilinos_ENABLE_NOX)
   ENDIF()
 ENDIF()
 
+IF(PyTrilinos_ENABLE_Tpetra)
+  IF(Tpetra_INST_INT_LONG_LONG)
+    SET(HAVE_PYTRILINOS_TPETRA_INST_INT_LONG_LONG ON)
+  ENDIF()
+ENDIF()
+
 #
 # Build the PyTrilinos configuration file
 TRIBITS_CONFIGURE_FILE(PyTrilinos_config.h)

packages/PyTrilinos/src/Tpetra.i

@@ -1477,7 +1477,9 @@ public:
 // Concrete scalar types for Tpetra classes //
 //////////////////////////////////////////////
 %tpetra_scalars(int       , int   )
+#ifdef HAVE_PYTRILINOS_TPETRA_INST_INT_LONG_LONG
 %tpetra_scalars(long long , long  )
+#endif
 %tpetra_scalars(double    , double)
 
 /////////////////////////////////////////////////////

packages/adelus/README.md

@@ -207,7 +207,7 @@ cmake \
 \
 -D Kokkos_ENABLE_SERIAL:BOOL=OFF \
 -D Kokkos_ENABLE_OPENMP:BOOL=ON \
--D Kokkos_ENABLE_PTHREAD:BOOL=OFF \
+-D Kokkos_ENABLE_THREADS:BOOL=OFF \
 -D Kokkos_ENABLE_CUDA:BOOL=ON \
 -D Kokkos_ENABLE_CUDA_UVM:BOOL=OFF \
 -D Kokkos_ENABLE_CUDA_LAMBDA:BOOL=ON \

packages/belos/src/BelosGmresPolySolMgr.hpp

@@ -152,9 +152,9 @@ template<class ScalarType, class MV, class OP>
 class GmresPolySolMgr : public SolverManager<ScalarType,MV,OP> {
 private:
 
-  typedef MultiVecTraits<ScalarType,MV> MVT;
-  typedef Teuchos::ScalarTraits<ScalarType> STS;
   typedef typename Teuchos::ScalarTraits<ScalarType>::magnitudeType MagnitudeType;
+
+  typedef Teuchos::ScalarTraits<MagnitudeType> MTS;
   typedef Belos::GmresPolyOp<ScalarType,MV,OP> gmres_poly_t;
   typedef Belos::GmresPolyMv<ScalarType,MV>    gmres_poly_mv_t;
 
@@ -217,6 +217,26 @@ class GmresPolySolMgr : public SolverManager<ScalarType,MV,OP> {
    */
   Teuchos::RCP<const Teuchos::ParameterList> getCurrentParameters() const override { return params_; }
 
+  /*! \brief Tolerance achieved by the last \c solve() invocation.
+    
+     This is the maximum over all right-hand sides' achieved
+     convergence tolerances, and is set whether or not the solve
+     actually managed to achieve the desired convergence tolerance.
+    
+     \warning This solver manager may be use as either a polynomial
+       preconditioned iterative method or a polynomial preconditioner.
+       In the later case, where a static polynomial is being applied
+       through each call to solve(), there is not an outer solve that 
+       can provide the achieved tolerance.
+     \warning This result may not be meaningful if there was a loss
+       of accuracy during the outer solve.  You should first call \c
+       isLOADetected() to check for a loss of accuracy during the
+       last solve.
+  */
+    MagnitudeType achievedTol() const override {
+      return achievedTol_;
+    }
+
   /*! \brief Return the timers for this object.
    *
    * The timers are ordered as follows:
@@ -331,7 +351,7 @@ class GmresPolySolMgr : public SolverManager<ScalarType,MV,OP> {
 #endif
 
   // Current solver values.
-  MagnitudeType polyTol_;
+  MagnitudeType polyTol_, achievedTol_;
   int maxDegree_, numIters_;
   int verbosity_;
   bool hasOuterSolver_;
@@ -363,6 +383,7 @@ template<class ScalarType, class MV, class OP>
 GmresPolySolMgr<ScalarType,MV,OP>::GmresPolySolMgr () :
   outputStream_ (Teuchos::rcp(outputStream_default_,false)),
   polyTol_ (DefaultSolverParameters::polyTol),
+  achievedTol_(MTS::zero()),
   maxDegree_ (maxDegree_default_),
   numIters_ (0),
   verbosity_ (verbosity_default_),
@@ -433,6 +454,8 @@ GmresPolySolMgr<ScalarType,MV,OP>::getValidParameters() const
       "The maximum degree allowed for any GMRES polynomial.");
     pl->set("Outer Solver", static_cast<const char *>(outerSolverType_default_),
       "The outer solver that this polynomial is used to precondition.");
+    pl->set("Outer Solver Params", Teuchos::ParameterList(),
+      "Parameter list for the outer solver.");
     pl->set("Verbosity", static_cast<int>(verbosity_default_),
       "What type(s) of solver information should be outputted\n"
       "to the output stream.");
@@ -464,7 +487,7 @@ setParameters (const Teuchos::RCP<Teuchos::ParameterList>& params)
     params_ = Teuchos::parameterList (*getValidParameters ());
   }
   else {
-    params->validateParameters (*getValidParameters ());
+    params->validateParameters (*getValidParameters (),0);
   }
 
   // Check which Gmres polynomial to use
@@ -689,6 +712,7 @@ ReturnType GmresPolySolMgr<ScalarType,MV,OP>::solve ()
     ret = solver->solve();
     numIters_ = solver->getNumIters();
     loaDetected_ = solver->isLOADetected();
+    achievedTol_ = solver->achievedTol();
 
   } // if (hasOuterSolver_ && maxDegree_)
   else if (hasOuterSolver_) {
@@ -704,12 +728,14 @@ ReturnType GmresPolySolMgr<ScalarType,MV,OP>::solve ()
     ret = solver->solve();
     numIters_ = solver->getNumIters();
     loaDetected_ = solver->isLOADetected();
+    achievedTol_ = solver->achievedTol();
 
   }
   else if (maxDegree_) {
 
     // Apply the polynomial to the current linear system
     poly_Op_->ApplyPoly( *problem_->getRHS(), *problem_->getLHS() );
+    achievedTol_ = MTS::one();
 
   }
 

packages/belos/test/BlockGmres/test_hybrid_gmres_complex_hb.cpp

@@ -249,8 +249,9 @@ int main(int argc, char *argv[]) {
     //
     Belos::ReturnType ret = solver->solve();
     //
-    // Compute actual residuals.
+    std::cout << "Belos::GmresPolySolMgr returned achievedTol: " << solver->achievedTol() << std::endl << std::endl;
     //
+    // Compute actual residuals.
     RCP<MyMultiVec<ST> > temp = rcp( new MyMultiVec<ST>(dim,numrhs) );
     OPT::Apply( *A, *soln, *temp );
     MVT::MvAddMv( one, *rhs, -one, *temp, *temp );

packages/common/auxiliarySoftware/SuiteSparse/cmake/TrilinosSS_config.h.in

@@ -1 +1,9 @@
 /* src/TrilinosSS_config.h.in.  Generated from configure.ac by autoheader.  */
+
+#ifndef TrilinosSS_CONFIG_H
+#define TrilinosSS_CONFIG_H
+
+/* define if we want to use OpenMP */
+#cmakedefine TRILINOSSS_HAVE_OMP
+
+#endif

packages/common/auxiliarySoftware/SuiteSparse/src/CMakeLists.txt

@@ -1,7 +1,27 @@
+######################################################################
+## Define the header and source files (and directories)
+######################################################################
+
 SET(HEADERS "")
 SET(SOURCES "")
 SET(IMPORTEDLIBS "")
 
+######################################################################
+## Package-specific configuration options
+######################################################################
+
+TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_OpenMP
+  TRILINOSSS_HAVE_OMP
+  "Enable OpenMP support in TrilinosSS."
+  ${${PROJECT_NAME}_ENABLE_OpenMP})
+
+TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h)
+
+SET(HEADERS ${HEADERS}
+  ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME}_config.h
+  )
+
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/AMD/Include)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/BTF/Include)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/CAMD/Include)
@@ -24,7 +44,7 @@ file(GLOB CCOLAMD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/CCOLAMD/Source/*.c)
 file(GLOB COLAMD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/COLAMD/Source/*.c)
 file(GLOB KLU_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/KLU/Source/*.c)
 
-SET(HEADERS ${AMD_HEADERS}
+APPEND_SET(HEADERS ${AMD_HEADERS}
    ${BTF_HEADERS}
    ${CAMD_HEADERS}
    ${CCOLAMD_HEADERS}

packages/common/auxiliarySoftware/SuiteSparse/src/KLU/Source/trilinos_klu_factor.c

@@ -6,6 +6,7 @@
  * or TRILINOS_KLU_analyze_given.
  */
 
+#include "TrilinosSS_config.h"
 #include "trilinos_klu_internal.h"
 
 /* ========================================================================== */
@@ -397,7 +398,7 @@ TRILINOS_KLU_numeric *TRILINOS_KLU_factor		/* returns NULL if error, or a valid
     Int *R ;
     TRILINOS_KLU_numeric *Numeric ;
     size_t n1, nzoff1, s, b6, n3 ;
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP 
     int num_threads;
 #endif
 
@@ -500,7 +501,7 @@ TRILINOS_KLU_numeric *TRILINOS_KLU_factor		/* returns NULL if error, or a valid
      *
      * If OpenMP is enabled the solve uses an Xwork of size num_threads * 4n.
      */
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP
 #pragma omp parallel
     num_threads = omp_get_num_threads();
     s = TRILINOS_KLU_mult_size_t (n, num_threads * sizeof (Entry), &ok) ;

packages/common/auxiliarySoftware/SuiteSparse/src/KLU/Source/trilinos_klu_tsolve.c

@@ -9,6 +9,7 @@
  * Numeric->Iwork).
  */
 
+#include "TrilinosSS_config.h"
 #include "trilinos_klu_internal.h"
 
 Int TRILINOS_KLU_tsolve
@@ -37,7 +38,7 @@ Int TRILINOS_KLU_tsolve
     Int *Q, *R, *Pnum, *Offp, *Offi, *Lip, *Uip, *Llen, *Ulen ;
     Unit **LUbx ;
     Int k1, k2, nk, k, block, pend, n, p, nblocks, chunk, nr, i ;
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP
     Entry *X1 ;
     int tid ;
 #endif
@@ -86,7 +87,7 @@ Int TRILINOS_KLU_tsolve
     Udiag = (double*) Numeric->Udiag ;
 
     Rs = Numeric->Rs ;
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP
     X1 = (Entry *) Numeric->Xwork ;
 #else
     X = (Entry *) Numeric->Xwork ;
@@ -97,12 +98,12 @@ Int TRILINOS_KLU_tsolve
     /* solve in chunks of 4 columns at a time */
     /* ---------------------------------------------------------------------- */
 
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP
 #pragma omp parallel for schedule(guided) private(tid, nr, k, block, k1, k2, nk, pend, p, s, i, Bz, X, x, offik, rs) 
 #endif
     for (chunk = 0 ; chunk < nrhs ; chunk += 4)
     {
-#ifdef KLU_ENABLE_OPENMP
+#ifdef TRILINOSSS_HAVE_OMP
         Bz  = ((Entry *) B) + d*chunk ;
         tid = omp_get_thread_num();
         X = X1 + (tid * n * 4);
@@ -476,7 +477,7 @@ Int TRILINOS_KLU_tsolve
 	/* go to the next chunk of B */
 	/* ------------------------------------------------------------------ */
 
-#ifndef KLU_ENABLE_OPENMP
+#ifndef TRILINOSSS_HAVE_OMP
 	Bz  += d*4 ;
 #endif
     }