diff --git a/CHANGELOG.md b/CHANGELOG.md index 010c9326a..e072e7a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s - Added support for `pyramid` and `wedge` elements. - Added a `topologies` option to the relay extract. This allows you to select which topologies are saved. This option can be used with the existing `fields` option, the result is the union of the selected topologies and fields. - Added `near_plane` and `far_plane` to the camera details provided in Ascent::info() +- Added `add_mpi_ranks` and `add_domain_ids` filters for adding rank and domain fields to a mesh ### Fixed - Resolved a few cases where MPI_COMM_WORLD was used instead instead of the selected MPI communicator. diff --git a/src/docs/sphinx/Actions/Pipelines.rst b/src/docs/sphinx/Actions/Pipelines.rst index 237bdbe94..3dbb2864f 100644 --- a/src/docs/sphinx/Actions/Pipelines.rst +++ b/src/docs/sphinx/Actions/Pipelines.rst @@ -984,6 +984,53 @@ The output field of the MIR Filter will be the name of the material set and can params["iterations"] = 8; //default: 0 params["max_error"] = 0.00001; //default: 0.00001 +Add MPI Ranks as Field Data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Ascent provides a filter to add MPI rank information to a mesh. +If the input data has multiple topolgies, the user must specify which topology to add the new field to. +The user also has the option of specifying the output name for the new field. + +.. code-block:: c++ + + conduit::Node pipelines; + // pipeline 1 + pipelines["pl1/f1/type"] = "add_mpi_ranks"; + //params optional + conduit::Node ¶ms = pipelines["pl1/f1/params"]; + params["output"] = "ranks";//default: "mpi_ranks" + params["topology"] = "topo"; //required if data has multiple topologies + +.. _addmpiranks: + +.. figure:: ../images/add_mpi_ranks.png + :scale: 50 % + :align: center + + An example of creating a pseudocolor plot of MPI ranks. + +Add Domain IDs as Field Data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Ascent provides a filter to add domain ID information to a mesh. +If the input data has multiple topolgies, the user must specify which topology to add the new field to. +The user also has the option of specifying the output name for the new field. + +.. code-block:: c++ + + conduit::Node pipelines; + // pipeline 1 + pipelines["pl1/f1/type"] = "add_domain_ids"; + //params optional + conduit::Node ¶ms = pipelines["pl1/f1/params"]; + params["output"] = "domain_ids";//default: "domain_ids" + params["topology"] = "topo"; //required if data has multiple topologies + +.. _adddomainids: + +.. figure:: ../images/add_domain_ids.png + :scale: 50 % + :align: center + + An example of creating a pseudocolor plot of domain IDs. Partitioning ~~~~~~~~~~~~ diff --git a/src/docs/sphinx/images/add_domain_ids.png b/src/docs/sphinx/images/add_domain_ids.png new file mode 100644 index 000000000..973763931 Binary files /dev/null and b/src/docs/sphinx/images/add_domain_ids.png differ diff --git a/src/docs/sphinx/images/add_mpi_ranks.png b/src/docs/sphinx/images/add_mpi_ranks.png new file mode 100644 index 000000000..697193a9e Binary files /dev/null and b/src/docs/sphinx/images/add_mpi_ranks.png differ diff --git a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_filters.cpp b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_filters.cpp index 8653c80ad..f83503059 100644 --- a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_filters.cpp +++ b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_filters.cpp @@ -112,6 +112,8 @@ register_builtin() AscentRuntime::register_filter_type(); AscentRuntime::register_filter_type(); // transforms, the current crop expect vtk-h input data + AscentRuntime::register_filter_type("transforms","add_domain_ids"); + AscentRuntime::register_filter_type("transforms","add_mpi_ranks"); AscentRuntime::register_filter_type("transforms","clip"); AscentRuntime::register_filter_type("transforms","clip_with_field"); AscentRuntime::register_filter_type("transforms","clean_grid"); diff --git a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.cpp b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.cpp index d50a5ce88..c0a88ad38 100644 --- a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.cpp +++ b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.cpp @@ -1187,6 +1187,219 @@ VTKHGhostStripper::execute() } } +//----------------------------------------------------------------------------- +VTKHAddRanks::VTKHAddRanks() +:Filter() +{ +// empty +} + +//----------------------------------------------------------------------------- +VTKHAddRanks::~VTKHAddRanks() +{ +// empty +} + +//----------------------------------------------------------------------------- +void +VTKHAddRanks::declare_interface(Node &i) +{ + i["type_name"] = "vtkh_add_mpi_ranks"; + i["port_names"].append() = "in"; + i["output_port"] = "true"; +} + +//----------------------------------------------------------------------------- +bool +VTKHAddRanks::verify_params(const conduit::Node ¶ms, + conduit::Node &info) +{ + info.reset(); + + bool res = check_string("topology",params, info, false); + res = check_string("output",params, info, false); + + std::vector valid_paths; + valid_paths.push_back("output"); + valid_paths.push_back("topology"); + + std::string surprises = surprise_check(valid_paths, params); + + if(surprises != "") + { + res = false; + info["errors"].append() = surprises; + } + + return res; +} + +//----------------------------------------------------------------------------- +void +VTKHAddRanks::execute() +{ + + if(!input(0).check_type()) + { + ASCENT_ERROR("VTKHAddRanks input must be a data object"); + } + + DataObject *data_object = input(0); + if(!data_object->is_valid()) + { + set_output(data_object); + return; + } + + int rank = 0; +#ifdef ASCENT_MPI_ENABLED + MPI_Comm mpi_comm = MPI_Comm_f2c(Workspace::default_mpi_comm()); + MPI_Comm_rank(mpi_comm, &rank); +#endif + + std::shared_ptr collection = data_object->as_vtkh_collection(); + + std::string output_field = "mpi_rank"; + if(params().has_child("output")) + { + output_field = params()["output"].as_string(); + } + + std::string topo_name = ""; + if(params().has_child("topology")) + { + topo_name = params()["topology"].as_string(); + } + else + { + bool throw_error = false; + topo_name = detail::resolve_topology(params(), + this->name(), + collection, + throw_error); + std::cerr << "topo_name: " << topo_name << std::endl; + if(topo_name == "") + { + // this creates a data object with an invalid source + set_output(new DataObject()); + return; + } + } + + vtkh::DataSet &data = collection->dataset_by_topology(topo_name); + VTKHCollection *new_coll = collection->copy_without_topology(topo_name); + data.AddConstantPointField(rank,output_field); + new_coll->add(data, topo_name); + + // re wrap in data object + DataObject *res = new DataObject(new_coll); + set_output(res); +} + +//----------------------------------------------------------------------------- +VTKHAddDomains::VTKHAddDomains() +:Filter() +{ +// empty +} + +//----------------------------------------------------------------------------- +VTKHAddDomains::~VTKHAddDomains() +{ +// empty +} + +//----------------------------------------------------------------------------- +void +VTKHAddDomains::declare_interface(Node &i) +{ + i["type_name"] = "vtkh_add_domain_ids"; + i["port_names"].append() = "in"; + i["output_port"] = "true"; +} + +//----------------------------------------------------------------------------- +bool +VTKHAddDomains::verify_params(const conduit::Node ¶ms, + conduit::Node &info) +{ + info.reset(); + + bool res = check_string("topology",params, info, false); + res = check_string("output",params, info, false); + + std::vector valid_paths; + valid_paths.push_back("output"); + valid_paths.push_back("topology"); + + std::string surprises = surprise_check(valid_paths, params); + + if(surprises != "") + { + res = false; + info["errors"].append() = surprises; + } + + return res; +} + +//----------------------------------------------------------------------------- +void +VTKHAddDomains::execute() +{ + + if(!input(0).check_type()) + { + ASCENT_ERROR("VTKHAddDomains input must be a data object"); + } + + DataObject *data_object = input(0); + if(!data_object->is_valid()) + { + set_output(data_object); + return; + } + + std::shared_ptr collection = data_object->as_vtkh_collection(); + + std::string output_field = "domain_ids"; + if(params().has_child("output")) + { + output_field = params()["output"].as_string(); + } + + std::string topo_name = ""; + if(params().has_child("topology")) + { + topo_name = params()["topology"].as_string(); + } + else + { + bool throw_error = false; + topo_name = detail::resolve_topology(params(), + this->name(), + collection, + throw_error); + std::cerr << "topo_name: " << topo_name << std::endl; + if(topo_name == "") + { + // this creates a data object with an invalid source + set_output(new DataObject()); + return; + } + } + + VTKHCollection *new_coll = collection->copy_without_topology(topo_name); + + vtkh::DataSet &data = collection->dataset_by_topology(topo_name); + data.AddDomainIdField(output_field); + new_coll->add(data, topo_name); + + // re wrap in data object + DataObject *res = new DataObject(new_coll); + set_output(res); +} + //----------------------------------------------------------------------------- VTKHThreshold::VTKHThreshold() :Filter() diff --git a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.hpp b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.hpp index 77b4a8e21..42521eec1 100644 --- a/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.hpp +++ b/src/libs/ascent/runtimes/flow_filters/ascent_runtime_vtkh_filters.hpp @@ -133,6 +133,35 @@ class ASCENT_API VTKHGhostStripper: public ::flow::Filter virtual void execute(); }; +//----------------------------------------------------------------------------- +class ASCENT_API VTKHAddRanks : public ::flow::Filter +{ +public: + VTKHAddRanks(); + virtual ~VTKHAddRanks(); + + virtual void declare_interface(conduit::Node &i); + virtual bool verify_params(const conduit::Node ¶ms, + conduit::Node &info); + virtual void execute(); +}; + +//----------------------------------------------------------------------------- +class ASCENT_API VTKHAddDomains : public ::flow::Filter +{ +public: + VTKHAddDomains(); + virtual ~VTKHAddDomains(); + + + virtual void declare_interface(conduit::Node &i); + virtual bool verify_params(const conduit::Node ¶ms, + conduit::Node &info); + virtual void execute(); +}; + + + //----------------------------------------------------------------------------- class ASCENT_API VTKHClip: public ::flow::Filter { diff --git a/src/libs/vtkh/DataSet.cpp b/src/libs/vtkh/DataSet.cpp index 96c3b958c..05311f33d 100644 --- a/src/libs/vtkh/DataSet.cpp +++ b/src/libs/vtkh/DataSet.cpp @@ -709,6 +709,22 @@ DataSet::AddLinearPointField(const vtkm::Float32 value, const std::string fieldn } } +void +DataSet::AddDomainIdField(const std::string fieldname) +{ + const size_t size = m_domain_ids.size(); + + for(size_t i = 0; i < size; ++i) + { + vtkm::Id domain_id = m_domain_ids[i]; + vtkm::Id num_points = m_domains[i].GetCoordinateSystem().GetData().GetNumberOfValues(); + vtkm::cont::ArrayHandle array; + detail::MemSet(array, (vtkm::Float32)domain_id, num_points); + vtkm::cont::Field field(fieldname, vtkm::cont::Field::Association::Points, array); + m_domains[i].AddField(field); + } +} + bool DataSet::FieldExists(const std::string &field_name) const { diff --git a/src/libs/vtkh/DataSet.hpp b/src/libs/vtkh/DataSet.hpp index 86eb1aa27..4662fab27 100644 --- a/src/libs/vtkh/DataSet.hpp +++ b/src/libs/vtkh/DataSet.hpp @@ -98,6 +98,7 @@ class VTKH_API DataSet // add a scalar field to this data set with a constant value void AddConstantPointField(const vtkm::Float32 value, const std::string fieldname); void AddLinearPointField(const vtkm::Float32 value, const std::string fieldname); + void AddDomainIdField(const std::string fieldname); bool HasDomainId(const vtkm::Id &domain_id) const; /*! \brief IsStructured returns true if all domains, globally, diff --git a/src/tests/_baseline_images/tout_mpi_add_domain_ids100.png b/src/tests/_baseline_images/tout_mpi_add_domain_ids100.png new file mode 100644 index 000000000..973763931 Binary files /dev/null and b/src/tests/_baseline_images/tout_mpi_add_domain_ids100.png differ diff --git a/src/tests/_baseline_images/tout_mpi_add_ranks100.png b/src/tests/_baseline_images/tout_mpi_add_ranks100.png new file mode 100644 index 000000000..e3beb1c01 Binary files /dev/null and b/src/tests/_baseline_images/tout_mpi_add_ranks100.png differ diff --git a/src/tests/ascent/CMakeLists.txt b/src/tests/ascent/CMakeLists.txt index 57173ee15..e8b2d630f 100644 --- a/src/tests/ascent/CMakeLists.txt +++ b/src/tests/ascent/CMakeLists.txt @@ -70,8 +70,8 @@ set(BASIC_TESTS t_ascent_smoke t_ascent_triggers t_ascent_blueprint_reductions t_ascent_sampling - t_ascent_uniform_grid - t_ascent_mir + t_ascent_uniform_grid + t_ascent_mir t_ascent_hola) set(DEVICE_TESTS t_ascent_execution_policies @@ -100,6 +100,8 @@ set(MPI_TESTS t_ascent_mpi_smoke t_ascent_mpi_slice t_ascent_mpi_uniform_grid t_ascent_mpi_vtk_file_extract + t_ascent_mpi_add_ranks + t_ascent_mpi_add_domain_ids t_ascent_mpi_unique_ids) # t_ascent_hola_mpi uses 8 mpi tasks, so its added manually diff --git a/src/tests/ascent/t_ascent_mpi_add_domain_ids.cpp b/src/tests/ascent/t_ascent_mpi_add_domain_ids.cpp new file mode 100644 index 000000000..8c990c81d --- /dev/null +++ b/src/tests/ascent/t_ascent_mpi_add_domain_ids.cpp @@ -0,0 +1,155 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +// Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +// other details. No copyright assignment is required to contribute to Ascent. +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +//----------------------------------------------------------------------------- +/// +/// file: t_ascent_mpi_add_domain_ids.cpp +/// +//----------------------------------------------------------------------------- + + +#include "gtest/gtest.h" + +#include + +#include +#include +#include + +#include +#include + +#include "t_config.hpp" +#include "t_utils.hpp" + + + + +using namespace std; +using namespace conduit; +using namespace ascent; + + +int NUM_DOMAINS = 8; + +//----------------------------------------------------------------------------- +TEST(ascent_mpi_add_domain_ids, test_mpi_add_domain_ids) +{ + // + //Set Up MPI + // + int par_rank; + int par_size; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm, &par_rank); + MPI_Comm_size(comm, &par_size); + + Node n; + ascent::about(n); + // only run this test if ascent was built with vtkm support + if(n["runtimes/ascent/vtkm/status"].as_string() == "disabled") + { + ASCENT_INFO("Ascent vtkm support disabled, skipping test"); + return; + } + + // + // Create an example mesh. + // + Node data, verify_info; + conduit::blueprint::mpi::mesh::examples::spiral_round_robin(NUM_DOMAINS,data,comm); + + EXPECT_TRUE(conduit::blueprint::mesh::verify(data,verify_info)); +// data["state/cycle"] = 100; +// data.print(); + + ASCENT_INFO("Testing adding domain ids to conduit::blueprint spiral input\n"); + + + string output_path = prepare_output_dir(); + string output_file = conduit::utils::join_file_path(output_path,"tout_mpi_add_domain_ids"); + string image_file = conduit::utils::join_file_path(output_path,"tout_mpi_add_domain_ids10"); + + // remove old images before rendering + if(par_rank == 0) + remove_test_image(output_file); + + // + // Create the actions. + // + + conduit::Node pipelines; + // pipeline 1 + pipelines["pl1/f1/type"] = "add_domain_ids"; + conduit::Node ¶ms = pipelines["pl1/f1/params"]; + params["topology"] = "topo"; + params["output"] = "domain_ids"; + + conduit::Node scenes; + scenes["s1/plots/p1/type"] = "pseudocolor"; + scenes["s1/plots/p1/field"] = "domain_ids"; + scenes["s1/plots/p1/pipeline"] = "pl1"; + scenes["s1/plots/p1/color_table/discrete"] = "true"; + + scenes["s1/image_prefix"] = image_file; + + conduit::Node actions; + // add the pipeline + conduit::Node &add_pipelines = actions.append(); + add_pipelines["action"] = "add_pipelines"; + add_pipelines["pipelines"] = pipelines; + // add the scenes + conduit::Node &add_scenes= actions.append(); + add_scenes["action"] = "add_scenes"; + add_scenes["scenes"] = scenes; + +// conduit::Node extracts; +// +// extracts["e1/type"] = "relay"; +// extracts["e1/params/path"] = output_file; +// extracts["e1/params/protocol"] = "blueprint/mesh/hdf5"; +// conduit::Node &add_ext= actions.append(); +// add_ext["action"] = "add_extracts"; +// add_ext["extracts"] = extracts; + + // + // Run Ascent + // + + Ascent ascent; + + Node ascent_opts; + ascent_opts["runtime/type"] = "ascent"; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(comm); + ascent_opts["exceptions"] = "forward"; + ascent.open(ascent_opts); + ascent.publish(data); + ascent.execute(actions); + ascent.close(); + + // check that we created an image + if(par_rank == 0) + { + EXPECT_TRUE(check_test_image(output_file)); + std::string msg = "An example of adding domain_ids to the data."; + ASCENT_ACTIONS_DUMP(actions,output_file,msg); + } +} + +//----------------------------------------------------------------------------- +int main(int argc, char* argv[]) +{ + int result = 0; + + ::testing::InitGoogleTest(&argc, argv); + MPI_Init(&argc, &argv); + + result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} + + diff --git a/src/tests/ascent/t_ascent_mpi_add_ranks.cpp b/src/tests/ascent/t_ascent_mpi_add_ranks.cpp new file mode 100644 index 000000000..ec13241e7 --- /dev/null +++ b/src/tests/ascent/t_ascent_mpi_add_ranks.cpp @@ -0,0 +1,155 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +// Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +// other details. No copyright assignment is required to contribute to Ascent. +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +//----------------------------------------------------------------------------- +/// +/// file: t_ascent_mpi_add_ranks.cpp +/// +//----------------------------------------------------------------------------- + + +#include "gtest/gtest.h" + +#include + +#include +#include +#include + +#include +#include + +#include "t_config.hpp" +#include "t_utils.hpp" + + + + +using namespace std; +using namespace conduit; +using namespace ascent; + + +int NUM_DOMAINS = 8; + +//----------------------------------------------------------------------------- +TEST(ascent_mpi_add_mpi_ranks, test_mpi_add_mpi_ranks) +{ + // + //Set Up MPI + // + int par_rank; + int par_size; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm, &par_rank); + MPI_Comm_size(comm, &par_size); + + Node n; + ascent::about(n); + // only run this test if ascent was built with vtkm support + if(n["runtimes/ascent/vtkm/status"].as_string() == "disabled") + { + ASCENT_INFO("Ascent vtkm support disabled, skipping test"); + return; + } + + // + // Create an example mesh. + // + Node data, verify_info; + conduit::blueprint::mpi::mesh::examples::spiral_round_robin(NUM_DOMAINS,data,comm); + + EXPECT_TRUE(conduit::blueprint::mesh::verify(data,verify_info)); +// data["state/cycle"] = 100; +// data.print(); + + ASCENT_INFO("Testing adding mpi ranks to conduit::blueprint spiral input\n"); + + + string output_path = prepare_output_dir(); + string output_file = conduit::utils::join_file_path(output_path,"tout_mpi_add_ranks"); + string image_file = conduit::utils::join_file_path(output_path,"tout_mpi_add_ranks10"); + + // remove old images before rendering + if(par_rank == 0) + remove_test_image(output_file); + + // + // Create the actions. + // + + conduit::Node pipelines; + // pipeline 1 + pipelines["pl1/f1/type"] = "add_mpi_ranks"; + conduit::Node ¶ms = pipelines["pl1/f1/params"]; + params["topology"] = "topo"; + params["output"] = "ranks"; + + conduit::Node scenes; + scenes["s1/plots/p1/type"] = "pseudocolor"; + scenes["s1/plots/p1/field"] = "ranks"; + scenes["s1/plots/p1/pipeline"] = "pl1"; + scenes["s1/plots/p1/color_table/discrete"] = "true"; + + scenes["s1/image_prefix"] = image_file; + + conduit::Node actions; + // add the pipeline + conduit::Node &add_pipelines = actions.append(); + add_pipelines["action"] = "add_pipelines"; + add_pipelines["pipelines"] = pipelines; + // add the scenes + conduit::Node &add_scenes= actions.append(); + add_scenes["action"] = "add_scenes"; + add_scenes["scenes"] = scenes; + +// conduit::Node extracts; +// +// extracts["e1/type"] = "relay"; +// extracts["e1/params/path"] = output_file; +// extracts["e1/params/protocol"] = "blueprint/mesh/hdf5"; +// conduit::Node &add_ext= actions.append(); +// add_ext["action"] = "add_extracts"; +// add_ext["extracts"] = extracts; + + // + // Run Ascent + // + + Ascent ascent; + + Node ascent_opts; + ascent_opts["runtime/type"] = "ascent"; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(comm); + ascent_opts["exceptions"] = "forward"; + ascent.open(ascent_opts); + ascent.publish(data); + ascent.execute(actions); + ascent.close(); + + // check that we created an image + if(par_rank == 0) + { + EXPECT_TRUE(check_test_image(output_file)); + std::string msg = "An example of adding mpi ranks to the data."; + ASCENT_ACTIONS_DUMP(actions,output_file,msg); + } +} + +//----------------------------------------------------------------------------- +int main(int argc, char* argv[]) +{ + int result = 0; + + ::testing::InitGoogleTest(&argc, argv); + MPI_Init(&argc, &argv); + + result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} + +