diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 784e8f71..56fd3574 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -165,7 +165,7 @@ jobs: with: repo-name: 'redev-openmpi' repo-path: 'SCOREC/redev' - repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376' + repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5' cache: true options: '-DCMAKE_CXX_COMPILER=`which mpicxx` -DMPIEXEC_EXECUTABLE=`which mpirun` diff --git a/.github/workflows/cmake-test.yml b/.github/workflows/cmake-test.yml index d736a952..40aeb88c 100644 --- a/.github/workflows/cmake-test.yml +++ b/.github/workflows/cmake-test.yml @@ -165,7 +165,7 @@ jobs: with: repo-name: 'redev' repo-path: 'SCOREC/redev' - repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376' + repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5' cache: true cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }} options: '-DCMAKE_CXX_COMPILER=`which mpicxx` diff --git a/src/pcms/coupler/CMakeLists.txt b/src/pcms/coupler/CMakeLists.txt index 82afceb3..6a498c62 100644 --- a/src/pcms/coupler/CMakeLists.txt +++ b/src/pcms/coupler/CMakeLists.txt @@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS field_exchange_planner.h partition.h overlap_mask.h + global_communicator.h ) diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index eef0f564..93635be3 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -11,10 +11,57 @@ #include "pcms/utility/assert.h" #include "pcms/utility/common.h" #include "pcms/utility/profile.h" +#include "pcms/coupler/global_communicator.h" #include namespace pcms { +template +class GlobalDataInterface +{ +public: + GlobalDataInterface(std::string name, + Rank1View data, + MPI_Comm mpi_comm, redev::Channel& channel) + : data_(data), + variable_name_(name), + mpi_comm_(mpi_comm), + comm_(name, mpi_comm_, channel) + { + PCMS_FUNCTION_TIMER; + } + + void Send(redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + + comm_.Send(data_.data_handle(), variable_name_, + static_cast(data_.extent(0)), mode); + } + + void Receive(redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + + auto received = comm_.Receive( + variable_name_, static_cast(data_.extent(0)), mode); + + PCMS_ALWAYS_ASSERT(received.size() == + static_cast(data_.extent(0))); + + std::copy(received.begin(), received.end(), data_.data_handle()); + } + +private: + Rank1View data_; + std::string variable_name_; + MPI_Comm mpi_comm_; + GlobalCommunicator comm_; +}; +using GlobalDataVariant = + std::variant, GlobalDataInterface, + GlobalDataInterface, GlobalDataInterface, + GlobalDataInterface>; class Application; @@ -31,6 +78,21 @@ class FieldHandle void Receive(redev::Mode mode = redev::Mode::Synchronous) const; [[nodiscard]] Field& GetField() const; +private: + Application* app_; + std::string name_; +}; +template +class DataHandle +{ +public: + DataHandle(Application* app, std::string name) + : app_(app), name_(std::move(name)) + { + } + void Send(redev::Mode mode = redev::Mode::Synchronous) const; + void Receive(redev::Mode mode = redev::Mode::Synchronous) const; + [[nodiscard]] const std::string& GetName() const noexcept { return name_; } private: @@ -99,6 +161,16 @@ class Application // Register a transferable field: like AddField but retains the function space // (via the stored Function) and returns a FunctionHandle usable in transfers. template + FieldHandle AddField(std::string name, Field&& field, + std::unique_ptr> serializer, + bool participates = true); + // Registers a reference to user-owned data. The Application does not take + // ownership; it uses the registered reference for subsequent send and receive + // operations. + template + DataHandle AddData(std::string name, std::vector& data, + MPI_Comm mpi_comm); + template FunctionHandle AddFunction(Function&& function, bool participates = true); @@ -127,6 +199,25 @@ class Application [mode](auto& field_communicator) { field_communicator->Receive(); }, detail::find_or_error(name, field_communicators_)); }; + void SendData(const std::string& name, + redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InSendPhase()); + + std::visit([mode](auto& data_interface) { data_interface.Send(mode); }, + detail::find_or_error(name, global_data_interfaces_)); + } + + void ReceiveData(const std::string& name, + redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InReceivePhase()); + + std::visit([mode](auto& data_interface) { data_interface.Receive(mode); }, + detail::find_or_error(name, global_data_interfaces_)); + } [[nodiscard]] bool InSendPhase() const noexcept { PCMS_FUNCTION_TIMER; @@ -205,8 +296,40 @@ class Application std::map> field_layout_communicators_; std::map> layout_overlap_masks_; + std::map global_data_interfaces_; }; +template +DataHandle Application::AddData(std::string name, std::vector& data, + MPI_Comm mpi_comm) +{ + PCMS_FUNCTION_TIMER; + auto data_view = make_array_view(data); + auto [it, inserted] = global_data_interfaces_.try_emplace( + name, std::in_place_type>, name, data_view, mpi_comm, + channel_); + + if (!inserted) { + throw pcms_error("Global data interface with this name already exists"); + } + return DataHandle{this, std::move(name)}; +} + +template +void DataHandle::Send(redev::Mode mode) const +{ + PCMS_ALWAYS_ASSERT(app_ != nullptr); + + app_->SendData(name_, mode); +} +template +void DataHandle::Receive(redev::Mode mode) const +{ + PCMS_ALWAYS_ASSERT(app_ != nullptr); + + app_->ReceiveData(name_, mode); +} + class Coupler { private: diff --git a/src/pcms/coupler/global_communicator.h b/src/pcms/coupler/global_communicator.h new file mode 100644 index 00000000..ac8481a4 --- /dev/null +++ b/src/pcms/coupler/global_communicator.h @@ -0,0 +1,52 @@ +#ifndef PCMS_GLOBAL_COMMUNICATOR_H +#define PCMS_GLOBAL_COMMUNICATOR_H +#include +#include +#include +namespace pcms +{ +using redev::Mode; +template +struct GlobalCommunicator +{ + using value_type = T; + +public: + GlobalCommunicator(std::string name, MPI_Comm mpi_comm, + redev::Channel& channel) + : mpi_comm(mpi_comm), channel_(channel), name_(std::move(name)) + { + PCMS_FUNCTION_TIMER; + comm_ = channel_.CreateComm(name_, mpi_comm, redev::CommType::Global); + } + GlobalCommunicator(const GlobalCommunicator&) = delete; + GlobalCommunicator& operator=(const GlobalCommunicator&) = delete; + GlobalCommunicator(GlobalCommunicator&&) = default; + GlobalCommunicator& operator=(GlobalCommunicator&&) = default; + + void Send(T* msg, std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase()); + comm_.SetCommParams(VarName, msg_size); + comm_.Send(msg, mode); + } + std::vector Receive(std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase()); + comm_.SetCommParams(VarName, msg_size); + auto data = comm_.Recv(mode); + return data; + } + +private: + MPI_Comm mpi_comm; + redev::Channel& channel_; + std::string name_; + redev::BidirectionalComm comm_; +}; +} // namespace pcms +#endif // PCMS_GLOBAL_COMMUNICATOR_H \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5efdc81e..9a7eb845 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -96,6 +96,37 @@ if(PCMS_ENABLE_OMEGA_H) ${d3d16p} ignored) endif() + add_exe(test_GDI) + tri_mpi_test( + TESTNAME + test_GDI + TIMEOUT + 20 + NAME1 + app + EXE1 + ./test_GDI + PROCS1 + 1 + ARGS1 + 1 + NAME2 + rdv + EXE2 + ./test_GDI + PROCS2 + 1 + ARGS2 + -1 + NAME3 + app + EXE3 + ./test_GDI + PROCS3 + 1 + ARGS3 + 0 + ) set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/) add_exe(test_twoClientOverlap) @@ -380,7 +411,7 @@ if(Catch2_FOUND) APPEND PCMS_UNIT_TEST_SOURCES test_error_handling.cpp - test_eqdsk.cpp + test_eqdsk.cpp test_uniform_grid.cpp test_field_evaluation.cpp test_field_interpolation.cpp @@ -402,7 +433,6 @@ if(Catch2_FOUND) test_omega_h_lagrange_field.cpp test_point_evaluator.cpp) endif() - if(PCMS_ENABLE_MESHFIELDS) list(APPEND PCMS_UNIT_TEST_SOURCES test_omega_h_form_integrator_utils.cpp) @@ -430,15 +460,6 @@ if(Catch2_FOUND) target_link_libraries(unit_tests PRIVATE PETSc::PETSc) endif() - target_link_libraries(unit_tests PUBLIC - Catch2::Catch2 - pcms::core - pcms_transfer - pcms_transfer - ) - - target_include_directories(unit_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - add_executable(test_interpolation_on_ltx_mesh test_interpolation_on_ltx_mesh.cpp) target_link_libraries(test_interpolation_on_ltx_mesh PUBLIC Catch2::Catch2WithMain pcms::core diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp new file mode 100644 index 00000000..a8467c6e --- /dev/null +++ b/test/test_GDI.cpp @@ -0,0 +1,133 @@ +#include +#include +#include "test_support.h" +#include "pcms/coupler/coupler.hpp" +#include +#include + +static constexpr bool done = true; +static constexpr int COMM_ROUNDS = 1; + +void xgc_delta_f(MPI_Comm comm) +{ + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f"); + + std::vector mean(1); + app->AddData("mean", mean, comm); + + mean[0] = 16; + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->BeginSendPhase(); + app->SendData("mean"); + app->EndSendPhase(); + printf("delta Sent mean:%ld\n", mean[0]); + app->BeginReceivePhase(); + app->ReceiveData("mean"); + app->EndReceivePhase(); + mean[0] = mean[0] / 2; + } + } while (!done); + printf("final Mean = %ld\n", mean[0]); + assert(std::fabs(mean[0] - 1.0) < 1e-12); + printf("GDI test successful.\n"); +} +void xgc_total_f(MPI_Comm comm) +{ + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_total_f"); + + std::vector mean(1); + app->AddData("mean", mean, comm); + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->BeginReceivePhase(); + app->ReceiveData("mean"); + app->EndReceivePhase(); + printf("total Recieved mean:%ld\n", mean[0]); + mean[0] = mean[0] / 2; + app->BeginSendPhase(); + app->SendData("mean"); + app->EndSendPhase(); + printf("total Sent mean:%ld\n", mean[0]); + } + } while (!done); +} +void xgc_coupler(MPI_Comm comm) +{ + // Define Partition + redev::LO dim = 3; + redev::LOs ranks(1); + std::iota(ranks.begin(), ranks.end(), 0); + redev::Reals cuts = {0}; + auto partition = redev::Partition{redev::RCBPtn{dim, ranks, cuts}}; + + pcms::Coupler cpl("proxy_couple", comm, true, partition); + auto* total_f = cpl.AddApplication("proxy_couple_xgc_total_f"); + auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f"); + + std::vector mean(1); + total_f->AddData("mean", mean, comm); + delta_f->AddData("mean", mean, comm); + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + delta_f->BeginReceivePhase(); + delta_f->ReceiveData("mean"); + delta_f->EndReceivePhase(); + printf("delta Received mean:%ld\n", mean[0]); + mean[0] = mean[0] / 2; + const auto msg_size = mean.size(); + total_f->BeginSendPhase(); + total_f->SendData("mean"); + total_f->EndSendPhase(); + printf("total sent mean:%ld\n", mean[0]); + total_f->BeginReceivePhase(); + total_f->ReceiveData("mean"); + total_f->EndReceivePhase(); + printf("delta Received mean:%ld\n", mean[0]); + mean[0] = mean[0] / 2; + delta_f->BeginSendPhase(); + delta_f->SendData("mean"); + delta_f->EndSendPhase(); + printf("delta sent mean:%ld\n", mean[0]); + } + } while (!done); +} + +int main(int argc, char** argv) +{ + try { + MPI_Init(&argc, &argv); + Kokkos::initialize(argc, argv); + OMEGA_H_CHECK(argc == 2); + + const auto clientId = std::atoi(argv[1]); + REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1); + + MPI_Comm comm = MPI_COMM_WORLD; + + switch (clientId) { + case -1: xgc_coupler(comm); break; + + case 0: xgc_delta_f(comm); break; + + case 1: xgc_total_f(comm); break; + default: + std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + + MPI_Finalize(); + return 0; + } catch (const std::exception& e) { + std::cerr << "Exception caught in main: " << e.what() << std::endl; + return 1; + } catch (...) { + std::cerr << "Unknown exception caught in main\n"; + return 1; + } +}