Add global comm

.github/workflows/clang-tidy.yml

@@ -165,7 +165,7 @@ jobs:
       with:
         repo-name: 'redev-openmpi'
         repo-path: 'SCOREC/redev'
-        repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
+        repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DMPIEXEC_EXECUTABLE=`which mpirun`

.github/workflows/cmake-test.yml

@@ -165,7 +165,7 @@ jobs:
       with:
         repo-name: 'redev'
         repo-path: 'SCOREC/redev'
-        repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
+        repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`

src/pcms/coupler/CMakeLists.txt

@@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS
         field_exchange_planner.h
         partition.h
         overlap_mask.h
+        global_communicator.h
 )
 
 

src/pcms/coupler/coupler.hpp

@@ -11,10 +11,57 @@
 #include "pcms/utility/assert.h"
 #include "pcms/utility/common.h"
 #include "pcms/utility/profile.h"
+#include "pcms/coupler/global_communicator.h"
 #include <memory>
 
 namespace pcms
 {
+template <typename T>
+class GlobalDataInterface
+{
+public:
+  GlobalDataInterface(std::string name,
+                      Rank1View<T, pcms::HostMemorySpace> data,
+                      MPI_Comm mpi_comm, redev::Channel& channel)
+    : data_(data),
+      variable_name_(name),
+      mpi_comm_(mpi_comm),
+      comm_(name, mpi_comm_, channel)
+  {
+    PCMS_FUNCTION_TIMER;
+  }
+
+  void Send(redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+
+    comm_.Send(data_.data_handle(), variable_name_,
+               static_cast<std::size_t>(data_.extent(0)), mode);
+  }
+
+  void Receive(redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+
+    auto received = comm_.Receive(
+      variable_name_, static_cast<std::size_t>(data_.extent(0)), mode);
+
+    PCMS_ALWAYS_ASSERT(received.size() ==
+                       static_cast<std::size_t>(data_.extent(0)));
+
+    std::copy(received.begin(), received.end(), data_.data_handle());
+  }
+
+private:
+  Rank1View<T, pcms::HostMemorySpace> data_;
+  std::string variable_name_;
+  MPI_Comm mpi_comm_;
+  GlobalCommunicator<T> comm_;
+};
+using GlobalDataVariant =
+  std::variant<GlobalDataInterface<int8_t>, GlobalDataInterface<int32_t>,
+               GlobalDataInterface<int64_t>, GlobalDataInterface<float>,
+               GlobalDataInterface<double>>;
 
 class Application;
 
@@ -31,6 +78,21 @@ class FieldHandle
   void Receive(redev::Mode mode = redev::Mode::Synchronous) const;
   [[nodiscard]] Field<T>& GetField() const;
 
+private:
+  Application* app_;
+  std::string name_;
+};
+template <typename T>
+class DataHandle
+{
+public:
+  DataHandle(Application* app, std::string name)
+    : app_(app), name_(std::move(name))
+  {
+  }
+  void Send(redev::Mode mode = redev::Mode::Synchronous) const;
+  void Receive(redev::Mode mode = redev::Mode::Synchronous) const;
+
   [[nodiscard]] const std::string& GetName() const noexcept { return name_; }
 
 private:
@@ -99,6 +161,16 @@ class Application
   // Register a transferable field: like AddField but retains the function space
   // (via the stored Function) and returns a FunctionHandle usable in transfers.
   template <typename T>
+  FieldHandle<T> AddField(std::string name, Field<T>&& field,
+                          std::unique_ptr<FieldSerializer<T>> serializer,
+                          bool participates = true);
+  // Registers a reference to user-owned data. The Application does not take
+  // ownership; it uses the registered reference for subsequent send and receive
+  // operations.
+  template <typename T>
+  DataHandle<T> AddData(std::string name, std::vector<T>& data,
+                        MPI_Comm mpi_comm);
+  template <typename T>
   FunctionHandle<T> AddFunction(Function<T>&& function,
                                 bool participates = true);
 
@@ -127,6 +199,25 @@ class Application
       [mode](auto& field_communicator) { field_communicator->Receive(); },
       detail::find_or_error(name, field_communicators_));
   };
+  void SendData(const std::string& name,
+                redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(InSendPhase());
+
+    std::visit([mode](auto& data_interface) { data_interface.Send(mode); },
+               detail::find_or_error(name, global_data_interfaces_));
+  }
+
+  void ReceiveData(const std::string& name,
+                   redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(InReceivePhase());
+
+    std::visit([mode](auto& data_interface) { data_interface.Receive(mode); },
+               detail::find_or_error(name, global_data_interfaces_));
+  }
   [[nodiscard]] bool InSendPhase() const noexcept
   {
     PCMS_FUNCTION_TIMER;
@@ -205,8 +296,40 @@ class Application
   std::map<std::string, std::unique_ptr<FieldLayoutCommunicator>>
     field_layout_communicators_;
   std::map<std::string, std::unique_ptr<OverlapMask>> layout_overlap_masks_;
+  std::map<std::string, GlobalDataVariant> global_data_interfaces_;
 };
 
+template <typename T>
+DataHandle<T> Application::AddData(std::string name, std::vector<T>& data,
+                                   MPI_Comm mpi_comm)
+{
+  PCMS_FUNCTION_TIMER;
+  auto data_view = make_array_view(data);
+  auto [it, inserted] = global_data_interfaces_.try_emplace(
+    name, std::in_place_type<GlobalDataInterface<T>>, name, data_view, mpi_comm,
+    channel_);
+
+  if (!inserted) {
+    throw pcms_error("Global data interface with this name already exists");
+  }
+  return DataHandle<T>{this, std::move(name)};
+}
+
+template <typename T>
+void DataHandle<T>::Send(redev::Mode mode) const
+{
+  PCMS_ALWAYS_ASSERT(app_ != nullptr);
+
+  app_->SendData(name_, mode);
+}
+template <typename T>
+void DataHandle<T>::Receive(redev::Mode mode) const
+{
+  PCMS_ALWAYS_ASSERT(app_ != nullptr);
+
+  app_->ReceiveData(name_, mode);
+}
+
 class Coupler
 {
 private:

src/pcms/coupler/global_communicator.h

@@ -0,0 +1,52 @@
+#ifndef PCMS_GLOBAL_COMMUNICATOR_H
+#define PCMS_GLOBAL_COMMUNICATOR_H
+#include <redev.h>
+#include <pcms/utility/profile.h>
+#include <pcms/utility/assert.h>
+namespace pcms
+{
+using redev::Mode;
+template <typename T>
+struct GlobalCommunicator
+{
+  using value_type = T;
+
+public:
+  GlobalCommunicator(std::string name, MPI_Comm mpi_comm,
+                     redev::Channel& channel)
+    : mpi_comm(mpi_comm), channel_(channel), name_(std::move(name))
+  {
+    PCMS_FUNCTION_TIMER;
+    comm_ = channel_.CreateComm<T>(name_, mpi_comm, redev::CommType::Global);
+  }
+  GlobalCommunicator(const GlobalCommunicator&) = delete;
+  GlobalCommunicator& operator=(const GlobalCommunicator&) = delete;
+  GlobalCommunicator(GlobalCommunicator&&) = default;
+  GlobalCommunicator& operator=(GlobalCommunicator&&) = default;
+
+  void Send(T* msg, std::string VarName, size_t msg_size,
+            Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase());
+    comm_.SetCommParams(VarName, msg_size);
+    comm_.Send(msg, mode);
+  }
+  std::vector<T> Receive(std::string VarName, size_t msg_size,
+                         Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase());
+    comm_.SetCommParams(VarName, msg_size);
+    auto data = comm_.Recv(mode);
+    return data;
+  }
+
+private:
+  MPI_Comm mpi_comm;
+  redev::Channel& channel_;
+  std::string name_;
+  redev::BidirectionalComm<T> comm_;
+};
+} // namespace pcms
+#endif // PCMS_GLOBAL_COMMUNICATOR_H

test/CMakeLists.txt

@@ -96,6 +96,37 @@ if(PCMS_ENABLE_OMEGA_H)
       ${d3d16p}
       ignored)
   endif()
+  add_exe(test_GDI)
+  tri_mpi_test(
+          TESTNAME
+          test_GDI
+          TIMEOUT
+          20
+          NAME1
+          app
+          EXE1
+          ./test_GDI
+          PROCS1
+          1
+          ARGS1
+          1
+          NAME2
+          rdv
+          EXE2
+          ./test_GDI
+          PROCS2
+          1
+          ARGS2
+          -1
+          NAME3
+          app
+          EXE3
+          ./test_GDI
+          PROCS3
+          1
+          ARGS3
+          0
+  )
 
   set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/)
   add_exe(test_twoClientOverlap)
@@ -380,7 +411,7 @@ if(Catch2_FOUND)
       APPEND
       PCMS_UNIT_TEST_SOURCES
       test_error_handling.cpp
-      test_eqdsk.cpp
+            test_eqdsk.cpp
       test_uniform_grid.cpp
       test_field_evaluation.cpp
       test_field_interpolation.cpp
@@ -402,7 +433,6 @@ if(Catch2_FOUND)
       test_omega_h_lagrange_field.cpp
       test_point_evaluator.cpp)
   endif()
-
   if(PCMS_ENABLE_MESHFIELDS)
     list(APPEND PCMS_UNIT_TEST_SOURCES
       test_omega_h_form_integrator_utils.cpp)
@@ -430,15 +460,6 @@ if(Catch2_FOUND)
     target_link_libraries(unit_tests PRIVATE PETSc::PETSc)
   endif()
 
-  target_link_libraries(unit_tests PUBLIC 
-    Catch2::Catch2 
-    pcms::core
-    pcms_transfer
-    pcms_transfer
-  )
-
- target_include_directories(unit_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
-
   add_executable(test_interpolation_on_ltx_mesh test_interpolation_on_ltx_mesh.cpp)
   target_link_libraries(test_interpolation_on_ltx_mesh PUBLIC Catch2::Catch2WithMain
                                                         pcms::core