External evaluation

.gitattributes

@@ -0,0 +1,3 @@
+# SWIG output regenerated from client.i; collapsed in GitHub diffs
+src/pcms/fortranapi/client_wrap.c linguist-generated
+src/pcms/fortranapi/pcms.f90 linguist-generated

src/pcms/capi/client.cpp

@@ -4,7 +4,6 @@
 #include "pcms/field/function_space/xgc.h"
 #include "pcms/field/data/xgc.h"
 #include "pcms/field/layout/xgc.h"
-#include "pcms/coupler/serializer/xgc.h"
 #include "pcms/discretization/discretization/xgc_reverse_classification.h"
 #include "pcms/coupler/coupler.hpp"
 #include "pcms/field/layout/empty.h"
@@ -79,7 +78,7 @@ class EmptyFieldFactory : public FieldFactory
           throw pcms_error(
             "EmptyFieldFactory::CreateField: requires SimpleFieldData");
         }
-        if (fd->GetDOFHolderDataHost().size() !=
+        if (fd->GetDOFAccess()->GetOwnedValues().size() !=
             detail::ExpectedFlatFieldDataSize(*layout_)) {
           throw pcms_error(
             "EmptyFieldFactory::CreateField: field data size does not match "
@@ -132,12 +131,9 @@ ClientState::HandleVariant RegisterField(
   auto field = registration.function_space.template CreateField<T>(
     std::move(name), std::make_unique<XGCFieldData<T>>(
                        registration.function_space.GetXGCLayout(), ValueBasis{},
-                       registration.data));
-  std::unique_ptr<FieldSerializer<T>> serializer =
-    std::make_unique<XGCFieldSerializer<T>>(registration.plane_comm,
-                                            participates);
+                       registration.data, registration.plane_comm));
   return ClientState::HandleVariant{
-    app.AddField(std::move(field), std::move(serializer), participates)};
+    app.AddField(std::move(field), participates)};
 }
 
 inline ClientState::HandleVariant RegisterField(
@@ -152,10 +148,8 @@ inline ClientState::HandleVariant RegisterField(
   auto function_space =
     detail::EmptyFieldFactory{name, csys::CylindricalRZ::Create()};
   auto field = function_space.CreateField<int>(std::move(name));
-  std::unique_ptr<FieldSerializer<int>> serializer =
-    std::make_unique<FieldSerializer<int>>();
   return ClientState::HandleVariant{
-    app.AddField(std::move(field), std::move(serializer), participates)};
+    app.AddField(std::move(field), participates)};
 }
 
 } // namespace pcms
@@ -189,6 +183,18 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle rc)
   delete reinterpret_cast<pcms::ReverseClassificationVertex*>(rc.pointer);
 }
 
+void pcms_set_overlap_mask(PcmsClientHandle client_handle, const char* name,
+                           in_overlap_function in_overlap)
+{
+  auto* app =
+    reinterpret_cast<pcms::ApplicationComm*>(client_handle.appPointer);
+  PCMS_ALWAYS_ASSERT(app != nullptr);
+  PCMS_ALWAYS_ASSERT(in_overlap != nullptr);
+  // Fields registered through the C API use their name as the layout name.
+  app->SetLayoutOverlapMask(name,
+                            std::make_unique<pcms::OverlapMask>(in_overlap));
+}
+
 PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name,
                                PcmsFieldAdapterHandle adapter_handle,
                                int participates)
@@ -253,11 +259,10 @@ template <typename T>
 void pcms_create_xgc_field_adapter_t(
   const char* /* name */, MPI_Comm comm, void* data, int size,
   const pcms::ReverseClassificationVertex& reverse_classification,
-  in_overlap_function in_overlap, pcms::FieldAdapterVariant& field_adapter)
+  pcms::FieldAdapterVariant& field_adapter)
 {
   PCMS_ALWAYS_ASSERT((size > 0) ? (data != nullptr) : true);
-  auto function_space =
-    pcms::XGCFieldFactory(reverse_classification, in_overlap, size);
+  auto function_space = pcms::XGCFieldFactory(reverse_classification, size);
   pcms::Rank1View<T, pcms::HostMemorySpace> data_view(
     reinterpret_cast<T*>(data), size);
   field_adapter.emplace<pcms::detail::XGCFieldRegistration<T>>(
@@ -266,7 +271,7 @@ void pcms_create_xgc_field_adapter_t(
 
 PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
   const char* name, MPI_Comm comm, void* data, int size, PcmsType data_type,
-  const PcmsReverseClassificationHandle rc, in_overlap_function in_overlap)
+  const PcmsReverseClassificationHandle rc)
 {
   auto* field_adapter = new pcms::FieldAdapterVariant{};
   PCMS_ALWAYS_ASSERT(rc.pointer != nullptr);
@@ -275,24 +280,20 @@ PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
   PCMS_ALWAYS_ASSERT(reverse_classification != nullptr);
   switch (data_type) {
     case PCMS_DOUBLE:
-      pcms_create_xgc_field_adapter_t<double>(name, comm, data, size,
-                                              *reverse_classification,
-                                              in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<double>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_FLOAT:
-      pcms_create_xgc_field_adapter_t<float>(name, comm, data, size,
-                                             *reverse_classification,
-                                             in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<float>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_INT:
-      pcms_create_xgc_field_adapter_t<int>(name, comm, data, size,
-                                           *reverse_classification, in_overlap,
-                                           *field_adapter);
+      pcms_create_xgc_field_adapter_t<int>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_LONG_INT:
-      pcms_create_xgc_field_adapter_t<pcms::GO>(name, comm, data, size,
-                                                *reverse_classification,
-                                                in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<pcms::GO>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     default:
       PCMS_ALWAYS_ASSERT(false, MPI_COMM_WORLD,

src/pcms/capi/client.h

@@ -65,13 +65,18 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle);
 // likely to be needed for production cases
 int pcms_reverse_classification_count_verts(PcmsReverseClassificationHandle);
 
+PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
+  const char* name, MPI_Comm plane_comm, void* data, int size,
+  PcmsType data_type, const PcmsReverseClassificationHandle rc);
+
 // takes in overlap function takes a geometric dimension and a geometric id
 // C doesn't have a builtin bool type, so we use int for compatability with C++
 typedef int8_t (*in_overlap_function)(int, int);
-PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
-  const char* name, MPI_Comm plane_comm, void* data, int size,
-  PcmsType data_type, const PcmsReverseClassificationHandle rc,
-  in_overlap_function in_overlap);
+// Restricts the field registered under `name` to the DOF holders whose
+// classification satisfies in_overlap (unclassified holders are passed as
+// (-1, -1)).
+void pcms_set_overlap_mask(PcmsClientHandle client_handle, const char* name,
+                           in_overlap_function in_overlap);
 
 PcmsFieldAdapterHandle pcms_create_dummy_field_adapter();
 

src/pcms/coupler/CMakeLists.txt

@@ -17,11 +17,6 @@ set(PCMS_COUPLER_SOURCES
         field_exchange_planner.cpp
 )
 
-if(PCMS_ENABLE_XGC)
-    list(APPEND PCMS_COUPLER_HEADERS
-            serializer/xgc.h)
-endif()
-
 add_library(pcms_coupler ${PCMS_COUPLER_SOURCES})
 set_target_properties(pcms_coupler PROPERTIES
         OUTPUT_NAME pcmscoupler

src/pcms/coupler/coupler.hpp

@@ -140,36 +140,24 @@ class ApplicationComm
 
   // Register a comm-only field, identified by its own name. The layout's
   // communicator is created on first use and shared by any later
-  // field/function whose layout has the same name.
+  // field/function whose layout has the same name. The field's data must
+  // provide DOFAccess: values are exchanged through it, and receiving writes
+  // them back through it on every rank, including ranks that do not
+  // participate, so backends that keep replicas can synchronize them.
   template <typename T>
   FieldHandle<T> AddField(Field<T>&& field, bool participates = true);
-
-  template <typename T>
-  FieldHandle<T> AddField(Field<T>&& field,
-                          std::unique_ptr<FieldSerializer<T>> serializer,
-                          bool participates = true);
-
-  // Register a transferable field: like AddField but retains the function space
-  // (via the stored Function) and returns a FunctionHandle usable in transfers.
-  template <typename T>
-  FieldHandle<T> AddField(std::string name, Field<T>&& field,
-                          std::unique_ptr<FieldSerializer<T>> serializer,
-                          bool participates = true);
   // Registers a reference to user-owned data. The Application does not take
   // ownership; it uses the registered reference for subsequent send and receive
   // operations.
   template <typename T>
   DataHandle<T> AddData(std::string name, Rank1View<T, HostMemorySpace> data,
                         MPI_Comm mpi_comm);
+  // Register a transferable field: like AddField but retains the function space
+  // (via the stored Function) and returns a FunctionHandle usable in transfers.
   template <typename T>
   FunctionHandle<T> AddFunction(Function<T>&& function,
                                 bool participates = true);
 
-  template <typename T>
-  FunctionHandle<T> AddFunction(Function<T>&& function,
-                                std::unique_ptr<FieldSerializer<T>> serializer,
-                                bool participates = true);
-
   void SendField(const std::string& name,
                  redev::Mode mode = redev::Mode::Synchronous)
   {
@@ -269,9 +257,7 @@ class ApplicationComm
     const FieldLayout& layout, bool participates);
 
   template <typename T>
-  void RegisterFieldCommunicator(Field<T>& field_obj,
-                                 std::unique_ptr<FieldSerializer<T>> serializer,
-                                 bool participates);
+  void RegisterFieldCommunicator(Field<T>& field_obj, bool participates);
 
   MPI_Comm mpi_comm_;
   redev::Redev& redev_;
@@ -416,16 +402,20 @@ pcms::Field<T>& pcms::ApplicationComm::GetField(const std::string& name)
 }
 
 template <typename T>
-void pcms::ApplicationComm::RegisterFieldCommunicator(
-  Field<T>& field_obj, std::unique_ptr<FieldSerializer<T>> serializer,
-  bool participates)
+void pcms::ApplicationComm::RegisterFieldCommunicator(Field<T>& field_obj,
+                                                      bool participates)
 {
   const std::string& name = field_obj.GetName();
+  if (field_obj.GetData().GetDOFAccess() == nullptr) {
+    throw pcms_error("Application: field '" + name +
+                     "' cannot be exchanged: its values are not accessible "
+                     "through PCMS (the field data provides no DOFAccess)");
+  }
   FieldLayoutCommunicator& layout_communicator =
     GetOrCreateLayoutCommunicator(field_obj.GetLayout(), participates);
   FieldCommunicator2Ptr field_communicator =
-    std::make_unique<FieldCommunicator<T>>(name, layout_communicator, field_obj,
-                                           std::move(serializer));
+    std::make_unique<FieldCommunicator<T>>(name, layout_communicator,
+                                           field_obj);
   auto [it, inserted] =
     field_communicators_.emplace(name, std::move(field_communicator));
   if (!inserted) {
@@ -436,15 +426,6 @@ void pcms::ApplicationComm::RegisterFieldCommunicator(
 template <typename T>
 pcms::FieldHandle<T> pcms::ApplicationComm::AddField(Field<T>&& field,
                                                      bool participates)
-{
-  return AddField(std::move(field), std::make_unique<FieldSerializer<T>>(),
-                  participates);
-}
-
-template <typename T>
-pcms::FieldHandle<T> pcms::ApplicationComm::AddField(
-  Field<T>&& field, std::unique_ptr<FieldSerializer<T>> serializer,
-  bool participates)
 {
   PCMS_FUNCTION_TIMER;
   std::string name = field.GetName();
@@ -457,7 +438,7 @@ pcms::FieldHandle<T> pcms::ApplicationComm::AddField(
   }
   try {
     RegisterFieldCommunicator<T>(std::get<Field<T>>(field_it->second),
-                                 std::move(serializer), participates);
+                                 participates);
   } catch (...) {
     fields_.erase(field_it);
     throw;
@@ -468,15 +449,6 @@ pcms::FieldHandle<T> pcms::ApplicationComm::AddField(
 template <typename T>
 pcms::FunctionHandle<T> pcms::ApplicationComm::AddFunction(
   Function<T>&& function, bool participates)
-{
-  return AddFunction(std::move(function),
-                     std::make_unique<FieldSerializer<T>>(), participates);
-}
-
-template <typename T>
-pcms::FunctionHandle<T> pcms::ApplicationComm::AddFunction(
-  Function<T>&& function, std::unique_ptr<FieldSerializer<T>> serializer,
-  bool participates)
 {
   PCMS_FUNCTION_TIMER;
   std::string name = function.GetName();
@@ -490,7 +462,7 @@ pcms::FunctionHandle<T> pcms::ApplicationComm::AddFunction(
   }
   try {
     RegisterFieldCommunicator<T>(std::get<Function<T>>(fn_it->second),
-                                 std::move(serializer), participates);
+                                 participates);
   } catch (...) {
     functions_.erase(fn_it);
     throw;

src/pcms/coupler/field_communicator.hpp

@@ -21,18 +21,7 @@ class FieldCommunicator
 public:
   FieldCommunicator(const std::string& name,
                     FieldLayoutCommunicator& layout_comm, Field<T>& field)
-    : FieldCommunicator(name, layout_comm, field,
-                        std::make_unique<FieldSerializer<T>>())
-  {
-  }
-
-  FieldCommunicator(const std::string& name,
-                    FieldLayoutCommunicator& layout_comm, Field<T>& field,
-                    std::unique_ptr<FieldSerializer<T>> serializer)
-    : comm_buffer_{},
-      layout_comm_(layout_comm),
-      field_(field),
-      serializer_(std::move(serializer))
+    : comm_buffer_{}, layout_comm_(layout_comm), field_(field)
   {
     PCMS_ALWAYS_ASSERT(layout_comm.GetLayout().GetName() ==
                        field.GetLayout().GetName());
@@ -51,7 +40,7 @@ class FieldCommunicator
     PCMS_FUNCTION_TIMER;
     PCMS_ALWAYS_ASSERT(layout_comm_.GetChannel().InSendCommunicationPhase());
     auto buffer = make_array_view(comm_buffer_);
-    serializer_->Serialize(field_, buffer, layout_comm_.GetPermutationArray());
+    serializer_.Serialize(field_, buffer, layout_comm_.GetPermutationArray());
     comm_.Send(buffer.data_handle(), mode);
   }
 
@@ -63,16 +52,16 @@ class FieldCommunicator
     // mode because we make an immediate call to deserialize after a call to
     // receive.
     auto data = comm_.Recv(redev::Mode::Synchronous);
-    serializer_->Deserialize(field_, make_const_array_view(data),
-                             layout_comm_.GetPermutationArray());
+    serializer_.Deserialize(field_, make_const_array_view(data),
+                            layout_comm_.GetPermutationArray());
   }
 
 private:
   std::vector<T> comm_buffer_;
   redev::BidirectionalComm<T> comm_;
   FieldLayoutCommunicator& layout_comm_;
   Field<T>& field_;
-  std::unique_ptr<FieldSerializer<T>> serializer_;
+  FieldSerializer<T> serializer_;
 };
 
 template <typename T>

src/pcms/coupler/field_layout_communicator.cpp

@@ -1,4 +1,5 @@
 #include "pcms/coupler/field_layout_communicator.h"
+#include "pcms/field/gid_permutation.hpp"
 
 namespace pcms
 {
@@ -108,6 +109,10 @@ void FieldLayoutCommunicator::UpdateLayout()
 void FieldLayoutCommunicator::UpdateLayoutNull()
 {
   PCMS_FUNCTION_TIMER;
+  // This rank exchanges nothing, but it still unpacks on receive (backends
+  // that keep replicas synchronize them there), so every holder is unmapped.
+  plan_.permutation = MakeUnmappedPermutation(
+    static_cast<size_t>(layout_.GetNumOwnedDofHolder()));
   if (redev_.GetProcessType() == redev::ProcessType::Client) {
     channel_.BeginSendCommunicationPhase();
     channel_.EndSendCommunicationPhase();

src/pcms/coupler/field_serializer.h

@@ -10,13 +10,16 @@
 namespace pcms
 {
 
+// Packs a field's owned values into, and unpacks them from, an exchange
+// buffer, through the field's DOFAccess. Any field with DOFAccess is
+// exchanged this way. Backends that need extra work on receive (e.g.,
+// synchronizing replicas) do it in DOFAccess::SetOwnedValues.
 template <typename T>
 class FieldSerializer
 {
 public:
-  virtual int Serialize(const Field<T>& field,
-                        Rank1View<T, HostMemorySpace> buffer,
-                        Rank1View<const LO, HostMemorySpace> permutation) const
+  int Serialize(const Field<T>& field, Rank1View<T, HostMemorySpace> buffer,
+                Rank1View<const LO, HostMemorySpace> permutation) const
   {
     const auto data = field.GetDOFHolderDataHost().GetValues();
     if (buffer.size() > 0) {
@@ -33,9 +36,8 @@ class FieldSerializer
     return static_cast<int>(buffer.size());
   }
 
-  virtual void Deserialize(
-    Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
-    Rank1View<const LO, HostMemorySpace> permutation) const
+  void Deserialize(Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
+                   Rank1View<const LO, HostMemorySpace> permutation) const
   {
     const auto& layout = field.GetLayout();
     const LO num_dof = layout.GetNumOwnedDofHolder();
@@ -62,8 +64,6 @@ class FieldSerializer
     field.SetDOFHolderDataUncheckedHost(
       Rank2View<const T, HostMemorySpace>(sorted.data(), num_dof, num_comp));
   }
-
-  virtual ~FieldSerializer() noexcept = default;
 };
 
 } // namespace pcms