Merge pull request #3309 from boutproject/parallel-regions

valid regions for parallel slices

Author: bendudson

include/bout/field3d.hxx

@@ -331,6 +331,7 @@ public:
   const Region<Ind3D>& getValidRegionWithDefault(const std::string& region_name) const;
   void setRegion(const std::string& region_name) override;
   void resetRegion() override { regionID.reset(); };
+  void resetRegionParallel();
   void setRegion(size_t id) override { regionID = id; };
   void setRegion(std::optional<size_t> id) override { regionID = id; };
   std::optional<size_t> getRegionID() const override { return regionID; };

src/field/field3d.cxx

@@ -145,6 +145,7 @@ void Field3D::splitParallelSlices() {
     yup_fields.emplace_back(fieldmesh);
     ydown_fields.emplace_back(fieldmesh);
   }
+  resetRegionParallel();
 }
 
 void Field3D::clearParallelSlices() {
@@ -841,6 +842,15 @@ void Field3D::setRegion(const std::string& region_name) {
   regionID = fieldmesh->getRegionID(region_name);
 }
 
+void Field3D::resetRegionParallel() {
+  if (isFci()) {
+    for (int i = 0; i < fieldmesh->ystart; ++i) {
+      yup_fields[i].setRegion(fmt::format("RGN_YPAR_{:+d}", i + 1));
+      ydown_fields[i].setRegion(fmt::format("RGN_YPAR_{:+d}", -i - 1));
+    }
+  }
+}
+
 Field3D& Field3D::enableTracking(const std::string& name,
                                  std::weak_ptr<Options> _tracking) {
   tracking = std::move(_tracking);

src/mesh/mesh.cxx

@@ -679,6 +679,12 @@ void Mesh::createDefaultRegions() {
                                 + getRegion3D("RGN_YGUARDS") + getRegion3D("RGN_ZGUARDS"))
                                    .unique());
 
+  for (int offset_ = -ystart; offset_ <= ystart; ++offset_) {
+    const auto region = fmt::format("RGN_YPAR_{:+d}", offset_);
+    addRegion3D(region, Region<Ind3D>(xstart, xend, ystart + offset_, yend + offset_, 0,
+                                      LocalNz - 1, LocalNy, LocalNz));
+  }
+
   //2D regions
   addRegion2D("RGN_ALL", Region<Ind2D>(0, LocalNx - 1, 0, LocalNy - 1, 0, 0, LocalNy, 1,
                                        maxregionblocksize));

src/mesh/parallel/fci.cxx

@@ -261,16 +261,6 @@ FCIMap::FCIMap(Mesh& mesh, [[maybe_unused]] const Coordinates::FieldMetric& dy,
   region_no_boundary = region_no_boundary.mask(to_remove);
 
   interp->setRegion(region_no_boundary);
-
-  const auto region = fmt::format("RGN_YPAR_{:+d}", offset_);
-  if (not map_mesh->hasRegion3D(region)) {
-    // The valid region for this slice
-    map_mesh->addRegion3D(region, Region<Ind3D>(map_mesh->xstart, map_mesh->xend,
-                                                map_mesh->ystart + offset_,
-                                                map_mesh->yend + offset_, 0,
-                                                map_mesh->LocalNz - 1, map_mesh->LocalNy,
-                                                map_mesh->LocalNz));
-  }
 }
 
 Field3D FCIMap::integrate(Field3D& f) const {