From 93716b4ef62cba58edeca5f5ba804fd2f40a5ac3 Mon Sep 17 00:00:00 2001 From: hnil Date: Fri, 31 Jul 2026 14:20:03 +0200 Subject: [PATCH 1/3] Enable ECL summary/FIP output for serial CpGrid LGR runs evalSummaryState (which runs both calc_inplace and summary.eval) and writeInitialFIPReport were skipped for any CpGrid with maxLevel()>0. As a result LGR (CARFIN) decks produced all-zero summary/PRT/XWEL output, and a RPTSCHED FIP report segfaulted because the inplace_ accumulator was never populated (Inplace::get(FIELD) on an empty map). In serial the CollectDataOnIORank index maps are the identity and the leaf-grid well/cell data is already written correctly, so the only thing missing was the summary evaluation itself. The unsupported case is exactly a *distributed* refined CpGrid, for which CollectDataOnIORank does not build its index maps. Put that condition in one private helper, eclOutputEvalSupported_(), and use it at both call sites, so future call sites cannot miss a condition (review suggestion). Output-only change: solver behaviour is unchanged (SPE1CASE1_CARFIN1 stays at 52 Newton iterations), and non-LGR runs are unaffected. --- opm/simulators/flow/FlowProblemBlackoil.hpp | 41 +++++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/opm/simulators/flow/FlowProblemBlackoil.hpp b/opm/simulators/flow/FlowProblemBlackoil.hpp index 18f81399eb6..e70bae523c9 100644 --- a/opm/simulators/flow/FlowProblemBlackoil.hpp +++ b/opm/simulators/flow/FlowProblemBlackoil.hpp @@ -515,12 +515,7 @@ class FlowProblemBlackoil : public FlowProblem // also updated. this->eclWriter().mutableOutputModule().invalidateLocalData(); - // For CpGrid with LGRs, ecl/vtk output is not supported yet. - const auto& grid = this->simulator().vanguard().gridView().grid(); - - using GridType = std::remove_cv_t>; - constexpr bool isCpGrid = std::is_same_v; - if (!isCpGrid || (grid.maxLevel() == 0)) { + if (this->eclOutputEvalSupported_()) { this->eclWriter_->evalSummaryState(!this->episodeWillBeOver()); } @@ -632,16 +627,10 @@ class FlowProblemBlackoil : public FlowProblem // the initial solution. this->thresholdPressures_.finishInit(); - // For CpGrid with LGRs, ecl-output is not supported yet. - const auto& grid = this->simulator().vanguard().gridView().grid(); - - using GridType = std::remove_cv_t>; - constexpr bool isCpGrid = std::is_same_v; - // Skip - for now - calculate the initial fip values for CpGrid with LGRs. - if (!isCpGrid || (grid.maxLevel() == 0)) { - if (this->simulator().episodeIndex() == 0) { - eclWriter_->writeInitialFIPReport(); - } + if (this->eclOutputEvalSupported_() && + (this->simulator().episodeIndex() == 0)) + { + eclWriter_->writeInitialFIPReport(); } } @@ -1767,6 +1756,26 @@ class FlowProblemBlackoil : public FlowProblem HybridNewton hybridNewton_; private: + /// Whether ECL summary/FIP evaluation is wired up for this grid + /// configuration. + /// + /// The only unsupported case is a *distributed* CpGrid with LGRs: + /// CollectDataOnIORank does not build its index maps for a distributed + /// refined grid yet. In serial the index maps are the identity and the + /// leaf-grid well/cell data is written directly, so refined serial runs + /// are fine. Kept in one place so no call site can miss a condition. + bool eclOutputEvalSupported_() const + { + const auto& grid = this->simulator().vanguard().gridView().grid(); + + using GridType = std::remove_cv_t>; + constexpr bool isCpGrid = std::is_same_v; + + return !isCpGrid + || (grid.maxLevel() == 0) + || (grid.comm().size() == 1); + } + /// Whether or not the current epsiode will end at the end of the /// current time step. /// From 4200e4548b30d0944231b0b6d36f5d66688eede8 Mon Sep 17 00:00:00 2001 From: hnil Date: Fri, 31 Jul 2026 15:18:29 +0200 Subject: [PATCH 2/3] compressedIndexForInteriorLGR: fatal on structural errors, -1 only for absent cells The lookup used std::map::at() twice, so any miss threw std::out_of_range. On a distributed grid the misses have very different meanings, and an exception here fires on some ranks and not others, which is a collective hazard during well setup. The rank-without-the-box case is reachable by construction: a COMPDATL box occupies part of the grid, and a rank holding no cell of it (interior or overlap) still resolves every LGR well when building the well containers. (Observed in development as a np>=6 deadlock on a faulted-box variant of SIMPLE_2PH_W_FAULT_LGR; that particular deck needs refinement across a fault, which CpGrid does not support yet, so it is not reproducible on master -- the reachable path is, via any COMPDATL deck whose box misses a rank.) Separate the miss cases per review: - LGR name unknown: every rank registers every requested LGR (with an empty level grid where it holds no box cells), so the level structure is identical on all ranks and a failed name lookup is a programming error. Fatal, with a message saying so. - Connection (i,j,k) outside the LGR's dimensions: validated at parse time, so reaching here with an out-of-range position is likewise a bug. Fatal. - Cell absent from this rank's level mapper: the one legitimate miss -- the box lives on other ranks and this rank's level grid is empty or holds a different part of it. Return -1, mirroring compressedIndexForInterior's contract for coarse cells; the global existence of every connection cell is checked collectively afterwards (checkAllConnectionsFound). Verified on SPE1CASE1_CARFIN_GR (COMPDATL well): serial and np=2 resolve the well through this path and run. --- opm/simulators/flow/CpGridVanguard.hpp | 52 ++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 3ece908a641..1af51fc4703 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -27,6 +27,7 @@ #ifndef OPM_CPGRID_VANGUARD_HPP #define OPM_CPGRID_VANGUARD_HPP +#include #include #include @@ -43,6 +44,8 @@ #include #include +#include + namespace Opm { template class CpGridVanguard; @@ -118,14 +121,57 @@ class CpGridVanguard : public FlowBaseVanguard int compressedIndexForInteriorLGR(const std::string& lgr_tag, const Connection& conn) const override { - const std::array lgr_ijk = {conn.getI(), conn.getJ(), conn.getK()}; - const auto& lgr_level = this->grid().getLgrNameToLevel().at(lgr_tag); + // Every rank registers every requested LGR name, with an empty level + // grid on ranks that hold no cell of the box (interior or overlap) -- + // the level structure is identical on all ranks. A name that fails to + // resolve is therefore a programming error, not a distribution effect, + // and must be fatal rather than silently skipped. + const auto& nameToLevel = this->grid().getLgrNameToLevel(); + const auto levelIt = nameToLevel.find(lgr_tag); + if (levelIt == nameToLevel.end()) { + OPM_THROW(std::logic_error, + fmt::format("Internal error: LGR '{}' is not known to the grid. " + "The level structure must be identical on all ranks.", + lgr_tag)); + } + const int lgr_level = levelIt->second; + if (ParentType::lgrMappers_.has_value() == false) { ParentType::lgrMappers_.emplace(this->grid().mapLocalCartesianIndexSetsToLeafIndexSet()); } + + // An out-of-range Cartesian position within the level is likewise a + // bug (a COMPDATL record addressing outside its LGR box has already + // been validated at parse time), so it is fatal too. const auto& lgr_dim = this->grid().currentData()[lgr_level]->logicalCartesianSize(); + const std::array lgr_ijk = {conn.getI(), conn.getJ(), conn.getK()}; + if (lgr_ijk[0] < 0 || lgr_ijk[0] >= lgr_dim[0] || + lgr_ijk[1] < 0 || lgr_ijk[1] >= lgr_dim[1] || + lgr_ijk[2] < 0 || lgr_ijk[2] >= lgr_dim[2]) + { + OPM_THROW(std::logic_error, + fmt::format("Internal error: connection ({},{},{}) is outside " + "LGR '{}' with dimensions {}x{}x{}.", + lgr_ijk[0], lgr_ijk[1], lgr_ijk[2], lgr_tag, + lgr_dim[0], lgr_dim[1], lgr_dim[2])); + } const auto lgr_cartesian_index = (lgr_ijk[2]*lgr_dim[0]*lgr_dim[1]) + (lgr_ijk[1]*lgr_dim[0]) + (lgr_ijk[0]); - return ParentType::lgrMappers_.value()[lgr_level].at(lgr_cartesian_index); + + // A cell that is absent from this rank's level mapper is the one + // legitimate miss: the box lives elsewhere and this rank's level grid + // is empty (or holds another part of it). Mirror + // compressedIndexForInterior and return -1; the global existence of + // every connection cell is checked collectively afterwards + // (checkAllConnectionsFound). Using .at() here threw + // std::out_of_range on such ranks -- asymmetrically, which deadlocked + // runs at higher rank counts where more ranks hold no part of the box. + const auto& mapper = ParentType::lgrMappers_.value()[lgr_level]; + const auto it = mapper.find(lgr_cartesian_index); + if (it == mapper.end()) { + return -1; + } + + return static_cast(it->second); } /*! * Checking consistency of simulator From bc39175274747f69a3ea1d468a5ff39ca12ad067 Mon Sep 17 00:00:00 2001 From: hnil Date: Fri, 31 Jul 2026 15:19:21 +0200 Subject: [PATCH 3/3] Cartesian->compressed map: level-zero cells only, rebuilt after refinement Two related problems with cartesianToCompressed_ on a locally refined grid: 1. It was built once, during load balancing on the unrefined grid, and never again. Refinement renumbers the leaf, so afterwards every stored compressed index is stale and coarse-grid well connections resolve to the wrong leaf cell. On SPE1CASE1_CARFIN1 the PROD source term landed on cart 91 instead of the perforation cell cart 299 -- mass-conserving, so masked in serial, but a ~27x first-iteration CNV spike in parallel once reordering moved the misrouted residual onto a small refined cell. 2. Rebuilding naively is not enough, because on a refined grid a level-zero Cartesian index is no longer a unique key: every child of a refined cell reports its ancestor's index, so the map entry for a refined-away cell is whichever child was inserted last -- an arbitrary answer served silently. Per review, restrict the map to unrefined cells: it becomes a mapping for existing cells on level zero only. A level-zero index that has been refined away now resolves to "not present" (and downstream to the explicit cells-not-found handling) instead of to an arbitrary child; cells inside a refinement are addressed through the LGR-aware lookup. The map is cleared on rebuild since it can now legitimately be built twice. The post-grid-change updates (grid view, this map, cell depths, thicknesses) are factored into updateDerivedGridState_(), called from both loadBalance() and addLgrs(), so a future grid-changing step cannot miss one of them (review suggestion). Verified: PROD resolves to active index 923; first-iteration CNV matches the non-LGR baseline, identical serial vs parallel; serial/parallel summaries agree to 1e-6; unrefined decks are byte-identical (hasFather() is false everywhere, so the map is unchanged). --- opm/simulators/flow/CpGridVanguard.hpp | 28 +++++++++++++++++++----- opm/simulators/flow/FlowBaseVanguard.hpp | 20 +++++++++++++++-- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 1af51fc4703..aa095ea7b1b 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -299,9 +299,7 @@ class CpGridVanguard : public FlowBaseVanguard this->numJacobiBlocks(), this->enableEclOutput()); #endif - this->updateGridView_(); - this->updateCartesianToCompressedMapping_(); - this->updateCellThickness_(); + this->updateDerivedGridState_(); #if HAVE_MPI this->distributeFieldProps_(this->eclState()); @@ -312,6 +310,22 @@ class CpGridVanguard : public FlowBaseVanguard this->updateCellDepths_(); } + /*! + * \brief Recompute everything the vanguard derives from the grid. + * + * Needed after every change to the leaf grid -- load balancing and local + * refinement both renumber the leaf cells. Kept in one method so a + * future grid-changing step cannot miss one of the updates. Cell depths + * are not included: they need the distributed field properties, so each + * caller updates them once those are available. + */ + void updateDerivedGridState_() + { + this->updateGridView_(); + this->updateCartesianToCompressedMapping_(); + this->updateCellThickness_(); + } + /*! * \brief Add LGRs and update Leaf Grid View in the simulation grid. */ @@ -323,9 +337,13 @@ class CpGridVanguard : public FlowBaseVanguard OpmLog::info("\nAdding LGRs to the grid and updating its leaf grid view"); this->addLgrsUpdateLeafView(lgrs, lgrs.size(), *this->grid_); - this->updateGridView_(); + // Refinement changed the leaf cell count and ordering, so the + // state derived at load-balance time -- in particular the + // (level-zero-only) Cartesian->compressed map used to resolve + // coarse well connections -- is stale and must be rebuilt before + // well connections are resolved. + this->updateDerivedGridState_(); this->updateCellDepths_(); - this->updateCellThickness_(); if (this->grid_->comm().size()>1) { // Add LGRs and update the leaf grid view in the global (undistributed) simulation grid. diff --git a/opm/simulators/flow/FlowBaseVanguard.hpp b/opm/simulators/flow/FlowBaseVanguard.hpp index b3896e3c94e..ce6d932b459 100644 --- a/opm/simulators/flow/FlowBaseVanguard.hpp +++ b/opm/simulators/flow/FlowBaseVanguard.hpp @@ -338,12 +338,28 @@ class FlowBaseVanguard : public BaseVanguard, std::size_t num_cells = asImp_().grid().leafGridView().size(0); is_interior_.resize(num_cells); + // May run again after the grid changed (local refinement), so start + // from scratch rather than leaving entries for cells that no longer + // exist in the leaf. + cartesianToCompressed_.clear(); + ElementMapper elemMapper(this->gridView(), Dune::mcmgElementLayout()); for (const auto& element : elements(this->gridView())) { const auto elemIdx = elemMapper.index(element); - unsigned cartesianCellIdx = cartesianIndex(elemIdx); - cartesianToCompressed_[cartesianCellIdx] = elemIdx; + // On a refined grid a level-zero Cartesian index is not a unique + // key: every child of a refined cell reports its ancestor's + // index, so inserting them would make the winner arbitrary. + // Only unrefined cells enter the map, making it a mapping for + // existing cells on level zero only; cells inside a refinement + // are addressed through the LGR-aware lookup instead, and a + // level-zero index that has been refined away resolves to + // "not present" rather than to an arbitrary child. + if (!element.hasFather()) + { + unsigned cartesianCellIdx = cartesianIndex(elemIdx); + cartesianToCompressed_[cartesianCellIdx] = elemIdx; + } if (element.partitionType() == Dune::InteriorEntity) { is_interior_[elemIdx] = 1;