From 97fd2720c88fa4241c8c7caaec5e038badccb75e Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 25 Sep 2026 10:38:04 +0200 Subject: [PATCH 1/4] feat(csr): one read-only Model(sparse=True) key for the sparse path (#976) Model(sparse=True) turns on sparse groupby-sum, sparse @ and frozen constraints for one model; v1 only, raises under legacy and with chunk; kept by copy and netcdf. Deprecate Model(freeze_constraints=...), its setter, sum(sparse=...) and options['sparse_groupby']; @ no longer reads the option. --- benchmarks/patterns/nodal_balance.py | 8 +- doc/api.rst | 2 +- doc/release_notes.rst | 5 + examples/creating-constraints.ipynb | 8 +- linopy/config.py | 11 ++ linopy/csr.py | 4 +- linopy/expressions.py | 27 +++-- linopy/io.py | 7 +- linopy/model.py | 69 +++++++++++-- pyproject.toml | 2 + test/test_csr.py | 147 ++++++++++++++++++++++++--- 11 files changed, 248 insertions(+), 42 deletions(-) diff --git a/benchmarks/patterns/nodal_balance.py b/benchmarks/patterns/nodal_balance.py index 4cc2e6884..4eafb3d84 100644 --- a/benchmarks/patterns/nodal_balance.py +++ b/benchmarks/patterns/nodal_balance.py @@ -9,7 +9,7 @@ current (dense) kernel. ``nodal_balance_sparse`` builds the identical constraint through the -CSR-backed path (``sum(sparse=True)`` + ``freeze=True`` under v1): the +CSR-backed path (``Model(sparse=True)`` under v1): the grouped sum never materializes the padded rectangle and is realized directly as a CSRConstraint, so its peak memory should stay flat across the severity sweep — the pair makes the padding cost visible. @@ -62,15 +62,15 @@ def _build(severity: int, sparse: bool) -> linopy.Model: buses = pd.RangeIndex(N_BUS, name="bus") rng = np.random.default_rng(1) - m = linopy.Model() + m = linopy.Model(sparse=sparse) gen = m.add_variables(lower=0, coords=[gens, time], name="gen") bus_of_gen = pd.Series(_bus_of_gen(severity), index=gens, name="bus") - supply = (1 * gen).groupby(bus_of_gen).sum(sparse=sparse) + supply = (1 * gen).groupby(bus_of_gen).sum() demand = xr.DataArray( rng.uniform(10.0, 100.0, size=(N_BUS, N_TIME)), coords=[buses, time] ) - m.add_constraints(supply == demand, name="balance", freeze=sparse) + m.add_constraints(supply == demand, name="balance") m.add_objective(gen.sum()) return m diff --git a/doc/api.rst b/doc/api.rst index 5a0aafa26..c27caebd7 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -411,7 +411,7 @@ CSRConstraint ============= Memory-efficient, immutable constraint representation backed by a scipy -CSR sparse matrix. Opt in via ``Model(freeze_constraints=True)`` or +CSR sparse matrix. Opt in via ``Model(sparse=True)`` or ``Model.add_constraints(..., freeze=True)``. See the :doc:`creating-constraints` guide for usage. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index dccad9b82..98cbd965f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -35,6 +35,11 @@ Upcoming Version * A frozen ``CSRConstraint`` is softened natively, without densifying: the slack terms are appended to its sparse rows in place, so ``add_constraints(..., penalty=..., freeze=True)`` works as well. The slack is kept by ``to_dense()``/``mutable()``, ``Constraint.freeze()`` and the netcdf round trip. (`#975 `__) * The slack variable(s) created by ``soften()`` can be retrieved afterwards via the new ``slack`` property of ``Constraint`` and ``CSRConstraint``. +*Sparse model* + +* New read-only model key ``Model(sparse=True)`` turns on the whole sparse (CSR) path for one model: ``groupby(...).sum()`` and ``@``/``dot`` against a constant return CSR-backed expressions, and ``add_constraints`` freezes every constraint unless ``freeze=False`` is passed. It requires the v1 semantics: ``Model(sparse=True)`` raises under legacy, and so do ``groupby(...).sum()`` and ``@`` on a sparse model, and ``read_netcdf`` of one, once the semantics are switched back to legacy. It rejects ``chunk``. The key is kept by ``Model.copy`` and the netcdf round trip. ``@`` no longer reads ``linopy.options["sparse_groupby"]``. (`#976 `__) +* Deprecated in favour of ``Model(sparse=True)``, each with a ``FutureWarning`` and to be removed with the legacy semantics: ``Model(freeze_constraints=...)`` and the ``Model.freeze_constraints`` setter, ``groupby(...).sum(sparse=...)`` and ``linopy.options["sparse_groupby"]``. They keep their current behaviour until then, except that ``@`` ignores ``sparse_groupby``, and netcdf files that store ``freeze_constraints`` still load. (`#976 `__) + *Other* * New method :meth:`linopy.Model.assign_coords` reassigns coordinate values across an existing model — variables, constraints (dense and CSR-backed), expressions and parameters — without changing the model's shape: ``m.assign_coords(snapshot=new_snapshots)``. Values-only: the new values must match the length of the dimension's full-index container, and containers holding subsets of the dimension are mapped by label, preserving the subset relation. Dataset variable order is preserved. Under v1 semantics, ``Model.solve()`` raises when containers carry labels on a shared dimension that are neither equal nor subsets of one another. Typical use is advancing the window in rolling-horizon optimization with the persistent solver interface. (https://github.com/PyPSA/linopy/issues/767) diff --git a/examples/creating-constraints.ipynb b/examples/creating-constraints.ipynb index 2827202d7..64a64afad 100644 --- a/examples/creating-constraints.ipynb +++ b/examples/creating-constraints.ipynb @@ -294,9 +294,9 @@ "id": "27", "metadata": {}, "source": [ - "### Freezing all constraints globally\n", + "### Building a sparse model\n", "\n", - "Set `freeze_constraints=True` on the `Model` to automatically freeze every constraint added via `add_constraints`:" + "Set `sparse=True` on the `Model` to build it on the sparse path: every constraint added via `add_constraints` is frozen, and `groupby(...).sum()` and `@` return CSR-backed expressions. Pass `freeze=False` to keep a single constraint mutable. The setting is fixed at creation and requires the v1 semantics:" ] }, { @@ -306,7 +306,7 @@ "metadata": {}, "outputs": [], "source": [ - "m3 = Model(freeze_constraints=True)\n", + "m3 = Model(sparse=True)\n", "z = m3.add_variables(coords=[np.arange(50)], name=\"z\")\n", "m3.add_constraints(z >= 0, name=\"lower\")\n", "m3.add_constraints(z <= 100, name=\"upper\")\n", @@ -382,7 +382,7 @@ "Additionally, if you don't need variable and constraint names in the solver (e.g. for batch solves), you can disable name export for extra speed:\n", "\n", "```python\n", - "m = Model(freeze_constraints=True, set_names_in_solver_io=False)\n", + "m = Model(sparse=True, set_names_in_solver_io=False)\n", "```" ] } diff --git a/linopy/config.py b/linopy/config.py index 53541c829..abc6f2429 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -12,6 +12,10 @@ LEGACY_SEMANTICS = "legacy" V1_SEMANTICS = "v1" VALID_SEMANTICS = {LEGACY_SEMANTICS, V1_SEMANTICS} +SPARSE_DEPRECATION = ( + "is deprecated and will be removed with the legacy semantics; use " + "Model(sparse=True) instead" +) class LinopySemanticsWarning(FutureWarning): @@ -50,6 +54,13 @@ def set_value(self, **kwargs: Any) -> None: f"Invalid semantics: {v!r}. " f"Must be one of {sorted(VALID_SEMANTICS)}." ) + if k == "sparse_groupby" and v: + from linopy.semantics import warn_outside_linopy + + warn_outside_linopy( + f'linopy.options["sparse_groupby"] {SPARSE_DEPRECATION}.', + FutureWarning, + ) self._current_values[k] = v def get_value(self, name: str) -> Any: diff --git a/linopy/csr.py b/linopy/csr.py index 38901a367..e20ac1c4b 100644 --- a/linopy/csr.py +++ b/linopy/csr.py @@ -1,8 +1,8 @@ """ The sparse backing of a LinearExpression: ``A @ x + c`` in CSR form. -Under v1 semantics, ``expr.groupby(g).sum(sparse=True)`` (or -``linopy.options["sparse_groupby"]``) returns an ordinary +In a sparse model (``Model(sparse=True)``, v1 semantics only), +``expr.groupby(g).sum()`` returns an ordinary :class:`~linopy.expressions.LinearExpression` backed by a :class:`CSRLinearExpression`: same public type, different backing, akin to dask-backed xarray objects. The CSR form is canonical (duplicate variables diff --git a/linopy/expressions.py b/linopy/expressions.py index b4b19bc28..3437dbfca 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -94,6 +94,7 @@ to_polars, ) from linopy.config import ( + SPARSE_DEPRECATION, options, ) from linopy.constants import ( @@ -127,6 +128,7 @@ join_fill, reindex_like_if_needed, warn_legacy, + warn_outside_linopy, ) from linopy.types import ( CONSTANT_TYPES, @@ -571,6 +573,7 @@ def sum( default one. Kept as an escape hatch. Leave at False unless the default misbehaves. Defaults to False. sparse : bool, optional + Deprecated, build the model with ``Model(sparse=True)`` instead. Build the grouped sum in CSR form behind the ordinary LinearExpression type — no group-size padding; a still-sparse lhs reaching ``Model.add_constraints`` with ``freeze=True`` @@ -579,7 +582,8 @@ def sum( Series/DataFrame, 1-D DataArray and coordinate-name-list groupers over an existing dimension; with a name list and ``observed=True`` the CSR result stays compact over the observed key combinations. - Requires v1 semantics. Defaults to + Requires v1 semantics. Defaults to True for a sparse model, a + CSR-backed expression or the deprecated ``linopy.options["sparse_groupby"]``. See :mod:`linopy.csr`. observed : bool Only applies when grouping by a list of coordinate names. If True, @@ -607,10 +611,17 @@ def sum( multikey_frame = None if use_fallback else _multikey_value_frame(group, labels) + self.model._check_sparse_semantics() csr = self._csr explicit_sparse = sparse is True + if sparse is not None: + warn_outside_linopy( + f"groupby(...).sum(sparse=...) {SPARSE_DEPRECATION}.", FutureWarning + ) if sparse is None: - sparse = is_v1() and (options["sparse_groupby"] or csr is not None) + sparse = is_v1() and ( + self.model.sparse or options["sparse_groupby"] or csr is not None + ) elif sparse and not is_v1(): raise ValueError( "sparse groupby-sum requires v1 semantics; opt in with " @@ -653,6 +664,8 @@ def sum( "DataArray or list of coordinate names as grouper over an " "existing dimension, without use_fallback." ) + if csr is None: + _densify_notice("groupby-sum with a grouper without a sparse path") if multikey_frame is not None: group = multikey_frame @@ -1586,7 +1599,8 @@ def dot(self, other: SideLike) -> Self | QuadraticExpression: Identical to ``@``. For a :class:`LinearExpression` that includes the sparse contraction under v1; :class:`QuadraticExpression` always takes - the dense path. There is no per-call ``sparse=`` flag. + the dense path. The result is CSR-backed for a sparse model + (``Model(sparse=True)``) or a CSR-backed input. """ return self.__matmul__(other) @@ -2829,8 +2843,9 @@ def __matmul__( form -- duplicate variables summed, terms label-ordered, explicit zeros pruned -- so its term count may differ from the dense path's while the values agree. Returns a CSR-backed result when ``self`` is - CSR-backed. + CSR-backed or the model is sparse. """ + self.model._check_sparse_semantics() other = as_constant(other) other_is_const = not isinstance(other, LinearExpression | variables.Variable) if other_is_const and is_v1() and type(self) is LinearExpression: @@ -2857,7 +2872,7 @@ def _sparse_matmul(self, other: ConstantLike) -> LinearExpression | None: grid, and an unlabelled output dimension. The result is the compact canonical form of :meth:`CSRLinearExpression.contracted`, so its term count may differ from the dense path's while the values agree; the - result is CSR-backed when the input was. + result is CSR-backed when the input was or the model is sparse. """ if is_nan_scalar(other): check_user_nan(op_kind="mul") @@ -2890,7 +2905,7 @@ def _sparse_matmul(self, other: ConstantLike) -> LinearExpression | None: ) new_indexes = [da.indexes[d].rename(d) for d in new_dims] res = csr.contracted(matrix, contracted, new_indexes) - if self._csr is not None or options["sparse_groupby"]: + if self._csr is not None or self.model.sparse: return type(self)._from_csr(res, self.model) return res.to_dense() diff --git a/linopy/io.py b/linopy/io.py index 4d008979a..4bf23d7bc 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -1099,6 +1099,7 @@ def with_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: params = [with_prefix(m.parameters, "parameters")] scalars = {k: getattr(m, k) for k in m.scalar_attrs} + scalars |= {"sparse": m.sparse, "freeze_constraints": m.freeze_constraints} ds = xr.merge(vars + cons + exprs + obj + params, combine_attrs="drop_conflicts") ds = ds.assign_attrs(scalars) ds.attrs[NETCDF_VERSION_ATTR] = version("linopy") @@ -1279,6 +1280,9 @@ def container_names(kind: str) -> list[str]: for k in m.scalar_attrs: if k in ds.attrs: setattr(m, k, ds.attrs[k]) + m._sparse = bool(ds.attrs.get("sparse", False)) + m._freeze_constraints = bool(ds.attrs.get("freeze_constraints", False)) + m._check_sparse_semantics() if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: m._dtypes["labels"] = np.int64 @@ -1354,7 +1358,6 @@ def copy(m: Model, include_solution: bool = False, deep: bool = True) -> Model: chunk=m._chunk, force_dim_names=m._force_dim_names, auto_mask=m._auto_mask, - freeze_constraints=m.freeze_constraints, set_names_in_solver_io=m.set_names_in_solver_io, solver_dir=str(m._solver_dir), ) @@ -1438,6 +1441,8 @@ def _copy_con(name: str, con: ConstraintBase) -> ConstraintBase: for attr in m.scalar_attrs: if include_solution or attr not in SOLVE_STATE_ATTRS: setattr(new_model, attr, getattr(m, attr)) + new_model._sparse = m.sparse + new_model._freeze_constraints = m.freeze_constraints if m._sos_reformulation_state is not None: new_model._sos_reformulation_state = _copy.deepcopy(m._sos_reformulation_state) diff --git a/linopy/model.py b/linopy/model.py index ec6ebd10a..1d358530c 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -41,6 +41,7 @@ replace_by_map, to_path, ) +from linopy.config import SPARSE_DEPRECATION from linopy.constants import ( FACTOR_DIM, GREATER_EQUAL, @@ -90,7 +91,7 @@ ) from linopy.remote import RemoteHandler from linopy.scaling import validate_scaling -from linopy.semantics import enforce_no_multiindex, is_v1 +from linopy.semantics import enforce_no_multiindex, is_v1, warn_outside_linopy try: from linopy.remote import OetcHandler @@ -171,6 +172,7 @@ class Model: _chunk: T_Chunks _force_dim_names: bool _freeze_constraints: bool + _sparse: bool _set_names_in_solver_io: bool _solver_dir: Path __slots__ = ( @@ -199,6 +201,7 @@ class Model: "_force_dim_names", "_auto_mask", "_freeze_constraints", + "_sparse", "_set_names_in_solver_io", "_solver_dir", "_relaxed_registry", @@ -233,9 +236,10 @@ def __init__( chunk: T_Chunks = None, force_dim_names: bool = False, auto_mask: bool = False, - freeze_constraints: bool = False, + freeze_constraints: bool | None = None, set_names_in_solver_io: bool = True, dtypes: Mapping[DtypeKey, type[np.signedinteger]] | None = None, + sparse: bool = False, ) -> None: """ Initialize the linopy model. @@ -259,9 +263,11 @@ def __init__( Whether to automatically mask variables and constraints where bounds, coefficients, or RHS values contain NaN. The default is False. - freeze_constraints : bool - Whether constraints added to the model should be frozen to the - CSR-backed representation by default. The default is False. + freeze_constraints : bool, optional + Deprecated, use ``sparse=True`` instead. Whether constraints added + to the model are frozen to the CSR-backed representation by + default, without making expressions sparse. Raises together with + ``sparse=True``. The default is False. set_names_in_solver_io : bool Whether direct solver exports should include variable and constraint names by default. The default is True. @@ -272,11 +278,26 @@ def __init__( halves label memory but caps the model at ~2.1 billion labels, after which it widens to ``np.int64`` automatically; pass ``np.int64`` upfront to avoid that mid-build upcast. + sparse : bool + Build the model on the sparse (CSR) path, exposed read-only as + ``Model.sparse``. ``groupby(...).sum()`` and ``@``/``dot`` against + a constant return CSR-backed expressions, and ``add_constraints`` + freezes constraints to a CSRConstraint unless ``freeze=False`` is + passed. Requires v1 semantics and no ``chunk``. See + :mod:`linopy.csr`. The default is False. Returns ------- linopy.Model """ + if sparse: + if not is_v1(): + raise ValueError( + "Model(sparse=True) requires v1 semantics; opt in with " + "linopy.options['semantics'] = 'v1'." + ) + if chunk: + raise ValueError("Model(sparse=True) does not support `chunk`.") self._dtypes: dict[DtypeKey, type[np.signedinteger]] = self._resolve_dtypes( dtypes ) @@ -299,7 +320,10 @@ def __init__( self._chunk: T_Chunks = chunk self._force_dim_names: bool = bool(force_dim_names) self._auto_mask: bool = bool(auto_mask) - self._freeze_constraints: bool = bool(freeze_constraints) + self._sparse: bool = bool(sparse) + self._freeze_constraints: bool = self._sparse + if freeze_constraints is not None: + self.freeze_constraints = freeze_constraints self._set_names_in_solver_io: bool = bool(set_names_in_solver_io) self._piecewise_formulations: dict[str, PiecewiseFormulation] = {} self._relaxed_registry: dict[str, str] = {} @@ -490,6 +514,8 @@ def chunk(self, value: T_Chunks) -> None: """ Set the chunk sizes of the model. """ + if value and self._sparse: + raise ValueError("A sparse model does not support `chunk`.") self._chunk = value @property @@ -534,8 +560,34 @@ def freeze_constraints(self) -> bool: @freeze_constraints.setter def freeze_constraints(self, value: bool) -> None: + if self._sparse: + raise ValueError( + "A sparse model freezes constraints by default; pass `freeze=` " + "to add_constraints instead." + ) + warn_outside_linopy( + f"Model.freeze_constraints {SPARSE_DEPRECATION}, or " + "add_constraints(..., freeze=True) per constraint.", + FutureWarning, + ) self._freeze_constraints = bool(value) + @property + def sparse(self) -> bool: + """ + Whether the model is built on the sparse (CSR) path, set once by + ``Model(sparse=True)``. + """ + return self._sparse + + def _check_sparse_semantics(self) -> None: + """Raise if the model is sparse but the semantics are legacy.""" + if self._sparse and not is_v1(): + raise ValueError( + "The model was created with sparse=True, which requires v1 " + "semantics, but linopy.options['semantics'] is 'legacy'." + ) + @property def set_names_in_solver_io(self) -> bool: """Whether direct solver exports include names by default.""" @@ -609,7 +661,6 @@ def scalar_attrs(self) -> list[str]: "_pwlCounter", "force_dim_names", "auto_mask", - "freeze_constraints", "set_names_in_solver_io", ] @@ -1283,8 +1334,8 @@ def add_constraints( Default is None. freeze : bool, optional If True, convert the constraint to an immutable CSR-backed CSRConstraint - for better memory efficiency. If None, uses the model default - ``Model.freeze_constraints`` setting (default False). + for better memory efficiency. If None, freezes if the model is + sparse (``Model(sparse=True)``). scaling : float/array_like, optional Positive finite scaling factor(s) for constraint rows. Solver-side left-hand-side coefficients and right-hand-side values are multiplied diff --git a/pyproject.toml b/pyproject.toml index b9c0c051f..ef1a9d4d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,8 @@ filterwarnings = [ # collection of ``linopy/variables.py`` in the source tree on # Windows CI. "ignore:piecewise:FutureWarning", + # The deprecated sparse switches (#976) stay covered until their removal. + "ignore:.*deprecated and will be removed with the legacy semantics:FutureWarning", ] [tool.coverage.run] diff --git a/test/test_csr.py b/test/test_csr.py index b38ed35c8..7435b2a3d 100644 --- a/test/test_csr.py +++ b/test/test_csr.py @@ -64,7 +64,10 @@ def balance_lhs(self, sparse: bool | None) -> LinearExpression: def base_model( - gens_per_bus: tuple[int, ...] = (7, 1, 3, 1, 2), n_snap: int = 3, seed: int = 0 + gens_per_bus: tuple[int, ...] = (7, 1, 3, 1, 2), + n_snap: int = 3, + seed: int = 0, + sparse: bool = False, ) -> Case: rng = np.random.default_rng(seed) n_bus = len(gens_per_bus) @@ -74,7 +77,7 @@ def base_model( lines = pd.Index([f"line{i}" for i in range(n_bus)], name="line") snaps = pd.Index(range(n_snap), name="snapshot") - m = linopy.Model() + m = linopy.Model(sparse=sparse) gen_p = m.add_variables(coords=[gens, snaps], name="gen_p") flow = m.add_variables(coords=[lines, snaps], name="flow") flow_t = m.add_variables(coords=[snaps, lines], name="flow_t") @@ -342,17 +345,17 @@ def test_namelist_sparse_grid_warns_and_observed_silences() -> None: require_v1() n = 200 s = pd.RangeIndex(n, name="s") - m = Model() + m = Model(sparse=True) x = m.add_variables(coords=[s], name="x") expr = (1.0 * x).assign_coords( period=xr.DataArray(np.arange(n), dims="s", coords={"s": s}), season=xr.DataArray(np.arange(n), dims="s", coords={"s": s}), ) with pytest.warns(UserWarning, match="dense .* grid"): - expr.groupby(["period", "season"]).sum(sparse=True) + expr.groupby(["period", "season"]).sum() with warnings.catch_warnings(): warnings.simplefilter("error") - res = expr.groupby(["period", "season"]).sum(sparse=True, observed=True) + res = expr.groupby(["period", "season"]).sum(observed=True) assert res._csr is not None assert res._csr.shape == (n,) @@ -455,16 +458,131 @@ def test_frozen_constraint_applies_row_scaling(sparse: bool) -> None: xr.testing.assert_equal(con.scaling, expected) -def test_option_gates_csr_and_freeze_model_default() -> None: +def sparse_model_results(c: Case) -> dict[str, Any]: + weights = xr.DataArray( + np.eye(len(c.gbus))[:, :2], + coords=[c.gbus.index, pd.Index(["a", "b"], name="k")], + ) + lhs = c.balance_lhs(sparse=None) + return { + "expression groupby": (1.0 * c.gen_p).groupby(c.gbus).sum(), + "variable groupby": c.gen_p.groupby(c.gbus).sum(), + "expression @": (1.0 * c.gen_p) @ weights, + "variable @": c.gen_p @ weights, + "constraint": c.m.add_constraints(lhs == c.load, name="bal"), + "unfrozen": c.m.add_constraints(lhs >= 0, name="free", freeze=False), + "copy": c.m.copy().constraints["bal"], + } + + +@pytest.mark.parametrize("sparse", [False, True]) +def test_model_sparse_switches_the_whole_build(sparse: bool) -> None: + require_v1() + res = sparse_model_results(base_model(sparse=sparse)) + unfrozen = res.pop("unfrozen") + assert isinstance(unfrozen, Constraint) + for key, obj in res.items(): + if isinstance(obj, LinearExpression): + assert obj.is_sparse is sparse, key + else: + assert isinstance(obj, CSRConstraint) is sparse, key + + +def test_model_sparse_persists_through_netcdf(tmp_path: Path) -> None: + require_v1() + c = base_model(sparse=True) + c.m.to_netcdf(tmp_path / "m.nc") + read = linopy.read_netcdf(tmp_path / "m.nc") + assert read.sparse and read.freeze_constraints + assert read.copy().sparse + linopy.options["semantics"] = "legacy" + with pytest.raises(ValueError, match="requires v1 semantics"): + linopy.read_netcdf(tmp_path / "m.nc") + + +def test_file_without_sparse_key_keeps_freeze_default(tmp_path: Path) -> None: + Model(freeze_constraints=True).to_netcdf(tmp_path / "m.nc") + ds = xr.load_dataset(tmp_path / "m.nc") + del ds.attrs["sparse"] + ds.to_netcdf(tmp_path / "old.nc") + read = linopy.read_netcdf(tmp_path / "old.nc") + assert read.freeze_constraints and not read.sparse + + +@pytest.mark.parametrize( + "op", + [ + lambda c: (1.0 * c.gen_p).groupby(c.gbus).sum(), + lambda c: (1.0 * c.gen_p) @ xr.DataArray(np.ones(len(c.gbus)), [c.gbus.index]), + ], + ids=["groupby", "matmul"], +) +def test_sparse_model_raises_under_legacy(op: Callable[[Case], Any]) -> None: + require_v1() + c = base_model(sparse=True) + linopy.options["semantics"] = "legacy" + with pytest.raises(ValueError, match="requires v1 semantics"): + op(c) + + +@pytest.mark.parametrize( + "build", + [ + lambda: Model(sparse=True, chunk=10), + lambda: Model(sparse=True, freeze_constraints=True), + lambda: setattr(Model(sparse=True), "chunk", 10), + lambda: setattr(Model(sparse=True), "freeze_constraints", False), + ], + ids=["chunk", "freeze_constraints", "set-chunk", "set-freeze_constraints"], +) +def test_sparse_model_rejects_conflicting_config(build: Callable[[], Any]) -> None: require_v1() + with pytest.raises(ValueError, match="sparse"): + build() + + +@pytest.mark.legacy +def test_model_sparse_requires_v1() -> None: + with pytest.raises(ValueError, match="requires v1 semantics"): + Model(sparse=True) + + +@pytest.mark.parametrize( + "switch", + [ + lambda c: Model(freeze_constraints=True), + lambda c: setattr(c.m, "freeze_constraints", True), + lambda c: (1.0 * c.gen_p).groupby(c.gbus).sum(sparse=False), + lambda c: linopy.options.set_value(sparse_groupby=True), + ], + ids=[ + "Model(freeze_constraints)", + "set-freeze_constraints", + "sum(sparse)", + "option", + ], +) +def test_deprecated_sparse_switches_warn(switch: Callable[[Case], Any]) -> None: c = base_model() - c.m.freeze_constraints = True - linopy.options["sparse_groupby"] = True - try: - con = c.m.add_constraints(c.balance_lhs(sparse=None) == c.load, name="bal") - finally: - linopy.options["sparse_groupby"] = False - assert isinstance(con, CSRConstraint) + with linopy.options, pytest.warns(FutureWarning, match="deprecated"): + switch(c) + + +def test_sparse_groupby_option_covers_groupby_only() -> None: + require_v1() + c = base_model() + weights = xr.DataArray(np.ones(len(c.gbus)), [c.gbus.index]) + with linopy.options: + linopy.options.set_value(sparse_groupby=True) + assert (1.0 * c.gen_p).groupby(c.gbus).sum().is_sparse + assert not ((1.0 * c.gen_p) @ weights).is_sparse + + +def test_sparse_model_notices_groupby_without_sparse_path() -> None: + require_v1() + c = base_model(sparse=True) + with no_densify(), pytest.raises(linopy.PerformanceWarning, match="grouper"): + (1.0 * c.gen_p).groupby(c.gbus).sum(use_fallback=True) def test_materialized_csr_still_freezes_via_dense_path() -> None: @@ -2062,9 +2180,8 @@ def softened( max_violation: float | None, freeze: bool, ) -> tuple[Case, ConstraintBase]: - c = base_model() + c = base_model(sparse=freeze and route == "penalty-default") c.m.add_objective(1.0 * c.gen_p.sum()) - c.m.freeze_constraints = freeze and route == "penalty-default" mask = MASK_KINDS[mask_kind](c) args = (c.balance_lhs(sparse=freeze), sign, c.load) kwargs: dict[str, Any] = dict(name="c", mask=mask) From 691c73031644a136aebc89e94331d73f7e367e3f Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 25 Sep 2026 11:32:39 +0200 Subject: [PATCH 2/4] refactor(csr): route sparse-model checks through setters and constructor --- linopy/expressions.py | 16 ++++++++-------- linopy/io.py | 6 ++---- linopy/model.py | 17 +++++------------ 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/linopy/expressions.py b/linopy/expressions.py index 3437dbfca..827a319ad 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -614,19 +614,19 @@ def sum( self.model._check_sparse_semantics() csr = self._csr explicit_sparse = sparse is True - if sparse is not None: - warn_outside_linopy( - f"groupby(...).sum(sparse=...) {SPARSE_DEPRECATION}.", FutureWarning - ) if sparse is None: sparse = is_v1() and ( self.model.sparse or options["sparse_groupby"] or csr is not None ) - elif sparse and not is_v1(): - raise ValueError( - "sparse groupby-sum requires v1 semantics; opt in with " - "linopy.options['semantics'] = 'v1'." + else: + warn_outside_linopy( + f"groupby(...).sum(sparse=...) {SPARSE_DEPRECATION}.", FutureWarning ) + if sparse and not is_v1(): + raise ValueError( + "sparse groupby-sum requires v1 semantics; opt in with " + "linopy.options['semantics'] = 'v1'." + ) if multikey_frame is not None and not observed: _warn_dense_grid(multikey_frame) diff --git a/linopy/io.py b/linopy/io.py index 4bf23d7bc..fc98f671a 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -1170,8 +1170,8 @@ def read_netcdf(path: Path | str, **kwargs: Any) -> Model: if isinstance(path, str): path = Path(path) - m = Model() ds = xr.load_dataset(path, **kwargs) + m = Model(sparse=bool(ds.attrs.get("sparse", False))) def has_prefix(k: str, prefix: str) -> bool: return k.rsplit("-", 1)[0] == prefix @@ -1280,9 +1280,7 @@ def container_names(kind: str) -> list[str]: for k in m.scalar_attrs: if k in ds.attrs: setattr(m, k, ds.attrs[k]) - m._sparse = bool(ds.attrs.get("sparse", False)) m._freeze_constraints = bool(ds.attrs.get("freeze_constraints", False)) - m._check_sparse_semantics() if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: m._dtypes["labels"] = np.int64 @@ -1360,6 +1358,7 @@ def copy(m: Model, include_solution: bool = False, deep: bool = True) -> Model: auto_mask=m._auto_mask, set_names_in_solver_io=m.set_names_in_solver_io, solver_dir=str(m._solver_dir), + sparse=m.sparse, ) new_model._variables = Variables( @@ -1441,7 +1440,6 @@ def _copy_con(name: str, con: ConstraintBase) -> ConstraintBase: for attr in m.scalar_attrs: if include_solution or attr not in SOLVE_STATE_ATTRS: setattr(new_model, attr, getattr(m, attr)) - new_model._sparse = m.sparse new_model._freeze_constraints = m.freeze_constraints if m._sos_reformulation_state is not None: diff --git a/linopy/model.py b/linopy/model.py index 1d358530c..31d8bb4ac 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -290,14 +290,8 @@ def __init__( ------- linopy.Model """ - if sparse: - if not is_v1(): - raise ValueError( - "Model(sparse=True) requires v1 semantics; opt in with " - "linopy.options['semantics'] = 'v1'." - ) - if chunk: - raise ValueError("Model(sparse=True) does not support `chunk`.") + self._sparse: bool = bool(sparse) + self._check_sparse_semantics() self._dtypes: dict[DtypeKey, type[np.signedinteger]] = self._resolve_dtypes( dtypes ) @@ -317,10 +311,9 @@ def __init__( self._pwlCounter: int = 0 self._blocks: DataArray | None = None - self._chunk: T_Chunks = chunk + self.chunk = chunk self._force_dim_names: bool = bool(force_dim_names) self._auto_mask: bool = bool(auto_mask) - self._sparse: bool = bool(sparse) self._freeze_constraints: bool = self._sparse if freeze_constraints is not None: self.freeze_constraints = freeze_constraints @@ -584,8 +577,8 @@ def _check_sparse_semantics(self) -> None: """Raise if the model is sparse but the semantics are legacy.""" if self._sparse and not is_v1(): raise ValueError( - "The model was created with sparse=True, which requires v1 " - "semantics, but linopy.options['semantics'] is 'legacy'." + "Model(sparse=True) requires v1 semantics; opt in with " + "linopy.options['semantics'] = 'v1'." ) @property From d5a8cfaca5bacad15c8f5a843694d3a2ca81ce8b Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 25 Sep 2026 12:20:27 +0200 Subject: [PATCH 3/4] refactor(csr): keep only the deprecated override in _freeze_constraints --- linopy/io.py | 4 ++-- linopy/model.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index fc98f671a..d7cd0063d 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -1099,7 +1099,7 @@ def with_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: params = [with_prefix(m.parameters, "parameters")] scalars = {k: getattr(m, k) for k in m.scalar_attrs} - scalars |= {"sparse": m.sparse, "freeze_constraints": m.freeze_constraints} + scalars |= {"sparse": m.sparse, "freeze_constraints": m._freeze_constraints} ds = xr.merge(vars + cons + exprs + obj + params, combine_attrs="drop_conflicts") ds = ds.assign_attrs(scalars) ds.attrs[NETCDF_VERSION_ATTR] = version("linopy") @@ -1440,7 +1440,7 @@ def _copy_con(name: str, con: ConstraintBase) -> ConstraintBase: for attr in m.scalar_attrs: if include_solution or attr not in SOLVE_STATE_ATTRS: setattr(new_model, attr, getattr(m, attr)) - new_model._freeze_constraints = m.freeze_constraints + new_model._freeze_constraints = m._freeze_constraints if m._sos_reformulation_state is not None: new_model._sos_reformulation_state = _copy.deepcopy(m._sos_reformulation_state) diff --git a/linopy/model.py b/linopy/model.py index 31d8bb4ac..0c6b5c68d 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -314,7 +314,7 @@ def __init__( self.chunk = chunk self._force_dim_names: bool = bool(force_dim_names) self._auto_mask: bool = bool(auto_mask) - self._freeze_constraints: bool = self._sparse + self._freeze_constraints: bool = False if freeze_constraints is not None: self.freeze_constraints = freeze_constraints self._set_names_in_solver_io: bool = bool(set_names_in_solver_io) @@ -549,7 +549,7 @@ def auto_mask(self, value: bool) -> None: @property def freeze_constraints(self) -> bool: """Whether constraints are frozen to CSR by default when added.""" - return self._freeze_constraints + return self._sparse or self._freeze_constraints @freeze_constraints.setter def freeze_constraints(self, value: bool) -> None: From 201aea84588314fa368621c4b28d29113ac0fa81 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 25 Sep 2026 12:39:08 +0200 Subject: [PATCH 4/4] test(csr): build through Model(sparse=True) instead of the deprecated switches --- pyproject.toml | 2 - test/test_constraint.py | 11 +- test/test_csr.py | 595 +++++++++++++++-------------- test/test_indicator_constraints.py | 11 +- test/test_linear_expression.py | 8 +- test/test_model.py | 30 +- test/test_solvers.py | 4 +- 7 files changed, 344 insertions(+), 317 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ef1a9d4d4..b9c0c051f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,8 +151,6 @@ filterwarnings = [ # collection of ``linopy/variables.py`` in the source tree on # Windows CI. "ignore:piecewise:FutureWarning", - # The deprecated sparse switches (#976) stay covered until their removal. - "ignore:.*deprecated and will be removed with the legacy semantics:FutureWarning", ] [tool.coverage.run] diff --git a/test/test_constraint.py b/test/test_constraint.py index 13928b2dd..ec8e297d3 100644 --- a/test/test_constraint.py +++ b/test/test_constraint.py @@ -86,8 +86,9 @@ def test_add_constraints_freeze(m: Model, x: linopy.Variable) -> None: assert c.ncons == 10 +@pytest.mark.v1 def test_add_constraints_uses_model_freeze_default() -> None: - m = Model(freeze_constraints=True) + m = Model(sparse=True) x = m.add_variables(coords=[pd.RangeIndex(10, name="first")], name="x") c = m.add_constraints(x >= 1, name="frozen_by_default") assert isinstance(c, linopy.constraints.CSRConstraint) @@ -122,11 +123,11 @@ def test_empty_constraints_repr() -> None: Model().constraints.__repr__() -@pytest.mark.parametrize("freeze_constraints", [True, False]) -def test_constraint_handles_empty_rows(freeze_constraints: bool) -> None: +@pytest.mark.parametrize("freeze", [True, False]) +def test_constraint_handles_empty_rows(freeze: bool) -> None: """An empty constraint group must be accepted and solve cleanly.""" - m = Model(freeze_constraints=freeze_constraints) + m = Model() x = m.add_variables( lower=0.0, coords=[range(3), range(2)], @@ -134,7 +135,7 @@ def test_constraint_handles_empty_rows(freeze_constraints: bool) -> None: name="x", ) empty = x.isel(time=range(1, 1)) - c = m.add_constraints(empty == 0, name="empty") + c = m.add_constraints(empty == 0, name="empty", freeze=freeze) assert isinstance(c, linopy.constraints.ConstraintBase) assert c.size == 0 # Solving a model with only an empty constraint group is also fine. diff --git a/test/test_csr.py b/test/test_csr.py index 7435b2a3d..5189575bb 100644 --- a/test/test_csr.py +++ b/test/test_csr.py @@ -55,13 +55,16 @@ class Case: bus1: pd.Series load: xr.DataArray - def balance_lhs(self, sparse: bool | None) -> LinearExpression: + def balance_lhs(self) -> LinearExpression: return ( - (self.eff * self.gen_p).groupby(self.gbus).sum(sparse=sparse) - + (1.0 * self.flow).groupby(self.bus0).sum(sparse=sparse) - - (1.0 * self.flow).groupby(self.bus1).sum(sparse=sparse) + self.gen_sum() + + (1.0 * self.flow).groupby(self.bus0).sum() + - (1.0 * self.flow).groupby(self.bus1).sum() ) + def gen_sum(self) -> LinearExpression: + return (self.eff * self.gen_p).groupby(self.gbus).sum() + def base_model( gens_per_bus: tuple[int, ...] = (7, 1, 3, 1, 2), @@ -92,6 +95,11 @@ def base_model( return Case(m, gen_p, flow, flow_t, eff, gbus, bus0, bus1, load) +def twin_models(**kwargs: Any) -> tuple[Case, Case]: + """A dense model and its sparse twin, built identically.""" + return base_model(**kwargs), base_model(sparse=True, **kwargs) + + def canon(df: pl.DataFrame) -> pl.DataFrame: return ( df.group_by(["labels", "vars"]) @@ -111,26 +119,29 @@ def assert_frozen_equal(con1: ConstraintBase, con2: ConstraintBase) -> None: assert np.array_equal(np.sort(labels[labels != -1]), np.sort(con2.active_labels())) -def reindexed_balance(c: Case, sparse: bool) -> LinearExpression: +def reindexed_balance(c: Case) -> LinearExpression: """Generation on all buses plus flow on two lines, both reindexed onto the load grid.""" lines = ["line0", "line1"] - gen = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=sparse) - flow = (1.0 * c.flow.loc[lines]).groupby(c.bus0.loc[lines]).sum(sparse=sparse) + gen = c.gen_sum() + flow = (1.0 * c.flow.loc[lines]).groupby(c.bus0.loc[lines]).sum() parts = [gen.reindex(bus=c.load.bus), flow.reindex(bus=c.load.bus)] return linopy.merge(parts, join="outer", cls=LinearExpression) def test_csr_requires_v1() -> None: c = base_model() + deprecated = pytest.warns(FutureWarning, match="deprecated") if is_v1(): - res = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) + with deprecated: + res = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) assert type(res) is LinearExpression return - with pytest.raises(ValueError, match="requires v1 semantics"): + with deprecated, pytest.raises(ValueError, match="requires v1 semantics"): (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - linopy.options["sparse_groupby"] = True + with pytest.warns(FutureWarning, match="deprecated"): + linopy.options["sparse_groupby"] = True try: - res = (c.eff * c.gen_p).groupby(c.gbus).sum() + res = c.gen_sum() finally: linopy.options["sparse_groupby"] = False assert res._csr is None @@ -138,57 +149,58 @@ def test_csr_requires_v1() -> None: def test_csr_is_plain_linear_expression_and_materializes_equivalently() -> None: require_v1() - c = base_model() - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - eager = (c.eff * c.gen_p).groupby(c.gbus).sum() + c1, c2 = twin_models() + sparse = c2.gen_sum() assert type(sparse) is LinearExpression - assert_linequal(sparse, eager) + assert sparse._csr is not None + assert_linequal(sparse, c1.gen_sum()) def test_csr_composition_materializes_equivalently() -> None: require_v1() - c = base_model() - sparse = c.balance_lhs(sparse=True) + c1, c2 = twin_models() + sparse = c2.balance_lhs() assert type(sparse) is LinearExpression - assert_linequal(sparse, c.balance_lhs(sparse=False)) + assert sparse._csr is not None + assert_linequal(sparse, c1.balance_lhs()) def test_scalar_ops_stay_csr() -> None: require_v1() - c = base_model() - sparse = -2.0 * (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) + c1, c2 = twin_models() + sparse = -2.0 * c2.gen_sum() assert sparse._csr is not None - assert_linequal(sparse, -2.0 * (c.eff * c.gen_p).groupby(c.gbus).sum()) + assert_linequal(sparse, -2.0 * c1.gen_sum()) @pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) def test_zero_coefficient_rows_stay_active(sparse: bool) -> None: require_v1() - c = base_model() - lhs = (0.0 * c.gen_p).groupby(c.gbus).sum(sparse=sparse) - lhs = lhs + (0.0 * c.flow).groupby(c.bus0).sum(sparse=sparse) + c = base_model(sparse=sparse) + lhs = (0.0 * c.gen_p).groupby(c.gbus).sum() + lhs = lhs + (0.0 * c.flow).groupby(c.bus0).sum() con = c.m.add_constraints(lhs == c.load, name="bal", freeze=True) assert len(con.active_labels()) == c.load.size def test_merge_keeps_absent_cell_absent() -> None: require_v1() - c = base_model() - dense = (c.eff * c.gen_p).groupby(c.gbus).sum() - flow = (1.0 * c.flow).groupby(c.bus0).sum() - mask = xr.DataArray(np.arange(len(c.load.bus)) % 2 == 0, coords=[c.load.bus]) - flow = flow.where(mask) + c1, c2 = twin_models() + mask = xr.DataArray(np.arange(len(c1.load.bus)) % 2 == 0, coords=[c1.load.bus]) + flow1, flow2 = ( + (1.0 * c.flow).groupby(c.bus0).sum(use_fallback=True).where(mask) + for c in (c1, c2) + ) - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - tot = linopy.merge([sparse, flow], join="outer", cls=LinearExpression) - assert tot._csr is not None + tot = linopy.merge([c2.gen_sum(), flow2], join="outer", cls=LinearExpression) + assert tot._csr is not None and flow2._csr is None assert_linequal( - tot, linopy.merge([dense, flow], join="outer", cls=LinearExpression) + tot, linopy.merge([c1.gen_sum(), flow1], join="outer", cls=LinearExpression) ) - con = c.m.add_constraints(tot >= c.load, name="bal", freeze=True) + con = c2.m.add_constraints(tot >= c2.load, name="bal") assert isinstance(con, CSRConstraint) - assert con.ncons == int(mask.sum()) * c.load.sizes["snapshot"] + assert con.ncons == int(mask.sum()) * c2.load.sizes["snapshot"] @pytest.mark.parametrize( @@ -203,12 +215,13 @@ def test_explicit_sparse_raises_on_unsupported_grouper( ) -> None: require_v1() c = base_model() - with pytest.raises(ValueError, match="sparse=True supports only"): + deprecated = pytest.warns(FutureWarning, match="deprecated") + with deprecated, pytest.raises(ValueError, match="sparse=True supports only"): (1.0 * c.gen_p).groupby(grouper).sum(sparse=True, **kwargs) def keyed_model( - member_first: bool = True, + member_first: bool = True, sparse: bool = False ) -> tuple[Model, LinearExpression, xr.DataArray]: """ ``(1 + x)`` over ``(s, snapshot)`` with ``period``/``season`` keys on ``s``; @@ -217,7 +230,7 @@ def keyed_model( n = 6 s = pd.RangeIndex(n, name="s") snaps = pd.Index(range(2), name="snapshot") - m = Model() + m = Model(sparse=sparse) coords = [s, snaps] if member_first else [snaps, s] x = m.add_variables(coords=coords, name="x") period = xr.DataArray(np.arange(n) // 2, dims="s", coords={"s": s}) @@ -235,10 +248,11 @@ def keyed_model( @pytest.mark.parametrize("observed", [False, True]) def test_namelist_sparse_matches_dense(observed: bool, member_first: bool) -> None: require_v1() - _, expr, _ = keyed_model(member_first) + _, dense_expr, _ = keyed_model(member_first) + _, sparse_expr, _ = keyed_model(member_first, sparse=True) keys = ["period", "season"] - sparse = expr.groupby(keys).sum(sparse=True, observed=observed) - dense = expr.groupby(keys).sum(observed=observed) + sparse = sparse_expr.groupby(keys).sum(observed=observed) + dense = dense_expr.groupby(keys).sum(observed=observed) csr = sparse._csr assert csr is not None if observed: @@ -252,48 +266,52 @@ def test_namelist_sparse_matches_dense(observed: bool, member_first: bool) -> No @pytest.mark.parametrize("as_namelist", [False, True]) def test_single_key_sparse_ignores_observed(as_namelist: bool) -> None: require_v1() - c = base_model() - expr = (c.eff * c.gen_p).assign_coords(bus=("gen", c.gbus.to_numpy())) - grouper = ["bus"] if as_namelist else c.gbus - sparse = expr.groupby(grouper).sum(sparse=True, observed=True) - assert sparse._csr is not None - assert_linequal(sparse, expr.groupby(grouper).sum()) + c1, c2 = twin_models() + dense, sparse = ( + (c.eff * c.gen_p).assign_coords(bus=("gen", c.gbus.to_numpy())) + for c in (c1, c2) + ) + grouper = ["bus"] if as_namelist else c1.gbus + res = sparse.groupby(grouper).sum(observed=True) + assert res._csr is not None + assert_linequal(res, dense.groupby(grouper).sum()) def test_sparse_keeps_aux_coords_on_surviving_dims() -> None: require_v1() - _, expr, _ = keyed_model() - expr = expr.assign_coords(tag=("snapshot", list("ab"))) + dense_expr, sparse_expr = ( + keyed_model(sparse=sparse_model)[1].assign_coords(tag=("snapshot", list("ab"))) + for sparse_model in (False, True) + ) for kwargs in ({}, {"observed": True}): - sparse = expr.groupby(["period", "season"]).sum(sparse=True, **kwargs) - dense = expr.groupby(["period", "season"]).sum(**kwargs) + sparse = sparse_expr.groupby(["period", "season"]).sum(**kwargs) + dense = dense_expr.groupby(["period", "season"]).sum(**kwargs) assert set(sparse.coords) == set(dense.coords) assert_linequal(sparse, dense) def test_dataframe_grouper_sparse_stays_compact() -> None: require_v1() - _, expr, _ = keyed_model() - df = expr.data[["period", "season"]].to_dataframe()[["period", "season"]] - sparse = expr.groupby(df).sum(sparse=True) + _, dense, _ = keyed_model() + _, expr, _ = keyed_model(sparse=True) + df = dense.data[["period", "season"]].to_dataframe()[["period", "season"]] + sparse = expr.groupby(df).sum() assert sparse._csr is not None assert sparse._csr.shape == (4, 2) - assert_linequal(sparse, expr.groupby(df).sum()) + assert_linequal(sparse, dense.groupby(df).sum()) def test_namelist_sparse_observed_freezes_compact() -> None: require_v1() m1, e1, rhs = keyed_model() - m2, e2, _ = keyed_model() + m2, e2, _ = keyed_model(sparse=True) dense = m1.add_constraints( e1.groupby(["period", "season"]).sum(observed=True) == rhs, name="c", freeze=True, ) sparse = m2.add_constraints( - e2.groupby(["period", "season"]).sum(sparse=True, observed=True) == rhs, - name="c", - freeze=True, + e2.groupby(["period", "season"]).sum(observed=True) == rhs, name="c" ) assert isinstance(sparse, CSRConstraint) assert sparse.ncons == rhs.size @@ -303,12 +321,10 @@ def test_namelist_sparse_observed_freezes_compact() -> None: def test_namelist_sparse_grid_absent_cells_inactive() -> None: require_v1() m1, e1, _ = keyed_model() - m2, e2, _ = keyed_model() + m2, e2, _ = keyed_model(sparse=True) keys = ["period", "season"] dense = m1.add_constraints(e1.groupby(keys).sum() == 0, name="c", freeze=True) - sparse = m2.add_constraints( - e2.groupby(keys).sum(sparse=True) == 0, name="c", freeze=True - ) + sparse = m2.add_constraints(e2.groupby(keys).sum() == 0, name="c") assert isinstance(sparse, CSRConstraint) assert sparse.ncons == 4 * 2 assert_conequal(dense, sparse, strict=False) @@ -319,24 +335,28 @@ def test_namelist_sparse_grid_absent_cells_inactive() -> None: def test_namelist_sparse_observed_keeps_aux_coords_through_merge() -> None: require_v1() - _, expr, _ = keyed_model() + _, dense_expr, _ = keyed_model() + _, sparse_expr, _ = keyed_model(sparse=True) keys = ["period", "season"] - sparse = expr.groupby(keys).sum(sparse=True, observed=True) - dense = expr.groupby(keys).sum(observed=True) - tot = sparse + dense + sparse = sparse_expr.groupby(keys).sum(observed=True) + dense = dense_expr.groupby(keys).sum(observed=True) + assert sparse._csr is not None + operand = sparse._csr.to_dense() + tot = sparse + operand assert tot._csr is not None assert set(tot._csr.grid.aux) == {"period", "season"} assert_linequal(tot, 2.0 * dense) - other = dense.assign_coords(region=("group", list("abcd"))) - tot = sparse + other + region = ("group", list("abcd")) + tot = sparse + operand.assign_coords(region=region) assert tot._csr is not None assert set(tot._csr.grid.aux) == {"period", "season", "region"} xr.testing.assert_equal( - tot.data.coords.to_dataset(), (dense + other).data.coords.to_dataset() + tot.data.coords.to_dataset(), + (dense + dense.assign_coords(region=region)).data.coords.to_dataset(), ) - conflicting = dense.assign_coords(season=("group", list("xyzw"))) + conflicting = operand.assign_coords(season=("group", list("xyzw"))) with pytest.raises(ValueError, match="conflicting values"): sparse + conflicting @@ -362,20 +382,18 @@ def test_namelist_sparse_grid_warns_and_observed_silences() -> None: def test_nan_multikey_grouper_raises_eagerly() -> None: require_v1() - _, expr, _ = keyed_model() + _, expr, _ = keyed_model(sparse=True) period = expr.data["period"] expr = expr.assign_coords(period=period.where(period > 0)) with pytest.raises(ValueError, match="NaN values"): - expr.groupby(["period", "season"]).sum(sparse=True) + expr.groupby(["period", "season"]).sum() def test_freeze_realizes_csr_without_dense_rectangle() -> None: require_v1() - c1, c2 = base_model(), base_model() - con1 = c1.m.add_constraints(c1.balance_lhs(sparse=False) == c1.load, name="bal") - con2 = c2.m.add_constraints( - c2.balance_lhs(sparse=True) == c2.load, name="bal", freeze=True - ) + c1, c2 = twin_models() + con1 = c1.m.add_constraints(c1.balance_lhs() == c1.load, name="bal") + con2 = c2.m.add_constraints(c2.balance_lhs() == c2.load, name="bal") assert isinstance(con2, CSRConstraint) assert_frozen_equal(con1, con2) @@ -384,9 +402,9 @@ def test_freeze_realizes_csr_without_dense_rectangle() -> None: def test_freeze_false_falls_back_to_identical_dense_constraint() -> None: """The fallback is canonical-form, so compare mathematically (strict=False).""" require_v1() - c1, c2 = base_model(), base_model() - con1 = c1.m.add_constraints(c1.balance_lhs(sparse=False) == c1.load, name="bal") - con2 = c2.m.add_constraints(c2.balance_lhs(sparse=True) == c2.load, name="bal") + c1, c2 = twin_models() + con1 = c1.m.add_constraints(c1.balance_lhs() == c1.load, name="bal") + con2 = c2.m.add_constraints(c2.balance_lhs() == c2.load, name="bal", freeze=False) assert isinstance(con2, Constraint) assert_conequal(con1, con2, strict=False) assert np.array_equal(con1.labels.values, con2.labels.values) @@ -394,9 +412,9 @@ def test_freeze_false_falls_back_to_identical_dense_constraint() -> None: def test_to_constraint_on_csr_lhs_is_unassigned_csr_constraint() -> None: require_v1() - c1, c2 = base_model(), base_model() - dense = c1.balance_lhs(sparse=False) == c1.load - con = c2.balance_lhs(sparse=True) == c2.load + c1, c2 = twin_models() + dense = c1.balance_lhs() == c1.load + con = c2.balance_lhs() == c2.load assert isinstance(con, CSRConstraint) assert not con.is_assigned assert con.type == "Constraint (unassigned)" @@ -410,7 +428,7 @@ def test_to_constraint_on_csr_lhs_is_unassigned_csr_constraint() -> None: c2.m.constraints.add(con) con1 = c1.m.add_constraints(dense, name="bal") - con2 = c2.m.add_constraints(con, name="bal", freeze=True) + con2 = c2.m.add_constraints(con, name="bal") assert isinstance(con2, CSRConstraint) assert con2.is_assigned assert np.array_equal( @@ -424,10 +442,10 @@ def test_group_without_terms_matches_dense_labels(freeze: bool) -> None: require_v1() def build(sparse: bool) -> ConstraintBase: - c = base_model() + c = base_model(sparse=sparse) gens = c.gen_p.indexes["gen"] gen_p = c.gen_p.where(xr.DataArray(gens != "gen7", coords=[gens])) - lhs = (c.eff * gen_p).groupby(c.gbus).sum(sparse=sparse) + lhs = (c.eff * gen_p).groupby(c.gbus).sum() return c.m.add_constraints(lhs == c.load, name="bal", freeze=freeze) dense, sparse = build(False), build(True) @@ -439,19 +457,19 @@ def build(sparse: bool) -> ConstraintBase: @pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) def test_frozen_invalid_infinite_rhs_raises(sparse: bool) -> None: require_v1() - c = base_model() + c = base_model(sparse=sparse) with pytest.raises(ValueError, match="incorrect infinite values"): - c.m.add_constraints(c.balance_lhs(sparse) <= -np.inf, name="bal", freeze=True) + c.m.add_constraints(c.balance_lhs() <= -np.inf, name="bal", freeze=True) @pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) def test_frozen_constraint_applies_row_scaling(sparse: bool) -> None: require_v1() - c = base_model() + c = base_model(sparse=sparse) snaps = c.load.indexes["snapshot"] scaling = xr.DataArray(np.arange(1.0, len(snaps) + 1), coords=[snaps]) con = c.m.add_constraints( - c.balance_lhs(sparse) == c.load, name="bal", freeze=True, scaling=scaling + c.balance_lhs() == c.load, name="bal", freeze=True, scaling=scaling ) assert isinstance(con, CSRConstraint) expected = scaling.broadcast_like(con.scaling).transpose(*con.scaling.dims) @@ -463,7 +481,7 @@ def sparse_model_results(c: Case) -> dict[str, Any]: np.eye(len(c.gbus))[:, :2], coords=[c.gbus.index, pd.Index(["a", "b"], name="k")], ) - lhs = c.balance_lhs(sparse=None) + lhs = c.balance_lhs() return { "expression groupby": (1.0 * c.gen_p).groupby(c.gbus).sum(), "variable groupby": c.gen_p.groupby(c.gbus).sum(), @@ -501,7 +519,9 @@ def test_model_sparse_persists_through_netcdf(tmp_path: Path) -> None: def test_file_without_sparse_key_keeps_freeze_default(tmp_path: Path) -> None: - Model(freeze_constraints=True).to_netcdf(tmp_path / "m.nc") + with pytest.warns(FutureWarning, match="deprecated"): + m = Model(freeze_constraints=True) + m.to_netcdf(tmp_path / "m.nc") ds = xr.load_dataset(tmp_path / "m.nc") del ds.attrs["sparse"] ds.to_netcdf(tmp_path / "old.nc") @@ -572,7 +592,7 @@ def test_sparse_groupby_option_covers_groupby_only() -> None: require_v1() c = base_model() weights = xr.DataArray(np.ones(len(c.gbus)), [c.gbus.index]) - with linopy.options: + with linopy.options, pytest.warns(FutureWarning, match="deprecated"): linopy.options.set_value(sparse_groupby=True) assert (1.0 * c.gen_p).groupby(c.gbus).sum().is_sparse assert not ((1.0 * c.gen_p) @ weights).is_sparse @@ -587,39 +607,39 @@ def test_sparse_model_notices_groupby_without_sparse_path() -> None: def test_materialized_csr_still_freezes_via_dense_path() -> None: require_v1() - c = base_model() - lhs = c.balance_lhs(sparse=True) + c = base_model(sparse=True) + lhs = c.balance_lhs() _ = lhs.nterm - con = c.m.add_constraints(lhs == c.load, name="bal", freeze=True) + con = c.m.add_constraints(lhs == c.load, name="bal") assert isinstance(con, CSRConstraint) @pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) def test_nan_rhs_raises(sparse: bool) -> None: require_v1() - c = base_model() + c = base_model(sparse=sparse) load = c.load.copy() load[0, 0] = np.nan with pytest.raises(ValueError, match="NaN"): - c.m.add_constraints(c.balance_lhs(sparse) == load, name="bal", freeze=True) + c.m.add_constraints(c.balance_lhs() == load, name="bal", freeze=True) @pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) def test_reordered_rhs_raises(sparse: bool) -> None: require_v1() - c = base_model() + c = base_model(sparse=sparse) load = c.load.isel(bus=slice(None, None, -1)) with pytest.raises(ValueError, match="[Cc]oordinate"): - c.m.add_constraints(c.balance_lhs(sparse) == load, name="bal", freeze=True) + c.m.add_constraints(c.balance_lhs() == load, name="bal", freeze=True) def test_nan_grouper_raises_eagerly() -> None: require_v1() - c = base_model() + c = base_model(sparse=True) gbus = c.gbus.copy() gbus.iloc[0] = np.nan with pytest.raises(ValueError, match="NaN values"): - (1.0 * c.gen_p).groupby(gbus).sum(sparse=True) + (1.0 * c.gen_p).groupby(gbus).sum() TERM_LINE = re.compile(r"^[+-][0-9.e+-]+ x[0-9]+$") @@ -641,12 +661,10 @@ def canon_lp(text: str) -> list[str]: def test_lp_files_identical(tmp_path: Path) -> None: require_v1() sizes = (7, 1, 3, 1, 2, 1, 1, 4, 1, 2, 1, 1) - c1, c2 = base_model(gens_per_bus=sizes), base_model(gens_per_bus=sizes) - c1.m.add_constraints(c1.balance_lhs(sparse=False) == c1.load, name="bal") + c1, c2 = twin_models(gens_per_bus=sizes) + c1.m.add_constraints(c1.balance_lhs() == c1.load, name="bal") c1.m.add_objective((1.0 * c1.gen_p).sum()) - c2.m.add_constraints( - c2.balance_lhs(sparse=True) == c2.load, name="bal", freeze=True - ) + c2.m.add_constraints(c2.balance_lhs() == c2.load, name="bal") c2.m.add_objective((1.0 * c2.gen_p).sum()) f1, f2 = tmp_path / "eager.lp", tmp_path / "sparse.lp" @@ -667,30 +685,28 @@ def test_lp_files_identical(tmp_path: Path) -> None: ) def test_reindex_stays_csr_and_matches_dense(indexers: dict) -> None: require_v1() - c = base_model() - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True).reindex(indexers) + c1, c2 = twin_models() + sparse = c2.gen_sum().reindex(indexers) assert sparse._csr is not None - dense = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=False).reindex(indexers) + dense = c1.gen_sum().reindex(indexers) assert_linequal(sparse, dense) def test_reindex_falls_back_to_dense_for_unsupported_kwargs() -> None: require_v1() - c = base_model() - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - res = sparse.reindex(bus=["bus3", "bus0"], copy=False) + c1, c2 = twin_models() + res = c2.gen_sum().reindex(bus=["bus3", "bus0"], copy=False) assert res._csr is None - dense = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=False) - assert_linequal(res, dense.reindex(bus=["bus3", "bus0"])) + assert_linequal(res, c1.gen_sum().reindex(bus=["bus3", "bus0"])) def test_reindex_merge_chain_freezes_csr() -> None: require_v1() - c1, c2 = base_model(), base_model() - con1 = c1.m.add_constraints(reindexed_balance(c1, False) == c1.load, name="bal") - tot = reindexed_balance(c2, True) + c1, c2 = twin_models() + con1 = c1.m.add_constraints(reindexed_balance(c1) == c1.load, name="bal") + tot = reindexed_balance(c2) assert tot._csr is not None - con2 = c2.m.add_constraints(tot == c2.load, name="bal", freeze=True) + con2 = c2.m.add_constraints(tot == c2.load, name="bal") assert isinstance(con2, CSRConstraint) assert_frozen_equal(con1, con2) @@ -700,12 +716,10 @@ def test_reindex_merge_chain_peak_memory() -> None: sizes = (200,) + (1,) * 299 n_snap = 50 dense_rectangle_bytes = len(sizes) * n_snap * max(sizes) * 16 - c = base_model(gens_per_bus=sizes, n_snap=n_snap) + c = base_model(gens_per_bus=sizes, n_snap=n_snap, sparse=True) tracemalloc.start() try: - c.m.add_constraints( - reindexed_balance(c, True) == c.load, name="bal", freeze=True - ) + c.m.add_constraints(reindexed_balance(c) == c.load, name="bal") _, peak = tracemalloc.get_traced_memory() finally: tracemalloc.stop() @@ -715,32 +729,29 @@ def test_reindex_merge_chain_peak_memory() -> None: @pytest.mark.parametrize("value", [0.0, 3.5]) def test_fillna_stays_csr_and_matches_dense(value: float) -> None: require_v1() - c = base_model() + c1, c2 = twin_models() wide = {"bus": [f"bus{i}" for i in range(7)]} - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True).reindex(wide) - filled = sparse.fillna(value) + filled = c2.gen_sum().reindex(wide).fillna(value) assert filled._csr is not None - dense = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=False).reindex(wide) + dense = c1.gen_sum().reindex(wide) assert_linequal(filled, dense.fillna(value)) def test_fillna_with_array_falls_back_to_dense() -> None: require_v1() - c = base_model() - fill = xr.zeros_like(c.load) - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - res = sparse.fillna(fill) + c1, c2 = twin_models() + fill = xr.zeros_like(c1.load) + res = c2.gen_sum().fillna(fill) assert res._csr is None - dense = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=False) - assert_linequal(res, dense.fillna(fill)) + assert_linequal(res, c1.gen_sum().fillna(fill)) def test_rename_stays_csr_and_matches_dense() -> None: require_v1() - c = base_model() - sparse = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True).rename(bus="node") + c1, c2 = twin_models() + sparse = c2.gen_sum().rename(bus="node") assert sparse._csr is not None - dense = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=False).rename(bus="node") + dense = c1.gen_sum().rename(bus="node") assert sparse.coord_dims == ("node", "snapshot") assert_linequal(sparse, dense) @@ -754,10 +765,11 @@ def test_namelist_sparse_observed_keeps_aux_coords_through_op( op: Callable[[LinearExpression], LinearExpression], ) -> None: require_v1() - _, expr, _ = keyed_model() + _, dense_expr, _ = keyed_model() + _, sparse_expr, _ = keyed_model(sparse=True) keys = ["period", "season"] - sparse = op(expr.groupby(keys).sum(sparse=True, observed=True)) - dense = op(expr.groupby(keys).sum(sparse=False, observed=True)) + sparse = op(sparse_expr.groupby(keys).sum(observed=True)) + dense = op(dense_expr.groupby(keys).sum(observed=True)) assert sparse._csr is not None for name in keys: xr.testing.assert_equal(sparse.coords[name], dense.coords[name]) @@ -765,13 +777,23 @@ def test_namelist_sparse_observed_keeps_aux_coords_through_op( def cross_grid_parts( - c: Case, sparse: bool, lines: tuple[str, ...] = ("line1", "line2") + c: Case, lines: tuple[str, ...] = ("line1", "line2") ) -> list[LinearExpression]: """Generation on all buses, flow on a line subset only, snapshot-major on the flow side.""" - gen = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=sparse) flow_t = 1.0 * c.flow_t.loc[:, list(lines)] - flow = flow_t.groupby(c.bus0.loc[list(lines)]).sum(sparse=sparse) - return [gen, flow] + return [c.gen_sum(), flow_t.groupby(c.bus0.loc[list(lines)]).sum()] + + +def twin_cross_grid_parts() -> tuple[list[LinearExpression], list[LinearExpression]]: + """``cross_grid_parts`` of a sparse model and of its dense twin.""" + c1, c2 = twin_models() + return cross_grid_parts(c2), cross_grid_parts(c1) + + +def densified(e: LinearExpression) -> LinearExpression: + """A dense operand in the model of the CSR-backed ``e``.""" + assert e._csr is not None + return e._csr.to_dense() def assert_terms_equal(a: LinearExpression, b: LinearExpression) -> None: @@ -791,8 +813,7 @@ def test_cross_grid_merge_stays_csr_and_matches_dense( join: JoinOptions, order: str ) -> None: require_v1() - c = base_model() - sparse, dense = cross_grid_parts(c, True), cross_grid_parts(c, False) + sparse, dense = twin_cross_grid_parts() if order == "flow-gen": sparse, dense = sparse[::-1], dense[::-1] res = linopy.merge(sparse, join=join, cls=LinearExpression) @@ -804,15 +825,15 @@ def test_cross_grid_merge_stays_csr_and_matches_dense( def test_transposed_grid_exact_merge_stays_csr_and_matches_dense() -> None: """Same labels in a transposed dim order stay sparse under the default join.""" require_v1() - c = base_model() - a = (1.0 * c.flow).groupby(c.bus0).sum(sparse=True) - b = (1.0 * c.flow_t).groupby(c.bus0).sum(sparse=True) + c1, c2 = twin_models() + a = (1.0 * c2.flow).groupby(c2.bus0).sum() + b = (1.0 * c2.flow_t).groupby(c2.bus0).sum() assert a.coord_dims == b.coord_dims[::-1] != b.coord_dims res = linopy.merge([a, b], cls=LinearExpression) assert res._csr is not None dense = [ - (1.0 * c.flow).groupby(c.bus0).sum(), - (1.0 * c.flow_t).groupby(c.bus0).sum(), + (1.0 * c1.flow).groupby(c1.bus0).sum(), + (1.0 * c1.flow_t).groupby(c1.bus0).sum(), ] assert_terms_equal(res, linopy.merge(dense, cls=LinearExpression)) @@ -820,10 +841,10 @@ def test_transposed_grid_exact_merge_stays_csr_and_matches_dense() -> None: @pytest.mark.parametrize("join", ["outer", "inner", "left", "right"]) def test_three_operand_cross_grid_merge_matches_dense(join: JoinOptions) -> None: require_v1() - c = base_model() + c1, c2 = twin_models() third_lines = ("line3", "line4") - sparse = cross_grid_parts(c, True) + cross_grid_parts(c, True, third_lines)[1:] - dense = cross_grid_parts(c, False) + cross_grid_parts(c, False, third_lines)[1:] + sparse = cross_grid_parts(c2) + cross_grid_parts(c2, third_lines)[1:] + dense = cross_grid_parts(c1) + cross_grid_parts(c1, third_lines)[1:] res = linopy.merge(sparse, join=join, cls=LinearExpression) assert res._csr is not None assert_terms_equal(res, linopy.merge(dense, join=join, cls=LinearExpression)) @@ -831,8 +852,7 @@ def test_three_operand_cross_grid_merge_matches_dense(join: JoinOptions) -> None def test_cross_grid_merge_absent_fill_matches_dense() -> None: require_v1() - c = base_model() - sparse, dense = cross_grid_parts(c, True), cross_grid_parts(c, False) + sparse, dense = twin_cross_grid_parts() res = linopy.merge( sparse, join="outer", fill_value=linopy.ABSENT, cls=LinearExpression ) @@ -844,25 +864,24 @@ def test_cross_grid_merge_absent_fill_matches_dense() -> None: assert filled._csr is not None assert_terms_equal(filled, expected.fillna(0)) assert_terms_equal(res, expected) - assert res.const.isnull().sum() == 3 * c.load.sizes["snapshot"] + assert res.const.isnull().sum() == 3 * res.sizes["snapshot"] def test_cross_grid_merge_keeps_absent_cell_absent() -> None: require_v1() - c = base_model() - sparse, dense = cross_grid_parts(c, True), cross_grid_parts(c, False) + sparse, dense = twin_cross_grid_parts() mask = xr.DataArray([True, False], coords=[dense[1].indexes["bus"]]) dense[1] = dense[1].where(mask) - res = linopy.merge([sparse[0], dense[1]], join="outer", cls=LinearExpression) + flow = densified(sparse[1]).where(mask) + res = linopy.merge([sparse[0], flow], join="outer", cls=LinearExpression) assert res._csr is not None assert_terms_equal(res, linopy.merge(dense, join="outer", cls=LinearExpression)) def test_cross_grid_merge_mixed_dense_operand_stays_csr() -> None: require_v1() - c = base_model() - sparse, dense = cross_grid_parts(c, True), cross_grid_parts(c, False) - res = sparse[0].add(dense[1], join="outer") + sparse, dense = twin_cross_grid_parts() + res = sparse[0].add(densified(sparse[1]), join="outer") assert isinstance(res, LinearExpression) assert res._csr is not None expected = dense[0].add(dense[1], join="outer") @@ -881,28 +900,29 @@ def test_cross_grid_merge_mixed_dense_operand_stays_csr() -> None: ) def test_cross_grid_merge_raises_like_dense(kwargs: dict, error: type) -> None: require_v1() - c = base_model() + sparse, dense = twin_cross_grid_parts() with pytest.raises(error): - linopy.merge(cross_grid_parts(c, False), **kwargs) + linopy.merge(dense, **kwargs) with pytest.raises(error): - linopy.merge(cross_grid_parts(c, True), **kwargs) + linopy.merge(sparse, **kwargs) @pytest.mark.parametrize("join", ["outer", "inner", "left", "right"]) def test_cross_grid_merge_with_aux_coord_operand_stays_csr(join: JoinOptions) -> None: """The aux coord follows its rows onto the joined grid, like dense.""" require_v1() - c = base_model() - sparse, dense = cross_grid_parts(c, True), cross_grid_parts(c, False) + sparse, dense = twin_cross_grid_parts() tag = xr.DataArray(["x", "y"], coords=[dense[1].indexes["bus"]]) - tagged = LinearExpression(dense[1].data.assign_coords(tag=tag), c.m) + tagged = densified(sparse[1]).assign_coords(tag=tag) with no_densify(): res = linopy.merge([sparse[0], tagged], join=join, cls=LinearExpression) assert res.is_sparse labels = res.indexes["bus"] want = tag.to_series().reindex(labels).to_numpy() assert pd.Series(res.coords["tag"].values).equals(pd.Series(want)) - want_expr = linopy.merge([dense[0], tagged], join=join, cls=LinearExpression) + want_expr = linopy.merge( + [dense[0], dense[1].assign_coords(tag=tag)], join=join, cls=LinearExpression + ) assert_sparse_matches(res, want_expr) @@ -919,41 +939,34 @@ def test_cross_grid_merge_aux_coord_conflict_raises_like_dense() -> None: def test_cross_grid_merge_with_duplicate_labels_raises_like_dense() -> None: require_v1() - c = base_model() dup = pd.Index(["bus1", "bus1", "bus2"], name="bus") - shed = 1.0 * c.m.add_variables( - coords=[dup, c.load.indexes["snapshot"]], name="shed" - ) - gen = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - dense = (c.eff * c.gen_p).groupby(c.gbus).sum() - with pytest.raises(ValueError, match="cannot reindex or align"): - linopy.merge([dense, shed], join="left") - with pytest.raises(ValueError, match="cannot reindex or align"): - linopy.merge([gen, shed], join="left") + for c in twin_models(): + shed = 1.0 * c.m.add_variables( + coords=[dup, c.load.indexes["snapshot"]], name="shed" + ) + with pytest.raises(ValueError, match="cannot reindex or align"): + linopy.merge([c.gen_sum(), shed], join="left") def test_override_merge_same_shape_stays_csr() -> None: require_v1() - c = base_model() - gen = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - flow = (1.0 * c.flow).groupby(c.bus1.str.upper()).sum(sparse=True) - res = linopy.merge([gen, flow], join="override", cls=LinearExpression) + dense, sparse = ( + [c.gen_sum(), (1.0 * c.flow).groupby(c.bus1.str.upper()).sum()] + for c in twin_models() + ) + res = linopy.merge(sparse, join="override", cls=LinearExpression) assert res._csr is not None - dense = [ - (c.eff * c.gen_p).groupby(c.gbus).sum(), - (1.0 * c.flow).groupby(c.bus1.str.upper()).sum(), - ] assert_terms_equal(res, linopy.merge(dense, join="override", cls=LinearExpression)) def test_cross_grid_balance_freezes_csr() -> None: require_v1() - c1, c2 = base_model(), base_model() - lhs1 = linopy.merge(cross_grid_parts(c1, False), join="outer") + c1, c2 = twin_models() + lhs1 = linopy.merge(cross_grid_parts(c1), join="outer") con1 = c1.m.add_constraints(lhs1 == c1.load, name="bal") - lhs2 = linopy.merge(cross_grid_parts(c2, True), join="outer", cls=LinearExpression) + lhs2 = linopy.merge(cross_grid_parts(c2), join="outer", cls=LinearExpression) assert lhs2._csr is not None - con2 = c2.m.add_constraints(lhs2 == c2.load, name="bal", freeze=True) + con2 = c2.m.add_constraints(lhs2 == c2.load, name="bal") assert isinstance(con2, CSRConstraint) assert_frozen_equal(con1, con2) @@ -963,11 +976,11 @@ def test_cross_grid_merge_peak_memory() -> None: sizes = (200,) + (1,) * 299 n_snap = 50 dense_rectangle_bytes = len(sizes) * n_snap * max(sizes) * 16 - c = base_model(gens_per_bus=sizes, n_snap=n_snap) + c = base_model(gens_per_bus=sizes, n_snap=n_snap, sparse=True) tracemalloc.start() try: - lhs = linopy.merge(cross_grid_parts(c, True), join="outer") - c.m.add_constraints(lhs == c.load, name="bal", freeze=True) + lhs = linopy.merge(cross_grid_parts(c), join="outer") + c.m.add_constraints(lhs == c.load, name="bal") _, peak = tracemalloc.get_traced_memory() finally: tracemalloc.stop() @@ -1127,13 +1140,12 @@ def loc_operand(index: pd.Index) -> xr.DataArray: def tagged_group(c: Case) -> LinearExpression: """Generation grouped into ``group`` with ``bus``/``tag`` as auxiliary coords.""" grouper = pd.DataFrame({"bus": c.gbus, "tag": c.gbus}) - return (1.0 * c.gen_p).groupby(grouper).sum(sparse=True, observed=True) + return (1.0 * c.gen_p).groupby(grouper).sum(observed=True) def test_contracted_keeps_aux_coords_on_kept_dims_only() -> None: require_v1() - c = base_model() - csr = tagged_group(c)._csr + csr = tagged_group(base_model(sparse=True))._csr assert csr is not None assert set(csr.grid.aux) == {"bus", "tag"} kept = csr.contracted( @@ -1173,14 +1185,14 @@ def test_grid_ops_carry_aux_coords( op: Callable[[Grid], Grid], aux_dims: dict[str, str] ) -> None: require_v1() - csr = tagged_group(base_model())._csr + csr = tagged_group(base_model(sparse=True))._csr assert csr is not None assert {n: d for n, (d, _) in op(csr.grid).aux.items()} == aux_dims def test_grid_conformed_reindexes_aux_coords_and_equality_sees_them() -> None: require_v1() - csr = tagged_group(base_model())._csr + csr = tagged_group(base_model(sparse=True))._csr assert csr is not None grid = csr.grid labels = grid.indexes["group"][::-1] @@ -1311,8 +1323,7 @@ def test_matmul_dimensionless_expression_matches_dense() -> None: def test_matmul_keeps_aux_coords_on_kept_dims_only() -> None: require_v1() - c = base_model() - grouped = tagged_group(c) + grouped = tagged_group(base_model(sparse=True)) assert grouped._csr is not None indexes = grouped._csr.grid.indexes @@ -1337,24 +1348,19 @@ def test_matmul_output_dim_order_matches_dense() -> None: def test_matmul_keeps_csr_backing_through_chain() -> None: require_v1() - c = base_model() - snaps = c.gen_p.indexes["snapshot"] - operand = loc_operand(snaps) - gen = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True) - flow = (1.0 * c.flow).groupby(c.bus0).sum(sparse=True) + c1, c2 = twin_models() + operand = loc_operand(c2.gen_p.indexes["snapshot"]) + gen = c2.gen_sum() + flow = (1.0 * c2.flow).groupby(c2.bus0).sum() res = gen @ operand assert res._csr is not None assert_linequal(res, gen.dot(operand)) - assert_cells_equal( - res, - (c.eff * c.gen_p).groupby(c.gbus).sum() @ operand, - ("bus", "loc"), - ) + assert_cells_equal(res, c1.gen_sum() @ operand, ("bus", "loc")) total = res + (flow @ operand) assert total._csr is not None - con = c.m.add_constraints(total <= 0, name="matmul", freeze=True) + con = c2.m.add_constraints(total <= 0, name="matmul") assert isinstance(con, CSRConstraint) @@ -1406,15 +1412,17 @@ def test_matmul_peak_memory() -> None: def full_contraction(c: Case) -> LinearExpression: - grouped = (1.0 * c.gen_p).groupby(c.gbus).sum(sparse=True) + grouped = (1.0 * c.gen_p).groupby(c.gbus).sum() ones = xr.DataArray(np.ones(grouped.shape[:2]), coords=grouped.coords) return grouped @ ones SPARSE_BUILDS: dict[str, Callable[[Case], LinearExpression]] = { - "grouped": lambda c: (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=True), + "grouped": Case.gen_sum, "aux": tagged_group, - "absent": lambda c: keyed_model()[1].groupby(["period", "season"]).sum(sparse=True), + "absent": lambda c: ( + keyed_model(sparse=c.m.sparse)[1].groupby(["period", "season"]).sum() + ), "zero-dim": full_contraction, } @@ -1423,12 +1431,19 @@ def sparse_and_dense( build: str, c: Case | None = None ) -> tuple[LinearExpression, LinearExpression]: """A sparse build and its dense conversion, the sparse one left sparse.""" - sparse = SPARSE_BUILDS[build](base_model() if c is None else c) + sparse = SPARSE_BUILDS[build](base_model(sparse=True) if c is None else c) csr = sparse._csr assert sparse.is_sparse and csr is not None return sparse, csr.to_dense() +def dense_build(build: str) -> LinearExpression: + """The build in a dense model, on the dense path.""" + dense = SPARSE_BUILDS[build](base_model()) + assert not dense.is_sparse + return dense + + METADATA: dict[str, Callable[[LinearExpression], Any]] = { "shape": lambda e: e.shape, "size": lambda e: e.size, @@ -1460,7 +1475,7 @@ def test_metadata_is_served_without_densifying(build: str, attr: str) -> None: def test_is_sparse_tracks_backing_and_repr_marks_it() -> None: require_v1() - c = base_model() + c = base_model(sparse=True) sparse = SPARSE_BUILDS["grouped"](c) assert sparse.is_sparse assert repr(sparse).startswith("LinearExpression (sparse) [") @@ -1471,8 +1486,11 @@ def test_is_sparse_tracks_backing_and_repr_marks_it() -> None: def add_chunked(e: LinearExpression, c: Case) -> Any: - c.m.chunk = {"bus": 2} - return c.m.add_constraints(e >= 1, freeze=True) + chunked = base_model() + chunked.m.chunk = {"bus": 2} + with pytest.warns(FutureWarning, match="deprecated"): + lhs = (chunked.eff * chunked.gen_p).groupby(chunked.gbus).sum(sparse=True) + return chunked.m.add_constraints(lhs >= 1, freeze=True) DENSIFY_OPS: dict[str, tuple[Callable[[LinearExpression, Case], Any], str]] = { @@ -1486,7 +1504,7 @@ def add_chunked(e: LinearExpression, c: Case) -> Any: "rhs-expr": (lambda e, c: e <= 1.0 * c.gen_p.sum("gen"), "over different dim"), "mutable": (lambda e, c: (e == c.load).mutable(), "`mutable\\(\\)`"), "add-unfrozen": ( - lambda e, c: c.m.add_constraints(e >= 1), + lambda e, c: c.m.add_constraints(e >= 1, freeze=False), "constraint added unfrozen, `freeze=False`", ), "add-chunked": (add_chunked, "chunked model"), @@ -1521,7 +1539,7 @@ def halves(e: LinearExpression) -> pd.Series: @pytest.mark.parametrize("op", list(DENSIFY_OPS)) def test_warn_on_densify_names_the_reason(op: str, enabled: bool) -> None: require_v1() - c = base_model() + c = base_model(sparse=True) sparse = SPARSE_BUILDS["grouped"](c) func, reason = DENSIFY_OPS[op] with linopy.options as opts, warnings.catch_warnings(record=True) as caught: @@ -1557,7 +1575,7 @@ def test_sparse_store_indices_follow_model_label_dtype(build: str) -> None: require_v1() sparse, _ = sparse_and_dense(build) m = sparse.model - con = m.add_constraints(sparse >= 1, name="con", freeze=True) + con = m.add_constraints(sparse >= 1, name="con") A = m.matrices.A assert isinstance(con, CSRConstraint) and sparse._csr is not None assert A is not None @@ -1568,10 +1586,10 @@ def test_sparse_store_indices_follow_model_label_dtype(build: str) -> None: @pytest.mark.parametrize("label_dtype", [np.int32, np.int64]) def test_sparse_group_sum_indices_widen_with_model(label_dtype: type) -> None: require_v1() - m = Model(dtypes={"labels": label_dtype}) + m = Model(dtypes={"labels": label_dtype}, sparse=True) x = m.add_variables(coords=[pd.RangeIndex(4, name="i")], name="x") group = xr.DataArray([0, 0, 1, 1], coords=[x.indexes["i"]], name="g") - expr = x.groupby(group).sum(sparse=True) + expr = x.groupby(group).sum() assert expr._csr is not None assert expr._csr.csr.indices.dtype == expr._csr.csr.indptr.dtype == label_dtype @@ -1711,7 +1729,7 @@ def test_merge_with_dense_expression_stays_csr_and_matches_dense( build: str, addend: str ) -> None: require_v1() - c = base_model() + c = base_model(sparse=True) sparse, dense = sparse_and_dense(build, c) other = DENSE_ADDENDS[addend](c, dense) with no_densify(): @@ -1793,12 +1811,11 @@ def test_join_with_constant_keeps_aux_coords_like_dense( def test_to_constraint_join_with_constant_matches_dense(join: JoinOptions) -> None: require_v1() cons = [] - for sparse in (False, True): - c = base_model() - lhs = (c.eff * c.gen_p).groupby(c.gbus).sum(sparse=sparse) + for c in twin_models(): + lhs = c.gen_sum() with no_densify(): con = lhs.to_constraint("<=", c.load.isel(bus=[0, 2]), join=join) - cons.append(c.m.add_constraints(con, name="c", freeze=sparse)) + cons.append(c.m.add_constraints(con, name="c")) dense, frozen = cons assert isinstance(frozen, CSRConstraint) xr.testing.assert_identical( @@ -1853,8 +1870,8 @@ def test_sum_stays_csr_and_matches_dense(build: str, dims: str) -> None: @pytest.mark.parametrize("drop_zeros", [False, True]) def test_sum_keeps_explicit_zeros_unless_dropped(drop_zeros: bool) -> None: require_v1() - c = base_model() - sparse = (0.0 * c.gen_p).groupby(c.gbus).sum(sparse=True) + c = base_model(sparse=True) + sparse = (0.0 * c.gen_p).groupby(c.gbus).sum() assert sparse._csr is not None dense = sparse._csr.to_dense() res = sparse.sum("snapshot", drop_zeros=drop_zeros) @@ -1888,25 +1905,28 @@ def test_sum_unknown_dim_raises_like_dense() -> None: @pytest.mark.parametrize("build", ["grouped", "aux", "absent"]) def test_chained_groupby_stays_csr_and_matches_dense(build: str, chain: str) -> None: require_v1() - sparse, dense = sparse_and_dense(build) + sparse, _ = sparse_and_dense(build) func = CHAINS[chain] with no_densify(): res = func(sparse) - assert_sparse_matches(res, func(dense)) + assert_sparse_matches(res, func(dense_build(build))) @pytest.mark.parametrize("observed", [False, True]) def test_chained_namelist_groupby_on_aux_coords_matches_dense(observed: bool) -> None: require_v1() - sparse, dense = sparse_and_dense("aux") + sparse, _ = sparse_and_dense("aux") + dense = dense_build("aux") res = sparse.groupby(["bus", "tag"]).sum(observed=observed) assert_sparse_matches(res, dense.groupby(["bus", "tag"]).sum(observed=observed)) def test_chained_groupby_sparse_false_densifies() -> None: require_v1() - sparse, dense = sparse_and_dense("grouped") - res = sparse.groupby(halves(sparse)).sum(sparse=False) + sparse, _ = sparse_and_dense("grouped") + dense = dense_build("grouped") + with pytest.warns(FutureWarning, match="deprecated"): + res = sparse.groupby(halves(sparse)).sum(sparse=False) assert not res.is_sparse assert_linequal(res, dense.groupby(halves(dense)).sum()) @@ -1969,7 +1989,7 @@ def test_selection_stays_csr_and_matches_dense(build: str, select: str) -> None: @pytest.mark.parametrize("op", list(ABSENT_KEEPING_OPS)) def test_absent_cells_stay_termless(op: str) -> None: require_v1() - sparse = SPARSE_BUILDS["absent"](base_model()) + sparse = SPARSE_BUILDS["absent"](base_model(sparse=True)) csr = ABSENT_KEEPING_OPS[op](sparse)._csr assert csr is not None absent = np.isnan(csr.const) @@ -1979,9 +1999,9 @@ def test_absent_cells_stay_termless(op: str) -> None: def test_scalar_selection_coords_merge_like_dense() -> None: require_v1() - c = base_model() + c = base_model(sparse=True) sparse, dense = sparse_and_dense("grouped", c) - flow = (1.0 * c.flow).groupby(c.bus0).sum() + flow = densified((1.0 * c.flow).groupby(c.bus0).sum()) with no_densify(): res = sparse.sel(snapshot=1) + flow.sel(snapshot=1) assert_sparse_matches(res, dense.sel(snapshot=1) + flow.sel(snapshot=1)) @@ -2010,7 +2030,7 @@ def test_where_on_mismatched_labels_raises_like_dense() -> None: @pytest.mark.parametrize("op", SELECTION_FALLBACKS) def test_selection_fallbacks_match_dense(op: str) -> None: require_v1() - c = base_model() + c = base_model(sparse=True) sparse, dense = sparse_and_dense("grouped", c) func = DENSIFY_OPS[op][0] res = func(sparse, c) @@ -2031,46 +2051,46 @@ def test_add_constraints_mask_freezes_sparse_and_matches_dense( mask: str, prebuilt: bool ) -> None: require_v1() - c1, c2 = base_model(), base_model() - lhs = c2.balance_lhs(sparse=True) + c1, c2 = twin_models() + lhs = c2.balance_lhs() m = MASKS[mask](lhs) - con1 = c1.m.add_constraints(c1.balance_lhs(False), ">=", c1.load, "bal", mask=m) + con1 = c1.m.add_constraints(c1.balance_lhs(), ">=", c1.load, "bal", mask=m) with no_densify(): if prebuilt: - con2 = c2.m.add_constraints(lhs >= c2.load, name="bal", mask=m, freeze=True) + con2 = c2.m.add_constraints(lhs >= c2.load, name="bal", mask=m) else: - con2 = c2.m.add_constraints(lhs, ">=", c2.load, "bal", mask=m, freeze=True) + con2 = c2.m.add_constraints(lhs, ">=", c2.load, "bal", mask=m) assert isinstance(con2, CSRConstraint) assert_frozen_equal(con1, con2) def observed_keys_group() -> LinearExpression: """The #941 reproducer: a multi-key observed grouping with aux coords on ``group``.""" - m = Model() + m = Model(sparse=True) x = m.add_variables(coords=[pd.RangeIndex(4, name="s")], name="x") expr = x.to_linexpr().assign_coords( period=("s", [1, 1, 2, 2]), region=("s", ["n", "s", "n", "s"]) ) - return expr.groupby(["period", "region"]).sum(observed=True, sparse=True) + return expr.groupby(["period", "region"]).sum(observed=True) def dashed_group() -> LinearExpression: """An observed grouping whose dims and aux coords carry dashes, the netcdf name separator.""" - m = Model() + m = Model(sparse=True) s = pd.RangeIndex(4, name="my-s") x = m.add_variables(coords=[s], name="x") grouper = pd.DataFrame( {"my-bus": ["a", "a", "b", "b"], "tag": [1, 1, 2, 2]}, index=s ) - grouped = x.to_linexpr().groupby(grouper).sum(sparse=True, observed=True) + grouped = x.to_linexpr().groupby(grouper).sum(observed=True) return grouped.rename({"group": "my-group"}) AUX_BUILDS: dict[str, Callable[[], LinearExpression]] = { "observed-keys": observed_keys_group, "dashed": dashed_group, - "aux": lambda: SPARSE_BUILDS["aux"](base_model()), - "scalar": lambda: SPARSE_BUILDS["grouped"](base_model()).sel(snapshot=1), + "aux": lambda: SPARSE_BUILDS["aux"](base_model(sparse=True)), + "scalar": lambda: SPARSE_BUILDS["grouped"](base_model(sparse=True)).sel(snapshot=1), } @@ -2082,9 +2102,9 @@ def test_frozen_constraint_keeps_aux_coords_like_dense( sparse = AUX_BUILDS[build]() m = sparse.model assert sparse._csr is not None - ref = m.add_constraints(sparse._csr.to_dense() >= 1, name="dense") + ref = m.add_constraints(sparse._csr.to_dense() >= 1, name="dense", freeze=False) with no_densify(): - con = m.add_constraints(sparse >= 1, name="sparse", freeze=True) + con = m.add_constraints(sparse >= 1, name="sparse") assert isinstance(con, CSRConstraint) want = xr.Dataset(coords=ref.coords) assert set(want.coords) > set(con.coord_names) @@ -2113,8 +2133,8 @@ def setter(attr: str) -> Callable[[CSRConstraint], None]: @pytest.mark.parametrize("op", list(FROZEN_MUTATIONS)) def test_frozen_constraint_mutation_names_mutable(op: str) -> None: require_v1() - c = base_model() - con = c.m.add_constraints(c.balance_lhs(sparse=True) >= c.load, freeze=True) + c = base_model(sparse=True) + con = c.m.add_constraints(c.balance_lhs() >= c.load) assert isinstance(con, CSRConstraint) with pytest.raises(AttributeError, match=rf"CSRConstraint\.{op} .*\.mutable\(\)"): FROZEN_MUTATIONS[op](con) @@ -2131,20 +2151,20 @@ def test_frozen_constraint_mutation_names_mutable(op: str) -> None: @pytest.mark.parametrize("op", list(FROZEN_UNSUPPORTED)) def test_frozen_unsupported_names_the_working_route(op: str) -> None: require_v1() - c = base_model() - con = c.m.add_constraints(c.balance_lhs(sparse=True) >= c.load, freeze=True) + c = base_model(sparse=True) + con = c.m.add_constraints(c.balance_lhs() >= c.load) assert isinstance(con, CSRConstraint) call, remedy = FROZEN_UNSUPPORTED[op] with pytest.raises(AttributeError, match=rf"CSRConstraint\.{op} .*{remedy}"): call(con) -EXPR_RHS: dict[str, Callable[[Case, bool], LinearExpression]] = { - "expr": lambda c, sparse: (1.0 * c.flow).groupby(c.bus1).sum(sparse=sparse), - "expr-const": lambda c, sparse: ( - (1.0 * c.flow).groupby(c.bus1).sum(sparse=sparse) + c.load +EXPR_RHS: dict[str, Callable[[Case], LinearExpression]] = { + "expr": lambda c: (1.0 * c.flow).groupby(c.bus1).sum(), + "expr-const": lambda c: (1.0 * c.flow).groupby(c.bus1).sum() + c.load, + "dense-expr-const": lambda c: ( + (1.0 * c.flow).groupby(c.bus1).sum(use_fallback=True) - 2.0 ), - "dense-expr-const": lambda c, sparse: (1.0 * c.flow).groupby(c.bus1).sum() - 2.0, } @@ -2152,16 +2172,17 @@ def test_frozen_unsupported_names_the_working_route(op: str) -> None: @pytest.mark.parametrize("rhs", list(EXPR_RHS)) def test_expression_rhs_freezes_sparse_and_matches_dense(rhs: str, form: str) -> None: require_v1() - c1, c2 = base_model(), base_model() - lhs1 = (c1.eff * c1.gen_p).groupby(c1.gbus).sum() - con1 = c1.m.add_constraints(lhs1 <= EXPR_RHS[rhs](c1, False), name="c", freeze=True) - lhs2 = (c2.eff * c2.gen_p).groupby(c2.gbus).sum(sparse=True) - rhs2 = EXPR_RHS[rhs](c2, True) + c1, c2 = twin_models() + con1 = c1.m.add_constraints( + c1.gen_sum() <= EXPR_RHS[rhs](c1), name="c", freeze=True + ) + lhs2 = c2.gen_sum() + rhs2 = EXPR_RHS[rhs](c2) with no_densify(): if form == "operator": - con2 = c2.m.add_constraints(lhs2 <= rhs2, name="c", freeze=True) + con2 = c2.m.add_constraints(lhs2 <= rhs2, name="c") else: - con2 = c2.m.add_constraints(lhs2, "<=", rhs2, name="c", freeze=True) + con2 = c2.m.add_constraints(lhs2, "<=", rhs2, name="c") assert isinstance(con2, CSRConstraint) assert_frozen_equal(con1, con2) @@ -2180,13 +2201,13 @@ def softened( max_violation: float | None, freeze: bool, ) -> tuple[Case, ConstraintBase]: - c = base_model(sparse=freeze and route == "penalty-default") + c = base_model(sparse=freeze) c.m.add_objective(1.0 * c.gen_p.sum()) mask = MASK_KINDS[mask_kind](c) - args = (c.balance_lhs(sparse=freeze), sign, c.load) + args = (c.balance_lhs(), sign, c.load) kwargs: dict[str, Any] = dict(name="c", mask=mask) if route == "soften": - con = c.m.add_constraints(*args, **kwargs, freeze=freeze) + con = c.m.add_constraints(*args, **kwargs) con.soften(penalty=2.0, max_violation=max_violation) else: kwargs["freeze"] = None if route == "penalty-default" else freeze @@ -2234,22 +2255,19 @@ def test_frozen_soften_matches_dense( def test_frozen_soften_keeps_sparse_objective() -> None: require_v1() - c = base_model() + c = base_model(sparse=True) with no_densify(): - c.m.add_objective((1.0 * c.gen_p).groupby(c.gbus).sum(sparse=True).sum()) - lhs = c.balance_lhs(sparse=True) - c.m.add_constraints(lhs >= c.load, freeze=True, penalty=2.0) + c.m.add_objective((1.0 * c.gen_p).groupby(c.gbus).sum().sum()) + c.m.add_constraints(c.balance_lhs() >= c.load, penalty=2.0) assert c.m.objective.expression.is_sparse def test_frozen_soften_max_sense_with_array_penalty() -> None: require_v1() - c = base_model() + c = base_model(sparse=True) c.m.add_objective(1.0 * c.gen_p.sum(), sense="max") with no_densify(): - con = c.m.add_constraints( - c.balance_lhs(sparse=True), ">=", c.load, name="c", freeze=True - ) + con = c.m.add_constraints(c.balance_lhs(), ">=", c.load, name="c") assert isinstance(con, CSRConstraint) penalty = xr.full_like(c.load, 2.0) slack = con.soften(penalty=penalty) @@ -2313,9 +2331,7 @@ def test_soften_rejects(error: str, freeze: bool) -> None: sign: Any = ">=" if error == "mixed-signs": sign = xr.DataArray(np.where(c.load > 4, ">=", "<="), coords=c.load.coords) - con = c.m.add_constraints( - c.balance_lhs(sparse=False), sign, c.load, name="c", freeze=freeze - ) + con = c.m.add_constraints(c.balance_lhs(), sign, c.load, name="c", freeze=freeze) assert isinstance(con, CSRConstraint) == freeze call, exc, match = SOFTEN_ERRORS[error] with pytest.raises(exc, match=match): @@ -2323,14 +2339,12 @@ def test_soften_rejects(error: str, freeze: bool) -> None: def frozen_model(soften: bool) -> tuple[Model, CSRConstraint]: - c = base_model() + c = base_model(sparse=True) for var in (c.gen_p, c.flow): var.update(lower=0, upper=1) with no_densify(): - c.m.add_objective((1.0 * c.gen_p).groupby(c.gbus).sum(sparse=True).sum()) - con = c.m.add_constraints( - c.balance_lhs(sparse=True), ">=", c.load, name="c", freeze=True - ) + c.m.add_objective((1.0 * c.gen_p).groupby(c.gbus).sum().sum()) + con = c.m.add_constraints(c.balance_lhs(), ">=", c.load, name="c") if soften: con.soften(penalty=2.0, max_violation=20.0) assert isinstance(con, CSRConstraint) @@ -2369,7 +2383,8 @@ def test_softening_frozen_copy_leaves_original(deep: bool) -> None: @pytest.mark.parametrize("deep", [True, False]) def test_copy_frozen_matrices(deep: bool) -> None: - m = Model(freeze_constraints=True) + require_v1() + m = Model(sparse=True) i = pd.RangeIndex(4, name="i") x = m.add_variables(lower=0, coords=[i], name="x") b = m.add_variables(coords=[i], binary=True, name="b") diff --git a/test/test_indicator_constraints.py b/test/test_indicator_constraints.py index 22962edd9..745f97b41 100644 --- a/test/test_indicator_constraints.py +++ b/test/test_indicator_constraints.py @@ -236,10 +236,12 @@ def test_copy_preserves(self, mbx: tuple[Model, Variable, Variable]) -> None: assert ic.binary_var is not None assert np.all(ic.binary_val == 1) - @pytest.mark.parametrize("freeze_constraints", [False, True]) - def test_netcdf_roundtrip(self, tmp_path: Path, freeze_constraints: bool) -> None: + @pytest.mark.parametrize( + "sparse", [False, pytest.param(True, marks=pytest.mark.v1)] + ) + def test_netcdf_roundtrip(self, tmp_path: Path, sparse: bool) -> None: """is_indicator and binary fields survive a netCDF round-trip.""" - m = Model(freeze_constraints=freeze_constraints) + m = Model(sparse=sparse) b = m.add_variables(name="b", binary=True) x = m.add_variables(lower=0, upper=10, name="x") m.add_constraints(x >= 0, name="regular") @@ -253,9 +255,10 @@ def test_netcdf_roundtrip(self, tmp_path: Path, freeze_constraints: bool) -> Non assert ic.binary_var is not None assert np.all(ic.binary_val == 1) + @pytest.mark.v1 def test_array_binval_roundtrip(self, tmp_path: Path) -> None: """A coords-based indicator has per-element binary_val that round-trips.""" - m = Model(freeze_constraints=True) + m = Model(sparse=True) idx = pd.RangeIndex(3, name="i") b = m.add_variables(coords=[idx], name="b", binary=True) x = m.add_variables(coords=[idx], lower=0, upper=10, name="x") diff --git a/test/test_linear_expression.py b/test/test_linear_expression.py index 186b78253..a01df59fc 100644 --- a/test/test_linear_expression.py +++ b/test/test_linear_expression.py @@ -31,6 +31,7 @@ options, ) from linopy.constants import FACTOR_DIM, HELPER_DIMS, TERM_DIM +from linopy.csr import CSRLinearExpression from linopy.expressions import ScalarLinearExpression from linopy.semantics import is_v1 from linopy.testing import assert_linequal, assert_quadequal @@ -689,14 +690,13 @@ def test_matmul_expr_and_const(x: Variable, y: Variable) -> None: def backed(expr: LinearExpression, backing: str) -> LinearExpression: - """The same expression, dense or CSR-backed through an identity groupby.""" + """The same expression, dense or CSR-backed.""" if backing == "dense": return expr if not is_v1(): pytest.skip("CSR backing requires v1 semantics") - dim = str(expr.coord_dims[-1]) - idx = expr.indexes[dim] - return expr.groupby(pd.Series(idx, index=idx, name=dim)).sum(sparse=True) + csr = CSRLinearExpression.from_dense(expr.data, expr.model) + return LinearExpression._from_csr(csr, expr.model) def assert_matmul_equal(res: LinearExpression, reference: LinearExpression) -> None: diff --git a/test/test_model.py b/test/test_model.py index de0b3cb61..1b93c5c60 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -44,16 +44,24 @@ def test_model_solver_dir() -> None: assert m.solver_dir == Path(d) -def test_model_config_defaults() -> None: - m = Model(freeze_constraints=True, set_names_in_solver_io=False) - assert m.freeze_constraints is True +SPARSE_CONFIG = pytest.mark.parametrize( + "sparse", [False, pytest.param(True, marks=pytest.mark.v1)] +) + + +@SPARSE_CONFIG +def test_model_config_defaults(sparse: bool) -> None: + m = Model(sparse=sparse, set_names_in_solver_io=False) + assert m.sparse is sparse + assert m.freeze_constraints is sparse assert m.set_names_in_solver_io is False -def test_model_copy_preserves_config() -> None: - m = Model(freeze_constraints=True, set_names_in_solver_io=False) - copied = m.copy() - assert copied.freeze_constraints is True +@SPARSE_CONFIG +def test_model_copy_preserves_config(sparse: bool) -> None: + copied = Model(sparse=sparse, set_names_in_solver_io=False).copy() + assert copied.sparse is sparse + assert copied.freeze_constraints is sparse assert copied.set_names_in_solver_io is False @@ -168,7 +176,7 @@ def test_remove_masked_variable_keeps_unrelated_constraints( freeze: bool, quadratic: bool ) -> None: # https://github.com/PyPSA/linopy/issues/883 - m: Model = Model(freeze_constraints=freeze) + m: Model = Model() i = pd.Index(range(3), name="i") mask = [True, False, True] @@ -178,10 +186,12 @@ def test_remove_masked_variable_keeps_unrelated_constraints( # `b` is masked, so the constraint carries empty term slots (-1), but it # never references `a` - without_a = m.add_constraints(b.sum() + c, EQUAL, 0, name="without_a") + without_a = m.add_constraints( + b.sum() + c, EQUAL, 0, name="without_a", freeze=freeze + ) assert not without_a.has_variable(a) - with_a = m.add_constraints(a.sum() + c, EQUAL, 0, name="with_a") + with_a = m.add_constraints(a.sum() + c, EQUAL, 0, name="with_a", freeze=freeze) assert with_a.has_variable(a) if quadratic: diff --git a/test/test_solvers.py b/test/test_solvers.py index 55069aa97..4a90b6a0c 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -170,9 +170,9 @@ def test_assign_result_explicit(simple_model: Model) -> None: def test_assign_result_with_csr_constraints_avoids_data_reconstruction( monkeypatch: pytest.MonkeyPatch, ) -> None: - m = Model(freeze_constraints=True) + m = Model() x = m.add_variables(coords=[range(3)], name="x") - m.add_constraints(x >= 0, name="c") + m.add_constraints(x >= 0, name="c", freeze=True) con = m.constraints["c"] assert isinstance(con, CSRConstraint)