diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a1805c27..ed490f9f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -16,9 +16,11 @@ Upcoming Version * conflicting auxiliary coordinates raise, instead of being silently dropped. * a reindexing ``join=`` fills the positions it creates per side: the linopy operand contributes the zero expression, the constant operand contributes ``fill_value=`` (new on ``.add`` / ``.sub`` / ``.mul`` / ``.div``), which defaults to "does not apply here" — so a missing divisor zeroes the term instead of leaving it unscaled, and a missing numerator no longer comes out as ``1 / divisor``. Absence carried in by an operand is untouched and still propagates. Pass ``fill_value=linopy.ABSENT`` to opt out of the fill altogether — the join's coordinates are kept but every position it creates stays absent, on constant and expression operands alike (``linopy.merge`` included). (https://github.com/PyPSA/linopy/issues/890) * a first-class ``pd.MultiIndex`` dimension is rejected in favour of a flat dimension with the levels as auxiliary coordinates. + * a quadratic product (``x * y``, ``expr * expr``) over shared dimensions with a differing label set or a reorder raises, instead of being silently outer-joined by label; align the factors with ``.sel`` / ``.reindex`` / ``linopy.align`` first, as the product takes no ``join=``. * a ``groupby`` grouper aligns to the grouped dimension by label — a grouper whose labels differ in set or order from the dimension raises instead of being matched positionally, and a multi-key grouper yields a flat ``group`` dimension (keys as aux coords) rather than a stacked ``group`` MultiIndex. (https://github.com/PyPSA/linopy/issues/827) * Every operation whose result changes under v1 emits a ``LinopySemanticsWarning`` under legacy, naming the fix — so a model can be migrated incrementally before opting in. The full rules are specified in :doc:`the arithmetic convention `. +* The legacy warnings now fire where the legacy and v1 results actually diverge, once per operation and pointing at the user's code. Arithmetic that fills an expression's absent slots with 0 (``x.diff("t") >= 0``, ``expr.shift(t=1) + y``, ``expr.where(mask) + 5``, ``expr.reindex(...) + 1``) warns, as does a constraint of a masked variable or expression against a constant (``x.where(mask) <= 3``, which legacy keeps as a labelled empty row and v1 drops), a reindexing ``join=`` that creates divisor positions on ``.div`` (filled with 1 under legacy, ``inf`` under v1) and a quadratic product over mismatched or reordered coordinates, which keeps integer labels under legacy. Reductions (``sum``, ``groupby(...).sum()``) of a masked variable no longer warn, as their results agree. The coordinate-mismatch messages describe the alignment legacy actually applies: positional when sizes match, otherwise an outer join by label for a merge; by label for a quadratic product, a constraint RHS (ignoring an explicit ``join=``, which v1 applies) and ``Variable * constant``. *Numerical scaling* diff --git a/linopy/alignment.py b/linopy/alignment.py index df85bd08..1f521fd9 100644 --- a/linopy/alignment.py +++ b/linopy/alignment.py @@ -791,8 +791,7 @@ def _reindex_reordered_dims( warn_legacy( _legacy_const_reorder_message( str(dim), expected_idx.values, actual_idx.values - ), - stacklevel=6, + ) ) arr = arr.reindex({dim: expected_idx}) return arr diff --git a/linopy/expressions.py b/linopy/expressions.py index bc811abd..2146513d 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -116,6 +116,7 @@ FillValueLike, _legacy_coord_mismatch_message, _legacy_group_multiindex_message, + _legacy_join_divisor_fill_message, _legacy_nan_rhs_constraint_message, _shared_dim_mismatch_message, absorb_absence, @@ -127,8 +128,10 @@ is_nan_scalar, is_v1, join_fill, + legacy_warnings_muted, reindex_like_if_needed, warn_legacy, + warn_legacy_absence, warn_outside_linopy, ) from linopy.types import ( @@ -1073,13 +1076,25 @@ def _multiply_by_linear_expression( # multiplication: (v1 + c1) * (v2 + c2) = v1 * v2 + c1 * v2 + c2 * v1 + c1 * c2 # with v being the variables and c the constants # merge on factor dimension only returns v1 * v2 + c1 * c2 + enforce_merge_dims( + [self.data, other.data], + concat_dim=FACTOR_DIM, + context="quadratic product", + alignment="by label, as an outer join", + joinable=False, + ) + warn_legacy_absence(self.const, other.const) ds = other.data[["coeffs", "vars"]].sel(_term=0).broadcast_like(self.data) ds = assign_multiindex_safe(ds, const=other.const) - res = merge([self, ds], dim=FACTOR_DIM, cls=QuadraticExpression) - if self.has_constant: - res = res + other.reset_const() * self.const.fillna(0) - if other.has_constant: - res = res + self.reset_const() * other.const.fillna(0) + join: JoinOptions = ( + "override" if _coord_sizes_agree([self.data, ds]) else "outer" + ) + res = merge([self, ds], dim=FACTOR_DIM, cls=QuadraticExpression, join=join) + with legacy_warnings_muted(): + if self.has_constant: + res = res + other.reset_const() * self.const.fillna(0) + if other.has_constant: + res = res + self.reset_const() * other.const.fillna(0) return cast(QuadraticExpression, res) def _absorb_join_absence(self, fill_value: FillValueLike) -> Self: @@ -1132,7 +1147,7 @@ def _align_constant( rather than by ``xr.align(join="exact")``, whose wording is not API-stable across xarray releases. """ - enforce_aux_conflict([self.const, other], stacklevel=4) + enforce_aux_conflict([self.const, other]) other_fill = {other.name: join_fill(fill_value, 0)} if join is None: if is_v1(): @@ -1143,8 +1158,7 @@ def _align_constant( warn_legacy( _legacy_coord_mismatch_message( "this operator's constant operand", *mismatch - ), - stacklevel=4, + ) ) if other.sizes == self.const.sizes: return self.const, other.assign_coords(coords=self.coords), False @@ -1258,10 +1272,12 @@ def _add_constant_legacy( if np.isscalar(other) and join is None: if is_nan_scalar(other): check_user_nan() + warn_legacy_absence(self.const) return self.assign(const=self.const.fillna(0) + other) self_const, da, needs_reindex = self._broadcast_and_align( other, fill_value, join ) + warn_legacy_absence(self_const) expr = self._reindexed_to(self_const, needs_reindex) return expr.assign(const=self_const.fillna(0) + da.fillna(0)) @@ -1315,6 +1331,7 @@ def _apply_constant_op_legacy( self_const, factor, needs_reindex = self._broadcast_and_align( other, factor_fill, join, op_kind ) + warn_legacy_absence(self_const) factor = factor.fillna(nan_fill) self_const = self_const.fillna(0) expr = self._reindexed_to(self_const, needs_reindex) @@ -1343,8 +1360,14 @@ def _divide_by_constant( join: JoinOptions | None = None, fill_value: FillValueLike = None, ) -> Self: - if fill_value is None: - fill_value = np.inf if is_v1() else 1 + if fill_value is None and is_v1(): + fill_value = np.inf + elif fill_value is None: # LEGACY: remove at 1.0 + if join in ("left", "outer") and not _labels_within( + self.const, as_dataarray(other) + ): + warn_legacy(_legacy_join_divisor_fill_message()) + fill_value = 1 return self._apply_constant_op( other, operator.truediv, @@ -1912,19 +1935,14 @@ def to_constraint( rhs = self._broadcast_rhs(rhs) if rhs.isnull().any(): check_user_nan() - else: # LEGACY: remove at 1.0 — see doc/design/legacy-removal.rst. - if isinstance(rhs, CONSTANT_TYPES): - rhs = self._broadcast_rhs(rhs) - mismatch = first_mismatched_dim(self.const, rhs) - if mismatch is not None: - warn_legacy( - _legacy_coord_mismatch_message("constraint RHS", *mismatch) - ) - if bool(rhs.isnull().any()): - warn_legacy(_legacy_nan_rhs_constraint_message()) - rhs = rhs.reindex_like(self.const, fill_value=np.nan) - if isinstance(rhs, DataArray): - rhs_nan_mask = rhs.isnull() + elif isinstance(rhs, CONSTANT_TYPES): # LEGACY: remove at 1.0 + rhs = self._broadcast_rhs(rhs) + self._warn_legacy_rhs_alignment(rhs, join) + rhs_nan = rhs.isnull() + if bool(rhs_nan.any()): + warn_legacy(_legacy_nan_rhs_constraint_message()) + rhs_nan_mask = rhs_nan.reindex_like(self.const, fill_value=True) + rhs = rhs.fillna(0).reindex_like(self.const, fill_value=0) all_to_lhs = self.sub(rhs, join=join).data computed_rhs = -all_to_lhs.const @@ -1935,6 +1953,40 @@ def to_constraint( ) return constraints.Constraint(data, model=self.model) + # LEGACY: remove at 1.0 + def _warn_legacy_rhs_alignment( + self, rhs: DataArray, join: JoinOptions | None + ) -> None: + """ + Legacy reindexes a constant RHS onto the expression by label, whatever + ``join``; warn unless the RHS covers every label and ``join`` would not + add, drop or reorder any. + """ + mismatch = first_mismatched_dim(self.const, rhs) + if mismatch is None: + return + alignment = ( + "by label onto the expression, no constraint where the RHS lacks a label" + ) + if join is None: + warn_legacy( + _legacy_coord_mismatch_message( + "constraint RHS", *mismatch, alignment=alignment + ) + ) + return + covered = join in ("left", "inner", "outer") and _labels_within(self.const, rhs) + if covered and (join != "outer" or _labels_within(rhs, self.const)): + return + warn_legacy( + _legacy_coord_mismatch_message( + "constraint RHS", + *mismatch, + alignment=f"{alignment}, ignoring `join={join!r}`", + v1=f"Under v1 `join={join!r}` is applied instead.", + ) + ) + def _broadcast_rhs(self, rhs: ConstantLike) -> DataArray: """Broadcast a constant RHS onto the expression's coordinates.""" rhs = broadcast_to_coords( @@ -3429,11 +3481,11 @@ class QuadraticExpression(BaseExpression): _fill_value = {"vars": -1, "coeffs": np.nan, "const": np.nan} def __init__(self, data: Dataset | None, model: Model) -> None: - super().__init__(data, model) - if data is None: da = xr.DataArray([[], []], dims=[FACTOR_DIM, TERM_DIM]) - data = Dataset({"coeffs": da, "vars": da, "const": 0}) + data = Dataset({"coeffs": da, "vars": da, "const": 0.0}) + super().__init__(data, model) + data = self._data if FACTOR_DIM not in data.vars.dims: raise ValueError(f"Data does not include dimension {FACTOR_DIM}") elif data.sizes[FACTOR_DIM] != 2: @@ -3709,6 +3761,18 @@ def _densify_all(exprs: Iterable[Any], reason: str) -> None: e._csr = None +def _coord_sizes_agree(objs: Sequence[Any]) -> bool: + """Whether all ``objs`` carry the same non-helper dimension sizes.""" + sizes = [{k: v for k, v in o.sizes.items() if k not in HELPER_DIMS} for o in objs] + return check_common_keys_values(sizes) + + +def _labels_within(a: DataArray, b: DataArray) -> bool: + """Whether every label of ``a`` on a dim shared with ``b`` is also in ``b``.""" + shared = set(a.indexes) & set(b.indexes) + return all(a.indexes[d].isin(b.indexes[d]).all() for d in shared) + + def _aligned( csrs: list[CSRLinearExpression], join: JoinOptions | None, fill: float ) -> list[CSRLinearExpression] | str: @@ -3929,10 +3993,7 @@ def merge( if join is not None: override = join == "override" elif issubclass(cls, linopy_types) and dim in HELPER_DIMS: - coord_dims = [ - {k: v for k, v in e.sizes.items() if k not in HELPER_DIMS} for e in exprs - ] - override = check_common_keys_values(coord_dims) # type: ignore + override = _coord_sizes_agree(exprs) else: override = False @@ -3955,6 +4016,7 @@ def merge( skipna = not is_v1() if dim == TERM_DIM: + warn_legacy_absence(*(d["const"] for d in data)) ds = xr.concat([d[["coeffs", "vars"]] for d in data], dim, **kwargs) subkwargs = {**kwargs, "fill_value": {"const": join_fill(fill_value, 0)}} const = xr.concat([d[["const"]] for d in data], dim, **subkwargs)["const"] diff --git a/linopy/semantics.py b/linopy/semantics.py index ea44af51..7471cda1 100644 --- a/linopy/semantics.py +++ b/linopy/semantics.py @@ -14,7 +14,9 @@ import os import sys -from collections.abc import Sequence +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any, TypeAlias from warnings import warn @@ -40,17 +42,27 @@ def _user_nan_message() -> str: ) -def _shared_dim_mismatch_message(dim: str, left: Any, right: Any) -> str: - """Shared-dim error text — names the dim and shows the disagreeing labels.""" +def _shared_dim_mismatch_message( + dim: str, left: Any, right: Any, *, joinable: bool = True +) -> str: + """ + Shared-dim error text — names the dim and shows the disagreeing labels. + ``joinable=False`` drops the ``join=`` hint for operations that reject it. + """ + join_hint = ( + ", or by passing an explicit `join=` argument to `.add` / `.sub` /" + " `.mul` / `.div` / `.le` / `.ge` / `.eq` (accepts inner / outer /" + " left / right / override)." + if joinable + else "." + ) return ( f"Coordinate mismatch on shared dimension {dim!r}: " f"left={_short_repr(left)}, right={_short_repr(right)}. " "Resolve with `.sel(...)` / `.reindex(...)` to align before " "combining, with `.assign_coords(...)` to relabel one side " "(positional alignment, made explicit), with `linopy.align(...)` " - "to pre-align several operands at once, or by passing an explicit " - "`join=` argument to `.add` / `.sub` / `.mul` / `.div` / `.le` / " - "`.ge` / `.eq` (accepts inner / outer / left / right / override)." + "to pre-align several operands at once" + join_hint ) @@ -158,14 +170,29 @@ def check_user_nan_breakpoints(*, context: str = "breakpoints") -> None: warn_legacy(_legacy_nan_breakpoint_message(context)) +_LEGACY_POSITIONAL_ALIGNMENT = "positional when sizes match, otherwise left-join" +_LEGACY_MERGE_ALIGNMENT = "positional when sizes match, otherwise outer join by label" +_V1_RAISES = "Under v1 this raises ValueError." +_JOIN_HINT = "\n or pass an explicit `join=` argument." + + +def _join_hint(joinable: bool) -> str: + """The trailing ``join=`` resolve line, omitted where ``join=`` is rejected.""" + return _JOIN_HINT if joinable else "" + + def _legacy_coord_mismatch_message( context: str, dim: str | None = None, left: Any = None, right: Any = None, + *, + alignment: str = _LEGACY_POSITIONAL_ALIGNMENT, + v1: str = _V1_RAISES, + joinable: bool = True, ) -> str: """ - Mismatched dim coords silently aligned (positional or left-join). + Mismatched dim coords silently aligned the way ``alignment`` describes. When ``dim`` / ``left`` / ``right`` are given, the message names the offending dim and shows the diff — same shape as the v1-raise text @@ -178,26 +205,35 @@ def _legacy_coord_mismatch_message( ) return ( f"Coordinate mismatch in {context} silently aligned by legacy" - " (positional when sizes match, otherwise left-join)." - " Under v1 this raises ValueError." + f" ({alignment}). {v1}" + diff + "\n Resolve: `.sel(...)` / `.reindex(...)` to align" "\n `.assign_coords(...)` to relabel one side" "\n `linopy.align(...)` to pre-align several operands" - "\n or pass an explicit `join=` argument." + _OPT_IN_HINT + + _join_hint(joinable) + + _OPT_IN_HINT ) -def _legacy_coord_reorder_message(context: str, dim: str, left: Any, right: Any) -> str: - """Same labels, different order — aligned positionally by legacy; v1 raises.""" +def _legacy_coord_reorder_message( + context: str, + dim: str, + left: Any, + right: Any, + *, + alignment: str = _LEGACY_MERGE_ALIGNMENT, + joinable: bool = True, +) -> str: + """Same labels, different order — aligned the way ``alignment`` describes.""" return ( - f"Coordinate order mismatch in {context} aligned positionally by legacy." + f"Coordinate order mismatch in {context} aligned by legacy ({alignment})." " Under v1 the same labels in a different order raise ValueError (§8);" " reindex or sort one side to align by label." f"\n Dim: {dim!r}: left={_short_repr(left)}, right={_short_repr(right)}" "\n Resolve: `.sel(...)` / `.reindex(...)` to align" "\n `.assign_coords(...)` to relabel one side" - "\n or pass an explicit `join=` argument." + _OPT_IN_HINT + + _join_hint(joinable) + + _OPT_IN_HINT ) @@ -259,9 +295,30 @@ def _legacy_masked_variable_message(name: str) -> str: " propagates through arithmetic instead (`x + y` becomes absent" " at the slot and the constraint drops)." f"\n Resolve: wrap with `{name}.fillna(0)` for the legacy" - " behaviour under v1" - "\n (no fix needed if you only use the variable in a" - " constraint LHS alone — `y >= 0` drops the same way in both)." + _OPT_IN_HINT + " behaviour under v1." + _OPT_IN_HINT + ) + + +def _legacy_absent_operand_message() -> str: + """An expression's absent slots silently filled by legacy arithmetic.""" + return ( + "An operand has absent slots (from `.where()` / `.shift()` / `.diff()`" + " / `.reindex()`) that legacy silently filled with 0 (so `expr + y`" + " keeps `y` there). Under v1 the absence propagates through arithmetic" + " instead (the result is absent at the slot and a constraint built from" + " it drops the row)." + "\n Resolve: `.fillna(0)` on the operand for the legacy behaviour under" + " v1." + _OPT_IN_HINT + ) + + +def _legacy_join_divisor_fill_message() -> str: + """A reindexing join created divisor positions, filled with 1 by legacy.""" + return ( + "The explicit `join=` created divisor positions, which legacy silently" + " filled with 1 (leaving the term unscaled). Under v1 they take" + " `fill_value=inf`, so the term is zeroed there." + "\n Resolve: pass `fill_value=` explicitly." + _OPT_IN_HINT ) @@ -323,16 +380,29 @@ def warn_outside_linopy(message: str, category: type[Warning]) -> None: warn(message, category, stacklevel=stacklevel) -def warn_legacy(message: str, *, stacklevel: int | None = None) -> None: +_LEGACY_WARNINGS_MUTED: ContextVar[bool] = ContextVar( + "_LEGACY_WARNINGS_MUTED", default=False +) + + +@contextmanager +def legacy_warnings_muted() -> Iterator[None]: + """Silence legacy warnings for an operation that already emitted its own.""" + token = _LEGACY_WARNINGS_MUTED.set(True) + try: + yield + finally: + _LEGACY_WARNINGS_MUTED.reset(token) + + +def warn_legacy(message: str) -> None: """ Emit a `LinopySemanticsWarning` pointing at the first call-stack frame - outside the linopy package (see :func:`warn_outside_linopy`). Pass an - explicit ``stacklevel`` to override (e.g. for tests). + outside the linopy package (see :func:`warn_outside_linopy`). Nothing is + emitted inside :func:`legacy_warnings_muted`. """ - if stacklevel is None: + if not _LEGACY_WARNINGS_MUTED.get(): warn_outside_linopy(message, LinopySemanticsWarning) - else: - warn(message, LinopySemanticsWarning, stacklevel=stacklevel) def _short_repr(values: Any, limit: int = 6) -> str: @@ -365,6 +435,17 @@ def is_nan_scalar(value: Any) -> bool: return isinstance(value, (float, np.floating)) and bool(np.isnan(value)) +def warn_legacy_absence(*consts: DataArray) -> None: + """ + Warn when legacy arithmetic is about to fill an operand's absent slots + (NaN ``const``) with 0, which v1 propagates instead (§6). + """ + if is_v1(): + return + if any(bool(c.isnull().any()) for c in consts): + warn_legacy(_legacy_absent_operand_message()) + + def check_user_nan(*, op_kind: str = "add") -> None: """ Enforce §5 for a user-supplied constant (scalar or array). @@ -375,7 +456,7 @@ def check_user_nan(*, op_kind: str = "add") -> None: """ if is_v1(): raise ValueError(_user_nan_message()) - warn_legacy(_legacy_nan_constant_message(op_kind), stacklevel=5) + warn_legacy(_legacy_nan_constant_message(op_kind)) class AbsentType: @@ -442,7 +523,7 @@ def check_join_fill_value(fill_value: FillValueLike, join: str | None) -> None: def enforce_aux_conflict( - datasets: Sequence[Any], *, concat_dim: str | None = None, stacklevel: int = 5 + datasets: Sequence[Any], *, concat_dim: str | None = None ) -> None: """ Enforce §11 across the given operands: v1 raises on aux-coord @@ -453,12 +534,10 @@ def enforce_aux_conflict( return if is_v1(): raise ValueError(_aux_conflict_message(*conflict)) - warn_legacy(_legacy_aux_conflict_message(*conflict), stacklevel=stacklevel) + warn_legacy(_legacy_aux_conflict_message(*conflict)) -def enforce_no_multiindex( - obj: Any, *, context: str = "input", stacklevel: int = 4 -) -> None: +def enforce_no_multiindex(obj: Any, *, context: str = "input") -> None: """Reject (v1) / deprecate (legacy) any MultiIndex dimension on ``obj``.""" indexes = getattr(obj, "indexes", None) or {} dim = next((d for d, i in indexes.items() if isinstance(i, pd.MultiIndex)), None) @@ -466,7 +545,7 @@ def enforce_no_multiindex( return if is_v1(): raise ValueError(_v1_multiindex_message(str(dim), context)) - warn_legacy(_legacy_multiindex_message(str(dim), context), stacklevel=stacklevel) + warn_legacy(_legacy_multiindex_message(str(dim), context)) def first_mismatched_dim( @@ -556,21 +635,36 @@ def shared_dim_mismatches( def enforce_merge_dims( - datasets: Sequence[Dataset], *, concat_dim: str, context: str + datasets: Sequence[Dataset], + *, + concat_dim: str, + context: str, + alignment: str = _LEGACY_MERGE_ALIGNMENT, + joinable: bool = True, ) -> None: """ Enforce §8 across the operands of a merge: v1 raises on a differing label set or a pure reorder, legacy warns and leaves the alignment to the merge. + ``alignment`` describes how legacy aligns them, ``joinable`` whether the + operation accepts ``join=``. """ mismatch, reorder = shared_dim_mismatches(datasets, concat_dim) if is_v1(): problem = mismatch or reorder if problem is not None: - raise ValueError(_shared_dim_mismatch_message(*problem)) + raise ValueError(_shared_dim_mismatch_message(*problem, joinable=joinable)) elif mismatch is not None: - warn_legacy(_legacy_coord_mismatch_message(context, *mismatch)) + warn_legacy( + _legacy_coord_mismatch_message( + context, *mismatch, alignment=alignment, joinable=joinable + ) + ) elif reorder is not None: - warn_legacy(_legacy_coord_reorder_message(context, *reorder)) + warn_legacy( + _legacy_coord_reorder_message( + context, *reorder, alignment=alignment, joinable=joinable + ) + ) def conflicting_aux_coord( diff --git a/linopy/variables.py b/linopy/variables.py index 0589756e..912aaeec 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -386,16 +386,18 @@ def to_linexpr( """ if not np.isscalar(coefficient): coeff_da = as_dataarray(coefficient) - enforce_aux_conflict([self.labels, coeff_da], stacklevel=4) + enforce_aux_conflict([self.labels, coeff_da]) mismatch = first_mismatched_dim(self.labels, coeff_da) if mismatch is not None: if is_v1(): raise ValueError(_shared_dim_mismatch_message(*mismatch)) warn_legacy( _legacy_coord_mismatch_message( - "this operator's constant operand", *mismatch - ), - stacklevel=4, + "this operator's constant operand", + *mismatch, + alignment="by label, the coefficient being 0 at the" + " variable's labels it lacks", + ) ) coefficient = broadcast_to_coords(coefficient, coords=self.coords, strict=False) if coefficient.isnull().any(): @@ -834,7 +836,7 @@ def groupby( A `LinearExpressionGroupBy` containing the xarray groups and ensuring the correct return type. """ - return self.to_linexpr().groupby( + return self.to_linexpr(_warn_absence=False).groupby( group=group, restore_coord_dims=restore_coord_dims ) @@ -1286,7 +1288,7 @@ def sum(self, dim: DimsLike | None = None, **kwargs: Any) -> LinearExpression: if kwargs: raise ValueError(f"Unknown keyword argument(s): {kwargs}") - return self.to_linexpr().sum(dim) + return self.to_linexpr(_warn_absence=False).sum(dim) def diff(self, dim: str, n: int = 1) -> LinearExpression: """ diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index db6b805d..b5ad50a7 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -46,6 +46,7 @@ class corresponds to a section of ``doc/design/convention.rst`` import contextlib import operator +import re import warnings from collections.abc import Generator from typing import Any, cast @@ -55,6 +56,7 @@ class corresponds to a section of ``doc/design/convention.rst`` import polars as pl import pytest import xarray as xr +from xarray.core.types import JoinOptions from linopy import LinearExpression, Model, QuadraticExpression from linopy.config import LinopySemanticsWarning @@ -693,7 +695,7 @@ def test_coord_mismatch_merge_path( assert msg == ( "Coordinate mismatch in merge along dim '_term' silently " "aligned by legacy (positional when sizes match, otherwise " - "left-join). Under v1 this raises ValueError." + "outer join by label). Under v1 this raises ValueError." "\n Dim: 'time': left=[0, 1, 2, 3, 4], " "right=[10, 11, 12, 13, 14]" "\n Resolve: `.sel(...)` / `.reindex(...)` to align" @@ -820,10 +822,7 @@ def test_masked_variable_in_arithmetic( "propagates through arithmetic instead (`x + y` becomes " "absent at the slot and the constraint drops)." "\n Resolve: wrap with `y.fillna(0)` for the legacy " - "behaviour under v1" - "\n (no fix needed if you only use the variable " - "in a constraint LHS alone — `y >= 0` drops the same way in " - "both)." + _OPT_IN_HINT + "behaviour under v1." + _OPT_IN_HINT ) @pytest.mark.legacy @@ -853,6 +852,335 @@ def test_warning_stacklevel_points_to_user_call( ) +class TestLegacyWarnsAtDivergence: + """ + Legacy warns exactly where its result departs from v1, and stays silent + where both agree. + """ + + @pytest.fixture + def mask(self, time: pd.RangeIndex) -> xr.DataArray: + return xr.DataArray([True, False, True, True, True], coords=[time]) + + @pytest.fixture + def far(self) -> xr.DataArray: + return xr.DataArray( + [2.0, 3.0], dims=["time"], coords={"time": pd.Index([0, 5], name="time")} + ) + + _HOLE_AT_1 = [False, True, False, False, False] + _HOLE_AT_0 = [True, False, False, False, False] + _ABSENT_EXPRS = [ + pytest.param( + lambda x, mask: (2 * x).where(mask) + x, _HOLE_AT_1, id="where_plus_var" + ), + pytest.param( + lambda x, mask: (1 * x).where(mask) + 5, _HOLE_AT_1, id="where_plus_scalar" + ), + pytest.param( + lambda x, mask: (1 * x).where(mask) * 2, _HOLE_AT_1, id="where_times_scalar" + ), + pytest.param( + lambda x, mask: (1 * x).shift(time=1) + x, _HOLE_AT_0, id="shift_plus_var" + ), + pytest.param(lambda x, mask: x.diff("time"), _HOLE_AT_0, id="var_diff"), + pytest.param( + lambda x, mask: (1 * x).reindex(time=range(6)) + 1, + [False] * 5 + [True], + id="reindex_plus_scalar", + ), + pytest.param( + lambda x, mask: (x * x).where(mask) + 1, + _HOLE_AT_1, + id="quad_where_plus_scalar", + ), + pytest.param( + lambda x, mask: (1 * x).where(mask) * x, _HOLE_AT_1, id="where_times_var" + ), + ] + + @pytest.mark.legacy + @pytest.mark.parametrize(("build", "absent"), _ABSENT_EXPRS) + def test_legacy_warns_on_filled_absence( + self, + x: Variable, + mask: xr.DataArray, + build: Any, + absent: list[bool], + unsilenced: None, + ) -> None: + with pytest.warns(LinopySemanticsWarning, match="absent slots"): + build(x, mask) + + @pytest.mark.v1 + @pytest.mark.parametrize(("build", "absent"), _ABSENT_EXPRS) + def test_absence_propagates( + self, x: Variable, mask: xr.DataArray, build: Any, absent: list[bool] + ) -> None: + assert build(x, mask).isnull().values.tolist() == absent + + _MASKED_CONSTRAINTS = [ + pytest.param(lambda x, mask: x.where(mask) <= 3, id="var_le_scalar"), + pytest.param(lambda x, mask: (1 * x).where(mask) >= 3, id="expr_ge_scalar"), + ] + + @pytest.mark.legacy + @pytest.mark.parametrize("build", _MASKED_CONSTRAINTS) + def test_legacy_masked_constraint_warns_once_and_keeps_row( + self, m: Model, x: Variable, mask: xr.DataArray, build: Any, unsilenced: None + ) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", LinopySemanticsWarning) + con = m.add_constraints(build(x, mask)) + assert len(caught) == 1 + assert "absent slots" in str(caught[0].message) + assert con.labels.values.tolist() == [0, 1, 2, 3, 4] + + @pytest.mark.v1 + @pytest.mark.parametrize("build", _MASKED_CONSTRAINTS) + def test_masked_constraint_drops_row( + self, m: Model, x: Variable, mask: xr.DataArray, build: Any + ) -> None: + assert m.add_constraints(build(x, mask)).labels.values.tolist() == [ + 0, + -1, + 2, + 3, + 4, + ] + + @pytest.mark.legacy + def test_legacy_diff_constraint_keeps_first_row( + self, x: Variable, unsilenced: None + ) -> None: + with pytest.warns(LinopySemanticsWarning, match="absent slots"): + con = x.diff("time") >= 0 + assert con.vars.values[0].tolist() == [0, -1] + + @pytest.mark.v1 + def test_diff_constraint_drops_first_row(self, x: Variable) -> None: + con = x.diff("time") >= 0 + assert con.vars.values[0].tolist() == [-1, -1] + assert np.isnan(con.rhs.values[0]) + + @pytest.mark.legacy + @pytest.mark.parametrize("join", ["left", "outer"]) + @pytest.mark.parametrize("operand", ["var", "expr"]) + def test_legacy_warns_on_join_created_divisor( + self, + x: Variable, + far: xr.DataArray, + join: JoinOptions, + operand: str, + unsilenced: None, + ) -> None: + lhs = x if operand == "var" else 1 * x + with pytest.warns(LinopySemanticsWarning, match="created divisor positions"): + result = lhs.div(far, join=join) + assert result.coeffs.values[1:5, 0].tolist() == [1.0] * 4 + + @pytest.mark.v1 + @pytest.mark.parametrize("join", ["left", "outer"]) + def test_join_created_divisor_zeroes_term( + self, x: Variable, far: xr.DataArray, join: JoinOptions + ) -> None: + assert x.div(far, join=join).coeffs.values[1:5, 0].tolist() == [0.0] * 4 + + @pytest.mark.legacy + @pytest.mark.parametrize( + "div", + [ + pytest.param(lambda x, far: x.div(far, join="inner"), id="inner"), + pytest.param( + lambda x, far: x.div(far, join="left", fill_value=1), id="fill_value" + ), + ], + ) + def test_legacy_divisor_join_without_divergence_is_silent( + self, x: Variable, far: xr.DataArray, div: Any, unsilenced: None + ) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error", LinopySemanticsWarning) + div(x, far) + + @staticmethod + def _subset(m: Model) -> Variable: + return m.add_variables(coords=[pd.Index([0, 1, 2], name="time")], name="s") + + _PAIRED = [[[i], [i + 5]] for i in range(3)] + [[[3], [-1]], [[4], [-1]]] + _MISMATCH = r"Coordinate mismatch in quadratic product silently aligned by legacy \(by label, as an outer join\)" + _REORDER = r"Coordinate order mismatch in quadratic product aligned by legacy \(by label, as an outer join\)" + + @pytest.mark.legacy + @pytest.mark.parametrize( + ("product", "expected_vars", "match"), + [ + pytest.param( + lambda x, s: x * s, _PAIRED, _MISMATCH, id="superset_times_subset" + ), + pytest.param( + lambda x, s: (1 * x) * (1 * s), _PAIRED, _MISMATCH, id="expressions" + ), + pytest.param( + lambda x, s: x * x.sel(time=[4, 3, 2, 1, 0]), + [[[i], [i]] for i in range(5)], + _REORDER, + id="reordered", + ), + pytest.param( + lambda x, s: (x + 1) * (s + 2), + [[[i, i + 5, i], [i + 5, -1, -1]] for i in range(3)] + + [[[i, -1, i], [-1, -1, -1]] for i in (3, 4)], + _MISMATCH, + id="both_constants", + ), + pytest.param( + lambda x, s: (x + 1) * s, + [[[i, i + 5], [i + 5, -1]] for i in range(3)] + + [[[i, -1], [-1, -1]] for i in (3, 4)], + _MISMATCH, + id="left_constant", + ), + pytest.param( + lambda x, s: x * (s + 2), + [[[i + 5, i], [i, -1]] for i in range(3)] + + [[[-1, i], [i, -1]] for i in (3, 4)], + _MISMATCH, + id="right_constant", + ), + ], + ) + def test_legacy_quadratic_mismatch_warns_once_with_int_labels( + self, + m: Model, + x: Variable, + product: Any, + expected_vars: list[Any], + match: str, + unsilenced: None, + ) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", LinopySemanticsWarning) + result = product(x, self._subset(m)) + assert len(caught) == 1 + message = str(caught[0].message) + assert re.match(match, message) + assert "join=" not in message + assert result.vars.values.tolist() == expected_vars + + @pytest.mark.v1 + @pytest.mark.parametrize( + "product", + [ + pytest.param(lambda x, s: x * s, id="superset_times_subset"), + pytest.param(lambda x, s: s * x, id="subset_times_superset"), + pytest.param(lambda x, s: (1 * x) * (1 * s), id="expressions"), + pytest.param(lambda x, s: x * x.sel(time=[4, 3, 2, 1, 0]), id="reordered"), + ], + ) + def test_quadratic_mismatch_raises( + self, m: Model, x: Variable, product: Any + ) -> None: + with pytest.raises( + ValueError, match="Coordinate mismatch on shared dimension" + ) as err: + product(x, self._subset(m)) + assert "join=" not in str(err.value) + + @pytest.mark.legacy + def test_legacy_rhs_explicit_join_message( + self, x: Variable, far: xr.DataArray, unsilenced: None + ) -> None: + msg = _one_legacy_warning(lambda: x.le(far, join="outer")) + assert msg == ( + "Coordinate mismatch in constraint RHS silently aligned by legacy " + "(by label onto the expression, no constraint where the RHS lacks " + "a label, ignoring `join='outer'`). Under v1 `join='outer'` is " + "applied instead." + "\n Dim: 'time': left=[0, 1, 2, 3, 4], right=[0, 5]" + "\n Resolve: `.sel(...)` / `.reindex(...)` to align" + "\n `.assign_coords(...)` to relabel one side" + "\n `linopy.align(...)` to pre-align several operands" + "\n or pass an explicit `join=` argument." + _OPT_IN_HINT + ) + + @pytest.mark.legacy + @pytest.mark.parametrize( + "build", + [ + pytest.param(lambda x, mask: x.where(mask).sum(), id="sum"), + pytest.param( + lambda x, mask: ( + x.where(mask) + .groupby(xr.DataArray([0, 0, 1, 1, 1], coords=[x.indexes["time"]])) + .sum() + ), + id="groupby_sum", + ), + pytest.param( + lambda x, mask: (1 * x).le( + xr.DataArray( + np.arange(6.0), coords=[pd.RangeIndex(6, name="time")] + ), + join="left", + ), + id="le_superset_join_left", + ), + ], + ) + def test_legacy_silent_without_divergence( + self, x: Variable, mask: xr.DataArray, build: Any, unsilenced: None + ) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error", LinopySemanticsWarning) + build(x, mask) + + @pytest.mark.legacy + @pytest.mark.parametrize( + "build", + [ + pytest.param( + lambda x: ( + (1 * x) + <= xr.DataArray([1.0, 2.0], coords=[pd.Index([0, 1], name="time")]) + ), + id="rhs_subset", + ), + pytest.param( + lambda x: ( + (1 * x) + + xr.DataArray( + [1.0, np.nan, 3.0, 4.0, 5.0], coords=[x.indexes["time"]] + ) + ), + id="nan_operand", + ), + ], + ) + def test_legacy_single_warning_points_at_user_code( + self, x: Variable, build: Any, unsilenced: None + ) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", LinopySemanticsWarning) + build(x) + assert [w.filename for w in caught] == [__file__] + + @pytest.mark.legacy + def test_legacy_variable_coefficient_mismatch_message( + self, x: Variable, unsilenced: None + ) -> None: + coeff = xr.DataArray( + [1.0, 2.0, 3.0, 4.0, 5.0], coords=[pd.Index([1, 2, 3, 4, 5], name="time")] + ) + msg = _one_legacy_warning(lambda: x * coeff) + assert msg.startswith( + "Coordinate mismatch in this operator's constant operand silently " + "aligned by legacy (by label, the coefficient being 0 at the " + "variable's labels it lacks). Under v1 this raises ValueError." + ) + assert (x * coeff).coeffs.values[:, 0].tolist() == [0.0, 1.0, 2.0, 3.0, 4.0] + + # ===================================================================== # §8 — Shared dimensions must match exactly (expr+expr / var+var, merge path) # ===================================================================== @@ -1081,7 +1409,8 @@ def test_reordered_merge_warns_legacy(self, m: Model) -> None: msg = _one_legacy_warning(lambda: (1 * a) + (1 * b)) assert msg == ( "Coordinate order mismatch in merge along dim '_term' aligned " - "positionally by legacy. Under v1 the same labels in a different " + "by legacy (positional when sizes match, otherwise outer join by " + "label). Under v1 the same labels in a different " "order raise ValueError (§8); reindex or sort one side to align " "by label." "\n Dim: 'e': left=['costs', 'penalty'], " @@ -1191,7 +1520,7 @@ def test_warn_on_var_plus_var_different_labels( assert msg == ( "Coordinate mismatch in merge along dim '_term' silently " "aligned by legacy (positional when sizes match, otherwise " - "left-join). Under v1 this raises ValueError." + "outer join by label). Under v1 this raises ValueError." "\n Dim: 'time': left=[0, 1, 2, 3, 4], " "right=[10, 11, 12, 13, 14]" "\n Resolve: `.sel(...)` / `.reindex(...)` to align" @@ -1904,8 +2233,8 @@ class TestConstraintRHS: ) _LEGACY_COORD_MISMATCH_RHS_MSG = ( "Coordinate mismatch in constraint RHS silently aligned by legacy " - "(positional when sizes match, otherwise left-join). Under v1 this " - "raises ValueError." + "(by label onto the expression, no constraint where the RHS lacks a " + "label). Under v1 this raises ValueError." "\n Dim: 'time': left=[0, 1, 2, 3, 4], right=[1, 3]" "\n Resolve: `.sel(...)` / `.reindex(...)` to align" "\n `.assign_coords(...)` to relabel one side"