diff --git a/flax/nnx/nn/attention.py b/flax/nnx/nn/attention.py index 4f0c4f0cd..16e09cfab 100644 --- a/flax/nnx/nn/attention.py +++ b/flax/nnx/nn/attention.py @@ -104,6 +104,16 @@ def dot_product_attention_weights( Returns: Output of shape `[batch..., num_heads, q_length, kv_length]`. """ + if not (0.0 <= dropout_rate < 1.0): + raise ValueError( + f'dropout_rate must be in the range [0, 1), got {dropout_rate!r}. ' + 'A negative or NaN value silently disables dropout without regularizing.' + ) + if not deterministic and dropout_rate > 0.0 and dropout_rng is None: + raise ValueError( + 'dropout_rng must be provided when dropout_rate > 0.0 and ' + 'deterministic=False. Pass a JAX PRNGKey as dropout_rng.' + ) query, key = promote_dtype((query, key), dtype=dtype) # type: ignore[bad-unpacking] dtype = query.dtype