From d0557817014d8f2de1d332590800368859407f3e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Nov 2025 18:07:32 -0500 Subject: [PATCH 1/7] Add support for schema downgrade migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change introduces the ability to downgrade metadata schemas to recent versions, allowing for more flexible schema version management. Key changes: - Allow migration to version 0.6.10 in addition to the current version - Implement SIMPLE_DOWNGRADES mechanism for safe field removal during downgrade - Remove restriction preventing downgrade to lower schema versions - Add validation to prevent data loss when downgrading with populated fields - Add comprehensive tests for downgrade functionality with releaseNotes field The downgrade mechanism ensures data integrity by raising an error if a field being removed during downgrade contains a non-empty value. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Conflicts: dandischema/metadata.py -- both wanted to always set schema version to migrated-to dandischema/tests/test_metadata.py -- just added tests conflicted placement Conflicts: dandischema/metadata.py -- again conflict on rebasing this downgrade... interesting that I got again this dialog :-/ --- dandischema/consts.py | 7 ++-- dandischema/metadata.py | 51 +++++++++++++++++------------- dandischema/tests/test_metadata.py | 50 +++++++++++++++++++++-------- 3 files changed, 68 insertions(+), 40 deletions(-) diff --git a/dandischema/consts.py b/dandischema/consts.py index c5091395..a9f801c2 100644 --- a/dandischema/consts.py +++ b/dandischema/consts.py @@ -20,10 +20,9 @@ DANDI_SCHEMA_VERSION, ] -# ATM we allow only for a single target version which is current -# migrate has a guard now for this since it cannot migrate to anything but current -# version -ALLOWED_TARGET_SCHEMAS = [DANDI_SCHEMA_VERSION] +# We establish migrations (back) to only a few recent versions. +# When adding changes, please consider whether a migration path should be added. +ALLOWED_TARGET_SCHEMAS = ["0.6.10", DANDI_SCHEMA_VERSION] # This allows multiple schemas for validation, whereas target schemas focus on # migration. diff --git a/dandischema/metadata.py b/dandischema/metadata.py index 8de8f8ee..9c9654d1 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -321,24 +321,18 @@ def migrate( schema version of the provided instance """ - # ATM, we only support the latest schema version as a target. See definition of - # `ALLOWED_TARGET_SCHEMAS` for details - if len(ALLOWED_TARGET_SCHEMAS) > 1: - msg = f"Only migration to current version, {DANDI_SCHEMA_VERSION}, is supported" - raise NotImplementedError(msg) - # -------------------------------------------------------------- # Validate DANDI schema version provided in the metadata instance # -------------------------------------------------------------- # DANDI schema version of the provided instance - obj_ver = obj.get("schemaVersion") - if obj_ver is None: + obj_version = obj.get("schemaVersion") + if obj_version is None: msg = ( - "The provided Dandiset metadata instance does not have a " + "The provided metadata instance does not have a " "`schemaVersion` field for specifying the DANDI schema version." ) raise ValueError(msg) - if not isinstance(obj_ver, str): + if not isinstance(obj_version, str): msg = ( "The provided Dandiset metadata instance has a non-string " "`schemaVersion` field for specifying the DANDI schema version." @@ -347,17 +341,17 @@ def migrate( # Check if `obj_ver` is a valid DANDI schema version try: # DANDI schema version of the provided instance in tuple form - obj_ver_tuple = version2tuple(obj_ver) + obj_version_tuple = version2tuple(obj_version) except ValueError as e: msg = ( "The provided Dandiset metadata instance has an invalid " "`schemaVersion` field for specifying the DANDI schema version." ) raise ValueError(msg) from e - if obj_ver not in ALLOWED_INPUT_SCHEMAS: + if obj_version not in ALLOWED_INPUT_SCHEMAS: msg = ( f"The DANDI schema version of the provided Dandiset metadata instance, " - f"{obj_ver!r}, is not one of the supported versions for input " + f"{obj_version!r}, is not one of the supported versions for input " f"Dandiset metadata instances. The supported versions are " f"{ALLOWED_INPUT_SCHEMAS}." ) @@ -370,7 +364,7 @@ def migrate( # Check if `to_version` is a valid DANDI schema version try: # The target DANDI schema version in tuple form - target_ver_tuple = version2tuple(to_version) + to_version_tuple = version2tuple(to_version) except ValueError as e: msg = ( "The provided target version, {to_version!r}, is not a valid DANDI schema " @@ -387,22 +381,17 @@ def migrate( raise ValueError(msg) # ---------------------------------------------------------------- - # Ensure the target DANDI schema version is at least the DANDI schema version - # of the provided instance - if obj_ver_tuple > target_ver_tuple: - raise ValueError(f"Cannot migrate from {obj_ver} to lower {to_version}.") - # Optionally validate the instance against the DANDI schema it specifies # before migration if not skip_validation: - validate_json(obj, _get_jsonschema_validator(obj_ver, "Dandiset")) + validate_json(obj, _get_jsonschema_validator(obj_version, "Dandiset")) obj_migrated = deepcopy(obj) - if obj_ver_tuple == target_ver_tuple: + if obj_version_tuple == to_version_tuple: return obj_migrated - if obj_ver_tuple < version2tuple("0.6.0") <= target_ver_tuple: + if obj_version_tuple < version2tuple("0.6.0") <= to_version_tuple: for val in obj_migrated.get("about", []): if "schemaKey" not in val: if "identifier" in val and "UBERON" in val["identifier"]: @@ -422,6 +411,24 @@ def migrate( if "schemaKey" not in obj_migrated: obj_migrated["schemaKey"] = "Dandiset" + # Downgrades + + # Simple downgrades that just require removing fields, which is totally fine + # if they are empty + SIMPLE_DOWNGRADES = [ + # version added, fields to remove + ("0.6.11", ["releaseNotes"]), + ] + for ver_added, fields in SIMPLE_DOWNGRADES: + # additional guards are via ALLOWED_TARGET_SCHEMAS + if (to_version_tuple < version2tuple(ver_added) <= obj_version_tuple): + for field in fields: + if field in obj_migrated: + if val := obj_migrated.get(field): + raise ValueError(f"Cannot downgrade to {to_version} from " + f"{obj_version} with {field}={val!r} present") + del obj_migrated[field] + # Always update schemaVersion when migrating obj_migrated["schemaVersion"] = to_version return obj_migrated diff --git a/dandischema/tests/test_metadata.py b/dandischema/tests/test_metadata.py index 7408eb02..01e8ca53 100644 --- a/dandischema/tests/test_metadata.py +++ b/dandischema/tests/test_metadata.py @@ -15,6 +15,7 @@ DANDISET_METADATA_DIR, INSTANCE_NAME, METADATA_DIR, + basic_publishmeta, skipif_instance_name_not_dandi, skipif_no_network, skipif_no_test_dandiset_metadata_dir, @@ -344,20 +345,6 @@ def test_migrate_value_errors(obj: Dict[str, Any], target: Any, msg: str) -> Non migrate(obj, to_version=target, skip_validation=True) -def test_migrate_value_errors_lesser_target(monkeypatch: pytest.MonkeyPatch) -> None: - """ - Test cases when `migrate()` is expected to raise a `ValueError` exception - when the target schema version is lesser than the schema version of the metadata - instance - """ - from dandischema import metadata - - monkeypatch.setattr(metadata, "ALLOWED_TARGET_SCHEMAS", ["0.6.0"]) - - with pytest.raises(ValueError, match="Cannot migrate from .* to lower"): - migrate({"schemaVersion": "0.6.7"}, to_version="0.6.0", skip_validation=True) - - @skipif_no_network @skipif_no_test_dandiset_metadata_dir # Skip for instance name not being DANDI because JSON schema version at `0.4.4`, the @@ -429,6 +416,41 @@ def test_migrate_schemaversion_update() -> None: ) +@pytest.mark.ai_generated +def test_migrate_downgrade_releasenotes() -> None: + """Test downgrade from 0.6.11 to 0.6.10 handling releaseNotes field""" + + # Create a basic PublishedDandiset metadata in 0.6.11 format + meta_dict = { + "schemaVersion": "0.6.11", + } + meta_dict.update(basic_publishmeta(dandi_id="999999")) + + # Test 1: Downgrade without releaseNotes (should succeed) + downgraded = migrate(meta_dict, to_version="0.6.10", skip_validation=True) + assert downgraded["schemaVersion"] == "0.6.10" + assert "releaseNotes" not in downgraded + + # Test 2: Downgrade with empty releaseNotes (should succeed) + meta_dict["releaseNotes"] = "" + downgraded = migrate(meta_dict, to_version="0.6.10", skip_validation=True) + assert downgraded["schemaVersion"] == "0.6.10" + assert "releaseNotes" not in downgraded + + # Test 3: Downgrade with non-empty releaseNotes (should fail) + meta_dict["releaseNotes"] = "Releasing during testing" + with pytest.raises(ValueError, match="Cannot downgrade to 0.6.10 from"): + migrate(meta_dict, to_version="0.6.10", skip_validation=True) + + # Test 4: No-op migration (already at target version) + meta_dict_0610 = meta_dict.copy() + meta_dict_0610["schemaVersion"] = "0.6.10" + meta_dict_0610.pop("releaseNotes") + migrated = migrate(meta_dict_0610, to_version="0.6.10", skip_validation=True) + assert migrated == meta_dict_0610 + assert migrated is not meta_dict_0610 # but we do create a copy + + @pytest.mark.parametrize( "files, summary", [ From 494e425d61d79222a39da38f054766ace9567c08 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 20 Nov 2025 16:27:16 -0800 Subject: [PATCH 2/7] Consider "empty" only None and empty containers and add a comment about that. --- dandischema/metadata.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/dandischema/metadata.py b/dandischema/metadata.py index 9c9654d1..cb198c86 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -414,20 +414,28 @@ def migrate( # Downgrades # Simple downgrades that just require removing fields, which is totally fine - # if they are empty + # if they are empty, as they are None or empty containers (list, tuple, etc). + # List only those for which such notion of "empty" applies. SIMPLE_DOWNGRADES = [ # version added, fields to remove ("0.6.11", ["releaseNotes"]), ] for ver_added, fields in SIMPLE_DOWNGRADES: # additional guards are via ALLOWED_TARGET_SCHEMAS - if (to_version_tuple < version2tuple(ver_added) <= obj_version_tuple): + if to_version_tuple < version2tuple(ver_added) <= obj_version_tuple: for field in fields: if field in obj_migrated: - if val := obj_migrated.get(field): - raise ValueError(f"Cannot downgrade to {to_version} from " - f"{obj_version} with {field}={val!r} present") - del obj_migrated[field] + value = obj_migrated.get(field) + # Explicit check for "empty" value per above description. + if value is None or ( + not value and isinstance(value, (list, tuple, dict, set)) + ): + del obj_migrated[field] + else: + raise ValueError( + f"Cannot downgrade to {to_version} from " + f"{obj_version} with {field}={value!r} present" + ) # Always update schemaVersion when migrating obj_migrated["schemaVersion"] = to_version From 5c601261abd47473ec931ec6f60cdf53b185f48b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Mar 2026 12:43:06 -0500 Subject: [PATCH 3/7] Fix downgrade migrations: correct version, add sameAs, handle empty strings - SIMPLE_DOWNGRADES: "0.6.11" -> "0.7.0" (no 0.6.11 was ever released) - Add sameAs to downgrade fields (added on master via PR #364) - Include str in empty-value check so releaseNotes="" is treated as empty - Rewrite test to use DANDI_SCHEMA_VERSION and minimal metadata dict (0.6.11 was not in ALLOWED_INPUT_SCHEMAS, basic_publishmeta needed instance_name positional arg) - Add test coverage for sameAs downgrade (empty list, non-empty list) Co-Authored-By: Claude Code 2.1.63 / Claude Opus 4.6 --- dandischema/metadata.py | 7 +++-- dandischema/tests/test_metadata.py | 49 ++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/dandischema/metadata.py b/dandischema/metadata.py index cb198c86..8dee913f 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -414,11 +414,12 @@ def migrate( # Downgrades # Simple downgrades that just require removing fields, which is totally fine - # if they are empty, as they are None or empty containers (list, tuple, etc). + # if they are empty, as they are None or empty containers (list, tuple, etc) + # or empty strings. # List only those for which such notion of "empty" applies. SIMPLE_DOWNGRADES = [ # version added, fields to remove - ("0.6.11", ["releaseNotes"]), + ("0.7.0", ["sameAs", "releaseNotes"]), ] for ver_added, fields in SIMPLE_DOWNGRADES: # additional guards are via ALLOWED_TARGET_SCHEMAS @@ -428,7 +429,7 @@ def migrate( value = obj_migrated.get(field) # Explicit check for "empty" value per above description. if value is None or ( - not value and isinstance(value, (list, tuple, dict, set)) + not value and isinstance(value, (list, tuple, dict, set, str)) ): del obj_migrated[field] else: diff --git a/dandischema/tests/test_metadata.py b/dandischema/tests/test_metadata.py index 01e8ca53..fd2e3f1e 100644 --- a/dandischema/tests/test_metadata.py +++ b/dandischema/tests/test_metadata.py @@ -15,7 +15,6 @@ DANDISET_METADATA_DIR, INSTANCE_NAME, METADATA_DIR, - basic_publishmeta, skipif_instance_name_not_dandi, skipif_no_network, skipif_no_test_dandiset_metadata_dir, @@ -417,19 +416,21 @@ def test_migrate_schemaversion_update() -> None: @pytest.mark.ai_generated -def test_migrate_downgrade_releasenotes() -> None: - """Test downgrade from 0.6.11 to 0.6.10 handling releaseNotes field""" +def test_migrate_downgrade() -> None: + """Test downgrade from 0.7.0 to 0.6.10 handling releaseNotes and sameAs fields""" - # Create a basic PublishedDandiset metadata in 0.6.11 format - meta_dict = { - "schemaVersion": "0.6.11", + # Minimal metadata at current (0.7.0) version + meta_dict: dict = { + "schemaKey": "Dandiset", + "schemaVersion": DANDI_SCHEMA_VERSION, + "identifier": "DANDI:000000", } - meta_dict.update(basic_publishmeta(dandi_id="999999")) - # Test 1: Downgrade without releaseNotes (should succeed) + # Test 1: Downgrade without new fields (should succeed) downgraded = migrate(meta_dict, to_version="0.6.10", skip_validation=True) assert downgraded["schemaVersion"] == "0.6.10" assert "releaseNotes" not in downgraded + assert "sameAs" not in downgraded # Test 2: Downgrade with empty releaseNotes (should succeed) meta_dict["releaseNotes"] = "" @@ -437,15 +438,37 @@ def test_migrate_downgrade_releasenotes() -> None: assert downgraded["schemaVersion"] == "0.6.10" assert "releaseNotes" not in downgraded - # Test 3: Downgrade with non-empty releaseNotes (should fail) + # Test 3: Downgrade with None releaseNotes (should succeed) + meta_dict["releaseNotes"] = None + downgraded = migrate(meta_dict, to_version="0.6.10", skip_validation=True) + assert downgraded["schemaVersion"] == "0.6.10" + assert "releaseNotes" not in downgraded + + # Test 4: Downgrade with empty sameAs list (should succeed) + meta_dict.pop("releaseNotes") + meta_dict["sameAs"] = [] + downgraded = migrate(meta_dict, to_version="0.6.10", skip_validation=True) + assert downgraded["schemaVersion"] == "0.6.10" + assert "sameAs" not in downgraded + + # Test 5: Downgrade with non-empty releaseNotes (should fail) + meta_dict.pop("sameAs") meta_dict["releaseNotes"] = "Releasing during testing" with pytest.raises(ValueError, match="Cannot downgrade to 0.6.10 from"): migrate(meta_dict, to_version="0.6.10", skip_validation=True) - # Test 4: No-op migration (already at target version) - meta_dict_0610 = meta_dict.copy() - meta_dict_0610["schemaVersion"] = "0.6.10" - meta_dict_0610.pop("releaseNotes") + # Test 6: Downgrade with non-empty sameAs (should fail) + meta_dict.pop("releaseNotes") + meta_dict["sameAs"] = ["dandi://DANDI-SANDBOX/123456"] + with pytest.raises(ValueError, match="Cannot downgrade to 0.6.10 from"): + migrate(meta_dict, to_version="0.6.10", skip_validation=True) + + # Test 7: No-op migration (already at target version) + meta_dict_0610 = { + "schemaKey": "Dandiset", + "schemaVersion": "0.6.10", + "identifier": "DANDI:000000", + } migrated = migrate(meta_dict_0610, to_version="0.6.10", skip_validation=True) assert migrated == meta_dict_0610 assert migrated is not meta_dict_0610 # but we do create a copy From 139bc77382d54eb4e8bacad81a391613235722ad Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Mar 2026 15:34:03 -0500 Subject: [PATCH 4/7] Fix downgrade as the sameAs was added after 0.7.0 schema release --- dandischema/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dandischema/metadata.py b/dandischema/metadata.py index 8dee913f..5480a0da 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -419,7 +419,8 @@ def migrate( # List only those for which such notion of "empty" applies. SIMPLE_DOWNGRADES = [ # version added, fields to remove - ("0.7.0", ["sameAs", "releaseNotes"]), + ("0.7.0", ["releaseNotes"]), + ("0.8.0", ["sameAs"]), ] for ver_added, fields in SIMPLE_DOWNGRADES: # additional guards are via ALLOWED_TARGET_SCHEMAS From f3d862a378ec9dfb5b91e69e1d87c4f5488f3618 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Mar 2026 15:36:18 -0500 Subject: [PATCH 5/7] Prep for 0.8.0 schema release to reflect addition of sameAs --- dandischema/consts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandischema/consts.py b/dandischema/consts.py index a9f801c2..863d7647 100644 --- a/dandischema/consts.py +++ b/dandischema/consts.py @@ -22,7 +22,7 @@ # We establish migrations (back) to only a few recent versions. # When adding changes, please consider whether a migration path should be added. -ALLOWED_TARGET_SCHEMAS = ["0.6.10", DANDI_SCHEMA_VERSION] +ALLOWED_TARGET_SCHEMAS = ["0.6.10", "0.7.0", DANDI_SCHEMA_VERSION] # This allows multiple schemas for validation, whereas target schemas focus on # migration. From de903499115f546652ed37c6865754dc968d243a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 4 Mar 2026 15:54:56 -0500 Subject: [PATCH 6/7] Aim to release as schema 0.7.1 --- dandischema/consts.py | 2 +- dandischema/metadata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dandischema/consts.py b/dandischema/consts.py index 863d7647..8ee5d97a 100644 --- a/dandischema/consts.py +++ b/dandischema/consts.py @@ -1,6 +1,6 @@ from packaging.version import Version as _Version -DANDI_SCHEMA_VERSION = "0.8.0" +DANDI_SCHEMA_VERSION = "0.7.1" ALLOWED_INPUT_SCHEMAS = [ "0.4.4", "0.5.1", diff --git a/dandischema/metadata.py b/dandischema/metadata.py index 5480a0da..fe0376ae 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -420,7 +420,7 @@ def migrate( SIMPLE_DOWNGRADES = [ # version added, fields to remove ("0.7.0", ["releaseNotes"]), - ("0.8.0", ["sameAs"]), + ("0.7.1", ["sameAs"]), ] for ver_added, fields in SIMPLE_DOWNGRADES: # additional guards are via ALLOWED_TARGET_SCHEMAS From bcda05081c52fc433f540a80c150cf442ee4434e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 3 Aug 2026 16:43:36 -0400 Subject: [PATCH 7/7] Fixup target schema version to be the current 0.8.0 Originally we were aiming for 0.7.1 but 0.8.0 was released, and in this PR we are not changing anything in the schema --- dandischema/consts.py | 2 +- dandischema/metadata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dandischema/consts.py b/dandischema/consts.py index 8ee5d97a..863d7647 100644 --- a/dandischema/consts.py +++ b/dandischema/consts.py @@ -1,6 +1,6 @@ from packaging.version import Version as _Version -DANDI_SCHEMA_VERSION = "0.7.1" +DANDI_SCHEMA_VERSION = "0.8.0" ALLOWED_INPUT_SCHEMAS = [ "0.4.4", "0.5.1", diff --git a/dandischema/metadata.py b/dandischema/metadata.py index fe0376ae..5480a0da 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -420,7 +420,7 @@ def migrate( SIMPLE_DOWNGRADES = [ # version added, fields to remove ("0.7.0", ["releaseNotes"]), - ("0.7.1", ["sameAs"]), + ("0.8.0", ["sameAs"]), ] for ver_added, fields in SIMPLE_DOWNGRADES: # additional guards are via ALLOWED_TARGET_SCHEMAS