Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 74 additions & 3 deletions tests/test_canonical_marker_atomic.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@
MARKER_NAME = ".biomero-canonical.json"


def _uses_processed_folder(path):
"""Whether the source keeps sidecars in BIOMERO folders.

Gates on the keyword the feature adds to the sidecar calls, not on log
text, so rewording a message fails the tests instead of skipping them.
"""
tree = ast.parse(path.read_text(encoding="utf-8"))
return any(
isinstance(node, ast.keyword) and node.arg == "processed_folder"
for node in ast.walk(tree))


def _marker_helpers():
tree = ast.parse(SCRIPT.read_text(encoding="utf-8"))
names = {"_write_json", "_canonical_marker_path", "_write_canonical_marker"}
Expand All @@ -29,8 +41,10 @@ def _marker_helpers():
"os": os,
"uuid4": uuid4,
"CANONICAL_MARKER_NAME": MARKER_NAME,
"external_canonical_marker_path": lambda path: path.parent / "external-marker",
"write_indexed_canonical_marker": lambda *args: pytest.fail(
"PROCESSED_DATA_FOLDER": ".processed",
"external_canonical_marker_path": (
lambda path, **_kwargs: path.parent / "external-marker"),
"write_indexed_canonical_marker": lambda *args, **kwargs: pytest.fail(
"Existing marker must be updated, not created again"
),
}
Expand All @@ -50,7 +64,7 @@ def test_existing_canonical_marker_is_replaced_without_direct_overwrite(tmp_path
"source": {"relativePath": "old"},
}), encoding="utf-8")
source = SimpleNamespace(to_dict=lambda: {"relativePath": "new"})
helpers["load_canonical_marker"] = lambda path: (
helpers["load_canonical_marker"] = lambda path, **_kwargs: (
source if json.loads((path / MARKER_NAME).read_text(
encoding="utf-8"
))["source"] == source.to_dict() else None
Expand Down Expand Up @@ -83,3 +97,60 @@ def test_failed_replacement_preserves_marker_and_cleans_temp_file(tmp_path):

assert marker.read_text(encoding="utf-8") == original
assert list(tmp_path.iterdir()) == [marker]


@pytest.mark.skipif(
not _uses_processed_folder(SCRIPT),
reason="sidecar location moved to BIOMERO folders in a later source",
)
def test_store_with_only_a_legacy_sidecar_is_rewritten_by_the_importer(tmp_path):
"""A legacy-only store has no current marker, so the importer writes it."""
helpers = _marker_helpers()
store = tmp_path / "image.zarr"
store.mkdir()
writes = []
helpers["write_indexed_canonical_marker"] = (
lambda path, source, **kwargs: writes.append((path, source, kwargs))
or tmp_path / "new-sidecar"
)
source = SimpleNamespace(to_dict=lambda: {})

result = helpers["_write_canonical_marker"](store, source)

assert result == tmp_path / "new-sidecar"
assert writes == [(store, source, {"processed_folder": ".processed"})]


def _move_store_function(move_marker):
tree = ast.parse(SCRIPT.read_text(encoding="utf-8"))
nodes = [node for node in tree.body
if isinstance(node, ast.FunctionDef)
and node.name == "_move_canonical_store"]
namespace = {
"PROCESSED_DATA_FOLDER": ".processed",
"logger": __import__("logging").getLogger("move-store-test"),
"move_canonical_marker": move_marker,
}
exec(compile(ast.Module(body=nodes, type_ignores=[]), str(SCRIPT), "exec"),
namespace)
return namespace["_move_canonical_store"]


@pytest.mark.skipif(
"move_canonical_marker" not in SCRIPT.read_text(encoding="utf-8"),
reason="sidecar move rollback arrives with a later source",
)
def test_failed_sidecar_move_renames_the_store_back(tmp_path):
def deny(*_args, **_kwargs):
raise PermissionError(13, "read-only")

move = _move_store_function(deny)
source = tmp_path / "old.zarr"
source.mkdir()
destination = tmp_path / "new.zarr"

with pytest.raises(PermissionError):
move(source, destination)

assert source.is_dir()
assert not destination.exists()
3 changes: 2 additions & 1 deletion tests/test_canonical_reuse.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ def test_cached_plate_upgrade_requires_matching_backing_path(
"canonical_pixel_verified": False})}) for i in source.images)})
ns["get_legacy_zarr_path"] = lambda _: path if same_backing else tmp_path / "other.zarr"
writes = []
ns["write_indexed_canonical_marker"] = lambda *args: writes.append(args)
ns["write_indexed_canonical_marker"] = (
lambda *args, **_kwargs: writes.append(args))
ns["attach_canonical_plate_source"] = lambda *args: None
result = ns["upgrade_reused_canonical"](None, object(), source, path)
assert all(i.source.canonical_pixel_verified for i in result.images) == same_backing
Expand Down
107 changes: 101 additions & 6 deletions tests/test_canonical_zarr_transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,22 @@
SCRIPT_PATH = SOURCE_ROOT / "_data" / "_SLURM_Image_Transfer.py"


def _uses_processed_folder(path):
"""Whether the source keeps sidecars in BIOMERO folders.

Gates on the keyword the feature adds to the sidecar calls, not on log
text, so rewording a message fails the tests instead of skipping them.
"""
tree = ast.parse(path.read_text(encoding="utf-8"))
return any(
isinstance(node, ast.keyword) and node.arg == "processed_folder"
for node in ast.walk(tree))


class InvalidCanonicalStore(RuntimeError):
"""Stand-in for biomero_importer's error for a corrupt or uncommitted marker."""


def _load_canonical_functions():
tree = ast.parse(SCRIPT_PATH.read_text(encoding="utf-8"))
wanted = {
Expand Down Expand Up @@ -103,8 +119,10 @@ def _load_canonical_functions():
"log": lambda text: None,
"monotonic": monotonic,
"ThreadPoolExecutor": ThreadPoolExecutor,
"load_canonical_marker": lambda _path: None,
"write_indexed_canonical_marker": lambda *_args: None,
"PROCESSED_DATA_FOLDER": ".processed",
"InvalidCanonicalStore": InvalidCanonicalStore,
"load_canonical_marker": lambda _path, **_kwargs: None,
"write_indexed_canonical_marker": lambda *_args, **_kwargs: None,
"pixel_identities_match": lambda left, right: left == right,
"os": os,
"shutil": shutil,
Expand Down Expand Up @@ -497,7 +515,7 @@ def test_storage_backed_plate_index_round_trip(pixel_identity):
plate = Object(9, [])
ns["load_group_storage_roots"] = lambda: {}
ns["resolve_managed_source_path"] = lambda *_args: Path("/plate.zarr")
ns["load_canonical_marker"] = lambda _path: canonical
ns["load_canonical_marker"] = lambda _path, **_kwargs: canonical

result = ns["attach_canonical_plate_source"](
"connection",
Expand Down Expand Up @@ -542,7 +560,7 @@ def test_storage_backed_plate_marker_can_be_shared_by_duplicate_imports(
)])
ns["load_group_storage_roots"] = lambda: {}
ns["resolve_managed_source_path"] = lambda *_args: Path("/plate.zarr")
ns["load_canonical_marker"] = lambda _path: marker_source
ns["load_canonical_marker"] = lambda _path, **_kwargs: marker_source

restored = ns["get_canonical_plate_source"](plate)

Expand Down Expand Up @@ -921,7 +939,7 @@ def test_trust_upgrade_updates_one_store_without_new_generation(
])
marker_writes = []
ns["write_indexed_canonical_marker"] = (
lambda *args: marker_writes.append(args)
lambda *args, **_kwargs: marker_writes.append(args)
)

upgraded = ns["upgrade_reused_canonical"](
Expand All @@ -936,6 +954,82 @@ def test_trust_upgrade_updates_one_store_without_new_generation(
assert canonical_annotation.values == upgraded.to_annotation_values()


@pytest.mark.skipif(
not _uses_processed_folder(SCRIPT_PATH),
reason="unwritable sidecar folder handled in a later source",
)
def test_trust_upgrade_survives_unwritable_sidecar_folder(
tmp_path, pixel_identity, caplog,
):
"""A read-only BIOMERO folder must not fail the workflow."""
ns = _load_canonical_functions()
path = tmp_path / ".processed/Image-7.ome.zarr"
path.mkdir(parents=True)
unverified = source(pixel_identity).model_copy(update={
"canonical_pixel_verified": False,
"pixel_identity_origin": "raw",
})
obj = Object(7, [
annotation_for(unverified),
Annotation("legacy", {"Filepath": str(path)}),
])

def read_only(*_args, **_kwargs):
raise PermissionError(13, "Read-only file system", str(path))

ns["write_indexed_canonical_marker"] = read_only

with caplog.at_level("WARNING"):
result = ns["upgrade_reused_canonical"]("connection", obj, unverified, path)

assert result == unverified
assert result.canonical_pixel_verified is False
assert "Could not write the canonical sidecar" in caplog.text


@pytest.mark.skipif(
not _uses_processed_folder(SCRIPT_PATH),
reason="unreadable sidecars handled in a later source",
)
@pytest.mark.parametrize("error", [
InvalidCanonicalStore("Canonical marker is not committed"),
PermissionError(13, "Permission denied"),
ValueError("Zarr path has no name"),
])
def test_trust_upgrade_survives_corrupt_or_unreadable_sidecar(
tmp_path, pixel_identity, caplog, error,
):
"""A sidecar problem costs the cache, never the transfer."""
ns = _load_canonical_functions()
path = tmp_path / ".processed/Image-7.ome.zarr"
path.mkdir(parents=True)
unverified = source(pixel_identity).model_copy(update={
"canonical_pixel_verified": False,
"pixel_identity_origin": "raw",
})
obj = Object(7, [
annotation_for(unverified),
Annotation("legacy", {"Filepath": str(path)}),
])
writes = []

def unreadable(*_args, **_kwargs):
raise error

ns["load_canonical_marker"] = unreadable
ns["write_indexed_canonical_marker"] = (
lambda *args, **_kwargs: writes.append(args))

with caplog.at_level("WARNING"):
result = ns["upgrade_reused_canonical"](
"connection", obj, unverified, path)

# Treated as no cached sidecar: the normal verified upgrade proceeds.
assert result.canonical_pixel_verified is True
assert "Ignoring the canonical sidecar" in caplog.text
assert str(path) in caplog.text


def test_managed_zarr_location_uses_most_specific_root(tmp_path):
ns = _load_canonical_functions()
import_root = tmp_path / "import"
Expand Down Expand Up @@ -1101,7 +1195,8 @@ def test_indexes_existing_plate_with_storage_marker_and_compact_annotation(
lambda *args, **kwargs: build_calls.append((args, kwargs)) or canonical
)
ns["write_indexed_canonical_marker"] = (
lambda path, source: marker_writes.append((Path(path), source))
lambda path, source, **_kwargs: marker_writes.append(
(Path(path), source))
)

indexed = ns["index_existing_plate_zarr"](
Expand Down
6 changes: 4 additions & 2 deletions tests/test_shallow_comparison_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ def load_functions(relative, names, **values):
root = Path(os.environ["BIOMERO_SCRIPTS_ROOT"])
tree = ast.parse((root / relative).read_text(encoding="utf-8"))
nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]
namespace = dict(json=json, UUID=UUID, logger=logging.getLogger(__name__), **values)
namespace = dict(json=json, UUID=UUID, logger=logging.getLogger(__name__),
PROCESSED_DATA_FOLDER=".processed",
InvalidCanonicalStore=RuntimeError, **values)
exec(compile(ast.Module(body=nodes, type_ignores=[]), relative, "exec"), namespace)
return namespace

Expand Down Expand Up @@ -171,7 +173,7 @@ def test_reuse_adopts_refreshed_current_cache_without_hashing(tmp_path):
"dataset_path": "s1", "comparison_shape": old.pixel_identity.shape})
updated = old.model_copy(update={"pixel_identity": identity})
ns = _load_canonical_functions()
ns["load_canonical_marker"] = lambda _: updated
ns["load_canonical_marker"] = lambda _, **_kwargs: updated
ns["get_legacy_zarr_path"] = lambda _: pytest.fail("Current cache should be reused")
writes = []
ns["_update_canonical_annotation"] = lambda annotation, values: writes.append(values)
Expand Down
Loading