Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
f7d264f
Import Zarr contracts from biomero-schema and own the workflow color
TorecLuik Nov 12, 2026
b0effc8
Address review: legible core version error, colour constants placement
TorecLuik Nov 12, 2026
419e559
Remove the input marker from results after import on every path
TorecLuik Nov 13, 2026
cad2f63
Import TRANSFER_INPUT_MARKER from biomero-schema
TorecLuik Nov 13, 2026
805ec94
Import CanonicalInput and TRANSFER_INPUT_MARKER from biomero-schema t…
TorecLuik Nov 13, 2026
03ad824
Keep CanonicalInput import for biomero-scripts#88 to move
TorecLuik Nov 13, 2026
fe3170f
Harden the input marker cleanup after review
TorecLuik Nov 13, 2026
8d6ed48
Read the Results_Path output name from biomero.constants.results
TorecLuik Nov 13, 2026
9c4344c
Merge #97 and master; require biomero>=2.9.0b26
TorecLuik Nov 13, 2026
dfc2243
Take the result folder name from core and stop the marker sweep at th…
TorecLuik Nov 16, 2026
8425807
Merge #97 into provenance-views-cleanup; state no minimum core version
TorecLuik Nov 16, 2026
866958e
Wrap a docstring line
TorecLuik Nov 16, 2026
5714bb4
Merge fix/remove-input-marker-after-import into provenance-views-cleanup
TorecLuik Nov 16, 2026
1133f5b
Make the result folder a module constant; check the workflow id, not …
TorecLuik Nov 16, 2026
329487a
Merge #97 (fixed result folder constant) into provenance-views-cleanup
TorecLuik Nov 16, 2026
d0499b5
Use core's Canonical_Inputs output name, the shallow-storage flag, an…
TorecLuik Nov 16, 2026
34d9fa5
Merge fix/remove-input-marker-after-import into provenance-views-cleanup
TorecLuik Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ custom deployment, follow the [scripts installation guide](https://nl-bioimaging
and install the scripts on the OMERO server and detached worker.

Deploy matching core and scripts releases together. This release requires core's
`biomero.provenance` and `biomero.maintenance` APIs even when optional features
`biomero.metadata_views` (renamed from `biomero.provenance`) and
`biomero.maintenance` APIs even when optional features
are disabled. Prerelease tag counters can differ between repositories; use the
component references supplied by the NL-BIOMERO release.

Expand Down
196 changes: 186 additions & 10 deletions __workflows/SLURM_Run_Workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@
from __future__ import print_function
import sys
import os
from pathlib import Path
import json
import re
from uuid import UUID, uuid4
import omero
from omero.grid import JobParams
Expand All @@ -79,7 +81,9 @@
os.getenv("BIOMERO_SHALLOW_ZARR", "false").lower() == "true"
)
if IMPORTER_ENABLED and SHALLOW_ZARR_ENABLED:
from biomero.zarr_contracts import CanonicalInput
from biomero_schema.zarr import CanonicalInput, TRANSFER_INPUT_MARKER
# The transfer writes input markers only when both are on.
SHALLOW_ZARR_STORAGE_ENABLED = IMPORTER_ENABLED and SHALLOW_ZARR_ENABLED

EXPORT_SCRIPTS = [constants.IMAGE_EXPORT_SCRIPT]
# Dynamically choose import script based on IMPORTER_ENABLED
Expand All @@ -102,8 +106,24 @@
constants.workflow.OUTPUT_CSV_TABLE,
constants.workflow.OUTPUT_ATTACH_FILE_OUTPUTS,
constants.workflow.OUTPUT_CREATE_ROIS]
# Blueprint qualitative color scheme, used for the ROIs of a workflow. Keep
# these values in Python because OMERO scripts do not load the Blueprint
# JavaScript package.
# https://blueprintjs.com/docs/#core/colors.qualitative-color-schemes
QUALITATIVE_COLOR_SCHEME = (
"#147EB3", # cerulean3
"#29A634", # forest3
"#D1980B", # gold3
"#D33D17", # vermilion3
"#9D3F9D", # violet3
"#00A396", # turquoise3
"#DB2C6F", # rose3
"#8EB125", # lime3
"#946638", # sepia3
"#7961DB", # indigo3
)
ROI_COLOR_PATTERN = re.compile(r"#[0-9A-F]{6}")
VERSION = "2.9.0b24"
CANONICAL_INPUTS_OUTPUT = "Canonical_Inputs"


def ensure_tracking_uuid(value):
Expand All @@ -115,16 +135,136 @@ def ensure_tracking_uuid(value):
return UUID(str(value))


def is_result_folder_of_workflow(results_root: str, workflow_id: UUID) -> bool:
"""Tell whether the parent folder of a path is named like this workflow.

``get_workflow_results_path`` in SLURM_Import_Results.py makes
``<...>/<workflow id>/<timestamp>``; this checks only the workflow id
part, so the result folder of another workflow, a workflow folder and
most unrelated paths are not accepted.

Args:
results_root (str): Resolved path to check.
workflow_id (UUID): Workflow UUID of this run.

Returns:
bool: True if the name of the parent folder is the workflow id.
"""
return Path(results_root).parent.name == str(workflow_id)


def log_walk_error(walk_error: OSError) -> None:
"""Warn about a folder the marker search could not read."""
logger.warning(f"Input marker search skipped a folder: {walk_error}")


def remove_input_markers(
results_path: str, marker_name: str, workflow_id: UUID) -> int:
"""Remove the transfer input markers from one BIOMERO result folder.

The workflow runner owns this cleanup: the marker is only a working file
of the transfer, and the importer needs it until the result is imported.
Only files named exactly like the marker are removed, and only below one
import's result folder of this workflow. Links are not
followed, and the search stops at the stores: the result folder holds the
exported stores directly, and the marker sits at the root of each.

Args:
results_path (str): Result folder of one import,
``<...>/<workflow id>/<timestamp>``.
marker_name (str): File name of the input marker; the caller passes
``TRANSFER_INPUT_MARKER``, which only loads with shallow Zarr.
workflow_id (UUID): Workflow UUID of this run; the result folder
must belong to it.

Returns:
int: Number of markers removed.
"""
results_root = os.path.realpath(results_path)
if not is_result_folder_of_workflow(results_root, workflow_id):
logger.warning(
f"Not removing input markers: {results_root} is not a result "
f"folder of the form <...>/{workflow_id}/<timestamp>")
return 0
if not os.path.isdir(results_root):
logger.warning(
f"Not removing input markers: result folder {results_root} "
"does not exist")
return 0
removed = 0
for directory, subdirectories, file_names in os.walk(
results_root, onerror=log_walk_error, followlinks=False):
if directory == results_root:
continue # the stores are its subfolders; no marker sits here
if marker_name in file_names:
marker_path = os.path.join(directory, marker_name)
try:
os.remove(marker_path)
removed += 1
logger.info(f"Removed input marker {marker_path}")
except OSError as remove_error:
logger.warning(
f"Could not remove input marker {marker_path}: "
f"{remove_error}")
subdirectories.clear() # nothing below a store root is a marker
logger.info(
f"Removed {removed} input marker(s) below {results_root}")
return removed


def remove_input_markers_after_import(
import_outputs: dict, workflow_id: UUID) -> int:
"""Remove the input markers from the result that was just imported.

Call this only after the result import succeeded: the importer reads the
marker during import, and removes it itself only after a local evaluation
or when the helper kept the full result. It stays in a result that was
shallowed on the HPC, and in a result the importer did not evaluate.
This one step covers all of those paths: the importer's removal is
consume-on-use, this one is the final sweep.

Args:
import_outputs (dict): Outputs of the result import script.
workflow_id (UUID): Workflow UUID of this run.

Returns:
int: Number of markers removed.
"""
if not SHALLOW_ZARR_STORAGE_ENABLED:
logger.debug(
"Not removing input markers: shallow Zarr storage is off, so "
"the transfer wrote none")
return 0
results_path_output = constants.results.RESULTS_PATH_OUTPUT
if not import_outputs or results_path_output not in import_outputs:
logger.warning(
f"Cannot remove input markers: the import script returned no "
f"{constants.results.RESULTS_PATH_OUTPUT} output")
return 0
results_path = unwrap(import_outputs[results_path_output])
if not results_path:
logger.warning(
f"Cannot remove input markers: empty "
f"{constants.results.RESULTS_PATH_OUTPUT} output")
return 0
logger.info(
f"Result import is done; removing input markers from {results_path}")
return remove_input_markers(
results_path, TRANSFER_INPUT_MARKER, workflow_id)


def parse_canonical_inputs_output(results):
"""Validate canonical input records returned by Image Transfer."""
if not results or CANONICAL_INPUTS_OUTPUT not in results:
if not results or constants.results.CANONICAL_INPUTS_OUTPUT not in results:
logger.debug(
f"Image Transfer returned no {CANONICAL_INPUTS_OUTPUT} output")
f"Image Transfer returned no "
f"{constants.results.CANONICAL_INPUTS_OUTPUT} output")
return ()
raw_value = unwrap(results[CANONICAL_INPUTS_OUTPUT])
raw_value = unwrap(results[constants.results.CANONICAL_INPUTS_OUTPUT])
if not raw_value:
logger.debug(
f"Image Transfer returned an empty {CANONICAL_INPUTS_OUTPUT} "
f"Image Transfer returned an empty "
f"{constants.results.CANONICAL_INPUTS_OUTPUT} "
"output")
return ()
payload = json.loads(raw_value)
Expand Down Expand Up @@ -344,6 +484,34 @@ def build_output_settings(client, selected_output) -> dict:
return settings


def resolve_workflow_color(color_override, workflow_id) -> str:
"""Return the ROI color of a workflow.

The result import scripts and Labels2Rois receive this resolved value, so
the color is decided once, here.

Args:
color_override: Optional ``#RRGGBB`` color chosen by the user.
workflow_id: UUID of the workflow, used to pick a palette color
deterministically when there is no override.

Returns:
The override in upper case, or the palette color of the workflow.

Raises:
ValueError: If the override is not in ``#RRGGBB`` format.
"""
color = str(color_override or "").strip().upper()
if color:
if not ROI_COLOR_PATTERN.fullmatch(color):
raise ValueError(
"Workflow color must be empty or use #RRGGBB format")
return color
return QUALITATIVE_COLOR_SCHEME[
UUID(str(workflow_id)).int % len(QUALITATIVE_COLOR_SCHEME)
]


def get_roi_script_capability(conn: BlitzGateway) -> dict:
"""Ask OMERO whether its optional Labels2Rois script is installed."""
svc = conn.getScriptService()
Expand Down Expand Up @@ -559,7 +727,7 @@ def validate_importer_write_access(slurmClient: SlurmClient, conn: BlitzGateway,
def validate_remote_shallower_ready(slurmClient: SlurmClient,
use_zarr_format: bool) -> None:
"""Fail before workflow submission when remote shallowing cannot run."""
if not (IMPORTER_ENABLED and SHALLOW_ZARR_ENABLED and use_zarr_format
if not (SHALLOW_ZARR_STORAGE_ENABLED and use_zarr_format
and slurmClient.remote_shallow_zarr):
logger.debug(
"Remote shallower readiness check skipped: "
Expand Down Expand Up @@ -612,7 +780,7 @@ def runScript():
and handled gracefully.
"""
shallow_state = ("enabled by flags"
if IMPORTER_ENABLED and SHALLOW_ZARR_ENABLED
if SHALLOW_ZARR_STORAGE_ENABLED
else "disabled, results stay full")
logger.info(
f"Shallow Zarr results: BIOMERO_SHALLOW_ZARR={SHALLOW_ZARR_ENABLED}, "
Expand Down Expand Up @@ -2009,7 +2177,7 @@ def exportImageToSLURM(client: omscripts.client,
logger.error(error_msg)
raise RuntimeError(error_msg)

if IMPORTER_ENABLED and SHALLOW_ZARR_ENABLED:
if SHALLOW_ZARR_STORAGE_ENABLED:
logger.info(
"Shallow Zarr enabled: reading canonical inputs returned by "
f"Image Transfer for workflow {wf_id}")
Expand Down Expand Up @@ -2436,7 +2604,7 @@ def importResultsToOmero(client: omscripts.client,
or rstring("Polygon"))
inputs[constants.results.ROI_NAME_PREFIX] = rstring(
f"{workflow_name}__{wf_id}")
roi_color = constants.resolve_workflow_color(
roi_color = resolve_workflow_color(
unwrap(client.getInput(constants.workflow.ROI_COLOR)), wf_id)
inputs[constants.results.ROI_COLOR] = rstring(roi_color)
logger.info(f"Using ROI color {roi_color} for workflow {wf_id}")
Expand Down Expand Up @@ -2526,6 +2694,14 @@ def importResultsToOmero(client: omscripts.client,
slurmClient.workflowTracker.fail_task(task_id, error_msg)
wf_failed = True
raise RuntimeError(error_msg)
# Reached only after a successful import: the importer needed the markers.
try:
remove_input_markers_after_import(rv, wf_id)
except Exception as cleanup_error:
# The result is imported; a cleanup problem must not fail the run.
logger.warning(
f"Input marker cleanup failed, leaving the markers: "
f"{cleanup_error}", exc_info=True)
return rv


Expand Down
22 changes: 14 additions & 8 deletions _data/SLURM_Get_Results.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,15 @@
import zipfile
import glob
from biomero import SlurmClient, constants
from biomero.provenance import render_workflow_metadata
try:
from biomero.metadata_views import (
format_large_field_reference, render_workflow_metadata)
except ImportError as error:
raise ImportError(
"These scripts need the biomero release that matches them, which provides "
"biomero.metadata_views (the module formerly named "
"biomero.provenance). Update biomero to the release matching these "
"scripts.") from error
import logging
import ezomero
# from aicsimageio import AICSImage
Expand Down Expand Up @@ -758,9 +766,8 @@ def write(values):
safe_key = "Long_key_sha256_" + hashlib.sha256(raw_key).hexdigest()
while safe_key in kv_dict or safe_key in reduced:
safe_key += "_"
reduced[safe_key] = (
"Large field; full value in {} (utf8_bytes={}; sha256={})".format(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest()))
reduced[safe_key] = format_large_field_reference(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest())
if reduced == kv_dict:
logger.warning("Index-size rejection for %s %s (%s); no large fields "
"could be reduced", object_type, object_id, ns)
Expand Down Expand Up @@ -2489,10 +2496,9 @@ def runScript():
roi_name_prefix=(unwrap(
client.getInput(constants.results.ROI_NAME_PREFIX))
or ""),
roi_color=constants.resolve_workflow_color(
unwrap(client.getInput(
constants.results.ROI_COLOR)),
wf_id),
roi_color=(unwrap(client.getInput(
constants.results.ROI_COLOR))
or ""),
clear_existing_rois=(unwrap(
client.getInput(constants.results.ROI_CLEAR_EXISTING))
or False),
Expand Down
Loading
Loading