Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f7d264f
Import Zarr contracts from biomero-schema and own the workflow color
TorecLuik Nov 12, 2026
3f97647
Detached runs: own launcher registration, resume decisions and cancel…
TorecLuik Nov 12, 2026
b0effc8
Address review: legible core version error, colour constants placement
TorecLuik Nov 12, 2026
e2c6b4c
Cancel passes through every stage; admin script to request a cancel; …
TorecLuik Nov 12, 2026
e79cf92
Round 2: WorkflowInterrupted passthrough, explicit stop_check, Pipeli…
TorecLuik Nov 12, 2026
f1e3425
Resume: use core's SlurmJob.ended_without_success
TorecLuik Nov 12, 2026
5dcc347
Round 3: claim checks before owner-only writes, scancel failure named…
TorecLuik Nov 12, 2026
419e559
Remove the input marker from results after import on every path
TorecLuik Nov 13, 2026
cad2f63
Import TRANSFER_INPUT_MARKER from biomero-schema
TorecLuik Nov 13, 2026
805ec94
Import CanonicalInput and TRANSFER_INPUT_MARKER from biomero-schema t…
TorecLuik Nov 13, 2026
03ad824
Keep CanonicalInput import for biomero-scripts#88 to move
TorecLuik Nov 13, 2026
fe3170f
Harden the input marker cleanup after review
TorecLuik Nov 13, 2026
4bff087
Merge remote-tracking branch 'origin/master' into detached-ownership
TorecLuik Nov 13, 2026
6dfd15d
Batched launcher registration as a function (register_detached_batches)
TorecLuik Nov 13, 2026
8d6ed48
Read the Results_Path output name from biomero.constants.results
TorecLuik Nov 13, 2026
9c4344c
Merge #97 and master; require biomero>=2.9.0b26
TorecLuik Nov 13, 2026
c706411
Merge #88 (with #97) into detached-ownership
TorecLuik Nov 13, 2026
dfc2243
Take the result folder name from core and stop the marker sweep at th…
TorecLuik Nov 16, 2026
8425807
Merge #97 into provenance-views-cleanup; state no minimum core version
TorecLuik Nov 16, 2026
50cfa08
Merge #88 into detached-ownership
TorecLuik Nov 16, 2026
866958e
Wrap a docstring line
TorecLuik Nov 16, 2026
5714bb4
Merge fix/remove-input-marker-after-import into provenance-views-cleanup
TorecLuik Nov 16, 2026
c7674dc
Merge provenance-views-cleanup into detached-ownership
TorecLuik Nov 16, 2026
1133f5b
Make the result folder a module constant; check the workflow id, not …
TorecLuik Nov 16, 2026
329487a
Merge #97 (fixed result folder constant) into provenance-views-cleanup
TorecLuik Nov 16, 2026
0207f8a
Merge #88 into detached-ownership
TorecLuik Nov 16, 2026
d0499b5
Use core's Canonical_Inputs output name, the shallow-storage flag, an…
TorecLuik Nov 16, 2026
34d9fa5
Merge fix/remove-input-marker-after-import into provenance-views-cleanup
TorecLuik Nov 16, 2026
f4ecc98
Merge provenance-views-cleanup into detached-ownership
TorecLuik Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ scripts follow. Deployment and administration are documented in NL-BIOMERO.
| `__workflows/` | `SLURM_Run_Workflow.py`, `SLURM_Run_Workflow_Batched.py` | Run an analysis and retrieve results; optionally split inputs into batches. |
| `_data/` | Image/File Transfer, Remote Conversion, Get Update | Export inputs, convert formats and monitor jobs. |
| `_data/` | `SLURM_Get_Results.py`, `SLURM_Import_Results.py` | Upload results through OMERO or import in place with BIOMERO.importer. |
| `admin/` | Slurm Init, Check Setup, Shallow Storage Migration, Cownary, Tail Logs | Initialize and inspect the cluster, migrate managed storage metadata, maintain workflow metadata and diagnose execution. |
| `admin/` | Slurm Init, Check Setup, Shallow Storage Migration, Cownary, Cancel Workflow, Tail Logs | Initialize and inspect the cluster, migrate managed storage metadata, maintain workflow metadata and diagnose execution. |

CellPose Segmentation and Example Minimal Slurm Script are examples, not the
standard workflow entry points; NL-BIOMERO does not install them by default.
Expand All @@ -42,7 +42,8 @@ custom deployment, follow the [scripts installation guide](https://nl-bioimaging
and install the scripts on the OMERO server and detached worker.

Deploy matching core and scripts releases together. This release requires core's
`biomero.provenance` and `biomero.maintenance` APIs even when optional features
`biomero.metadata_views` (renamed from `biomero.provenance`) and
`biomero.maintenance` APIs even when optional features
are disabled. Prerelease tag counters can differ between repositories; use the
component references supplied by the NL-BIOMERO release.

Expand Down
993 changes: 742 additions & 251 deletions __workflows/SLURM_Run_Workflow.py

Large diffs are not rendered by default.

73 changes: 51 additions & 22 deletions __workflows/SLURM_Run_Workflow_Batched.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import omero.util.script_utils as script_utils
import omero.scripts as omscripts
from biomero import SlurmClient, constants
from biomero.detached import detached_mode_enabled
import logging
from itertools import islice
import time as timesleep
Expand All @@ -77,18 +78,54 @@

# Batched runs are queued the same way single runs are: this script resolves
# the batches and records them in a launcher task, and the supervisor spawns
# one child workflow per batch (see biomero.detached and, in NL-BIOMERO, the
# WorkflowSupervisor in processor.py). Without a biomero that ships the
# launcher contract, batches are run inline as sub-scripts as before.
try:
from biomero import detached
except ImportError: # pragma: no cover - older biomero release
detached = None
# one child workflow per batch (see WorkflowTracker.add_launcher_task and, in
# NL-BIOMERO, the WorkflowSupervisor in processor.py). With detached mode off,
# batches are run inline as sub-scripts.


def detached_mode_enabled() -> bool:
"""Whether to queue the batches for the supervisor instead of running them."""
return detached is not None and detached.detached_mode_enabled()
def register_detached_batches(client, slurmClient, wf_id,
selected_workflow_names, batch_ids_list,
batch_size, effective_type, inputs,
selected_output, group, image_ids):
"""Record a batched request as a launcher task for the detached supervisor.

The task's params are the contract with the supervisor's batched parent:
every input of this script, the selected workflows, the ``batches`` with
their ``batch_size`` and ``effective_type``, the ``base_inputs`` each child
starts from, the ``selected_output``, the ``group`` and all image IDs.

Args:
client: OMERO script client whose inputs are recorded.
slurmClient: Active SLURM client, whose tracker records the task.
wf_id: Workflow the launcher belongs to.
selected_workflow_names: Workflows the user chose.
batch_ids_list: The IDs of each batch.
batch_size: Items per batch.
effective_type: Data type of the items after batching.
inputs: Script inputs as rtypes, as sent to each batch.
selected_output: Resolved output options.
group: Active OMERO group ID.
image_ids: All item IDs of the request.

Returns:
UUID: The launcher task ID.
"""
launcher_params = {
**(client.getInputs(unwrap=True) or {}),
"workflows": selected_workflow_names,
"batches": [list(batch) for batch in batch_ids_list],
"batch_size": batch_size,
"effective_type": effective_type,
"base_inputs": {key: unwrap(value) for key, value in inputs.items()},
"selected_output": selected_output,
"group": group,
constants.transfer.IDS: list(image_ids)}
launcher_task_id = slurmClient.workflowTracker.add_launcher_task(
wf_id, constants.RUN_WF_BATCHED_SCRIPT, VERSION,
launcher_params[constants.transfer.IDS], launcher_params)
logger.info(f"Queued detached workflow {wf_id} (launcher task "
f"{launcher_task_id}) for {selected_workflow_names}")
return launcher_task_id


def runScript():
Expand Down Expand Up @@ -540,18 +577,10 @@ def runScript():
selected_workflow_names = [name for name, selected
in selected_workflows.items()
if selected]
detached.register_detached_launcher(
client, slurmClient.workflowTracker, wf_id,
constants.RUN_WF_BATCHED_SCRIPT, VERSION,
selected_workflow_names,
{"batches": [list(batch) for batch in batch_ids_list],
"batch_size": batch_size,
"effective_type": effective_type,
"base_inputs": {key: unwrap(value)
for key, value in inputs.items()},
"selected_output": selected_output,
"group": group,
constants.transfer.IDS: list(image_ids)})
register_detached_batches(
client, slurmClient, wf_id, selected_workflow_names,
batch_ids_list, batch_size, effective_type, inputs,
selected_output, group, image_ids)
client.setOutput("Message", rstring(
f"Your {len(batch_ids_list)} batches are queued and run "
f"in the background: you can close this tab. Results are "
Expand Down
22 changes: 14 additions & 8 deletions _data/SLURM_Get_Results.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,15 @@
import zipfile
import glob
from biomero import SlurmClient, constants
from biomero.provenance import render_workflow_metadata
try:
from biomero.metadata_views import (
format_large_field_reference, render_workflow_metadata)
except ImportError as error:
raise ImportError(
"These scripts need the biomero release that matches them, which provides "
"biomero.metadata_views (the module formerly named "
"biomero.provenance). Update biomero to the release matching these "
"scripts.") from error
import logging
import ezomero
# from aicsimageio import AICSImage
Expand Down Expand Up @@ -758,9 +766,8 @@ def write(values):
safe_key = "Long_key_sha256_" + hashlib.sha256(raw_key).hexdigest()
while safe_key in kv_dict or safe_key in reduced:
safe_key += "_"
reduced[safe_key] = (
"Large field; full value in {} (utf8_bytes={}; sha256={})".format(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest()))
reduced[safe_key] = format_large_field_reference(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest())
if reduced == kv_dict:
logger.warning("Index-size rejection for %s %s (%s); no large fields "
"could be reduced", object_type, object_id, ns)
Expand Down Expand Up @@ -2489,10 +2496,9 @@ def runScript():
roi_name_prefix=(unwrap(
client.getInput(constants.results.ROI_NAME_PREFIX))
or ""),
roi_color=constants.resolve_workflow_color(
unwrap(client.getInput(
constants.results.ROI_COLOR)),
wf_id),
roi_color=(unwrap(client.getInput(
constants.results.ROI_COLOR))
or ""),
clear_existing_rois=(unwrap(
client.getInput(constants.results.ROI_CLEAR_EXISTING))
or False),
Expand Down
42 changes: 32 additions & 10 deletions _data/SLURM_Import_Results.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@
directory once import completes **unless** it was attached to OMERO (in which case
OMERO may reference the file directly via an in-place symlink and it must remain).

OUTPUTS:
- Message: concise summary of the import.
- Workflow_UUID: the workflow the results belong to.
- Results_Path: the ``.analyzed/uuid/timestamp`` folder that was imported from;
SLURM_Run_Workflow.py uses it to remove the transfer input markers afterwards.

PERMANENT STORAGE (otherwise preserved):
- Permanent storage: .analyzed/uuid/timestamp/ directories and extracted contents
- Complete workflow preservation: images, CSVs, metadata, logs, analysis results
Expand Down Expand Up @@ -102,7 +108,15 @@
import ezomero

from biomero import SlurmClient, constants
from biomero.provenance import render_workflow_metadata
try:
from biomero.metadata_views import (
format_large_field_reference, render_workflow_metadata)
except ImportError as error:
raise ImportError(
"These scripts need the biomero release that matches them, which "
"provides biomero.metadata_views (the module formerly named "
"biomero.provenance). Update biomero to the release matching these "
"scripts.") from error
try:
from omero_upload import upload_ln_s
INPLACE_ATTACHMENTS_AVAILABLE = True
Expand All @@ -113,6 +127,9 @@

logger = logging.getLogger(__name__)

# BIOMERO's result folder under the group folder; configurability is tracked
# in NL-BIOMERO#111.
RESULTS_FOLDER = ".analyzed"
RESULTS_RETRIEVED_MARKER = ".biomero-results-retrieved.json"
RESULT_IMPORT_STATE = ".biomero-result-import.json"
IMPORT_POLL_TIMEOUT_SECONDS = 24 * 60 * 60
Expand Down Expand Up @@ -1940,7 +1957,7 @@ def get_workflow_results_path(group_name: str, wf_id: Optional[str]) -> str:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")

results_path = os.path.join(
base_path, '.analyzed', wf_id, timestamp)
base_path, RESULTS_FOLDER, wf_id, timestamp)
return results_path


Expand All @@ -1957,7 +1974,7 @@ def find_existing_result_storage(
"""
base_path = get_importer_group_base_path(group_name)
workflow_path = os.path.join(
base_path, '.analyzed', str(wf_id)) if base_path else None
base_path, RESULTS_FOLDER, str(wf_id)) if base_path else None
if not workflow_path or not os.path.isdir(workflow_path):
return None, None

Expand Down Expand Up @@ -3580,9 +3597,8 @@ def write(values):
safe_key = "Long_key_sha256_" + hashlib.sha256(raw_key).hexdigest()
while safe_key in kv_dict or safe_key in reduced:
safe_key += "_"
reduced[safe_key] = (
"Large field; full value in {} (utf8_bytes={}; sha256={})".format(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest()))
reduced[safe_key] = format_large_field_reference(
reference, len(raw_value), hashlib.sha256(raw_value).hexdigest())
if reduced == kv_dict:
logger.warning("Index-size rejection for %s %s (%s); no large fields "
"could be reduced", object_type, object_id, ns)
Expand Down Expand Up @@ -5258,6 +5274,7 @@ def runScript() -> None:
slurm_job_id = None
wf_id = None
slurm_data_path = None
permanent_storage_path = None
folder = None
log_file = None
script_failed = False
Expand Down Expand Up @@ -5419,7 +5436,7 @@ def runScript() -> None:
# including concurrent first attempts before a result folder exists.
result_recovery = recovery_locks.enter_context(result_import_recovery(
os.path.join(get_importer_group_base_path(group_name),
".analyzed", str(wf_id)),
RESULTS_FOLDER, str(wf_id)),
name=f".biomero-finalize-{uuid.uuid5(uuid.NAMESPACE_URL, str(slurm_job_id))}.lock"))
# Update task status to IMPORTING if task_id is available
if task_id and slurmClient.track_workflows:
Expand Down Expand Up @@ -5676,9 +5693,8 @@ def runScript() -> None:
or False),
roi_name_prefix=(unwrap(client.getInput(
constants.results.ROI_NAME_PREFIX)) or ""),
roi_color=constants.resolve_workflow_color(
unwrap(client.getInput(
constants.results.ROI_COLOR)), wf_id),
roi_color=(unwrap(client.getInput(
constants.results.ROI_COLOR)) or ""),
clear_existing_rois=(unwrap(client.getInput(
constants.results.ROI_CLEAR_EXISTING)) or False),
clear_roi_filter=(unwrap(client.getInput(
Expand Down Expand Up @@ -5907,6 +5923,12 @@ def runScript() -> None:
try:
if wf_id is not None:
client.setOutput(constants.results.WORKFLOW_UUID_OUTPUT, rstring(str(wf_id)))
# Tell the workflow runner which result folder was imported,
# so it can clean up the transfer's working files there.
if permanent_storage_path:
client.setOutput(
constants.results.RESULTS_PATH_OUTPUT,
rstring(str(permanent_storage_path)))
# Only set success message if we haven't already set a failure message
if not script_failed:
client.setOutput("Message", rstring(str(message)))
Expand Down
5 changes: 2 additions & 3 deletions _data/_SLURM_Image_Transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
except ImportError:
import Image
from biomero import SlurmClient, constants
from biomero.zarr_contracts import (
from biomero_schema.zarr import (
CANONICAL_PLATE_IMAGE_NAMESPACE,
CANONICAL_PLATE_LABEL_NAMESPACE,
CANONICAL_PLATE_SOURCE_NAMESPACE,
Expand Down Expand Up @@ -157,7 +157,6 @@
)
IMPORT_MOUNT_PATH = os.getenv("IMPORT_MOUNT_PATH", "/data")
IMPORT_MOUNT_STORAGE_ROOT = "import-mount-data"
CANONICAL_INPUTS_OUTPUT = "Canonical_Inputs"
SCRIPT_CLIENT_KEEPALIVE_SECONDS = 60

# keep track of log strings.
Expand Down Expand Up @@ -3331,7 +3330,7 @@ def run_script():
_require_keepalive(conn.keepAlive, "writing Image Transfer outputs")
client.setOutput("Message", rstring(message))
client.setOutput(
CANONICAL_INPUTS_OUTPUT,
constants.results.CANONICAL_INPUTS_OUTPUT,
rstring(json.dumps([
item.to_dict() for item in canonical_inputs
], separators=(",", ":"), sort_keys=True)),
Expand Down
Loading
Loading