From 32ea660a050338769bdf7f6d4410cec0ad4fc8a1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 13 Mar 2026 13:21:07 -0400 Subject: [PATCH 01/53] add api route for retrieving a bucket policy --- apps/api/src/hyp3_api/handlers.py | 28 ++++++++++++++++++++++++++++ apps/api/src/hyp3_api/routes.py | 6 ++++++ apps/api/src/hyp3_api/util.py | 7 +++++++ 3 files changed, 41 insertions(+) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 4aabb1088..b2a2454e1 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -132,3 +132,31 @@ def _get_names_for_user(user: str) -> list[str]: jobs.extend(new_jobs) names = {job['name'] for job in jobs if 'name' in job} return sorted(list(names)) + + +def get_bucket_policy(bucket_name: str): + account_arn = util.get_account_arn() + + policy = f''' + {{ + "Version": "2012-10-17", + "Statement": [ + {{ + "Sid": "write permission", + "Effect": "Allow", + "Principal": {{ "AWS": "{account_arn}:root" }}, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::{bucket_name}/*" + }}, + {{ + "Sid": "get bucket location permission", + "Effect": "Allow", + "Principal": {{ "AWS": "arn:aws:iam::{account_arn}:root" }}, + "Action": "s3:GetBucketLocation", + "Resource": "arn:aws:s3:::{bucket_name}" + }} + ] + }} + ''' + + return policy diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 4ede62204..f4cd9c228 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -193,3 +193,9 @@ def user_patch() -> Response: @openapi def user_get() -> Response: return jsonify(handlers.get_user(g.user)) + + +@app.route('/bucket-policy/', methods=['GET']) +@openapi +def bucket_policy_get(bucket_name: str) -> Response: + return jsonify(handlers.get_bucket_policy(bucket_name)) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 1b6b1ad3d..ac7075c40 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -4,6 +4,7 @@ from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +import boto3 class TokenDeserializeError(Exception): """Raised when paging results and `start_token` fails to deserialize.""" @@ -46,3 +47,9 @@ def build_next_url(url: str, start_token: str, x_forwarded_host: str | None = No url_parts[4] = urlencode(query) return urlunparse(url_parts) + + +def get_current_account_arn(): + sts = boto3.client('sts') + account = sts.get_caller_identity().split(':user')[0] + return account From 3eba802e27a9a90a296dd28d288dce4876e300dd Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 13 Mar 2026 13:22:01 -0400 Subject: [PATCH 02/53] formatting --- apps/api/src/hyp3_api/handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index b2a2454e1..033a3b38c 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -136,7 +136,6 @@ def _get_names_for_user(user: str) -> list[str]: def get_bucket_policy(bucket_name: str): account_arn = util.get_account_arn() - policy = f''' {{ "Version": "2012-10-17", @@ -158,5 +157,4 @@ def get_bucket_policy(bucket_name: str): ] }} ''' - return policy From c9656f9828baa973ae0e6b6292b7035ea829ef4a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 13 Mar 2026 13:25:11 -0400 Subject: [PATCH 03/53] add boto3 req --- requirements-apps-api.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt index d9f129612..c0c0eafbf 100644 --- a/requirements-apps-api.txt +++ b/requirements-apps-api.txt @@ -1,3 +1,4 @@ +boto3==1.42.46 flask==3.1.3 Flask-Cors==6.0.2 jsonschema==4.26.0 From 9795e68d4c4163ca4ac2eff779e0b90db24dce12 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:16:20 -0400 Subject: [PATCH 04/53] add bucket and bucket prefix from api ref --- job_spec/AK_FIRE_SAFE.yml | 4 ++-- job_spec/ARIA_AUTORIFT.yml | 4 ++-- job_spec/ARIA_RAIDER.yml | 4 ++-- job_spec/ARIA_S1_COSEIS.yml | 4 ++-- job_spec/ARIA_S1_GUNW.yml | 4 ++-- job_spec/AUTORIFT.yml | 4 ++-- job_spec/INSAR_GAMMA.yml | 4 ++-- job_spec/INSAR_ISCE.yml | 4 ++-- job_spec/INSAR_ISCE_BURST.yml | 4 ++-- job_spec/INSAR_ISCE_MULTI_BURST.yml | 4 ++-- job_spec/ITS_LIVE_AUTORIFT.yml | 4 ++-- job_spec/ITS_LIVE_CROP.yml | 10 ++++++---- job_spec/ITS_LIVE_CROP_BULK.yml | 10 ++++++---- job_spec/ITS_LIVE_META_BULK.yml | 5 +++-- job_spec/OPERA_DISP_TMS.yml | 8 ++++---- job_spec/OPERA_DIST_S1.yml | 6 +++--- job_spec/OPERA_DIST_S1_CONFIRMATION.yml | 6 +++--- job_spec/PISM_TERRA_EXECUTE.yml | 4 ++-- job_spec/PISM_TERRA_PREP_ENSEMBLE.yml | 4 ++-- job_spec/PISM_TERRA_RUN.yml | 4 ++-- job_spec/RTC_GAMMA.yml | 4 ++-- job_spec/SLIMSAR_TDBP.yml | 4 ++-- job_spec/SRG_GSLC.yml | 4 ++-- job_spec/SRG_TIME_SERIES.yml | 4 ++-- job_spec/VOLCSARVATORY_MINTPY.yml | 5 +++-- job_spec/VOLCSARVATORY_MULTI_BURST.yml | 5 +++-- job_spec/WATER_MAP.yml | 12 ++++++------ job_spec/WATER_MAP_EQ.yml | 8 ++++---- 28 files changed, 77 insertions(+), 70 deletions(-) diff --git a/job_spec/AK_FIRE_SAFE.yml b/job_spec/AK_FIRE_SAFE.yml index c14c903bb..fab79c1fb 100644 --- a/job_spec/AK_FIRE_SAFE.yml +++ b/job_spec/AK_FIRE_SAFE.yml @@ -39,9 +39,9 @@ AK_FIRE_SAFE: image: ghcr.io/asfhyp3/hyp3-gather-landsat command: - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --start-date - Ref::start_date - --end-date diff --git a/job_spec/ARIA_AUTORIFT.yml b/job_spec/ARIA_AUTORIFT.yml index ddbf7667b..aa1a119d0 100644 --- a/job_spec/ARIA_AUTORIFT.yml +++ b/job_spec/ARIA_AUTORIFT.yml @@ -148,9 +148,9 @@ AUTORIFT: command: - Ref::granules - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --parameter-file - Ref::parameter_file - --naming-scheme diff --git a/job_spec/ARIA_RAIDER.yml b/job_spec/ARIA_RAIDER.yml index 1ed85034f..35f2d62b7 100644 --- a/job_spec/ARIA_RAIDER.yml +++ b/job_spec/ARIA_RAIDER.yml @@ -30,9 +30,9 @@ ARIA_RAIDER: - ++process - calcDelaysGUNW - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --weather-model - Ref::weather_model - --input-bucket-prefix diff --git a/job_spec/ARIA_S1_COSEIS.yml b/job_spec/ARIA_S1_COSEIS.yml index a01007a2d..5ba366bc0 100644 --- a/job_spec/ARIA_S1_COSEIS.yml +++ b/job_spec/ARIA_S1_COSEIS.yml @@ -104,9 +104,9 @@ ARIA_S1_COSEIS: - ++omp-num-threads - '4' # 8 vCPUs per 16 GB RAM for the C instance family; 4 for M; 2 for R - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --reference-scenes - Ref::granules - --secondary-scenes diff --git a/job_spec/ARIA_S1_GUNW.yml b/job_spec/ARIA_S1_GUNW.yml index 40d19f0a0..16a3f3f41 100644 --- a/job_spec/ARIA_S1_GUNW.yml +++ b/job_spec/ARIA_S1_GUNW.yml @@ -61,9 +61,9 @@ ARIA_S1_GUNW: - ++process - calcDelaysGUNW - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --weather-model - HRRR timeout: 10800 # 3 hr diff --git a/job_spec/AUTORIFT.yml b/job_spec/AUTORIFT.yml index 03f6efd30..c33633b8c 100644 --- a/job_spec/AUTORIFT.yml +++ b/job_spec/AUTORIFT.yml @@ -42,9 +42,9 @@ AUTORIFT: command: - Ref::granules - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --parameter-file - '/vsicurl/https://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp' - --naming-scheme diff --git a/job_spec/INSAR_GAMMA.yml b/job_spec/INSAR_GAMMA.yml index 30e9d139b..c59cde5df 100644 --- a/job_spec/INSAR_GAMMA.yml +++ b/job_spec/INSAR_GAMMA.yml @@ -86,9 +86,9 @@ INSAR_GAMMA: - ++omp-num-threads - '4' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --include-look-vectors - Ref::include_look_vectors - --include-los-displacement diff --git a/job_spec/INSAR_ISCE.yml b/job_spec/INSAR_ISCE.yml index 13db0fa51..6d678a455 100644 --- a/job_spec/INSAR_ISCE.yml +++ b/job_spec/INSAR_ISCE.yml @@ -102,9 +102,9 @@ INSAR_ISCE: - ++omp-num-threads - '4' # 8 vCPUs per 16 GB RAM for the C instance family; 4 for M; 2 for R - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --reference-scenes - Ref::granules - --secondary-scenes diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 1fa93173a..831f83799 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -50,9 +50,9 @@ INSAR_ISCE_BURST: - ++omp-num-threads - '1' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --apply-water-mask - Ref::apply_water_mask - --looks diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml index e05e52e64..0034b038b 100644 --- a/job_spec/INSAR_ISCE_MULTI_BURST.yml +++ b/job_spec/INSAR_ISCE_MULTI_BURST.yml @@ -109,9 +109,9 @@ INSAR_ISCE_MULTI_BURST: - ++process - insar_tops_multi_burst - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --apply-water-mask - Ref::apply_water_mask - --looks diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index b6e9ac27f..0c8e48cde 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -199,9 +199,9 @@ AUTORIFT: - ++plugin - meta - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --publish-bucket - Ref::publish_bucket - --stac-items-endpoint diff --git a/job_spec/ITS_LIVE_CROP.yml b/job_spec/ITS_LIVE_CROP.yml index 4bb86b0d6..584840480 100644 --- a/job_spec/ITS_LIVE_CROP.yml +++ b/job_spec/ITS_LIVE_CROP.yml @@ -41,9 +41,10 @@ ITS_LIVE_CROP: - crop_netcdf_product - Ref::granule_uri - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 10800 @@ -61,9 +62,10 @@ ITS_LIVE_CROP: - --granule-uri - Ref::granule_uri - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket - --stac-items-endpoint diff --git a/job_spec/ITS_LIVE_CROP_BULK.yml b/job_spec/ITS_LIVE_CROP_BULK.yml index 285c72183..eee3423ea 100644 --- a/job_spec/ITS_LIVE_CROP_BULK.yml +++ b/job_spec/ITS_LIVE_CROP_BULK.yml @@ -47,9 +47,10 @@ ITS_LIVE_CROP_BULK: - --stop-idx - Ref::stop_idx - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 @@ -71,9 +72,10 @@ ITS_LIVE_CROP_BULK: - --stop-idx - Ref::stop_idx - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index 16b2e1a7b..e584bbdc6 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -47,9 +47,10 @@ ITS_LIVE_META_BULK: - --stop-idx - Ref::stop_idx - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 diff --git a/job_spec/OPERA_DISP_TMS.yml b/job_spec/OPERA_DISP_TMS.yml index d6930216f..409cab16f 100644 --- a/job_spec/OPERA_DISP_TMS.yml +++ b/job_spec/OPERA_DISP_TMS.yml @@ -34,9 +34,9 @@ OPERA_DISP_TMS: - '20140101' - '20300101' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix timeout: 1800 # 30 min compute_environment: Default vcpu: 1 @@ -51,9 +51,9 @@ OPERA_DISP_TMS: - create_tile_map - velocity - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix timeout: 21600 # 6 hr compute_environment: Default vcpu: 1 diff --git a/job_spec/OPERA_DIST_S1.yml b/job_spec/OPERA_DIST_S1.yml index 228c54d45..13a857a52 100644 --- a/job_spec/OPERA_DIST_S1.yml +++ b/job_spec/OPERA_DIST_S1.yml @@ -164,9 +164,9 @@ OPERA_DIST_S1: - --post_date_buffer_days - Ref::post_date_buffer_days - --bucket - - '!Ref Bucket' - - --bucket_prefix - - Ref::job_id + - Ref::bucket + - --bucket-prefix + - Ref::bucket_prefix - --memory_strategy - Ref::memory_strategy - --low_confidence_alert_threshold diff --git a/job_spec/OPERA_DIST_S1_CONFIRMATION.yml b/job_spec/OPERA_DIST_S1_CONFIRMATION.yml index c7ca39ed3..9397bb3ef 100644 --- a/job_spec/OPERA_DIST_S1_CONFIRMATION.yml +++ b/job_spec/OPERA_DIST_S1_CONFIRMATION.yml @@ -31,9 +31,9 @@ OPERA_DIST_S1_CONFIRMATION: - --dist_s1_data_bucket - Ref::dist_s1_data_bucket - --bucket - - '!Ref Bucket' - - --bucket_prefix - - Ref::job_id + - Ref::bucket + - --bucket-prefix + - Ref::bucket_prefix timeout: 3600 # 1 hr compute_environment: Default vcpu: 1 diff --git a/job_spec/PISM_TERRA_EXECUTE.yml b/job_spec/PISM_TERRA_EXECUTE.yml index bfd09b199..6a0e8efe8 100644 --- a/job_spec/PISM_TERRA_EXECUTE.yml +++ b/job_spec/PISM_TERRA_EXECUTE.yml @@ -26,9 +26,9 @@ PISM_TERRA_EXECUTE: - pism-glacier-execute - Ref::run_script - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --job-id - Ref::ensemble_job_id secrets: diff --git a/job_spec/PISM_TERRA_PREP_ENSEMBLE.yml b/job_spec/PISM_TERRA_PREP_ENSEMBLE.yml index 0ecb9d178..591ffdac9 100644 --- a/job_spec/PISM_TERRA_PREP_ENSEMBLE.yml +++ b/job_spec/PISM_TERRA_PREP_ENSEMBLE.yml @@ -45,9 +45,9 @@ PISM_TERRA_PREP_ENSEMBLE: - Ref::run_template - Ref::uq_config - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --ntasks - Ref::ntasks secrets: diff --git a/job_spec/PISM_TERRA_RUN.yml b/job_spec/PISM_TERRA_RUN.yml index e84bfd723..758d75656 100644 --- a/job_spec/PISM_TERRA_RUN.yml +++ b/job_spec/PISM_TERRA_RUN.yml @@ -39,9 +39,9 @@ PISM_TERRA_RUN: - Ref::pism_config - Ref::run_template - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --ntasks - Ref::ntasks - --execute diff --git a/job_spec/RTC_GAMMA.yml b/job_spec/RTC_GAMMA.yml index e0663cc18..1709d6213 100644 --- a/job_spec/RTC_GAMMA.yml +++ b/job_spec/RTC_GAMMA.yml @@ -106,9 +106,9 @@ RTC_GAMMA: - ++omp-num-threads - '4' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --resolution - Ref::resolution - --dem-name diff --git a/job_spec/SLIMSAR_TDBP.yml b/job_spec/SLIMSAR_TDBP.yml index b09757ed2..01d9eaac6 100644 --- a/job_spec/SLIMSAR_TDBP.yml +++ b/job_spec/SLIMSAR_TDBP.yml @@ -62,9 +62,9 @@ SLIMSAR_TDBP: - --pols - Ref::pols - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix timeout: 126000 # 35 hours compute_environment: SlimSAR vcpu: 1 diff --git a/job_spec/SRG_GSLC.yml b/job_spec/SRG_GSLC.yml index 18182b40e..b0a29a324 100644 --- a/job_spec/SRG_GSLC.yml +++ b/job_spec/SRG_GSLC.yml @@ -51,9 +51,9 @@ SRG_GSLC: - Ref::bounds - --gpu - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - Ref::granules timeout: 10800 compute_environment: SrgGslc diff --git a/job_spec/SRG_TIME_SERIES.yml b/job_spec/SRG_TIME_SERIES.yml index 4b6d64831..1c47e7b39 100644 --- a/job_spec/SRG_TIME_SERIES.yml +++ b/job_spec/SRG_TIME_SERIES.yml @@ -74,9 +74,9 @@ SRG_TIME_SERIES: - --bounds - Ref::bounds - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --use-gslc-prefix timeout: 86400 # 24 hr compute_environment: SrgTimeSeries diff --git a/job_spec/VOLCSARVATORY_MINTPY.yml b/job_spec/VOLCSARVATORY_MINTPY.yml index 340735f26..db736153a 100644 --- a/job_spec/VOLCSARVATORY_MINTPY.yml +++ b/job_spec/VOLCSARVATORY_MINTPY.yml @@ -65,9 +65,9 @@ VOLCSARVATORY_MINTPY: image: ghcr.io/asfhyp3/hyp3-mintpy command: - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --start-date - Ref::start_date - --end-date @@ -80,6 +80,7 @@ VOLCSARVATORY_MINTPY: - Ref::input_bucket - --input-prefix - Ref::input_prefix + # FIXME: - --publish-bucket - Ref::publish_bucket - --publish-prefix diff --git a/job_spec/VOLCSARVATORY_MULTI_BURST.yml b/job_spec/VOLCSARVATORY_MULTI_BURST.yml index 984c041bc..330ecd751 100644 --- a/job_spec/VOLCSARVATORY_MULTI_BURST.yml +++ b/job_spec/VOLCSARVATORY_MULTI_BURST.yml @@ -77,9 +77,9 @@ INSAR_ISCE_MULTI_BURST: - ++process - insar_tops_multi_burst - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --apply-water-mask - Ref::apply_water_mask - --looks @@ -88,6 +88,7 @@ INSAR_ISCE_MULTI_BURST: - Ref::reference - --secondary - Ref::secondary + # FIXME: - --publish-bucket - Ref::publish_bucket - --publish-prefix diff --git a/job_spec/WATER_MAP.yml b/job_spec/WATER_MAP.yml index dfc4c0eb4..64c5f3d5e 100644 --- a/job_spec/WATER_MAP.yml +++ b/job_spec/WATER_MAP.yml @@ -122,9 +122,9 @@ WATER_MAP: - ++omp-num-threads - '4' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --resolution - Ref::resolution - --speckle-filter @@ -159,9 +159,9 @@ WATER_MAP: - ++process - HYDRO30 - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --max-vv-threshold - Ref::max_vv_threshold - --max-vh-threshold @@ -182,9 +182,9 @@ WATER_MAP: - ++process - FD30 - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --estimator - Ref::flood_depth_estimator - --water-level-sigma diff --git a/job_spec/WATER_MAP_EQ.yml b/job_spec/WATER_MAP_EQ.yml index c3d4725b2..f3e71f194 100644 --- a/job_spec/WATER_MAP_EQ.yml +++ b/job_spec/WATER_MAP_EQ.yml @@ -64,9 +64,9 @@ WATER_MAP_EQ: - ++omp-num-threads - '4' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --resolution - Ref::resolution - --speckle-filter @@ -99,9 +99,9 @@ WATER_MAP_EQ: image: 845172464411.dkr.ecr.us-west-2.amazonaws.com/water-map-equal-percent-solution command: - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --max-vv-threshold - Ref::max_vv_threshold - --hand-threshold From a7bf25d2bbe51a2a65787d3c8e26259c4a569cad Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:17:20 -0400 Subject: [PATCH 05/53] pass content bucket to api --- apps/api/api-cf.yml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/api/api-cf.yml.j2 b/apps/api/api-cf.yml.j2 index 946e28531..1fbe7dd71 100644 --- a/apps/api/api-cf.yml.j2 +++ b/apps/api/api-cf.yml.j2 @@ -24,6 +24,9 @@ Parameters: SystemAvailable: Type: String + ContentBucket: + Type: String + {% if security_environment == 'EDC' %} VpcId: Type: String @@ -193,6 +196,7 @@ Resources: DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser DEFAULT_APPLICATION_STATUS: !Ref DefaultApplicationStatus SYSTEM_AVAILABLE: !Ref SystemAvailable + CONTENT_BUCKET: !Ref ContentBucket Code: src/ Handler: hyp3_api.lambda_handler.handler MemorySize: 3008 From 2d396a2313bc25eed77759bec82d905d5511a5d0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:21:40 -0400 Subject: [PATCH 06/53] add user provided publish bucket option for all jobs --- .../hyp3_api/api-spec/job_parameters.yml.j2 | 4 +++ .../src/hyp3_api/api-spec/openapi-spec.yml.j2 | 23 ++++++++++++++++ apps/api/src/hyp3_api/handlers.py | 27 +++++++++++++++++++ apps/get-files/src/get_files.py | 9 ++++++- 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 index 83ea5c963..d99538a24 100644 --- a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 @@ -54,6 +54,10 @@ components: - {{ job_type }} name: $ref: "./openapi-spec.yml#/components/schemas/name" + bucket: + $ref: "./openapi-spec.yml#/components/schemas/bucket" + bucket_prefix: + $ref: "./openapi-spec.yml#/components/schemas/bucket_prefix" job_parameters: $ref: "#/components/schemas/{{ job_type }}Parameters" diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 36898e79d..611cdeb0b 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -354,6 +354,10 @@ components: $ref: "#/components/schemas/execution_started" name: $ref: "#/components/schemas/name" + bucket: + $ref: "#/components/schemas/bucket" + bucket_prefix: + $ref: "#/components/schemas/bucket_prefix" files: $ref: "#/components/schemas/list_of_files" browse_images: @@ -438,6 +442,25 @@ components: maxLength: 100 example: Job Name + bucket: + description: User provided text to define the product bucket, null for default + type: string + nullable: true + minLength: 3 + maxLength: 63 + example: my-s3-bucket + # TODO: Do we use S3 Transfer Acceleration - https://stackoverflow.com/questions/50480924/regex-for-s3-bucket-name + pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})\.){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"" + + bucket_prefix: + description: User provided text to define the bucket prefix to place the product in + type: string + nullable: true + minLength: 1 + maxLength: 100 + example: job-id # TODO: + pattern: "^(?!/)(?!.*//)[A-Za-z0-9._/-]{1,1024}$" + start_token: description: Token used for fetching subsequent results for large queries type: string diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 033a3b38c..60fe61182 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -1,4 +1,5 @@ from http.client import responses +from os import environ from flask import Response, abort, jsonify, request @@ -22,6 +23,30 @@ def problem_format(status: int, message: str) -> Response: return response +def _handle_content_bucket(jobs: list): + content_bucket = environ['CONTENT_BUCKET'] + example_bucket = 'my-s3-bucket' + example_bucket_prefix = 'job-id' + + for i in range(len(jobs)): + job_id = jobs[i]['job_id'] + user_bucket = jobs[i]['bucket'] + # TODO: What kind of input sanitization is needed? + # TODO: Should we check for write permissions here, or in validate_jobs? + if user_bucket and user_bucket not in [content_bucket, example_bucket]: + if prefix := jobs[i]['bucket_prefix']: + # TODO: Do we want this to be madnatory? + if prefix is not example_bucket_prefix: + jobs[i]['bucket_prefix'] = prefix.format(job_id=job_id, name=jobs[i]['name']) # FIXME: + else: + jobs[i]['bucket_prefix'] = job_id + else: + jobs[i]['bucket'] = content_bucket + jobs[i]['bucket_prefix'] = job_id + + return jobs + + def post_jobs(body: dict, user: str) -> dict: print(body) @@ -32,6 +57,8 @@ def post_jobs(body: dict, user: str) -> dict: except (ValidationError, MultiBurstValidationError) as e: abort(problem_format(400, str(e))) + body['jobs'] = _handle_content_bucket(jobs=body['jobs']) + try: body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs'], dry_run=bool(body.get('validate_only'))) except UnexpectedApplicationStatusError as e: diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 83663c31b..8e22fd3c3 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -90,9 +90,16 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict: } +# FIXME: def lambda_handler(event: dict, context: object) -> None: bucket = environ['BUCKET'] + job_id = event['job_id'] + job = dynamo.jobs.get_job(job_id=job_id) - response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=event['job_id']) + if job['bucket'] is not bucket and bucket is not "": + bucket = job['bucket'] + prefix = job['bucket_prefix'] + + response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix) files = organize_files(response['Contents'], bucket) dynamo.jobs.update_job({'job_id': event['job_id'], **files}) From e223823f95669c582ca533e15b772e5adc07000c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:25:02 -0400 Subject: [PATCH 07/53] add bucket and bucket_prefix to batch params --- apps/render_cf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/render_cf.py b/apps/render_cf.py index baeb3b13e..e8afd6eaf 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -130,6 +130,10 @@ def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None = for param in step_params: if param == 'job_id': batch_params['job_id.$'] = '$.job_id' + elif param == 'bucket': + batch_params['bucket.$'] = '$.bucket' + elif param == 'bucket_prefix': + batch_params['bucket_prefix.$'] = '$.bucket_prefix' elif param == map_item: batch_params[f'{map_item}.$'] = "States.Format('{}', $$.Map.Item.Value)" else: From b49e150ffbce458abdd8ce897ed65e74c10629fc Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:29:50 -0400 Subject: [PATCH 08/53] bucket read permissions --- apps/api/src/hyp3_api/handlers.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 60fe61182..270159f77 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -161,6 +161,7 @@ def _get_names_for_user(user: str) -> list[str]: return sorted(list(names)) +# TODO: Are these permissions going to allow all elements of the stack bucket access? def get_bucket_policy(bucket_name: str): account_arn = util.get_account_arn() policy = f''' @@ -174,6 +175,27 @@ def get_bucket_policy(bucket_name: str): "Action": "s3:PutObject", "Resource": "arn:aws:s3:::{bucket_name}/*" }}, + {{ + "Sid": "write tagging permission", + "Effect": "Allow", + "Principal": {{ "AWS": "{account_arn}:root" }}, + "Action": "s3:PutObjectTagging", + "Resource": "arn:aws:s3:::{bucket_name}/*" + }}, + {{ + "Sid": "read permission", + "Effect": "Allow", + "Principal": {{ "AWS": "{account_arn}:root" }}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::{bucket_name}/*" + }}, + {{ + "Sid": "read tagging permission", + "Effect": "Allow", + "Principal": {{ "AWS": "{account_arn}:root" }}, + "Action": "s3:GetObjectTagging", + "Resource": "arn:aws:s3:::{bucket_name}/*" + }}, {{ "Sid": "get bucket location permission", "Effect": "Allow", From 53f77c346f19406fa63761d294673922e179b4c0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 26 Mar 2026 15:39:36 -0400 Subject: [PATCH 09/53] cleaner get_files handler --- apps/get-files/src/get_files.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 8e22fd3c3..9da3c1139 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -90,15 +90,12 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict: } -# FIXME: def lambda_handler(event: dict, context: object) -> None: - bucket = environ['BUCKET'] - job_id = event['job_id'] - job = dynamo.jobs.get_job(job_id=job_id) + job = dynamo.jobs.get_job(job_id=event['job_id']) - if job['bucket'] is not bucket and bucket is not "": - bucket = job['bucket'] - prefix = job['bucket_prefix'] + # `bucket` and `bucket_prefix` are set in the `api.handlers.post_jobs` + bucket = job['bucket'] + prefix = job['bucket_prefix'] response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix) files = organize_files(response['Contents'], bucket) From c8c14ab020940ef268ee24076e7ea58b66b822a3 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 14:49:53 -0400 Subject: [PATCH 10/53] cleaner handler for content bucket --- apps/api/src/hyp3_api/handlers.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 270159f77..26cffe253 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -1,5 +1,6 @@ from http.client import responses from os import environ +from pathlib import Path from flask import Response, abort, jsonify, request @@ -24,22 +25,17 @@ def problem_format(status: int, message: str) -> Response: def _handle_content_bucket(jobs: list): - content_bucket = environ['CONTENT_BUCKET'] - example_bucket = 'my-s3-bucket' - example_bucket_prefix = 'job-id' + content_bucket = environ.get('CONTENT_BUCKET', '') + example_bucket = 'default-s3-bucket' for i in range(len(jobs)): job_id = jobs[i]['job_id'] user_bucket = jobs[i]['bucket'] - # TODO: What kind of input sanitization is needed? - # TODO: Should we check for write permissions here, or in validate_jobs? if user_bucket and user_bucket not in [content_bucket, example_bucket]: if prefix := jobs[i]['bucket_prefix']: - # TODO: Do we want this to be madnatory? - if prefix is not example_bucket_prefix: - jobs[i]['bucket_prefix'] = prefix.format(job_id=job_id, name=jobs[i]['name']) # FIXME: - else: - jobs[i]['bucket_prefix'] = job_id + jobs[i]['bucket_prefix'] = prefix.format(job_id=job_id, name=jobs[i]['name']) + else: + jobs[i]['bucket_prefix'] = job_id else: jobs[i]['bucket'] = content_bucket jobs[i]['bucket_prefix'] = job_id From f8ae4cfce37b3b6e927e7bd9f5ab74a81030b254 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 14:50:24 -0400 Subject: [PATCH 11/53] changed defaults for bucket and prefix --- apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 611cdeb0b..64cfb5cfe 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -448,8 +448,7 @@ components: nullable: true minLength: 3 maxLength: 63 - example: my-s3-bucket - # TODO: Do we use S3 Transfer Acceleration - https://stackoverflow.com/questions/50480924/regex-for-s3-bucket-name + example: "default-s3-bucket" pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})\.){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"" bucket_prefix: @@ -458,7 +457,7 @@ components: nullable: true minLength: 1 maxLength: 100 - example: job-id # TODO: + example: "{job_id}" pattern: "^(?!/)(?!.*//)[A-Za-z0-9._/-]{1,1024}$" start_token: From 5175a22155c0c7ff30921b9cc98f0e4b191d08fb Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 14:59:50 -0400 Subject: [PATCH 12/53] add return type for get_current_account_arn --- apps/api/src/hyp3_api/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index ac7075c40..63423db12 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -49,7 +49,7 @@ def build_next_url(url: str, start_token: str, x_forwarded_host: str | None = No return urlunparse(url_parts) -def get_current_account_arn(): +def get_current_account_arn() -> str: sts = boto3.client('sts') account = sts.get_caller_identity().split(':user')[0] return account From 86dfb411486e226bf2af055a0fc845b12eb190ad Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 15:01:08 -0400 Subject: [PATCH 13/53] add return type for get_bucket_policy --- apps/api/src/hyp3_api/handlers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 26cffe253..af1f636fb 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -1,6 +1,5 @@ from http.client import responses from os import environ -from pathlib import Path from flask import Response, abort, jsonify, request @@ -157,8 +156,7 @@ def _get_names_for_user(user: str) -> list[str]: return sorted(list(names)) -# TODO: Are these permissions going to allow all elements of the stack bucket access? -def get_bucket_policy(bucket_name: str): +def get_bucket_policy(bucket_name: str) -> str: account_arn = util.get_account_arn() policy = f''' {{ From c0d180d0a1dcbb2122ff7559bcfe7f5fae06d998 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 15:01:48 -0400 Subject: [PATCH 14/53] ruff --- apps/api/src/hyp3_api/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 63423db12..5cb146430 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -6,6 +6,7 @@ import boto3 + class TokenDeserializeError(Exception): """Raised when paging results and `start_token` fails to deserialize.""" From a534722efaf70e1b38a90ee6a1eced31ce247ae3 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 15:02:01 -0400 Subject: [PATCH 15/53] add return type for _handle_content_bucket --- apps/api/src/hyp3_api/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index af1f636fb..1874996fb 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -23,7 +23,7 @@ def problem_format(status: int, message: str) -> Response: return response -def _handle_content_bucket(jobs: list): +def _handle_content_bucket(jobs: list) -> dict: content_bucket = environ.get('CONTENT_BUCKET', '') example_bucket = 'default-s3-bucket' From de284315a426a69aaccf8c760249731f728526b7 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 1 Apr 2026 15:04:17 -0400 Subject: [PATCH 16/53] mypy --- apps/api/src/hyp3_api/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 1874996fb..28a421716 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -23,7 +23,7 @@ def problem_format(status: int, message: str) -> Response: return response -def _handle_content_bucket(jobs: list) -> dict: +def _handle_content_bucket(jobs: list) -> list: content_bucket = environ.get('CONTENT_BUCKET', '') example_bucket = 'default-s3-bucket' @@ -157,7 +157,7 @@ def _get_names_for_user(user: str) -> list[str]: def get_bucket_policy(bucket_name: str) -> str: - account_arn = util.get_account_arn() + account_arn = util.get_current_account_arn() policy = f''' {{ "Version": "2012-10-17", From 34be2fd156713eea038adf6f8e4127327b0731a0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 12:54:29 -0400 Subject: [PATCH 17/53] better regex for bucket and bucket prefix --- apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 64cfb5cfe..4bcbdb4de 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -449,7 +449,7 @@ components: minLength: 3 maxLength: 63 example: "default-s3-bucket" - pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})\.){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"" + pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})\.){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" bucket_prefix: description: User provided text to define the bucket prefix to place the product in @@ -458,7 +458,7 @@ components: minLength: 1 maxLength: 100 example: "{job_id}" - pattern: "^(?!/)(?!.*//)[A-Za-z0-9._/-]{1,1024}$" + pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|\{job_id\}|\{name\}){1,1024}$" start_token: description: Token used for fetching subsequent results for large queries From 4c39afbef986feab19d334ff624c78e544608cd0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 12:59:28 -0400 Subject: [PATCH 18/53] fixed regex escape characters --- apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 4bcbdb4de..8056974f7 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -449,7 +449,7 @@ components: minLength: 3 maxLength: 63 example: "default-s3-bucket" - pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})\.){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" + pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2}).){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" bucket_prefix: description: User provided text to define the bucket prefix to place the product in @@ -458,7 +458,7 @@ components: minLength: 1 maxLength: 100 example: "{job_id}" - pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|\{job_id\}|\{name\}){1,1024}$" + pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|{job_id}|{name}){1,1024}$" start_token: description: Token used for fetching subsequent results for large queries From 12689377abea209b2f9823bc418f8e82bc49a58d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 13:23:42 -0400 Subject: [PATCH 19/53] use enumerate rather than range --- apps/api/src/hyp3_api/handlers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 28a421716..40127276b 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -27,17 +27,17 @@ def _handle_content_bucket(jobs: list) -> list: content_bucket = environ.get('CONTENT_BUCKET', '') example_bucket = 'default-s3-bucket' - for i in range(len(jobs)): - job_id = jobs[i]['job_id'] - user_bucket = jobs[i]['bucket'] + for idx, job in enumerate(jobs): + job_id = job['job_id'] + user_bucket = job['bucket'] if user_bucket and user_bucket not in [content_bucket, example_bucket]: - if prefix := jobs[i]['bucket_prefix']: - jobs[i]['bucket_prefix'] = prefix.format(job_id=job_id, name=jobs[i]['name']) + if prefix := job['bucket_prefix']: + jobs[idx]['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) else: - jobs[i]['bucket_prefix'] = job_id + jobs[idx]['bucket_prefix'] = job_id else: - jobs[i]['bucket'] = content_bucket - jobs[i]['bucket_prefix'] = job_id + jobs[idx]['bucket'] = content_bucket + jobs[idx]['bucket_prefix'] = job_id return jobs From 18dd47d715a85c825c026a0e6535ffff4aff97ed Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 15:51:01 -0400 Subject: [PATCH 20/53] move bucket handling to dynamo, handle nulls, and add env var for tests --- apps/api/src/hyp3_api/handlers.py | 28 ++-------------------------- lib/dynamo/dynamo/jobs.py | 20 ++++++++++++++++++++ tests/cfg.env | 1 + 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 40127276b..aa0efefe2 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -1,5 +1,4 @@ from http.client import responses -from os import environ from flask import Response, abort, jsonify, request @@ -23,37 +22,14 @@ def problem_format(status: int, message: str) -> Response: return response -def _handle_content_bucket(jobs: list) -> list: - content_bucket = environ.get('CONTENT_BUCKET', '') - example_bucket = 'default-s3-bucket' - - for idx, job in enumerate(jobs): - job_id = job['job_id'] - user_bucket = job['bucket'] - if user_bucket and user_bucket not in [content_bucket, example_bucket]: - if prefix := job['bucket_prefix']: - jobs[idx]['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) - else: - jobs[idx]['bucket_prefix'] = job_id - else: - jobs[idx]['bucket'] = content_bucket - jobs[idx]['bucket_prefix'] = job_id - - return jobs - - def post_jobs(body: dict, user: str) -> dict: print(body) - try: validate_jobs(body['jobs']) except CmrError as e: abort(problem_format(503, str(e))) except (ValidationError, MultiBurstValidationError) as e: abort(problem_format(400, str(e))) - - body['jobs'] = _handle_content_bucket(jobs=body['jobs']) - try: body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs'], dry_run=bool(body.get('validate_only'))) except UnexpectedApplicationStatusError as e: @@ -158,7 +134,7 @@ def _get_names_for_user(user: str) -> list[str]: def get_bucket_policy(bucket_name: str) -> str: account_arn = util.get_current_account_arn() - policy = f''' + policy = f""" {{ "Version": "2012-10-17", "Statement": [ @@ -199,5 +175,5 @@ def get_bucket_policy(bucket_name: str) -> str: }} ] }} - ''' + """ return policy diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 1b29799fa..c0cae73dc 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -84,6 +84,25 @@ def _raise_for_application_status(application_status: str, user_id: str) -> None raise InvalidApplicationStatusError(user_id, application_status) +def _handle_content_bucket(job: dict) -> dict: + content_bucket = environ['CONTENT_BUCKET'] + example_bucket = 'default-s3-bucket' + + job_id = job['job_id'] + user_bucket = job.get('bucket', '') + if user_bucket and user_bucket not in [content_bucket, example_bucket]: + if prefix := job.get('bucket_prefix', ''): + job['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) + else: + job['bucket_prefix'] = job_id + else: + job['bucket'] = content_bucket + job['bucket_prefix'] = job_id + + return job + + +# TODO: Is anything needed for user provided bucket here? def _prepare_job_for_database( job: dict, user_id: str, @@ -107,6 +126,7 @@ def _prepare_job_for_database( 'priority': priority, **job, } + prepared_job = _handle_content_bucket(prepared_job) if 'job_type' in prepared_job: prepared_job['job_parameters'] = { **DEFAULT_PARAMS_BY_JOB_TYPE[prepared_job['job_type']], diff --git a/tests/cfg.env b/tests/cfg.env index a8e781711..0e2ab9fdd 100644 --- a/tests/cfg.env +++ b/tests/cfg.env @@ -12,3 +12,4 @@ AWS_ACCESS_KEY_ID=testing AWS_SECRET_ACCESS_KEY=testing AWS_SECURITY_TOKEN=testing AWS_SESSION_TOKEN=testing +CONTENT_BUCKET=test-bucket From 800c72c585470bd908188bcae025879b38761cee Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 15:59:09 -0400 Subject: [PATCH 21/53] update tests for bucket and bucket_prefix handling --- tests/test_dynamo/test_jobs.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 5a6d9ca71..2d91e1c1f 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -314,7 +314,13 @@ def test_get_credit_cost_validate_keys(): def test_put_jobs(tables, monkeypatch, approved_user): monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '10') - payload = [{'name': 'name1'}, {'name': 'name1'}, {'name': 'name2'}] + payload = [ + {'name': 'name1'}, + {'name': 'name1'}, + {'name': 'name2'}, + {'name': 'name3', 'bucket': 'test-bucket-1', 'bucket_prefix': 'prefix/{name}/{job_id}'}, + {'name': 'name4', 'bucket': '', 'bucket_prefix': 'example/prefix/'}, + ] with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month: mock_get_current_month.return_value = '2024-02' @@ -334,12 +340,24 @@ def test_put_jobs(tables, monkeypatch, approved_user): 'request_time', 'priority', 'credit_cost', + 'bucket', + 'bucket_prefix', } assert job['request_time'] <= current_utc_time() assert job['user_id'] == approved_user assert job['status_code'] == 'PENDING' assert job['execution_started'] is False assert job['credit_cost'] == 1 + assert job['bucket'] == 'test-bucket' + assert job['bucket_prefix'] == job['job_id'] + + if job['name'] == 'name3': + assert job['bucket'] == 'test-bucket-1' + assert job['bucket_prefix'] == f'prefix/{job["name"]}/{job["job_id"]}' + + if job['name'] == 'name4': + assert job['bucket'] == 'test-bucket' + assert job['bucket_prefix'] == job['job_id'] assert tables.jobs_table.scan()['Items'] == sorted(jobs, key=lambda item: item['job_id']) From 2f6e8d91b02be02cd264147cc269c0c3e6b59100 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:00:17 -0400 Subject: [PATCH 22/53] update test_put_jobs --- tests/test_dynamo/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 2d91e1c1f..38419b67e 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -329,7 +329,7 @@ def test_put_jobs(tables, monkeypatch, approved_user): mock_get_current_month.assert_called_once_with() - assert len(jobs) == 3 + assert len(jobs) == 5 for job in jobs: assert set(job.keys()) == { 'name', From db700098b408ce785137d23fa3e9f0527bc2d3a2 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:09:17 -0400 Subject: [PATCH 23/53] cleaner _handle_content_bucket and fixed test --- lib/dynamo/dynamo/jobs.py | 7 +++---- tests/test_dynamo/test_jobs.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index c0cae73dc..4e782219a 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -90,11 +90,10 @@ def _handle_content_bucket(job: dict) -> dict: job_id = job['job_id'] user_bucket = job.get('bucket', '') + prefix = job.get('bucket_prefix', '') + if user_bucket and user_bucket not in [content_bucket, example_bucket]: - if prefix := job.get('bucket_prefix', ''): - job['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) - else: - job['bucket_prefix'] = job_id + job['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) if prefix else job_id else: job['bucket'] = content_bucket job['bucket_prefix'] = job_id diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 38419b67e..0f2f67e62 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -348,14 +348,14 @@ def test_put_jobs(tables, monkeypatch, approved_user): assert job['status_code'] == 'PENDING' assert job['execution_started'] is False assert job['credit_cost'] == 1 - assert job['bucket'] == 'test-bucket' - assert job['bucket_prefix'] == job['job_id'] if job['name'] == 'name3': assert job['bucket'] == 'test-bucket-1' assert job['bucket_prefix'] == f'prefix/{job["name"]}/{job["job_id"]}' - - if job['name'] == 'name4': + elif job['name'] == 'name4': + assert job['bucket'] == 'test-bucket' + assert job['bucket_prefix'] == job['job_id'] + else: assert job['bucket'] == 'test-bucket' assert job['bucket_prefix'] == job['job_id'] From fa0b2ae026ec48b01c71a1712e35e0cce5915c26 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:28:05 -0400 Subject: [PATCH 24/53] add error for attempting to use custom prefix with default bucket --- apps/api/src/hyp3_api/handlers.py | 3 +++ lib/dynamo/dynamo/exceptions.py | 4 ++++ lib/dynamo/dynamo/jobs.py | 5 +++++ tests/test_dynamo/test_jobs.py | 6 +++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index aa0efefe2..999304e0c 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -5,6 +5,7 @@ import dynamo from dynamo.exceptions import ( AccessCodeError, + CustomPrefixForDefaultBucketError, InsufficientCreditsError, UnexpectedApplicationStatusError, UpdateJobForDifferentUserError, @@ -36,6 +37,8 @@ def post_jobs(body: dict, user: str) -> dict: abort(problem_format(403, str(e))) except InsufficientCreditsError as e: abort(problem_format(400, str(e))) + except CustomPrefixForDefaultBucketError as e: + abort(problem_format(400, str(e))) return body diff --git a/lib/dynamo/dynamo/exceptions.py b/lib/dynamo/dynamo/exceptions.py index 6d32a67c8..bdbe9ca88 100644 --- a/lib/dynamo/dynamo/exceptions.py +++ b/lib/dynamo/dynamo/exceptions.py @@ -25,6 +25,10 @@ class UpdateJobForDifferentUserError(Exception): """Raised when a user attempts to update a different user's job.""" +class CustomPrefixForDefaultBucketError(Exception): + """Raised when a user attempts to use a custom bucket prefix with the default hyp3 content bucket.""" + + class InvalidApplicationStatusError(Exception): """Raised for an invalid user application status.""" diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 4e782219a..9dca9f8aa 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -9,6 +9,7 @@ import dynamo.user from dynamo.exceptions import ( + CustomPrefixForDefaultBucketError, InsufficientCreditsError, InvalidApplicationStatusError, NotStartedApplicationError, @@ -95,6 +96,10 @@ def _handle_content_bucket(job: dict) -> dict: if user_bucket and user_bucket not in [content_bucket, example_bucket]: job['bucket_prefix'] = prefix.format(job_id=job_id, name=job['name']) if prefix else job_id else: + if prefix and prefix != '{job_id}': + raise CustomPrefixForDefaultBucketError( + 'A custom bucket prefix can not be used with the default content bucket.' + ) job['bucket'] = content_bucket job['bucket_prefix'] = job_id diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 0f2f67e62..b44b701aa 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -319,7 +319,7 @@ def test_put_jobs(tables, monkeypatch, approved_user): {'name': 'name1'}, {'name': 'name2'}, {'name': 'name3', 'bucket': 'test-bucket-1', 'bucket_prefix': 'prefix/{name}/{job_id}'}, - {'name': 'name4', 'bucket': '', 'bucket_prefix': 'example/prefix/'}, + {'name': 'name4', 'bucket': 'test-bucket-1', 'bucket_prefix': 'example/prefix/'}, ] with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month: @@ -353,8 +353,8 @@ def test_put_jobs(tables, monkeypatch, approved_user): assert job['bucket'] == 'test-bucket-1' assert job['bucket_prefix'] == f'prefix/{job["name"]}/{job["job_id"]}' elif job['name'] == 'name4': - assert job['bucket'] == 'test-bucket' - assert job['bucket_prefix'] == job['job_id'] + assert job['bucket'] == 'test-bucket-1' + assert job['bucket_prefix'] == 'example/prefix/' else: assert job['bucket'] == 'test-bucket' assert job['bucket_prefix'] == job['job_id'] From 991eea4d752d41b3269e273aab7996128fba9825 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:33:27 -0400 Subject: [PATCH 25/53] add ref to content bucket for api --- apps/main-cf.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/main-cf.yml.j2 b/apps/main-cf.yml.j2 index 277036da1..5a86c5202 100644 --- a/apps/main-cf.yml.j2 +++ b/apps/main-cf.yml.j2 @@ -126,6 +126,7 @@ Resources: DefaultCreditsPerUser: !Ref DefaultCreditsPerUser DefaultApplicationStatus: !Ref DefaultApplicationStatus SystemAvailable: !Ref SystemAvailable + ContentBucket: !Ref ContentBucket {% if security_environment == 'EDC' %} VpcId: !Ref VpcId SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId From 7e03b40487ebca9c988978c1bd10c009980f50d1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:36:20 -0400 Subject: [PATCH 26/53] fixed test_put_jobs credit count --- tests/test_dynamo/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index b44b701aa..01265fe51 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -364,7 +364,7 @@ def test_put_jobs(tables, monkeypatch, approved_user): assert tables.users_table.scan()['Items'] == [ { 'user_id': approved_user, - 'remaining_credits': Decimal(7), + 'remaining_credits': Decimal(5), '_month_of_last_credit_reset': '2024-02', 'application_status': APPLICATION_APPROVED, } From 90d2e9c8da95a3c3267777c0a5e3edf13143b2e8 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 2 Apr 2026 16:50:59 -0400 Subject: [PATCH 27/53] updated changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2dbca2ff..3cbda1ed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.17.0] + +### Added +- `bucket` and `bucket_prefix` top level parameters for all job types. + - The `bucket` parameter allows for overwriting the content bucket that a job's products will be placed into. + - The `bucket_prefix` parameter: + - allows for overwriting the default prefix (the job ID) for the S3 bucket. + - can only be used if also using a custom bucket. + - allows for inserting the job's ID and name by including `{job_id}` or `{name}`. +- `/bucket-policy/` route for retrieving AWS policy to allow HyP3 to write to a custom bucket + ## [10.16.4] ### Added From 639e99b95fa7e3d31ca2655a09ab27af6ffe29d0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 3 Apr 2026 13:48:09 -0400 Subject: [PATCH 28/53] add patch for dynamo.jobs.get_jobs --- tests/test_get_files.py | 128 +++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 55 deletions(-) diff --git a/tests/test_get_files.py b/tests/test_get_files.py index 7818c93e4..d938714de 100644 --- a/tests/test_get_files.py +++ b/tests/test_get_files.py @@ -119,31 +119,37 @@ def test_get_files_zipped_product(s3_stubber: Stubber): stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse_rgb.png', 'rgb_browse') event = {'job_id': 'myJobId'} - with patch('dynamo.jobs.update_job') as mock_update_job: - get_files.lambda_handler(event, None) - mock_update_job.assert_called_once_with( - { + with patch('dynamo.jobs.get_job') as mock_get_job: + with patch('dynamo.jobs.update_job') as mock_update_job: + mock_get_job.return_value = { 'job_id': 'myJobId', - 'expiration_time': '2020-01-01T00:00:00+00:00', - 'files': [ - { - 'url': 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myProduct.zip', - 's3': { - 'bucket': 'myBucket', - 'key': 'myJobId/myProduct.zip', - }, - 'size': 50, - 'filename': 'myProduct.zip', - } - ], - 'browse_images': [ - 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse.png', - 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse_rgb.png', - ], - 'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'], - 'logs': [], + 'bucket': 'myBucket', + 'bucket_prefix': 'myJobId', } - ) + get_files.lambda_handler(event, None) + mock_update_job.assert_called_once_with( + { + 'job_id': 'myJobId', + 'expiration_time': '2020-01-01T00:00:00+00:00', + 'files': [ + { + 'url': 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myProduct.zip', + 's3': { + 'bucket': 'myBucket', + 'key': 'myJobId/myProduct.zip', + }, + 'size': 50, + 'filename': 'myProduct.zip', + } + ], + 'browse_images': [ + 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse.png', + 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse_rgb.png', + ], + 'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'], + 'logs': [], + } + ) def test_get_files_netcdf_product(s3_stubber: Stubber): @@ -168,30 +174,36 @@ def test_get_files_netcdf_product(s3_stubber: Stubber): stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse.png', 'amp_browse') event = {'job_id': 'myJobId'} - with patch('dynamo.jobs.update_job') as mock_update_job: - get_files.lambda_handler(event, None) - mock_update_job.assert_called_once_with( - { + with patch('dynamo.jobs.get_job') as mock_get_job: + with patch('dynamo.jobs.update_job') as mock_update_job: + mock_get_job.return_value = { 'job_id': 'myJobId', - 'expiration_time': '2020-01-01T00:00:00+00:00', - 'files': [ - { - 'url': 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myProduct.nc', - 's3': { - 'bucket': 'myBucket', - 'key': 'myJobId/myProduct.nc', - }, - 'size': 50, - 'filename': 'myProduct.nc', - } - ], - 'browse_images': [ - 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse.png', - ], - 'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'], - 'logs': [], + 'bucket': 'myBucket', + 'bucket_prefix': 'myJobId', } - ) + get_files.lambda_handler(event, None) + mock_update_job.assert_called_once_with( + { + 'job_id': 'myJobId', + 'expiration_time': '2020-01-01T00:00:00+00:00', + 'files': [ + { + 'url': 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myProduct.nc', + 's3': { + 'bucket': 'myBucket', + 'key': 'myJobId/myProduct.nc', + }, + 'size': 50, + 'filename': 'myProduct.nc', + } + ], + 'browse_images': [ + 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse.png', + ], + 'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'], + 'logs': [], + } + ) def test_get_files_failed_job(s3_stubber: Stubber): @@ -206,15 +218,21 @@ def test_get_files_failed_job(s3_stubber: Stubber): stub_expiration(s3_stubber, 'myBucket', 'myJobId/myJobId.log') event = {'job_id': 'myJobId'} - with patch('dynamo.jobs.update_job') as mock_update_job: - get_files.lambda_handler(event, None) - mock_update_job.assert_called_once_with( - { + with patch('dynamo.jobs.get_job') as mock_get_job: + with patch('dynamo.jobs.update_job') as mock_update_job: + mock_get_job.return_value = { 'job_id': 'myJobId', - 'expiration_time': '2020-01-01T00:00:00+00:00', - 'files': [], - 'browse_images': [], - 'thumbnail_images': [], - 'logs': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myJobId.log'], + 'bucket': 'myBucket', + 'bucket_prefix': 'myJobId', } - ) + get_files.lambda_handler(event, None) + mock_update_job.assert_called_once_with( + { + 'job_id': 'myJobId', + 'expiration_time': '2020-01-01T00:00:00+00:00', + 'files': [], + 'browse_images': [], + 'thumbnail_images': [], + 'logs': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myJobId.log'], + } + ) From 425a610716d7619823447b076ba658a943c46cba Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 3 Apr 2026 13:54:29 -0400 Subject: [PATCH 29/53] revert --bucket-prefix to --bucket_prefix --- job_spec/OPERA_DIST_S1.yml | 2 +- job_spec/OPERA_DIST_S1_CONFIRMATION.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/OPERA_DIST_S1.yml b/job_spec/OPERA_DIST_S1.yml index 13a857a52..38a3b81cc 100644 --- a/job_spec/OPERA_DIST_S1.yml +++ b/job_spec/OPERA_DIST_S1.yml @@ -165,7 +165,7 @@ OPERA_DIST_S1: - Ref::post_date_buffer_days - --bucket - Ref::bucket - - --bucket-prefix + - --bucket_prefix - Ref::bucket_prefix - --memory_strategy - Ref::memory_strategy diff --git a/job_spec/OPERA_DIST_S1_CONFIRMATION.yml b/job_spec/OPERA_DIST_S1_CONFIRMATION.yml index 9397bb3ef..f4ca581aa 100644 --- a/job_spec/OPERA_DIST_S1_CONFIRMATION.yml +++ b/job_spec/OPERA_DIST_S1_CONFIRMATION.yml @@ -32,7 +32,7 @@ OPERA_DIST_S1_CONFIRMATION: - Ref::dist_s1_data_bucket - --bucket - Ref::bucket - - --bucket-prefix + - --bucket_prefix - Ref::bucket_prefix timeout: 3600 # 1 hr compute_environment: Default From e0a6f4f140e5c8d14ebaa947aebc1d335a1d32e4 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 3 Apr 2026 13:56:25 -0400 Subject: [PATCH 30/53] remove todo --- lib/dynamo/dynamo/jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 9dca9f8aa..d1e0cc764 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -106,7 +106,6 @@ def _handle_content_bucket(job: dict) -> dict: return job -# TODO: Is anything needed for user provided bucket here? def _prepare_job_for_database( job: dict, user_id: str, From 69ad74c0287a09680d5d0087332d72d3c8af8c18 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 3 Apr 2026 14:00:12 -0400 Subject: [PATCH 31/53] removed comment --- apps/get-files/src/get_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 9da3c1139..7dd85224d 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -93,7 +93,6 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict: def lambda_handler(event: dict, context: object) -> None: job = dynamo.jobs.get_job(job_id=event['job_id']) - # `bucket` and `bucket_prefix` are set in the `api.handlers.post_jobs` bucket = job['bucket'] prefix = job['bucket_prefix'] From f669e7f7b55313a40b9efc30bad9664fbecf77e8 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 13 May 2026 13:38:04 -0800 Subject: [PATCH 32/53] fix boto3 conflicts after rebase --- requirements-apps-api.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt index c0c0eafbf..ee07eda77 100644 --- a/requirements-apps-api.txt +++ b/requirements-apps-api.txt @@ -1,4 +1,4 @@ -boto3==1.42.46 +boto3==1.42.97 flask==3.1.3 Flask-Cors==6.0.2 jsonschema==4.26.0 From 39d6373687cd6e9e2fdd2af2ee23605922589045 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 13 May 2026 14:51:22 -0800 Subject: [PATCH 33/53] Update permissions for HyP3 + S3 interactions outside the hyp3 content bucket --- apps/api/src/hyp3_api/handlers.py | 23 +++++++++++++++-------- apps/compute-cf.yml.j2 | 16 +++++++++++++--- apps/get-files/get-files-cf.yml.j2 | 9 +++++++++ apps/upload-log/upload-log-cf.yml.j2 | 8 ++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 999304e0c..77b72b9a8 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -142,31 +142,38 @@ def get_bucket_policy(bucket_name: str) -> str: "Version": "2012-10-17", "Statement": [ {{ - "Sid": "write permission", + "Sid": "list permission", "Effect": "Allow", "Principal": {{ "AWS": "{account_arn}:root" }}, - "Action": "s3:PutObject", + "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::{bucket_name}/*" }}, {{ - "Sid": "write tagging permission", + "Sid": "read permission", "Effect": "Allow", "Principal": {{ "AWS": "{account_arn}:root" }}, - "Action": "s3:PutObjectTagging", + "Action": "s3:GetObject", "Resource": "arn:aws:s3:::{bucket_name}/*" }}, {{ - "Sid": "read permission", + "Sid": "read tagging permission", "Effect": "Allow", "Principal": {{ "AWS": "{account_arn}:root" }}, - "Action": "s3:GetObject", + "Action": "s3:GetObjectTagging", "Resource": "arn:aws:s3:::{bucket_name}/*" }}, {{ - "Sid": "read tagging permission", + "Sid": "write permission", "Effect": "Allow", "Principal": {{ "AWS": "{account_arn}:root" }}, - "Action": "s3:GetObjectTagging", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::{bucket_name}/*" + }}, + {{ + "Sid": "write tagging permission", + "Effect": "Allow", + "Principal": {{ "AWS": "{account_arn}:root" }}, + "Action": "s3:PutObjectTagging", "Resource": "arn:aws:s3:::{bucket_name}/*" }}, {{ diff --git a/apps/compute-cf.yml.j2 b/apps/compute-cf.yml.j2 index 1b0a40f66..d75e474d9 100644 --- a/apps/compute-cf.yml.j2 +++ b/apps/compute-cf.yml.j2 @@ -173,10 +173,20 @@ Resources: Version: 2012-10-17 Statement: - Effect: Allow - Action: s3:PutObject - Resource: !Sub "arn:aws:s3:::${ContentBucket}/*" + Action: + - s3:ListBucket + - s3:GetObject + - s3:GetObjectTagging + - s3:PutObject + - s3:PutObjectTagging + Resource: "*" + Condition: + StringNotEquals: + s3:ResourceAccount: !Ref "AWS::AccountId" - Effect: Allow - Action: s3:PutObjectTagging + Action: + - s3:PutObject + - s3:PutObjectTagging Resource: !Sub "arn:aws:s3:::${ContentBucket}/*" - Effect: Allow Action: sns:Publish diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index 4ed19c081..dbf225a43 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -75,6 +75,15 @@ Resources: - s3:GetObject - s3:GetObjectTagging Resource: !Sub "arn:aws:s3:::${Bucket}/*" + - Effect: Allow + Action: + - s3:ListBucket + - s3:GetObject + - s3:GetObjectTagging + Resource: "*" + Condition: + StringNotEquals: + s3:ResourceAccount: !Ref "AWS::AccountId" - Effect: Allow Action: dynamodb:UpdateItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*" diff --git a/apps/upload-log/upload-log-cf.yml.j2 b/apps/upload-log/upload-log-cf.yml.j2 index 75992dcfa..3987bdfa9 100644 --- a/apps/upload-log/upload-log-cf.yml.j2 +++ b/apps/upload-log/upload-log-cf.yml.j2 @@ -66,6 +66,14 @@ Resources: - s3:PutObject - s3:PutObjectTagging Resource: !Sub "arn:aws:s3:::${Bucket}/*" + - Effect: Allow + Action: + - s3:PutObject + - s3:PutObjectTagging + Resource: "*" + Condition: + StringNotEquals: + s3:ResourceAccount: !Ref "AWS::AccountId" - Effect: Allow Action: logs:GetLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/batch/job*" From 2b62ad212a77b8d250beb5049a9ea813b5bd0416 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 13 May 2026 16:28:56 -0800 Subject: [PATCH 34/53] add pydantic to fix pip errors --- requirements-apps-api.txt | 1 + requirements-apps-disable-private-dns.txt | 1 + requirements-apps-get-files.txt | 1 + requirements-apps-handle-batch-event.txt | 1 + requirements-apps-search-archive.txt | 1 + requirements-apps-start-execution.txt | 1 + requirements-apps-update-db.txt | 1 + 7 files changed, 7 insertions(+) diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt index ee07eda77..6520a23ac 100644 --- a/requirements-apps-api.txt +++ b/requirements-apps-api.txt @@ -1,4 +1,5 @@ boto3==1.42.97 +pydantic==2.12.5 flask==3.1.3 Flask-Cors==6.0.2 jsonschema==4.26.0 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index 722724ed8..0bc011ff9 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1,2 @@ boto3==1.42.97 +pydantic==2.12.5 diff --git a/requirements-apps-get-files.txt b/requirements-apps-get-files.txt index 9a8bf92d8..d369b4029 100644 --- a/requirements-apps-get-files.txt +++ b/requirements-apps-get-files.txt @@ -1,2 +1,3 @@ boto3==1.42.97 +pydantic==2.12.5 ./lib/dynamo/ diff --git a/requirements-apps-handle-batch-event.txt b/requirements-apps-handle-batch-event.txt index ea4b060f3..a8b7730fb 100644 --- a/requirements-apps-handle-batch-event.txt +++ b/requirements-apps-handle-batch-event.txt @@ -1 +1,2 @@ ./lib/dynamo/ +pydantic==2.12.5 diff --git a/requirements-apps-search-archive.txt b/requirements-apps-search-archive.txt index 9ebb713dd..c653321b0 100644 --- a/requirements-apps-search-archive.txt +++ b/requirements-apps-search-archive.txt @@ -2,3 +2,4 @@ ./lib/lambda_logging/ asf_enumeration==0.4.0 asf_search==12.0.7 +pydantic==2.12.5 diff --git a/requirements-apps-start-execution.txt b/requirements-apps-start-execution.txt index ec3ca50ea..12742d9e0 100644 --- a/requirements-apps-start-execution.txt +++ b/requirements-apps-start-execution.txt @@ -1,3 +1,4 @@ boto3==1.42.97 +pydantic==2.12.5 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-update-db.txt b/requirements-apps-update-db.txt index ea4b060f3..a8b7730fb 100644 --- a/requirements-apps-update-db.txt +++ b/requirements-apps-update-db.txt @@ -1 +1,2 @@ ./lib/dynamo/ +pydantic==2.12.5 From 9fc2fa5de50f4edd2a985a8c7331ea9e8b0608cf Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 13 May 2026 19:34:04 -0800 Subject: [PATCH 35/53] try removing openapi decorator --- apps/api/src/hyp3_api/routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index f4cd9c228..7f62ded9d 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -196,6 +196,5 @@ def user_get() -> Response: @app.route('/bucket-policy/', methods=['GET']) -@openapi def bucket_policy_get(bucket_name: str) -> Response: return jsonify(handlers.get_bucket_policy(bucket_name)) From 2eafd487817c0aab29a29229bfceb99b33855c99 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 13 May 2026 20:08:20 -0800 Subject: [PATCH 36/53] fix get_caller_identity --- apps/api/src/hyp3_api/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 5cb146430..eeefe0b57 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -52,5 +52,4 @@ def build_next_url(url: str, start_token: str, x_forwarded_host: str | None = No def get_current_account_arn() -> str: sts = boto3.client('sts') - account = sts.get_caller_identity().split(':user')[0] - return account + return sts.get_caller_identity()['Account'] From bfd2d9385ec085618bc7d7d06bc1e134fda19051 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 12:26:37 -0800 Subject: [PATCH 37/53] Return dictionaries from bucket-policy handler instead of a big string --- apps/api/src/hyp3_api/handlers.py | 68 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 77b72b9a8..fc3409ffe 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -135,55 +135,65 @@ def _get_names_for_user(user: str) -> list[str]: return sorted(list(names)) -def get_bucket_policy(bucket_name: str) -> str: +def get_bucket_policy(bucket_name: str) -> dict: account_arn = util.get_current_account_arn() - policy = f""" - {{ + policy = { "Version": "2012-10-17", "Statement": [ - {{ + { "Sid": "list permission", "Effect": "Allow", - "Principal": {{ "AWS": "{account_arn}:root" }}, + "Principal": { + "AWS": f"{account_arn}:root" + }, "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::{bucket_name}/*" - }}, - {{ + "Resource": f"arn:aws:s3:::{bucket_name}/*" + }, + { "Sid": "read permission", "Effect": "Allow", - "Principal": {{ "AWS": "{account_arn}:root" }}, + "Principal": { + "AWS": f"{account_arn}:root" + }, "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::{bucket_name}/*" - }}, - {{ + "Resource": f"arn:aws:s3:::{bucket_name}/*" + }, + { "Sid": "read tagging permission", "Effect": "Allow", - "Principal": {{ "AWS": "{account_arn}:root" }}, + "Principal": { + "AWS": f"{account_arn}:root" + }, "Action": "s3:GetObjectTagging", - "Resource": "arn:aws:s3:::{bucket_name}/*" - }}, - {{ + "Resource": f"arn:aws:s3:::{bucket_name}/*" + }, + { "Sid": "write permission", "Effect": "Allow", - "Principal": {{ "AWS": "{account_arn}:root" }}, + "Principal": { + "AWS": f"{account_arn}:root" + }, "Action": "s3:PutObject", - "Resource": "arn:aws:s3:::{bucket_name}/*" - }}, - {{ + "Resource": f"arn:aws:s3:::{bucket_name}/*" + }, + { "Sid": "write tagging permission", "Effect": "Allow", - "Principal": {{ "AWS": "{account_arn}:root" }}, + "Principal": { + "AWS": f"{account_arn}:root" + }, "Action": "s3:PutObjectTagging", - "Resource": "arn:aws:s3:::{bucket_name}/*" - }}, - {{ + "Resource": f"arn:aws:s3:::{bucket_name}/*" + }, + { "Sid": "get bucket location permission", "Effect": "Allow", - "Principal": {{ "AWS": "arn:aws:iam::{account_arn}:root" }}, + "Principal": { + "AWS": f"arn:aws:iam::{account_arn}:root" + }, "Action": "s3:GetBucketLocation", - "Resource": "arn:aws:s3:::{bucket_name}" - }} + "Resource": f"arn:aws:s3:::{bucket_name}" + } ] - }} - """ + } return policy From c93f210247bf888885f6816ea7d4881c3cb46ba0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 13:15:41 -0800 Subject: [PATCH 38/53] Simplify bucket-policy response --- apps/api/src/hyp3_api/handlers.py | 60 +++++++++---------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index fc3409ffe..a928dd4c2 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -141,58 +141,30 @@ def get_bucket_policy(bucket_name: str) -> dict: "Version": "2012-10-17", "Statement": [ { - "Sid": "list permission", + "Sid": "HyP3 bucket-level publish permissions", "Effect": "Allow", "Principal": { - "AWS": f"{account_arn}:root" + "AWS": f"{account_arn}:root", }, - "Action": "s3:ListBucket", - "Resource": f"arn:aws:s3:::{bucket_name}/*" + "Action": [ + "s3:ListBucket", + "s3:getBucketLocation", + ], + "Resource": f"arn:aws:s3:::{bucket_name}", }, { - "Sid": "read permission", + "Sid": "HyP3 object-level publish permissions", "Effect": "Allow", "Principal": { - "AWS": f"{account_arn}:root" + "AWS": f"{account_arn}:root", }, - "Action": "s3:GetObject", - "Resource": f"arn:aws:s3:::{bucket_name}/*" - }, - { - "Sid": "read tagging permission", - "Effect": "Allow", - "Principal": { - "AWS": f"{account_arn}:root" - }, - "Action": "s3:GetObjectTagging", - "Resource": f"arn:aws:s3:::{bucket_name}/*" - }, - { - "Sid": "write permission", - "Effect": "Allow", - "Principal": { - "AWS": f"{account_arn}:root" - }, - "Action": "s3:PutObject", - "Resource": f"arn:aws:s3:::{bucket_name}/*" - }, - { - "Sid": "write tagging permission", - "Effect": "Allow", - "Principal": { - "AWS": f"{account_arn}:root" - }, - "Action": "s3:PutObjectTagging", - "Resource": f"arn:aws:s3:::{bucket_name}/*" - }, - { - "Sid": "get bucket location permission", - "Effect": "Allow", - "Principal": { - "AWS": f"arn:aws:iam::{account_arn}:root" - }, - "Action": "s3:GetBucketLocation", - "Resource": f"arn:aws:s3:::{bucket_name}" + "Action": [ + "s3:GetObject", + "s3:GetObjectTagging", + "s3:PutObject", + "s3:PutObjectTagging", + ], + "Resource": f"arn:aws:s3:::{bucket_name}/*", } ] } From 0042abd168655fecc80fe1c78c732fcb9225bc4c Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 13:18:39 -0800 Subject: [PATCH 39/53] Add dynamodb:GetItem permissions to get-files --- apps/get-files/get-files-cf.yml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index dbf225a43..b3e19e52a 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -85,7 +85,9 @@ Resources: StringNotEquals: s3:ResourceAccount: !Ref "AWS::AccountId" - Effect: Allow - Action: dynamodb:UpdateItem + Action: + - dynamodb:GetItem + - dynamodb:UpdateItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*" Lambda: From 485d413220401a4fa57d49cf193b8e438f5d6421 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 13:22:21 -0800 Subject: [PATCH 40/53] put the logs in the user bucket as well --- apps/upload-log/src/upload_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index 6da5025ec..8320432d5 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -76,4 +76,4 @@ def lambda_handler(event: dict, context: object) -> None: assert 'Error' in result log_content = get_log_content_from_failed_attempts(json.loads(result['Cause'])) - write_log_to_s3(environ['BUCKET'], event['prefix'], log_content) + write_log_to_s3(event['bucket'], event['bucket_prefix'], log_content) From 0e7f7b4759ef2c389c06a3cc5864a8a904ff78b1 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 13:32:36 -0800 Subject: [PATCH 41/53] actually, use event context instead of dynamo.jobs.get_job for get-files --- apps/get-files/get-files-cf.yml.j2 | 4 +--- apps/get-files/src/get_files.py | 9 ++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index b3e19e52a..dbf225a43 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -85,9 +85,7 @@ Resources: StringNotEquals: s3:ResourceAccount: !Ref "AWS::AccountId" - Effect: Allow - Action: - - dynamodb:GetItem - - dynamodb:UpdateItem + Action: dynamodb:UpdateItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*" Lambda: diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 7dd85224d..463692f42 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -91,11 +91,6 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict: def lambda_handler(event: dict, context: object) -> None: - job = dynamo.jobs.get_job(job_id=event['job_id']) - - bucket = job['bucket'] - prefix = job['bucket_prefix'] - - response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix) - files = organize_files(response['Contents'], bucket) + response = S3_CLIENT.list_objects_v2(Bucket=event['bucket'], Prefix=event['bucket_prefix']) + files = organize_files(response['Contents'], event['bucket']) dynamo.jobs.update_job({'job_id': event['job_id'], **files}) From f0aa3efc0c8fc57a0ecd66f128f0b33da9afc364 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 13:55:45 -0800 Subject: [PATCH 42/53] add bucket and bucket_prefix as parameters to get-files and upload-logs in step function --- apps/step-function.json.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 7cd14c89a..38355941e 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -133,6 +133,8 @@ "Resource": "${UploadLogLambdaArn}", "Parameters": { "prefix.$": "$.job_id", + "bucket.$": "$.bucket", + "bucket_prefix.$": "$.bucket_prefix", "log_group": "/aws/batch/job", "processing_results.$": "$.results.processing_results" }, @@ -160,7 +162,9 @@ "Type": "Task", "Resource": "${GetFilesLambdaArn}", "Parameters": { - "job_id.$": "$.job_id" + "job_id.$": "$.job_id", + "bucket.$": "$.bucket", + "bucket_prefix.$": "$.bucket_prefix" }, "Retry": [ { From d6d412b02e718387307fe2eedeba7f0ec64e0ef3 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 14 May 2026 14:21:23 -0800 Subject: [PATCH 43/53] tweak bucket-policy --- apps/api/src/hyp3_api/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index a928dd4c2..07b80dc15 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -144,7 +144,7 @@ def get_bucket_policy(bucket_name: str) -> dict: "Sid": "HyP3 bucket-level publish permissions", "Effect": "Allow", "Principal": { - "AWS": f"{account_arn}:root", + "AWS": f"{account_arn}", }, "Action": [ "s3:ListBucket", @@ -156,7 +156,7 @@ def get_bucket_policy(bucket_name: str) -> dict: "Sid": "HyP3 object-level publish permissions", "Effect": "Allow", "Principal": { - "AWS": f"{account_arn}:root", + "AWS": f"{account_arn}", }, "Action": [ "s3:GetObject", From 84ad8527066deb1b71a433b46db2bd062bf07aab Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 15 May 2026 13:39:05 -0800 Subject: [PATCH 44/53] Fix get-files: exiration time optional and fix distribution url --- apps/get-files/get-files-cf.yml.j2 | 5 ++++- apps/get-files/src/get_files.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index dbf225a43..0aa32533a 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -68,7 +68,9 @@ Resources: - logs:PutLogEvents Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" - Effect: Allow - Action: s3:ListBucket + Action: + - s3:ListBucket + - s3:getBucketLocation Resource: !Sub "arn:aws:s3:::${Bucket}" - Effect: Allow Action: @@ -78,6 +80,7 @@ Resources: - Effect: Allow Action: - s3:ListBucket + - s3:getBucketLocation - s3:GetObject - s3:GetObjectTagging Resource: "*" diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 463692f42..9e460377b 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -13,16 +13,21 @@ def get_download_url(bucket: str, key: str) -> str: - if distribution_url := os.getenv('DISTRIBUTION_URL'): - download_url = urllib.parse.urljoin(distribution_url, key) - else: - region = environ['AWS_REGION'] - download_url = f'https://{bucket}.s3.{region}.amazonaws.com/{key}' - return download_url + if (bucket == environ['BUCKET']) and (distribution_url := os.getenv('DISTRIBUTION_URL')): + return urllib.parse.urljoin(distribution_url, key) + + region = S3_CLIENT.head_bucket(Bucket=bucket)['BucketRegion'] + return f'https://{bucket}.s3.{region}.amazonaws.com/{key}' def get_expiration_time(bucket: str, key: str) -> str: s3_object = S3_CLIENT.get_object(Bucket=bucket, Key=key) + expiration = s3_object.get('Expiration') + if expiration is None: + # HyP3's 100th birthday; 100 years since first non-ASF job + # https://hyp3-api.asf.alaska.edu/jobs/969ab836-aa95-4613-8673-2a0415949afa + return '2120-10-21T00:00:00+00:00' + expiration_string = s3_object['Expiration'].split('"')[1] expiration_datetime = datetime.strptime(expiration_string, '%a, %d %b %Y %H:%M:%S %Z') return expiration_datetime.isoformat(timespec='seconds') + '+00:00' From d931c74b75ddb2f5eb91ea8c2dd8aed5b4f59646 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 15 May 2026 13:41:35 -0800 Subject: [PATCH 45/53] ruff ruff --- apps/api/src/hyp3_api/handlers.py | 44 +++++++++++++++---------------- apps/get-files/src/get_files.py | 2 +- apps/upload-log/src/upload_log.py | 1 - 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 07b80dc15..d07ac2a1a 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -138,34 +138,34 @@ def _get_names_for_user(user: str) -> list[str]: def get_bucket_policy(bucket_name: str) -> dict: account_arn = util.get_current_account_arn() policy = { - "Version": "2012-10-17", - "Statement": [ + 'Version': '2012-10-17', + 'Statement': [ { - "Sid": "HyP3 bucket-level publish permissions", - "Effect": "Allow", - "Principal": { - "AWS": f"{account_arn}", + 'Sid': 'HyP3 bucket-level publish permissions', + 'Effect': 'Allow', + 'Principal': { + 'AWS': f'{account_arn}', }, - "Action": [ - "s3:ListBucket", - "s3:getBucketLocation", + 'Action': [ + 's3:ListBucket', + 's3:getBucketLocation', ], - "Resource": f"arn:aws:s3:::{bucket_name}", + 'Resource': f'arn:aws:s3:::{bucket_name}', }, { - "Sid": "HyP3 object-level publish permissions", - "Effect": "Allow", - "Principal": { - "AWS": f"{account_arn}", + 'Sid': 'HyP3 object-level publish permissions', + 'Effect': 'Allow', + 'Principal': { + 'AWS': f'{account_arn}', }, - "Action": [ - "s3:GetObject", - "s3:GetObjectTagging", - "s3:PutObject", - "s3:PutObjectTagging", + 'Action': [ + 's3:GetObject', + 's3:GetObjectTagging', + 's3:PutObject', + 's3:PutObjectTagging', ], - "Resource": f"arn:aws:s3:::{bucket_name}/*", - } - ] + 'Resource': f'arn:aws:s3:::{bucket_name}/*', + }, + ], } return policy diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 9e460377b..d2edfd292 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -14,7 +14,7 @@ def get_download_url(bucket: str, key: str) -> str: if (bucket == environ['BUCKET']) and (distribution_url := os.getenv('DISTRIBUTION_URL')): - return urllib.parse.urljoin(distribution_url, key) + return urllib.parse.urljoin(distribution_url, key) region = S3_CLIENT.head_bucket(Bucket=bucket)['BucketRegion'] return f'https://{bucket}.s3.{region}.amazonaws.com/{key}' diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index 8320432d5..d058e8f4d 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -1,5 +1,4 @@ import json -from os import environ import boto3 from botocore.config import Config From ac1756d28b9a6be2e4e163788dc7b086ac813878 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 15 May 2026 14:28:49 -0800 Subject: [PATCH 46/53] update get-files and upload-logs tests --- tests/test_get_files.py | 50 ++++++++++++++++++++++++++++++++-------- tests/test_upload_log.py | 9 +++++--- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/tests/test_get_files.py b/tests/test_get_files.py index d938714de..5a3792ae9 100644 --- a/tests/test_get_files.py +++ b/tests/test_get_files.py @@ -21,11 +21,19 @@ def s3_stubber(): stubber.assert_no_pending_responses() -def test_get_download_url(monkeypatch): +def stub_bucket_head(s3_stubber: Stubber, bucket, region): + params = {'Bucket': bucket} + s3_response = {'BucketRegion': region} + s3_stubber.add_response(method='head_bucket', expected_params=params, service_response=s3_response) + + +def test_get_download_url(monkeypatch, s3_stubber: Stubber): assert os.getenv('DISTRIBUTION_URL') is None + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') assert get_files.get_download_url('myBucket', 'myKey') == 'https://myBucket.s3.myRegion.amazonaws.com/myKey' monkeypatch.setenv('DISTRIBUTION_URL', '') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') assert get_files.get_download_url('myBucket', 'myKey') == 'https://myBucket.s3.myRegion.amazonaws.com/myKey' monkeypatch.setenv('DISTRIBUTION_URL', 'https://foo.com/') @@ -34,21 +42,34 @@ def test_get_download_url(monkeypatch): monkeypatch.setenv('DISTRIBUTION_URL', 'https://foo.com') assert get_files.get_download_url('myBucket', 'myKey') == 'https://foo.com/myKey' + stub_bucket_head(s3_stubber, 'userBucket', 'userRegion') + assert get_files.get_download_url('userBucket', 'myKey') == 'https://userBucket.s3.userRegion.amazonaws.com/myKey' + + monkeypatch.delenv('DISTRIBUTION_URL') + stub_bucket_head(s3_stubber, 'userBucket', 'userRegion') + assert get_files.get_download_url('userBucket', 'myKey') == 'https://userBucket.s3.userRegion.amazonaws.com/myKey' + -def stub_expiration(s3_stubber: Stubber, bucket, key): +def stub_expiration(s3_stubber: Stubber, bucket, key, expires=True): params = {'Bucket': bucket, 'Key': key} - s3_response = { - 'Expiration': 'expiry-date="Wed, 01 Jan 2020 00:00:00 UTC", ' - 'rule-id="MDQxMzRmZTgtNDFlMi00Y2UwLWIyZjEtMTEzYTllNDNjYjJk"' - } + s3_response = {} + if expires: + s3_response['Expiration'] = ( + 'expiry-date="Wed, 01 Jan 2020 00:00:00 UTC", rule-id="MDQxMzRmZTgtNDFlMi00Y2UwLWIyZjEtMTEzYTllNDNjYjJk"' + ) + s3_stubber.add_response(method='get_object', expected_params=params, service_response=s3_response) def test_get_expiration(s3_stubber: Stubber): - stub_expiration(s3_stubber, 'myBucket', 'myKey') + stub_expiration(s3_stubber, 'myBucket', 'myKey', expires=True) response = get_files.get_expiration_time('myBucket', 'myKey') assert response == '2020-01-01T00:00:00+00:00' + stub_expiration(s3_stubber, 'userBucket', 'userKey', expires=False) + response = get_files.get_expiration_time('userBucket', 'userKey') + assert response == '2120-10-21T00:00:00+00:00' + def stub_get_object_tagging(s3_stubber: Stubber, bucket, key, file_type): params = {'Bucket': bucket, 'Key': key} @@ -111,14 +132,19 @@ def test_get_files_zipped_product(s3_stubber: Stubber): }, ] stub_list_files(s3_stubber, 'myJobId', 'myBucket', files) + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.zip', 'product') stub_expiration(s3_stubber, 'myBucket', 'myJobId/myProduct.zip') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.tif', 'product') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myThumbnail.png', 'amp_thumbnail') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse.png', 'amp_browse') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse_rgb.png', 'rgb_browse') - event = {'job_id': 'myJobId'} + event = {'job_id': 'myJobId', 'bucket': 'myBucket', 'bucket_prefix': 'myJobId'} with patch('dynamo.jobs.get_job') as mock_get_job: with patch('dynamo.jobs.update_job') as mock_update_job: mock_get_job.return_value = { @@ -168,12 +194,15 @@ def test_get_files_netcdf_product(s3_stubber: Stubber): }, ] stub_list_files(s3_stubber, 'myJobId', 'myBucket', files) + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.nc', 'product') stub_expiration(s3_stubber, 'myBucket', 'myJobId/myProduct.nc') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myThumbnail.png', 'amp_thumbnail') + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse.png', 'amp_browse') - event = {'job_id': 'myJobId'} + event = {'job_id': 'myJobId', 'bucket': 'myBucket', 'bucket_prefix': 'myJobId'} with patch('dynamo.jobs.get_job') as mock_get_job: with patch('dynamo.jobs.update_job') as mock_update_job: mock_get_job.return_value = { @@ -214,10 +243,11 @@ def test_get_files_failed_job(s3_stubber: Stubber): }, ] stub_list_files(s3_stubber, 'myJobId', 'myBucket', files) + stub_bucket_head(s3_stubber, 'myBucket', 'myRegion') stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myJobId.log', 'log') stub_expiration(s3_stubber, 'myBucket', 'myJobId/myJobId.log') - event = {'job_id': 'myJobId'} + event = {'job_id': 'myJobId', 'bucket': 'myBucket', 'bucket_prefix': 'myJobId'} with patch('dynamo.jobs.get_job') as mock_get_job: with patch('dynamo.jobs.update_job') as mock_update_job: mock_get_job.return_value = { diff --git a/tests/test_upload_log.py b/tests/test_upload_log.py index 8d1d14a9c..ccbc5c3e8 100644 --- a/tests/test_upload_log.py +++ b/tests/test_upload_log.py @@ -113,7 +113,8 @@ def test_upload_log_to_s3(s3_stubber): def test_lambda_handler(mock_get_log_content: MagicMock, mock_write_log_to_s3: MagicMock): mock_get_log_content.return_value = 'here is some test log content' event = { - 'prefix': 'test-prefix', + 'bucket': 'test-bucket', + 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', 'processing_results': {'step_0': {'Container': {'LogStreamName': 'test-log-stream'}}}, } @@ -128,7 +129,8 @@ def test_lambda_handler(mock_get_log_content: MagicMock, mock_write_log_to_s3: M @patch.dict(os.environ, {'BUCKET': 'test-bucket'}, clear=True) def test_lambda_handler_no_log_stream(mock_write_log_to_s3: MagicMock): event = { - 'prefix': 'test-prefix', + 'bucket': 'test-bucket', + 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', 'processing_results': { 'step_0': { @@ -152,7 +154,8 @@ def mock_get_log_events(**kwargs): ) event = { - 'prefix': 'test-prefix', + 'bucket': 'test-bucket', + 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', 'processing_results': { 'step_0': { From 4648b8bd937916fe85357b641fccb169c3aaa951 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 15 May 2026 15:04:21 -0800 Subject: [PATCH 47/53] remove redundant parameter from upload-logs step in step function --- apps/step-function.json.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 38355941e..df5eacad8 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -132,7 +132,6 @@ "Type": "Task", "Resource": "${UploadLogLambdaArn}", "Parameters": { - "prefix.$": "$.job_id", "bucket.$": "$.bucket", "bucket_prefix.$": "$.bucket_prefix", "log_group": "/aws/batch/job", From 084e04bdc9a169a2c716e49dc9d8831ee8855862 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 15 May 2026 15:16:03 -0800 Subject: [PATCH 48/53] remove prefix == job_id assumption in upload logs --- apps/step-function.json.j2 | 1 + apps/upload-log/src/upload_log.py | 6 +++--- tests/test_upload_log.py | 17 ++++++++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index df5eacad8..25b8b9aaf 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -132,6 +132,7 @@ "Type": "Task", "Resource": "${UploadLogLambdaArn}", "Parameters": { + "job_id.$": "$.job_id", "bucket.$": "$.bucket", "bucket_prefix.$": "$.bucket_prefix", "log_group": "/aws/batch/job", diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index d058e8f4d..f0149b5c7 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -43,8 +43,8 @@ def get_log_content_from_failed_attempts(cause: dict) -> str: return content -def write_log_to_s3(bucket: str, prefix: str, content: str) -> None: - key = f'{prefix}/{prefix}.log' +def write_log_to_s3(job_id: str, bucket: str, prefix: str, content: str) -> None: + key = f'{prefix}/{job_id}.log' S3.put_object(Bucket=bucket, Key=key, Body=content, ContentType='text/plain') tag_set = { 'TagSet': [ @@ -75,4 +75,4 @@ def lambda_handler(event: dict, context: object) -> None: assert 'Error' in result log_content = get_log_content_from_failed_attempts(json.loads(result['Cause'])) - write_log_to_s3(event['bucket'], event['bucket_prefix'], log_content) + write_log_to_s3(event['job_id'], event['bucket'], event['bucket_prefix'], log_content) diff --git a/tests/test_upload_log.py b/tests/test_upload_log.py index ccbc5c3e8..b5a966fb4 100644 --- a/tests/test_upload_log.py +++ b/tests/test_upload_log.py @@ -104,7 +104,7 @@ def test_upload_log_to_s3(s3_stubber): s3_stubber.add_response(method='put_object', expected_params=expected_params, service_response={}) s3_stubber.add_response(method='put_object_tagging', expected_params=tag_params, service_response={}) - upload_log.write_log_to_s3('myBucket', 'myJobId', 'myContent') + upload_log.write_log_to_s3('myJobId', 'myBucket', 'myJobId', 'myContent') @patch('upload_log.write_log_to_s3') @@ -113,6 +113,7 @@ def test_upload_log_to_s3(s3_stubber): def test_lambda_handler(mock_get_log_content: MagicMock, mock_write_log_to_s3: MagicMock): mock_get_log_content.return_value = 'here is some test log content' event = { + 'job_id': 'job-id', 'bucket': 'test-bucket', 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', @@ -122,13 +123,16 @@ def test_lambda_handler(mock_get_log_content: MagicMock, mock_write_log_to_s3: M upload_log.lambda_handler(event, None) mock_get_log_content.assert_called_once_with('test-log-group', 'test-log-stream') - mock_write_log_to_s3.assert_called_once_with('test-bucket', 'test-prefix', mock_get_log_content.return_value) + mock_write_log_to_s3.assert_called_once_with( + 'job-id', 'test-bucket', 'test-prefix', mock_get_log_content.return_value + ) @patch('upload_log.write_log_to_s3') @patch.dict(os.environ, {'BUCKET': 'test-bucket'}, clear=True) def test_lambda_handler_no_log_stream(mock_write_log_to_s3: MagicMock): event = { + 'job_id': 'job-id', 'bucket': 'test-bucket', 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', @@ -142,7 +146,7 @@ def test_lambda_handler_no_log_stream(mock_write_log_to_s3: MagicMock): upload_log.lambda_handler(event, None) - mock_write_log_to_s3.assert_called_once_with('test-bucket', 'test-prefix', 'foo reason') + mock_write_log_to_s3.assert_called_once_with('job-id', 'test-bucket', 'test-prefix', 'foo reason') def test_lambda_handler_log_stream_does_not_exist(): @@ -154,6 +158,7 @@ def mock_get_log_events(**kwargs): ) event = { + 'job_id': 'job-id', 'bucket': 'test-bucket', 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', @@ -179,7 +184,7 @@ def mock_get_log_events(**kwargs): upload_log.lambda_handler(event, None) mock_write_log_to_s3.assert_called_once_with( - 'test-bucket', 'test-prefix', 'error message 1\nerror message 2\nerror message 3' + 'job-id', 'test-bucket', 'test-prefix', 'error message 1\nerror message 2\nerror message 3' ) @@ -192,7 +197,9 @@ def mock_get_log_events(**kwargs): ) event = { - 'prefix': 'test-prefix', + 'job_id': 'job-id', + 'bucket': 'test-bucket', + 'bucket_prefix': 'test-prefix', 'log_group': 'test-log-group', 'processing_results': { 'step_0': { From 73624e8c085016e32060863fc9d8a6979233fabe Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 18 May 2026 14:09:04 -0800 Subject: [PATCH 49/53] Add OpenAPI spec for bucket-policy --- .../src/hyp3_api/api-spec/openapi-spec.yml.j2 | 35 +++++++++++++++++-- apps/api/src/hyp3_api/routes.py | 1 + 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 8056974f7..5a4278c7e 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -95,7 +95,6 @@ paths: in: query schema: $ref: "#/components/schemas/start_token" - responses: "200": description: 200 response @@ -136,7 +135,6 @@ paths: schema: $ref: "#/components/schemas/job_id" required: true - responses: "200": description: 200 response @@ -172,6 +170,23 @@ paths: schema: $ref: "#/components/schemas/user" + /bucket-policy/{bucket_name}: + get: + description: Get a bucket policy for `bucket_name` which allows HyP3 to write to that bucket. + parameters: + - name: bucket_name + in: path + schema: + $ref: "#/components/schemas/bucket" + required: true + responses: + "200": + description: 200 response + content: + application/json: + schema: + $ref: "#/components/schemas/bucket_policy" + components: schemas: @@ -460,6 +475,22 @@ components: example: "{job_id}" pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|{job_id}|{name}){1,1024}$" + bucket_policy: + description: An AWS S3 Bucket Policy for a bucket that will allow HyP3 to write to the bucket. + type: object + required: + - Version + - Statement + additionalProperties: false + properties: + Version: + type: string + Statement: + type: array + items: + type: object + + start_token: description: Token used for fetching subsequent results for large queries type: string diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 7f62ded9d..f4cd9c228 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -196,5 +196,6 @@ def user_get() -> Response: @app.route('/bucket-policy/', methods=['GET']) +@openapi def bucket_policy_get(bucket_name: str) -> Response: return jsonify(handlers.get_bucket_policy(bucket_name)) From 521887aa7f1927df6a0912a6f4bc21cdd1584493 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 18 May 2026 14:28:57 -0800 Subject: [PATCH 50/53] Add OpenAPI spec for bucket-policy GET endpoint --- .../src/hyp3_api/api-spec/openapi-spec.yml.j2 | 35 +++++++++++++++++-- lib/dynamo/dynamo/jobs.py | 2 +- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 5a4278c7e..b90ec04e1 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -463,7 +463,7 @@ components: nullable: true minLength: 3 maxLength: 63 - example: "default-s3-bucket" + example: "my-example-bucket" pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2}).){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" bucket_prefix: @@ -476,7 +476,7 @@ components: pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|{job_id}|{name}){1,1024}$" bucket_policy: - description: An AWS S3 Bucket Policy for a bucket that will allow HyP3 to write to the bucket. + description: An AWS S3 Bucket Policy that, when applied, will allow HyP3 to write to the bucket. type: object required: - Version @@ -485,10 +485,41 @@ components: properties: Version: type: string + format: date + example: "2012-10-17" Statement: type: array items: type: object + example: >- + [ + { + "Sid": "HyP3 bucket-level publish permissions", + "Effect": "Allow", + "Principal": { + "AWS": "123456789012" + }, + "Action": [ + "s3:ListBucket", + "s3:getBucketLocation" + ], + "Resource": "arn:aws:s3:::my-example-bucket" + }, + { + "Sid": "HyP3 object-level publish permissions", + "Effect": "Allow", + "Principal": { + "AWS": "123456789012" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectTagging", + "s3:PutObject", + "s3:PutObjectTagging" + ], + "Resource": "arn:aws:s3:::my-example-bucket/*" + } + ] start_token: diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index d1e0cc764..5a44b0040 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -87,7 +87,7 @@ def _raise_for_application_status(application_status: str, user_id: str) -> None def _handle_content_bucket(job: dict) -> dict: content_bucket = environ['CONTENT_BUCKET'] - example_bucket = 'default-s3-bucket' + example_bucket = 'my-example-bucket' job_id = job['job_id'] user_bucket = job.get('bucket', '') From f1bf40bfd3b0cead8138de045a25a464e1fd5025 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 18 May 2026 15:16:17 -0800 Subject: [PATCH 51/53] Add note to handlers to also update openapi spec --- apps/api/src/hyp3_api/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index d07ac2a1a..66d52fe57 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -137,6 +137,7 @@ def _get_names_for_user(user: str) -> list[str]: def get_bucket_policy(bucket_name: str) -> dict: account_arn = util.get_current_account_arn() + # NOTE: Reflect any edits here in api-spec/openapi-spec.yml.j2 as well policy = { 'Version': '2012-10-17', 'Statement': [ From 6b015b58f9e203f61a43360f272bb71cc9a0baa6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 18 May 2026 16:46:51 -0800 Subject: [PATCH 52/53] Fix bucket + bucket_prefix in some job specs --- job_spec/ARIA_S1_COSEIS.yml | 4 ++-- job_spec/ARIA_S1_GUNW.yml | 4 ++-- job_spec/INSAR_ISCE.yml | 4 ++-- job_spec/INSAR_ISCE_BURST_EC2.yml | 4 ++-- job_spec/INSAR_ISCE_BURST_SPOT.yml | 6 +++--- job_spec/ITS_LIVE_AUTORIFT.yml | 4 ++-- job_spec/SRG_TIME_SERIES.yml | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/job_spec/ARIA_S1_COSEIS.yml b/job_spec/ARIA_S1_COSEIS.yml index 5ba366bc0..48e27d0ee 100644 --- a/job_spec/ARIA_S1_COSEIS.yml +++ b/job_spec/ARIA_S1_COSEIS.yml @@ -142,9 +142,9 @@ ARIA_S1_COSEIS: - ++process - calcDelaysGUNW - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --weather-model - Ref::weather_model timeout: 21600 diff --git a/job_spec/ARIA_S1_GUNW.yml b/job_spec/ARIA_S1_GUNW.yml index 16a3f3f41..43225044f 100644 --- a/job_spec/ARIA_S1_GUNW.yml +++ b/job_spec/ARIA_S1_GUNW.yml @@ -37,9 +37,9 @@ ARIA_S1_GUNW: - ++omp-num-threads - '4' # 8 vCPUs per 16 GB RAM for the C instance family; 4 for M; 2 for R - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --reference-date - Ref::reference_date - --secondary-date diff --git a/job_spec/INSAR_ISCE.yml b/job_spec/INSAR_ISCE.yml index 6d678a455..93daeb372 100644 --- a/job_spec/INSAR_ISCE.yml +++ b/job_spec/INSAR_ISCE.yml @@ -140,9 +140,9 @@ INSAR_ISCE: - ++process - calcDelaysGUNW - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --weather-model - Ref::weather_model timeout: 10800 diff --git a/job_spec/INSAR_ISCE_BURST_EC2.yml b/job_spec/INSAR_ISCE_BURST_EC2.yml index e5c37d027..ac947735e 100644 --- a/job_spec/INSAR_ISCE_BURST_EC2.yml +++ b/job_spec/INSAR_ISCE_BURST_EC2.yml @@ -50,9 +50,9 @@ INSAR_ISCE_BURST_EC2: - ++omp-num-threads - '1' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --apply-water-mask - Ref::apply_water_mask - --looks diff --git a/job_spec/INSAR_ISCE_BURST_SPOT.yml b/job_spec/INSAR_ISCE_BURST_SPOT.yml index 9969e5925..9dca56f76 100644 --- a/job_spec/INSAR_ISCE_BURST_SPOT.yml +++ b/job_spec/INSAR_ISCE_BURST_SPOT.yml @@ -19,7 +19,7 @@ INSAR_ISCE_BURST: example: S1_136231_IW2_20200604T022312_VV_7C85-BURST apply_water_mask: api_schema: - description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping. + description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping. default: false type: boolean looks: @@ -50,9 +50,9 @@ INSAR_ISCE_BURST: - ++omp-num-threads - '1' - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --apply-water-mask - Ref::apply_water_mask - --looks diff --git a/job_spec/ITS_LIVE_AUTORIFT.yml b/job_spec/ITS_LIVE_AUTORIFT.yml index 0c8e48cde..234191b40 100644 --- a/job_spec/ITS_LIVE_AUTORIFT.yml +++ b/job_spec/ITS_LIVE_AUTORIFT.yml @@ -167,9 +167,9 @@ AUTORIFT: command: - Ref::granules - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --parameter-file - Ref::parameter_file - --publish-bucket diff --git a/job_spec/SRG_TIME_SERIES.yml b/job_spec/SRG_TIME_SERIES.yml index 1c47e7b39..254ad282f 100644 --- a/job_spec/SRG_TIME_SERIES.yml +++ b/job_spec/SRG_TIME_SERIES.yml @@ -53,9 +53,9 @@ SRG_TIME_SERIES: - Ref::bounds - --gpu - --bucket - - '!Ref Bucket' + - Ref::bucket - --bucket-prefix - - Ref::job_id + - Ref::bucket_prefix - --use-gslc-prefix - Ref::granule timeout: 10800 From 9201162e4bb8fabbeb35b865a52ddc3002a2d024 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 20 May 2026 15:37:28 -0800 Subject: [PATCH 53/53] Drop publish_bucket FIXME reminder comments Co-authored-by: Joseph H Kennedy --- job_spec/ITS_LIVE_CROP.yml | 2 -- job_spec/ITS_LIVE_CROP_BULK.yml | 2 -- job_spec/ITS_LIVE_META_BULK.yml | 1 - job_spec/VOLCSARVATORY_MINTPY.yml | 1 - job_spec/VOLCSARVATORY_MULTI_BURST.yml | 1 - 5 files changed, 7 deletions(-) diff --git a/job_spec/ITS_LIVE_CROP.yml b/job_spec/ITS_LIVE_CROP.yml index 584840480..0ca9eaf2a 100644 --- a/job_spec/ITS_LIVE_CROP.yml +++ b/job_spec/ITS_LIVE_CROP.yml @@ -44,7 +44,6 @@ ITS_LIVE_CROP: - Ref::bucket - --bucket-prefix - Ref::bucket_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 10800 @@ -65,7 +64,6 @@ ITS_LIVE_CROP: - Ref::bucket - --bucket-prefix - Ref::bucket_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket - --stac-items-endpoint diff --git a/job_spec/ITS_LIVE_CROP_BULK.yml b/job_spec/ITS_LIVE_CROP_BULK.yml index eee3423ea..48aaeae57 100644 --- a/job_spec/ITS_LIVE_CROP_BULK.yml +++ b/job_spec/ITS_LIVE_CROP_BULK.yml @@ -50,7 +50,6 @@ ITS_LIVE_CROP_BULK: - Ref::bucket - --bucket-prefix - Ref::bucket_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 @@ -75,7 +74,6 @@ ITS_LIVE_CROP_BULK: - Ref::bucket - --bucket-prefix - Ref::bucket_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 diff --git a/job_spec/ITS_LIVE_META_BULK.yml b/job_spec/ITS_LIVE_META_BULK.yml index e584bbdc6..6b5108c94 100644 --- a/job_spec/ITS_LIVE_META_BULK.yml +++ b/job_spec/ITS_LIVE_META_BULK.yml @@ -50,7 +50,6 @@ ITS_LIVE_META_BULK: - Ref::bucket - --bucket-prefix - Ref::bucket_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket timeout: 21600 diff --git a/job_spec/VOLCSARVATORY_MINTPY.yml b/job_spec/VOLCSARVATORY_MINTPY.yml index db736153a..e0d52909b 100644 --- a/job_spec/VOLCSARVATORY_MINTPY.yml +++ b/job_spec/VOLCSARVATORY_MINTPY.yml @@ -80,7 +80,6 @@ VOLCSARVATORY_MINTPY: - Ref::input_bucket - --input-prefix - Ref::input_prefix - # FIXME: - --publish-bucket - Ref::publish_bucket - --publish-prefix diff --git a/job_spec/VOLCSARVATORY_MULTI_BURST.yml b/job_spec/VOLCSARVATORY_MULTI_BURST.yml index 330ecd751..c1d0c6e82 100644 --- a/job_spec/VOLCSARVATORY_MULTI_BURST.yml +++ b/job_spec/VOLCSARVATORY_MULTI_BURST.yml @@ -88,7 +88,6 @@ INSAR_ISCE_MULTI_BURST: - Ref::reference - --secondary - Ref::secondary - # FIXME: - --publish-bucket - Ref::publish_bucket - --publish-prefix