Release v10.17.0 -- Write products to user buckets

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [10.17.0]
+
+### Added
+- `bucket` and `bucket_prefix` top level parameters for all job types.
+  - The `bucket` parameter allows for overwriting the content bucket that a job's products will be placed into.
+  - The `bucket_prefix` parameter:
+    - allows for overwriting the default prefix (the job ID) for the S3 bucket.
+    - can only be used if also using a custom bucket.
+    - allows for inserting the job's ID and name by including `{job_id}` or `{name}` in the prefix string. 
+- `/bucket-policy/<bucket_name>` route for retrieving an AWS policy that will allow HyP3 to write to a the `<bucket_name>` S3 Bucket.
+
 ## [10.16.5]
 
 ### Added

apps/api/api-cf.yml.j2

@@ -24,6 +24,9 @@ Parameters:
   SystemAvailable:
     Type: String
 
+  ContentBucket:
+    Type: String
+
   {% if security_environment == 'EDC' %}
   VpcId:
     Type: String
@@ -193,6 +196,7 @@ Resources:
           DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser
           DEFAULT_APPLICATION_STATUS: !Ref DefaultApplicationStatus
           SYSTEM_AVAILABLE: !Ref SystemAvailable
+          CONTENT_BUCKET: !Ref ContentBucket
       Code: src/
       Handler: hyp3_api.lambda_handler.handler
       MemorySize: 3008

apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2

@@ -54,6 +54,10 @@ components:
             - {{ job_type }}
         name:
           $ref: "./openapi-spec.yml#/components/schemas/name"
+        bucket:
+          $ref: "./openapi-spec.yml#/components/schemas/bucket"
+        bucket_prefix: 
+          $ref: "./openapi-spec.yml#/components/schemas/bucket_prefix"
         job_parameters:
           $ref: "#/components/schemas/{{ job_type }}Parameters"
 

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -95,7 +95,6 @@ paths:
           in: query
           schema:
             $ref: "#/components/schemas/start_token"
-
       responses:
         "200":
           description: 200 response
@@ -136,7 +135,6 @@ paths:
           schema:
             $ref: "#/components/schemas/job_id"
           required: true
-
       responses:
         "200":
           description: 200 response
@@ -172,6 +170,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/user"
 
+  /bucket-policy/{bucket_name}:
+    get:
+      description: Get a bucket policy for `bucket_name` which allows HyP3 to write to that bucket.
+      parameters:
+        - name: bucket_name
+          in: path
+          schema:
+            $ref: "#/components/schemas/bucket"
+          required: true
+      responses:
+        "200":
+          description: 200 response
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/bucket_policy"
+
 components:
   schemas:
 
@@ -354,6 +369,10 @@ components:
           $ref: "#/components/schemas/execution_started"
         name:
           $ref: "#/components/schemas/name"
+        bucket:
+          $ref: "#/components/schemas/bucket"
+        bucket_prefix:
+          $ref: "#/components/schemas/bucket_prefix"
         files:
           $ref: "#/components/schemas/list_of_files"
         browse_images:
@@ -438,6 +457,71 @@ components:
       maxLength: 100
       example: Job Name
 
+    bucket:
+      description: User provided text to define the product bucket, null for default
+      type: string
+      nullable: true
+      minLength: 3
+      maxLength: 63
+      example: "my-example-bucket"
+      pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2}).){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"
+
+    bucket_prefix:
+      description: User provided text to define the bucket prefix to place the product in
+      type: string
+      nullable: true
+      minLength: 1
+      maxLength: 100
+      example: "{job_id}"
+      pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|{job_id}|{name}){1,1024}$"
+
+    bucket_policy:
+      description: An AWS S3 Bucket Policy that, when applied, will allow HyP3 to write to the bucket.
+      type: object
+      required:
+        - Version
+        - Statement
+      additionalProperties: false
+      properties:
+        Version:
+          type: string
+          format: date
+          example: "2012-10-17"
+        Statement:
+          type: array
+          items:
+            type: object
+          example: >-
+            [
+              {
+                "Sid": "HyP3 bucket-level publish permissions",
+                "Effect": "Allow",
+                "Principal": {
+                  "AWS": "123456789012"
+                },
+                  "Action": [
+                  "s3:ListBucket",
+                  "s3:getBucketLocation"
+                ],
+                "Resource": "arn:aws:s3:::my-example-bucket"
+              },
+              {
+                "Sid": "HyP3 object-level publish permissions",
+                "Effect": "Allow",
+                "Principal": {
+                  "AWS": "123456789012"
+                },
+                "Action": [
+                  "s3:GetObject",
+                  "s3:GetObjectTagging",
+                  "s3:PutObject",
+                  "s3:PutObjectTagging"
+                ],
+                "Resource": "arn:aws:s3:::my-example-bucket/*"
+              }
+            ]
+
+
     start_token:
       description: Token used for fetching subsequent results for large queries
       type: string

apps/api/src/hyp3_api/handlers.py

@@ -5,6 +5,7 @@
 import dynamo
 from dynamo.exceptions import (
     AccessCodeError,
+    CustomPrefixForDefaultBucketError,
     InsufficientCreditsError,
     UnexpectedApplicationStatusError,
     UpdateJobForDifferentUserError,
@@ -24,20 +25,20 @@ def problem_format(status: int, message: str) -> Response:
 
 def post_jobs(body: dict, user: str) -> dict:
     print(body)
-
     try:
         validate_jobs(body['jobs'])
     except CmrError as e:
         abort(problem_format(503, str(e)))
     except (ValidationError, MultiBurstValidationError) as e:
         abort(problem_format(400, str(e)))
-
     try:
         body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs'], dry_run=bool(body.get('validate_only')))
     except UnexpectedApplicationStatusError as e:
         abort(problem_format(403, str(e)))
     except InsufficientCreditsError as e:
         abort(problem_format(400, str(e)))
+    except CustomPrefixForDefaultBucketError as e:
+        abort(problem_format(400, str(e)))
     return body
 
 
@@ -132,3 +133,40 @@ def _get_names_for_user(user: str) -> list[str]:
         jobs.extend(new_jobs)
     names = {job['name'] for job in jobs if 'name' in job}
     return sorted(list(names))
+
+
+def get_bucket_policy(bucket_name: str) -> dict:
+    account_arn = util.get_current_account_arn()
+    # NOTE: Reflect any edits here in api-spec/openapi-spec.yml.j2 as well
+    policy = {
+        'Version': '2012-10-17',
+        'Statement': [
+            {
+                'Sid': 'HyP3 bucket-level publish permissions',
+                'Effect': 'Allow',
+                'Principal': {
+                    'AWS': f'{account_arn}',
+                },
+                'Action': [
+                    's3:ListBucket',
+                    's3:getBucketLocation',
+                ],
+                'Resource': f'arn:aws:s3:::{bucket_name}',
+            },
+            {
+                'Sid': 'HyP3 object-level publish permissions',
+                'Effect': 'Allow',
+                'Principal': {
+                    'AWS': f'{account_arn}',
+                },
+                'Action': [
+                    's3:GetObject',
+                    's3:GetObjectTagging',
+                    's3:PutObject',
+                    's3:PutObjectTagging',
+                ],
+                'Resource': f'arn:aws:s3:::{bucket_name}/*',
+            },
+        ],
+    }
+    return policy

apps/api/src/hyp3_api/routes.py

@@ -193,3 +193,9 @@ def user_patch() -> Response:
 @openapi
 def user_get() -> Response:
     return jsonify(handlers.get_user(g.user))
+
+
+@app.route('/bucket-policy/<bucket_name>', methods=['GET'])
+@openapi
+def bucket_policy_get(bucket_name: str) -> Response:
+    return jsonify(handlers.get_bucket_policy(bucket_name))

apps/api/src/hyp3_api/util.py

@@ -4,6 +4,8 @@
 from typing import Any
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
+import boto3
+
 
 class TokenDeserializeError(Exception):
     """Raised when paging results and `start_token` fails to deserialize."""
@@ -46,3 +48,8 @@ def build_next_url(url: str, start_token: str, x_forwarded_host: str | None = No
     url_parts[4] = urlencode(query)
 
     return urlunparse(url_parts)
+
+
+def get_current_account_arn() -> str:
+    sts = boto3.client('sts')
+    return sts.get_caller_identity()['Account']

apps/compute-cf.yml.j2

@@ -173,10 +173,20 @@ Resources:
         Version: 2012-10-17
         Statement:
           - Effect: Allow
-            Action: s3:PutObject
-            Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
+            Action:
+              - s3:ListBucket
+              - s3:GetObject
+              - s3:GetObjectTagging
+              - s3:PutObject
+              - s3:PutObjectTagging
+            Resource: "*"
+            Condition:
+              StringNotEquals:
+                s3:ResourceAccount: !Ref "AWS::AccountId"
           - Effect: Allow
-            Action: s3:PutObjectTagging
+            Action:
+              - s3:PutObject
+              - s3:PutObjectTagging
             Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
           - Effect: Allow
             Action: sns:Publish

apps/get-files/get-files-cf.yml.j2

@@ -68,13 +68,25 @@ Resources:
               - logs:PutLogEvents
             Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*"
           - Effect: Allow
-            Action: s3:ListBucket
+            Action:
+             - s3:ListBucket
+             - s3:getBucketLocation
             Resource: !Sub "arn:aws:s3:::${Bucket}"
           - Effect: Allow
             Action:
               - s3:GetObject
               - s3:GetObjectTagging
             Resource: !Sub "arn:aws:s3:::${Bucket}/*"
+          - Effect: Allow
+            Action:
+              - s3:ListBucket
+              - s3:getBucketLocation
+              - s3:GetObject
+              - s3:GetObjectTagging
+            Resource: "*"
+            Condition:
+              StringNotEquals:
+                s3:ResourceAccount: !Ref "AWS::AccountId"
           - Effect: Allow
             Action: dynamodb:UpdateItem
             Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*"

apps/get-files/src/get_files.py

@@ -13,16 +13,21 @@
 
 
 def get_download_url(bucket: str, key: str) -> str:
-    if distribution_url := os.getenv('DISTRIBUTION_URL'):
-        download_url = urllib.parse.urljoin(distribution_url, key)
-    else:
-        region = environ['AWS_REGION']
-        download_url = f'https://{bucket}.s3.{region}.amazonaws.com/{key}'
-    return download_url
+    if (bucket == environ['BUCKET']) and (distribution_url := os.getenv('DISTRIBUTION_URL')):
+        return urllib.parse.urljoin(distribution_url, key)
+
+    region = S3_CLIENT.head_bucket(Bucket=bucket)['BucketRegion']
+    return f'https://{bucket}.s3.{region}.amazonaws.com/{key}'
 
 
 def get_expiration_time(bucket: str, key: str) -> str:
     s3_object = S3_CLIENT.get_object(Bucket=bucket, Key=key)
+    expiration = s3_object.get('Expiration')
+    if expiration is None:
+        # HyP3's 100th birthday; 100 years since first non-ASF job
+        # https://hyp3-api.asf.alaska.edu/jobs/969ab836-aa95-4613-8673-2a0415949afa
+        return '2120-10-21T00:00:00+00:00'
+
     expiration_string = s3_object['Expiration'].split('"')[1]
     expiration_datetime = datetime.strptime(expiration_string, '%a, %d %b %Y %H:%M:%S %Z')
     return expiration_datetime.isoformat(timespec='seconds') + '+00:00'
@@ -91,8 +96,6 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict:
 
 
 def lambda_handler(event: dict, context: object) -> None:
-    bucket = environ['BUCKET']
-
-    response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=event['job_id'])
-    files = organize_files(response['Contents'], bucket)
+    response = S3_CLIENT.list_objects_v2(Bucket=event['bucket'], Prefix=event['bucket_prefix'])
+    files = organize_files(response['Contents'], event['bucket'])
     dynamo.jobs.update_job({'job_id': event['job_id'], **files})

apps/main-cf.yml.j2

@@ -126,6 +126,7 @@ Resources:
         DefaultCreditsPerUser: !Ref DefaultCreditsPerUser
         DefaultApplicationStatus: !Ref DefaultApplicationStatus
         SystemAvailable: !Ref SystemAvailable
+        ContentBucket: !Ref ContentBucket
         {% if security_environment == 'EDC' %}
         VpcId: !Ref VpcId
         SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId

apps/render_cf.py

@@ -130,6 +130,10 @@ def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None =
     for param in step_params:
         if param == 'job_id':
             batch_params['job_id.$'] = '$.job_id'
+        elif param == 'bucket':
+            batch_params['bucket.$'] = '$.bucket'
+        elif param == 'bucket_prefix':
+            batch_params['bucket_prefix.$'] = '$.bucket_prefix'
         elif param == map_item:
             batch_params[f'{map_item}.$'] = "States.Format('{}', $$.Map.Item.Value)"
         else: