From ef8b783b6bdff7f46d737137f84c5f6375bb6a87 Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Mon, 17 Nov 2025 20:08:07 -0800 Subject: [PATCH 1/6] v1 of auto destroy feature --- src/deployml/cli/cli.py | 142 +++++++++++++ .../templates/gcp/cloud_run/main.tf.j2 | 47 +++++ .../templates/gcp/cloud_run/variables.tf.j2 | 13 ++ .../teardown/cloud/gcp/cloud_function/main.py | 191 ++++++++++++++++++ .../cloud/gcp/cloud_function/requirements.txt | 3 + .../modules/teardown/cloud/gcp/main.tf | 140 +++++++++++++ .../modules/teardown/cloud/gcp/outputs.tf | 20 ++ .../modules/teardown/cloud/gcp/variables.tf | 38 ++++ src/deployml/utils/teardown.py | 50 +++++ 9 files changed, 644 insertions(+) create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/requirements.txt create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/main.tf create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/variables.tf create mode 100644 src/deployml/utils/teardown.py diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index cf81b61..0c65421 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -39,10 +39,16 @@ run_infracost_analysis, format_cost_for_confirmation, ) +from deployml.utils.teardown import ( + save_deployment_metadata, + load_deployment_metadata, + calculate_cron_from_timestamp, +) import re import time import json +from datetime import datetime, timedelta cli = typer.Typer() @@ -480,6 +486,19 @@ def deploy( name_material = f"{workspace_name}:{project_id}".encode("utf-8") name_hash = hashlib.sha1(name_material).hexdigest()[:6] + # Handle teardown configuration + teardown_config = config.get("teardown", {}) + teardown_enabled = teardown_config.get("enabled", False) + teardown_cron_schedule = "" + teardown_scheduled_timestamp = 0 + + if teardown_enabled: + duration_hours = teardown_config.get("duration_hours", 24) + deployed_at = datetime.utcnow() + teardown_at = deployed_at + timedelta(hours=duration_hours) + teardown_scheduled_timestamp = int(teardown_at.timestamp()) + teardown_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + # Render templates if deployment_type == "cloud_vm": main_tf = main_template.render( @@ -493,6 +512,9 @@ def deploy( zone=config["provider"].get("zone", f"{region}-a"), stack_name=workspace_name, name_hash=name_hash, + teardown_config=teardown_config if teardown_enabled else None, + teardown_cron_schedule=teardown_cron_schedule, + teardown_scheduled_timestamp=teardown_scheduled_timestamp, ) else: main_tf = main_template.render( @@ -504,6 +526,9 @@ def deploy( project_id=project_id, stack_name=workspace_name, name_hash=name_hash, + teardown_config=teardown_config if teardown_enabled else None, + teardown_cron_schedule=teardown_cron_schedule, + teardown_scheduled_timestamp=teardown_scheduled_timestamp, ) variables_tf = var_template.render( stack=stack, @@ -652,6 +677,26 @@ def deploy( ) if result_code == 0 or result_code == 1: typer.echo("✅ Deployment complete!") + + # Handle auto-teardown metadata + if teardown_enabled: + duration_hours = teardown_config.get("duration_hours", 24) + deployed_at = datetime.utcnow() + teardown_at = deployed_at + timedelta(hours=duration_hours) + + metadata = { + "deployed_at": deployed_at.isoformat(), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours, + "scheduler_job_name": f"deployml-teardown-{workspace_name}" + } + save_deployment_metadata(DEPLOYML_DIR, metadata) + + typer.echo(f"\n⏰ Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f" (in {duration_hours} hours)") + typer.echo(f" To cancel: deployml teardown cancel --config-path {config_path}") + # Show all Terraform outputs in a user-friendly way output_proc = subprocess.run( ["terraform", "output", "-json"], @@ -836,6 +881,103 @@ def status(): typer.echo("Checking deployment status...") +@cli.command() +def teardown( + action: str = typer.Argument(..., help="Action: cancel, status, or schedule"), + config_path: Path = typer.Option( + ..., "--config-path", "-c", help="Path to YAML config file" + ), +): + """ + Manage auto-teardown: cancel scheduled teardown, check status, or schedule new teardown. + """ + if not config_path.exists(): + typer.echo(f"❌ Config file not found: {config_path}") + raise typer.Exit(code=1) + + config = yaml.safe_load(config_path.read_text()) + workspace_name = config.get("name") or "default" + DEPLOYML_DIR = Path.cwd() / ".deployml" / workspace_name + + if action == "cancel": + cancel_teardown(config, DEPLOYML_DIR, workspace_name) + elif action == "status": + show_teardown_status(DEPLOYML_DIR, workspace_name) + elif action == "schedule": + schedule_teardown(config, DEPLOYML_DIR, workspace_name) + else: + typer.echo(f"❌ Unknown action: {action}. Use: cancel, status, or schedule") + raise typer.Exit(code=1) + + +def cancel_teardown(config: dict, deployml_dir: Path, workspace_name: str): + """Cancel scheduled teardown.""" + project_id = config["provider"]["project_id"] + region = config["provider"]["region"] + + # Delete Cloud Scheduler job + scheduler_job_name = f"deployml-teardown-{workspace_name}" + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "delete", scheduler_job_name, + "--project", project_id, "--region", region, "--quiet"], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + typer.echo("✅ Scheduled teardown cancelled") + # Update metadata + metadata = load_deployment_metadata(deployml_dir) + if metadata: + metadata["teardown_enabled"] = False + save_deployment_metadata(deployml_dir, metadata) + else: + typer.echo(f"⚠️ Could not cancel teardown: {result.stderr}") + typer.echo(" The scheduler job may not exist or may have already been deleted.") + + +def show_teardown_status(deployml_dir: Path, workspace_name: str): + """Show teardown status.""" + metadata = load_deployment_metadata(deployml_dir) + + if not metadata or not metadata.get("teardown_enabled"): + typer.echo("ℹ️ Auto-teardown is not enabled for this workspace") + return + + deployed_at = datetime.fromisoformat(metadata["deployed_at"]) + teardown_at = datetime.fromisoformat(metadata["teardown_scheduled_at"]) + now = datetime.utcnow() + + if now >= teardown_at: + typer.echo("⚠️ Teardown time has passed (may have already executed)") + else: + time_remaining = teardown_at - now + hours = int(time_remaining.total_seconds() // 3600) + minutes = int((time_remaining.total_seconds() % 3600) // 60) + typer.echo(f"⏰ Scheduled teardown: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f" Time remaining: {hours}h {minutes}m") + typer.echo(f" To cancel: deployml teardown cancel --config-path ") + + +def schedule_teardown(config: dict, deployml_dir: Path, workspace_name: str): + """Schedule a new teardown.""" + duration_hours = typer.prompt("Hours until teardown", default=24, type=int) + deployed_at = datetime.utcnow() + teardown_at = deployed_at + timedelta(hours=duration_hours) + + metadata = { + "deployed_at": deployed_at.isoformat(), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours + } + save_deployment_metadata(deployml_dir, metadata) + + typer.echo(f"✅ Teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo("⚠️ Note: This only updates local metadata. To actually schedule teardown,") + typer.echo(" you need to redeploy with teardown.enabled: true in your config.") + + @cli.command() def init( provider: str = typer.Option( diff --git a/src/deployml/templates/gcp/cloud_run/main.tf.j2 b/src/deployml/templates/gcp/cloud_run/main.tf.j2 index 0d3b7b0..758d1a2 100644 --- a/src/deployml/templates/gcp/cloud_run/main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/main.tf.j2 @@ -40,6 +40,26 @@ provider "{% if cloud == 'gcp' %}google{% elif cloud == 'aws' %}aws{% else %}{{c {% endfor %} {% endfor %} +{# Auto-teardown module #} +{% if teardown_config and teardown_config.get("enabled", false) %} +module "teardown" { + source = "./modules/teardown/cloud/gcp" + + project_id = var.project_id + region = var.region + workspace_name = "{{ stack_name }}" + + {% set duration_hours = teardown_config.get("duration_hours", 24) %} + {% set deployed_at = deployment_timestamp %} + {% set teardown_time = deployed_at + (duration_hours * 3600) %} + {% set teardown_cron = calculate_cron_from_timestamp(teardown_time) %} + schedule = "{{ teardown_cron }}" + + time_zone = "{{ teardown_config.get('time_zone', 'UTC') }}" + terraform_state_bucket = var.terraform_state_bucket +} +{% endif %} + # Create Cloud SQL PostgreSQL instance if needed {% if flags.needs_postgres %} module "cloud_sql_postgres" { @@ -307,3 +327,30 @@ output "metrics_connection_string_cloud_sql" { } {% endif %} +{# Auto-teardown module #} +{% if teardown_config and teardown_config.get("enabled", false) %} +module "teardown" { + source = "./modules/teardown/cloud/gcp" + + project_id = var.project_id + region = var.region + workspace_name = "{{ stack_name }}" + + schedule = "{{ teardown_cron_schedule }}" + + time_zone = "{{ teardown_config.get('time_zone', 'UTC') }}" + terraform_state_bucket = var.terraform_state_bucket + terraform_files_bucket = var.terraform_files_bucket +} + +output "teardown_scheduler_job" { + value = module.teardown.scheduler_job_name + description = "Name of the Cloud Scheduler job for auto-teardown" +} + +output "teardown_scheduled_at" { + value = "{{ teardown_scheduled_timestamp }}" + description = "Unix timestamp when teardown is scheduled" +} +{% endif %} + diff --git a/src/deployml/templates/gcp/cloud_run/variables.tf.j2 b/src/deployml/templates/gcp/cloud_run/variables.tf.j2 index 7667caf..df4be7b 100644 --- a/src/deployml/templates/gcp/cloud_run/variables.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/variables.tf.j2 @@ -217,3 +217,16 @@ variable "feast_create_bigquery_dataset" { default = true } +# Teardown-specific variables +variable "terraform_state_bucket" { + description = "GCS bucket for storing Terraform state (optional, for teardown)" + type = string + default = "" +} + +variable "terraform_files_bucket" { + description = "GCS bucket for storing Terraform files (optional, for teardown)" + type = string + default = "" +} + diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py new file mode 100644 index 0000000..87aad00 --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py @@ -0,0 +1,191 @@ +""" +Cloud Function to automatically teardown DeployML infrastructure. +Triggered by Cloud Scheduler. + +This function uses the DeployML CLI approach: it calls terraform destroy +on the workspace directory stored in GCS, or uses terraform remote state. +""" +import os +import json +import subprocess +import tempfile +import shutil +from pathlib import Path +from google.cloud import storage as gcs +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def teardown_infrastructure(request): + """ + Cloud Function entry point. + Expects JSON payload with: + - workspace_name: Name of the workspace to destroy + - project_id: GCP project ID + - terraform_files_bucket: (optional) GCS bucket with Terraform files + - terraform_state_bucket: (optional) GCS bucket with Terraform state + """ + try: + # Parse request + request_json = request.get_json(silent=True) + if not request_json: + return {'error': 'No JSON payload provided'}, 400 + + workspace_name = request_json.get('workspace_name') + project_id = request_json.get('project_id') + terraform_files_bucket = request_json.get('terraform_files_bucket', '') + terraform_state_bucket = request_json.get('terraform_state_bucket', '') + + if not workspace_name or not project_id: + return {'error': 'Missing required fields: workspace_name, project_id'}, 400 + + logger.info(f"Starting teardown for workspace: {workspace_name}") + + # Create temporary directory for terraform + with tempfile.TemporaryDirectory() as tmpdir: + terraform_dir = Path(tmpdir) / "terraform" + terraform_dir.mkdir(parents=True, exist_ok=True) + + # Download terraform files from GCS if provided + if terraform_files_bucket: + download_terraform_files(terraform_files_bucket, workspace_name, terraform_dir) + + # Download terraform state if stored in GCS + if terraform_state_bucket: + download_terraform_state(terraform_state_bucket, workspace_name, terraform_dir) + + # Install terraform if not available (Cloud Functions don't have it by default) + # We'll use a workaround: call terraform via a container or use gcloud commands + # For now, we'll assume terraform is available or use alternative method + + # Set GCP project + try: + subprocess.run( + ['gcloud', 'config', 'set', 'project', project_id], + cwd=terraform_dir, + check=True, + capture_output=True, + timeout=30 + ) + except Exception as e: + logger.warning(f"Could not set gcloud project: {e}") + + # Try to initialize terraform (if files exist) + if (terraform_dir / "main.tf").exists(): + # Initialize terraform + init_result = subprocess.run( + ['terraform', 'init'], + cwd=terraform_dir, + check=False, + capture_output=True, + text=True, + timeout=120 + ) + + if init_result.returncode != 0: + logger.warning(f"Terraform init failed: {init_result.stderr}") + # Try alternative: use terraform with remote state only + return destroy_via_remote_state(project_id, workspace_name, terraform_state_bucket) + + # Run terraform destroy + destroy_result = subprocess.run( + ['terraform', 'destroy', '-auto-approve'], + cwd=terraform_dir, + capture_output=True, + text=True, + timeout=600 # 10 minutes timeout + ) + + if destroy_result.returncode == 0: + logger.info(f"Successfully destroyed infrastructure for {workspace_name}") + return { + 'status': 'success', + 'message': f'Infrastructure destroyed for workspace: {workspace_name}', + 'workspace': workspace_name + }, 200 + else: + logger.error(f"Terraform destroy failed: {destroy_result.stderr}") + return { + 'status': 'error', + 'message': f'Destroy failed: {destroy_result.stderr}', + 'workspace': workspace_name + }, 500 + else: + # No terraform files found, try alternative approach + logger.info("No terraform files found, attempting alternative teardown method") + return destroy_via_remote_state(project_id, workspace_name, terraform_state_bucket) + + except subprocess.TimeoutExpired: + logger.error("Terraform operation timed out") + return { + 'status': 'error', + 'message': 'Teardown operation timed out' + }, 500 + except Exception as e: + logger.error(f"Error in teardown function: {str(e)}") + return { + 'status': 'error', + 'message': str(e) + }, 500 + + +def download_terraform_files(bucket_name: str, workspace_name: str, terraform_dir: Path): + """Download Terraform files from GCS bucket.""" + try: + storage_client = gcs.Client() + bucket = storage_client.bucket(bucket_name) + + # List and download all terraform files + prefix = f"{workspace_name}/terraform/" + blobs = bucket.list_blobs(prefix=prefix) + + for blob in blobs: + # Get relative path + relative_path = blob.name.replace(prefix, "") + if relative_path: # Skip the prefix itself + local_path = terraform_dir / relative_path + local_path.parent.mkdir(parents=True, exist_ok=True) + blob.download_to_filename(local_path) + logger.info(f"Downloaded {blob.name} to {local_path}") + except Exception as e: + logger.warning(f"Could not download terraform files: {e}") + + +def download_terraform_state(bucket_name: str, workspace_name: str, terraform_dir: Path): + """Download Terraform state files from GCS bucket.""" + try: + storage_client = gcs.Client() + bucket = storage_client.bucket(bucket_name) + + # Download terraform.tfstate if exists + blob_name = f"{workspace_name}/terraform/terraform.tfstate" + blob = bucket.blob(blob_name) + if blob.exists(): + blob.download_to_filename(terraform_dir / "terraform.tfstate") + logger.info(f"Downloaded terraform state from {blob_name}") + except Exception as e: + logger.warning(f"Could not download terraform state: {e}") + + +def destroy_via_remote_state(project_id: str, workspace_name: str, state_bucket: str): + """ + Alternative teardown method using terraform with remote state backend. + Creates a minimal terraform config that uses remote state. + """ + try: + logger.info("Attempting teardown via remote state") + # This would require creating a backend config and running terraform destroy + # For now, return an error suggesting manual teardown + return { + 'status': 'error', + 'message': f'Could not automatically teardown. Please run: deployml destroy --config-path ', + 'workspace': workspace_name + }, 500 + except Exception as e: + logger.error(f"Remote state teardown failed: {e}") + return { + 'status': 'error', + 'message': str(e) + }, 500 \ No newline at end of file diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/requirements.txt b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/requirements.txt new file mode 100644 index 0000000..748abc9 --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-storage>=2.10.0 +google-cloud-secret-manager>=2.16.0 + diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf b/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf new file mode 100644 index 0000000..777e71c --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf @@ -0,0 +1,140 @@ +# Teardown module - Creates Cloud Function and Cloud Scheduler for auto-teardown + +resource "google_project_service" "required" { + for_each = toset([ + "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", + "storage-api.googleapis.com" + ]) + project = var.project_id + service = each.value + disable_on_destroy = false +} + +data "google_project" "current" {} + +# Archive Cloud Function source code +data "archive_file" "function_source" { + type = "zip" + source_dir = "${path.module}/cloud_function" + output_path = "${path.module}/function.zip" +} + +# Cloud Storage bucket for function source code +resource "google_storage_bucket" "function_source" { + name = "${var.project_id}-teardown-functions-${random_id.bucket_suffix.hex}" + location = var.region + project = var.project_id + + uniform_bucket_level_access = true + force_destroy = true +} + +resource "random_id" "bucket_suffix" { + byte_length = 4 +} + +# Upload function source code +resource "google_storage_bucket_object" "function_source" { + name = "teardown-function-${data.archive_file.function_source.output_md5}.zip" + bucket = google_storage_bucket.function_source.name + source = data.archive_file.function_source.output_path +} + +# Cloud Function +resource "google_cloudfunctions_function" "teardown" { + name = "deployml-teardown-${var.workspace_name}" + description = "Auto-teardown function for DeployML workspace: ${var.workspace_name}" + runtime = "python311" + region = var.region + project = var.project_id + + available_memory_mb = 256 + source_archive_bucket = google_storage_bucket.function_source.name + source_archive_object = google_storage_bucket_object.function_source.name + trigger { + http_trigger { + security_level = "SECURE_ALWAYS" + } + } + entry_point = "teardown_infrastructure" + + environment_variables = { + PROJECT_ID = var.project_id + } + + service_account_email = google_service_account.teardown.email + + depends_on = [google_project_service.required] +} + +# Service account for Cloud Function +resource "google_service_account" "teardown" { + account_id = "deployml-teardown-${substr(replace(var.workspace_name, "-", ""), 0, 20)}" + display_name = "DeployML Teardown Service Account" + project = var.project_id +} + +# Grant permissions to service account +resource "google_project_iam_member" "teardown_permissions" { + for_each = toset([ + "roles/run.admin", # To destroy Cloud Run services + "roles/compute.instanceAdmin.v1", # To destroy VMs + "roles/storage.admin", # To destroy storage buckets + "roles/cloudsql.admin", # To destroy Cloud SQL instances + "roles/iam.serviceAccountUser", # To use service accounts + "roles/resourcemanager.projectIamAdmin" # To clean up IAM bindings + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:${google_service_account.teardown.email}" +} + +# Cloud Scheduler job +resource "google_cloud_scheduler_job" "teardown" { + name = "deployml-teardown-${var.workspace_name}" + description = "Auto-teardown job for DeployML workspace: ${var.workspace_name}" + schedule = var.schedule + time_zone = var.time_zone + region = var.region + project = var.project_id + + http_target { + http_method = "POST" + uri = google_cloudfunctions_function.teardown.https_trigger_url + + body = base64encode(jsonencode({ + workspace_name = var.workspace_name + project_id = var.project_id + terraform_state_bucket = var.terraform_state_bucket + terraform_files_bucket = var.terraform_files_bucket + })) + + headers = { + "Content-Type" = "application/json" + } + + oidc_token { + service_account_email = google_service_account.teardown.email + } + } + + depends_on = [google_project_service.required] +} + +# Outputs +output "function_url" { + value = google_cloudfunctions_function.teardown.https_trigger_url + description = "URL of the teardown Cloud Function" +} + +output "scheduler_job_name" { + value = google_cloud_scheduler_job.teardown.name + description = "Name of the Cloud Scheduler job" +} + +output "scheduler_job_id" { + value = google_cloud_scheduler_job.teardown.id + description = "ID of the Cloud Scheduler job" +} \ No newline at end of file diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf b/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf new file mode 100644 index 0000000..a9fd816 --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf @@ -0,0 +1,20 @@ +output "function_url" { + value = google_cloudfunctions_function.teardown.https_trigger_url + description = "URL of the teardown Cloud Function" +} + +output "scheduler_job_name" { + value = google_cloud_scheduler_job.teardown.name + description = "Name of the Cloud Scheduler job" +} + +output "scheduler_job_id" { + value = google_cloud_scheduler_job.teardown.id + description = "ID of the Cloud Scheduler job" +} + +output "service_account_email" { + value = google_service_account.teardown.email + description = "Email of the service account used for teardown" +} + diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/variables.tf b/src/deployml/terraform/modules/teardown/cloud/gcp/variables.tf new file mode 100644 index 0000000..a174ef9 --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/variables.tf @@ -0,0 +1,38 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "region" { + description = "GCP region" + type = string +} + +variable "workspace_name" { + description = "Name of the DeployML workspace to teardown" + type = string +} + +variable "schedule" { + description = "Cron schedule for teardown (e.g., '0 2 * * *' for daily at 2 AM)" + type = string +} + +variable "time_zone" { + description = "Time zone for the schedule" + type = string + default = "UTC" +} + +variable "terraform_state_bucket" { + description = "GCS bucket containing Terraform state (optional)" + type = string + default = "" +} + +variable "terraform_files_bucket" { + description = "GCS bucket containing Terraform files (optional, for teardown)" + type = string + default = "" +} + diff --git a/src/deployml/utils/teardown.py b/src/deployml/utils/teardown.py new file mode 100644 index 0000000..3050eea --- /dev/null +++ b/src/deployml/utils/teardown.py @@ -0,0 +1,50 @@ +"""Utilities for managing auto-teardown functionality.""" +import json +from pathlib import Path +from datetime import datetime, timedelta +from typing import Optional, Dict, Any + + +METADATA_FILE = "deployment_metadata.json" + + +def save_deployment_metadata(workspace_dir: Path, metadata: Dict[str, Any]): + """Save deployment metadata including teardown schedule.""" + metadata_path = workspace_dir / METADATA_FILE + metadata_path.parent.mkdir(parents=True, exist_ok=True) + + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + + +def load_deployment_metadata(workspace_dir: Path) -> Optional[Dict[str, Any]]: + """Load deployment metadata.""" + metadata_path = workspace_dir / METADATA_FILE + if not metadata_path.exists(): + return None + + with open(metadata_path, 'r') as f: + return json.load(f) + + +def calculate_teardown_schedule(deployed_at: datetime, duration_hours: int) -> str: + """ + Calculate cron expression for teardown time. + Returns cron string like "0 2 15 1 *" for Jan 15 at 2:00 AM UTC + """ + teardown_time = deployed_at + timedelta(hours=duration_hours) + + # Format: minute hour day month day-of-week + minute = teardown_time.minute + hour = teardown_time.hour + day = teardown_time.day + month = teardown_time.month + + return f"{minute} {hour} {day} {month} *" + + +def calculate_cron_from_timestamp(timestamp: int) -> str: + """Convert Unix timestamp to cron expression.""" + dt = datetime.utcfromtimestamp(timestamp) + return f"{dt.minute} {dt.hour} {dt.day} {dt.month} *" + From 05b33e3bf420cc2939c3c1f1a8b9030b8696465f Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Mon, 24 Nov 2025 22:21:25 -0800 Subject: [PATCH 2/6] auto-teardown --- src/deployml/cli/cli.py | 442 +++++++++++++++++- .../templates/gcp/cloud_run/main.tf.j2 | 2 +- .../templates/gcp/cloud_run/mlflow_main.tf.j2 | 27 ++ .../teardown/cloud/gcp/cloudbuild.yaml | 88 ++++ .../modules/teardown/cloud/gcp/main.tf | 185 ++++---- .../modules/teardown/cloud/gcp/outputs.tf | 15 +- .../teardown/cloud/gcp/teardown_script.sh | 231 +++++++++ src/deployml/utils/helpers.py | 12 + 8 files changed, 903 insertions(+), 99 deletions(-) create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/cloudbuild.yaml create mode 100644 src/deployml/terraform/modules/teardown/cloud/gcp/teardown_script.sh diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index 0c65421..0c05270 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -3,6 +3,7 @@ import typer import shutil import subprocess +import re from deployml.utils.banner import display_banner from deployml.utils.menu import prompt, show_menu from deployml.utils.constants import ( @@ -45,6 +46,395 @@ calculate_cron_from_timestamp, ) + +def upload_terraform_files_to_gcs(terraform_dir: Path, project_id: str, workspace_name: str): + """ + Upload Terraform files to GCS bucket for Cloud Build teardown. + Gets bucket name from Terraform state. + """ + try: + # Get terraform files bucket from Terraform state + # The bucket is created by the teardown module + state_proc = subprocess.run( + ["terraform", "state", "list"], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if state_proc.returncode != 0: + typer.echo(f"⚠️ Could not read Terraform state: {state_proc.stderr}") + return + + # Find the terraform_files bucket resource + bucket_resource = None + for line in state_proc.stdout.split('\n'): + if 'module.teardown.google_storage_bucket.terraform_files' in line: + bucket_resource = line.strip() + break + + if not bucket_resource: + typer.echo("⚠️ Teardown module bucket not found in state. Skipping upload.") + return + + # Get bucket name from state + show_proc = subprocess.run( + ["terraform", "state", "show", bucket_resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if show_proc.returncode != 0: + typer.echo(f"⚠️ Could not get bucket name: {show_proc.stderr}") + return + + # Extract bucket name from terraform state show output + bucket_name = None + for line in show_proc.stdout.split('\n'): + if 'name' in line and '=' in line: + bucket_name = line.split('=')[1].strip().strip('"') + break + + if not bucket_name: + typer.echo("⚠️ Could not extract bucket name from state.") + return + + # Upload Terraform files + storage_client = storage.Client(project=project_id) + bucket = storage_client.bucket(bucket_name) + + # Upload all .tf files, .tfvars, and state files + terraform_files = list(terraform_dir.glob("*.tf")) + list(terraform_dir.glob("*.tfvars")) + terraform_files += list(terraform_dir.glob("terraform.tfstate*")) # Include state files + terraform_files += list((terraform_dir / "modules").rglob("*.tf")) if (terraform_dir / "modules").exists() else [] + + uploaded_count = 0 + for tf_file in terraform_files: + if tf_file.is_file(): + # Create relative path from terraform_dir + relative_path = tf_file.relative_to(terraform_dir) + blob_path = f"{workspace_name}/terraform/{relative_path}" + + blob = bucket.blob(blob_path) + blob.upload_from_filename(str(tf_file)) + uploaded_count += 1 + + typer.echo(f"✅ Uploaded {uploaded_count} Terraform files to gs://{bucket_name}/{workspace_name}/terraform/") + + except Exception as e: + typer.echo(f"⚠️ Error uploading Terraform files: {e}") + import traceback + typer.echo(traceback.format_exc()) + + +def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_name: str, region: str) -> dict: + """ + Extract resource details from Terraform outputs and state. + Returns a manifest dictionary with all resources that need to be deleted. + """ + import json + import subprocess + from urllib.parse import urlparse + + manifest = { + "workspace_name": workspace_name, + "project_id": project_id, + "region": region, + "resources": { + "cloud_run_services": [], + "cloud_run_jobs": [], + "cloud_sql_instances": [], + "storage_buckets": [], + "cloud_scheduler_jobs": [], + "pubsub_topics": [], + "secret_manager_secrets": [], + "service_accounts": [], + "cloud_build_triggers": [], + } + } + + # Get Terraform outputs + output_proc = subprocess.run( + ["terraform", "output", "-json"], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if output_proc.returncode == 0: + outputs = json.loads(output_proc.stdout) + + # Extract Cloud Run service names from URLs + for key, value in outputs.items(): + output_val = value.get('value', '') + + # Cloud Run services - we'll extract from Terraform state instead of URLs + # (URLs contain hash suffixes that don't match actual service names) + pass # Skip URL parsing, will get from state below + + # Storage buckets + if '_bucket' in key and output_val: + if isinstance(output_val, str) and output_val: + manifest["resources"]["storage_buckets"].append({ + "name": output_val + }) + + # Cloud SQL instance connection name + if 'instance_connection_name' in key and output_val: + # Format: project:region:instance + parts = str(output_val).split(':') + if len(parts) == 3: + manifest["resources"]["cloud_sql_instances"].append({ + "name": parts[2], + "region": parts[1] + }) + + # Query Terraform state for additional resources + state_proc = subprocess.run( + ["terraform", "state", "list"], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if state_proc.returncode == 0: + state_resources = [r.strip() for r in state_proc.stdout.strip().split('\n') if r.strip()] + + for resource in state_resources: + try: + # Cloud Run services (v1 and v2) + if 'google_cloud_run_service' in resource and 'google_cloud_run_v2_job' not in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + service_name = None + service_region = region + for line in show_proc.stdout.split('\n'): + # Strip ANSI escape codes + clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + # Check for name field (but not location, id, or other fields) + if (clean_line.strip().startswith('name') or ' = name' in clean_line.lower()) and '=' in clean_line and 'location' not in clean_line.lower() and 'id' not in clean_line.lower() and 'latest' not in clean_line.lower(): + parts = clean_line.split('=') + if len(parts) >= 2: + potential_name = parts[1].strip().strip('"').strip("'") + # Remove ANSI codes from the name itself + potential_name = re.sub(r'\x1b\[[0-9;]*m', '', potential_name) + # Skip null, empty, or invalid values + if potential_name and potential_name.lower() != 'null' and '/' not in potential_name and '@' not in potential_name and len(potential_name) < 200: + service_name = potential_name + break + # Check for location or region field + elif (clean_line.strip().startswith('location') or clean_line.strip().startswith('region')) and '=' in clean_line: + parts = clean_line.split('=') + if len(parts) >= 2: + service_region = parts[1].strip().strip('"').strip("'") + service_region = re.sub(r'\x1b\[[0-9;]*m', '', service_region) + if service_name: + manifest["resources"]["cloud_run_services"].append({ + "name": service_name, + "region": service_region + }) + + # Cloud Run Jobs + elif 'google_cloud_run_v2_job' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + for line in show_proc.stdout.split('\n'): + if 'name' in line.lower() and '=' in line and 'location' not in line.lower(): + job_name = line.split('=')[1].strip().strip('"').strip("'") + if job_name: + manifest["resources"]["cloud_run_jobs"].append({ + "name": job_name, + "region": region + }) + break + + # Cloud Scheduler jobs + elif 'google_cloud_scheduler_job' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + job_name = None + job_region = region + for line in show_proc.stdout.split('\n'): + if 'name' in line.lower() and '=' in line and 'location' not in line.lower(): + job_name = line.split('=')[1].strip().strip('"').strip("'") + # Extract job name from URL if it's a full URL + if job_name and '/jobs/' in job_name: + job_name = job_name.split('/jobs/')[-1].split(':')[0].split('/')[-1] + elif 'region' in line.lower() and '=' in line: + job_region = line.split('=')[1].strip().strip('"').strip("'") + if job_name: + manifest["resources"]["cloud_scheduler_jobs"].append({ + "name": job_name, + "region": job_region + }) + + # Pub/Sub topics + elif 'google_pubsub_topic' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + for line in show_proc.stdout.split('\n'): + if 'name' in line.lower() and '=' in line: + topic_name = line.split('=')[1].strip().strip('"').strip("'") + if topic_name: + manifest["resources"]["pubsub_topics"].append({ + "name": topic_name + }) + break + + # Secret Manager secrets + elif 'google_secret_manager_secret' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + for line in show_proc.stdout.split('\n'): + if 'secret_id' in line.lower() and '=' in line: + secret_name = line.split('=')[1].strip().strip('"').strip("'") + if secret_name: + manifest["resources"]["secret_manager_secrets"].append({ + "name": secret_name + }) + break + + # Service accounts (only teardown ones to avoid deleting user SAs) + elif 'google_service_account' in resource and 'teardown' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + for line in show_proc.stdout.split('\n'): + if 'email' in line.lower() and '@' in line and '=' in line: + sa_email = line.split('=')[1].strip().strip('"').strip("'") + if sa_email: + manifest["resources"]["service_accounts"].append({ + "email": sa_email + }) + break + + # Cloud Build triggers + elif 'google_cloudbuild_trigger' in resource: + show_proc = subprocess.run( + ["terraform", "state", "show", resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + if show_proc.returncode == 0: + for line in show_proc.stdout.split('\n'): + if 'name' in line.lower() and '=' in line: + trigger_name = line.split('=')[1].strip().strip('"').strip("'") + if trigger_name: + manifest["resources"]["cloud_build_triggers"].append({ + "name": trigger_name + }) + break + except Exception as e: + # Skip resources that can't be parsed + continue + + # Remove duplicates + for resource_type in manifest["resources"]: + seen = set() + unique_resources = [] + for res in manifest["resources"][resource_type]: + if resource_type in ["cloud_run_services", "cloud_run_jobs", "cloud_sql_instances", "cloud_scheduler_jobs"]: + key = (res.get("name"), res.get("region")) + elif resource_type == "service_accounts": + key = res.get("email") + else: + key = res.get("name") + + if key and key not in seen: + seen.add(key) + unique_resources.append(res) + manifest["resources"][resource_type] = unique_resources + + return manifest + + +def upload_resource_manifest(manifest: dict, terraform_dir: Path, project_id: str, workspace_name: str): + """Upload resource manifest to GCS bucket.""" + import json + + try: + # Get bucket name from Terraform state (same logic as upload_terraform_files_to_gcs) + state_proc = subprocess.run( + ["terraform", "state", "list"], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if state_proc.returncode != 0: + raise Exception(f"Could not read Terraform state: {state_proc.stderr}") + + bucket_resource = None + for line in state_proc.stdout.split('\n'): + if 'module.teardown.google_storage_bucket.terraform_files' in line: + bucket_resource = line.strip() + break + + if not bucket_resource: + raise Exception("Teardown module bucket not found in state") + + show_proc = subprocess.run( + ["terraform", "state", "show", bucket_resource], + cwd=terraform_dir, + capture_output=True, + text=True, + ) + + if show_proc.returncode != 0: + raise Exception(f"Could not get bucket name: {show_proc.stderr}") + + bucket_name = None + for line in show_proc.stdout.split('\n'): + if 'name' in line and '=' in line: + bucket_name = line.split('=')[1].strip().strip('"') + break + + if not bucket_name: + raise Exception("Could not extract bucket name from state") + + # Upload manifest + storage_client = storage.Client(project=project_id) + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(f"{workspace_name}/resource-manifest.json") + blob.upload_from_string(json.dumps(manifest, indent=2)) + + typer.echo(f"✅ Resource manifest uploaded to gs://{bucket_name}/{workspace_name}/resource-manifest.json") + + except Exception as e: + typer.echo(f"⚠️ Error uploading resource manifest: {e}") + raise + import re import time import json @@ -408,12 +798,17 @@ def deploy( } ) + # Handle teardown configuration (needed before copying modules) + teardown_config = config.get("teardown", {}) + teardown_enabled = teardown_config.get("enabled", False) + typer.echo("📦 Copying module templates...") copy_modules_to_workspace( DEPLOYML_MODULES_DIR, stack=stack, deployment_type=deployment_type, cloud=cloud, + teardown_enabled=teardown_enabled, ) # --- UNIFIED BUCKET CONFIGURATION APPROACH --- # Collect all bucket configurations in a structured way (similar to VM creation) @@ -486,9 +881,7 @@ def deploy( name_material = f"{workspace_name}:{project_id}".encode("utf-8") name_hash = hashlib.sha1(name_material).hexdigest()[:6] - # Handle teardown configuration - teardown_config = config.get("teardown", {}) - teardown_enabled = teardown_config.get("enabled", False) + # Calculate teardown schedule (teardown_config and teardown_enabled already defined above) teardown_cron_schedule = "" teardown_scheduled_timestamp = 0 @@ -675,9 +1068,30 @@ def deploy( DEPLOYML_TERRAFORM_DIR, minutes, ) - if result_code == 0 or result_code == 1: + if result_code == 0: typer.echo("✅ Deployment complete!") + # Upload Terraform files and resource manifest to GCS for teardown (if enabled) + if teardown_enabled: + try: + typer.echo("📤 Uploading Terraform files to GCS for teardown...") + upload_terraform_files_to_gcs(DEPLOYML_TERRAFORM_DIR, project_id, workspace_name) + + # Extract and upload resource manifest + typer.echo("📋 Extracting resource manifest...") + manifest = extract_resource_manifest( + DEPLOYML_TERRAFORM_DIR, + project_id, + workspace_name, + region + ) + upload_resource_manifest(manifest, DEPLOYML_TERRAFORM_DIR, project_id, workspace_name) + typer.echo("✅ Resource manifest uploaded successfully") + + except Exception as e: + typer.echo(f"⚠️ Warning: Could not upload files/manifest to GCS: {e}") + typer.echo(" Teardown may not work automatically. Manual teardown required.") + # Handle auto-teardown metadata if teardown_enabled: duration_hours = teardown_config.get("duration_hours", 24) @@ -693,7 +1107,7 @@ def deploy( } save_deployment_metadata(DEPLOYML_DIR, metadata) - typer.echo(f"\n⏰ Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f"\n Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") typer.echo(f" (in {duration_hours} hours)") typer.echo(f" To cancel: deployml teardown cancel --config-path {config_path}") @@ -766,7 +1180,23 @@ def deploy( else: typer.echo("⚠️ Could not retrieve Terraform outputs.") else: - typer.echo("❌ Terraform apply failed") + log_file = DEPLOYML_TERRAFORM_DIR / "terraform_apply.log" + typer.secho(f"\n❌ Terraform apply failed with exit code {result_code}", fg=typer.colors.RED, bold=True) + typer.echo(f"\n📋 Check the Terraform log for details:") + typer.echo(f" {log_file}") + typer.echo(f"\n💡 Common issues:") + typer.echo(" - Required GCP APIs may not be enabled (check log for API activation URLs)") + typer.echo(" - Insufficient IAM permissions") + typer.echo(" - Resource conflicts or quota limits") + typer.echo("\n🔍 Last 20 lines of the log:") + if log_file.exists(): + try: + with open(log_file, 'r') as f: + lines = f.readlines() + for line in lines[-20:]: + typer.echo(f" {line.rstrip()}") + except Exception: + pass raise typer.Exit(code=1) else: typer.echo("❌ Deployment cancelled") diff --git a/src/deployml/templates/gcp/cloud_run/main.tf.j2 b/src/deployml/templates/gcp/cloud_run/main.tf.j2 index 758d1a2..b42e649 100644 --- a/src/deployml/templates/gcp/cloud_run/main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/main.tf.j2 @@ -127,7 +127,7 @@ module "{{ stage_name }}_{{ tool.name }}" { image = var.{{ stage_name }}_{{ tool.name }}_image != "" ? var.{{ stage_name }}_{{ tool.name }}_image : var.global_image {% elif key == "artifact_bucket" %} # Skip - already handled above - {% elif key not in resource_params and key != "create_artifact_bucket" and key != "backend_store_uri" and key != "use_postgres" %} + {% elif key not in resource_params and key != "create_artifact_bucket" and key != "backend_store_uri" and key != "use_postgres" and key != "allow_public_access" %} {{ key }} = var.{{ key }} {% endif %} {% endfor %} diff --git a/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 b/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 index 791f96d..e8b3da1 100644 --- a/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2 @@ -376,4 +376,31 @@ output "{{ stage_name }}_{{ tool.name }}_jobs_summary" { {% endfor %} {% endfor %} +{# Auto-teardown module #} +{% if teardown_config and teardown_config.get("enabled", false) %} +module "teardown" { + source = "./modules/teardown/cloud/gcp" + + project_id = var.project_id + region = var.region + workspace_name = "{{ stack_name }}" + + schedule = "{{ teardown_cron_schedule }}" + + time_zone = "{{ teardown_config.get('time_zone', 'UTC') }}" + terraform_state_bucket = var.terraform_state_bucket + terraform_files_bucket = var.terraform_files_bucket +} + +output "teardown_scheduler_job" { + value = module.teardown.scheduler_job_name + description = "Name of the Cloud Scheduler job for auto-teardown" +} + +output "teardown_scheduled_at" { + value = "{{ teardown_scheduled_timestamp }}" + description = "Unix timestamp when teardown is scheduled" +} +{% endif %} + {% endblock %} \ No newline at end of file diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/cloudbuild.yaml b/src/deployml/terraform/modules/teardown/cloud/gcp/cloudbuild.yaml new file mode 100644 index 0000000..f76ef3c --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/cloudbuild.yaml @@ -0,0 +1,88 @@ +steps: + # Authenticate with service account first + - name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + echo "$$SERVICE_ACCOUNT_KEY" | base64 -d > /workspace/sa-key.json + gcloud auth activate-service-account --key-file=/workspace/sa-key.json + export GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json + id: 'authenticate' + secretEnv: ['SERVICE_ACCOUNT_KEY'] + + # Download Terraform files from GCS + - name: 'gcr.io/cloud-builders/gsutil' + entrypoint: 'bash' + args: + - '-c' + - | + mkdir -p /workspace + gsutil -m cp -r gs://${_TERRAFORM_FILES_BUCKET}/${_WORKSPACE_NAME}/terraform/* /workspace/ || echo "No terraform files found in GCS" + id: 'download-terraform-files' + env: + - 'GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json' + secretEnv: ['SERVICE_ACCOUNT_KEY'] + waitFor: ['authenticate'] + + # Initialize Terraform (try remote backend first, fallback to local state) + - name: 'hashicorp/terraform:1.5.0' + entrypoint: 'bash' + args: + - '-c' + - | + cd /workspace + export GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json + + # Try to initialize with remote backend if state bucket is provided + if [ -n "${_TERRAFORM_STATE_BUCKET}" ] && [ "${_TERRAFORM_STATE_BUCKET}" != "" ]; then + terraform init \ + -backend-config="bucket=${_TERRAFORM_STATE_BUCKET}" \ + -backend-config="prefix=${_WORKSPACE_NAME}/terraform/state" \ + -reconfigure || terraform init -reconfigure + else + # Use local state file if available + if [ -f "terraform.tfstate" ]; then + terraform init -reconfigure + else + echo "No state file found. Attempting to initialize..." + terraform init -reconfigure + fi + fi + id: 'terraform-init' + env: + - 'GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json' + secretEnv: ['SERVICE_ACCOUNT_KEY'] + waitFor: ['download-terraform-files'] + + # Destroy infrastructure + - name: 'hashicorp/terraform:1.5.0' + entrypoint: 'bash' + args: + - '-c' + - | + cd /workspace + export GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json + terraform destroy -auto-approve + id: 'terraform-destroy' + env: + - 'GOOGLE_APPLICATION_CREDENTIALS=/workspace/sa-key.json' + secretEnv: ['SERVICE_ACCOUNT_KEY'] + waitFor: ['terraform-init'] + +options: + logging: CLOUD_LOGGING_ONLY + machineType: 'N1_HIGHCPU_8' + +availableSecrets: + secretManager: + - versionName: projects/${PROJECT_ID}/secrets/deployml-teardown-sa-key-${_WORKSPACE_NAME_NO_DASH}/versions/latest + env: 'SERVICE_ACCOUNT_KEY' + +substitutions: + _WORKSPACE_NAME: '' + _WORKSPACE_NAME_NO_DASH: '' + _TERRAFORM_STATE_BUCKET: '' + _TERRAFORM_FILES_BUCKET: '' + +timeout: '1800s' # 30 minutes diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf b/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf index 777e71c..b6d5416 100644 --- a/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/main.tf @@ -1,10 +1,11 @@ -# Teardown module - Creates Cloud Function and Cloud Scheduler for auto-teardown +# Teardown module - Creates Cloud Run Job and Cloud Scheduler for auto-teardown resource "google_project_service" "required" { for_each = toset([ - "cloudfunctions.googleapis.com", + "run.googleapis.com", "cloudscheduler.googleapis.com", - "storage-api.googleapis.com" + "storage-api.googleapis.com", + "secretmanager.googleapis.com" ]) project = var.project_id service = each.value @@ -13,64 +14,28 @@ resource "google_project_service" "required" { data "google_project" "current" {} -# Archive Cloud Function source code -data "archive_file" "function_source" { - type = "zip" - source_dir = "${path.module}/cloud_function" - output_path = "${path.module}/function.zip" -} - -# Cloud Storage bucket for function source code -resource "google_storage_bucket" "function_source" { - name = "${var.project_id}-teardown-functions-${random_id.bucket_suffix.hex}" +# GCS bucket for storing Terraform files and resource manifest +resource "google_storage_bucket" "terraform_files" { + name = "${var.project_id}-deployml-terraform-${random_id.bucket_suffix.hex}" location = var.region project = var.project_id uniform_bucket_level_access = true force_destroy = true + + versioning { + enabled = true + } } resource "random_id" "bucket_suffix" { byte_length = 4 } -# Upload function source code -resource "google_storage_bucket_object" "function_source" { - name = "teardown-function-${data.archive_file.function_source.output_md5}.zip" - bucket = google_storage_bucket.function_source.name - source = data.archive_file.function_source.output_path -} - -# Cloud Function -resource "google_cloudfunctions_function" "teardown" { - name = "deployml-teardown-${var.workspace_name}" - description = "Auto-teardown function for DeployML workspace: ${var.workspace_name}" - runtime = "python311" - region = var.region - project = var.project_id - - available_memory_mb = 256 - source_archive_bucket = google_storage_bucket.function_source.name - source_archive_object = google_storage_bucket_object.function_source.name - trigger { - http_trigger { - security_level = "SECURE_ALWAYS" - } - } - entry_point = "teardown_infrastructure" - - environment_variables = { - PROJECT_ID = var.project_id - } - - service_account_email = google_service_account.teardown.email - - depends_on = [google_project_service.required] -} - -# Service account for Cloud Function +# Service account for Cloud Run Job resource "google_service_account" "teardown" { - account_id = "deployml-teardown-${substr(replace(var.workspace_name, "-", ""), 0, 20)}" + # account_id must be 6-30 chars. "deployml-teardown-" is 18 chars, leaving 12 for workspace name + account_id = "deployml-teardown-${substr(replace(var.workspace_name, "-", ""), 0, 12)}" display_name = "DeployML Teardown Service Account" project = var.project_id } @@ -78,12 +43,17 @@ resource "google_service_account" "teardown" { # Grant permissions to service account resource "google_project_iam_member" "teardown_permissions" { for_each = toset([ - "roles/run.admin", # To destroy Cloud Run services - "roles/compute.instanceAdmin.v1", # To destroy VMs - "roles/storage.admin", # To destroy storage buckets - "roles/cloudsql.admin", # To destroy Cloud SQL instances - "roles/iam.serviceAccountUser", # To use service accounts - "roles/resourcemanager.projectIamAdmin" # To clean up IAM bindings + "roles/run.admin", # To destroy Cloud Run services and jobs + "roles/compute.instanceAdmin.v1", # To destroy VMs + "roles/storage.admin", # To destroy storage buckets + "roles/cloudsql.admin", # To destroy Cloud SQL instances + "roles/iam.serviceAccountUser", # To use service accounts + "roles/resourcemanager.projectIamAdmin", # To clean up IAM bindings + "roles/storage.objectAdmin", # To read/write files in GCS + "roles/secretmanager.secretAccessor", # To access secrets + "roles/pubsub.admin", # To delete Pub/Sub topics + "roles/cloudscheduler.admin", # To delete scheduler jobs + "roles/cloudbuild.builds.builder" # To delete Cloud Build triggers (if any exist from old deployments) ]) project = var.project_id @@ -91,7 +61,72 @@ resource "google_project_iam_member" "teardown_permissions" { member = "serviceAccount:${google_service_account.teardown.email}" } -# Cloud Scheduler job +# Upload teardown script to GCS +resource "google_storage_bucket_object" "teardown_script" { + name = "${var.workspace_name}/teardown.sh" + bucket = google_storage_bucket.terraform_files.name + source = "${path.module}/teardown_script.sh" +} + +# Cloud Run Job for teardown +resource "google_cloud_run_v2_job" "teardown" { + name = "deployml-teardown-${var.workspace_name}" + location = var.region + project = var.project_id + deletion_protection = false + + template { + template { + service_account = google_service_account.teardown.email + containers { + image = "gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + + command = ["bash"] + args = [ + "-c", + <<-EOT + # Download teardown script + gsutil cp gs://${google_storage_bucket.terraform_files.name}/${var.workspace_name}/teardown.sh /tmp/teardown.sh + chmod +x /tmp/teardown.sh + + # Set environment variables + export WORKSPACE_NAME="${var.workspace_name}" + export PROJECT_ID="${var.project_id}" + export REGION="${var.region}" + + # Run teardown script + /tmp/teardown.sh + EOT + ] + + resources { + limits = { + cpu = "2" + memory = "2Gi" + } + } + } + + timeout = "1800s" # 30 minutes + } + } + + depends_on = [ + google_project_service.required, + google_storage_bucket_object.teardown_script + ] +} + +# Grant Cloud Scheduler (via compute service account) permission to invoke the Cloud Run Job +resource "google_cloud_run_v2_job_iam_member" "scheduler_invoker" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_job.teardown.name + role = "roles/run.invoker" + member = "serviceAccount:${data.google_project.current.number}-compute@developer.gserviceaccount.com" +} + +# Cloud Scheduler job that triggers Cloud Run Job resource "google_cloud_scheduler_job" "teardown" { name = "deployml-teardown-${var.workspace_name}" description = "Auto-teardown job for DeployML workspace: ${var.workspace_name}" @@ -101,40 +136,16 @@ resource "google_cloud_scheduler_job" "teardown" { project = var.project_id http_target { + uri = "https://${var.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${var.project_id}/jobs/${google_cloud_run_v2_job.teardown.name}:run" http_method = "POST" - uri = google_cloudfunctions_function.teardown.https_trigger_url - - body = base64encode(jsonencode({ - workspace_name = var.workspace_name - project_id = var.project_id - terraform_state_bucket = var.terraform_state_bucket - terraform_files_bucket = var.terraform_files_bucket - })) - headers = { - "Content-Type" = "application/json" - } - - oidc_token { - service_account_email = google_service_account.teardown.email + oauth_token { + service_account_email = "${data.google_project.current.number}-compute@developer.gserviceaccount.com" } } - depends_on = [google_project_service.required] + depends_on = [ + google_project_service.required, + google_cloud_run_v2_job_iam_member.scheduler_invoker + ] } - -# Outputs -output "function_url" { - value = google_cloudfunctions_function.teardown.https_trigger_url - description = "URL of the teardown Cloud Function" -} - -output "scheduler_job_name" { - value = google_cloud_scheduler_job.teardown.name - description = "Name of the Cloud Scheduler job" -} - -output "scheduler_job_id" { - value = google_cloud_scheduler_job.teardown.id - description = "ID of the Cloud Scheduler job" -} \ No newline at end of file diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf b/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf index a9fd816..a9564ab 100644 --- a/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/outputs.tf @@ -1,8 +1,3 @@ -output "function_url" { - value = google_cloudfunctions_function.teardown.https_trigger_url - description = "URL of the teardown Cloud Function" -} - output "scheduler_job_name" { value = google_cloud_scheduler_job.teardown.name description = "Name of the Cloud Scheduler job" @@ -18,3 +13,13 @@ output "service_account_email" { description = "Email of the service account used for teardown" } +output "terraform_files_bucket" { + value = google_storage_bucket.terraform_files.name + description = "GCS bucket storing Terraform files and resource manifest for teardown" +} + +output "cloud_run_job_name" { + value = google_cloud_run_v2_job.teardown.name + description = "Name of the Cloud Run Job for teardown" +} + diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/teardown_script.sh b/src/deployml/terraform/modules/teardown/cloud/gcp/teardown_script.sh new file mode 100644 index 0000000..9d21fa0 --- /dev/null +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/teardown_script.sh @@ -0,0 +1,231 @@ +#!/bin/bash +set -e + +# Environment variables set by Cloud Run Job +WORKSPACE_NAME="${WORKSPACE_NAME}" +PROJECT_ID="${PROJECT_ID}" +REGION="${REGION}" + +echo "🗑️ Starting teardown for workspace: ${WORKSPACE_NAME}" +echo "📋 Project: ${PROJECT_ID}, Region: ${REGION}" + +# Find the bucket (it has a pattern: {project_id}-deployml-terraform-*) +BUCKET_NAME=$(gsutil ls | grep "${PROJECT_ID}-deployml-terraform-" | head -1 | xargs basename || echo "") + +if [ -z "$BUCKET_NAME" ]; then + echo "❌ Could not find deployml Terraform bucket" + exit 1 +fi + +echo "📥 Downloading resource manifest from gs://${BUCKET_NAME}/${WORKSPACE_NAME}/resource-manifest.json" + +# Download resource manifest +gsutil cp "gs://${BUCKET_NAME}/${WORKSPACE_NAME}/resource-manifest.json" /tmp/manifest.json || { + echo "❌ Failed to download manifest" + exit 1 +} + +# Use Python to parse JSON and delete resources (Python is available in cloud-sdk image) +python3 << PYTHON_EOF +import json +import subprocess +import sys + +# Ensure output is flushed immediately +sys.stdout.flush() +sys.stderr.flush() + +try: + # Load manifest + with open('/tmp/manifest.json', 'r') as f: + manifest = json.load(f) + + resources = manifest.get('resources', {}) + project_id = manifest.get('project_id', '${PROJECT_ID}') + region = manifest.get('region', '${REGION}') + + print("✅ Manifest downloaded. Found resources to delete:") + sys.stdout.flush() + for resource_type, items in resources.items(): + if items: + print(f" {resource_type}: {len(items)} items") + sys.stdout.flush() + + # Set project + subprocess.run(['gcloud', 'config', 'set', 'project', project_id], check=True) + + # Delete Cloud Run services + if resources.get('cloud_run_services'): + print("\n🗑️ Deleting Cloud Run services...") + sys.stdout.flush() + for service in resources['cloud_run_services']: + name = service.get('name') + svc_region = service.get('region', region) + if name: + print(f" Deleting Cloud Run service: {name} (region: {svc_region})") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'run', 'services', 'delete', name, '--region', svc_region, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Cloud Run Jobs + if resources.get('cloud_run_jobs'): + print("\n🗑️ Deleting Cloud Run Jobs...") + sys.stdout.flush() + for job in resources['cloud_run_jobs']: + name = job.get('name') + job_region = job.get('region', region) + if name: + print(f" Deleting Cloud Run Job: {name} (region: {job_region})") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'run', 'jobs', 'delete', name, '--region', job_region, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Cloud Scheduler jobs + if resources.get('cloud_scheduler_jobs'): + print("\n🗑️ Deleting Cloud Scheduler jobs...") + sys.stdout.flush() + for job in resources['cloud_scheduler_jobs']: + name = job.get('name') + # Extract job name from URL if it's a full URL + if name and '/jobs/' in name: + name = name.split('/jobs/')[-1].split(':')[0] + job_region = job.get('region', region) + if name: + print(f" Deleting Scheduler job: {name} (region: {job_region})") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'scheduler', 'jobs', 'delete', name, '--location', job_region, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Cloud Build triggers + if resources.get('cloud_build_triggers'): + print("\n🗑️ Deleting Cloud Build triggers...") + sys.stdout.flush() + for trigger in resources['cloud_build_triggers']: + name = trigger.get('name') + if name: + print(f" Deleting Cloud Build trigger: {name}") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'builds', 'triggers', 'delete', name, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Pub/Sub topics + if resources.get('pubsub_topics'): + print("\n🗑️ Deleting Pub/Sub topics...") + sys.stdout.flush() + for topic in resources['pubsub_topics']: + name = topic.get('name') + if name: + print(f" Deleting Pub/Sub topic: {name}") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'pubsub', 'topics', 'delete', name, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Cloud SQL instances + if resources.get('cloud_sql_instances'): + print("\n🗑️ Deleting Cloud SQL instances...") + sys.stdout.flush() + for instance in resources['cloud_sql_instances']: + name = instance.get('name') + if name: + print(f" Deleting Cloud SQL instance: {name}") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'sql', 'instances', 'delete', name, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Storage buckets + if resources.get('storage_buckets'): + print("\n🗑️ Deleting Storage buckets...") + sys.stdout.flush() + for bucket in resources['storage_buckets']: + name = bucket.get('name') + if name: + print(f" Deleting Storage bucket: {name}") + sys.stdout.flush() + # Use gcloud storage rm -r to delete bucket with all contents + result = subprocess.run( + ['gcloud', 'storage', 'rm', '-r', f'gs://{name}', '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Secret Manager secrets + if resources.get('secret_manager_secrets'): + print("\n🗑️ Deleting Secret Manager secrets...") + sys.stdout.flush() + for secret in resources['secret_manager_secrets']: + name = secret.get('name') + if name: + print(f" Deleting secret: {name}") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'secrets', 'delete', name, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {name}: {result.stderr}") + sys.stdout.flush() + + # Delete Service Accounts (last, as they may be used by other resources) + # Skip deleting the teardown service account itself (can't delete itself) + if resources.get('service_accounts'): + print("\n🗑️ Deleting Service Accounts...") + sys.stdout.flush() + for sa in resources['service_accounts']: + email = sa.get('email') + if email: + # Skip deleting the teardown service account itself + if 'deployml-teardown' in email: + print(f" ⏭️ Skipping teardown service account (cannot delete itself): {email}") + sys.stdout.flush() + continue + + print(f" Deleting service account: {email}") + sys.stdout.flush() + result = subprocess.run( + ['gcloud', 'iam', 'service-accounts', 'delete', email, '--quiet'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ⚠️ Failed to delete {email}: {result.stderr}") + sys.stdout.flush() + + print("\n✅ Teardown complete!") + sys.stdout.flush() +except Exception as e: + print(f"❌ Error during teardown: {e}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + sys.exit(1) +PYTHON_EOF diff --git a/src/deployml/utils/helpers.py b/src/deployml/utils/helpers.py index 450dc62..3002796 100644 --- a/src/deployml/utils/helpers.py +++ b/src/deployml/utils/helpers.py @@ -58,6 +58,7 @@ def copy_modules_to_workspace( stack: list | None = None, deployment_type: str | None = None, cloud: str = "gcp", + teardown_enabled: bool = False, ) -> None: """ Copy only the required Terraform module templates to the workspace directory. @@ -92,6 +93,10 @@ def copy_modules_to_workspace( tool_name = tool.get("name") if tool_name: used_modules.add(tool_name) + + # Add teardown module to used_modules if teardown is enabled + if teardown_enabled: + used_modules.add("teardown") # Only copy the modules that are being used, and only the specific deployment type for module_path in MODULE_TEMPLATES_DIR.iterdir(): @@ -103,6 +108,13 @@ def copy_modules_to_workspace( shutil.rmtree(dest_module_path) shutil.copytree(module_path, dest_module_path) continue + # Special case: always copy full teardown module (if it exists) + if module_path.name == "teardown": + dest_module_path = modules_dir / module_path.name + if dest_module_path.exists(): + shutil.rmtree(dest_module_path) + shutil.copytree(module_path, dest_module_path) + continue # Create the destination module directory dest_module_path = modules_dir / module_path.name if dest_module_path.exists(): From de562f24d7ed15ecd0231ccbb2fa86a4641e4c33 Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Tue, 25 Nov 2025 15:30:46 -0800 Subject: [PATCH 3/6] fix commands and scheduling --- src/deployml/cli/cli.py | 291 ++++++++++++++++++++++++++++++--- src/deployml/utils/teardown.py | 4 +- 2 files changed, 271 insertions(+), 24 deletions(-) diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index 0c05270..7800bf0 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -882,13 +882,15 @@ def deploy( name_hash = hashlib.sha1(name_material).hexdigest()[:6] # Calculate teardown schedule (teardown_config and teardown_enabled already defined above) + # Note: This is a preliminary schedule - will be updated after Terraform completes with exact time teardown_cron_schedule = "" teardown_scheduled_timestamp = 0 if teardown_enabled: duration_hours = teardown_config.get("duration_hours", 24) deployed_at = datetime.utcnow() - teardown_at = deployed_at + timedelta(hours=duration_hours) + # Add buffer to ensure schedule is in future (will be updated to exact time after deployment) + teardown_at = deployed_at + timedelta(hours=duration_hours, minutes=10) teardown_scheduled_timestamp = int(teardown_at.timestamp()) teardown_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) @@ -1092,22 +1094,55 @@ def deploy( typer.echo(f"⚠️ Warning: Could not upload files/manifest to GCS: {e}") typer.echo(" Teardown may not work automatically. Manual teardown required.") - # Handle auto-teardown metadata + # Handle auto-teardown metadata and update scheduler schedule if teardown_enabled: duration_hours = teardown_config.get("duration_hours", 24) + # Calculate teardown time AFTER deployment completes (not before) deployed_at = datetime.utcnow() teardown_at = deployed_at + timedelta(hours=duration_hours) + # Calculate the correct cron schedule based on actual deployment completion time + teardown_scheduled_timestamp = int(teardown_at.timestamp()) + correct_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + time_zone = teardown_config.get("time_zone", "UTC") + + # Update the Cloud Scheduler job with the correct schedule + scheduler_job_name = f"deployml-teardown-{workspace_name}" + try: + typer.echo(f"⏰ Updating teardown schedule to: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + update_result = subprocess.run( + [ + "gcloud", "scheduler", "jobs", "update", "http", scheduler_job_name, + "--location", region, + "--schedule", correct_cron_schedule, + "--time-zone", time_zone, + "--project", project_id, + "--quiet" + ], + capture_output=True, + text=True, + ) + if update_result.returncode == 0: + typer.echo(f"✅ Teardown schedule updated successfully") + else: + typer.echo(f"⚠️ Warning: Could not update scheduler schedule: {update_result.stderr}") + typer.echo(f" Schedule may be incorrect. Check manually with:") + typer.echo(f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}") + except Exception as e: + typer.echo(f"⚠️ Warning: Could not update scheduler schedule: {e}") + typer.echo(f" Schedule may be incorrect. Check manually with:") + typer.echo(f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}") + metadata = { "deployed_at": deployed_at.isoformat(), "teardown_scheduled_at": teardown_at.isoformat(), "teardown_enabled": True, "duration_hours": duration_hours, - "scheduler_job_name": f"deployml-teardown-{workspace_name}" + "scheduler_job_name": scheduler_job_name } save_deployment_metadata(DEPLOYML_DIR, metadata) - typer.echo(f"\n Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f"\n✅ Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") typer.echo(f" (in {duration_hours} hours)") typer.echo(f" To cancel: deployml teardown cancel --config-path {config_path}") @@ -1319,7 +1354,7 @@ def teardown( ), ): """ - Manage auto-teardown: cancel scheduled teardown, check status, or schedule new teardown. + Manage auto-teardown: cancel scheduled teardown, check status, update schedule, or schedule new teardown. """ if not config_path.exists(): typer.echo(f"❌ Config file not found: {config_path}") @@ -1332,11 +1367,13 @@ def teardown( if action == "cancel": cancel_teardown(config, DEPLOYML_DIR, workspace_name) elif action == "status": - show_teardown_status(DEPLOYML_DIR, workspace_name) + show_teardown_status(config, DEPLOYML_DIR, workspace_name) + elif action == "update": + update_teardown_schedule(config, DEPLOYML_DIR, workspace_name) elif action == "schedule": schedule_teardown(config, DEPLOYML_DIR, workspace_name) else: - typer.echo(f"❌ Unknown action: {action}. Use: cancel, status, or schedule") + typer.echo(f"❌ Unknown action: {action}. Use: cancel, status, update, or schedule") raise typer.Exit(code=1) @@ -1366,27 +1403,235 @@ def cancel_teardown(config: dict, deployml_dir: Path, workspace_name: str): typer.echo(" The scheduler job may not exist or may have already been deleted.") -def show_teardown_status(deployml_dir: Path, workspace_name: str): - """Show teardown status.""" - metadata = load_deployment_metadata(deployml_dir) +def show_teardown_status(config: dict, deployml_dir: Path, workspace_name: str): + """Show teardown status by querying Cloud Scheduler.""" + project_id = config["provider"]["project_id"] + region = config["provider"]["region"] + scheduler_job_name = f"deployml-teardown-{workspace_name}" + + # Query Cloud Scheduler job + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "describe", scheduler_job_name, + "--project", project_id, "--location", region, "--format", "json"], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + typer.echo(f"⚠️ Cloud Scheduler job not found: {scheduler_job_name}") + typer.echo(" Teardown may not be scheduled or may have already been cancelled.") + + # Check local metadata as fallback + metadata = load_deployment_metadata(deployml_dir) + if metadata and metadata.get("teardown_enabled"): + teardown_at = datetime.fromisoformat(metadata["teardown_scheduled_at"]) + typer.echo(f"\n📋 Local metadata shows teardown was scheduled for:") + typer.echo(f" {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + return + + # Parse Cloud Scheduler job details + try: + job_info = json.loads(result.stdout) + except json.JSONDecodeError: + typer.echo("❌ Failed to parse Cloud Scheduler job information") + return + + # Extract information + schedule = job_info.get("schedule", "N/A") + time_zone = job_info.get("timeZone", "UTC") + state = job_info.get("state", "UNKNOWN") + schedule_time = job_info.get("scheduleTime", "") + last_attempt_time = job_info.get("lastAttemptTime", "") + + # Display comprehensive status + typer.echo("📋 Auto-Teardown Status") + typer.echo("=" * 60) + + # Job state + state_emoji = "✅" if state == "ENABLED" else "⏸️" if state == "PAUSED" else "❌" + typer.echo(f"{state_emoji} Status: {state}") + + # Cron schedule + typer.echo(f"📅 Cron Schedule: {schedule}") + typer.echo(f"🌍 Timezone: {time_zone}") + + # Next execution time + if schedule_time: + try: + # Parse ISO format with Z suffix (UTC) + time_str = schedule_time.replace('Z', '+00:00') + next_run = datetime.fromisoformat(time_str) + # Get current UTC time as timezone-aware + from datetime import timezone + now = datetime.now(timezone.utc) + typer.echo(f"⏰ Next Execution: {next_run.strftime('%Y-%m-%d %H:%M:%S UTC')}") + + if now < next_run: + time_remaining = next_run - now + hours = int(time_remaining.total_seconds() // 3600) + minutes = int((time_remaining.total_seconds() % 3600) // 60) + typer.echo(f" ⏳ Time Remaining: {hours}h {minutes}m") + else: + time_passed = now - next_run + hours_passed = int(time_passed.total_seconds() // 3600) + minutes_passed = int((time_passed.total_seconds() % 3600) // 60) + typer.echo(f" ⚠️ Scheduled time passed {hours_passed}h {minutes_passed}m ago") + except Exception as e: + typer.echo(f"⏰ Next Execution: {schedule_time}") + + # Last attempt + if last_attempt_time and last_attempt_time != "1970-01-01T00:00:00Z": + try: + # Parse ISO format with Z suffix (UTC) + time_str = last_attempt_time.replace('Z', '+00:00') + last_run = datetime.fromisoformat(time_str) + typer.echo(f"🕐 Last Execution: {last_run.strftime('%Y-%m-%d %H:%M:%S UTC')}") + except Exception: + typer.echo(f"🕐 Last Execution: {last_attempt_time}") + else: + typer.echo("🕐 Last Execution: Never") + + # Job name for reference + typer.echo(f"\n📌 Job Name: {scheduler_job_name}") + + # Actions + typer.echo("\n💡 Actions:") + typer.echo(f" Update: deployml teardown update --config-path ") + typer.echo(f" Cancel: deployml teardown cancel --config-path ") + typer.echo(f" View in Console: https://console.cloud.google.com/cloudscheduler/jobs/edit/{region}/{scheduler_job_name}?project={project_id}") + + +def update_teardown_schedule(config: dict, deployml_dir: Path, workspace_name: str): + """Update the scheduled teardown time.""" + project_id = config["provider"]["project_id"] + region = config["provider"]["region"] + scheduler_job_name = f"deployml-teardown-{workspace_name}" + + # Check if Cloud Scheduler job exists + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "describe", scheduler_job_name, + "--project", project_id, "--location", region, "--format", "json"], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + typer.echo(f"❌ Cloud Scheduler job not found: {scheduler_job_name}") + typer.echo(" Cannot update schedule. The teardown job may not exist.") + typer.echo(" Use 'deployml deploy' with teardown.enabled: true to create it.") + raise typer.Exit(code=1) + + # Get current job info to preserve timezone + try: + job_info = json.loads(result.stdout) + time_zone = job_info.get("timeZone", "UTC") + except json.JSONDecodeError: + time_zone = "UTC" + + # Prompt for new schedule + typer.echo("📅 Update Teardown Schedule") + typer.echo("=" * 60) + + # Show current schedule + try: + schedule_time = job_info.get("scheduleTime", "") + if schedule_time: + time_str = schedule_time.replace('Z', '+00:00') + current_time = datetime.fromisoformat(time_str) + typer.echo(f"⏰ Current Schedule: {current_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + except Exception: + pass + + # Get new duration + duration_hours = typer.prompt("Hours until new teardown time", default=24, type=int) + + if duration_hours < 0: + typer.echo("❌ Duration must be positive") + raise typer.Exit(code=1) - if not metadata or not metadata.get("teardown_enabled"): - typer.echo("ℹ️ Auto-teardown is not enabled for this workspace") + # Calculate new teardown time + from datetime import timezone + now = datetime.now(timezone.utc) + teardown_at = now + timedelta(hours=duration_hours) + # Use UTC timestamp directly to avoid timezone issues + teardown_scheduled_timestamp = int(teardown_at.timestamp()) + new_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + + typer.echo(f"\n⏰ New Schedule: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + + # Confirm update + confirm = typer.confirm("Update the teardown schedule?", default=True) + if not confirm: + typer.echo("❌ Update cancelled") return - deployed_at = datetime.fromisoformat(metadata["deployed_at"]) - teardown_at = datetime.fromisoformat(metadata["teardown_scheduled_at"]) - now = datetime.utcnow() + # Update Cloud Scheduler job + typer.echo("\n🔄 Updating Cloud Scheduler job...") + typer.echo(f" Cron schedule: {new_cron_schedule}") + update_result = subprocess.run( + [ + "gcloud", "scheduler", "jobs", "update", "http", scheduler_job_name, + "--location", region, + "--schedule", new_cron_schedule, + "--time-zone", time_zone, + "--project", project_id, + "--quiet" + ], + capture_output=True, + text=True, + ) + + if update_result.returncode != 0: + typer.echo(f"❌ Failed to update schedule: {update_result.stderr}") + if update_result.stdout: + typer.echo(f" stdout: {update_result.stdout}") + raise typer.Exit(code=1) + + # Verify the update by querying the job again + verify_result = subprocess.run( + ["gcloud", "scheduler", "jobs", "describe", scheduler_job_name, + "--project", project_id, "--location", region, "--format", "json"], + capture_output=True, + text=True, + ) - if now >= teardown_at: - typer.echo("⚠️ Teardown time has passed (may have already executed)") + if verify_result.returncode == 0: + try: + updated_job_info = json.loads(verify_result.stdout) + updated_schedule_time = updated_job_info.get("scheduleTime", "") + updated_schedule = updated_job_info.get("schedule", "") + + if updated_schedule_time: + time_str = updated_schedule_time.replace('Z', '+00:00') + actual_time = datetime.fromisoformat(time_str) + typer.echo(f"\n✅ Teardown schedule updated successfully!") + typer.echo(f" Scheduled time: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f" Cron schedule: {updated_schedule}") + + # Check if it matches what we intended + if actual_time.strftime('%Y-%m-%d %H:%M') != teardown_at.strftime('%Y-%m-%d %H:%M'): + typer.echo(f"\n⚠️ Warning: Scheduled time differs from intended time") + typer.echo(f" Intended: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo(f" Actual: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + else: + typer.echo(f"✅ Teardown schedule updated successfully!") + typer.echo(f" Cron schedule: {updated_schedule}") + except Exception as e: + typer.echo(f"✅ Teardown schedule updated (verification failed: {e})") else: - time_remaining = teardown_at - now - hours = int(time_remaining.total_seconds() // 3600) - minutes = int((time_remaining.total_seconds() % 3600) // 60) - typer.echo(f"⏰ Scheduled teardown: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") - typer.echo(f" Time remaining: {hours}h {minutes}m") - typer.echo(f" To cancel: deployml teardown cancel --config-path ") + typer.echo(f"✅ Teardown schedule updated successfully!") + typer.echo(f" (Could not verify - check with: gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id})") + + # Update local metadata + metadata = load_deployment_metadata(deployml_dir) or {} + metadata.update({ + "deployed_at": metadata.get("deployed_at", now.isoformat()), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours, + "scheduler_job_name": scheduler_job_name + }) + save_deployment_metadata(deployml_dir, metadata) def schedule_teardown(config: dict, deployml_dir: Path, workspace_name: str): diff --git a/src/deployml/utils/teardown.py b/src/deployml/utils/teardown.py index 3050eea..b730d61 100644 --- a/src/deployml/utils/teardown.py +++ b/src/deployml/utils/teardown.py @@ -45,6 +45,8 @@ def calculate_teardown_schedule(deployed_at: datetime, duration_hours: int) -> s def calculate_cron_from_timestamp(timestamp: int) -> str: """Convert Unix timestamp to cron expression.""" - dt = datetime.utcfromtimestamp(timestamp) + # Use UTC timezone-aware datetime to ensure correct conversion + from datetime import timezone + dt = datetime.fromtimestamp(timestamp, tz=timezone.utc) return f"{dt.minute} {dt.hour} {dt.day} {dt.month} *" From 424b7bd34f52ac56def3eeed79e7ee10c503804c Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Tue, 25 Nov 2025 15:45:44 -0800 Subject: [PATCH 4/6] python api for auto-teardown --- src/deployml/__init__.py | 8 +- src/deployml/api.py | 262 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 src/deployml/api.py diff --git a/src/deployml/__init__.py b/src/deployml/__init__.py index 2794b35..49f535f 100644 --- a/src/deployml/__init__.py +++ b/src/deployml/__init__.py @@ -4,6 +4,9 @@ # Import diagnostics for easy access from .diagnostics import run_doctor, check_system, DeployMLDoctor +# Import Python API for teardown management +from .api import get_teardown_status, update_teardown_schedule, cancel_teardown + __all__ = [ 'deploy', 'load', @@ -11,5 +14,8 @@ 'ServiceURLs', 'run_doctor', 'check_system', - 'DeployMLDoctor' + 'DeployMLDoctor', + 'get_teardown_status', + 'update_teardown_schedule', + 'cancel_teardown' ] diff --git a/src/deployml/api.py b/src/deployml/api.py new file mode 100644 index 0000000..6d322ed --- /dev/null +++ b/src/deployml/api.py @@ -0,0 +1,262 @@ +""" +DeployML Python API + +Programmatic interface for managing DeployML deployments and teardown schedules. + +Example usage: + import deployml.api as api + + # Get teardown status + status = api.get_teardown_status( + project_id="my-project", + region="us-west1", + workspace_name="my-stack" + ) + + # Update teardown schedule + result = api.update_teardown_schedule( + project_id="my-project", + region="us-west1", + workspace_name="my-stack", + duration_hours=6 + ) + + # Cancel teardown + api.cancel_teardown( + project_id="my-project", + region="us-west1", + workspace_name="my-stack" + ) +""" +import json +import subprocess +from pathlib import Path +from datetime import datetime, timedelta, timezone +from typing import Optional, Dict, Any + +from .utils.teardown import ( + calculate_cron_from_timestamp, + load_deployment_metadata, + save_deployment_metadata +) + + +def get_teardown_status( + project_id: str, + region: str, + workspace_name: str, + deployml_dir: Optional[Path] = None +) -> Dict[str, Any]: + """ + Get teardown status from Cloud Scheduler. + + Args: + project_id: GCP project ID + region: GCP region + workspace_name: Workspace name + deployml_dir: Optional path to deployment directory for metadata + + Returns: + Dictionary with status information including: + - exists: bool - Whether the scheduler job exists + - state: str - Job state (ENABLED, PAUSED, etc.) + - schedule: str - Cron schedule + - schedule_time: datetime - Next scheduled execution time + - time_zone: str - Timezone + - last_attempt_time: Optional[datetime] - Last execution time + - metadata: Optional[Dict] - Local metadata if available + """ + scheduler_job_name = f"deployml-teardown-{workspace_name}" + + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "describe", scheduler_job_name, + "--project", project_id, "--location", region, "--format", "json"], + capture_output=True, + text=True, + ) + + status = { + "exists": False, + "scheduler_job_name": scheduler_job_name, + } + + # Load local metadata if available + if deployml_dir: + metadata = load_deployment_metadata(deployml_dir) + status["metadata"] = metadata + + if result.returncode != 0: + return status + + try: + job_info = json.loads(result.stdout) + status["exists"] = True + status["state"] = job_info.get("state", "UNKNOWN") + status["schedule"] = job_info.get("schedule", "") + status["time_zone"] = job_info.get("timeZone", "UTC") + + # Parse schedule time + schedule_time = job_info.get("scheduleTime", "") + if schedule_time: + time_str = schedule_time.replace('Z', '+00:00') + status["schedule_time"] = datetime.fromisoformat(time_str) + + # Parse last attempt time + last_attempt_time = job_info.get("lastAttemptTime", "") + if last_attempt_time and last_attempt_time != "1970-01-01T00:00:00Z": + time_str = last_attempt_time.replace('Z', '+00:00') + status["last_attempt_time"] = datetime.fromisoformat(time_str) + + return status + except json.JSONDecodeError: + return status + + +def update_teardown_schedule( + project_id: str, + region: str, + workspace_name: str, + duration_hours: int, + deployml_dir: Optional[Path] = None, + time_zone: str = "UTC" +) -> Dict[str, Any]: + """ + Update teardown schedule programmatically. + + Args: + project_id: GCP project ID + region: GCP region + workspace_name: Workspace name + duration_hours: Hours until teardown + deployml_dir: Optional path to deployment directory for metadata + time_zone: Timezone (default: UTC) + + Returns: + Dictionary with update result: + - success: bool - Whether update succeeded + - scheduled_time: datetime - New scheduled time + - cron_schedule: str - Cron expression + - error: Optional[str] - Error message if failed + """ + scheduler_job_name = f"deployml-teardown-{workspace_name}" + + # Check if job exists and get current timezone + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "describe", scheduler_job_name, + "--project", project_id, "--location", region, "--format", "json"], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + return { + "success": False, + "error": f"Cloud Scheduler job not found: {scheduler_job_name}" + } + + # Get timezone from existing job if not provided + try: + job_info = json.loads(result.stdout) + time_zone = job_info.get("timeZone", time_zone) + except json.JSONDecodeError: + pass + + # Calculate new teardown time + now = datetime.now(timezone.utc) + teardown_at = now + timedelta(hours=duration_hours) + teardown_scheduled_timestamp = int(teardown_at.timestamp()) + new_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + + # Update Cloud Scheduler job + update_result = subprocess.run( + [ + "gcloud", "scheduler", "jobs", "update", "http", scheduler_job_name, + "--location", region, + "--schedule", new_cron_schedule, + "--time-zone", time_zone, + "--project", project_id, + "--quiet" + ], + capture_output=True, + text=True, + ) + + if update_result.returncode != 0: + return { + "success": False, + "error": update_result.stderr, + "scheduled_time": teardown_at, + "cron_schedule": new_cron_schedule + } + + # Update local metadata if directory provided + if deployml_dir: + metadata = load_deployment_metadata(deployml_dir) or {} + metadata.update({ + "deployed_at": metadata.get("deployed_at", now.isoformat()), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours, + "scheduler_job_name": scheduler_job_name + }) + save_deployment_metadata(deployml_dir, metadata) + + return { + "success": True, + "scheduled_time": teardown_at, + "cron_schedule": new_cron_schedule, + "time_zone": time_zone + } + + +def cancel_teardown( + project_id: str, + region: str, + workspace_name: str, + deployml_dir: Optional[Path] = None +) -> Dict[str, Any]: + """ + Cancel scheduled teardown programmatically. + + Args: + project_id: GCP project ID + region: GCP region + workspace_name: Workspace name + deployml_dir: Optional path to deployment directory for metadata + + Returns: + Dictionary with cancellation result: + - success: bool - Whether cancellation succeeded + - error: Optional[str] - Error message if failed + """ + scheduler_job_name = f"deployml-teardown-{workspace_name}" + + result = subprocess.run( + ["gcloud", "scheduler", "jobs", "delete", scheduler_job_name, + "--project", project_id, "--location", region, "--quiet"], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + return { + "success": False, + "error": result.stderr + } + + # Update metadata if directory provided + if deployml_dir: + metadata = load_deployment_metadata(deployml_dir) + if metadata: + metadata["teardown_enabled"] = False + save_deployment_metadata(deployml_dir, metadata) + + return {"success": True} + + +__all__ = [ + 'get_teardown_status', + 'update_teardown_schedule', + 'cancel_teardown' +] + From 7b6012061fc9922d55ee3ad14c7bc5e5d19b7561 Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Wed, 26 Nov 2025 17:52:07 -0800 Subject: [PATCH 5/6] update auto-delete --- demo/auto-destory-demo.yaml | 30 ++++ src/deployml/__init__.py | 8 +- src/deployml/notebook/deployment.py | 79 ++++++++- src/deployml/notebook/stack.py | 154 ++++++++++++++++++ .../templates/gcp/cloud_run/main.tf.j2 | 16 +- .../modules/cloud_sql_postgres/main.tf | 64 +++++++- .../modules/cloud_sql_postgres/variables.tf | 12 ++ 7 files changed, 345 insertions(+), 18 deletions(-) create mode 100644 demo/auto-destory-demo.yaml diff --git a/demo/auto-destory-demo.yaml b/demo/auto-destory-demo.yaml new file mode 100644 index 0000000..60cf2a2 --- /dev/null +++ b/demo/auto-destory-demo.yaml @@ -0,0 +1,30 @@ +name: teardown-demo-stack +provider: + name: gcp + project_id: mldeploy-468919 + region: us-west1 +deployment: + type: cloud_run + +teardown: + enabled: true + duration_hours: 2 + time_zone: "UTC" + +stack: + - experiment_tracking: + name: mlflow + params: + image: gcr.io/mldeploy-468919/mlflow/mlflow:latest + service_name: teardown-demo-mlflow-server + + - artifact_tracking: + name: mlflow + params: + image: gcr.io/mldeploy-468919/mlflow/mlflow:latest + + - model_registry: + name: mlflow + params: + image: gcr.io/mldeploy-468919/mlflow/mlflow:latest + backend_store_uri: postgresql \ No newline at end of file diff --git a/src/deployml/__init__.py b/src/deployml/__init__.py index 49f535f..2794b35 100644 --- a/src/deployml/__init__.py +++ b/src/deployml/__init__.py @@ -4,9 +4,6 @@ # Import diagnostics for easy access from .diagnostics import run_doctor, check_system, DeployMLDoctor -# Import Python API for teardown management -from .api import get_teardown_status, update_teardown_schedule, cancel_teardown - __all__ = [ 'deploy', 'load', @@ -14,8 +11,5 @@ 'ServiceURLs', 'run_doctor', 'check_system', - 'DeployMLDoctor', - 'get_teardown_status', - 'update_teardown_schedule', - 'cancel_teardown' + 'DeployMLDoctor' ] diff --git a/src/deployml/notebook/deployment.py b/src/deployml/notebook/deployment.py index 6ba58e9..3dd6457 100644 --- a/src/deployml/notebook/deployment.py +++ b/src/deployml/notebook/deployment.py @@ -21,8 +21,29 @@ def deploy(config_path: str, show_progress: bool = True) -> DeploymentStack: """ config_file = Path(config_path) + # If relative path doesn't exist, try common locations if not config_file.exists(): - raise FileNotFoundError(f"Configuration file not found: {config_path}") + # Try in current directory + if not config_file.is_absolute(): + # Try demo/ subdirectory + demo_path = Path("demo") / config_file.name + if demo_path.exists(): + config_file = demo_path + print(f"ℹ️ Found config file in demo/ directory: {config_file}") + else: + # Try example/config/ subdirectory + example_path = Path("example/config") / config_file.name + if example_path.exists(): + config_file = example_path + print(f"ℹ️ Found config file in example/config/ directory: {config_file}") + else: + raise FileNotFoundError( + f"Configuration file not found: {config_path}\n" + f"Tried locations:\n" + f" - {Path(config_path).absolute()}\n" + f" - {Path('demo') / Path(config_path).name}\n" + f" - {Path('example/config') / Path(config_path).name}" + ) # Load configuration with open(config_file, 'r') as f: @@ -44,8 +65,8 @@ def deploy(config_path: str, show_progress: bool = True) -> DeploymentStack: workspace_name = config.get('name', 'default') workspace_dir = Path.cwd() / ".deployml" / workspace_name - # Run deployment using CLI command with logs - _deploy_with_cli(config_path, workspace_dir) + # Run deployment using CLI command with logs (use resolved config_file path) + _deploy_with_cli(str(config_file), workspace_dir) # Create and return deployment stack stack = DeploymentStack(config, workspace_dir) @@ -212,6 +233,9 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: print("📝 DEPLOYMENT LOG") print("="*60) + # Store output lines for error reporting + output_lines = [] + # Run with live output process = subprocess.Popen( cmd, @@ -223,6 +247,7 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: # Stream and format output line by line for line in iter(process.stdout.readline, ''): + output_lines.append(line) formatted_line = _format_deployment_line(line) if formatted_line is not None: print(formatted_line) @@ -231,6 +256,48 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: process.wait() if process.returncode != 0: - print(f"\nDEPLOYMENT FAILED (Exit Code: {process.returncode})") - print("="*60) - raise RuntimeError(f"Deployment failed with exit code {process.returncode}") \ No newline at end of file + # Force flush any pending output + sys.stdout.flush() + sys.stderr.flush() + + print(f"\n" + "="*60, flush=True) + print(f"❌ DEPLOYMENT FAILED (Exit Code: {process.returncode})", flush=True) + print("="*60, flush=True) + + # Show last 50 lines of output to help debug + print("\n📋 Last 50 lines of output:", flush=True) + print("-" * 60, flush=True) + for line in output_lines[-50:]: + # Show unformatted lines for error context + clean_line = line.rstrip() + if clean_line: + print(clean_line, flush=True) + + # Try to extract error message + error_lines = [line for line in output_lines if any(keyword in line.lower() for keyword in ['error', 'failed', 'fatal', 'exception', 'traceback'])] + if error_lines: + print("\n🔍 Error Summary:", flush=True) + print("-" * 60, flush=True) + for line in error_lines[-10:]: # Last 10 error-related lines + clean_line = line.rstrip() + if clean_line: + print(clean_line, flush=True) + else: + print("\n⚠️ No obvious error keywords found in output.", flush=True) + print(" Showing all output lines for debugging:", flush=True) + print("-" * 60, flush=True) + for line in output_lines: + clean_line = line.rstrip() + if clean_line: + print(clean_line, flush=True) + + print("="*60, flush=True) + sys.stdout.flush() + sys.stderr.flush() + + raise RuntimeError( + f"Deployment failed with exit code {process.returncode}. " + f"Check the output above for details. " + f"Total output lines: {len(output_lines)}, " + f"Error-related lines: {len([l for l in output_lines if any(k in l.lower() for k in ['error', 'failed', 'fatal'])])}" + ) \ No newline at end of file diff --git a/src/deployml/notebook/stack.py b/src/deployml/notebook/stack.py index ca2fa68..8b12385 100644 --- a/src/deployml/notebook/stack.py +++ b/src/deployml/notebook/stack.py @@ -2,6 +2,7 @@ import subprocess from pathlib import Path from typing import Dict, Any +from datetime import datetime, timezone import pandas as pd import mlflow from mlflow.tracking import MlflowClient @@ -385,4 +386,157 @@ def show_status(self) -> None: for label, value in status_data: print(f"{label:20}: {value}") + print("="*80) + + def get_teardown_status(self) -> Dict[str, Any]: + """ + Get teardown status for this deployment stack. + + Returns: + Dictionary with status information including: + - exists: bool - Whether the scheduler job exists + - state: str - Job state (ENABLED, PAUSED, etc.) + - schedule: str - Cron schedule + - schedule_time: datetime - Next scheduled execution time + - time_zone: str - Timezone + - last_attempt_time: Optional[datetime] - Last execution time + - metadata: Optional[Dict] - Local metadata if available + """ + from deployml.api import get_teardown_status as _get_teardown_status + + project_id = self.provider.get('project_id') + region = self.provider.get('region') + workspace_name = self.name + + if not project_id or not region: + return { + "exists": False, + "error": "Missing project_id or region in configuration" + } + + return _get_teardown_status( + project_id=project_id, + region=region, + workspace_name=workspace_name, + deployml_dir=self.workspace_dir + ) + + def update_teardown_schedule( + self, + duration_hours: int, + time_zone: str = "UTC" + ) -> Dict[str, Any]: + """ + Update teardown schedule for this deployment stack. + + Args: + duration_hours: Hours until teardown + time_zone: Timezone (default: UTC) + + Returns: + Dictionary with update result: + - success: bool - Whether update succeeded + - scheduled_time: datetime - New scheduled time + - cron_schedule: str - Cron expression + - error: Optional[str] - Error message if failed + """ + from deployml.api import update_teardown_schedule as _update_teardown_schedule + + project_id = self.provider.get('project_id') + region = self.provider.get('region') + workspace_name = self.name + + if not project_id or not region: + return { + "success": False, + "error": "Missing project_id or region in configuration" + } + + return _update_teardown_schedule( + project_id=project_id, + region=region, + workspace_name=workspace_name, + duration_hours=duration_hours, + deployml_dir=self.workspace_dir, + time_zone=time_zone + ) + + def cancel_teardown(self) -> Dict[str, Any]: + """ + Cancel scheduled teardown for this deployment stack. + + Returns: + Dictionary with cancellation result: + - success: bool - Whether cancellation succeeded + - error: Optional[str] - Error message if failed + """ + from deployml.api import cancel_teardown as _cancel_teardown + + project_id = self.provider.get('project_id') + region = self.provider.get('region') + workspace_name = self.name + + if not project_id or not region: + return { + "success": False, + "error": "Missing project_id or region in configuration" + } + + return _cancel_teardown( + project_id=project_id, + region=region, + workspace_name=workspace_name, + deployml_dir=self.workspace_dir + ) + + def show_teardown_status(self): + """Display teardown status in a formatted way""" + status = self.get_teardown_status() + + print("\n" + "="*80) + print("AUTO-TEARDOWN STATUS") + print("="*80) + + if status.get("error"): + print(f"❌ Error: {status['error']}") + print("="*80) + return + + if not status.get("exists"): + print("ℹ️ Auto-teardown is not scheduled for this workspace") + print("="*80) + return + + state_emoji = "✅" if status.get("state") == "ENABLED" else "⏸️" if status.get("state") == "PAUSED" else "❌" + print(f"{state_emoji} Status: {status.get('state', 'UNKNOWN')}") + print(f"📅 Cron Schedule: {status.get('schedule', 'N/A')}") + print(f"🌍 Timezone: {status.get('time_zone', 'UTC')}") + + schedule_time = status.get("schedule_time") + if schedule_time: + print(f"⏰ Next Execution: {schedule_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + + now = datetime.now(timezone.utc) + if now < schedule_time: + time_remaining = schedule_time - now + hours = int(time_remaining.total_seconds() // 3600) + minutes = int((time_remaining.total_seconds() % 3600) // 60) + print(f" ⏳ Time Remaining: {hours}h {minutes}m") + else: + time_passed = now - schedule_time + hours_passed = int(time_passed.total_seconds() // 3600) + minutes_passed = int((time_passed.total_seconds() % 3600) // 60) + print(f" ⚠️ Scheduled time passed {hours_passed}h {minutes_passed}m ago") + + last_attempt = status.get("last_attempt_time") + if last_attempt: + print(f"🕐 Last Execution: {last_attempt.strftime('%Y-%m-%d %H:%M:%S UTC')}") + else: + print("🕐 Last Execution: Never") + + print(f"\n📌 Job Name: {status.get('scheduler_job_name')}") + print("\n💡 Actions:") + print(f" Update: stack.update_teardown_schedule(duration_hours=6)") + print(f" Cancel: stack.cancel_teardown()") + print("="*80) \ No newline at end of file diff --git a/src/deployml/templates/gcp/cloud_run/main.tf.j2 b/src/deployml/templates/gcp/cloud_run/main.tf.j2 index b42e649..95165e4 100644 --- a/src/deployml/templates/gcp/cloud_run/main.tf.j2 +++ b/src/deployml/templates/gcp/cloud_run/main.tf.j2 @@ -62,13 +62,27 @@ module "teardown" { # Create Cloud SQL PostgreSQL instance if needed {% if flags.needs_postgres %} +{% set has_feast = false %} +{% set has_grafana = false %} +{% for stage in stack %} + {% for stage_name, tool in stage.items() %} + {% if stage_name == "feature_store" and tool.name == "feast" %} + {% set has_feast = true %} + {% endif %} + {% if stage_name == "model_monitoring" and tool.name == "grafana" %} + {% set has_grafana = true %} + {% endif %} + {% endfor %} +{% endfor %} module "cloud_sql_postgres" { - source = "./modules/{{ flags.first_tool_name }}/cloud/gcp/cloud_sql_postgres" + source = "./modules/cloud_sql_postgres" project_id = var.project_id region = var.region db_instance_name = "{{ flags.first_tool_name }}-postgres--{{ project_id }}" db_name = "{{ flags.first_tool_name }}" db_user = "{{ flags.first_tool_name }}" + create_feast_db = {{ "true" if has_feast else "false" }} + create_metrics_db = {{ "true" if has_grafana else "false" }} } {% endif %} diff --git a/src/deployml/terraform/modules/cloud_sql_postgres/main.tf b/src/deployml/terraform/modules/cloud_sql_postgres/main.tf index d8369e6..c213431 100644 --- a/src/deployml/terraform/modules/cloud_sql_postgres/main.tf +++ b/src/deployml/terraform/modules/cloud_sql_postgres/main.tf @@ -28,18 +28,68 @@ resource "google_sql_database_instance" "postgres" { deletion_protection = false } +# Wait for Cloud SQL instance to be fully ready before creating databases +# Cloud SQL instances can take 2-5 minutes to become fully operational +resource "time_sleep" "wait_for_instance" { + depends_on = [google_sql_database_instance.postgres] + create_duration = "180s" # Wait 3 minutes for instance to be fully ready +} + +# Additional check: Use a null_resource to verify instance is actually running +# This helps catch cases where the instance exists but is stopped +resource "null_resource" "verify_instance_running" { + depends_on = [time_sleep.wait_for_instance] + + provisioner "local-exec" { + command = <<-EOT + set +e + echo "Checking Cloud SQL instance status..." + if ! command -v gcloud &> /dev/null; then + echo "gcloud CLI not found, skipping status check (relying on time_sleep)" + exit 0 + fi + + for i in {1..30}; do + STATE=$(gcloud sql instances describe ${google_sql_database_instance.postgres.name} --project=${var.project_id} --format="value(state)" 2>/dev/null || echo "NOT_FOUND") + if [ "$STATE" = "RUNNABLE" ]; then + echo "✓ Instance is RUNNABLE, proceeding..." + exit 0 + elif [ "$STATE" = "STOPPED" ] || [ "$STATE" = "SUSPENDED" ]; then + echo "⚠ Instance is $STATE. Attempting to start..." + gcloud sql instances patch ${google_sql_database_instance.postgres.name} --project=${var.project_id} --activation-policy=ALWAYS 2>/dev/null || true + sleep 30 + elif [ "$STATE" = "NOT_FOUND" ]; then + echo "Instance not found yet, waiting... (attempt $i/30)" + sleep 10 + else + echo "Instance state: $STATE (attempt $i/30)" + sleep 10 + fi + done + echo "⚠ Warning: Could not verify instance is RUNNABLE, but proceeding..." + exit 0 # Don't fail the entire deployment if this check fails + EOT + } + + triggers = { + instance_name = google_sql_database_instance.postgres.name + instance_id = google_sql_database_instance.postgres.id + } +} + resource "google_sql_database" "db" { name = var.db_name instance = google_sql_database_instance.postgres.name project = var.project_id - depends_on = [google_sql_database_instance.postgres] + depends_on = [null_resource.verify_instance_running] } resource "google_sql_database" "feast_db" { + count = var.create_feast_db ? 1 : 0 name = "feast" instance = google_sql_database_instance.postgres.name project = var.project_id - depends_on = [google_sql_database_instance.postgres] + depends_on = [null_resource.verify_instance_running] lifecycle { ignore_changes = [name] @@ -47,10 +97,11 @@ resource "google_sql_database" "feast_db" { } resource "google_sql_database" "metrics_db" { + count = var.create_metrics_db ? 1 : 0 name = "metrics" instance = google_sql_database_instance.postgres.name project = var.project_id - depends_on = [google_sql_database_instance.postgres] + depends_on = [null_resource.verify_instance_running] lifecycle { ignore_changes = [name] @@ -62,7 +113,12 @@ resource "google_sql_user" "users" { instance = google_sql_database_instance.postgres.name password = random_password.db_password.result project = var.project_id - depends_on = [google_sql_database_instance.postgres] + depends_on = [null_resource.verify_instance_running] + + lifecycle { + # Retry on failure in case instance is still starting + create_before_destroy = true + } } resource "google_project_service" "required" { diff --git a/src/deployml/terraform/modules/cloud_sql_postgres/variables.tf b/src/deployml/terraform/modules/cloud_sql_postgres/variables.tf index 71d31bd..9e07a89 100644 --- a/src/deployml/terraform/modules/cloud_sql_postgres/variables.tf +++ b/src/deployml/terraform/modules/cloud_sql_postgres/variables.tf @@ -38,6 +38,18 @@ variable "max_connections" { default = "100" } +variable "create_feast_db" { + description = "Whether to create the Feast database" + type = bool + default = false +} + +variable "create_metrics_db" { + description = "Whether to create the Metrics database (for Grafana)" + type = bool + default = false +} + variable "gcp_service_list" { description = "The list of APIs necessary for MLflow with Cloud SQL" type = list(string) From 04105f2456482a99070ab285cc05d4b7556ba940 Mon Sep 17 00:00:00 2001 From: VenkatachalamSubramanianPeriyaSubbu Date: Sat, 6 Dec 2025 13:30:27 -0800 Subject: [PATCH 6/6] auto-teardown.md --- AUTO_TEARDOWN.md | 143 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 AUTO_TEARDOWN.md diff --git a/AUTO_TEARDOWN.md b/AUTO_TEARDOWN.md new file mode 100644 index 0000000..1e4f6d7 --- /dev/null +++ b/AUTO_TEARDOWN.md @@ -0,0 +1,143 @@ +# Auto-Teardown Demo + +This demo shows how to test the auto-teardown feature with a 10-minute duration. + +## Quick Start + +### 1. Update Configuration + +Edit `teardown-demo.yaml` and update: +- `project_id`: Your GCP project ID +- `region`: Your preferred GCP region + +### 2. Deploy with Auto-Teardown + +```bash +deployml deploy --config-path demo/teardown-demo.yaml +``` + +You should see output like: +``` +✅ Deployment complete! + +⏰ Auto-teardown scheduled for: 2025-01-15 14:10:00 UTC + (in 24 hours) + To cancel: deployml teardown cancel --config-path demo/teardown-demo.yaml +``` + +### 3. Check Teardown Status + +```bash +deployml teardown status --config-path demo/teardown-demo.yaml +``` + +### 4. Monitor Teardown + +**Watch the countdown:** +```bash +watch -n 60 "deployml teardown status --config-path demo/teardown-demo.yaml" +``` + +**Check Cloud Scheduler:** +```bash +gcloud scheduler jobs describe deployml-teardown-teardown-demo-stack \ + --location us-west1 \ + --project YOUR_PROJECT_ID +``` + +**Check Cloud Function logs:** +```bash +gcloud functions logs read deployml-teardown-teardown-demo-stack \ + --region us-west1 \ + --project YOUR_PROJECT_ID \ + --limit 50 +``` + +### 5. Verify Teardown Executed + +After 10 minutes, check: + +```bash +# Check if infrastructure still exists +gcloud run services list --project YOUR_PROJECT_ID + +# Check Cloud Function logs for teardown execution +gcloud functions logs read deployml-teardown-teardown-demo-stack \ + --region us-west1 \ + --project YOUR_PROJECT_ID \ + --limit 10 +``` + +### 6. Cancel Teardown (Optional) + +If you want to cancel before the scheduled time: + +```bash +deployml teardown cancel --config-path demo/teardown-demo.yaml +``` + +## What Happens + +1. **Deployment**: Infrastructure is deployed normally +2. **Scheduler Created**: Cloud Scheduler job is created to trigger teardown in 10 minutes +3. **Wait**: Wait 10 minutes +4. **Teardown**: Cloud Scheduler triggers Cloud Function +5. **Destroy**: Cloud Function runs `terraform destroy` +6. **Cleanup**: All infrastructure resources are destroyed + +## Configuration Details + +- **duration_hours**: `0.167` = 10 minutes (10/60 hours) +- **time_zone**: `UTC` (you can change this) +- **enabled**: `true` to enable auto-teardown + +## Troubleshooting + +### Teardown didn't execute + +1. Check Cloud Scheduler job status: + ```bash + gcloud scheduler jobs describe deployml-teardown-teardown-demo-stack \ + --location us-west1 \ + --project YOUR_PROJECT_ID + ``` + +2. Check Cloud Function logs: + ```bash + gcloud functions logs read deployml-teardown-teardown-demo-stack \ + --region us-west1 \ + --project YOUR_PROJECT_ID + ``` + +3. Manually trigger the function (for testing): + ```bash + # Get the function URL + FUNCTION_URL=$(gcloud functions describe deployml-teardown-teardown-demo-stack \ + --region us-west1 \ + --project YOUR_PROJECT_ID \ + --format="value(httpsTrigger.url)") + + # Trigger it manually + curl -X POST $FUNCTION_URL \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_name": "teardown-demo-stack", + "project_id": "YOUR_PROJECT_ID" + }' + ``` + +## Cleanup After Demo + +After the demo, clean up any remaining resources: + +```bash +# If teardown didn't execute automatically +deployml destroy --config-path demo/teardown-demo.yaml + +# Clean up the scheduler job if it still exists +gcloud scheduler jobs delete deployml-teardown-teardown-demo-stack \ + --location us-west1 \ + --project YOUR_PROJECT_ID \ + --quiet +``` +