From da56a5fa7ac92d57593805fe187c077faaca92e1 Mon Sep 17 00:00:00 2001 From: silversurfer Date: Tue, 16 Jun 2026 14:02:55 -0700 Subject: [PATCH 1/5] feat: add infracost v2 support with deployml estimate and deployml costs commands - Rewrites infracost.py for v2 CLI (scan + inspect --json, dropped deprecated breakdown command) - Adds deployml estimate: pre-deploy cost prediction from config, no GCP credentials needed - Adds deployml costs: cost check against live deployed Terraform workspace - Adds check_infracost_authenticated() with macOS + Linux credential path support - Fixes copy_modules_to_workspace to include cloud_sql_postgres when mlflow uses postgresql - Updates deployml doctor to show infracost install + auth status - Adds 14 unit tests for infracost.py - Updates docs/features/costs.md and tutorial with estimate/costs commands --- docs/features/costs.md | 62 +++-- docs/tutorials/gcp-cloud-run.md | 23 ++ src/deployml/cli/cli.py | 259 +++++++++++++++--- src/deployml/diagnostics/doctor.py | 26 +- src/deployml/utils/helpers.py | 10 + src/deployml/utils/infracost.py | 412 ++++++++++++----------------- tests/test_infracost.py | 169 ++++++++++++ 7 files changed, 646 insertions(+), 315 deletions(-) create mode 100644 tests/test_infracost.py diff --git a/docs/features/costs.md b/docs/features/costs.md index db732ec..8e2e704 100644 --- a/docs/features/costs.md +++ b/docs/features/costs.md @@ -1,46 +1,50 @@ # Cost Estimates -deployml integrates with [Infracost](https://github.com/infracost/infracost) to provide cost estimates before deploying your infrastructure, helping you manage cloud costs effectively in academic settings. - -## Overview - -Cost analysis runs automatically during deployment, showing monthly cost estimates for your entire stack, cost breakdowns by component, and warnings if costs exceed your configured threshold. The process analyzes your Terraform configuration before deployment, allowing you to adjust your configuration based on estimates. +deployml integrates with [Infracost](https://www.infracost.io) to show infrastructure costs. Both commands read your Terraform configuration — they price always-on resources like Cloud SQL accurately, but usage-based services (BigQuery, GCS, Cloud Run) show $0 since costs depend on actual usage. Check the GCP Billing Console for real usage charges. ## Setup -Install Infracost and register for a free API key using the instructions [here](https://www.infracost.io/docs/#quick-start). +```bash +brew install infracost +infracost auth login +``` -## Configuration +Run `deployml doctor` to confirm infracost is installed and authenticated. -Cost analysis is enabled by default. Configure it in your YAML file to enable or disable cost analysis, set a warning threshold in USD (default $100/month), and choose the currency for cost display. +## Commands -Here is an example of what this might look like: -```yaml -cost_analysis: - enabled: true # Enable/disable cost analysis (default: true) - warning_threshold: 50.0 # Warn if monthly cost exceeds this amount (default: 100.0) - currency: "USD" - bucket_amount: 200 # GB stored across GCS buckets - cloudsql_amount: 50 # GB of Cloud SQL storage +**Before deploying** — estimates cost from your config without touching any infrastructure: +```bash +deployml estimate ``` -## Typical Costs +**While deployed** — scans your actual deployed Terraform workspace: +```bash +deployml costs +``` -Here are estimated typical costs for several **GCP** services, but please do not simply believe these numbers without keeping track of costs yourself. +Both commands show a breakdown of which resources cost money and how much. -- Cloud Run services cost $10-30 per month depending on traffic. -- Cloud SQL PostgreSQL ranges from $7/month for small instances to $25+ for production. -- Google Cloud Storage costs approximately $0.020 per GB per month. -- BigQuery storage costs $0.020 per GB per month with query costs based on data scanned. -- Cloud VMs cost approximately $25 per month for medium instances. -- GKE clusters have no management fee, but you pay for VM instances and load balancers. Note that the GKE can get very expensive very quickly. +## Cost shown during deploy +`deployml deploy` automatically runs a cost estimate after `terraform plan` and shows it before the confirmation prompt: +``` + Deploy stack? Monthly cost: ~$34.55 USD [y/N]: +``` + +## Configuration + +```yaml +cost_analysis: + enabled: true # set to false to skip (default: true) + warning_threshold: 50.0 # warn if monthly cost exceeds this (default: 100.0) +``` +## Typical costs (Cloud Run stack) -## Cost Optimization +A standard MLflow + FastAPI + Grafana deployment runs around **$34/month**, almost entirely Cloud SQL. Cloud Run, BigQuery, and GCS scale to zero and cost nothing at idle. -Here are some tips to keep the costs low while you are learning: +## Keeping costs low -- Use SQLite instead of Cloud SQL whenever possible, particularly for development purposes and when your data is small. -- Enable auto-teardown to prevent forgotten deployments. -- Use Cloud Run for variable workloads to take advantage of scale-to-zero pricing. \ No newline at end of file +- Always `deployml destroy` when done — Cloud SQL bills continuously. +- Use `backend_store_uri: sqlite` instead of `postgresql` during development to eliminate Cloud SQL entirely. diff --git a/docs/tutorials/gcp-cloud-run.md b/docs/tutorials/gcp-cloud-run.md index 9bae181..57033d5 100644 --- a/docs/tutorials/gcp-cloud-run.md +++ b/docs/tutorials/gcp-cloud-run.md @@ -9,6 +9,11 @@ Make sure `deployml doctor` passes before starting. You will need: - `gcloud` CLI, authenticated (`gcloud auth login` and `gcloud auth application-default login`) - Docker (running) - Terraform +- Infracost (optional, for cost estimates): + ```bash + brew install infracost + infracost auth login + ``` ## 1. Create a GCP Project @@ -71,6 +76,16 @@ stack: - `model_serving` — deploys a FastAPI container that pulls the latest registered model from MLflow on startup - `model_monitoring` — deploys Grafana connected to the Postgres `metrics` database +## 3.5 Estimate Costs (Optional) + +Check what the stack will cost before committing to a 20-minute deploy: + +```bash +deployml estimate +``` + +No GCP credentials required, no infrastructure touched. A standard Cloud Run stack runs around **$34/month** — almost entirely Cloud SQL. See [Cost Estimates](../features/costs.md). + ## 4. Build Docker Images Build and push the service images to Artifact Registry: @@ -152,6 +167,14 @@ You should see `offline_features`, `predictions`, `ground_truth`, and `drift_met With the stack running, follow the [example walkthrough](example.md) to train a model, register it, serve predictions through FastAPI, and visualize drift metrics in Grafana. +## 8.5 Check Running Costs + +```bash +deployml costs +``` + +Shows what your deployed stack is currently costing. Cloud SQL is the main driver at ~$34/month — everything else scales to zero. + ## 9. Teardown When you are done, destroy all infrastructure to avoid ongoing charges: diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index fe9dc14..056cf9d 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -41,6 +41,7 @@ ) from deployml.utils.infracost import ( check_infracost_available, + check_infracost_authenticated, run_infracost_analysis, format_cost_for_confirmation, ) @@ -529,6 +530,13 @@ def doctor( # Infracost if infracost_installed: typer.secho("\n Infracost is installed", fg=typer.colors.GREEN) + if check_infracost_authenticated(): + typer.secho(" Infracost is authenticated", fg=typer.colors.GREEN) + else: + typer.secho( + " Infracost not authenticated — run: infracost auth login", + fg=typer.colors.YELLOW, + ) else: typer.secho( "\nWARNING: Infracost not installed (optional)", fg=typer.colors.YELLOW @@ -778,6 +786,212 @@ def terraform( output_dir = Path(output_dir) +@cli.command() +def estimate( + config_path: Path = typer.Option( + Path("config.yaml"), "--config-path", "-c", help="Path to YAML config file" + ), +): + """Estimate monthly infrastructure cost without deploying anything.""" + import tempfile + import shutil as _shutil + import hashlib as _hashlib + + if not check_infracost_available(): + typer.echo(" Infracost is not installed.") + typer.echo(" Install: https://www.infracost.io/docs/#quick-start") + raise typer.Exit(code=1) + + if not check_infracost_authenticated(): + typer.echo(" Infracost is not authenticated.") + typer.echo(" Run: infracost auth login") + typer.echo(" Or set: export INFRACOST_API_KEY=") + raise typer.Exit(code=1) + + if not config_path.exists(): + typer.echo(f" Config file not found: {config_path}") + raise typer.Exit(code=1) + + config = yaml.safe_load(config_path.read_text()) + cloud = config["provider"]["name"] + project_id = config["provider"]["project_id"] + region = config["provider"]["region"] + deployment_type = config["deployment"]["type"] + stack = config.get("stack", []) + workspace_name = config.get("name") or "development" + + if deployment_type == "gke": + typer.echo(" Cost estimation is not supported for GKE deployments.") + raise typer.Exit(code=1) + + teardown_config = config.get("teardown", {}) + teardown_enabled = teardown_config.get("enabled", False) + + # Auto-resolve image URIs so templates render with valid image paths + _TOOL_IMAGE_NAMES = { + "mlflow": "mlflow", + "feast": "feast", + "fastapi": "fastapi", + "grafana": "grafana-container", + "wandb": "wandb", + } + _ar_base = f"{region}-docker.pkg.dev/{project_id}/mlops-images" + for stage in stack: + for stage_name, tool in stage.items(): + tool_name = tool.get("name", "") + params = tool.setdefault("params", {}) + existing_image = params.get("image", "") + if not existing_image or existing_image.startswith("gcr.io/"): + image_name = _TOOL_IMAGE_NAMES.get(tool_name) + if image_name: + params["image"] = f"{_ar_base}/{image_name}:latest" + if stage_name == "workflow_orchestration" and tool_name == "cron": + for job in params.get("jobs", []): + if not job.get("image") or job.get("image", "").startswith("gcr.io/"): + job_name = job.get("service_name", "") + job["image"] = f"{_ar_base}/{job_name}:latest" + + # Build bucket_configs without GCS calls — estimate doesn't need live cloud state + bucket_configs = [] + for stage in stack: + for stage_name, tool in stage.items(): + if tool.get("params", {}).get("artifact_bucket"): + bucket_configs.append({ + "stage": stage_name, + "tool": tool["name"], + "bucket_name": tool["params"]["artifact_bucket"], + "create": tool["params"].get("create_artifact_bucket", True), + "exists": False, + }) + create_artifact_bucket = any(c["create"] for c in bucket_configs) + + name_material = f"{workspace_name}:{project_id}".encode("utf-8") + name_hash = _hashlib.sha1(name_material).hexdigest()[:6] + + warning_threshold = config.get("cost_analysis", {}).get("warning_threshold", 100.0) + + temp_dir = Path(tempfile.mkdtemp()) + modules_dir = temp_dir / "modules" + modules_dir.mkdir() + + try: + copy_modules_to_workspace( + modules_dir, + stack=stack, + deployment_type=deployment_type, + cloud=cloud, + teardown_enabled=teardown_enabled, + ) + + env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) + if deployment_type == "cloud_run": + if any(tool.get("name") == "wandb" for stage in stack for tool in stage.values()): + main_template = env.get_template(f"{cloud}/{deployment_type}/wandb_main.tf.j2") + elif any(tool.get("name") == "mlflow" for stage in stack for tool in stage.values()): + main_template = env.get_template(f"{cloud}/{deployment_type}/mlflow_main.tf.j2") + else: + main_template = env.get_template(f"{cloud}/{deployment_type}/main.tf.j2") + else: + main_template = env.get_template(f"{cloud}/{deployment_type}/main.tf.j2") + + var_template = env.get_template(f"{cloud}/{deployment_type}/variables.tf.j2") + tfvars_template = env.get_template(f"{cloud}/{deployment_type}/terraform.tfvars.j2") + + render_kwargs = dict( + cloud=cloud, + stack=stack, + deployment_type=deployment_type, + create_artifact_bucket=create_artifact_bucket, + bucket_configs=bucket_configs, + project_id=project_id, + stack_name=workspace_name, + name_hash=name_hash, + teardown_config=None, + teardown_cron_schedule="", + teardown_scheduled_timestamp=0, + ) + if deployment_type == "cloud_vm": + render_kwargs["region"] = region + render_kwargs["zone"] = config["provider"].get("zone", f"{region}-a") + + main_tf = main_template.render(**render_kwargs) + variables_tf = var_template.render( + stack=stack, + cloud=cloud, + project_id=project_id, + stack_name=workspace_name, + name_hash=name_hash, + ) + tfvars_content = tfvars_template.render( + project_id=project_id, + region=region, + zone=config["provider"].get("zone", f"{region}-a"), + stack=stack, + cloud=cloud, + create_artifact_bucket=create_artifact_bucket, + stack_name=workspace_name, + name_hash=name_hash, + ) + + (temp_dir / "main.tf").write_text(main_tf) + (temp_dir / "variables.tf").write_text(variables_tf) + (temp_dir / "terraform.tfvars").write_text(tfvars_content) + + typer.echo(f" Estimating cost for: {workspace_name}") + analysis = run_infracost_analysis(temp_dir, warning_threshold, show_resources=True) + + if analysis is None: + typer.secho(" Cost estimate unavailable.", fg=typer.colors.YELLOW) + raise typer.Exit(code=1) + + except typer.Exit: + raise + except Exception as e: + typer.echo(f" Estimate failed: {e}") + raise typer.Exit(code=1) + finally: + _shutil.rmtree(temp_dir, ignore_errors=True) + + +@cli.command() +def costs( + config_path: Path = typer.Option( + Path("config.yaml"), "--config-path", "-c", help="Path to YAML config file" + ), +): + """Show the monthly cost of your currently running deployment.""" + if not check_infracost_available(): + typer.echo(" Infracost is not installed.") + typer.echo(" Install: https://www.infracost.io/docs/#quick-start") + raise typer.Exit(code=1) + + if not check_infracost_authenticated(): + typer.echo(" Infracost is not authenticated.") + typer.echo(" Run: infracost auth login") + raise typer.Exit(code=1) + + if not config_path.exists(): + typer.echo(f" Config file not found: {config_path}") + raise typer.Exit(code=1) + + config = yaml.safe_load(config_path.read_text()) + workspace_name = config.get("name") or "development" + terraform_dir = Path.cwd() / ".deployml" / workspace_name / "terraform" + + if not terraform_dir.exists(): + typer.echo(f" No deployment found at {terraform_dir}") + typer.echo(" Run 'deployml deploy' to deploy, or 'deployml estimate' for a pre-deploy cost prediction.") + raise typer.Exit(code=1) + + warning_threshold = config.get("cost_analysis", {}).get("warning_threshold", 100.0) + typer.echo(f" Checking costs for running deployment: {workspace_name}") + analysis = run_infracost_analysis(terraform_dir, warning_threshold, show_resources=True) + + if analysis is None: + typer.secho(" Cost check unavailable.", fg=typer.colors.YELLOW) + raise typer.Exit(code=1) + + @cli.command() def deploy( config_path: Path = typer.Option( @@ -1210,50 +1424,7 @@ def deploy( cost_analysis = None if cost_enabled: - usage_file_path = cost_config.get("usage_file") - usage_file = Path(usage_file_path) if usage_file_path else None - - # If no explicit usage file provided, generate one from high-level YAML values - if usage_file is None: - try: - bucket_amount = cost_config.get("bucket_amount") - cloudsql_amount = cost_config.get( - "cloudSQL_amount" - ) or cost_config.get("cloudsql_amount") - bigquery_amount = cost_config.get( - "bigQuery_amount" - ) or cost_config.get("bigquery_amount") - - resource_type_default_usage = {} - # Map high-level amounts to Infracost resource defaults - if bucket_amount is not None: - resource_type_default_usage["google_storage_bucket"] = { - "storage_gb": float(bucket_amount) - } - if cloudsql_amount is not None: - resource_type_default_usage[ - "google_sql_database_instance" - ] = {"storage_gb": float(cloudsql_amount)} - if bigquery_amount is not None: - resource_type_default_usage["google_bigquery_table"] = { - "storage_gb": float(bigquery_amount) - } - - if resource_type_default_usage: - usage_yaml = { - "version": "0.1", - "resource_type_default_usage": resource_type_default_usage, - } - usage_file = DEPLOYML_TERRAFORM_DIR / "infracost-usage.yml" - with open(usage_file, "w") as f: - yaml.safe_dump(usage_yaml, f, sort_keys=False) - except Exception: - # If usage-file generation fails, continue without it - usage_file = None - - cost_analysis = run_infracost_analysis( - DEPLOYML_TERRAFORM_DIR, warning_threshold, usage_file=usage_file - ) + cost_analysis = run_infracost_analysis(DEPLOYML_TERRAFORM_DIR, warning_threshold) # Format confirmation message with cost information if cost_analysis: diff --git a/src/deployml/diagnostics/doctor.py b/src/deployml/diagnostics/doctor.py index 791ff97..9c36a38 100644 --- a/src/deployml/diagnostics/doctor.py +++ b/src/deployml/diagnostics/doctor.py @@ -87,7 +87,8 @@ def run_all_checks(self) -> List[CheckResult]: # Development tools self._check_git() self._check_infracost() - + self._check_infracost_authenticated() + # Permissions and access self._check_docker_permissions() self._check_cloud_authentication() @@ -388,7 +389,28 @@ def _check_infracost(self): message="Installed but version check failed", required=False )) - + + def _check_infracost_authenticated(self): + """Check if infracost is authenticated via credentials file or API key env var""" + if not shutil.which("infracost"): + return # _check_infracost already reported not-installed + from deployml.utils.infracost import check_infracost_authenticated + if check_infracost_authenticated(): + self._add_result(CheckResult( + name="Infracost Auth", + status=CheckStatus.PASS, + message="Infracost is authenticated", + required=False + )) + else: + self._add_result(CheckResult( + name="Infracost Auth", + status=CheckStatus.WARNING, + message="Infracost not authenticated — cost analysis will be skipped", + fix_command="infracost auth login OR export INFRACOST_API_KEY=", + required=False + )) + def _check_docker_permissions(self): """Check Docker permissions""" try: diff --git a/src/deployml/utils/helpers.py b/src/deployml/utils/helpers.py index 97e8e39..e005ae3 100644 --- a/src/deployml/utils/helpers.py +++ b/src/deployml/utils/helpers.py @@ -102,6 +102,16 @@ def copy_modules_to_workspace( # BigQuery is always included — provides the mlops dataset and tables used_modules.add("bigquery") + # cloud_sql_postgres is needed when mlflow or feast uses a postgresql backend. + # This mirrors the template's flags.needs_postgres logic so the module source + # reference in the rendered main.tf can always be resolved. + for stage in stack: + for stage_name, tool in stage.items(): + if tool.get("name") in ("mlflow", "feast"): + backend = tool.get("params", {}).get("backend_store_uri", "") + if backend.startswith("postgresql"): + used_modules.add("cloud_sql_postgres") + # Only copy the modules that are being used, and only the specific deployment type for module_path in MODULE_TEMPLATES_DIR.iterdir(): if module_path.is_dir() and module_path.name in used_modules: diff --git a/src/deployml/utils/infracost.py b/src/deployml/utils/infracost.py index 5418ef2..7bb675a 100644 --- a/src/deployml/utils/infracost.py +++ b/src/deployml/utils/infracost.py @@ -1,52 +1,50 @@ import json +import os import subprocess import typer from pathlib import Path -from typing import Dict, Optional, List -from dataclasses import dataclass - - -@dataclass -class CostComponent: - """Represents a single cost component for a resource""" - - name: str - unit: str - monthly_cost: float - hourly_cost: float - usage_based: bool = False - - -@dataclass -class ResourceCost: - """Represents cost information for a single resource""" - - name: str - resource_type: str - monthly_cost: float - hourly_cost: float - components: List[CostComponent] +from typing import Dict, List, Optional, Tuple +from dataclasses import dataclass, field + + +_RESOURCE_TYPE_LABELS = { + "google_sql_database_instance": "Cloud SQL", + "google_cloud_run_service": "Cloud Run", + "google_cloud_run_v2_service": "Cloud Run", + "google_storage_bucket": "GCS Bucket", + "google_bigquery_dataset": "BigQuery", + "google_bigquery_table": "BigQuery", + "google_compute_instance": "Compute Engine VM", + "google_container_cluster": "GKE Cluster", + "google_redis_instance": "Cloud Memorystore", + "google_pubsub_topic": "Pub/Sub", + "google_pubsub_subscription": "Pub/Sub", +} + + +def _resource_label(address: str) -> str: + """Extract a human-readable label from a terraform resource address.""" + # address looks like: module.cloud_sql_postgres.google_sql_database_instance.postgres + parts = address.split(".") + for part in parts: + if part in _RESOURCE_TYPE_LABELS: + return _RESOURCE_TYPE_LABELS[part] + if part.startswith("google_"): + return part # fall back to raw type name + return address @dataclass class CostAnalysis: - """Represents the complete cost analysis results""" - total_monthly_cost: float - total_hourly_cost: float currency: str - resources: List[ResourceCost] - detected_resources: int - supported_resources: int + resources: int + costed_resources: int + free_resources: int + resource_costs: List[Tuple[str, float]] = field(default_factory=list) def check_infracost_available() -> bool: - """ - Check if infracost CLI is available in the system PATH. - - Returns: - bool: True if infracost is available, False otherwise. - """ try: result = subprocess.run( ["infracost", "--version"], @@ -55,269 +53,203 @@ def check_infracost_available() -> bool: timeout=10, ) return result.returncode == 0 - except ( - subprocess.TimeoutExpired, - FileNotFoundError, - subprocess.SubprocessError, - ): + except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): return False -def run_infracost_breakdown(terraform_dir: Path, usage_file: Optional[Path] = None) -> Optional[Dict]: - """ - Run infracost breakdown analysis on the terraform directory. +def check_infracost_authenticated() -> bool: + if os.environ.get("INFRACOST_API_KEY"): + return True + # v2 on macOS stores token here; v1 / Linux uses ~/.config/infracost/credentials.yml + macos_token = Path.home() / "Library" / "Application Support" / "infracost" / "token.json" + linux_creds = Path.home() / ".config" / "infracost" / "credentials.yml" + return macos_token.exists() or linux_creds.exists() - Args: - terraform_dir: Path to the terraform directory - Returns: - Dict containing the infracost JSON output, or None if failed +def run_infracost_scan(terraform_dir: Path) -> Optional[Dict]: """ - if not check_infracost_available(): - return None + Run infracost v2 scan and return parsed JSON. + Uses `infracost scan --json`. Falls back to flag-before-subcommand + form if needed, since --json is a global flag in v2. + Does not require terraform init — infracost v2 parses HCL directly. + """ + cmds = [ + ["infracost", "scan", str(terraform_dir), "--json"], + ["infracost", "--json", "scan", str(terraform_dir)], + ] + for cmd in cmds: + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode == 0: + return json.loads(result.stdout) + if "unknown flag" not in result.stderr: + typer.echo(f"Infracost scan failed: {result.stderr}") + return None + # unknown flag → try next form + except subprocess.TimeoutExpired: + typer.echo("Infracost scan timed out") + return None + except json.JSONDecodeError: + typer.echo("Failed to parse infracost JSON output") + return None + except (FileNotFoundError, subprocess.SubprocessError) as e: + typer.echo(f"Infracost error: {e}") + return None + typer.echo("Infracost scan failed: could not find working --json flag form") + return None - try: - # Run infracost breakdown and capture JSON output - cmd = [ - "infracost", - "breakdown", - "--path", - str(terraform_dir), - "--format", - "json", - ] - if usage_file is not None: - cmd.extend(["--usage-file", str(usage_file)]) +def fetch_resource_costs() -> List[Tuple[str, float]]: + """ + Run `infracost inspect --group-by resource --json` against the last cached + scan and return a list of (label, monthly_cost) for non-zero cost resources. + """ + try: result = subprocess.run( - cmd, - cwd=terraform_dir, + ["infracost", "inspect", "--group-by", "resource", "--json"], capture_output=True, text=True, - timeout=60, + timeout=30, ) - - if result.returncode == 0: - return json.loads(result.stdout) - else: - typer.echo(f"⚠️ Infracost analysis failed: {result.stderr}") - return None - - except subprocess.TimeoutExpired: - typer.echo("⚠️ Infracost analysis timed out") - return None - except json.JSONDecodeError: - typer.echo("⚠️ Failed to parse infracost output") - return None - except Exception as e: - typer.echo(f"⚠️ Infracost error: {e}") - return None - - -def parse_infracost_data(data: Dict) -> Optional[CostAnalysis]: + if result.returncode != 0: + return [] + rows = json.loads(result.stdout) + costs = [] + seen_labels = set() + for row in rows: + cost = float(row.get("cost", 0) or 0) + if cost <= 0: + continue + address = row.get("columns", {}).get("resource", "") + label = _resource_label(address) + if label in seen_labels: + # Accumulate duplicate resource types (e.g. multiple Cloud Run services) + for i, (lbl, c) in enumerate(costs): + if lbl == label: + costs[i] = (lbl, c + cost) + break + else: + seen_labels.add(label) + costs.append((label, cost)) + return sorted(costs, key=lambda x: x[1], reverse=True) + except Exception: + return [] + + +def parse_infracost_scan_data(data: Dict) -> Optional[CostAnalysis]: """ - Parse infracost JSON data into structured cost analysis. - - Args: - data: Raw infracost JSON data + Parse v2 infracost JSON into a CostAnalysis. - Returns: - CostAnalysis object or None if parsing failed + Two schemas exist depending on how infracost is invoked: + - `infracost scan --json`: fields under a top-level "summary" key, + cost field is "total_monthly_cost" + - `infracost inspect --json`: fields at top level, cost field is "monthly_cost" """ try: - resources = [] - - for project in data.get("projects", []): - breakdown = project.get("breakdown", {}) - - for resource_data in breakdown.get("resources", []): - components = [] - - # Parse cost components - for comp_data in resource_data.get("costComponents", []): - monthly_cost = comp_data.get("monthlyCost") - hourly_cost = comp_data.get("hourlyCost") - - # Skip components without cost data - if monthly_cost is None and hourly_cost is None: - continue - - component = CostComponent( - name=comp_data.get("name", ""), - unit=comp_data.get("unit", ""), - monthly_cost=float(monthly_cost or 0), - hourly_cost=float(hourly_cost or 0), - usage_based=comp_data.get("usageBased", False), - ) - components.append(component) - - resource = ResourceCost( - name=resource_data.get("name", ""), - resource_type=resource_data.get("resourceType", ""), - monthly_cost=float(resource_data.get("monthlyCost", 0)), - hourly_cost=float(resource_data.get("hourlyCost", 0)), - components=components, - ) - resources.append(resource) - + if "summary" in data: + # infracost scan --json format + summary = data["summary"] + monthly_cost_str = summary.get("total_monthly_cost", "0") or "0" + return CostAnalysis( + total_monthly_cost=float(monthly_cost_str), + currency=data.get("currency", "USD"), + resources=int(summary.get("resources", 0)), + costed_resources=int(summary.get("costed_resources", 0)), + free_resources=int(summary.get("free_resources", 0)), + ) + # infracost inspect --json format + monthly_cost_str = data.get("monthly_cost", "0") or "0" return CostAnalysis( - total_monthly_cost=float(data.get("totalMonthlyCost", 0)), - total_hourly_cost=float(data.get("totalHourlyCost", 0)), + total_monthly_cost=float(monthly_cost_str), currency=data.get("currency", "USD"), - resources=resources, - detected_resources=data.get("summary", {}).get( - "totalDetectedResources", 0 - ), - supported_resources=data.get("summary", {}).get( - "totalSupportedResources", 0 - ), + resources=int(data.get("resources", 0)), + costed_resources=int(data.get("costed_resources", 0)), + free_resources=int(data.get("free_resources", 0)), ) - - except Exception as e: - typer.echo(f"⚠️ Failed to parse cost data: {e}") + except (ValueError, TypeError) as e: + typer.echo(f"Failed to parse cost data: {e}") return None def display_cost_breakdown( - analysis: CostAnalysis, warning_threshold: float = 100.0 + analysis: CostAnalysis, + warning_threshold: float = 100.0, + show_resources: bool = False, ) -> None: - """ - Display a user-friendly cost breakdown. - - Args: - analysis: CostAnalysis object with cost data - warning_threshold: Monthly cost threshold for warnings (default: $100) - """ typer.echo("\n" + "=" * 60) - typer.secho("💰 COST ANALYSIS", fg=typer.colors.BRIGHT_CYAN, bold=True) + typer.secho("COST ANALYSIS", fg=typer.colors.BRIGHT_CYAN, bold=True) typer.echo("=" * 60) - # Overall costs monthly_cost = analysis.total_monthly_cost + color = typer.colors.BRIGHT_GREEN if monthly_cost < warning_threshold else typer.colors.BRIGHT_YELLOW typer.secho( f"Monthly Cost: ${monthly_cost:.2f} {analysis.currency}", - fg=( - typer.colors.BRIGHT_GREEN - if monthly_cost < warning_threshold - else typer.colors.BRIGHT_YELLOW - ), + fg=color, bold=True, ) typer.echo( - f"Hourly Cost: ${analysis.total_hourly_cost:.4f} {analysis.currency}" - ) - typer.echo( - f"Resources: {analysis.supported_resources} supported, {analysis.detected_resources} total" + f"Resources: {analysis.costed_resources} costed, " + f"{analysis.free_resources} free, {analysis.resources} total" ) - # Warning for high costs + if show_resources and analysis.resource_costs: + typer.echo("\nWhat costs money:") + for label, cost in analysis.resource_costs: + typer.secho(f" ${cost:.2f} {label}", fg=typer.colors.BRIGHT_BLUE) + typer.echo( + "\nNote: Cloud Run, BigQuery, and GCS show $0 at idle — they scale to\n" + "zero and charge only for actual usage." + ) + if monthly_cost > warning_threshold: typer.echo() typer.secho( - f"⚠️ WARNING: Monthly cost exceeds ${warning_threshold:.0f} threshold!", + f"WARNING: Monthly cost exceeds ${warning_threshold:.0f} threshold!", fg=typer.colors.BRIGHT_RED, bold=True, ) - - # Resource breakdown - if analysis.resources: - typer.echo("\n📋 Resource Breakdown:") - typer.echo("-" * 40) - - # Sort resources by monthly cost (highest first) - sorted_resources = sorted( - analysis.resources, key=lambda r: r.monthly_cost, reverse=True - ) - - for resource in sorted_resources: - if resource.monthly_cost > 0: - typer.echo(f"\n• {resource.name}") - typer.echo(f" Type: {resource.resource_type}") - typer.secho( - f" Monthly Cost: ${resource.monthly_cost:.2f}", - fg=typer.colors.BRIGHT_BLUE, - ) - - # Show top cost components - if resource.components: - for component in resource.components[ - :3 - ]: # Show top 3 components - if component.monthly_cost > 0: - usage_note = ( - " (usage-based)" - if component.usage_based - else "" - ) - typer.echo( - f" └─ {component.name}: ${component.monthly_cost:.2f}{usage_note}" - ) - - # Usage-based resources note - usage_based_resources = [ - r - for r in analysis.resources - if any(c.usage_based for c in r.components) - ] - - if usage_based_resources: - typer.echo("\n📊 Note: Some resources have usage-based pricing") - typer.echo(" Actual costs may vary based on usage patterns") - typer.echo() def format_cost_for_confirmation(monthly_cost: float, currency: str) -> str: - """ - Format cost information for the deployment confirmation prompt. - - Args: - monthly_cost: Monthly cost in the specified currency - currency: Currency code (e.g., 'USD') - - Returns: - Formatted string for confirmation prompt - """ if monthly_cost > 0: - return f"💰 Monthly cost: ~${monthly_cost:.2f} {currency}" + return f"Monthly cost: ~${monthly_cost:.2f} {currency}" else: - return "💰 Monthly cost: Variable (usage-based pricing)" + return "Monthly cost: Variable (usage-based pricing)" def run_infracost_analysis( - terraform_dir: Path, warning_threshold: float = 100.0, usage_file: Optional[Path] = None + terraform_dir: Path, + warning_threshold: float = 100.0, + show_resources: bool = False, ) -> Optional[CostAnalysis]: - """ - Run complete infracost analysis workflow. - - Args: - terraform_dir: Path to terraform directory - warning_threshold: Cost warning threshold - - Returns: - CostAnalysis object or None if analysis failed - """ - # Check if infracost is available if not check_infracost_available(): - typer.echo( - "💡 Tip: Install infracost CLI for cost analysis before deployment" - ) + typer.echo("Tip: Install infracost CLI for cost analysis before deployment") typer.echo(" Visit: https://www.infracost.io/docs/#quick-start") return None - typer.echo("💰 Running cost analysis...") + if not check_infracost_authenticated(): + typer.echo("Tip: Authenticate infracost to enable cost analysis") + typer.echo(" Run: infracost auth login") + typer.echo(" Or set: export INFRACOST_API_KEY=") + return None - # Run infracost breakdown - raw_data = run_infracost_breakdown(terraform_dir, usage_file=usage_file) - if not raw_data: + typer.echo("Running cost analysis...") + raw_data = run_infracost_scan(terraform_dir) + if raw_data is None: return None - # Parse the data - analysis = parse_infracost_data(raw_data) - if not analysis: + analysis = parse_infracost_scan_data(raw_data) + if analysis is None: return None - # Display the breakdown - display_cost_breakdown(analysis, warning_threshold) + if show_resources: + analysis.resource_costs = fetch_resource_costs() + display_cost_breakdown(analysis, warning_threshold, show_resources=show_resources) return analysis diff --git a/tests/test_infracost.py b/tests/test_infracost.py new file mode 100644 index 0000000..26d9227 --- /dev/null +++ b/tests/test_infracost.py @@ -0,0 +1,169 @@ +"""Unit tests for src/deployml/utils/infracost.py (v2 rewrite). + +Covers: + 1. check_infracost_available — OS-boundary mock (subprocess) + 2. check_infracost_authenticated — file-system + env var mocks + 3. parse_infracost_scan_data — pure parsing, no I/O + 4. format_cost_for_confirmation — pure function +""" + +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch, patch as _patch + +from deployml.utils.infracost import ( + CostAnalysis, + check_infracost_available, + check_infracost_authenticated, + format_cost_for_confirmation, + parse_infracost_scan_data, +) + + +# --------------------------------------------------------------------------- +# check_infracost_available +# --------------------------------------------------------------------------- + +def test_check_infracost_available_returns_true(): + with patch("deployml.utils.infracost.subprocess.run") as mock_run: + mock_run.return_value = SimpleNamespace(returncode=0, stdout="infracost v2.4.2", stderr="") + assert check_infracost_available() is True + + +def test_check_infracost_available_returns_false_when_not_installed(): + with patch("deployml.utils.infracost.subprocess.run", side_effect=FileNotFoundError): + assert check_infracost_available() is False + + +def test_check_infracost_available_returns_false_on_nonzero_returncode(): + with patch("deployml.utils.infracost.subprocess.run") as mock_run: + mock_run.return_value = SimpleNamespace(returncode=1, stdout="", stderr="error") + assert check_infracost_available() is False + + +# --------------------------------------------------------------------------- +# check_infracost_authenticated +# --------------------------------------------------------------------------- + +def test_check_infracost_authenticated_returns_true_with_credentials_file(tmp_path): + creds_file = tmp_path / ".config" / "infracost" / "credentials.yml" + creds_file.parent.mkdir(parents=True) + creds_file.write_text("api_key: test123\n") + with patch("deployml.utils.infracost.Path.home", return_value=tmp_path): + # INFRACOST_API_KEY must be absent so we exercise the file-path branch + env_without_key = {k: v for k, v in os.environ.items() if k != "INFRACOST_API_KEY"} + with patch.dict(os.environ, env_without_key, clear=True): + assert check_infracost_authenticated() is True + + +def test_check_infracost_authenticated_returns_true_with_env_var(tmp_path): + with patch("deployml.utils.infracost.Path.home", return_value=tmp_path): + with patch.dict(os.environ, {"INFRACOST_API_KEY": "ic-test-key-abc"}): + assert check_infracost_authenticated() is True + + +def test_check_infracost_authenticated_returns_false_when_neither(tmp_path): + # tmp_path has no credentials.yml and INFRACOST_API_KEY is cleared + env_without_key = {k: v for k, v in os.environ.items() if k != "INFRACOST_API_KEY"} + with patch.dict(os.environ, env_without_key, clear=True): + with patch("deployml.utils.infracost.Path.home", return_value=tmp_path): + assert check_infracost_authenticated() is False + + +# --------------------------------------------------------------------------- +# parse_infracost_scan_data (pure parsing, no subprocess) +# --------------------------------------------------------------------------- + +# infracost inspect --json format (fields at top level) +_INSPECT_JSON = { + "projects": 1, + "resources": 6, + "costed_resources": 1, + "free_resources": 5, + "monthly_cost": "26.4591683", + "currency": "USD", + "project_details": [], + "failing_policy_list": [], +} + +# infracost scan --json format (fields nested under "summary") +_SCAN_JSON = { + "currency": "USD", + "summary": { + "projects": 1, + "resources": 71, + "costed_resources": 10, + "free_resources": 61, + "total_monthly_cost": "34.55", + }, + "projects": [], +} + + +def test_parse_infracost_scan_data_inspect_format_extracts_all_fields(): + result = parse_infracost_scan_data(_INSPECT_JSON) + assert result is not None + assert abs(result.total_monthly_cost - 26.4591683) < 0.0001 + assert result.currency == "USD" + assert result.resources == 6 + assert result.costed_resources == 1 + assert result.free_resources == 5 + + +def test_parse_infracost_scan_data_scan_format_extracts_all_fields(): + # infracost scan --json wraps fields under "summary" with "total_monthly_cost" + result = parse_infracost_scan_data(_SCAN_JSON) + assert result is not None + assert abs(result.total_monthly_cost - 34.55) < 0.001 + assert result.currency == "USD" + assert result.resources == 71 + assert result.costed_resources == 10 + assert result.free_resources == 61 + + +def test_parse_infracost_scan_data_handles_empty_dict(): + result = parse_infracost_scan_data({}) + assert result is not None + assert result.total_monthly_cost == 0.0 + assert result.currency == "USD" + assert result.resources == 0 + assert result.costed_resources == 0 + assert result.free_resources == 0 + + +def test_parse_infracost_scan_data_handles_null_monthly_cost(): + data = {**_INSPECT_JSON, "monthly_cost": None} + result = parse_infracost_scan_data(data) + assert result is not None + assert result.total_monthly_cost == 0.0 + + +def test_parse_infracost_scan_data_handles_invalid_cost_string(): + # float("not-a-number") raises ValueError → function should return None + data = {**_INSPECT_JSON, "monthly_cost": "not-a-number"} + result = parse_infracost_scan_data(data) + assert result is None + + +def test_parse_infracost_scan_data_handles_zero_cost(): + data = {**_INSPECT_JSON, "monthly_cost": "0", "costed_resources": 0, "free_resources": 6} + result = parse_infracost_scan_data(data) + assert result is not None + assert result.total_monthly_cost == 0.0 + assert result.costed_resources == 0 + + +# --------------------------------------------------------------------------- +# format_cost_for_confirmation (pure function) +# --------------------------------------------------------------------------- + +def test_format_cost_for_confirmation_nonzero_cost(): + result = format_cost_for_confirmation(26.46, "USD") + assert "$26.46" in result + assert "USD" in result + + +def test_format_cost_for_confirmation_zero_cost(): + result = format_cost_for_confirmation(0.0, "USD") + assert "Variable" in result or "usage-based" in result.lower() From ed65724b5df2f6706042ec4dda67c10e4263096a Mon Sep 17 00:00:00 2001 From: silversurfer Date: Tue, 14 Jul 2026 14:58:29 -0700 Subject: [PATCH 2/5] feat: make deployml estimate usage-aware with light/heavy profiles Infracost defaults usage-based services (Cloud Run, BigQuery, GCS) to zero usage, so the previous estimate only surfaced the always-on Cloud SQL cost and showed $0 for everything else -- misleading for students. The estimate command now feeds infracost a realistic usage profile (--profile light|heavy, default light) and renders a two-bucket view: ALWAYS-ON (fixed 24/7) vs USAGE-BASED (scales with activity), each line labelled in plain English, plus a 'biggest lever' call-out. - add usage_profiles.py: LIGHT/HEAVY dicts + render_usage_yaml() - infracost.py: ResourceCost dataclass, run_infracost_scan_with_usage(), fetch_resource_costs_detailed(), display_estimate(), run_estimate_analysis(); pin 'infracost inspect --file ' instead of the global scan cache - cli.py: estimate gains --profile and calls the new flow - tests: 22 passing --- src/deployml/cli/cli.py | 14 +- src/deployml/utils/infracost.py | 358 +++++++++++++++++++++++++-- src/deployml/utils/usage_profiles.py | 81 ++++++ tests/test_infracost.py | 139 ++++++++++- 4 files changed, 565 insertions(+), 27 deletions(-) create mode 100644 src/deployml/utils/usage_profiles.py diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index 056cf9d..3b2c36d 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -43,6 +43,7 @@ check_infracost_available, check_infracost_authenticated, run_infracost_analysis, + run_estimate_analysis, format_cost_for_confirmation, ) from deployml.utils.teardown import ( @@ -791,12 +792,21 @@ def estimate( config_path: Path = typer.Option( Path("config.yaml"), "--config-path", "-c", help="Path to YAML config file" ), + profile: str = typer.Option( + "light", "--profile", "-p", + help="Usage profile for the estimate: 'light' (typical student demo) or 'heavy'.", + ), ): """Estimate monthly infrastructure cost without deploying anything.""" import tempfile import shutil as _shutil import hashlib as _hashlib + profile = profile.lower() + if profile not in ("light", "heavy"): + typer.echo(f" Unknown profile '{profile}'. Use 'light' or 'heavy'.") + raise typer.Exit(code=1) + if not check_infracost_available(): typer.echo(" Infracost is not installed.") typer.echo(" Install: https://www.infracost.io/docs/#quick-start") @@ -938,7 +948,9 @@ def estimate( (temp_dir / "terraform.tfvars").write_text(tfvars_content) typer.echo(f" Estimating cost for: {workspace_name}") - analysis = run_infracost_analysis(temp_dir, warning_threshold, show_resources=True) + analysis = run_estimate_analysis( + temp_dir, profile_name=profile, warning_threshold=warning_threshold + ) if analysis is None: typer.secho(" Cost estimate unavailable.", fg=typer.colors.YELLOW) diff --git a/src/deployml/utils/infracost.py b/src/deployml/utils/infracost.py index 7bb675a..a64d513 100644 --- a/src/deployml/utils/infracost.py +++ b/src/deployml/utils/infracost.py @@ -1,7 +1,10 @@ import json import os +import shutil import subprocess +import tempfile import typer +from collections import OrderedDict from pathlib import Path from typing import Dict, List, Optional, Tuple from dataclasses import dataclass, field @@ -34,6 +37,63 @@ def _resource_label(address: str) -> str: return address +# Resource types that bill 24/7 whether or not the stack is used. Anything with a +# cost that is NOT in this set is treated as usage-based (scales with activity). +_FIXED_TYPES = { + "google_sql_database_instance", # Cloud SQL — always-on Postgres + "google_redis_instance", # Memorystore — always-on + "google_container_cluster", # GKE control plane — always-on + "google_compute_instance", # VM — always-on +} + +# Plain-English "what it is" text. Module-prefix descriptions are more specific +# (they name the tool), so they win over the generic per-type fallback below. +_MODULE_DESCRIPTIONS = { + "cloud_sql_postgres": "MLflow's backend database", + "experiment_tracking_mlflow": "MLflow tracking server", + "artifact_tracking": "MLflow model artifacts", + "model_serving_fastapi": "FastAPI model server", + "model_monitoring_grafana": "Grafana dashboards", + "bigquery": "prediction logging & analytics", +} +_TYPE_DESCRIPTIONS = { + "google_sql_database_instance": "always-on Postgres database", + "google_cloud_run_service": "serverless container", + "google_cloud_run_v2_service": "serverless container", + "google_storage_bucket": "object / artifact storage", + "google_bigquery_dataset": "analytics queries", + "google_bigquery_table": "analytics storage", + "google_compute_instance": "virtual machine", + "google_container_cluster": "Kubernetes control plane", + "google_redis_instance": "in-memory cache", +} + + +def _resource_type_from_address(address: str) -> str: + """Pull the google_* resource type out of a terraform address.""" + for part in address.split("."): + if part.startswith("google_"): + return part + return "" + + +def _classify_category(resource_type: str) -> str: + """'fixed' if the resource bills 24/7, else 'usage'.""" + return "fixed" if resource_type in _FIXED_TYPES else "usage" + + +def _resource_description(address: str, resource_type: str) -> str: + """Prefer a tool-specific description from the module name, else per-type text.""" + parts = address.split(".") + if len(parts) >= 2 and parts[0] == "module": + # strip the count index, e.g. model_serving_fastapi[0] -> model_serving_fastapi + module_name = parts[1].split("[")[0] + desc = _MODULE_DESCRIPTIONS.get(module_name) + if desc: + return desc + return _TYPE_DESCRIPTIONS.get(resource_type, resource_type or "resource") + + @dataclass class CostAnalysis: total_monthly_cost: float @@ -44,6 +104,32 @@ class CostAnalysis: resource_costs: List[Tuple[str, float]] = field(default_factory=list) +@dataclass +class ResourceCost: + """A single costed resource, classified and labelled for the estimate view.""" + address: str # module.cloud_sql_postgres.google_sql_database_instance.postgres + resource_type: str # google_sql_database_instance + monthly_cost: float + category: str # "fixed" | "usage" + label: str # "Cloud SQL" + description: str # "MLflow's backend database" + + +def _row_to_resource_cost(row: Dict) -> ResourceCost: + """Map one `infracost inspect --group-by resource` row to a ResourceCost.""" + cost = float(row.get("cost", 0) or 0) + address = row.get("columns", {}).get("resource", "") + resource_type = _resource_type_from_address(address) + return ResourceCost( + address=address, + resource_type=resource_type, + monthly_cost=cost, + category=_classify_category(resource_type), + label=_resource_label(address), + description=_resource_description(address, resource_type), + ) + + def check_infracost_available() -> bool: try: result = subprocess.run( @@ -104,43 +190,129 @@ def run_infracost_scan(terraform_dir: Path) -> Optional[Dict]: return None -def fetch_resource_costs() -> List[Tuple[str, float]]: +def run_infracost_scan_with_usage( + terraform_dir: Path, usage_profile: Dict +) -> Optional[Dict]: + """ + Run infracost v2 with a usage profile applied and return parsed JSON. + + v2 has no --usage-file flag; usage must be supplied via an auto-discovered + infracost.yml. That config route also does NOT auto-load terraform.tfvars and + requires paths relative to the config file (absolute paths return 0 + resources). So we: + 1. copy the rendered terraform into /tf + 2. write /infracost-usage.yml and /infracost.yml + (relative `path: tf`, usage_file, terraform_var_files: [terraform.tfvars]) + 3. run `infracost scan --json` with cwd = config_dir + """ + from deployml.utils.usage_profiles import render_usage_yaml + + config_dir = Path(tempfile.mkdtemp()) + try: + shutil.copytree(terraform_dir, config_dir / "tf") + (config_dir / "infracost-usage.yml").write_text( + render_usage_yaml(usage_profile) + ) + (config_dir / "infracost.yml").write_text( + "version: 0.1\n" + "projects:\n" + " - path: tf\n" + " usage_file: infracost-usage.yml\n" + " terraform_var_files: [terraform.tfvars]\n" + ) + result = subprocess.run( + ["infracost", "scan", "--json"], + cwd=config_dir, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode != 0: + typer.echo(f"Infracost scan failed: {result.stderr}") + return None + return json.loads(result.stdout) + except subprocess.TimeoutExpired: + typer.echo("Infracost scan timed out") + return None + except json.JSONDecodeError: + typer.echo("Failed to parse infracost JSON output") + return None + except (FileNotFoundError, subprocess.SubprocessError, OSError) as e: + typer.echo(f"Infracost error: {e}") + return None + finally: + shutil.rmtree(config_dir, ignore_errors=True) + + +def _run_inspect_rows(scan_json_path: Path) -> List[Dict]: """ - Run `infracost inspect --group-by resource --json` against the last cached - scan and return a list of (label, monthly_cost) for non-zero cost resources. + Run `infracost inspect --file --group-by resource --json` and + return the raw rows (or [] on any failure). + + Passing --file pins inspect to the exact scan we just produced. Without it, + inspect reads infracost's global "most recent scan" cache, so a prior or + concurrent scan of a different workspace could be reported here instead. """ try: result = subprocess.run( - ["infracost", "inspect", "--group-by", "resource", "--json"], + [ + "infracost", + "inspect", + "--file", + str(scan_json_path), + "--group-by", + "resource", + "--json", + ], capture_output=True, text=True, timeout=30, ) if result.returncode != 0: return [] - rows = json.loads(result.stdout) - costs = [] - seen_labels = set() - for row in rows: - cost = float(row.get("cost", 0) or 0) - if cost <= 0: - continue - address = row.get("columns", {}).get("resource", "") - label = _resource_label(address) - if label in seen_labels: - # Accumulate duplicate resource types (e.g. multiple Cloud Run services) - for i, (lbl, c) in enumerate(costs): - if lbl == label: - costs[i] = (lbl, c + cost) - break - else: - seen_labels.add(label) - costs.append((label, cost)) - return sorted(costs, key=lambda x: x[1], reverse=True) - except Exception: + return json.loads(result.stdout) + except (subprocess.SubprocessError, json.JSONDecodeError, ValueError, OSError): return [] +def fetch_resource_costs(scan_json_path: Path) -> List[Tuple[str, float]]: + """ + Return a list of (label, monthly_cost) for non-zero cost resources, grouped + by GCP service label. Used by the simple deploy/costs breakdown view. + """ + costs: List[Tuple[str, float]] = [] + seen_labels = set() + for row in _run_inspect_rows(scan_json_path): + cost = float(row.get("cost", 0) or 0) + if cost <= 0: + continue + address = row.get("columns", {}).get("resource", "") + label = _resource_label(address) + if label in seen_labels: + # Accumulate duplicate resource types (e.g. multiple Cloud Run services) + for i, (lbl, c) in enumerate(costs): + if lbl == label: + costs[i] = (lbl, c + cost) + break + else: + seen_labels.add(label) + costs.append((label, cost)) + return sorted(costs, key=lambda x: x[1], reverse=True) + + +def fetch_resource_costs_detailed(scan_json_path: Path) -> List[ResourceCost]: + """ + Return per-resource ResourceCost records (non-zero cost only), each classified + as fixed vs usage and labelled in plain English. Drives the estimate view. + """ + resources = [ + _row_to_resource_cost(row) + for row in _run_inspect_rows(scan_json_path) + ] + resources = [r for r in resources if r.monthly_cost > 0] + return sorted(resources, key=lambda r: r.monthly_cost, reverse=True) + + def parse_infracost_scan_data(data: Dict) -> Optional[CostAnalysis]: """ Parse v2 infracost JSON into a CostAnalysis. @@ -249,7 +421,143 @@ def run_infracost_analysis( return None if show_resources: - analysis.resource_costs = fetch_resource_costs() + # Write the scan result to a temp file and inspect it explicitly, rather + # than letting infracost read its global "most recent scan" cache. + scan_dir = Path(tempfile.mkdtemp()) + scan_json_path = scan_dir / "infracost-scan.json" + try: + scan_json_path.write_text(json.dumps(raw_data)) + analysis.resource_costs = fetch_resource_costs(scan_json_path) + finally: + shutil.rmtree(scan_dir, ignore_errors=True) display_cost_breakdown(analysis, warning_threshold, show_resources=show_resources) return analysis + + +def display_estimate( + resources: List[ResourceCost], + analysis: CostAnalysis, + profile_name: str, + warning_threshold: float = 100.0, +) -> None: + """ + Render the estimate view: a headline split into fixed vs usage cost, the two + buckets with plain-English labels, and a call-out for the biggest cost lever. + """ + fixed = [r for r in resources if r.category == "fixed"] + usage = [r for r in resources if r.category == "usage"] + fixed_total = sum(r.monthly_cost for r in fixed) + usage_total = sum(r.monthly_cost for r in usage) + total = analysis.total_monthly_cost + currency = analysis.currency + + typer.echo("\n" + "=" * 60) + typer.secho(" MONTHLY COST", fg=typer.colors.BRIGHT_CYAN, bold=True) + typer.echo("=" * 60) + headline_color = ( + typer.colors.BRIGHT_GREEN if total < warning_threshold + else typer.colors.BRIGHT_YELLOW + ) + typer.secho( + f" ~${total:,.0f} / month " + f"(${fixed_total:,.2f} fixed + ~${usage_total:,.2f} usage) {currency}", + fg=headline_color, + bold=True, + ) + + if fixed: + typer.echo() + typer.secho(" ALWAYS-ON (billed 24/7 even if you never use the stack)", bold=True) + for r in fixed: + typer.secho( + f" ${r.monthly_cost:>8.2f} {r.label:<12} {r.description}", + fg=typer.colors.BRIGHT_BLUE, + ) + + if usage: + typer.echo() + typer.secho( + f" USAGE-BASED (scales with activity · profile: {profile_name})", bold=True + ) + # Collapse identical rows (e.g. the four BigQuery tables) into one line. + grouped: "OrderedDict[Tuple[str, str], Tuple[float, int]]" = OrderedDict() + for r in usage: + key = (r.label, r.description) + cost, count = grouped.get(key, (0.0, 0)) + grouped[key] = (cost + r.monthly_cost, count + 1) + for (label, description), (cost, count) in grouped.items(): + suffix = f" (x{count})" if count > 1 else "" + typer.secho( + f" ${cost:>8.2f} {label:<12} {description}{suffix}", + fg=typer.colors.BRIGHT_BLUE, + ) + + # Biggest lever: if one always-on resource dominates, tell the student. + if fixed and total > 0: + top = max(fixed, key=lambda r: r.monthly_cost) + share = top.monthly_cost / total + if share > 0.5: + typer.echo() + typer.secho( + f" Biggest lever: {top.label} is {share * 100:.0f}% of your cost and runs 24/7.", + fg=typer.colors.BRIGHT_MAGENTA, + bold=True, + ) + if top.resource_type == "google_sql_database_instance": + typer.secho( + " Switch MLflow to a SQLite backend to drop this to ~$0/month.", + fg=typer.colors.BRIGHT_MAGENTA, + ) + + omitted = max(analysis.resources - len(resources), 0) + if omitted: + typer.echo() + typer.echo(f" {omitted} free / API-enablement resources omitted (no cost).") + + typer.echo( + "\n Note: usage costs are estimated at this profile's load. Heavier use\n" + " raises the usage line, not the fixed baseline." + ) + + if total > warning_threshold: + typer.echo() + typer.secho( + f" WARNING: exceeds ${warning_threshold:.0f}/month threshold!", + fg=typer.colors.BRIGHT_RED, + bold=True, + ) + typer.echo() + + +def run_estimate_analysis( + terraform_dir: Path, + profile_name: str = "light", + warning_threshold: float = 100.0, +) -> Optional[CostAnalysis]: + """ + Scan `terraform_dir` with a usage profile applied, then render the estimate + view. Assumes infracost availability/auth has already been checked by the + caller. Returns the CostAnalysis, or None on failure. + """ + from deployml.utils.usage_profiles import get_profile + + typer.echo(f"Running cost estimate (usage profile: {profile_name})...") + raw_data = run_infracost_scan_with_usage(terraform_dir, get_profile(profile_name)) + if raw_data is None: + return None + + analysis = parse_infracost_scan_data(raw_data) + if analysis is None: + return None + + scan_dir = Path(tempfile.mkdtemp()) + scan_json_path = scan_dir / "infracost-scan.json" + try: + scan_json_path.write_text(json.dumps(raw_data)) + resources = fetch_resource_costs_detailed(scan_json_path) + finally: + shutil.rmtree(scan_dir, ignore_errors=True) + + display_estimate(resources, analysis, profile_name, warning_threshold) + return analysis diff --git a/src/deployml/utils/usage_profiles.py b/src/deployml/utils/usage_profiles.py new file mode 100644 index 0000000..ec3332e --- /dev/null +++ b/src/deployml/utils/usage_profiles.py @@ -0,0 +1,81 @@ +""" +Usage profiles for cost estimation. + +Infracost prices usage-based resources (Cloud Run, BigQuery, GCS) at *zero usage* +by default, so `deployml estimate` would show $0 for everything except the +always-on Cloud SQL instance. These profiles feed infracost a realistic +assumption of how much a student actually uses the stack, so the estimate +reflects a real monthly bill. + +Field names below are the exact keys infracost expects for each resource type +(see infracost's usage-file schema). To tune assumptions, edit the numbers here. +""" + +# A typical student demo: clicking around the UIs + running the example serving +# script a few times over a month. Small storage, light query volume. +LIGHT = { + "google_cloud_run_service": { + "monthly_requests": 50000, + "average_request_duration_ms": 300, + "concurrent_requests_per_instance": 10, + }, + "google_storage_bucket": { + "storage_gb": 5, + "monthly_class_a_operations": 20000, + "monthly_class_b_operations": 100000, + "monthly_data_retrieval_gb": 5, + }, + "google_bigquery_dataset": { + "monthly_queries_tb": 0.05, # ~50 GB scanned / month + }, + "google_bigquery_table": { + "monthly_active_storage_gb": 1, + "monthly_streaming_inserts_mb": 200, + }, +} + +# A heavier course project / small team: more traffic, more data, more queries. +# Roughly 10-20x light — used to show students a rough ceiling, not a hard limit. +HEAVY = { + "google_cloud_run_service": { + "monthly_requests": 1000000, + "average_request_duration_ms": 400, + "concurrent_requests_per_instance": 20, + }, + "google_storage_bucket": { + "storage_gb": 100, + "monthly_class_a_operations": 500000, + "monthly_class_b_operations": 2000000, + "monthly_data_retrieval_gb": 100, + }, + "google_bigquery_dataset": { + "monthly_queries_tb": 1.0, # ~1 TB scanned / month + }, + "google_bigquery_table": { + "monthly_active_storage_gb": 25, + "monthly_streaming_inserts_mb": 5000, + }, +} + +PROFILES = {"light": LIGHT, "heavy": HEAVY} + + +def get_profile(name: str) -> dict: + """Return a usage profile by name, defaulting to light for unknown names.""" + return PROFILES.get(name, LIGHT) + + +def render_usage_yaml(profile: dict) -> str: + """ + Render a usage profile into an infracost-usage.yml document. + + Uses `resource_type_default_usage`, which applies the same assumptions to + every resource of a given type (e.g. all three Cloud Run services), which is + exactly what a coarse light/heavy profile wants. + """ + lines = ["version: 0.1", "resource_type_default_usage:"] + for resource_type, fields in profile.items(): + lines.append(f" {resource_type}:") + for key, value in fields.items(): + lines.append(f" {key}: {value}") + return "\n".join(lines) + "\n" diff --git a/tests/test_infracost.py b/tests/test_infracost.py index 26d9227..9429476 100644 --- a/tests/test_infracost.py +++ b/tests/test_infracost.py @@ -7,18 +7,28 @@ 4. format_cost_for_confirmation — pure function """ +import json import os from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch, patch as _patch +from unittest.mock import patch + +import yaml from deployml.utils.infracost import ( CostAnalysis, + ResourceCost, + _classify_category, + _row_to_resource_cost, check_infracost_available, check_infracost_authenticated, + display_estimate, + fetch_resource_costs, + fetch_resource_costs_detailed, format_cost_for_confirmation, parse_infracost_scan_data, ) +from deployml.utils.usage_profiles import LIGHT, render_usage_yaml # --------------------------------------------------------------------------- @@ -154,6 +164,39 @@ def test_parse_infracost_scan_data_handles_zero_cost(): assert result.costed_resources == 0 +# --------------------------------------------------------------------------- +# fetch_resource_costs (argv construction + row parsing) +# --------------------------------------------------------------------------- + +def test_fetch_resource_costs_passes_file_flag_to_inspect(tmp_path): + # Guards against regressing to the global-cache form of `infracost inspect`: + # inspect must be pinned to the scan JSON we pass, via --file. + scan_json = tmp_path / "infracost-scan.json" + scan_json.write_text("{}") + with patch("deployml.utils.infracost.subprocess.run") as mock_run: + mock_run.return_value = SimpleNamespace(returncode=0, stdout="[]", stderr="") + fetch_resource_costs(scan_json) + argv = mock_run.call_args[0][0] + assert argv[:2] == ["infracost", "inspect"] + assert "--file" in argv + assert argv[argv.index("--file") + 1] == str(scan_json) + + +def test_fetch_resource_costs_accumulates_duplicate_resource_types(tmp_path): + # Two Cloud Run services should collapse into one row with summed cost. + rows = [ + {"cost": "5.00", "columns": {"resource": "module.a.google_cloud_run_v2_service.x"}}, + {"cost": "3.00", "columns": {"resource": "module.b.google_cloud_run_v2_service.y"}}, + {"cost": "0", "columns": {"resource": "module.c.google_storage_bucket.z"}}, + ] + with patch("deployml.utils.infracost.subprocess.run") as mock_run: + mock_run.return_value = SimpleNamespace( + returncode=0, stdout=json.dumps(rows), stderr="" + ) + result = fetch_resource_costs(tmp_path / "scan.json") + assert result == [("Cloud Run", 8.0)] # zero-cost bucket dropped + + # --------------------------------------------------------------------------- # format_cost_for_confirmation (pure function) # --------------------------------------------------------------------------- @@ -167,3 +210,97 @@ def test_format_cost_for_confirmation_nonzero_cost(): def test_format_cost_for_confirmation_zero_cost(): result = format_cost_for_confirmation(0.0, "USD") assert "Variable" in result or "usage-based" in result.lower() + + +# --------------------------------------------------------------------------- +# usage profiles (render_usage_yaml) +# --------------------------------------------------------------------------- + +def test_render_usage_yaml_is_valid_and_has_profile_keys(): + text = render_usage_yaml(LIGHT) + data = yaml.safe_load(text) # must parse as valid YAML + assert data["version"] == 0.1 + defaults = data["resource_type_default_usage"] + assert "google_cloud_run_service" in defaults + assert defaults["google_bigquery_dataset"]["monthly_queries_tb"] == 0.05 + + +# --------------------------------------------------------------------------- +# classification (_classify_category / _row_to_resource_cost) +# --------------------------------------------------------------------------- + +def test_classify_category_fixed_vs_usage(): + assert _classify_category("google_sql_database_instance") == "fixed" + for usage_type in ( + "google_cloud_run_service", + "google_bigquery_dataset", + "google_storage_bucket", + ): + assert _classify_category(usage_type) == "usage" + + +def test_row_to_resource_cost_maps_and_labels(): + row = { + "cost": "34.55", + "columns": {"resource": "module.cloud_sql_postgres.google_sql_database_instance.postgres"}, + } + rc = _row_to_resource_cost(row) + assert rc.resource_type == "google_sql_database_instance" + assert rc.category == "fixed" + assert rc.label == "Cloud SQL" + assert rc.description == "MLflow's backend database" # module-specific wins + assert abs(rc.monthly_cost - 34.55) < 0.001 + + +# --------------------------------------------------------------------------- +# fetch_resource_costs_detailed (filter zero + sort + classify) +# --------------------------------------------------------------------------- + +def test_fetch_resource_costs_detailed_filters_zero_and_sorts(tmp_path): + rows = [ + {"cost": "0.08", "columns": {"resource": "module.model_serving_fastapi[0].google_cloud_run_service.fastapi"}}, + {"cost": "34.55", "columns": {"resource": "module.cloud_sql_postgres.google_sql_database_instance.pg"}}, + {"cost": "0", "columns": {"resource": "module.bigquery.google_bigquery_table.predictions"}}, + ] + with patch("deployml.utils.infracost._run_inspect_rows", return_value=rows): + result = fetch_resource_costs_detailed(tmp_path / "scan.json") + # zero-cost row dropped, remaining sorted by cost descending + assert [round(r.monthly_cost, 2) for r in result] == [34.55, 0.08] + assert result[0].category == "fixed" + assert result[1].category == "usage" + assert result[1].description == "FastAPI model server" + + +# --------------------------------------------------------------------------- +# display_estimate (lever logic — capture printed output) +# --------------------------------------------------------------------------- + +def _rc(addr, rtype, cost, category, label, desc): + return ResourceCost(addr, rtype, cost, category, label, desc) + + +def test_display_estimate_fires_lever_when_fixed_dominates(capsys): + resources = [ + _rc("module.cloud_sql_postgres.google_sql_database_instance.pg", + "google_sql_database_instance", 34.55, "fixed", "Cloud SQL", "MLflow's backend database"), + _rc("module.bigquery.google_bigquery_dataset.mlops", + "google_bigquery_dataset", 0.31, "usage", "BigQuery", "prediction logging & analytics"), + ] + analysis = CostAnalysis(34.86, "USD", 71, 10, 61) + display_estimate(resources, analysis, "light") + out = capsys.readouterr().out + assert "ALWAYS-ON" in out and "USAGE-BASED" in out + assert "Biggest lever" in out + assert "SQLite" in out + + +def test_display_estimate_no_lever_when_balanced(capsys): + resources = [ + _rc("m.google_sql_database_instance.a", "google_sql_database_instance", 10.0, "fixed", "Cloud SQL", "db"), + _rc("m.google_compute_instance.b", "google_compute_instance", 10.0, "fixed", "Compute Engine VM", "vm"), + _rc("m.google_storage_bucket.c", "google_storage_bucket", 5.0, "usage", "GCS Bucket", "storage"), + ] + analysis = CostAnalysis(25.0, "USD", 30, 3, 27) + display_estimate(resources, analysis, "light") + out = capsys.readouterr().out + assert "Biggest lever" not in out # top fixed is 40% of total, below the 50% cutoff From bd420376617034c292652443e8a53250d56708fa Mon Sep 17 00:00:00 2001 From: silversurfer Date: Sat, 12 Sep 2026 19:36:30 -0700 Subject: [PATCH 3/5] feat: price deployml costs from real GCP usage `deployml costs` re-ran the zero-usage infracost scan against the deployed terraform, so it reported the same misleading numbers the estimate redesign existed to fix: Cloud Run, BigQuery and GCS at $0, only Cloud SQL real. It answered "what would this cost if nobody used it", which nobody is asking. It now measures what the student actually did. New utils/measured_usage.py reads Cloud Monitoring over a window (--days, default 30), maps each metric onto the infracost usage key it corresponds to, and keys it to the terraform address via `terraform show -json`, so per-service usage prices per service instead of charging every Cloud Run service the whole stack's traffic. Deliberately no billing integration: GCP has no API for "my spend so far", and the BigQuery billing export needs billing-account admin plus a ~24h lag, which is out of reach for a student. Measuring usage needs only monitoring.viewer. Design rule is that nothing is invented. The scan gets an empty profile and measured per-address usage only, so anything Monitoring could not report stays at zero and the reason prints under "About these numbers". A silent $0 is the bug being fixed; an explained $0 is fine. The three approximations (GCS class A/B inferred from method names, project-scoped BigQuery bytes split evenly, Cloud Run concurrency left at infracost's default) are disclosed the same way. Measured usage also retires the guessed profiles: classify_usage_profile compares real request volume against light/heavy on a log scale, so those stop being pricing inputs and become a label ("you are a LIGHT user"). Also folded in, since they are the same code path: - deploy's confirmation prompt was built from the zero-usage scan, i.e. it asked students to approve a number we know is wrong. It now uses the same usage-aware estimate, which let run_infracost_scan, run_infracost_analysis, display_cost_breakdown and fetch_resource_costs be deleted as duplicates. - check_infracost_authenticated can only prove a token file exists, so an expired token passed pre-flight and failed at scan time with raw JSON stderr. The failure is now recognised and answered with "infracost auth login". Verified against a real expired token. - the `bigquery` module description won for both the dataset and the tables, so storage rows were labelled as query cost. Split into per-type descriptions. - google_cloud_run_v2_job (teardown, offline_scoring, explainability) was in neither label map and printed as a raw google_* type name. - a failed `infracost inspect` returned [] silently, rendering as "nothing costs money" under a non-zero total. It now says so. - costs defaulted to workspace "development" while deploy/destroy/get-urls use "default", so it could look in a directory deploy never wrote. - costs and estimate bypassed _load_config_or_exit, turning a missing key into " Estimate failed: 'provider'". Metric and aligner pairs were validated against the live Monitoring API. database/up is GAUGE/INT64 rather than BOOL, so ALIGN_FRACTION_TRUE is rejected with HTTP 400; ALIGN_MEAN over a 0/1 gauge is the same uptime fraction and is pinned by a test. Tests: 85 -> 106, all passing. --- config.example.yaml | 5 + src/deployml/cli/cli.py | 94 +++- src/deployml/utils/constants.py | 2 + src/deployml/utils/infracost.py | 429 ++++++++++++------ src/deployml/utils/measured_usage.py | 651 +++++++++++++++++++++++++++ src/deployml/utils/usage_profiles.py | 72 ++- tests/test_infracost.py | 179 +++++++- tests/test_measured_usage.py | 328 ++++++++++++++ 8 files changed, 1577 insertions(+), 183 deletions(-) create mode 100644 src/deployml/utils/measured_usage.py create mode 100644 tests/test_measured_usage.py diff --git a/config.example.yaml b/config.example.yaml index 67caa00..e77b320 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -27,3 +27,8 @@ stack: name: grafana params: service_name: grafana-server + +# Optional. Controls the cost commands and the estimate shown before a deploy. +# cost_analysis: +# enabled: true # show a cost estimate in the deploy confirmation +# warning_threshold: 100.0 # warn in USD/month above this figure diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index 028fdc2..607d50e 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -46,8 +46,8 @@ from deployml.utils.infracost import ( check_infracost_available, check_infracost_authenticated, - run_infracost_analysis, run_estimate_analysis, + run_actual_cost_analysis, format_cost_for_confirmation, ) from deployml.utils.teardown import ( @@ -973,8 +973,24 @@ def costs( config_path: Path = typer.Option( Path("config.yaml"), "--config-path", "-c", help="Path to YAML config file" ), + days: int = typer.Option( + 30, "--days", "-d", + help="How many days of real usage to measure (1-90).", + ), ): - """Show the monthly cost of your currently running deployment.""" + """ + Show what your running deployment actually costs, from real GCP usage. + + Where `deployml estimate` prices a stack that does not exist yet and has to + assume how hard you will use it, this reads your real usage out of Cloud + Monitoring -- requests served, bytes stored, bytes scanned -- and prices + that. Use it to see whether the estimate was right and where your money is + actually going. + """ + if days < 1 or days > 90: + typer.echo(" --days must be between 1 and 90.") + raise typer.Exit(code=1) + if not check_infracost_available(): typer.echo(" Infracost is not installed.") typer.echo(" Install: https://www.infracost.io/docs/#quick-start") @@ -989,18 +1005,67 @@ def costs( typer.echo(f" Config file not found: {config_path}") raise typer.Exit(code=1) - config = yaml.safe_load(config_path.read_text()) - workspace_name = config.get("name") or "development" - terraform_dir = Path.cwd() / ".deployml" / workspace_name / "terraform" + config = _load_config_or_exit(config_path) + + try: + project_id = config["provider"]["project_id"] + except (KeyError, TypeError): + typer.echo(" Config is missing provider.project_id.") + raise typer.Exit(code=1) + + # Measured usage comes from the Monitoring REST API, which authenticates with + # Application Default Credentials rather than the gcloud user login. + if not check_gcp_adc(): + typer.echo(" Application Default Credentials are not configured.") + typer.echo(" Run: gcloud auth application-default login") + raise typer.Exit(code=1) + + # Matches the workspace name deploy/destroy/get-urls use. This previously + # defaulted to "development", so costs looked in a directory deploy never wrote. + workspace_name = config.get("name") or "default" + deployml_dir = Path.cwd() / ".deployml" / workspace_name + terraform_dir = deployml_dir / "terraform" if not terraform_dir.exists(): typer.echo(f" No deployment found at {terraform_dir}") typer.echo(" Run 'deployml deploy' to deploy, or 'deployml estimate' for a pre-deploy cost prediction.") raise typer.Exit(code=1) + # How long the stack has existed, so we can project what it has cost so far. + # Only written when auto-teardown is enabled, so treat it as optional. + from datetime import datetime as _datetime, timezone as _timezone + + deployed_days = None + metadata = load_deployment_metadata(deployml_dir) + if metadata and metadata.get("deployed_at"): + try: + deployed_at = _datetime.fromisoformat(metadata["deployed_at"]) + if deployed_at.tzinfo is None: + deployed_at = deployed_at.replace(tzinfo=_timezone.utc) + elapsed = _datetime.now(_timezone.utc) - deployed_at + deployed_days = max(elapsed.total_seconds() / 86400.0, 0.0) + except (ValueError, TypeError): + deployed_days = None + + # Measuring a window longer than the deployment has existed would divide real + # usage across days that never happened and understate the run-rate. + if deployed_days is not None and 0 < deployed_days < days: + typer.secho( + f" Deployment is only {deployed_days:.1f} days old; " + f"measuring that window instead of {days} days.", + fg=typer.colors.YELLOW, + ) + days = max(deployed_days, 1.0 / 24.0) + warning_threshold = config.get("cost_analysis", {}).get("warning_threshold", 100.0) - typer.echo(f" Checking costs for running deployment: {workspace_name}") - analysis = run_infracost_analysis(terraform_dir, warning_threshold, show_resources=True) + analysis = run_actual_cost_analysis( + terraform_dir, + project_id=project_id, + workspace_name=workspace_name, + days=days, + warning_threshold=warning_threshold, + deployed_days=deployed_days, + ) if analysis is None: typer.secho(" Cost check unavailable.", fg=typer.colors.YELLOW) @@ -1489,7 +1554,20 @@ def deploy( cost_analysis = None if cost_enabled: - cost_analysis = run_infracost_analysis(DEPLOYML_TERRAFORM_DIR, warning_threshold) + # Infracost is optional, so a missing or unauthenticated install skips the + # estimate rather than blocking the deploy. + if check_infracost_available() and check_infracost_authenticated(): + cost_analysis = run_estimate_analysis( + DEPLOYML_TERRAFORM_DIR, + profile_name="light", + warning_threshold=warning_threshold, + ) + else: + typer.secho( + " Skipping cost estimate (infracost not installed or not authenticated).", + fg=typer.colors.YELLOW, + ) + typer.echo(" Install: https://www.infracost.io/docs/#quick-start") # Format confirmation message with cost information if cost_analysis: diff --git a/src/deployml/utils/constants.py b/src/deployml/utils/constants.py index 643d4f1..d916951 100644 --- a/src/deployml/utils/constants.py +++ b/src/deployml/utils/constants.py @@ -63,4 +63,6 @@ "roles/bigquery.admin", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", + # deployml costs reads real usage from Cloud Monitoring to price the deployment. + "roles/monitoring.viewer", ] \ No newline at end of file diff --git a/src/deployml/utils/infracost.py b/src/deployml/utils/infracost.py index a9c3a83..9ac4278 100644 --- a/src/deployml/utils/infracost.py +++ b/src/deployml/utils/infracost.py @@ -6,16 +6,20 @@ import typer from collections import OrderedDict from pathlib import Path -from typing import Dict, List, Optional, Tuple -from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from dataclasses import dataclass from deployml.utils.platform_compat import run_tool +if TYPE_CHECKING: # imported for typing only; avoids a runtime import cycle + from deployml.utils.measured_usage import MeasuredUsage + _RESOURCE_TYPE_LABELS = { "google_sql_database_instance": "Cloud SQL", "google_cloud_run_service": "Cloud Run", "google_cloud_run_v2_service": "Cloud Run", + "google_cloud_run_v2_job": "Cloud Run Job", "google_storage_bucket": "GCS Bucket", "google_bigquery_dataset": "BigQuery", "google_bigquery_table": "BigQuery", @@ -56,15 +60,15 @@ def _resource_label(address: str) -> str: "artifact_tracking": "MLflow model artifacts", "model_serving_fastapi": "FastAPI model server", "model_monitoring_grafana": "Grafana dashboards", - "bigquery": "prediction logging & analytics", } _TYPE_DESCRIPTIONS = { "google_sql_database_instance": "always-on Postgres database", "google_cloud_run_service": "serverless container", "google_cloud_run_v2_service": "serverless container", + "google_cloud_run_v2_job": "scheduled batch job", "google_storage_bucket": "object / artifact storage", - "google_bigquery_dataset": "analytics queries", - "google_bigquery_table": "analytics storage", + "google_bigquery_dataset": "prediction & feature queries", + "google_bigquery_table": "prediction & feature storage", "google_compute_instance": "virtual machine", "google_container_cluster": "Kubernetes control plane", "google_redis_instance": "in-memory cache", @@ -103,7 +107,6 @@ class CostAnalysis: resources: int costed_resources: int free_resources: int - resource_costs: List[Tuple[str, float]] = field(default_factory=list) @dataclass @@ -154,46 +157,37 @@ def check_infracost_authenticated() -> bool: return macos_token.exists() or linux_creds.exists() -def run_infracost_scan(terraform_dir: Path) -> Optional[Dict]: +# Substrings infracost uses when the problem is credentials rather than terraform. +_AUTH_ERROR_HINTS = ( + "unauthorized", "unauthenticated", "invalid api key", "invalid token", + "expired", "log in", "login", "not authenticated", "401", "403", +) + + +def _report_scan_failure(stderr: str) -> None: """ - Run infracost v2 scan and return parsed JSON. - Uses `infracost scan --json`. Falls back to flag-before-subcommand - form if needed, since --json is a global flag in v2. - Does not require terraform init — infracost v2 parses HCL directly. + Explain a failed scan, and name the fix when it is an auth problem. + + check_infracost_authenticated() can only see whether a token file exists, so + an expired token passes the pre-flight check and fails here instead. Rather + than dump a raw stderr at the student, recognise the auth case and tell them + what to run. """ - cmds = [ - ["infracost", "scan", str(terraform_dir), "--json"], - ["infracost", "--json", "scan", str(terraform_dir)], - ] - for cmd in cmds: - try: - result = run_tool( - cmd[0], cmd[1:], - capture_output=True, - text=True, - timeout=120, - ) - if result.returncode == 0: - return json.loads(result.stdout) - if "unknown flag" not in result.stderr: - typer.echo(f"Infracost scan failed: {result.stderr}") - return None - # unknown flag → try next form - except subprocess.TimeoutExpired: - typer.echo("Infracost scan timed out") - return None - except json.JSONDecodeError: - typer.echo("Failed to parse infracost JSON output") - return None - except (FileNotFoundError, subprocess.SubprocessError) as e: - typer.echo(f"Infracost error: {e}") - return None - typer.echo("Infracost scan failed: could not find working --json flag form") - return None + text = (stderr or "").strip() + if any(hint in text.lower() for hint in _AUTH_ERROR_HINTS): + typer.secho( + " Infracost rejected your credentials (the saved token may have expired).", + fg=typer.colors.YELLOW, + ) + typer.echo(" Run: infracost auth login") + return + typer.echo(f"Infracost scan failed: {text}") def run_infracost_scan_with_usage( - terraform_dir: Path, usage_profile: Dict + terraform_dir: Path, + usage_profile: Dict, + resource_usage: Optional[Dict[str, Dict[str, float]]] = None, ) -> Optional[Dict]: """ Run infracost v2 with a usage profile applied and return parsed JSON. @@ -206,6 +200,10 @@ def run_infracost_scan_with_usage( 2. write /infracost-usage.yml and /infracost.yml (relative `path: tf`, usage_file, terraform_var_files: [terraform.tfvars]) 3. run `infracost scan --json` with cwd = config_dir + + `usage_profile` sets type-wide defaults (the light/heavy estimate path); + `resource_usage` pins usage to individual terraform addresses (the measured + path used by `deployml costs`). Either may be empty. """ from deployml.utils.usage_profiles import render_usage_yaml @@ -213,7 +211,7 @@ def run_infracost_scan_with_usage( try: shutil.copytree(terraform_dir, config_dir / "tf") (config_dir / "infracost-usage.yml").write_text( - render_usage_yaml(usage_profile) + render_usage_yaml(usage_profile, resource_usage) ) (config_dir / "infracost.yml").write_text( "version: 0.1\n" @@ -230,7 +228,7 @@ def run_infracost_scan_with_usage( timeout=120, ) if result.returncode != 0: - typer.echo(f"Infracost scan failed: {result.stderr}") + _report_scan_failure(result.stderr) return None return json.loads(result.stdout) except subprocess.TimeoutExpired: @@ -271,37 +269,21 @@ def _run_inspect_rows(scan_json_path: Path) -> List[Dict]: timeout=30, ) if result.returncode != 0: + typer.secho( + " Could not read the per-resource cost breakdown " + "(infracost inspect failed); showing totals only.", + fg=typer.colors.YELLOW, + ) return [] return json.loads(result.stdout) except (subprocess.SubprocessError, json.JSONDecodeError, ValueError, OSError): + typer.secho( + " Could not read the per-resource cost breakdown; showing totals only.", + fg=typer.colors.YELLOW, + ) return [] -def fetch_resource_costs(scan_json_path: Path) -> List[Tuple[str, float]]: - """ - Return a list of (label, monthly_cost) for non-zero cost resources, grouped - by GCP service label. Used by the simple deploy/costs breakdown view. - """ - costs: List[Tuple[str, float]] = [] - seen_labels = set() - for row in _run_inspect_rows(scan_json_path): - cost = float(row.get("cost", 0) or 0) - if cost <= 0: - continue - address = row.get("columns", {}).get("resource", "") - label = _resource_label(address) - if label in seen_labels: - # Accumulate duplicate resource types (e.g. multiple Cloud Run services) - for i, (lbl, c) in enumerate(costs): - if lbl == label: - costs[i] = (lbl, c + cost) - break - else: - seen_labels.add(label) - costs.append((label, cost)) - return sorted(costs, key=lambda x: x[1], reverse=True) - - def fetch_resource_costs_detailed(scan_json_path: Path) -> List[ResourceCost]: """ Return per-resource ResourceCost records (non-zero cost only), each classified @@ -350,46 +332,6 @@ def parse_infracost_scan_data(data: Dict) -> Optional[CostAnalysis]: return None -def display_cost_breakdown( - analysis: CostAnalysis, - warning_threshold: float = 100.0, - show_resources: bool = False, -) -> None: - typer.echo("\n" + "=" * 60) - typer.secho("COST ANALYSIS", fg=typer.colors.BRIGHT_CYAN, bold=True) - typer.echo("=" * 60) - - monthly_cost = analysis.total_monthly_cost - color = typer.colors.BRIGHT_GREEN if monthly_cost < warning_threshold else typer.colors.BRIGHT_YELLOW - typer.secho( - f"Monthly Cost: ${monthly_cost:.2f} {analysis.currency}", - fg=color, - bold=True, - ) - typer.echo( - f"Resources: {analysis.costed_resources} costed, " - f"{analysis.free_resources} free, {analysis.resources} total" - ) - - if show_resources and analysis.resource_costs: - typer.echo("\nWhat costs money:") - for label, cost in analysis.resource_costs: - typer.secho(f" ${cost:.2f} {label}", fg=typer.colors.BRIGHT_BLUE) - typer.echo( - "\nNote: Cloud Run, BigQuery, and GCS show $0 at idle — they scale to\n" - "zero and charge only for actual usage." - ) - - if monthly_cost > warning_threshold: - typer.echo() - typer.secho( - f"WARNING: Monthly cost exceeds ${warning_threshold:.0f} threshold!", - fg=typer.colors.BRIGHT_RED, - bold=True, - ) - typer.echo() - - def format_cost_for_confirmation(monthly_cost: float, currency: str) -> str: if monthly_cost > 0: return f"Monthly cost: ~${monthly_cost:.2f} {currency}" @@ -397,46 +339,6 @@ def format_cost_for_confirmation(monthly_cost: float, currency: str) -> str: return "Monthly cost: Variable (usage-based pricing)" -def run_infracost_analysis( - terraform_dir: Path, - warning_threshold: float = 100.0, - show_resources: bool = False, -) -> Optional[CostAnalysis]: - if not check_infracost_available(): - typer.echo("Tip: Install infracost CLI for cost analysis before deployment") - typer.echo(" Visit: https://www.infracost.io/docs/#quick-start") - return None - - if not check_infracost_authenticated(): - typer.echo("Tip: Authenticate infracost to enable cost analysis") - typer.echo(" Run: infracost auth login") - typer.echo(" Or set: export INFRACOST_API_KEY=") - return None - - typer.echo("Running cost analysis...") - raw_data = run_infracost_scan(terraform_dir) - if raw_data is None: - return None - - analysis = parse_infracost_scan_data(raw_data) - if analysis is None: - return None - - if show_resources: - # Write the scan result to a temp file and inspect it explicitly, rather - # than letting infracost read its global "most recent scan" cache. - scan_dir = Path(tempfile.mkdtemp()) - scan_json_path = scan_dir / "infracost-scan.json" - try: - scan_json_path.write_text(json.dumps(raw_data)) - analysis.resource_costs = fetch_resource_costs(scan_json_path) - finally: - shutil.rmtree(scan_dir, ignore_errors=True) - - display_cost_breakdown(analysis, warning_threshold, show_resources=show_resources) - return analysis - - def display_estimate( resources: List[ResourceCost], analysis: CostAnalysis, @@ -563,3 +465,234 @@ def run_estimate_analysis( display_estimate(resources, analysis, profile_name, warning_threshold) return analysis + + +def display_actual_costs( + resources: List[ResourceCost], + analysis: CostAnalysis, + measured: "MeasuredUsage", + workspace_name: str, + profile_label: str, + profile_ratio: float, + deployed_days: Optional[float] = None, + warning_threshold: float = 100.0, +) -> None: + """ + Render the measured-cost view for a running deployment. + + Same two-bucket shape as display_estimate, with three additions that only + make sense once the stack actually exists: each usage line is annotated with + the real numbers behind it, the always-on line reports how much of the window + the database was up, and the student gets told which profile their real usage + resembles. + """ + fixed = [r for r in resources if r.category == "fixed"] + usage = [r for r in resources if r.category == "usage"] + fixed_total = sum(r.monthly_cost for r in fixed) + usage_total = sum(r.monthly_cost for r in usage) + total = analysis.total_monthly_cost + currency = analysis.currency + window = measured.window_days + + typer.echo("\n" + "=" * 60) + typer.secho(f" ACTUAL COST · {workspace_name}", fg=typer.colors.BRIGHT_CYAN, bold=True) + typer.echo("=" * 60) + typer.echo(f" Measured from your real GCP usage over the last {window:g} days.") + typer.echo() + + headline_color = ( + typer.colors.BRIGHT_GREEN if total < warning_threshold + else typer.colors.BRIGHT_YELLOW + ) + typer.secho( + f" Run-rate: ~${total:,.0f} / month " + f"(${fixed_total:,.2f} always-on + ~${usage_total:,.2f} measured usage) {currency}", + fg=headline_color, + bold=True, + ) + if deployed_days: + # Straight-line projection from the monthly run-rate. Not a billed figure: + # for that a student needs a BigQuery billing export. + so_far = total * (deployed_days / 30.0) + typer.secho( + f" Charged so far: ~${so_far:,.2f} " + f"({deployed_days:.1f} days since deploy, approximate)", + fg=headline_color, + ) + + if fixed: + typer.echo() + typer.secho(" ALWAYS-ON (billed 24/7 while the stack exists)", bold=True) + for r in fixed: + line = f" ${r.monthly_cost:>8.2f} {r.label:<12} {r.description}" + uptime = measured.sql_uptime.get(r.address) + if uptime is not None: + line += f" · up {uptime * 100:.0f}% of window" + typer.secho(line, fg=typer.colors.BRIGHT_BLUE) + + if usage: + typer.echo() + typer.secho( + f" USAGE-BASED (measured over {window:g} days, scaled to a month)", bold=True + ) + # Collapse identical rows the way display_estimate does, but carry each + # group's measured detail through so the numbers stay visible. + grouped: "OrderedDict[Tuple[str, str], List]" = OrderedDict() + for r in usage: + key = (r.label, r.description) + entry = grouped.setdefault(key, [0.0, 0, []]) + entry[0] += r.monthly_cost + entry[1] += 1 + detail = measured.details.get(r.address) + if detail and detail not in entry[2]: + entry[2].append(detail) + for (label, description), (cost, count, details) in grouped.items(): + suffix = f" (x{count})" if count > 1 else "" + line = f" ${cost:>8.2f} {label:<12} {description}{suffix}" + if details: + line += " · " + "; ".join(details) + typer.secho(line, fg=typer.colors.BRIGHT_BLUE) + + # The payoff of measuring: light/heavy become labels, not pricing inputs. + if measured.cloud_run_monthly_requests > 0: + typer.echo() + typer.secho( + f" You are a {profile_label.upper()} user — your real usage is " + f"{profile_ratio:.1f}x the '{profile_label}' profile.", + fg=typer.colors.BRIGHT_CYAN, + ) + + if fixed and total > 0: + top = max(fixed, key=lambda r: r.monthly_cost) + share = top.monthly_cost / total + if share > 0.5: + typer.echo() + typer.secho( + f" Biggest lever: {top.label} is {share * 100:.0f}% of your cost " + f"and runs 24/7.", + fg=typer.colors.BRIGHT_MAGENTA, + bold=True, + ) + typer.secho( + " Tear the stack down when you are not using it: deployml destroy", + fg=typer.colors.BRIGHT_MAGENTA, + ) + + if measured.notes: + # Covers both what Monitoring could not report (priced at $0) and the + # places where a measurement is an approximation. Either way the student + # sees why a number is what it is. + typer.echo() + typer.secho(" About these numbers:", bold=True) + for note in measured.notes: + typer.echo(f" - {note}") + + typer.echo( + "\n Note: this prices your real usage, it is not a billing statement.\n" + " Check the GCP Billing Console for the amount you were actually charged." + ) + + if total > warning_threshold: + typer.echo() + typer.secho( + f" WARNING: exceeds ${warning_threshold:.0f}/month threshold!", + fg=typer.colors.BRIGHT_RED, + bold=True, + ) + typer.echo() + + +def run_actual_cost_analysis( + terraform_dir: Path, + project_id: str, + workspace_name: str, + days: float = 30.0, + warning_threshold: float = 100.0, + deployed_days: Optional[float] = None, +) -> Optional[CostAnalysis]: + """ + Price a running deployment from its real usage, then render the result. + + Same contract as run_estimate_analysis: assumes the caller already checked + infracost availability and auth, prints its own diagnostics, and returns the + CostAnalysis or None. + + The usage profile passed to infracost is deliberately empty. Only the + measured per-address numbers are supplied, so anything Monitoring could not + tell us stays at zero instead of falling back to a light/heavy guess, and + the reason is printed in the "Not measured" section. + """ + from deployml.utils.measured_usage import ( + MonitoringError, + build_resource_map, + collect_measured_usage, + ) + from deployml.utils.usage_profiles import classify_usage_profile + + deployed = build_resource_map(terraform_dir) + if not deployed: + typer.secho( + " Could not read any deployed resources from the terraform state.", + fg=typer.colors.YELLOW, + ) + typer.echo( + " Run 'deployml deploy' first, or 'deployml estimate' for a " + "pre-deploy prediction." + ) + return None + + typer.echo( + f"Measuring real usage for {len(deployed)} deployed resources " + f"over the last {days:g} days..." + ) + try: + measured = collect_measured_usage(project_id, deployed, days) + except MonitoringError as e: + typer.secho(f" {e}", fg=typer.colors.BRIGHT_RED) + if e.hint: + for line in e.hint.splitlines(): + typer.echo(f" {line}") + return None + + if measured.is_empty(): + typer.secho( + " No usage has been recorded for this deployment yet.", + fg=typer.colors.YELLOW, + ) + typer.echo( + " Monitoring data takes a few minutes to appear after a deploy. " + "Showing always-on cost only." + ) + + raw_data = run_infracost_scan_with_usage( + terraform_dir, {}, measured.resource_usage + ) + if raw_data is None: + return None + + analysis = parse_infracost_scan_data(raw_data) + if analysis is None: + return None + + scan_dir = Path(tempfile.mkdtemp()) + scan_json_path = scan_dir / "infracost-scan.json" + try: + scan_json_path.write_text(json.dumps(raw_data)) + resources = fetch_resource_costs_detailed(scan_json_path) + finally: + shutil.rmtree(scan_dir, ignore_errors=True) + + profile_label, profile_ratio = classify_usage_profile( + measured.cloud_run_monthly_requests + ) + display_actual_costs( + resources, + analysis, + measured, + workspace_name, + profile_label, + profile_ratio, + deployed_days=deployed_days, + warning_threshold=warning_threshold, + ) + return analysis diff --git a/src/deployml/utils/measured_usage.py b/src/deployml/utils/measured_usage.py new file mode 100644 index 0000000..603690d --- /dev/null +++ b/src/deployml/utils/measured_usage.py @@ -0,0 +1,651 @@ +""" +Measure a deployment's *real* GCP usage and express it in infracost's usage keys. + +Why this exists +--------------- +`deployml estimate` prices a stack that does not exist yet, so it has to *assume* +how much a student will use it (the light/heavy profiles in `usage_profiles.py`). +Once the stack is deployed we can do better: Cloud Monitoring already records how +many requests each Cloud Run service served, how many bytes sit in GCS, and how +many bytes BigQuery scanned. This module reads those counters and converts them +into the exact keys infracost expects, so `deployml costs` prices the student's +actual behaviour instead of a guess. + +There is deliberately no billing integration here. GCP has no API that returns +"my spend so far" -- that needs a BigQuery billing export, which requires +billing-account admin and has a ~24h lag, so it is out of reach for most +students. Measuring usage and pricing it through infracost needs nothing beyond +`roles/monitoring.viewer`. + +How usage maps onto infracost keys +---------------------------------- +| infracost key | Monitoring metric | +|--------------------------------------------|------------------------------------------------| +| `monthly_requests` (Cloud Run) | `run.googleapis.com/request_count` | +| `average_request_duration_ms` (Cloud Run) | `run.googleapis.com/request_latencies` (p50) | +| `storage_gb` (GCS) | `storage.googleapis.com/storage/total_bytes` | +| `monthly_class_a_operations` (GCS) | `storage.googleapis.com/api/request_count` | +| `monthly_class_b_operations` (GCS) | `storage.googleapis.com/api/request_count` | +| `monthly_data_retrieval_gb` (GCS) | `storage.googleapis.com/network/sent_bytes_count` | +| `monthly_queries_tb` (BQ dataset) | `bigquery.googleapis.com/query/scanned_bytes` | +| `monthly_active_storage_gb` (BQ table) | `bigquery.googleapis.com/storage/stored_bytes` | +| `monthly_streaming_inserts_mb`(BQ table) | `bigquery.googleapis.com/storage/uploaded_bytes` | +| (display only) Cloud SQL uptime | `cloudsql.googleapis.com/database/up` | + +Three honest approximations, each surfaced to the user in `MeasuredUsage.notes`: + +1. **Class A vs B operations.** GCS reports operations by method name, not by + billing class, so we bucket mutating methods and List as class A and Get as + class B. That matches Google's pricing table for the methods this stack + actually calls, but it is a mapping we maintain, not a number GCP reports. +2. **BigQuery attribution.** Scanned and stored bytes arrive project-scoped, not + per-table, so we split them evenly across the deployment's BigQuery + addresses. BigQuery storage and query pricing are both linear in bytes, so + the *total* cost is exact -- only the per-row split is approximate. +3. **Flow vs stock.** Counters (requests, bytes scanned) are summed over the + window and scaled to 30 days. Levels (bytes stored) are averaged over the + window and used as-is, because a monthly average level is already what + infracost's `storage_gb` means. Cloud SQL uptime is the mean of a 0/1 gauge, + which is the fraction of the window the instance was running. + +Design rule: never invent a number. Anything Monitoring does not return is left +absent, so infracost prices it at zero, and the reason lands in `notes` for the +display to print. A silent $0 is the bug `deployml costs` exists to fix; an +explained $0 is fine. +""" + +import json +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Dict, List, Optional + +import requests + +from deployml.utils.platform_compat import run_tool + +MONITORING_ENDPOINT = "https://monitoring.googleapis.com/v3/projects/{project}/timeSeries" + +# A month, for scaling a measurement window up to the monthly figures infracost wants. +DAYS_PER_MONTH = 30.0 + +_BYTES_PER_GB = 1024 ** 3 +_BYTES_PER_TB = 1024 ** 4 +_BYTES_PER_MB = 1024 ** 2 + +# GCS reports operations by method name; pricing bills them by class. Methods that +# create, modify, or enumerate objects are class A; plain reads are class B. +# Anything unrecognised is counted as class B, the cheaper of the two, so an +# unknown method can never silently inflate the estimate. +_CLASS_A_METHOD_HINTS = ( + "insert", "create", "update", "patch", "delete", "copy", + "compose", "rewrite", "list", "setiampolicy", "post", +) + +# Cloud Run resource types, for matching a terraform address to a service name. +_CLOUD_RUN_TYPES = ("google_cloud_run_service", "google_cloud_run_v2_service") + +# Where each resource type keeps its GCP-side name in `terraform show -json`. +# Falls back to "name" for everything not listed. +_NAME_KEYS = { + "google_bigquery_table": ("table_id",), + "google_bigquery_dataset": ("dataset_id",), + "google_sql_database_instance": ("name",), +} + + +class MonitoringError(Exception): + """ + The Monitoring API could not be read at all. + + Carries a `hint` with the command that fixes it, so the CLI can tell the + student what to do instead of dumping an HTTP status. + """ + + def __init__(self, message: str, hint: str = ""): + super().__init__(message) + self.hint = hint + + +@dataclass +class DeployedResource: + """One resource in the deployed terraform state, with its GCP-side name.""" + address: str # module.cloud_sql_postgres.google_sql_database_instance.postgres + resource_type: str # google_sql_database_instance + name: str # mlops-postgres + + +@dataclass +class MeasuredUsage: + """Real usage over a window, keyed the way infracost wants it.""" + window_days: float + # terraform address -> infracost usage keys. Fed straight into the usage file. + resource_usage: Dict[str, Dict[str, float]] = field(default_factory=dict) + # terraform address -> human annotation ("1.2M requests, 240 ms median") + details: Dict[str, str] = field(default_factory=dict) + # terraform address -> fraction of the window the instance was up (0.0-1.0) + sql_uptime: Dict[str, float] = field(default_factory=dict) + # Things we could not measure, printed so a $0 is never unexplained. + notes: List[str] = field(default_factory=list) + # Headline number used to label the student light/heavy. + cloud_run_monthly_requests: float = 0.0 + + def is_empty(self) -> bool: + """True when Monitoring returned nothing usable for any resource.""" + return not self.resource_usage + + +# --------------------------------------------------------------------------- # +# Reading the deployed terraform state +# --------------------------------------------------------------------------- # + +def _walk_module(module: Dict) -> List[Dict]: + """Flatten a `terraform show -json` module tree into a list of resources.""" + resources = list(module.get("resources") or []) + for child in module.get("child_modules") or []: + resources.extend(_walk_module(child)) + return resources + + +def _gcp_name(resource: Dict) -> str: + """Pull the GCP-side resource name out of one `terraform show -json` entry.""" + values = resource.get("values") or {} + for key in _NAME_KEYS.get(resource.get("type", ""), ("name",)): + name = values.get(key) + if name: + return str(name) + return "" + + +def build_resource_map(terraform_dir: Path) -> List[DeployedResource]: + """ + List the deployed google_* resources with both their terraform address and + their GCP-side name. + + Uses `terraform show -json`, the machine-readable form, rather than parsing + the human output of `terraform state show` line by line. Monitoring reports + usage against GCP names while infracost keys usage against terraform + addresses, so this mapping is what lets measured usage reach the right + resource. + + Returns [] when the state cannot be read; callers treat that as "nothing + deployed here" rather than an error. + """ + try: + result = run_tool( + "terraform", ["show", "-json"], + cwd=terraform_dir, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode != 0: + return [] + data = json.loads(result.stdout) + except (subprocess.SubprocessError, json.JSONDecodeError, OSError, ValueError): + return [] + + root = (data.get("values") or {}).get("root_module") or {} + deployed: List[DeployedResource] = [] + for resource in _walk_module(root): + # Data sources show up here too (module.x.data.google_project.current). + # They are lookups, not billable resources, and infracost never costs them. + if resource.get("mode", "managed") != "managed": + continue + resource_type = resource.get("type", "") + if not resource_type.startswith("google_"): + continue + name = _gcp_name(resource) + if not name: + continue + deployed.append( + DeployedResource( + address=resource.get("address", ""), + resource_type=resource_type, + name=name, + ) + ) + return deployed + + +def _addresses_of_type(resources: List[DeployedResource], *types: str) -> List[str]: + return [r.address for r in resources if r.resource_type in types] + + +def _address_by_name( + resources: List[DeployedResource], name: str, *types: str +) -> Optional[str]: + """Find the terraform address of the resource GCP calls `name`.""" + for r in resources: + if r.resource_type in types and r.name == name: + return r.address + return None + + +# --------------------------------------------------------------------------- # +# Talking to Cloud Monitoring +# --------------------------------------------------------------------------- # + +def access_token() -> str: + """ + Mint an access token from Application Default Credentials. + + Uses the same `gcloud auth application-default print-access-token` call that + `helpers.check_gcp_adc()` probes with, so if `deployml doctor` says ADC is + configured this will succeed. + """ + try: + result = run_tool( + "gcloud", ["auth", "application-default", "print-access-token"], + capture_output=True, + text=True, + timeout=60, + ) + except (subprocess.SubprocessError, FileNotFoundError, OSError) as e: + raise MonitoringError( + f"Could not run gcloud to get credentials: {e}", + hint="gcloud auth application-default login", + ) + if result.returncode != 0 or not result.stdout.strip(): + raise MonitoringError( + "No Application Default Credentials available.", + hint="gcloud auth application-default login", + ) + return result.stdout.strip() + + +def _point_value(point: Dict) -> float: + """Read a numeric value out of one Monitoring point, whatever its type.""" + value = point.get("value") or {} + if "doubleValue" in value: + return float(value["doubleValue"]) + if "int64Value" in value: + # int64 comes back as a JSON string. + return float(value["int64Value"]) + if "distributionValue" in value: + dist = value["distributionValue"] or {} + return float(dist.get("mean", 0) or 0) + if "boolValue" in value: + return 1.0 if value["boolValue"] else 0.0 + return 0.0 + + +def query_timeseries( + project_id: str, + token: str, + metric_type: str, + start: datetime, + end: datetime, + aligner: str, +) -> List[Dict]: + """ + Fetch one metric's time series over [start, end]. + + The alignment period is the whole window, so each series collapses to a + single point: a total for counters (ALIGN_SUM) or an average for levels + (ALIGN_MEAN), which for a 0/1 gauge is also the uptime fraction. + + Deliberately no crossSeriesReducer/groupByFields: we want the raw per-series + breakdown so `resource.labels` still tells us which bucket or service each + number belongs to, and we aggregate in Python instead. Cloud Run splits + request_count by response code and GCS splits api/request_count by method, + so there are several series per resource either way. + + Raises MonitoringError when the API itself is unreachable (bad credentials, + API disabled, missing IAM). Returns [] when the API works but has no data. + """ + window_seconds = max(int((end - start).total_seconds()), 60) + params = { + "filter": f'metric.type = "{metric_type}"', + "interval.startTime": start.strftime("%Y-%m-%dT%H:%M:%SZ"), + "interval.endTime": end.strftime("%Y-%m-%dT%H:%M:%SZ"), + "aggregation.alignmentPeriod": f"{window_seconds}s", + "aggregation.perSeriesAligner": aligner, + } + url = MONITORING_ENDPOINT.format(project=project_id) + series: List[Dict] = [] + page_token = None + # Cap pages so a pathological project can't hang the command. + for _ in range(10): + if page_token: + params["pageToken"] = page_token + try: + response = requests.get( + url, + params=params, + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + except requests.RequestException as e: + raise MonitoringError( + f"Could not reach the Cloud Monitoring API: {e}", + hint="Check your network connection and retry.", + ) + + if response.status_code in (401, 403): + raise MonitoringError( + "Not allowed to read Cloud Monitoring metrics for this project.", + hint=( + "Enable the API and grant yourself the viewer role:\n" + f" gcloud services enable monitoring.googleapis.com --project {project_id}\n" + " then ask your project owner for roles/monitoring.viewer" + ), + ) + if response.status_code != 200: + raise MonitoringError( + f"Cloud Monitoring returned HTTP {response.status_code}.", + hint="Retry in a moment; if it persists, check the GCP status page.", + ) + + try: + payload = response.json() + except ValueError: + raise MonitoringError( + "Cloud Monitoring returned a response we could not parse.", + hint="Retry in a moment.", + ) + + series.extend(payload.get("timeSeries") or []) + page_token = payload.get("nextPageToken") + if not page_token: + break + return series + + +def _sum_points(entry: Dict) -> float: + """Total a single series' points (normally just one, given our alignment).""" + return sum(_point_value(p) for p in entry.get("points") or []) + + +def _mean_points(entry: Dict) -> float: + points = entry.get("points") or [] + if not points: + return 0.0 + return sum(_point_value(p) for p in points) / len(points) + + +def _by_resource_label( + series: List[Dict], label: str, reducer=_sum_points +) -> Dict[str, float]: + """Collapse series into {resource label value: number}, summing duplicates.""" + totals: Dict[str, float] = {} + for entry in series: + key = ((entry.get("resource") or {}).get("labels") or {}).get(label) + if not key: + continue + totals[key] = totals.get(key, 0.0) + reducer(entry) + return totals + + +# --------------------------------------------------------------------------- # +# Formatting helpers for the display annotations +# --------------------------------------------------------------------------- # + +def human_count(value: float) -> str: + """Compact a request count: 1234567 -> '1.2M'.""" + if value >= 1_000_000: + return f"{value / 1_000_000:.1f}M" + if value >= 1_000: + return f"{value / 1_000:.1f}k" + return f"{value:.0f}" + + +def _scale_to_month(total: float, days: float) -> float: + """Scale a counter measured over `days` up to a 30-day month.""" + if days <= 0: + return 0.0 + return total * (DAYS_PER_MONTH / days) + + +# --------------------------------------------------------------------------- # +# Per-service collectors. Each fills in the infracost keys it can measure and +# appends a note for anything it cannot, then leaves the rest alone. +# --------------------------------------------------------------------------- # + +def _collect_cloud_run(fetch, resources: List[DeployedResource], + usage: MeasuredUsage, days: float) -> None: + if not _addresses_of_type(resources, *_CLOUD_RUN_TYPES): + return + + # request_count is the first call made, so it carries required=True: if the + # API is unreachable we want one clear error, not a note on every metric. + per_service = _by_resource_label( + fetch("run.googleapis.com/request_count", "ALIGN_SUM", required=True), + "service_name", + ) + per_service_latency = _by_resource_label( + fetch("run.googleapis.com/request_latencies", "ALIGN_PERCENTILE_50"), + "service_name", + _mean_points, + ) + + if not per_service: + usage.notes.append( + "Cloud Run served no requests in this window, so it is priced at $0." + ) + return + + matched = False + for service_name, count in per_service.items(): + address = _address_by_name(resources, service_name, *_CLOUD_RUN_TYPES) + if address is None: + # A Cloud Run service in this project that this stack did not deploy. + continue + matched = True + monthly = _scale_to_month(count, days) + entry: Dict[str, float] = {"monthly_requests": round(monthly, 2)} + detail = f"{human_count(monthly)} req/mo" + + latency = per_service_latency.get(service_name) + if latency: + entry["average_request_duration_ms"] = round(latency, 1) + detail += f", {latency:.0f} ms median" + + usage.resource_usage[address] = entry + usage.details[address] = detail + usage.cloud_run_monthly_requests += monthly + + if matched: + usage.notes.append( + "Cloud Run concurrency is not exposed as a metric, so requests per " + "instance uses infracost's default." + ) + + +def _collect_gcs(fetch, resources: List[DeployedResource], + usage: MeasuredUsage, days: float) -> None: + if not _addresses_of_type(resources, "google_storage_bucket"): + return + + stored = _by_resource_label( + fetch("storage.googleapis.com/storage/total_bytes", "ALIGN_MEAN"), + "bucket_name", + _mean_points, + ) + retrieved = _by_resource_label( + fetch("storage.googleapis.com/network/sent_bytes_count", "ALIGN_SUM"), + "bucket_name", + ) + + # Operations arrive split by method name, which we map onto billing classes. + class_a: Dict[str, float] = {} + class_b: Dict[str, float] = {} + for entry in fetch("storage.googleapis.com/api/request_count", "ALIGN_SUM"): + bucket = ((entry.get("resource") or {}).get("labels") or {}).get("bucket_name") + if not bucket: + continue + method = (((entry.get("metric") or {}).get("labels") or {}) + .get("method", "") or "").lower() + target = class_a if any(h in method for h in _CLASS_A_METHOD_HINTS) else class_b + target[bucket] = target.get(bucket, 0.0) + _sum_points(entry) + + buckets = set(stored) | set(retrieved) | set(class_a) | set(class_b) + if not buckets: + usage.notes.append( + "GCS reported no storage or traffic in this window, so it is priced at $0." + ) + return + + matched_ops = False + for bucket in buckets: + address = _address_by_name(resources, bucket, "google_storage_bucket") + if address is None: + continue + entry: Dict[str, float] = {} + details: List[str] = [] + + stored_gb = stored.get(bucket, 0.0) / _BYTES_PER_GB + if stored_gb > 0: + entry["storage_gb"] = round(stored_gb, 4) + details.append(f"{stored_gb:.2f} GB stored") + + retrieved_gb = _scale_to_month(retrieved.get(bucket, 0.0), days) / _BYTES_PER_GB + if retrieved_gb > 0: + entry["monthly_data_retrieval_gb"] = round(retrieved_gb, 4) + details.append(f"{retrieved_gb:.2f} GB/mo read") + + ops_a = _scale_to_month(class_a.get(bucket, 0.0), days) + ops_b = _scale_to_month(class_b.get(bucket, 0.0), days) + if ops_a > 0 or ops_b > 0: + matched_ops = True + entry["monthly_class_a_operations"] = round(ops_a, 2) + entry["monthly_class_b_operations"] = round(ops_b, 2) + details.append(f"{human_count(ops_a + ops_b)} ops/mo") + + if entry: + usage.resource_usage[address] = entry + usage.details[address] = " · ".join(details) + + if matched_ops: + usage.notes.append( + "GCS class A/B operations are inferred from method names, since GCS " + "reports methods rather than billing classes." + ) + + +def _collect_bigquery(fetch, resources: List[DeployedResource], + usage: MeasuredUsage, days: float) -> None: + dataset_addresses = _addresses_of_type(resources, "google_bigquery_dataset") + table_addresses = _addresses_of_type(resources, "google_bigquery_table") + if not dataset_addresses and not table_addresses: + return + + scanned_bytes = sum( + _sum_points(e) + for e in fetch("bigquery.googleapis.com/query/scanned_bytes", "ALIGN_SUM") + ) + stored_bytes = sum( + _mean_points(e) + for e in fetch("bigquery.googleapis.com/storage/stored_bytes", "ALIGN_MEAN") + ) + uploaded_bytes = sum( + _sum_points(e) + for e in fetch("bigquery.googleapis.com/storage/uploaded_bytes", "ALIGN_SUM") + ) + + # These metrics are project-scoped, so they are split evenly across the + # deployment's addresses. BigQuery query and storage pricing are both linear + # in bytes, so the total cost is exact; only the per-row split is a guess. + if dataset_addresses and scanned_bytes > 0: + scanned_tb = _scale_to_month(scanned_bytes, days) / _BYTES_PER_TB + share = scanned_tb / len(dataset_addresses) + for address in dataset_addresses: + usage.resource_usage.setdefault(address, {})["monthly_queries_tb"] = round(share, 6) + usage.details[address] = f"{scanned_tb:.3f} TB/mo scanned" + elif dataset_addresses: + usage.notes.append( + "BigQuery ran no queries in this window, so query cost is $0." + ) + + if table_addresses: + stored_gb = stored_bytes / _BYTES_PER_GB + uploaded_mb = _scale_to_month(uploaded_bytes, days) / _BYTES_PER_MB + if stored_gb > 0 or uploaded_mb > 0: + storage_share = stored_gb / len(table_addresses) + insert_share = uploaded_mb / len(table_addresses) + for address in table_addresses: + entry = usage.resource_usage.setdefault(address, {}) + if stored_gb > 0: + entry["monthly_active_storage_gb"] = round(storage_share, 6) + if uploaded_mb > 0: + entry["monthly_streaming_inserts_mb"] = round(insert_share, 4) + details = [] + if stored_gb > 0: + details.append(f"{stored_gb:.2f} GB stored") + if uploaded_mb > 0: + details.append(f"{uploaded_mb:.1f} MB/mo inserted") + # Annotate the first table; the display collapses identical rows anyway. + usage.details[table_addresses[0]] = " · ".join(details) + if uploaded_mb <= 0: + usage.notes.append( + "BigQuery streaming inserts were not reported for this window, " + "so insert cost shows as $0." + ) + if len(table_addresses) > 1 and stored_gb > 0: + usage.notes.append( + f"BigQuery storage is reported per project, so it is split evenly " + f"across {len(table_addresses)} tables. The total is exact." + ) + + +def _collect_cloud_sql(fetch, resources: List[DeployedResource], + usage: MeasuredUsage) -> None: + """ + Record how much of the window the database was actually up. + + Cloud SQL is billed for existing, not for being used, so this does not change + what infracost charges. It changes what we can *tell* the student: the + always-on line is the whole bill for most deployments, and the only lever is + not running it. + """ + if not _addresses_of_type(resources, "google_sql_database_instance"): + return + + # database/up is GAUGE/INT64, not BOOL, so ALIGN_FRACTION_TRUE is rejected + # with HTTP 400. The value is 1 while the instance is up and 0 while it is + # down, so the mean across the window is the uptime fraction. + uptime = _by_resource_label( + fetch("cloudsql.googleapis.com/database/up", "ALIGN_MEAN"), + "database_id", + _mean_points, + ) + for database_id, fraction in uptime.items(): + # database_id is "project:instance" + instance = database_id.split(":")[-1] + address = _address_by_name(resources, instance, "google_sql_database_instance") + if address is not None: + usage.sql_uptime[address] = min(max(fraction, 0.0), 1.0) + + +def collect_measured_usage( + project_id: str, + resources: List[DeployedResource], + days: float = 30.0, +) -> MeasuredUsage: + """ + Measure real usage for the deployed resources over the last `days` days. + + Raises MonitoringError if Cloud Monitoring cannot be read at all, so the CLI + can print one actionable message. Individual metrics that fail become notes + and are priced at zero rather than guessed. + """ + end = datetime.now(timezone.utc).replace(microsecond=0) + start = end - timedelta(days=days) + token = access_token() + usage = MeasuredUsage(window_days=float(days)) + + def fetch(metric_type: str, aligner: str, required: bool = False) -> List[Dict]: + try: + return query_timeseries(project_id, token, metric_type, start, end, aligner) + except MonitoringError as e: + if required: + raise + usage.notes.append(f"Could not read {metric_type}: {e}") + return [] + + _collect_cloud_run(fetch, resources, usage, days) + _collect_gcs(fetch, resources, usage, days) + _collect_bigquery(fetch, resources, usage, days) + _collect_cloud_sql(fetch, resources, usage) + return usage diff --git a/src/deployml/utils/usage_profiles.py b/src/deployml/utils/usage_profiles.py index ec3332e..24ffbda 100644 --- a/src/deployml/utils/usage_profiles.py +++ b/src/deployml/utils/usage_profiles.py @@ -11,6 +11,9 @@ (see infracost's usage-file schema). To tune assumptions, edit the numbers here. """ +import math +from typing import Dict, Optional, Tuple + # A typical student demo: clicking around the UIs + running the example serving # script a few times over a month. Small storage, light query volume. LIGHT = { @@ -65,17 +68,66 @@ def get_profile(name: str) -> dict: return PROFILES.get(name, LIGHT) -def render_usage_yaml(profile: dict) -> str: +def render_usage_yaml( + profile: Dict, + resource_usage: Optional[Dict[str, Dict[str, float]]] = None, +) -> str: """ - Render a usage profile into an infracost-usage.yml document. + Render an infracost-usage.yml document. + + Two sections, either of which may be omitted: + + - `resource_type_default_usage` applies one assumption to every resource of a + type, so all three Cloud Run services get the same request count. That is + what a coarse light/heavy *estimate* wants. + - `resource_usage` pins usage to a single terraform address. That is what + measured usage needs: real numbers differ per service, and a type default + would charge every service the whole deployment's total. + + Per-address entries win over type defaults, which is why `deployml costs` + passes an empty profile plus measured `resource_usage`: anything it could not + measure then stays at infracost's zero default instead of being invented. - Uses `resource_type_default_usage`, which applies the same assumptions to - every resource of a given type (e.g. all three Cloud Run services), which is - exactly what a coarse light/heavy profile wants. + Values are formatted straight into YAML, so only scalars are supported. """ - lines = ["version: 0.1", "resource_type_default_usage:"] - for resource_type, fields in profile.items(): - lines.append(f" {resource_type}:") - for key, value in fields.items(): - lines.append(f" {key}: {value}") + lines = ["version: 0.1"] + + if profile: + lines.append("resource_type_default_usage:") + for resource_type, fields in profile.items(): + lines.append(f" {resource_type}:") + for key, value in fields.items(): + lines.append(f" {key}: {value}") + + if resource_usage: + lines.append("resource_usage:") + for address, fields in resource_usage.items(): + # Addresses contain dots and brackets, so they need quoting. + lines.append(f' "{address}":') + for key, value in fields.items(): + lines.append(f" {key}: {value}") + return "\n".join(lines) + "\n" + + +def classify_usage_profile(monthly_requests: float) -> Tuple[str, float]: + """ + Label measured usage against the light and heavy profiles. + + This is the payoff of measuring. Once we know a student's real request + volume, light and heavy stop being pricing *inputs* (guesses we bill them + for) and become pricing *labels* ("you are a light user"). Compared on a log + scale, because the profiles are 20x apart and the useful question is which + order of magnitude the student is in, not the arithmetic distance. + + Returns (profile name, ratio of measured usage to that profile). + """ + light = LIGHT["google_cloud_run_service"]["monthly_requests"] + heavy = HEAVY["google_cloud_run_service"]["monthly_requests"] + if monthly_requests <= 0: + return "light", 0.0 + light_distance = abs(math.log(monthly_requests / light)) + heavy_distance = abs(math.log(monthly_requests / heavy)) + if heavy_distance < light_distance: + return "heavy", monthly_requests / heavy + return "light", monthly_requests / light diff --git a/tests/test_infracost.py b/tests/test_infracost.py index 0659e0f..b76359d 100644 --- a/tests/test_infracost.py +++ b/tests/test_infracost.py @@ -19,11 +19,13 @@ CostAnalysis, ResourceCost, _classify_category, + _report_scan_failure, + _resource_description, _row_to_resource_cost, + display_actual_costs, check_infracost_available, check_infracost_authenticated, display_estimate, - fetch_resource_costs, fetch_resource_costs_detailed, format_cost_for_confirmation, parse_infracost_scan_data, @@ -165,17 +167,19 @@ def test_parse_infracost_scan_data_handles_zero_cost(): # --------------------------------------------------------------------------- -# fetch_resource_costs (argv construction + row parsing) +# _run_inspect_rows (argv construction) # --------------------------------------------------------------------------- -def test_fetch_resource_costs_passes_file_flag_to_inspect(tmp_path): +def test_inspect_is_pinned_to_the_scan_file(tmp_path): # Guards against regressing to the global-cache form of `infracost inspect`: - # inspect must be pinned to the scan JSON we pass, via --file. + # inspect must be pinned to the scan JSON we pass, via --file. Without it, + # inspect reads infracost's "most recent scan" and can report a different + # workspace's costs. scan_json = tmp_path / "infracost-scan.json" scan_json.write_text("{}") with patch("deployml.utils.infracost.run_tool") as mock_run: mock_run.return_value = SimpleNamespace(returncode=0, stdout="[]", stderr="") - fetch_resource_costs(scan_json) + fetch_resource_costs_detailed(scan_json) tool, argv = mock_run.call_args[0][0], mock_run.call_args[0][1] # run_tool resolves the tool name to a real path, so the name is passed # separately from the args rather than as argv[0]. @@ -185,19 +189,14 @@ def test_fetch_resource_costs_passes_file_flag_to_inspect(tmp_path): assert argv[argv.index("--file") + 1] == str(scan_json) -def test_fetch_resource_costs_accumulates_duplicate_resource_types(tmp_path): - # Two Cloud Run services should collapse into one row with summed cost. - rows = [ - {"cost": "5.00", "columns": {"resource": "module.a.google_cloud_run_v2_service.x"}}, - {"cost": "3.00", "columns": {"resource": "module.b.google_cloud_run_v2_service.y"}}, - {"cost": "0", "columns": {"resource": "module.c.google_storage_bucket.z"}}, - ] +def test_inspect_failure_warns_instead_of_reporting_no_costs(capsys, tmp_path): + # A failed inspect used to render as "nothing costs money" underneath a + # non-zero headline total. It must say so instead. with patch("deployml.utils.infracost.run_tool") as mock_run: - mock_run.return_value = SimpleNamespace( - returncode=0, stdout=json.dumps(rows), stderr="" - ) - result = fetch_resource_costs(tmp_path / "scan.json") - assert result == [("Cloud Run", 8.0)] # zero-cost bucket dropped + mock_run.return_value = SimpleNamespace(returncode=1, stdout="", stderr="boom") + result = fetch_resource_costs_detailed(tmp_path / "scan.json") + assert result == [] + assert "per-resource cost breakdown" in capsys.readouterr().out # --------------------------------------------------------------------------- @@ -307,3 +306,149 @@ def test_display_estimate_no_lever_when_balanced(capsys): display_estimate(resources, analysis, "light") out = capsys.readouterr().out assert "Biggest lever" not in out # top fixed is 40% of total, below the 50% cutoff + + +# --------------------------------------------------------------------------- +# render_usage_yaml (per-address measured usage) +# --------------------------------------------------------------------------- + +def test_render_usage_yaml_pins_usage_to_terraform_addresses(): + # Measured usage differs per service, so it must key on the address. A type + # default would charge every Cloud Run service the whole stack's traffic. + out = render_usage_yaml({}, { + "module.experiment_tracking_mlflow.google_cloud_run_service.mlflow": { + "monthly_requests": 30000.0, + }, + "module.bigquery.google_bigquery_table.predictions[0]": { + "monthly_active_storage_gb": 1.5, + }, + }) + data = yaml.safe_load(out) + assert data["version"] == 0.1 + # An empty profile emits no type defaults, so unmeasured usage stays at zero + # rather than falling back to a light/heavy guess. + assert "resource_type_default_usage" not in data + pinned = data["resource_usage"] + assert pinned[ + "module.experiment_tracking_mlflow.google_cloud_run_service.mlflow" + ]["monthly_requests"] == 30000.0 + # Addresses with a count index stay intact through quoting. + assert "module.bigquery.google_bigquery_table.predictions[0]" in pinned + + +def test_render_usage_yaml_can_emit_both_sections(): + out = render_usage_yaml(LIGHT, {"module.a.google_storage_bucket.b": {"storage_gb": 2}}) + data = yaml.safe_load(out) + assert "resource_type_default_usage" in data and "resource_usage" in data + + +# --------------------------------------------------------------------------- +# _report_scan_failure (expired-token handling) +# --------------------------------------------------------------------------- + +def test_scan_failure_names_the_login_command_for_auth_errors(capsys): + # check_infracost_authenticated() can only see that a token file exists, so + # an expired token gets through the pre-flight check and fails here. + _report_scan_failure("Error: 401 Unauthorized: invalid API key") + out = capsys.readouterr().out + assert "infracost auth login" in out + + +def test_scan_failure_passes_through_non_auth_errors(capsys): + _report_scan_failure("Error: could not parse main.tf at line 12") + out = capsys.readouterr().out + assert "could not parse main.tf" in out + assert "auth login" not in out + + +# --------------------------------------------------------------------------- +# _resource_description (BigQuery queries vs storage) +# --------------------------------------------------------------------------- + +def test_bigquery_dataset_and_tables_get_distinct_descriptions(): + # Both live in the `bigquery` module, so a module-level description applied + # to both and labelled storage rows as query cost. + dataset = _resource_description( + "module.bigquery.google_bigquery_dataset.mlops", "google_bigquery_dataset" + ) + table = _resource_description( + "module.bigquery.google_bigquery_table.predictions", "google_bigquery_table" + ) + assert dataset != table + assert "quer" in dataset + assert "storage" in table + + +def test_cloud_run_jobs_are_labelled(): + # Emitted by the teardown / offline_scoring / explainability modules. + rc = _row_to_resource_cost( + {"cost": "0.02", "columns": {"resource": "module.teardown.google_cloud_run_v2_job.j"}} + ) + assert rc.label == "Cloud Run Job" + assert rc.description == "scheduled batch job" + + +# --------------------------------------------------------------------------- +# display_actual_costs (measured view) +# --------------------------------------------------------------------------- + +def _measured(**kwargs): + from deployml.utils.measured_usage import MeasuredUsage + return MeasuredUsage(**kwargs) + + +def test_display_actual_costs_shows_measurements_uptime_and_profile_label(capsys): + sql = "module.cloud_sql_postgres.google_sql_database_instance.pg" + run = "module.experiment_tracking_mlflow.google_cloud_run_service.mlflow" + resources = [ + _rc(sql, "google_sql_database_instance", 34.55, "fixed", "Cloud SQL", + "MLflow's backend database"), + _rc(run, "google_cloud_run_service", 0.71, "usage", "Cloud Run", + "MLflow tracking server"), + ] + measured = _measured( + window_days=7.0, + resource_usage={run: {"monthly_requests": 30000.0}}, + details={run: "30.0k req/mo, 240 ms median"}, + sql_uptime={sql: 0.5}, + notes=["BigQuery streaming inserts were not reported."], + cloud_run_monthly_requests=30000.0, + ) + display_actual_costs( + resources, CostAnalysis(35.26, "USD", 71, 10, 61), measured, + "gcp-mlops-stack", "light", 0.6, deployed_days=7.0, + ) + out = capsys.readouterr().out + + assert "ACTUAL COST" in out and "gcp-mlops-stack" in out + assert "last 7 days" in out + assert "Run-rate" in out + # The measured numbers are shown next to the price they produced. + assert "30.0k req/mo, 240 ms median" in out + # Cloud SQL bills for existing, so uptime is reported rather than discounted. + assert "up 50% of window" in out + assert "$ 34.55" in out + # light/heavy is now a label on real usage, not a pricing input. + assert "LIGHT user" in out + assert "Biggest lever" in out and "deployml destroy" in out + # A $0 is always explained. + assert "About these numbers" in out and "streaming inserts" in out + # ~$8.23 = 35.26 * 7/30 + assert "Charged so far" in out + + +def test_display_actual_costs_omits_profile_label_without_traffic(capsys): + sql = "module.cloud_sql_postgres.google_sql_database_instance.pg" + resources = [ + _rc(sql, "google_sql_database_instance", 34.55, "fixed", "Cloud SQL", "db"), + ] + measured = _measured(window_days=30.0, notes=["Cloud Run served no requests."]) + display_actual_costs( + resources, CostAnalysis(34.55, "USD", 71, 10, 61), measured, + "ws", "light", 0.0, + ) + out = capsys.readouterr().out + # Calling someone a "light user" on zero data would be a guess. + assert "LIGHT user" not in out + assert "Charged so far" not in out # no deploy timestamp available + assert "Cloud Run served no requests." in out diff --git a/tests/test_measured_usage.py b/tests/test_measured_usage.py new file mode 100644 index 0000000..2cb6bbb --- /dev/null +++ b/tests/test_measured_usage.py @@ -0,0 +1,328 @@ +""" +Tests for reading real GCP usage out of Cloud Monitoring. + +Mocked at two boundaries, matching tests/test_infracost.py: `run_tool` for the +terraform/gcloud shell-outs, and `requests.get` for the Monitoring REST API. +Nothing here touches the network or a real project. +""" + +import json +from types import SimpleNamespace +from unittest.mock import patch + +from deployml.utils.measured_usage import ( + DeployedResource, + MonitoringError, + build_resource_map, + collect_measured_usage, + human_count, + query_timeseries, +) +from deployml.utils.usage_profiles import classify_usage_profile + +_GB = 1024 ** 3 +_TB = 1024 ** 4 + + +# --------------------------------------------------------------------------- +# Canned data helpers +# --------------------------------------------------------------------------- + +def _response(payload, status=200): + return SimpleNamespace(status_code=status, json=lambda: payload) + + +def _series(resource_labels, value, metric_labels=None): + """One Monitoring time series collapsed to a single aligned point.""" + return { + "metric": {"labels": metric_labels or {}}, + "resource": {"labels": resource_labels}, + "points": [{"value": {"doubleValue": float(value)}}], + } + + +def _fake_get(by_metric): + """Serve canned series based on which metric the filter asks for.""" + def _get(url, params=None, headers=None, timeout=None): + wanted = (params or {}).get("filter", "") + for metric, series in by_metric.items(): + if metric in wanted: + return _response({"timeSeries": series}) + return _response({"timeSeries": []}) + return _get + + +def _resources(): + return [ + DeployedResource("module.experiment_tracking_mlflow.google_cloud_run_service.mlflow", + "google_cloud_run_service", "mlflow-server"), + DeployedResource("module.model_serving_fastapi.google_cloud_run_service.api", + "google_cloud_run_service", "fastapi-mlflow-server"), + DeployedResource("module.artifact_tracking.google_storage_bucket.artifacts", + "google_storage_bucket", "mlflow-artifacts-demo"), + DeployedResource("module.bigquery.google_bigquery_dataset.mlops", + "google_bigquery_dataset", "mlops"), + DeployedResource("module.bigquery.google_bigquery_table.predictions", + "google_bigquery_table", "predictions"), + DeployedResource("module.bigquery.google_bigquery_table.ground_truth", + "google_bigquery_table", "ground_truth"), + DeployedResource("module.cloud_sql_postgres.google_sql_database_instance.pg", + "google_sql_database_instance", "mlops-postgres"), + ] + + +def _measure(by_metric, days=7.0, resources=None): + with patch("deployml.utils.measured_usage.access_token", return_value="token"), \ + patch("deployml.utils.measured_usage.requests.get", side_effect=_fake_get(by_metric)): + return collect_measured_usage("demo-project", resources or _resources(), days) + + +# --------------------------------------------------------------------------- +# build_resource_map (terraform address <-> GCP name) +# --------------------------------------------------------------------------- + +_SHOW_JSON = { + "values": { + "root_module": { + "resources": [ + {"address": "google_project_service.run", "type": "google_project_service", + "values": {"service": "run.googleapis.com"}}, + ], + "child_modules": [ + { + "address": "module.cloud_sql_postgres", + "resources": [ + {"address": "module.cloud_sql_postgres.google_sql_database_instance.pg", + "type": "google_sql_database_instance", + "values": {"name": "mlops-postgres"}}, + ], + "child_modules": [ + { + "address": "module.bigquery", + "resources": [ + # BigQuery keeps its name under table_id, not name. + {"address": "module.bigquery.google_bigquery_table.predictions", + "type": "google_bigquery_table", + "values": {"table_id": "predictions"}}, + ], + } + ], + } + ], + } + } +} + + +def test_build_resource_map_walks_nested_modules_and_type_specific_names(tmp_path): + with patch("deployml.utils.measured_usage.run_tool") as mock_run: + mock_run.return_value = SimpleNamespace( + returncode=0, stdout=json.dumps(_SHOW_JSON), stderr="" + ) + result = build_resource_map(tmp_path) + argv = mock_run.call_args[0][1] + + # Machine-readable form, not the human output of `terraform state show`. + assert argv == ["show", "-json"] + by_address = {r.address: r for r in result} + # Nested child_modules are reached, and google_project_service has no name so it drops. + assert set(by_address) == { + "module.cloud_sql_postgres.google_sql_database_instance.pg", + "module.bigquery.google_bigquery_table.predictions", + } + assert by_address["module.bigquery.google_bigquery_table.predictions"].name == "predictions" + + +def test_build_resource_map_returns_empty_when_state_unreadable(tmp_path): + with patch("deployml.utils.measured_usage.run_tool") as mock_run: + mock_run.return_value = SimpleNamespace(returncode=1, stdout="", stderr="no state") + assert build_resource_map(tmp_path) == [] + + +# --------------------------------------------------------------------------- +# query_timeseries (request shape + error mapping) +# --------------------------------------------------------------------------- + +def test_query_timeseries_sends_window_as_alignment_period(): + from datetime import datetime, timedelta, timezone + end = datetime(2026, 9, 12, tzinfo=timezone.utc) + start = end - timedelta(days=7) + with patch("deployml.utils.measured_usage.requests.get") as mock_get: + mock_get.return_value = _response({"timeSeries": []}) + query_timeseries("p", "tok", "run.googleapis.com/request_count", + start, end, "ALIGN_SUM") + params = mock_get.call_args.kwargs["params"] + headers = mock_get.call_args.kwargs["headers"] + + # One aligned point per series, covering the whole window. + assert params["aggregation.alignmentPeriod"] == f"{7 * 24 * 3600}s" + assert params["aggregation.perSeriesAligner"] == "ALIGN_SUM" + assert 'metric.type = "run.googleapis.com/request_count"' == params["filter"] + # No cross-series reducer: we need resource.labels intact to attribute usage. + assert "aggregation.crossSeriesReducer" not in params + assert headers["Authorization"] == "Bearer tok" + + +def test_query_timeseries_raises_with_a_fix_hint_on_permission_denied(): + from datetime import datetime, timedelta, timezone + end = datetime(2026, 9, 12, tzinfo=timezone.utc) + with patch("deployml.utils.measured_usage.requests.get") as mock_get: + mock_get.return_value = _response({}, status=403) + try: + query_timeseries("demo-project", "tok", "run.googleapis.com/request_count", + end - timedelta(days=1), end, "ALIGN_SUM") + except MonitoringError as e: + assert "monitoring.viewer" in e.hint + assert "demo-project" in e.hint + else: + raise AssertionError("expected MonitoringError") + + +def test_monitoring_failure_on_the_first_metric_propagates(): + # The first metric acts as the probe: a dead API must surface as one clear + # error, not as a note attached to every metric. + with patch("deployml.utils.measured_usage.access_token", return_value="token"), \ + patch("deployml.utils.measured_usage.requests.get") as mock_get: + mock_get.return_value = _response({}, status=403) + try: + collect_measured_usage("demo-project", _resources(), 7.0) + except MonitoringError: + pass + else: + raise AssertionError("expected MonitoringError") + + +# --------------------------------------------------------------------------- +# collect_measured_usage (metric -> infracost key mapping) +# --------------------------------------------------------------------------- + +def test_cloud_run_requests_are_scaled_from_the_window_to_a_month(): + usage = _measure({ + "run.googleapis.com/request_count": [ + # Split by response code, as Cloud Run actually reports it. + _series({"service_name": "mlflow-server"}, 5000, {"response_code_class": "2xx"}), + _series({"service_name": "mlflow-server"}, 2000, {"response_code_class": "4xx"}), + _series({"service_name": "fastapi-mlflow-server"}, 700), + # A service in the project that this stack did not deploy. + _series({"service_name": "someone-elses-api"}, 999999), + ], + "run.googleapis.com/request_latencies": [ + _series({"service_name": "mlflow-server"}, 240), + ], + }, days=7.0) + + mlflow = "module.experiment_tracking_mlflow.google_cloud_run_service.mlflow" + # 7000 requests over 7 days -> 30000 per 30-day month. + assert usage.resource_usage[mlflow]["monthly_requests"] == 30000.0 + assert usage.resource_usage[mlflow]["average_request_duration_ms"] == 240.0 + assert usage.resource_usage[ + "module.model_serving_fastapi.google_cloud_run_service.api" + ]["monthly_requests"] == 3000.0 + # Unrelated services must not leak into this deployment's cost. + assert usage.cloud_run_monthly_requests == 33000.0 + assert "240 ms median" in usage.details[mlflow] + + +def test_gcs_operations_are_split_into_billing_classes_by_method(): + usage = _measure({ + "run.googleapis.com/request_count": [], + "storage.googleapis.com/storage/total_bytes": [ + _series({"bucket_name": "mlflow-artifacts-demo"}, 4 * _GB, {"storage_class": "STANDARD"}), + ], + "storage.googleapis.com/api/request_count": [ + _series({"bucket_name": "mlflow-artifacts-demo"}, 70, {"method": "storage.objects.insert"}), + _series({"bucket_name": "mlflow-artifacts-demo"}, 140, {"method": "storage.objects.list"}), + _series({"bucket_name": "mlflow-artifacts-demo"}, 700, {"method": "storage.objects.get"}), + ], + }, days=7.0) + + bucket = "module.artifact_tracking.google_storage_bucket.artifacts" + entry = usage.resource_usage[bucket] + assert entry["storage_gb"] == 4.0 # a level, not scaled + assert entry["monthly_class_a_operations"] == 900.0 # (70+140) * 30/7 + assert entry["monthly_class_b_operations"] == 3000.0 # 700 * 30/7 + # The approximation is disclosed rather than hidden. + assert any("class A/B" in note for note in usage.notes) + + +def test_bigquery_storage_splits_evenly_but_the_total_stays_exact(): + usage = _measure({ + "run.googleapis.com/request_count": [], + "bigquery.googleapis.com/query/scanned_bytes": [ + _series({"project_id": "demo-project"}, 0.7 * _TB), + ], + "bigquery.googleapis.com/storage/stored_bytes": [ + _series({"dataset_id": "mlops"}, 6 * _GB), + ], + }, days=7.0) + + tables = [ + "module.bigquery.google_bigquery_table.predictions", + "module.bigquery.google_bigquery_table.ground_truth", + ] + per_table = [usage.resource_usage[t]["monthly_active_storage_gb"] for t in tables] + assert per_table == [3.0, 3.0] + # BigQuery prices storage linearly, so an even split gives the exact total. + assert abs(sum(per_table) - 6.0) < 1e-6 + dataset = "module.bigquery.google_bigquery_dataset.mlops" + assert abs(usage.resource_usage[dataset]["monthly_queries_tb"] - 3.0) < 1e-3 + + +def test_cloud_sql_uptime_is_recorded_against_its_address(): + usage = _measure({ + "run.googleapis.com/request_count": [], + "cloudsql.googleapis.com/database/up": [ + _series({"database_id": "demo-project:mlops-postgres"}, 0.5), + ], + }, days=7.0) + assert usage.sql_uptime[ + "module.cloud_sql_postgres.google_sql_database_instance.pg" + ] == 0.5 + + +def test_cloud_sql_uptime_uses_an_aligner_the_api_accepts(): + # database/up is GAUGE/INT64, not BOOL. ALIGN_FRACTION_TRUE reads like the + # right choice but the API rejects it with HTTP 400, which degraded the + # uptime line to a "could not read" note. ALIGN_MEAN over a 0/1 gauge is the + # same number and is accepted. + seen = {} + + def _get(url, params=None, headers=None, timeout=None): + wanted = params["filter"] + if "database/up" in wanted: + seen["aligner"] = params["aggregation.perSeriesAligner"] + return _response({"timeSeries": []}) + + with patch("deployml.utils.measured_usage.access_token", return_value="token"), \ + patch("deployml.utils.measured_usage.requests.get", side_effect=_get): + collect_measured_usage("demo-project", _resources(), 7.0) + + assert seen["aligner"] == "ALIGN_MEAN" + + +def test_nothing_measured_means_notes_not_invented_numbers(): + usage = _measure({"run.googleapis.com/request_count": []}, days=7.0) + assert usage.is_empty() + assert usage.resource_usage == {} + # Every $0 must come with a reason the display can print. + assert any("no requests" in note for note in usage.notes) + assert any("GCS" in note for note in usage.notes) + + +# --------------------------------------------------------------------------- +# Formatting + profile labelling +# --------------------------------------------------------------------------- + +def test_human_count_compacts_large_numbers(): + assert human_count(1_250_000) == "1.2M" + assert human_count(30_000) == "30.0k" + assert human_count(42) == "42" + + +def test_classify_usage_profile_picks_the_nearer_profile_on_a_log_scale(): + assert classify_usage_profile(50_000) == ("light", 1.0) + assert classify_usage_profile(1_000_000) == ("heavy", 1.0) + # Crossover is the geometric mean of the two profiles (~224k), not the midpoint. + assert classify_usage_profile(200_000)[0] == "light" + assert classify_usage_profile(300_000)[0] == "heavy" + assert classify_usage_profile(0) == ("light", 0.0) From fd8f24c3ed24d98a6c8289c9344d8fcd1dd98950 Mon Sep 17 00:00:00 2001 From: silversurfer Date: Sat, 12 Sep 2026 19:36:39 -0700 Subject: [PATCH 4/5] docs: document estimate and costs, and the fixed-vs-usage split docs/api/cli-commands.md never mentioned either cost command. Both now have a section with options, sample output, and their prerequisites. docs/features/costs.md claimed both commands show $0 for usage-based services, which stopped being true when estimate became usage-aware and is now the whole point of costs. Rewritten around the two questions the commands answer -- what will this cost (assumed usage) versus what is it costing me (measured usage) -- with the fixed-vs-usage distinction up front, since that is what makes the numbers legible: how hard you use the stack barely matters next to whether the database is running. tests/README.md described only the helpers tests. Now lists all six test modules, the mocking conventions, and the regression guards worth not breaking. --- docs/api/cli-commands.md | 95 ++++++++++++++++++++++++ docs/features/costs.md | 155 ++++++++++++++++++++++++++++++++------- tests/README.md | 57 ++++++++++---- 3 files changed, 267 insertions(+), 40 deletions(-) diff --git a/docs/api/cli-commands.md b/docs/api/cli-commands.md index 786bb50..0b0ed5a 100644 --- a/docs/api/cli-commands.md +++ b/docs/api/cli-commands.md @@ -59,6 +59,49 @@ Builds run on Cloud Build, so a local Docker daemon is not required for GCP mode --- +## `deployml estimate` + +Predict monthly cost from your config, without deploying or touching any cloud +resources. Reads the Terraform deployml would generate and prices it. + +```bash +deployml estimate +deployml estimate --profile heavy +``` + +**Options:** +- `--config-path`, `-c`: Path to config YAML. Default `config.yaml`. +- `--profile`, `-p`: Usage assumption, `light` (default) or `heavy`. Light is a + typical student working through the coursework; heavy is a busy course project, + roughly 10-20x the traffic. + +Infracost prices usage-based services at zero usage by default, which would report +Cloud Run, BigQuery, and GCS at $0 and make the estimate look far cheaper than +reality. The profile supplies a realistic usage assumption instead, and the output +separates the two kinds of cost: + +``` + ~$35 / month ($34.55 fixed + ~$0.92 usage) USD + + ALWAYS-ON (billed 24/7 even if you never use the stack) + $ 34.55 Cloud SQL MLflow's backend database + + USAGE-BASED (scales with activity · profile: light) + $ 0.44 BigQuery prediction & feature queries + $ 0.24 GCS Bucket MLflow model artifacts + $ 0.24 Cloud Run MLflow tracking server + + Biggest lever: Cloud SQL is 97% of your cost and runs 24/7. +``` + +Changing the profile moves only the usage line; the always-on baseline does not +move, which is the point. Not supported for GKE deployments. + +Requires infracost installed and authenticated (`infracost auth login`), but no +cloud credentials. + +--- + ## `deployml deploy` Deploy infrastructure from a YAML config file. Prompts for confirmation by default. @@ -80,6 +123,58 @@ First-time deploy takes about 20 minutes because Cloud SQL Postgres provisioning --- +## `deployml costs` + +Show what a running deployment actually costs, measured from real GCP usage +rather than assumed. + +```bash +deployml costs +deployml costs --days 7 +``` + +**Options:** +- `--config-path`, `-c`: Path to config YAML. Default `config.yaml`. +- `--days`, `-d`: How many days of real usage to measure, 1-90. Default 30. If the + deployment is younger than the window, the window shrinks to the deployment's age + so the run-rate is not diluted by days that never happened. + +Where `estimate` has to assume your usage, this reads it: requests served per Cloud +Run service, bytes stored in GCS, bytes scanned by BigQuery, and how much of the +window the database was up. Those measurements are pulled from Cloud Monitoring and +priced through infracost, so each line shows the number that produced it: + +``` + ACTUAL COST · gcp-mlops-stack + Measured from your real GCP usage over the last 7 days. + + Run-rate: ~$36 / month ($34.55 always-on + ~$1.42 measured usage) USD + Charged so far: ~$8.40 (7.0 days since deploy, approximate) + + ALWAYS-ON (billed 24/7 while the stack exists) + $ 34.55 Cloud SQL MLflow's backend database · up 100% of window + + USAGE-BASED (measured over 7 days, scaled to a month) + $ 0.71 Cloud Run MLflow tracking server · 30.0k req/mo, 240 ms median + $ 0.44 BigQuery prediction & feature queries · 0.041 TB/mo scanned + + You are a LIGHT user — your real usage is 0.6x the 'light' profile. +``` + +Anything Cloud Monitoring cannot report is priced at $0 and listed under +"About these numbers", along with the places a measurement is an approximation, +so a zero is always explained rather than silently assumed. + +This is not a billing statement. It prices your measured usage at list rates; +for the amount you were actually charged, see the GCP Billing Console. + +**Requires:** +- infracost installed and authenticated +- Application Default Credentials: `gcloud auth application-default login` +- `roles/monitoring.viewer` on the project (checked by `deployml doctor --project-id`) + +--- + ## `deployml get-urls` Print service URLs from the last deployment and write them to a `.env` file. Database credentials are masked. diff --git a/docs/features/costs.md b/docs/features/costs.md index 9ab3224..35c278f 100644 --- a/docs/features/costs.md +++ b/docs/features/costs.md @@ -1,6 +1,15 @@ # Cost Estimates -deployml integrates with [Infracost](https://www.infracost.io) to show infrastructure costs. Both commands read your Terraform configuration — they price always-on resources like Cloud SQL accurately, but usage-based services (BigQuery, GCS, Cloud Run) show $0 since costs depend on actual usage. Check the GCP Billing Console for real usage charges. +deployml uses [Infracost](https://www.infracost.io) to answer two different +questions about money, with one command each: + +| Command | Question | Where the usage numbers come from | +|---|---|---| +| `deployml estimate` | What *will* this cost? | A light/heavy assumption, since the stack does not exist yet | +| `deployml costs` | What *is* this costing me? | Cloud Monitoring — your real requests, bytes, and uptime | + +Neither is a billing statement. Both price resources at list rates; for the amount +you were actually charged, check the GCP Billing Console. ## Setup @@ -9,55 +18,151 @@ brew install infracost infracost auth login ``` -Run `deployml doctor` to confirm infracost is installed and authenticated. +`deployml costs` additionally needs Application Default Credentials and read access +to metrics: + +```bash +gcloud auth application-default login +``` + +Run `deployml doctor --project-id YOUR_PROJECT` to confirm infracost is installed and +authenticated and that you hold `roles/monitoring.viewer`. + +## Fixed cost vs usage cost + +This is the distinction that makes the numbers make sense, and both commands are +built around it. + +- **Always-on** resources bill 24/7 for existing, whether or not anyone touches + them. Cloud SQL is the one that matters here. +- **Usage-based** resources bill per request, per GB, per byte scanned. Cloud Run, + BigQuery, and GCS all scale to zero. -## Commands +Infracost assumes **zero usage** unless you tell it otherwise. That is why a naive +estimate reports $0 for every usage-based service and makes the stack look free +apart from the database. Both commands fix this, in different ways. + +## Before deploying: `deployml estimate` -**Before deploying** — estimates cost from your config without touching any infrastructure: ```bash deployml estimate +deployml estimate --profile heavy ``` -**While deployed** — scans your actual deployed Terraform workspace: +Renders the Terraform your config would produce, prices it against a usage profile, +and splits the result: + +``` + ~$35 / month ($34.55 fixed + ~$0.92 usage) USD + + ALWAYS-ON (billed 24/7 even if you never use the stack) + $ 34.55 Cloud SQL MLflow's backend database + + USAGE-BASED (scales with activity · profile: light) + $ 0.44 BigQuery prediction & feature queries + $ 0.24 GCS Bucket MLflow model artifacts + $ 0.24 Cloud Run MLflow tracking server + + Biggest lever: Cloud SQL is 97% of your cost and runs 24/7. + Switch MLflow to a SQLite backend to drop this to ~$0/month. +``` + +`light` is a student working through the coursework; `heavy` is a busy course +project at roughly 10-20x the traffic. Switching profiles moves only the usage line +— the always-on baseline does not budge. That is the lesson: how hard you use the +stack barely matters next to whether the database is running. + +No cloud credentials needed, and nothing is deployed. + +## While deployed: `deployml costs` + ```bash deployml costs +deployml costs --days 7 +``` + +Once the stack exists we can stop guessing. This reads your real usage out of Cloud +Monitoring — requests per service, bytes stored, bytes scanned, database uptime — +and prices that instead: + +``` + ACTUAL COST · gcp-mlops-stack + Measured from your real GCP usage over the last 7 days. + + Run-rate: ~$36 / month ($34.55 always-on + ~$1.42 measured usage) USD + Charged so far: ~$8.40 (7.0 days since deploy, approximate) + + ALWAYS-ON (billed 24/7 while the stack exists) + $ 34.55 Cloud SQL MLflow's backend database · up 100% of window + + USAGE-BASED (measured over 7 days, scaled to a month) + $ 0.71 Cloud Run MLflow tracking server · 30.0k req/mo, 240 ms median + $ 0.44 BigQuery prediction & feature queries · 0.041 TB/mo scanned + + You are a LIGHT user — your real usage is 0.6x the 'light' profile. + + Biggest lever: Cloud SQL is 96% of your cost and runs 24/7. + Tear the stack down when you are not using it: deployml destroy ``` -Both commands show a breakdown of which resources cost money and how much. +Every usage line carries the measurement behind it, so you can see *why* a number +is what it is. Once you have run this, `light` and `heavy` stop being assumptions +you are billed for and become a label on your actual behaviour. + +Counters (requests, bytes scanned) are summed over the window and scaled to 30 +days. Levels (bytes stored) are averaged over the window. Anything Monitoring does +not report is priced at $0 and listed under "About these numbers", together with +the places where a measurement is an approximation — a zero is always explained, +never silently assumed. ## Cost shown during deploy -`deployml deploy` automatically runs a cost estimate after `terraform plan` and shows it before the confirmation prompt: +`deployml deploy` runs the usage-aware estimate after `terraform plan` and shows the +total before the confirmation prompt: + ``` - Deploy stack? Monthly cost: ~$34.55 USD [y/N]: + Deploy stack? Monthly cost: ~$35.47 USD [y/N]: ``` +If infracost is missing or unauthenticated the estimate is skipped and the deploy +continues. + ## Configuration ```yaml cost_analysis: - enabled: true # set to false to skip (default: true) + enabled: true # set to false to skip the estimate during deploy (default: true) warning_threshold: 50.0 # warn if monthly cost exceeds this (default: 100.0) ``` ## Typical costs (Cloud Run stack) -A standard MLflow + FastAPI + Grafana deployment runs around **$34/month**, almost entirely Cloud SQL. Cloud Run, BigQuery, and GCS scale to zero and cost nothing at idle. - -- Cloud Run services cost $10-30 per month depending on traffic. -- Cloud SQL PostgreSQL ranges from $7/month for small instances to $25+ for production. -- Google Cloud Storage costs approximately $0.020 per GB per month. -- BigQuery storage costs $0.020 per GB per month with query costs based on data scanned. -- Cloud VMs cost approximately $25 per month for medium instances. -- GKE clusters have no management fee, but you pay for VM instances and load balancers. MLflow on GKE also provisions a small PersistentDisk for its data, a few cents per GB-month. Note that GKE can get expensive quickly. - - +A standard MLflow + FastAPI + Grafana deployment runs around **$35/month**, and +roughly 97% of that is the Cloud SQL instance. The usage-based services add well +under a dollar at student traffic levels. + +- Cloud SQL PostgreSQL: from ~$7/month for the smallest instance to $25+ for + production sizes. Billed continuously. +- Cloud Run: scales to zero, so a few cents per month at light traffic; $10-30/month + under sustained load. +- Google Cloud Storage: ~$0.020 per GB per month. +- BigQuery: ~$0.020 per GB per month storage, plus query cost per byte scanned. +- Cloud VMs: ~$25/month for medium instances. +- GKE: no management fee for one zonal cluster, but you pay for the VM instances and + load balancers, and MLflow on GKE provisions a small PersistentDisk. This adds up + quickly. ## Cost Optimization -Here are some tips to keep the costs low while you are learning: - -- Use SQLite instead of Cloud SQL whenever possible, particularly for development purposes and when your data is small. Set `backend_store_uri: sqlite` instead of `postgresql` to eliminate the Cloud SQL instance entirely. The minikube and GKE MLflow paths already do this, sqlite on a PersistentVolumeClaim, so they avoid the always-on Cloud SQL cost. -- Always run `deployml destroy` when you are done — Cloud SQL bills continuously, whether or not anyone is using it. -- Enable auto-teardown to prevent forgotten deployments. -- Use Cloud Run for variable workloads to take advantage of scale-to-zero pricing. +Tips to keep costs low while you are learning: + +- **Use SQLite instead of Cloud SQL** where you can. Set + `backend_store_uri: sqlite` instead of `postgresql` to remove the Cloud SQL + instance entirely — that is the single biggest saving available, since it deletes + ~97% of the bill. The minikube and GKE MLflow paths already do this, with sqlite on + a PersistentVolumeClaim. +- **Always run `deployml destroy`** when you are done. Cloud SQL bills continuously, + used or not. `deployml costs` reports database uptime so you can see how much of + the month you actually left it running. +- **Enable auto-teardown** so a forgotten deployment cannot bill all semester. +- **Prefer Cloud Run for variable workloads** to take advantage of scale-to-zero. diff --git a/tests/README.md b/tests/README.md index 600769e..c9384c2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,28 +1,55 @@ # Tests -Unit tests for the pure-Python helpers and validators in `deployml.cli` and -`deployml.utils.helpers`. These tests do NOT touch GCP. Subprocess calls are -mocked. +Unit tests for the pure-Python helpers, validators, and cost logic in +`deployml.cli` and `deployml.utils`. These tests do NOT touch GCP. Subprocess +calls and HTTP calls are mocked. Run with: ```bash -conda run -n ml pytest tests/ -v +PYTHONPATH=src pytest tests/ -v ``` -Or from the project root: +Or, if the package is installed in your environment: ```bash -pytest tests/ +conda run -n ml pytest tests/ -v ``` -What is covered: -- `_load_config_or_exit`: valid mapping, malformed YAML, non-mapping, empty. -- `_validate_deploy_config_or_exit`: every documented error path. -- `validate_gcp_project`, `validate_gcp_region`: subprocess mocked. -- `get_missing_iam_roles`: owner short-circuit and the diff path. -- `check_gcp_adc`, `check_bq`, `check_docker_daemon`, `get_terraform_version`: subprocess mocked. +## Layout + +| File | Covers | +|---|---| +| `test_helpers.py` | config loading/validation, GCP validators, tool + auth probes | +| `test_doctor.py` | the prereq checker's tool-detection pattern | +| `test_infracost.py` | infracost invocation, cost parsing, and the estimate/actual display views | +| `test_measured_usage.py` | reading real usage from Cloud Monitoring and mapping it to infracost keys | +| `test_platform_compat.py` | `run_tool` / `resolve_tool` cross-platform behaviour | +| `test_gke_destroy.py` | GKE teardown manifest and disk cleanup logic | + +## Conventions + +- Plain module-level `def test_*` functions. No test classes. +- Only built-in fixtures (`tmp_path`, `capsys`); no `conftest.py`. +- `unittest.mock.patch` used as a **context manager**, not a decorator. +- External tools are mocked at the `run_tool` import site in the module under + test, returning `SimpleNamespace(returncode=, stdout=, stderr=)` — not by + patching `subprocess.run`. +- HTTP is mocked at `requests.get` in the module under test. + +## Notable regression guards + +- `test_inspect_is_pinned_to_the_scan_file`: `infracost inspect` must be passed + `--file`. Without it, inspect reads infracost's global "most recent scan" cache + and can report a different workspace's costs. +- `test_nothing_measured_means_notes_not_invented_numbers`: unmeasurable usage + must stay at $0 *with a printed reason*, never fall back to a guessed profile. +- `test_bigquery_storage_splits_evenly_but_the_total_stays_exact`: BigQuery + metrics are project-scoped, so per-table attribution is an even split; the test + pins the property that makes this acceptable — linear pricing keeps the total exact. + +## What is NOT covered -What is NOT covered: -- Anything that requires a real GCP project (deploy, destroy, init API enable). - Those are validated by the end-to-end walkthrough in CLAUDE_INSTRUCTIONS. +- Anything requiring a real GCP project (deploy, destroy, init API enable) or a + real infracost API call. Those are validated by the end-to-end walkthrough. +- The Typer command bodies themselves; there is no `CliRunner` harness yet. From 8cd4146163035493de7242315deebda7aa624ef7 Mon Sep 17 00:00:00 2001 From: silversurfer Date: Sun, 13 Sep 2026 18:04:56 -0700 Subject: [PATCH 5/5] test: pin the infracost address contract, and escape for_each addresses Verified the one load-bearing assumption in the measured-usage path: that the terraform address `terraform show -json` reports is byte-identical to the address infracost keys `resource_usage` on. If they disagree the usage file is silently ignored and every usage line prices at $0 -- the exact bug measured usage exists to fix, just relocated. Checked against a real cached `infracost scan` of the rendered deployml Cloud Run stack (infracost records each resource's address in its "name" field) and diffed it against build_resource_map() run on the deployed state. All 10 costable addresses match exactly, count indices included: module.experiment_tracking_mlflow[0].google_cloud_run_service.mlflow[0] google_storage_bucket.artifact_tracking_mlflow_artifact ... Those strings are now pinned in a test, since the tempting "cleanup" here is to normalise or strip the count index, which would break the binding silently. Reading the real scan also turned up an unescaped-quote bug. for_each addresses embed their own quotes, e.g. module.cloud_sql_postgres.google_project_service.required["cloudkms.googleapis.com"] and render_usage_yaml wrapped addresses with a naive f-string, so such an address closed the YAML string early and made the whole usage file unparseable. Not reachable today -- no measured type uses for_each -- but it would have been a silent trap for whoever added one. Now emitted via json.dumps, whose escaping is valid YAML double-quoted style. Tests: 106 -> 108. --- src/deployml/utils/usage_profiles.py | 10 ++- tests/test_infracost.py | 13 +++- tests/test_measured_usage.py | 99 ++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/src/deployml/utils/usage_profiles.py b/src/deployml/utils/usage_profiles.py index 24ffbda..b0024b8 100644 --- a/src/deployml/utils/usage_profiles.py +++ b/src/deployml/utils/usage_profiles.py @@ -11,6 +11,7 @@ (see infracost's usage-file schema). To tune assumptions, edit the numbers here. """ +import json import math from typing import Dict, Optional, Tuple @@ -102,8 +103,13 @@ def render_usage_yaml( if resource_usage: lines.append("resource_usage:") for address, fields in resource_usage.items(): - # Addresses contain dots and brackets, so they need quoting. - lines.append(f' "{address}":') + # Addresses contain dots and brackets, so they need quoting -- and a + # for_each address embeds its own quotes, e.g. + # google_project_service.required["cloudkms.googleapis.com"] + # which would close the string early and produce invalid YAML. + # json.dumps emits a correctly escaped double-quoted scalar, which is + # also valid YAML double-quoted style. + lines.append(f" {json.dumps(address)}:") for key, value in fields.items(): lines.append(f" {key}: {value}") diff --git a/tests/test_infracost.py b/tests/test_infracost.py index b76359d..bc2ebd6 100644 --- a/tests/test_infracost.py +++ b/tests/test_infracost.py @@ -7,9 +7,7 @@ 4. format_cost_for_confirmation — pure function """ -import json import os -from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -452,3 +450,14 @@ def test_display_actual_costs_omits_profile_label_without_traffic(capsys): assert "LIGHT user" not in out assert "Charged so far" not in out # no deploy timestamp available assert "Cloud Run served no requests." in out + + +def test_render_usage_yaml_escapes_for_each_addresses(): + # A for_each address embeds its own quotes, e.g. + # google_project_service.required["cloudkms.googleapis.com"] + # Naive f-string quoting closes the YAML string early and the whole usage file + # becomes unparseable, which infracost would reject or misread. + addr = 'module.x.google_storage_bucket.buckets["prod"]' + out = render_usage_yaml({}, {addr: {"storage_gb": 5}}) + data = yaml.safe_load(out) # must not raise + assert data["resource_usage"][addr]["storage_gb"] == 5 diff --git a/tests/test_measured_usage.py b/tests/test_measured_usage.py index 2cb6bbb..cbb605f 100644 --- a/tests/test_measured_usage.py +++ b/tests/test_measured_usage.py @@ -326,3 +326,102 @@ def test_classify_usage_profile_picks_the_nearer_profile_on_a_log_scale(): assert classify_usage_profile(200_000)[0] == "light" assert classify_usage_profile(300_000)[0] == "heavy" assert classify_usage_profile(0) == ("light", 0.0) + + +# --------------------------------------------------------------------------- +# Address contract with infracost +# --------------------------------------------------------------------------- + +# The exact addresses infracost reported for the deployml Cloud Run stack, taken +# from a real `infracost scan` of the rendered workspace (it keys each resource by +# its terraform address in the "name" field). Measured usage is pinned to these +# strings via the usage file's `resource_usage` block, so if terraform's address +# and infracost's address ever stop agreeing, the usage file is silently ignored +# and every usage line prices at $0 -- the exact bug measured usage exists to fix. +# +# Note the count indices: `module.experiment_tracking_mlflow[0]` and `mlflow[0]` +# are part of the key. Stripping or normalising them breaks the binding. +_INFRACOST_ADDRESSES = { + "google_storage_bucket.artifact_tracking_mlflow_artifact", + "module.bigquery.google_bigquery_dataset.mlops", + "module.bigquery.google_bigquery_table.drift_metrics", + "module.bigquery.google_bigquery_table.ground_truth", + "module.bigquery.google_bigquery_table.offline_features", + "module.bigquery.google_bigquery_table.predictions", + "module.cloud_sql_postgres.google_sql_database_instance.postgres", + "module.experiment_tracking_mlflow[0].google_cloud_run_service.mlflow[0]", + "module.model_monitoring_grafana[0].google_cloud_run_service.grafana", + "module.model_serving_fastapi[0].google_cloud_run_service.fastapi", +} + +# Shaped like `terraform show -json` output for that same stack: addresses exactly +# as terraform emits them, plus a data source and a free resource that must drop. +_REAL_SHAPE = { + "values": { + "root_module": { + "resources": [ + {"address": "google_storage_bucket.artifact_tracking_mlflow_artifact", + "mode": "managed", "type": "google_storage_bucket", + "values": {"name": "mlflow-artifacts-deployml-test-dev"}}, + ], + "child_modules": [ + {"address": "module.bigquery", "resources": [ + {"address": "module.bigquery.google_bigquery_dataset.mlops", + "mode": "managed", "type": "google_bigquery_dataset", + "values": {"dataset_id": "mlops"}}, + ] + [ + {"address": f"module.bigquery.google_bigquery_table.{t}", + "mode": "managed", "type": "google_bigquery_table", + "values": {"table_id": t}} + for t in ("drift_metrics", "ground_truth", "offline_features", "predictions") + ]}, + {"address": "module.cloud_sql_postgres", "resources": [ + {"address": "module.cloud_sql_postgres.google_sql_database_instance.postgres", + "mode": "managed", "type": "google_sql_database_instance", + "values": {"name": "mlflow-postgres-deployml-test-dev"}}, + # Free resource: enabling an API costs nothing and infracost + # marks it is_free, but it still has a name so it lands in the map. + {"address": 'module.cloud_sql_postgres.google_project_service.required["cloudkms.googleapis.com"]', + "mode": "managed", "type": "google_project_service", + "values": {"service": "cloudkms.googleapis.com"}}, + ]}, + {"address": "module.experiment_tracking_mlflow[0]", "resources": [ + {"address": "module.experiment_tracking_mlflow[0].google_cloud_run_service.mlflow[0]", + "mode": "managed", "type": "google_cloud_run_service", + "values": {"name": "mlflow-server"}}, + # Data source, not a billable resource. + {"address": "module.experiment_tracking_mlflow[0].data.google_project.current", + "mode": "data", "type": "google_project", + "values": {"name": "deployml-TEST-DEV"}}, + ]}, + {"address": "module.model_monitoring_grafana[0]", "resources": [ + {"address": "module.model_monitoring_grafana[0].google_cloud_run_service.grafana", + "mode": "managed", "type": "google_cloud_run_service", + "values": {"name": "grafana-server"}}, + ]}, + {"address": "module.model_serving_fastapi[0]", "resources": [ + {"address": "module.model_serving_fastapi[0].google_cloud_run_service.fastapi", + "mode": "managed", "type": "google_cloud_run_service", + "values": {"name": "fastapi-mlflow-server"}}, + ]}, + ], + } + } +} + + +def test_addresses_match_what_infracost_keys_usage_on(tmp_path): + with patch("deployml.utils.measured_usage.run_tool") as mock_run: + mock_run.return_value = SimpleNamespace( + returncode=0, stdout=json.dumps(_REAL_SHAPE), stderr="" + ) + addresses = {r.address for r in build_resource_map(tmp_path)} + + # Terraform's addresses are passed through verbatim, count indices included. + assert _INFRACOST_ADDRESSES <= addresses, ( + "addresses infracost expects are missing: " + f"{sorted(_INFRACOST_ADDRESSES - addresses)}" + ) + # The data source dropped; the free API-enablement resource is allowed through + # because infracost prices it at $0 anyway and it never matches a metric label. + assert not any(".data." in a for a in addresses)