From 3f4c0c7c83bb7341bd112e4dc3758a47b6f14337 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Wed, 1 Jul 2026 12:56:09 +0200 Subject: [PATCH 01/14] feat: first draft tracing skill. --- .claude-plugin/marketplace.json | 14 ++ .../setup-tracing/.claude-plugin/plugin.json | 22 ++ .../setup-tracing/.codex-plugin/plugin.json | 34 +++ plugins/setup-tracing/skills/setup-tracing | 1 + skills/README.md | 1 + skills/setup-tracing/SKILL.md | 238 ++++++++++++++++++ 6 files changed, 310 insertions(+) create mode 100644 plugins/setup-tracing/.claude-plugin/plugin.json create mode 100644 plugins/setup-tracing/.codex-plugin/plugin.json create mode 120000 plugins/setup-tracing/skills/setup-tracing create mode 100644 skills/setup-tracing/SKILL.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 568144f..cf13d1b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -64,6 +64,20 @@ "homepage": "https://github.com/entur/ai", "repository": "https://github.com/entur/ai", "license": "EUPL-1.2" + }, + { + "name": "setup-tracing", + "source": "./plugins/setup-tracing", + "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "category": "observability", + "keywords": ["entur", "tracing", "opentelemetry", "cloud-trace", "observability", "kotlin", "java", "go"], + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2" } ] } diff --git a/plugins/setup-tracing/.claude-plugin/plugin.json b/plugins/setup-tracing/.claude-plugin/plugin.json new file mode 100644 index 0000000..858eb69 --- /dev/null +++ b/plugins/setup-tracing/.claude-plugin/plugin.json @@ -0,0 +1,22 @@ +{ + "name": "setup-tracing", + "version": "0.1.0", + "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2", + "keywords": [ + "entur", + "tracing", + "opentelemetry", + "cloud-trace", + "observability", + "kotlin", + "java", + "go" + ] +} diff --git a/plugins/setup-tracing/.codex-plugin/plugin.json b/plugins/setup-tracing/.codex-plugin/plugin.json new file mode 100644 index 0000000..389af80 --- /dev/null +++ b/plugins/setup-tracing/.codex-plugin/plugin.json @@ -0,0 +1,34 @@ +{ + "name": "setup-tracing", + "version": "0.1.0", + "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2", + "keywords": [ + "entur", + "tracing", + "opentelemetry", + "cloud-trace", + "observability", + "kotlin", + "java", + "go" + ], + "skills": "./skills/", + "interface": { + "displayName": "Tracing Setup", + "shortDescription": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace.", + "longDescription": "Wires distributed tracing into a Kotlin/Java (OpenTelemetry Java Agent) or Go (manual OpenTelemetry SDK) service, following the Entur tracing golden path: Terraform API/IAM setup, application instrumentation, runtime env vars, and log-trace correlation.", + "developerName": "Team Kvalitet", + "category": "DevOps", + "capabilities": [ + "Write" + ], + "websiteURL": "https://github.com/entur/ai" + } +} diff --git a/plugins/setup-tracing/skills/setup-tracing b/plugins/setup-tracing/skills/setup-tracing new file mode 120000 index 0000000..6ee49f4 --- /dev/null +++ b/plugins/setup-tracing/skills/setup-tracing @@ -0,0 +1 @@ +../../../skills/setup-tracing \ No newline at end of file diff --git a/skills/README.md b/skills/README.md index 3298767..d5c79c1 100644 --- a/skills/README.md +++ b/skills/README.md @@ -93,6 +93,7 @@ gh skill install entur/ai | [entur-project-bootstrap](entur-project-bootstrap/) | Bootstrap a new Entur app: self-service manifests, Helm, Terraform, Docker, CI/CD | | [setup-cicd-workflows](setup-cicd-workflows/) | Generate CI/CD GitHub Actions workflows using Entur reusable workflows | | [scr-situation-complication-resolution](scr-situation-complication-resolution/) | Structure problems and decisions in SCR format for leadership | +| [setup-tracing](setup-tracing/) | Instrument a Kotlin/Java or Go service with OpenTelemetry and ship traces to Google Cloud Trace | ## 🤝 Contributing diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md new file mode 100644 index 0000000..6628e42 --- /dev/null +++ b/skills/setup-tracing/SKILL.md @@ -0,0 +1,238 @@ +--- +name: setup-tracing +description: > + Instrument an Entur application with OpenTelemetry and ship traces to Google + Cloud Trace, following the Entur tracing golden path. Covers Terraform + (Cloud Trace API + IAM), the OpenTelemetry Java Agent for Spring Boot + (Kotlin/Java), manual OpenTelemetry SDK wiring for Go, runtime env vars, and + log-trace correlation. Use this skill when the user says "add tracing", + "set up OpenTelemetry", "instrument for Cloud Trace", or asks to wire + distributed tracing into a Kotlin/Java or Go service. Confirms the repo + language and that Cloud Trace storage has already been enabled in the + console before making any changes. +--- + +# Set Up Tracing (Golden Path) + +Wire distributed tracing into an Entur service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Kotlin/Java (Spring Boot, via the OpenTelemetry Java Agent)** and **Go (manual OpenTelemetry SDK)** only. + +## Step 0: Confirm prerequisites -- do not guess any of these + +Ask the user directly; do not infer silently and do not proceed until all four are answered. + +1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java, `go.mod` → Go. State the detected language and ask the user to confirm it. + - If the repo is neither (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation (Java/Kotlin) and manual SDK wiring (Go) -- do not improvise OpenTelemetry setup for another language. +2. **Step 1 -- Cloud Trace storage.** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* + - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. + - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. +3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present; otherwise ask. +4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. + +## Step 1: Enable the Cloud Trace API in Terraform + +Add `cloudtrace.googleapis.com` to the per-environment service-activation block: + +```hcl +# terraform/main.tf +resource "google_project_service" "services" { + for_each = toset([ + "cloudtrace.googleapis.com", + # ... other APIs your service needs + ]) + project = module.init.app.project_id + service = each.value + disable_on_destroy = false +} +``` + +If the repo has no `terraform/` directory yet, do not create one ad hoc -- tell the user to bootstrap Terraform with the Entur `terraform-google-init` module first. + +## Step 2: Grant the workload service account `roles/cloudtrace.agent` + +```hcl +# terraform/main.tf +resource "google_project_iam_member" "runtime_cloudtrace_agent" { + project = module.init.app.project_id + role = "roles/cloudtrace.agent" + member = "serviceAccount:${module.init.service_accounts.runtime.email}" + depends_on = [google_project_service.services] +} +``` + +Bind exactly this role -- it is write-only. Never bind `roles/cloudtrace.user`, `.admin`, or `.editor` to the workload; read access is granted separately at the folder level. + +## Step 3: Instrument the application + +### Kotlin/Java -- OpenTelemetry Java Agent (default; do not hand-instrument without a specific reason) + +Attach the agent via `-javaagent` in a multi-stage Dockerfile -- a temporary Alpine stage downloads the JARs, the distroless final stage only carries the JARs themselves: + +```dockerfile +# Dockerfile +FROM alpine:3.24 AS otel +RUN mkdir /otel && \ + wget -q -O /otel/opentelemetry-javaagent.jar \ + https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.29.0/opentelemetry-javaagent.jar && \ + wget -q -O /otel/gcp-auth-extension.jar \ + https://repo1.maven.org/maven2/io/opentelemetry/contrib/opentelemetry-gcp-auth-extension/1.58.0-alpha/opentelemetry-gcp-auth-extension-1.58.0-alpha-shadow.jar + +FROM gcr.io/distroless/java25-debian13:nonroot +WORKDIR /app +COPY --from=otel /otel /otel +COPY build/libs/app.jar app.jar +CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ + "-Dotel.javaagent.extensions=/otel/gcp-auth-extension.jar", \ + "-Dotel.javaagent.logging=application", \ + "-jar", "/app/app.jar"] +``` + +- Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a Workload Identity token to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. +- Check the OpenTelemetry Java agent's own release notes for a newer version before pinning; `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension) are current as of this guide. +- If the project also uses Cloud Profiler, merge its `-javaagent`/flags into this **same** `CMD` -- a Dockerfile only honors its last `CMD`, a second one silently disables the first. + +### Go -- manual OpenTelemetry SDK wiring + +```go +// internal/tracing/tracing.go +package tracing + +import ( + "context" + "fmt" + + googlegrpc "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +func Init(ctx context.Context, projectID, serviceName, serviceVersion string) (func(context.Context) error, error) { + exp, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithEndpoint("telemetry.googleapis.com:443"), + otlptracegrpc.WithDialOption( + grpc.WithTransportCredentials(googlegrpc.NewDefaultCredentials()), + ), + ) + if err != nil { + return nil, fmt.Errorf("otlp trace exporter: %w", err) + } + + res, _ := resource.New(ctx, resource.WithAttributes( + semconv.ServiceName(serviceName), + semconv.ServiceVersion(serviceVersion), + )) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), + // sdktrace.WithSampler(sdktrace.AlwaysSample()), use this instead if the service runs on Cloud Run + ) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + return tp.Shutdown, nil +} +``` + +- `googlegrpc.NewDefaultCredentials()` bundles TLS + Application Default Credentials -- no API keys or credential files in the container; auth resolves through the Workload Identity binding from Step 2. +- Wrap the outermost handler so every inbound request gets a span, and filter probe/metrics paths so they don't drown the trace stream: + + ```go + // main.go + import ( + "net/http" + "strings" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + ) + + handler := otelhttp.NewHandler(mux, "", + otelhttp.WithFilter(func(r *http.Request) bool { + return r.URL.Path != "/metrics" && !strings.HasPrefix(r.URL.Path, "/health/") + }), + ) + http.ListenAndServe(":8080", handler) + ``` + +- Call `tracing.Init` at the top of `main()`, guarded by `TRACING_ENABLED`, and always `defer shutdown(ctx)` so buffered spans flush before exit: + + ```go + // main.go + ctx := context.Background() + if os.Getenv("TRACING_ENABLED") == "true" { + projectID := os.Getenv("GCP_PROJECT_ID") + shutdown, err := tracing.Init(ctx, projectID, "my-service", "1.0.0") + if err != nil { + logging.Fatal().Err(err).Msg("failed to initialize tracing") + } + defer shutdown(ctx) + } + ``` + +- Any manual span from `tracer.Start(ctx, ...)` must `defer span.End()` -- an unended span never exports. + +## Step 4: Set runtime environment variables + +Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. + +**Kotlin/Java (Java Agent auto-configures from env vars):** + +| Variable | Kubernetes | Cloud Run | +|---|---|---| +| `OTEL_TRACES_EXPORTER` | `otlp` | `otlp` | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `https://telemetry.googleapis.com` | `https://telemetry.googleapis.com` | +| `OTEL_SERVICE_NAME` | `` | `` | +| `TRACING_ENABLED` | `true` | `true` | +| `GCP_PROJECT_ID` | `ent--` | `ent--` | +| `OTEL_TRACES_SAMPLER` | `parentbased_always_on` | **omit** | + +Do not set `OTEL_TRACES_SAMPLER` on Cloud Run: the Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. + +**Go (sampler/exporter already hardcoded in `tracing.Init`; only these two are needed):** + +| Variable | Kubernetes | Cloud Run | +|---|---|---| +| `TRACING_ENABLED` | `true` | `true` | +| `GCP_PROJECT_ID` | `ent--` | `ent--` | + +Set the Go sampler in code to match the runtime, mirroring the Java table above: `ParentBased(AlwaysSample())` on Kubernetes, bare `AlwaysSample()` on Cloud Run. + +## Step 5: Correlate logs with traces + +Every log line inside a traced request needs `logging.googleapis.com/trace` (`projects//traces/`), `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. + +- **Kotlin/Java:** use `entur/cloud-logging` with `spring-boot-starter-gcp-web` and plain SLF4J logging. The fields are injected automatically via Micrometer Tracing -- no manual extraction. +- **Go:** `go get github.com/entur/go-logging`, then extract the span context per handler and attach the fields to every log call inside that handler (info/warn/error alike): + + ```go + sc := trace.SpanFromContext(r.Context()).SpanContext() + projectID := os.Getenv("GCP_PROJECT_ID") + logging.Info(). + Str("logging.googleapis.com/trace", fmt.Sprintf("projects/%s/traces/%s", projectID, sc.TraceID())). + Str("logging.googleapis.com/spanId", sc.SpanID().String()). + Bool("logging.googleapis.com/trace_sampled", sc.IsSampled()). + Msg("handling request") + ``` + + Health-probe endpoints filtered out of tracing in Step 3 don't need these fields. + +## Step 6: Verify + +- GCP Console: **Trace → Trace Explorer** under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). +- Programmatically: if the entur-kompass MCP server is available, use `list_cloud_traces` / `get_cloud_trace` against the app's per-env project. + +## Critical Rules + +- **Step 1 (trace storage) is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. +- **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. +- **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. +- **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. +- **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags. +- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. +- **Do not improvise instrumentation for languages outside Kotlin/Java and Go** -- this golden path doesn't cover them; say so and stop. From df1ea416e45b0c739f9fb39fa1722ad16949ec65 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Wed, 1 Jul 2026 13:18:46 +0200 Subject: [PATCH 02/14] perf: compress and tidy up skill. --- skills/setup-tracing/SKILL.md | 39 +++++++++++++++-------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md index 6628e42..f1317a7 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing/SKILL.md @@ -1,15 +1,10 @@ --- name: setup-tracing description: > - Instrument an Entur application with OpenTelemetry and ship traces to Google - Cloud Trace, following the Entur tracing golden path. Covers Terraform - (Cloud Trace API + IAM), the OpenTelemetry Java Agent for Spring Boot - (Kotlin/Java), manual OpenTelemetry SDK wiring for Go, runtime env vars, and - log-trace correlation. Use this skill when the user says "add tracing", - "set up OpenTelemetry", "instrument for Cloud Trace", or asks to wire - distributed tracing into a Kotlin/Java or Go service. Confirms the repo - language and that Cloud Trace storage has already been enabled in the - console before making any changes. + Wire distributed tracing into an Entur service following the golden path. + Use when the user says "add tracing", "set up OpenTelemetry", "instrument + for Cloud Trace", or asks to add distributed tracing to a Kotlin/Java or + Go service. --- # Set Up Tracing (Golden Path) @@ -25,7 +20,7 @@ Ask the user directly; do not infer silently and do not proceed until all four a 2. **Step 1 -- Cloud Trace storage.** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. -3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present; otherwise ask. +3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. 4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. ## Step 1: Enable the Cloud Trace API in Terraform @@ -141,6 +136,8 @@ func Init(ctx context.Context, projectID, serviceName, serviceVersion string) (f } ``` +Run `go mod tidy` after pasting. + - `googlegrpc.NewDefaultCredentials()` bundles TLS + Application Default Credentials -- no API keys or credential files in the container; auth resolves through the Workload Identity binding from Step 2. - Wrap the outermost handler so every inbound request gets a span, and filter probe/metrics paths so they don't drown the trace stream: @@ -194,12 +191,7 @@ Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) f Do not set `OTEL_TRACES_SAMPLER` on Cloud Run: the Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. -**Go (sampler/exporter already hardcoded in `tracing.Init`; only these two are needed):** - -| Variable | Kubernetes | Cloud Run | -|---|---|---| -| `TRACING_ENABLED` | `true` | `true` | -| `GCP_PROJECT_ID` | `ent--` | `ent--` | +**Go (sampler/exporter already hardcoded in `tracing.Init`; only these two are needed):** set `TRACING_ENABLED` to `true` and `GCP_PROJECT_ID` to `ent--`, the same on both Kubernetes and Cloud Run. Set the Go sampler in code to match the runtime, mirroring the Java table above: `ParentBased(AlwaysSample())` on Kubernetes, bare `AlwaysSample()` on Cloud Run. @@ -222,10 +214,14 @@ Every log line inside a traced request needs `logging.googleapis.com/trace` (`pr Health-probe endpoints filtered out of tracing in Step 3 don't need these fields. -## Step 6: Verify +## Step 6: Tell the user what's left + +Steps 0-5 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: -- GCP Console: **Trace → Trace Explorer** under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). -- Programmatically: if the entur-kompass MCP server is available, use `list_cloud_traces` / `get_cloud_trace` against the app's per-env project. +1. Commit and merge the changes, and let the normal CD pipeline deploy to the confirmed environment(s) from Step 0. +2. Send a few requests to the deployed service to generate spans. +3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). +4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). ## Critical Rules @@ -233,6 +229,5 @@ Every log line inside a traced request needs `logging.googleapis.com/trace` (`pr - **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. - **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. - **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. -- **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags. -- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. -- **Do not improvise instrumentation for languages outside Kotlin/Java and Go** -- this golden path doesn't cover them; say so and stop. +- **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. +- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. \ No newline at end of file From a5a26dd9c493979b43a71e20a55abd9f6072b960 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Wed, 1 Jul 2026 14:43:37 +0200 Subject: [PATCH 03/14] test: scenario tests for tracing --- .../28-tracing-iam-least-privilege.md | 35 +++++++++++++++ .../29-tracing-java-agent-default.md | 42 +++++++++++++++++ .../30-tracing-dockerfile-single-cmd.md | 45 +++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tests/scenarios/28-tracing-iam-least-privilege.md create mode 100644 tests/scenarios/29-tracing-java-agent-default.md create mode 100644 tests/scenarios/30-tracing-dockerfile-single-cmd.md diff --git a/tests/scenarios/28-tracing-iam-least-privilege.md b/tests/scenarios/28-tracing-iam-least-privilege.md new file mode 100644 index 0000000..ac8f50b --- /dev/null +++ b/tests/scenarios/28-tracing-iam-least-privilege.md @@ -0,0 +1,35 @@ +# Scenario: Tracing IAM Role Is Write-Only + +## Description + +Verifies the agent grants exactly roles/cloudtrace.agent to the workload service +account, not a broader trace role. + +## Prompt + +What IAM role should be bound to the workload service account so it can export +traces to Cloud Trace, per Entur's tracing golden path? + +Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` +format on its own line: + +- role: + +## Assertions + +```json +{ + "must_contain": [ + "role: roles/cloudtrace.agent" + ], + "must_not_contain": [ + "role: roles/cloudtrace.user", + "role: roles/cloudtrace.admin", + "role: roles/cloudtrace.editor" + ] +} +``` + +## Budget + +0.08 diff --git a/tests/scenarios/29-tracing-java-agent-default.md b/tests/scenarios/29-tracing-java-agent-default.md new file mode 100644 index 0000000..1048ee3 --- /dev/null +++ b/tests/scenarios/29-tracing-java-agent-default.md @@ -0,0 +1,42 @@ +# Scenario: Java Agent Is the Default for Java/Kotlin, Not Go + +## Description + +Verifies the agent defaults a Kotlin/Java service to the OpenTelemetry Java Agent +without a specific reason to hand-instrument, AND correctly does NOT recommend the +Java Agent for a Go service in the same prompt -- confirming it understands the +language scope of the rule rather than pattern-matching "just get tracing working" +to one fixed answer. + +## Prompt + +Two engineers ask for tracing, no specific reason for manual instrumentation given +by either: + +1. "We need tracing on our Spring Boot service, nothing fancy, just get it working." +2. "We need tracing on our Go service, nothing fancy, just get it working." + +Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` +format on its own line: + +- spring_boot_approach: +- go_approach: + +## Assertions + +```json +{ + "must_contain": [ + "spring_boot_approach: java agent", + "go_approach: manual sdk" + ], + "must_not_contain": [ + "spring_boot_approach: manual sdk", + "go_approach: java agent" + ] +} +``` + +## Budget + +0.08 diff --git a/tests/scenarios/30-tracing-dockerfile-single-cmd.md b/tests/scenarios/30-tracing-dockerfile-single-cmd.md new file mode 100644 index 0000000..aded62a --- /dev/null +++ b/tests/scenarios/30-tracing-dockerfile-single-cmd.md @@ -0,0 +1,45 @@ +# Scenario: Tracing and Profiler Flags Merge into One CMD + +## Description + +Verifies the agent knows a Dockerfile only honors its last CMD instruction, so +adding the tracing Java Agent to a service that already runs Cloud Profiler must +merge flags into the existing CMD, not append a second one -- the common LLM +instinct when told to "add" a step to an existing file. + +## Prompt + +A service's Dockerfile already has this line to run Cloud Profiler: + +CMD ["-javaagent:/profiler/profiler_java_agent.so", "-jar", "/app/app.jar"] + +Now add OpenTelemetry Java Agent tracing per Entur's golden path. + +Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` +format on its own line: + +- number_of_cmd_instructions: +- both_agents_in_same_cmd: +- why: + +## Assertions + +```json +{ + "must_contain": [ + "number_of_cmd_instructions: 1", + "both_agents_in_same_cmd: yes" + ], + "must_not_contain": [ + "number_of_cmd_instructions: 2", + "both_agents_in_same_cmd: no" + ], + "must_match": [ + "last CMD|only.*(last|one) CMD|second CMD.*(disable|override|ignore)" + ] +} +``` + +## Budget + +0.08 From a1ffa7d16f8bcd4a2856a6276619f90cc348bc4c Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Wed, 1 Jul 2026 14:44:33 +0200 Subject: [PATCH 04/14] docs: lint run --- skills/setup-tracing/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md index f1317a7..bd67c37 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing/SKILL.md @@ -230,4 +230,4 @@ Steps 0-5 are everything this skill can do by editing the repo. Verifying traces - **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. - **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. -- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. \ No newline at end of file +- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. From e8d43b05ac4a86db98d17152697b7be85a8e284e Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Wed, 1 Jul 2026 15:02:42 +0200 Subject: [PATCH 05/14] fix: naming bug in tracing skill. --- skills/setup-tracing/SKILL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md index bd67c37..e10a54f 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing/SKILL.md @@ -17,10 +17,10 @@ Ask the user directly; do not infer silently and do not proceed until all four a 1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java, `go.mod` → Go. State the detected language and ask the user to confirm it. - If the repo is neither (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation (Java/Kotlin) and manual SDK wiring (Go) -- do not improvise OpenTelemetry setup for another language. -2. **Step 1 -- Cloud Trace storage.** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* +2. **Cloud Trace storage (manual, console-only).** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. -3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. +3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. 4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. ## Step 1: Enable the Cloud Trace API in Terraform @@ -225,7 +225,7 @@ Steps 0-5 are everything this skill can do by editing the repo. Verifying traces ## Critical Rules -- **Step 1 (trace storage) is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. +- **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. - **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. - **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. - **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. From 5ca762d425f7a21affae0c10232c0bf167c0db10 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 3 Jul 2026 13:44:41 +0200 Subject: [PATCH 06/14] feat: updated to not track metrics and logs. --- skills/setup-tracing/SKILL.md | 44 ++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md index e10a54f..d973f03 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing/SKILL.md @@ -34,7 +34,7 @@ resource "google_project_service" "services" { "cloudtrace.googleapis.com", # ... other APIs your service needs ]) - project = module.init.app.project_id + project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias service = each.value disable_on_destroy = false } @@ -47,7 +47,7 @@ If the repo has no `terraform/` directory yet, do not create one ad hoc -- tell ```hcl # terraform/main.tf resource "google_project_iam_member" "runtime_cloudtrace_agent" { - project = module.init.app.project_id + project = module.init.app.project_id # adapt "init" to this repo's module alias role = "roles/cloudtrace.agent" member = "serviceAccount:${module.init.service_accounts.runtime.email}" depends_on = [google_project_service.services] @@ -82,7 +82,8 @@ CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ ``` - Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a Workload Identity token to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. -- Check the OpenTelemetry Java agent's own release notes for a newer version before pinning; `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension) are current as of this guide. +- Use the pinned versions above as-is -- `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension). Do not spend time checking upstream for a newer release. In Step 6's summary, tell the user which versions were used and point to the `otel` build stage in the Dockerfile as where to bump them later. +- `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 6 summary as something the user may need to change. - If the project also uses Cloud Profiler, merge its `-javaagent`/flags into this **same** `CMD` -- a Dockerfile only honors its last `CMD`, a second one silently disables the first. ### Go -- manual OpenTelemetry SDK wiring @@ -178,6 +179,36 @@ Run `go mod tidy` after pasting. Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. +**If `helm//env/values-kub-ent-.yaml` doesn't exist yet for a confirmed environment**, ask the user: *"No Helm values file exists yet for `` -- do you want me to create `helm//env/values-kub-ent-.yaml` with the tracing env vars?"* Only create it if they say yes, and only for the environment(s) confirmed in Step 0. Use this shape: + +```yaml +# helm//env/values-kub-ent-.yaml +common: + env: + container: + env: + - name: OTEL_TRACES_EXPORTER + value: otlp + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: https://telemetry.googleapis.com + - name: OTEL_SERVICE_NAME + value: + - name: TRACING_ENABLED + value: "true" + - name: GCP_PROJECT_ID + value: ent-- + - name: OTEL_TRACES_SAMPLER + value: parentbased_always_on + - name: OTEL_METRICS_EXPORTER + value: none + - name: OTEL_LOGS_EXPORTER + value: none +``` + +If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. Omit `OTEL_TRACES_SAMPLER` on Cloud Run, per the table below. + +Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). + **Kotlin/Java (Java Agent auto-configures from env vars):** | Variable | Kubernetes | Cloud Run | @@ -188,6 +219,10 @@ Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) f | `TRACING_ENABLED` | `true` | `true` | | `GCP_PROJECT_ID` | `ent--` | `ent--` | | `OTEL_TRACES_SAMPLER` | `parentbased_always_on` | **omit** | +| `OTEL_METRICS_EXPORTER` | `none` | `none` | +| `OTEL_LOGS_EXPORTER` | `none` | `none` | + +Always set `OTEL_METRICS_EXPORTER=none` and `OTEL_LOGS_EXPORTER=none` on both runtimes -- the agent defaults both to `otlp` like traces, but Step 2's IAM grant doesn't authorize either. Do not set `OTEL_TRACES_SAMPLER` on Cloud Run: the Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. @@ -223,6 +258,8 @@ Steps 0-5 are everything this skill can do by editing the repo. Verifying traces 3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). 4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). +For Kotlin/Java, also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. + ## Critical Rules - **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. @@ -231,3 +268,4 @@ Steps 0-5 are everything this skill can do by editing the repo. Verifying traces - **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. - **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. +- **Kotlin/Java also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. \ No newline at end of file From cf421615cef0051832fb71a5c1372301410bc57c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:45:10 +0000 Subject: [PATCH 07/14] chore: apply markdownlint --fix --- skills/setup-tracing/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing/SKILL.md index d973f03..7b1b310 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing/SKILL.md @@ -268,4 +268,4 @@ For Kotlin/Java, also state in the summary: the OpenTelemetry Java agent (`v2.29 - **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. - **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. -- **Kotlin/Java also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. \ No newline at end of file +- **Kotlin/Java also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. From becfc013da5b0cdb34571aac021c541decc8a64e Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Tue, 28 Jul 2026 10:03:31 +0200 Subject: [PATCH 08/14] refactor: split java and go setup into seperate skills. --- .agents/plugins/marketplace.json | 24 ++ .claude-plugin/marketplace.json | 22 +- .../.claude-plugin/plugin.json | 20 ++ .../.codex-plugin/plugin.json | 32 +++ .../setup-tracing-go/skills/setup-tracing-go | 1 + .../.claude-plugin/plugin.json | 6 +- .../.codex-plugin/plugin.json | 12 +- .../skills/setup-tracing-java | 1 + plugins/setup-tracing/skills/setup-tracing | 1 - skills/README.md | 3 +- skills/setup-tracing-go/SKILL.md | 208 ++++++++++++++++++ .../SKILL.md | 152 ++----------- .../28-tracing-iam-least-privilege.md | 2 +- .../29-tracing-java-agent-default.md | 13 +- .../30-tracing-dockerfile-single-cmd.md | 2 +- 15 files changed, 343 insertions(+), 156 deletions(-) create mode 100644 plugins/setup-tracing-go/.claude-plugin/plugin.json create mode 100644 plugins/setup-tracing-go/.codex-plugin/plugin.json create mode 120000 plugins/setup-tracing-go/skills/setup-tracing-go rename plugins/{setup-tracing => setup-tracing-java}/.claude-plugin/plugin.json (60%) rename plugins/{setup-tracing => setup-tracing-java}/.codex-plugin/plugin.json (51%) create mode 120000 plugins/setup-tracing-java/skills/setup-tracing-java delete mode 120000 plugins/setup-tracing/skills/setup-tracing create mode 100644 skills/setup-tracing-go/SKILL.md rename skills/{setup-tracing => setup-tracing-java}/SKILL.md (57%) diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 10cbdf3..c921dfa 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -51,6 +51,30 @@ "authentication": "ON_INSTALL" }, "category": "Documentation" + }, + { + "name": "setup-tracing-java", + "source": { + "source": "local", + "path": "./plugins/setup-tracing-java" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "DevOps" + }, + { + "name": "setup-tracing-go", + "source": { + "source": "local", + "path": "./plugins/setup-tracing-go" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "DevOps" } ] } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index cf13d1b..697a696 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -66,11 +66,25 @@ "license": "EUPL-1.2" }, { - "name": "setup-tracing", - "source": "./plugins/setup-tracing", - "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "name": "setup-tracing-java", + "source": "./plugins/setup-tracing-java", + "description": "Instrument a Kotlin/Java (Spring Boot) service with the OpenTelemetry Java Agent and ship traces to Google Cloud Trace, following the Entur tracing golden path.", "category": "observability", - "keywords": ["entur", "tracing", "opentelemetry", "cloud-trace", "observability", "kotlin", "java", "go"], + "keywords": ["entur", "tracing", "opentelemetry", "cloud-trace", "observability", "kotlin", "java", "spring-boot"], + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2" + }, + { + "name": "setup-tracing-go", + "source": "./plugins/setup-tracing-go", + "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", + "category": "observability", + "keywords": ["entur", "tracing", "opentelemetry", "cloud-trace", "observability", "go"], "author": { "name": "Team Kvalitet", "email": "support@entur.org" diff --git a/plugins/setup-tracing-go/.claude-plugin/plugin.json b/plugins/setup-tracing-go/.claude-plugin/plugin.json new file mode 100644 index 0000000..2170336 --- /dev/null +++ b/plugins/setup-tracing-go/.claude-plugin/plugin.json @@ -0,0 +1,20 @@ +{ + "name": "setup-tracing-go", + "version": "0.1.0", + "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2", + "keywords": [ + "entur", + "tracing", + "opentelemetry", + "cloud-trace", + "observability", + "go" + ] +} diff --git a/plugins/setup-tracing-go/.codex-plugin/plugin.json b/plugins/setup-tracing-go/.codex-plugin/plugin.json new file mode 100644 index 0000000..af5f600 --- /dev/null +++ b/plugins/setup-tracing-go/.codex-plugin/plugin.json @@ -0,0 +1,32 @@ +{ + "name": "setup-tracing-go", + "version": "0.1.0", + "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", + "author": { + "name": "Team Kvalitet", + "email": "support@entur.org" + }, + "homepage": "https://github.com/entur/ai", + "repository": "https://github.com/entur/ai", + "license": "EUPL-1.2", + "keywords": [ + "entur", + "tracing", + "opentelemetry", + "cloud-trace", + "observability", + "go" + ], + "skills": "./skills/", + "interface": { + "displayName": "Tracing Setup (Go)", + "shortDescription": "Instrument a Go service with OpenTelemetry and ship traces to Google Cloud Trace.", + "longDescription": "Wires distributed tracing into a Go service via manual OpenTelemetry SDK instrumentation, following the Entur tracing golden path: Terraform API/IAM setup, application instrumentation, runtime env vars, and log-trace correlation.", + "developerName": "Team Kvalitet", + "category": "DevOps", + "capabilities": [ + "Write" + ], + "websiteURL": "https://github.com/entur/ai" + } +} diff --git a/plugins/setup-tracing-go/skills/setup-tracing-go b/plugins/setup-tracing-go/skills/setup-tracing-go new file mode 120000 index 0000000..da8c011 --- /dev/null +++ b/plugins/setup-tracing-go/skills/setup-tracing-go @@ -0,0 +1 @@ +../../../skills/setup-tracing-go \ No newline at end of file diff --git a/plugins/setup-tracing/.claude-plugin/plugin.json b/plugins/setup-tracing-java/.claude-plugin/plugin.json similarity index 60% rename from plugins/setup-tracing/.claude-plugin/plugin.json rename to plugins/setup-tracing-java/.claude-plugin/plugin.json index 858eb69..852f3d4 100644 --- a/plugins/setup-tracing/.claude-plugin/plugin.json +++ b/plugins/setup-tracing-java/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { - "name": "setup-tracing", + "name": "setup-tracing-java", "version": "0.1.0", - "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "description": "Instrument a Kotlin/Java (Spring Boot) service with the OpenTelemetry Java Agent and ship traces to Google Cloud Trace, following the Entur tracing golden path.", "author": { "name": "Team Kvalitet", "email": "support@entur.org" @@ -17,6 +17,6 @@ "observability", "kotlin", "java", - "go" + "spring-boot" ] } diff --git a/plugins/setup-tracing/.codex-plugin/plugin.json b/plugins/setup-tracing-java/.codex-plugin/plugin.json similarity index 51% rename from plugins/setup-tracing/.codex-plugin/plugin.json rename to plugins/setup-tracing-java/.codex-plugin/plugin.json index 389af80..6b10419 100644 --- a/plugins/setup-tracing/.codex-plugin/plugin.json +++ b/plugins/setup-tracing-java/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { - "name": "setup-tracing", + "name": "setup-tracing-java", "version": "0.1.0", - "description": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace, following the Entur tracing golden path (Kotlin/Java Agent, Go SDK).", + "description": "Instrument a Kotlin/Java (Spring Boot) service with the OpenTelemetry Java Agent and ship traces to Google Cloud Trace, following the Entur tracing golden path.", "author": { "name": "Team Kvalitet", "email": "support@entur.org" @@ -17,13 +17,13 @@ "observability", "kotlin", "java", - "go" + "spring-boot" ], "skills": "./skills/", "interface": { - "displayName": "Tracing Setup", - "shortDescription": "Instrument a service with OpenTelemetry and ship traces to Google Cloud Trace.", - "longDescription": "Wires distributed tracing into a Kotlin/Java (OpenTelemetry Java Agent) or Go (manual OpenTelemetry SDK) service, following the Entur tracing golden path: Terraform API/IAM setup, application instrumentation, runtime env vars, and log-trace correlation.", + "displayName": "Tracing Setup (Java)", + "shortDescription": "Instrument a Kotlin/Java service with OpenTelemetry and ship traces to Google Cloud Trace.", + "longDescription": "Wires distributed tracing into a Kotlin/Java (Spring Boot) service via the OpenTelemetry Java Agent, following the Entur tracing golden path: Terraform API/IAM setup, application instrumentation, runtime env vars, and log-trace correlation.", "developerName": "Team Kvalitet", "category": "DevOps", "capabilities": [ diff --git a/plugins/setup-tracing-java/skills/setup-tracing-java b/plugins/setup-tracing-java/skills/setup-tracing-java new file mode 120000 index 0000000..e5dae63 --- /dev/null +++ b/plugins/setup-tracing-java/skills/setup-tracing-java @@ -0,0 +1 @@ +../../../skills/setup-tracing-java \ No newline at end of file diff --git a/plugins/setup-tracing/skills/setup-tracing b/plugins/setup-tracing/skills/setup-tracing deleted file mode 120000 index 6ee49f4..0000000 --- a/plugins/setup-tracing/skills/setup-tracing +++ /dev/null @@ -1 +0,0 @@ -../../../skills/setup-tracing \ No newline at end of file diff --git a/skills/README.md b/skills/README.md index d5c79c1..b5da398 100644 --- a/skills/README.md +++ b/skills/README.md @@ -93,7 +93,8 @@ gh skill install entur/ai | [entur-project-bootstrap](entur-project-bootstrap/) | Bootstrap a new Entur app: self-service manifests, Helm, Terraform, Docker, CI/CD | | [setup-cicd-workflows](setup-cicd-workflows/) | Generate CI/CD GitHub Actions workflows using Entur reusable workflows | | [scr-situation-complication-resolution](scr-situation-complication-resolution/) | Structure problems and decisions in SCR format for leadership | -| [setup-tracing](setup-tracing/) | Instrument a Kotlin/Java or Go service with OpenTelemetry and ship traces to Google Cloud Trace | +| [setup-tracing-java](setup-tracing-java/) | Instrument a Kotlin/Java (Spring Boot) service with the OpenTelemetry Java Agent and ship traces to Google Cloud Trace | +| [setup-tracing-go](setup-tracing-go/) | Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace | ## 🤝 Contributing diff --git a/skills/setup-tracing-go/SKILL.md b/skills/setup-tracing-go/SKILL.md new file mode 100644 index 0000000..14f2849 --- /dev/null +++ b/skills/setup-tracing-go/SKILL.md @@ -0,0 +1,208 @@ +--- +name: setup-tracing-go +description: > + Wire distributed tracing into an Entur Go service using manual + OpenTelemetry SDK instrumentation, following the golden path to Cloud + Trace. Use when the user says "add tracing", "set up OpenTelemetry", + "instrument for Cloud Trace", or "add distributed tracing" for a Go + service -- typically a repo with `go.mod`. For a Kotlin/Java or Spring + Boot service, use the `setup-tracing-java` skill instead. +--- + +# Set Up Tracing -- Go (Golden Path) + +Wire distributed tracing into an Entur Go service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **manual OpenTelemetry SDK wiring** only -- it does not cover Kotlin/Java, Python, or any other language. + +## Step 0: Confirm prerequisites -- do not guess any of these + +Ask the user directly; do not infer silently and do not proceed until all four are answered. + +1. **Language.** Detect from the repo: `go.mod` → Go. State the detected language and ask the user to confirm it. + - If the repo is Kotlin/Java (`build.gradle.kts`), **stop** and point the user to the `setup-tracing-java` skill instead. + - If the repo is neither Go nor Kotlin/Java (Python, Node, etc.), **stop**. Tell the user this golden path only documents manual SDK wiring for Go -- do not improvise OpenTelemetry setup for another language. +2. **Cloud Trace storage (manual, console-only).** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* + - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. + - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. +3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. +4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. + +## Step 1: Enable the Cloud Trace API in Terraform + +Add `cloudtrace.googleapis.com` to the per-environment service-activation block: + +```hcl +# terraform/main.tf +resource "google_project_service" "services" { + for_each = toset([ + "cloudtrace.googleapis.com", + # ... other APIs your service needs + ]) + project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias + service = each.value + disable_on_destroy = false +} +``` + +If the repo has no `terraform/` directory yet, do not create one ad hoc -- tell the user to bootstrap Terraform with the Entur `terraform-google-init` module first. + +## Step 2: Grant the workload service account `roles/cloudtrace.agent` + +```hcl +# terraform/main.tf +resource "google_project_iam_member" "runtime_cloudtrace_agent" { + project = module.init.app.project_id # adapt "init" to this repo's module alias + role = "roles/cloudtrace.agent" + member = "serviceAccount:${module.init.service_accounts.runtime.email}" + depends_on = [google_project_service.services] +} +``` + +Bind exactly this role -- it is write-only. Never bind `roles/cloudtrace.user`, `.admin`, or `.editor` to the workload; read access is granted separately at the folder level. + +## Step 3: Instrument the application -- manual OpenTelemetry SDK wiring + +```go +// internal/tracing/tracing.go +package tracing + +import ( + "context" + "fmt" + + googlegrpc "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +func Init(ctx context.Context, projectID, serviceName, serviceVersion string) (func(context.Context) error, error) { + exp, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithEndpoint("telemetry.googleapis.com:443"), + otlptracegrpc.WithDialOption( + grpc.WithTransportCredentials(googlegrpc.NewDefaultCredentials()), + ), + ) + if err != nil { + return nil, fmt.Errorf("otlp trace exporter: %w", err) + } + + res, _ := resource.New(ctx, resource.WithAttributes( + semconv.ServiceName(serviceName), + semconv.ServiceVersion(serviceVersion), + )) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), + // sdktrace.WithSampler(sdktrace.AlwaysSample()), use this instead if the service runs on Cloud Run + ) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + return tp.Shutdown, nil +} +``` + +Run `go mod tidy` after pasting. + +- `googlegrpc.NewDefaultCredentials()` bundles TLS + Application Default Credentials -- no API keys or credential files in the container; auth resolves through the Workload Identity binding from Step 2. +- Wrap the outermost handler so every inbound request gets a span, and filter probe/metrics paths so they don't drown the trace stream: + + ```go + // main.go + import ( + "net/http" + "strings" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + ) + + handler := otelhttp.NewHandler(mux, "", + otelhttp.WithFilter(func(r *http.Request) bool { + return r.URL.Path != "/metrics" && !strings.HasPrefix(r.URL.Path, "/health/") + }), + ) + http.ListenAndServe(":8080", handler) + ``` + +- Call `tracing.Init` at the top of `main()`, guarded by `TRACING_ENABLED`, and always `defer shutdown(ctx)` so buffered spans flush before exit: + + ```go + // main.go + ctx := context.Background() + if os.Getenv("TRACING_ENABLED") == "true" { + projectID := os.Getenv("GCP_PROJECT_ID") + shutdown, err := tracing.Init(ctx, projectID, "my-service", "1.0.0") + if err != nil { + logging.Fatal().Err(err).Msg("failed to initialize tracing") + } + defer shutdown(ctx) + } + ``` + +- Any manual span from `tracer.Start(ctx, ...)` must `defer span.End()` -- an unended span never exports. + +## Step 4: Set runtime environment variables + +Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. + +**If `helm//env/values-kub-ent-.yaml` doesn't exist yet for a confirmed environment**, ask the user: *"No Helm values file exists yet for `` -- do you want me to create `helm//env/values-kub-ent-.yaml` with the tracing env vars?"* Only create it if they say yes, and only for the environment(s) confirmed in Step 0. Use this shape: + +```yaml +# helm//env/values-kub-ent-.yaml +common: + env: + container: + env: + - name: TRACING_ENABLED + value: "true" + - name: GCP_PROJECT_ID + value: ent-- +``` + +If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. + +Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). + +The sampler and exporter are already hardcoded in `tracing.Init` (Step 3), so only the two env vars shown above are needed -- set the same on both Kubernetes and Cloud Run. Set the sampler in code to match the runtime instead: `ParentBased(AlwaysSample())` on Kubernetes, bare `AlwaysSample()` on Cloud Run (see the commented-out line in Step 3). + +## Step 5: Correlate logs with traces + +Every log line inside a traced request needs `logging.googleapis.com/trace` (`projects//traces/`), `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. + +`go get github.com/entur/go-logging`, then extract the span context per handler and attach the fields to every log call inside that handler (info/warn/error alike): + +```go +sc := trace.SpanFromContext(r.Context()).SpanContext() +projectID := os.Getenv("GCP_PROJECT_ID") +logging.Info(). + Str("logging.googleapis.com/trace", fmt.Sprintf("projects/%s/traces/%s", projectID, sc.TraceID())). + Str("logging.googleapis.com/spanId", sc.SpanID().String()). + Bool("logging.googleapis.com/trace_sampled", sc.IsSampled()). + Msg("handling request") +``` + +Health-probe endpoints filtered out of tracing in Step 3 don't need these fields. + +## Step 6: Tell the user what's left + +Steps 0-5 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: + +1. Commit and merge the changes, and let the normal CD pipeline deploy to the confirmed environment(s) from Step 0. +2. Send a few requests to the deployed service to generate spans. +3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). +4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). + +## Critical Rules + +- **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. +- **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. +- **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. +- **Kubernetes vs Cloud Run sampler differs** -- see Step 3/4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. +- **Any manual span must `defer span.End()`** -- an unended span never exports. diff --git a/skills/setup-tracing/SKILL.md b/skills/setup-tracing-java/SKILL.md similarity index 57% rename from skills/setup-tracing/SKILL.md rename to skills/setup-tracing-java/SKILL.md index 7b1b310..480cfd5 100644 --- a/skills/setup-tracing/SKILL.md +++ b/skills/setup-tracing-java/SKILL.md @@ -1,22 +1,25 @@ --- -name: setup-tracing +name: setup-tracing-java description: > - Wire distributed tracing into an Entur service following the golden path. - Use when the user says "add tracing", "set up OpenTelemetry", "instrument - for Cloud Trace", or asks to add distributed tracing to a Kotlin/Java or - Go service. + Wire distributed tracing into an Entur Kotlin/Java (Spring Boot) service + using the OpenTelemetry Java Agent, following the golden path to Cloud + Trace. Use when the user says "add tracing", "set up OpenTelemetry", + "instrument for Cloud Trace", or "add distributed tracing" for a Kotlin, + Java, or Spring Boot service -- typically a repo with `build.gradle.kts`. + For a Go service, use the `setup-tracing-go` skill instead. --- -# Set Up Tracing (Golden Path) +# Set Up Tracing -- Kotlin/Java (Golden Path) -Wire distributed tracing into an Entur service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Kotlin/Java (Spring Boot, via the OpenTelemetry Java Agent)** and **Go (manual OpenTelemetry SDK)** only. +Wire distributed tracing into an Entur Kotlin/Java service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Spring Boot via the OpenTelemetry Java Agent** only -- it does not cover Go, Python, or any other language. ## Step 0: Confirm prerequisites -- do not guess any of these Ask the user directly; do not infer silently and do not proceed until all four are answered. -1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java, `go.mod` → Go. State the detected language and ask the user to confirm it. - - If the repo is neither (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation (Java/Kotlin) and manual SDK wiring (Go) -- do not improvise OpenTelemetry setup for another language. +1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java. State the detected language and ask the user to confirm it. + - If the repo is Go (`go.mod`), **stop** and point the user to the `setup-tracing-go` skill instead. + - If the repo is neither Kotlin/Java nor Go (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation for Kotlin/Java -- do not improvise OpenTelemetry setup for another language. 2. **Cloud Trace storage (manual, console-only).** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. @@ -58,7 +61,7 @@ Bind exactly this role -- it is write-only. Never bind `roles/cloudtrace.user`, ## Step 3: Instrument the application -### Kotlin/Java -- OpenTelemetry Java Agent (default; do not hand-instrument without a specific reason) +### OpenTelemetry Java Agent (default; do not hand-instrument without a specific reason) Attach the agent via `-javaagent` in a multi-stage Dockerfile -- a temporary Alpine stage downloads the JARs, the distroless final stage only carries the JARs themselves: @@ -86,95 +89,6 @@ CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ - `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 6 summary as something the user may need to change. - If the project also uses Cloud Profiler, merge its `-javaagent`/flags into this **same** `CMD` -- a Dockerfile only honors its last `CMD`, a second one silently disables the first. -### Go -- manual OpenTelemetry SDK wiring - -```go -// internal/tracing/tracing.go -package tracing - -import ( - "context" - "fmt" - - googlegrpc "google.golang.org/grpc/credentials/google" - "google.golang.org/grpc" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.26.0" -) - -func Init(ctx context.Context, projectID, serviceName, serviceVersion string) (func(context.Context) error, error) { - exp, err := otlptracegrpc.New(ctx, - otlptracegrpc.WithEndpoint("telemetry.googleapis.com:443"), - otlptracegrpc.WithDialOption( - grpc.WithTransportCredentials(googlegrpc.NewDefaultCredentials()), - ), - ) - if err != nil { - return nil, fmt.Errorf("otlp trace exporter: %w", err) - } - - res, _ := resource.New(ctx, resource.WithAttributes( - semconv.ServiceName(serviceName), - semconv.ServiceVersion(serviceVersion), - )) - - tp := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exp), - sdktrace.WithResource(res), - sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), - // sdktrace.WithSampler(sdktrace.AlwaysSample()), use this instead if the service runs on Cloud Run - ) - otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - )) - return tp.Shutdown, nil -} -``` - -Run `go mod tidy` after pasting. - -- `googlegrpc.NewDefaultCredentials()` bundles TLS + Application Default Credentials -- no API keys or credential files in the container; auth resolves through the Workload Identity binding from Step 2. -- Wrap the outermost handler so every inbound request gets a span, and filter probe/metrics paths so they don't drown the trace stream: - - ```go - // main.go - import ( - "net/http" - "strings" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - ) - - handler := otelhttp.NewHandler(mux, "", - otelhttp.WithFilter(func(r *http.Request) bool { - return r.URL.Path != "/metrics" && !strings.HasPrefix(r.URL.Path, "/health/") - }), - ) - http.ListenAndServe(":8080", handler) - ``` - -- Call `tracing.Init` at the top of `main()`, guarded by `TRACING_ENABLED`, and always `defer shutdown(ctx)` so buffered spans flush before exit: - - ```go - // main.go - ctx := context.Background() - if os.Getenv("TRACING_ENABLED") == "true" { - projectID := os.Getenv("GCP_PROJECT_ID") - shutdown, err := tracing.Init(ctx, projectID, "my-service", "1.0.0") - if err != nil { - logging.Fatal().Err(err).Msg("failed to initialize tracing") - } - defer shutdown(ctx) - } - ``` - -- Any manual span from `tracer.Start(ctx, ...)` must `defer span.End()` -- an unended span never exports. - ## Step 4: Set runtime environment variables Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. @@ -205,49 +119,21 @@ common: value: none ``` -If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. Omit `OTEL_TRACES_SAMPLER` on Cloud Run, per the table below. +If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. Omit `OTEL_TRACES_SAMPLER` on Cloud Run, per the note below. Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). -**Kotlin/Java (Java Agent auto-configures from env vars):** - -| Variable | Kubernetes | Cloud Run | -|---|---|---| -| `OTEL_TRACES_EXPORTER` | `otlp` | `otlp` | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | `https://telemetry.googleapis.com` | `https://telemetry.googleapis.com` | -| `OTEL_SERVICE_NAME` | `` | `` | -| `TRACING_ENABLED` | `true` | `true` | -| `GCP_PROJECT_ID` | `ent--` | `ent--` | -| `OTEL_TRACES_SAMPLER` | `parentbased_always_on` | **omit** | -| `OTEL_METRICS_EXPORTER` | `none` | `none` | -| `OTEL_LOGS_EXPORTER` | `none` | `none` | +**Java Agent auto-configures from env vars.** On Cloud Run, set the same variables shown above on the `cloudrun.yaml` container spec, with one difference: omit `OTEL_TRACES_SAMPLER` entirely. Always set `OTEL_METRICS_EXPORTER=none` and `OTEL_LOGS_EXPORTER=none` on both runtimes -- the agent defaults both to `otlp` like traces, but Step 2's IAM grant doesn't authorize either. Do not set `OTEL_TRACES_SAMPLER` on Cloud Run: the Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. -**Go (sampler/exporter already hardcoded in `tracing.Init`; only these two are needed):** set `TRACING_ENABLED` to `true` and `GCP_PROJECT_ID` to `ent--`, the same on both Kubernetes and Cloud Run. - -Set the Go sampler in code to match the runtime, mirroring the Java table above: `ParentBased(AlwaysSample())` on Kubernetes, bare `AlwaysSample()` on Cloud Run. - ## Step 5: Correlate logs with traces Every log line inside a traced request needs `logging.googleapis.com/trace` (`projects//traces/`), `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. -- **Kotlin/Java:** use `entur/cloud-logging` with `spring-boot-starter-gcp-web` and plain SLF4J logging. The fields are injected automatically via Micrometer Tracing -- no manual extraction. -- **Go:** `go get github.com/entur/go-logging`, then extract the span context per handler and attach the fields to every log call inside that handler (info/warn/error alike): - - ```go - sc := trace.SpanFromContext(r.Context()).SpanContext() - projectID := os.Getenv("GCP_PROJECT_ID") - logging.Info(). - Str("logging.googleapis.com/trace", fmt.Sprintf("projects/%s/traces/%s", projectID, sc.TraceID())). - Str("logging.googleapis.com/spanId", sc.SpanID().String()). - Bool("logging.googleapis.com/trace_sampled", sc.IsSampled()). - Msg("handling request") - ``` - - Health-probe endpoints filtered out of tracing in Step 3 don't need these fields. +Use `entur/cloud-logging` with `spring-boot-starter-gcp-web` and plain SLF4J logging. The fields are injected automatically via Micrometer Tracing -- no manual extraction. ## Step 6: Tell the user what's left @@ -258,14 +144,14 @@ Steps 0-5 are everything this skill can do by editing the repo. Verifying traces 3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). 4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). -For Kotlin/Java, also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. +Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. ## Critical Rules - **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. - **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. - **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. -- **Java/Kotlin defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. +- **Defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. - **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. -- **Kotlin/Java also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. +- **Also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. diff --git a/tests/scenarios/28-tracing-iam-least-privilege.md b/tests/scenarios/28-tracing-iam-least-privilege.md index ac8f50b..c788a9a 100644 --- a/tests/scenarios/28-tracing-iam-least-privilege.md +++ b/tests/scenarios/28-tracing-iam-least-privilege.md @@ -10,7 +10,7 @@ account, not a broader trace role. What IAM role should be bound to the workload service account so it can export traces to Cloud Trace, per Entur's tracing golden path? -Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` +Read `skills/setup-tracing-java/SKILL.md` in this repository and answer in `key: value` format on its own line: - role: diff --git a/tests/scenarios/29-tracing-java-agent-default.md b/tests/scenarios/29-tracing-java-agent-default.md index 1048ee3..78d0173 100644 --- a/tests/scenarios/29-tracing-java-agent-default.md +++ b/tests/scenarios/29-tracing-java-agent-default.md @@ -3,10 +3,11 @@ ## Description Verifies the agent defaults a Kotlin/Java service to the OpenTelemetry Java Agent -without a specific reason to hand-instrument, AND correctly does NOT recommend the -Java Agent for a Go service in the same prompt -- confirming it understands the -language scope of the rule rather than pattern-matching "just get tracing working" -to one fixed answer. +(per `setup-tracing-java`) without a specific reason to hand-instrument, AND +correctly does NOT recommend the Java Agent for a Go service (per +`setup-tracing-go`) in the same prompt -- confirming it pulls the answer from the +language-scoped skill rather than pattern-matching "just get tracing working" to +one fixed answer. ## Prompt @@ -16,8 +17,8 @@ by either: 1. "We need tracing on our Spring Boot service, nothing fancy, just get it working." 2. "We need tracing on our Go service, nothing fancy, just get it working." -Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` -format on its own line: +Read `skills/setup-tracing-java/SKILL.md` and `skills/setup-tracing-go/SKILL.md` in +this repository and answer in `key: value` format on its own line: - spring_boot_approach: - go_approach: diff --git a/tests/scenarios/30-tracing-dockerfile-single-cmd.md b/tests/scenarios/30-tracing-dockerfile-single-cmd.md index aded62a..38a482b 100644 --- a/tests/scenarios/30-tracing-dockerfile-single-cmd.md +++ b/tests/scenarios/30-tracing-dockerfile-single-cmd.md @@ -15,7 +15,7 @@ CMD ["-javaagent:/profiler/profiler_java_agent.so", "-jar", "/app/app.jar"] Now add OpenTelemetry Java Agent tracing per Entur's golden path. -Read `skills/setup-tracing/SKILL.md` in this repository and answer in `key: value` +Read `skills/setup-tracing-java/SKILL.md` in this repository and answer in `key: value` format on its own line: - number_of_cmd_instructions: From 1d7ee5bb211c9c3ff776ada9a2c361c50f748715 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Tue, 28 Jul 2026 12:23:54 +0200 Subject: [PATCH 09/14] fix: telemetry API activation. --- skills/setup-tracing-go/SKILL.md | 5 +++-- skills/setup-tracing-java/SKILL.md | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/skills/setup-tracing-go/SKILL.md b/skills/setup-tracing-go/SKILL.md index 14f2849..8b0f1c6 100644 --- a/skills/setup-tracing-go/SKILL.md +++ b/skills/setup-tracing-go/SKILL.md @@ -26,15 +26,16 @@ Ask the user directly; do not infer silently and do not proceed until all four a 3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. 4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. -## Step 1: Enable the Cloud Trace API in Terraform +## Step 1: Enable the Cloud Trace and Telemetry APIs in Terraform -Add `cloudtrace.googleapis.com` to the per-environment service-activation block: +Add both `cloudtrace.googleapis.com` and `telemetry.googleapis.com` to the per-environment service-activation block -- `cloudtrace.googleapis.com` covers the Cloud Trace backend, while `telemetry.googleapis.com` enables the OTLP ingestion endpoint the service exports to: ```hcl # terraform/main.tf resource "google_project_service" "services" { for_each = toset([ "cloudtrace.googleapis.com", + "telemetry.googleapis.com", # ... other APIs your service needs ]) project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias diff --git a/skills/setup-tracing-java/SKILL.md b/skills/setup-tracing-java/SKILL.md index 480cfd5..ac9b647 100644 --- a/skills/setup-tracing-java/SKILL.md +++ b/skills/setup-tracing-java/SKILL.md @@ -26,15 +26,16 @@ Ask the user directly; do not infer silently and do not proceed until all four a 3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. 4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. -## Step 1: Enable the Cloud Trace API in Terraform +## Step 1: Enable the Cloud Trace and Telemetry APIs in Terraform -Add `cloudtrace.googleapis.com` to the per-environment service-activation block: +Add both `cloudtrace.googleapis.com` and `telemetry.googleapis.com` to the per-environment service-activation block -- `cloudtrace.googleapis.com` covers the Cloud Trace backend, while `telemetry.googleapis.com` enables the OTLP ingestion endpoint the service exports to: ```hcl # terraform/main.tf resource "google_project_service" "services" { for_each = toset([ "cloudtrace.googleapis.com", + "telemetry.googleapis.com", # ... other APIs your service needs ]) project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias @@ -111,6 +112,8 @@ common: value: "true" - name: GCP_PROJECT_ID value: ent-- + - name: GOOGLE_CLOUD_PROJECT + value: ent-- - name: OTEL_TRACES_SAMPLER value: parentbased_always_on - name: OTEL_METRICS_EXPORTER @@ -121,6 +124,8 @@ common: If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. Omit `OTEL_TRACES_SAMPLER` on Cloud Run, per the note below. +`GCP_PROJECT_ID` and `GOOGLE_CLOUD_PROJECT` carry the same value but serve different consumers: `GCP_PROJECT_ID` is read by application code that needs the project ID directly; `GOOGLE_CLOUD_PROJECT` is the variable Google's client libraries auto-detect to determine the target project. Set both. + Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). **Java Agent auto-configures from env vars.** On Cloud Run, set the same variables shown above on the `cloudrun.yaml` container spec, with one difference: omit `OTEL_TRACES_SAMPLER` entirely. From 41f905300964cd44319e564fa46744b450d1e2e5 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 4 Sep 2026 15:10:46 +0200 Subject: [PATCH 10/14] docs(setup-tracing-java): align skill with latest tracing.md Traces land in the shared host project (ent-kub-), and IAM roles plus required APIs are provisioned automatically via the common Helm chart -- remove the manual Terraform steps, the Cloud Trace storage console prerequisite, and the ent-- project references that contradicted this. Trim runtime env vars to just the sampler, matching tracing.md, and add its calls-per-minute sampling ratio table. --- skills/setup-tracing-java/SKILL.md | 133 ++++++++++------------------- 1 file changed, 46 insertions(+), 87 deletions(-) diff --git a/skills/setup-tracing-java/SKILL.md b/skills/setup-tracing-java/SKILL.md index ac9b647..0259c44 100644 --- a/skills/setup-tracing-java/SKILL.md +++ b/skills/setup-tracing-java/SKILL.md @@ -11,56 +11,19 @@ description: > # Set Up Tracing -- Kotlin/Java (Golden Path) -Wire distributed tracing into an Entur Kotlin/Java service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Spring Boot via the OpenTelemetry Java Agent** only -- it does not cover Go, Python, or any other language. +Wire distributed tracing into an Entur Kotlin/Java service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Spring Boot via the OpenTelemetry Java Agent** only -- it does not cover Go, Python, or any other language. Trace spans are exported to a shared host project (`ent-kub-`), not the application's own project -- this is what makes it possible to correlate a trace with the logs from the same request. -## Step 0: Confirm prerequisites -- do not guess any of these +## Step 0: Confirm prerequisites -- do not guess these -Ask the user directly; do not infer silently and do not proceed until all four are answered. +Ask the user directly; do not infer silently. 1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java. State the detected language and ask the user to confirm it. - If the repo is Go (`go.mod`), **stop** and point the user to the `setup-tracing-go` skill instead. - If the repo is neither Kotlin/Java nor Go (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation for Kotlin/Java -- do not improvise OpenTelemetry setup for another language. -2. **Cloud Trace storage (manual, console-only).** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* - - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. - - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. -3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. -4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. - -## Step 1: Enable the Cloud Trace and Telemetry APIs in Terraform - -Add both `cloudtrace.googleapis.com` and `telemetry.googleapis.com` to the per-environment service-activation block -- `cloudtrace.googleapis.com` covers the Cloud Trace backend, while `telemetry.googleapis.com` enables the OTLP ingestion endpoint the service exports to: - -```hcl -# terraform/main.tf -resource "google_project_service" "services" { - for_each = toset([ - "cloudtrace.googleapis.com", - "telemetry.googleapis.com", - # ... other APIs your service needs - ]) - project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias - service = each.value - disable_on_destroy = false -} -``` - -If the repo has no `terraform/` directory yet, do not create one ad hoc -- tell the user to bootstrap Terraform with the Entur `terraform-google-init` module first. - -## Step 2: Grant the workload service account `roles/cloudtrace.agent` - -```hcl -# terraform/main.tf -resource "google_project_iam_member" "runtime_cloudtrace_agent" { - project = module.init.app.project_id # adapt "init" to this repo's module alias - role = "roles/cloudtrace.agent" - member = "serviceAccount:${module.init.service_accounts.runtime.email}" - depends_on = [google_project_service.services] -} -``` - -Bind exactly this role -- it is write-only. Never bind `roles/cloudtrace.user`, `.admin`, or `.editor` to the workload; read access is granted separately at the folder level. +2. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where the sampler env var in Step 2 is set. +3. **Manifest check.** Confirm a `GoogleCloudApplication` manifest exists under `.entur/*.yaml` and the service is deployed on Kubernetes or Cloud Run. IAM roles and the required Google Cloud APIs for tracing are provisioned automatically through the common Helm chart -- do not add Terraform for either. -## Step 3: Instrument the application +## Step 1: Instrument the application ### OpenTelemetry Java Agent (default; do not hand-instrument without a specific reason) @@ -85,78 +48,74 @@ CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ "-jar", "/app/app.jar"] ``` -- Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a Workload Identity token to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. -- Use the pinned versions above as-is -- `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension). Do not spend time checking upstream for a newer release. In Step 6's summary, tell the user which versions were used and point to the `otel` build stage in the Dockerfile as where to bump them later. -- `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 6 summary as something the user may need to change. +- Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a valid GCP access token (via Application Default Credentials) to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. +- Use the pinned versions above as-is -- `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension). Do not spend time checking upstream for a newer release. In Step 4's summary, tell the user which versions were used and point to the `otel` build stage in the Dockerfile as where to bump them later. +- `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 4 summary as something the user may need to change. - If the project also uses Cloud Profiler, merge its `-javaagent`/flags into this **same** `CMD` -- a Dockerfile only honors its last `CMD`, a second one silently disables the first. -## Step 4: Set runtime environment variables +## Step 2: Set the sampler explicitly -Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. +Sampling must be set explicitly per environment -- do not rely on the default everywhere. -**If `helm//env/values-kub-ent-.yaml` doesn't exist yet for a confirmed environment**, ask the user: *"No Helm values file exists yet for `` -- do you want me to create `helm//env/values-kub-ent-.yaml` with the tracing env vars?"* Only create it if they say yes, and only for the environment(s) confirmed in Step 0. Use this shape: +**If `helm//env/values-kub-ent-.yaml` doesn't exist yet for an environment**, ask the user: *"No Helm values file exists yet for `` -- do you want me to create `helm//env/values-kub-ent-.yaml` with the sampler env vars?"* Only create it if they say yes. ```yaml -# helm//env/values-kub-ent-.yaml +# values-kub-ent-dev.yaml common: - env: container: env: - - name: OTEL_TRACES_EXPORTER - value: otlp - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: https://telemetry.googleapis.com - - name: OTEL_SERVICE_NAME - value: - - name: TRACING_ENABLED - value: "true" - - name: GCP_PROJECT_ID - value: ent-- - - name: GOOGLE_CLOUD_PROJECT - value: ent-- - name: OTEL_TRACES_SAMPLER - value: parentbased_always_on - - name: OTEL_METRICS_EXPORTER - value: none - - name: OTEL_LOGS_EXPORTER - value: none + value: parentbased_always_on # sample everything, easiest for debugging ``` -If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. Omit `OTEL_TRACES_SAMPLER` on Cloud Run, per the note below. +```yaml +# values-kub-ent-prd.yaml / values-kub-ent-tst.yaml +common: + container: + env: + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio # sample a fixed ratio of requests + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.1" # 10% of requests +``` -`GCP_PROJECT_ID` and `GOOGLE_CLOUD_PROJECT` carry the same value but serve different consumers: `GCP_PROJECT_ID` is read by application code that needs the project ID directly; `GOOGLE_CLOUD_PROJECT` is the variable Google's client libraries auto-detect to determine the target project. Set both. +If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. -Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). +These are recommended defaults, not hard requirements -- adjust `OTEL_TRACES_SAMPLER_ARG` if a service needs a different ratio: -**Java Agent auto-configures from env vars.** On Cloud Run, set the same variables shown above on the `cloudrun.yaml` container spec, with one difference: omit `OTEL_TRACES_SAMPLER` entirely. +| Calls per minute | Sampling ratio | +|-------------------|----------------| +| Fewer than 50 | 100% | +| 50 to 100 | 50% | +| More than 1,000 | 1% | +| More than 5,000 | 0.01% | -Always set `OTEL_METRICS_EXPORTER=none` and `OTEL_LOGS_EXPORTER=none` on both runtimes -- the agent defaults both to `otlp` like traces, but Step 2's IAM grant doesn't authorize either. +Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill. -Do not set `OTEL_TRACES_SAMPLER` on Cloud Run: the Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. +**On Cloud Run**, set the variable on the `cloudrun.yaml` container spec instead, with one difference: omit `OTEL_TRACES_SAMPLER`/`OTEL_TRACES_SAMPLER_ARG` entirely. The Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. -## Step 5: Correlate logs with traces +## Step 3: Correlate logs with traces -Every log line inside a traced request needs `logging.googleapis.com/trace` (`projects//traces/`), `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. +Every log line inside a traced request needs `logging.googleapis.com/trace`, `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. -Use `entur/cloud-logging` with `spring-boot-starter-gcp-web` and plain SLF4J logging. The fields are injected automatically via Micrometer Tracing -- no manual extraction. +For Java/Kotlin Spring Boot services, no manual work is needed. If `entur/cloud-logging` (v7.1.0) together with `spring-boot-starter-gcp-web` is correctly set up, those two handle all of this automatically -- the library injects the fields via Micrometer Tracing, and standard SLF4J logging picks them up on every log line within a traced request. -## Step 6: Tell the user what's left +## Step 4: Tell the user what's left -Steps 0-5 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: +Steps 0-3 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: -1. Commit and merge the changes, and let the normal CD pipeline deploy to the confirmed environment(s) from Step 0. +1. Commit and merge the changes, and let the normal CD pipeline deploy. 2. Send a few requests to the deployed service to generate spans. -3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). -4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). +3. Check **Monitoring → Trace → Trace Explorer** in the GCP Console under the shared host project `ent-kub-` (not the application project, even for Kubernetes workloads -- traces always land in the host project). Since traces from multiple applications land in the same project, filter Trace Explorer by `service.name` to scope the view to just this service. +4. If no spans show up: trace storage provisions automatically the first time a span is successfully written to the project, and it isn't instant -- give it a few minutes before assuming something is broken. Also double check the sampler env var actually reached the deployed container for that environment (a common miss is setting it in the wrong Helm values file). Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. ## Critical Rules -- **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. -- **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. -- **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. +- **Traces land in the shared host project `ent-kub-`**, never the application's own project -- this applies to Kubernetes and Cloud Run alike. No `GCP_PROJECT_ID`/`GOOGLE_CLOUD_PROJECT` env var configuration is needed for tracing to work. +- **IAM roles and the required Google Cloud APIs are provisioned automatically** through the common Helm chart. Never add Terraform to enable `cloudtrace.googleapis.com`/`telemetry.googleapis.com` or to grant a trace-related IAM role for this. +- **Trace storage auto-provisions** on first successful span write -- never tell the user to manually enable it in the console, and never script it. - **Defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. -- **Kubernetes vs Cloud Run sampler differs** -- see Step 4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. -- **Also disables metrics/logs export** -- see Step 4. They default to `otlp` too if left unset, but Step 2 doesn't authorize either. +- **Kubernetes vs Cloud Run sampler differs** -- see Step 2. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. From 7a69d52a232bca3076d5723d97242f2c2da0bdc7 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 4 Sep 2026 15:25:05 +0200 Subject: [PATCH 11/14] feat: removed go skill and updated java skill --- .agents/plugins/marketplace.json | 12 - .claude-plugin/marketplace.json | 14 -- .../.claude-plugin/plugin.json | 20 -- .../.codex-plugin/plugin.json | 32 --- .../setup-tracing-go/skills/setup-tracing-go | 1 - skills/README.md | 1 - skills/setup-tracing-go/SKILL.md | 209 ------------------ skills/setup-tracing-java/SKILL.md | 29 +-- 8 files changed, 2 insertions(+), 316 deletions(-) delete mode 100644 plugins/setup-tracing-go/.claude-plugin/plugin.json delete mode 100644 plugins/setup-tracing-go/.codex-plugin/plugin.json delete mode 120000 plugins/setup-tracing-go/skills/setup-tracing-go delete mode 100644 skills/setup-tracing-go/SKILL.md diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index c921dfa..95dabb9 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -63,18 +63,6 @@ "authentication": "ON_INSTALL" }, "category": "DevOps" - }, - { - "name": "setup-tracing-go", - "source": { - "source": "local", - "path": "./plugins/setup-tracing-go" - }, - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL" - }, - "category": "DevOps" } ] } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 697a696..f5b5afa 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -78,20 +78,6 @@ "homepage": "https://github.com/entur/ai", "repository": "https://github.com/entur/ai", "license": "EUPL-1.2" - }, - { - "name": "setup-tracing-go", - "source": "./plugins/setup-tracing-go", - "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", - "category": "observability", - "keywords": ["entur", "tracing", "opentelemetry", "cloud-trace", "observability", "go"], - "author": { - "name": "Team Kvalitet", - "email": "support@entur.org" - }, - "homepage": "https://github.com/entur/ai", - "repository": "https://github.com/entur/ai", - "license": "EUPL-1.2" } ] } diff --git a/plugins/setup-tracing-go/.claude-plugin/plugin.json b/plugins/setup-tracing-go/.claude-plugin/plugin.json deleted file mode 100644 index 2170336..0000000 --- a/plugins/setup-tracing-go/.claude-plugin/plugin.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "setup-tracing-go", - "version": "0.1.0", - "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", - "author": { - "name": "Team Kvalitet", - "email": "support@entur.org" - }, - "homepage": "https://github.com/entur/ai", - "repository": "https://github.com/entur/ai", - "license": "EUPL-1.2", - "keywords": [ - "entur", - "tracing", - "opentelemetry", - "cloud-trace", - "observability", - "go" - ] -} diff --git a/plugins/setup-tracing-go/.codex-plugin/plugin.json b/plugins/setup-tracing-go/.codex-plugin/plugin.json deleted file mode 100644 index af5f600..0000000 --- a/plugins/setup-tracing-go/.codex-plugin/plugin.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "setup-tracing-go", - "version": "0.1.0", - "description": "Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace, following the Entur tracing golden path.", - "author": { - "name": "Team Kvalitet", - "email": "support@entur.org" - }, - "homepage": "https://github.com/entur/ai", - "repository": "https://github.com/entur/ai", - "license": "EUPL-1.2", - "keywords": [ - "entur", - "tracing", - "opentelemetry", - "cloud-trace", - "observability", - "go" - ], - "skills": "./skills/", - "interface": { - "displayName": "Tracing Setup (Go)", - "shortDescription": "Instrument a Go service with OpenTelemetry and ship traces to Google Cloud Trace.", - "longDescription": "Wires distributed tracing into a Go service via manual OpenTelemetry SDK instrumentation, following the Entur tracing golden path: Terraform API/IAM setup, application instrumentation, runtime env vars, and log-trace correlation.", - "developerName": "Team Kvalitet", - "category": "DevOps", - "capabilities": [ - "Write" - ], - "websiteURL": "https://github.com/entur/ai" - } -} diff --git a/plugins/setup-tracing-go/skills/setup-tracing-go b/plugins/setup-tracing-go/skills/setup-tracing-go deleted file mode 120000 index da8c011..0000000 --- a/plugins/setup-tracing-go/skills/setup-tracing-go +++ /dev/null @@ -1 +0,0 @@ -../../../skills/setup-tracing-go \ No newline at end of file diff --git a/skills/README.md b/skills/README.md index b5da398..66f1f6c 100644 --- a/skills/README.md +++ b/skills/README.md @@ -94,7 +94,6 @@ gh skill install entur/ai | [setup-cicd-workflows](setup-cicd-workflows/) | Generate CI/CD GitHub Actions workflows using Entur reusable workflows | | [scr-situation-complication-resolution](scr-situation-complication-resolution/) | Structure problems and decisions in SCR format for leadership | | [setup-tracing-java](setup-tracing-java/) | Instrument a Kotlin/Java (Spring Boot) service with the OpenTelemetry Java Agent and ship traces to Google Cloud Trace | -| [setup-tracing-go](setup-tracing-go/) | Instrument a Go service with manual OpenTelemetry SDK wiring and ship traces to Google Cloud Trace | ## 🤝 Contributing diff --git a/skills/setup-tracing-go/SKILL.md b/skills/setup-tracing-go/SKILL.md deleted file mode 100644 index 8b0f1c6..0000000 --- a/skills/setup-tracing-go/SKILL.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -name: setup-tracing-go -description: > - Wire distributed tracing into an Entur Go service using manual - OpenTelemetry SDK instrumentation, following the golden path to Cloud - Trace. Use when the user says "add tracing", "set up OpenTelemetry", - "instrument for Cloud Trace", or "add distributed tracing" for a Go - service -- typically a repo with `go.mod`. For a Kotlin/Java or Spring - Boot service, use the `setup-tracing-java` skill instead. ---- - -# Set Up Tracing -- Go (Golden Path) - -Wire distributed tracing into an Entur Go service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **manual OpenTelemetry SDK wiring** only -- it does not cover Kotlin/Java, Python, or any other language. - -## Step 0: Confirm prerequisites -- do not guess any of these - -Ask the user directly; do not infer silently and do not proceed until all four are answered. - -1. **Language.** Detect from the repo: `go.mod` → Go. State the detected language and ask the user to confirm it. - - If the repo is Kotlin/Java (`build.gradle.kts`), **stop** and point the user to the `setup-tracing-java` skill instead. - - If the repo is neither Go nor Kotlin/Java (Python, Node, etc.), **stop**. Tell the user this golden path only documents manual SDK wiring for Go -- do not improvise OpenTelemetry setup for another language. -2. **Cloud Trace storage (manual, console-only).** Ask: *"Have you already enabled Cloud Trace storage in the GCP Console (Monitoring → Trace Explorer → Enable trace storage) for every environment you're setting up, in the application project `ent--` -- not the cluster host project `ent-kub-`?"* - - If no, or unsure, **stop** and tell them to do this first, once per environment (dev, tst, prd). This is a console-only action -- there is no Terraform resource for it yet. Do not attempt to script it. - - Only proceed with the environments the user confirms are done. If they've only enabled it for `dev`, scope the rest of this skill to `dev`. -3. **App ID.** Needed to build `ent--`. Read `metadata.id` from the self-service manifest under `.entur/*.yaml` if present. If the manifest exists but `metadata.id` is missing or empty, ask the user for it instead of guessing -- do not derive it from the repo name or any other field. If no manifest exists at all, ask the user directly. -4. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where env vars in Step 4 are set. - -## Step 1: Enable the Cloud Trace and Telemetry APIs in Terraform - -Add both `cloudtrace.googleapis.com` and `telemetry.googleapis.com` to the per-environment service-activation block -- `cloudtrace.googleapis.com` covers the Cloud Trace backend, while `telemetry.googleapis.com` enables the OTLP ingestion endpoint the service exports to: - -```hcl -# terraform/main.tf -resource "google_project_service" "services" { - for_each = toset([ - "cloudtrace.googleapis.com", - "telemetry.googleapis.com", - # ... other APIs your service needs - ]) - project = module.init.app.project_id # "init" assumes that's this repo's terraform-google-init alias - service = each.value - disable_on_destroy = false -} -``` - -If the repo has no `terraform/` directory yet, do not create one ad hoc -- tell the user to bootstrap Terraform with the Entur `terraform-google-init` module first. - -## Step 2: Grant the workload service account `roles/cloudtrace.agent` - -```hcl -# terraform/main.tf -resource "google_project_iam_member" "runtime_cloudtrace_agent" { - project = module.init.app.project_id # adapt "init" to this repo's module alias - role = "roles/cloudtrace.agent" - member = "serviceAccount:${module.init.service_accounts.runtime.email}" - depends_on = [google_project_service.services] -} -``` - -Bind exactly this role -- it is write-only. Never bind `roles/cloudtrace.user`, `.admin`, or `.editor` to the workload; read access is granted separately at the folder level. - -## Step 3: Instrument the application -- manual OpenTelemetry SDK wiring - -```go -// internal/tracing/tracing.go -package tracing - -import ( - "context" - "fmt" - - googlegrpc "google.golang.org/grpc/credentials/google" - "google.golang.org/grpc" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.26.0" -) - -func Init(ctx context.Context, projectID, serviceName, serviceVersion string) (func(context.Context) error, error) { - exp, err := otlptracegrpc.New(ctx, - otlptracegrpc.WithEndpoint("telemetry.googleapis.com:443"), - otlptracegrpc.WithDialOption( - grpc.WithTransportCredentials(googlegrpc.NewDefaultCredentials()), - ), - ) - if err != nil { - return nil, fmt.Errorf("otlp trace exporter: %w", err) - } - - res, _ := resource.New(ctx, resource.WithAttributes( - semconv.ServiceName(serviceName), - semconv.ServiceVersion(serviceVersion), - )) - - tp := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exp), - sdktrace.WithResource(res), - sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), - // sdktrace.WithSampler(sdktrace.AlwaysSample()), use this instead if the service runs on Cloud Run - ) - otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - )) - return tp.Shutdown, nil -} -``` - -Run `go mod tidy` after pasting. - -- `googlegrpc.NewDefaultCredentials()` bundles TLS + Application Default Credentials -- no API keys or credential files in the container; auth resolves through the Workload Identity binding from Step 2. -- Wrap the outermost handler so every inbound request gets a span, and filter probe/metrics paths so they don't drown the trace stream: - - ```go - // main.go - import ( - "net/http" - "strings" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - ) - - handler := otelhttp.NewHandler(mux, "", - otelhttp.WithFilter(func(r *http.Request) bool { - return r.URL.Path != "/metrics" && !strings.HasPrefix(r.URL.Path, "/health/") - }), - ) - http.ListenAndServe(":8080", handler) - ``` - -- Call `tracing.Init` at the top of `main()`, guarded by `TRACING_ENABLED`, and always `defer shutdown(ctx)` so buffered spans flush before exit: - - ```go - // main.go - ctx := context.Background() - if os.Getenv("TRACING_ENABLED") == "true" { - projectID := os.Getenv("GCP_PROJECT_ID") - shutdown, err := tracing.Init(ctx, projectID, "my-service", "1.0.0") - if err != nil { - logging.Fatal().Err(err).Msg("failed to initialize tracing") - } - defer shutdown(ctx) - } - ``` - -- Any manual span from `tracer.Start(ctx, ...)` must `defer span.End()` -- an unended span never exports. - -## Step 4: Set runtime environment variables - -Set on Helm values (`values-kub-ent-.yaml`, under `common.container.env`) for Kubernetes, or on the container spec (`cloudrun.yaml`) for Cloud Run. `GCP_PROJECT_ID` is always the application's per-env project (`ent--`) -- never the cluster host project. - -**If `helm//env/values-kub-ent-.yaml` doesn't exist yet for a confirmed environment**, ask the user: *"No Helm values file exists yet for `` -- do you want me to create `helm//env/values-kub-ent-.yaml` with the tracing env vars?"* Only create it if they say yes, and only for the environment(s) confirmed in Step 0. Use this shape: - -```yaml -# helm//env/values-kub-ent-.yaml -common: - env: - container: - env: - - name: TRACING_ENABLED - value: "true" - - name: GCP_PROJECT_ID - value: ent-- -``` - -If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. - -Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill (same rule as Step 1's Terraform bootstrap). - -The sampler and exporter are already hardcoded in `tracing.Init` (Step 3), so only the two env vars shown above are needed -- set the same on both Kubernetes and Cloud Run. Set the sampler in code to match the runtime instead: `ParentBased(AlwaysSample())` on Kubernetes, bare `AlwaysSample()` on Cloud Run (see the commented-out line in Step 3). - -## Step 5: Correlate logs with traces - -Every log line inside a traced request needs `logging.googleapis.com/trace` (`projects//traces/`), `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. - -`go get github.com/entur/go-logging`, then extract the span context per handler and attach the fields to every log call inside that handler (info/warn/error alike): - -```go -sc := trace.SpanFromContext(r.Context()).SpanContext() -projectID := os.Getenv("GCP_PROJECT_ID") -logging.Info(). - Str("logging.googleapis.com/trace", fmt.Sprintf("projects/%s/traces/%s", projectID, sc.TraceID())). - Str("logging.googleapis.com/spanId", sc.SpanID().String()). - Bool("logging.googleapis.com/trace_sampled", sc.IsSampled()). - Msg("handling request") -``` - -Health-probe endpoints filtered out of tracing in Step 3 don't need these fields. - -## Step 6: Tell the user what's left - -Steps 0-5 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: - -1. Commit and merge the changes, and let the normal CD pipeline deploy to the confirmed environment(s) from Step 0. -2. Send a few requests to the deployed service to generate spans. -3. Check **Trace → Trace Explorer** in the GCP Console under `ent--` (not `ent-kub-`, even for Kubernetes workloads -- traces always land in the application project). -4. If no spans show up: re-check that Step 0's trace storage was enabled for the *deployed* environment, and that `TRACING_ENABLED`/`GCP_PROJECT_ID` reached the running container (a common miss is setting them in the wrong Helm values file for the target environment). - -## Critical Rules - -- **The Step 0 trace-storage check is a manual console action, once per environment.** Never attempt to Terraform it; never proceed with Steps 1-5 for an environment the user hasn't confirmed is enabled. -- **Grant exactly `roles/cloudtrace.agent`** to the workload -- never a broader trace role. -- **`GCP_PROJECT_ID` is always the application project (`ent--`)**, never the cluster host project, even on Kubernetes. -- **Kubernetes vs Cloud Run sampler differs** -- see Step 3/4. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. -- **Any manual span must `defer span.End()`** -- an unended span never exports. diff --git a/skills/setup-tracing-java/SKILL.md b/skills/setup-tracing-java/SKILL.md index 0259c44..b08aa37 100644 --- a/skills/setup-tracing-java/SKILL.md +++ b/skills/setup-tracing-java/SKILL.md @@ -6,22 +6,12 @@ description: > Trace. Use when the user says "add tracing", "set up OpenTelemetry", "instrument for Cloud Trace", or "add distributed tracing" for a Kotlin, Java, or Spring Boot service -- typically a repo with `build.gradle.kts`. - For a Go service, use the `setup-tracing-go` skill instead. --- # Set Up Tracing -- Kotlin/Java (Golden Path) Wire distributed tracing into an Entur Kotlin/Java service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Spring Boot via the OpenTelemetry Java Agent** only -- it does not cover Go, Python, or any other language. Trace spans are exported to a shared host project (`ent-kub-`), not the application's own project -- this is what makes it possible to correlate a trace with the logs from the same request. -## Step 0: Confirm prerequisites -- do not guess these - -Ask the user directly; do not infer silently. - -1. **Language.** Detect from the repo: `build.gradle.kts` → Kotlin/Java. State the detected language and ask the user to confirm it. - - If the repo is Go (`go.mod`), **stop** and point the user to the `setup-tracing-go` skill instead. - - If the repo is neither Kotlin/Java nor Go (Python, Node, etc.), **stop**. Tell the user this golden path only documents Java Agent instrumentation for Kotlin/Java -- do not improvise OpenTelemetry setup for another language. -2. **Runtime.** Kubernetes (`helm//env/values-kub-ent-.yaml`) or Cloud Run (`cloudrun.yaml`)? Determines where the sampler env var in Step 2 is set. -3. **Manifest check.** Confirm a `GoogleCloudApplication` manifest exists under `.entur/*.yaml` and the service is deployed on Kubernetes or Cloud Run. IAM roles and the required Google Cloud APIs for tracing are provisioned automatically through the common Helm chart -- do not add Terraform for either. ## Step 1: Instrument the application @@ -51,7 +41,6 @@ CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ - Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a valid GCP access token (via Application Default Credentials) to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. - Use the pinned versions above as-is -- `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension). Do not spend time checking upstream for a newer release. In Step 4's summary, tell the user which versions were used and point to the `otel` build stage in the Dockerfile as where to bump them later. - `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 4 summary as something the user may need to change. -- If the project also uses Cloud Profiler, merge its `-javaagent`/flags into this **same** `CMD` -- a Dockerfile only honors its last `CMD`, a second one silently disables the first. ## Step 2: Set the sampler explicitly @@ -81,32 +70,19 @@ common: If the file already exists with its own `common.container.env` entries, append these to the existing list instead of overwriting the file -- check for entries with the same `name` first and update their `value` in place rather than duplicating. -These are recommended defaults, not hard requirements -- adjust `OTEL_TRACES_SAMPLER_ARG` if a service needs a different ratio: - -| Calls per minute | Sampling ratio | -|-------------------|----------------| -| Fewer than 50 | 100% | -| 50 to 100 | 50% | -| More than 1,000 | 1% | -| More than 5,000 | 0.01% | - Only create the single env file this way -- do not scaffold `helm//Chart.yaml` or `helm//values.yaml` ad hoc. If those don't exist either, Helm hasn't been bootstrapped for this service at all, which is out of scope for this skill. -**On Cloud Run**, set the variable on the `cloudrun.yaml` container spec instead, with one difference: omit `OTEL_TRACES_SAMPLER`/`OTEL_TRACES_SAMPLER_ARG` entirely. The Cloud Run load balancer injects its own (thin) sampling decision upstream, and `parentbased_always_on` would defer to it, sampling far less than intended. Omitting the variable lets the agent default to `always_on`. - ## Step 3: Correlate logs with traces -Every log line inside a traced request needs `logging.googleapis.com/trace`, `logging.googleapis.com/spanId`, and `logging.googleapis.com/trace_sampled` -- otherwise Cloud Logging can't join it to the trace in Trace Explorer. - For Java/Kotlin Spring Boot services, no manual work is needed. If `entur/cloud-logging` (v7.1.0) together with `spring-boot-starter-gcp-web` is correctly set up, those two handle all of this automatically -- the library injects the fields via Micrometer Tracing, and standard SLF4J logging picks them up on every log line within a traced request. ## Step 4: Tell the user what's left -Steps 0-3 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: +Steps 1-3 are everything this skill can do by editing the repo. Verifying traces actually arrive requires a live deploy and real traffic, which happens outside this skill -- tell the user to: 1. Commit and merge the changes, and let the normal CD pipeline deploy. 2. Send a few requests to the deployed service to generate spans. -3. Check **Monitoring → Trace → Trace Explorer** in the GCP Console under the shared host project `ent-kub-` (not the application project, even for Kubernetes workloads -- traces always land in the host project). Since traces from multiple applications land in the same project, filter Trace Explorer by `service.name` to scope the view to just this service. +3. Check **Monitoring → Trace → Trace Explorer** in the GCP Console under the shared host project `ent-kub-` (not the application project, traces always land in the host project). Since traces from multiple applications land in the same project, filter Trace Explorer by `service.name` to scope the view to just this service. 4. If no spans show up: trace storage provisions automatically the first time a span is successfully written to the project, and it isn't instant -- give it a few minutes before assuming something is broken. Also double check the sampler env var actually reached the deployed container for that environment (a common miss is setting it in the wrong Helm values file). Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. @@ -118,4 +94,3 @@ Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth - **Trace storage auto-provisions** on first successful span write -- never tell the user to manually enable it in the console, and never script it. - **Defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. - **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. -- **Kubernetes vs Cloud Run sampler differs** -- see Step 2. Getting this backwards silently under-samples on Cloud Run or double-samples on Kubernetes. From 4107a6d00c08c9907c42c3866ba1dca2e9d59c7a Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 4 Sep 2026 15:28:25 +0200 Subject: [PATCH 12/14] chore(tests): remove obsolete Java-vs-Go tracing test scenario Scenario 29 read skills/setup-tracing-go/SKILL.md to compare its behavior against setup-tracing-java, but the Go skill was removed -- the file it depended on no longer exists. --- .../29-tracing-java-agent-default.md | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 tests/scenarios/29-tracing-java-agent-default.md diff --git a/tests/scenarios/29-tracing-java-agent-default.md b/tests/scenarios/29-tracing-java-agent-default.md deleted file mode 100644 index 78d0173..0000000 --- a/tests/scenarios/29-tracing-java-agent-default.md +++ /dev/null @@ -1,43 +0,0 @@ -# Scenario: Java Agent Is the Default for Java/Kotlin, Not Go - -## Description - -Verifies the agent defaults a Kotlin/Java service to the OpenTelemetry Java Agent -(per `setup-tracing-java`) without a specific reason to hand-instrument, AND -correctly does NOT recommend the Java Agent for a Go service (per -`setup-tracing-go`) in the same prompt -- confirming it pulls the answer from the -language-scoped skill rather than pattern-matching "just get tracing working" to -one fixed answer. - -## Prompt - -Two engineers ask for tracing, no specific reason for manual instrumentation given -by either: - -1. "We need tracing on our Spring Boot service, nothing fancy, just get it working." -2. "We need tracing on our Go service, nothing fancy, just get it working." - -Read `skills/setup-tracing-java/SKILL.md` and `skills/setup-tracing-go/SKILL.md` in -this repository and answer in `key: value` format on its own line: - -- spring_boot_approach: -- go_approach: - -## Assertions - -```json -{ - "must_contain": [ - "spring_boot_approach: java agent", - "go_approach: manual sdk" - ], - "must_not_contain": [ - "spring_boot_approach: manual sdk", - "go_approach: java agent" - ] -} -``` - -## Budget - -0.08 From 20ccba40b8018c74fea692661853ca9385a42800 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 4 Sep 2026 15:31:57 +0200 Subject: [PATCH 13/14] chore(tests): remove tracing test scenarios invalidated by the golden-path update - 28-tracing-iam-least-privilege.md tested a manual roles/cloudtrace.agent grant that setup-tracing-java no longer performs -- IAM is now provisioned automatically via the common Helm chart. - 17-trace-project-routing.md asserted the old app-project (ent--) trace routing model by reading guides/reference/tracing.md directly, which still documents that old model on this branch and now contradicts the updated skill. Removed rather than updated since the reference doc it depends on hasn't been synced yet. --- tests/scenarios/17-trace-project-routing.md | 47 ------------------- .../28-tracing-iam-least-privilege.md | 35 -------------- 2 files changed, 82 deletions(-) delete mode 100644 tests/scenarios/17-trace-project-routing.md delete mode 100644 tests/scenarios/28-tracing-iam-least-privilege.md diff --git a/tests/scenarios/17-trace-project-routing.md b/tests/scenarios/17-trace-project-routing.md deleted file mode 100644 index 01890aa..0000000 --- a/tests/scenarios/17-trace-project-routing.md +++ /dev/null @@ -1,47 +0,0 @@ -# Scenario: Cloud Trace Project Routing for Kubernetes Workloads - -## Description - -Verifies that the agent picks the **application's** per-env project (`ent--`) for Cloud Trace reads on a Kubernetes workload, **not** the cluster host project (`ent-kub-`). This is the most-confused routing rule: Cloud Logging routes k8s logs to the cluster host project, but Cloud Trace (and Cloud Profiler) route to the application project regardless of runtime. - -## Prompt - -You are helping an Entur engineer debug a slow request in a Kubernetes-runtime service. - -Details: - -- Repository: `entur/some-repo` -- App ID (metadata.id): `someapp` -- Runtime: Kubernetes (deployed via the common Helm chart) -- Environment: prd -- Cluster host project: `ent-kub-prd` - -The engineer wants to query Cloud Trace for slow traces from the last hour. - -Read the Entur AI documentation in this repository (start with AGENTS.md, then read the tracing and observability references) and answer in `key: value` format on its own line: - -- trace_project_id: -- logs_project_id: -- one_line_reason: - -## Assertions - -```json -{ - "must_contain": [ - "trace_project_id: ent-someapp-prd", - "logs_project_id: ent-kub-prd" - ], - "must_not_contain": [ - "trace_project_id: ent-kub-prd", - "logs_project_id: ent-someapp-prd" - ], - "must_match": [ - "(workload|application|app).*(report|stamp|write|emit|own).*project|kubelet.*cluster|trace.*agent.*workload|exporter.*workload" - ] -} -``` - -## Budget - -0.10 diff --git a/tests/scenarios/28-tracing-iam-least-privilege.md b/tests/scenarios/28-tracing-iam-least-privilege.md deleted file mode 100644 index c788a9a..0000000 --- a/tests/scenarios/28-tracing-iam-least-privilege.md +++ /dev/null @@ -1,35 +0,0 @@ -# Scenario: Tracing IAM Role Is Write-Only - -## Description - -Verifies the agent grants exactly roles/cloudtrace.agent to the workload service -account, not a broader trace role. - -## Prompt - -What IAM role should be bound to the workload service account so it can export -traces to Cloud Trace, per Entur's tracing golden path? - -Read `skills/setup-tracing-java/SKILL.md` in this repository and answer in `key: value` -format on its own line: - -- role: - -## Assertions - -```json -{ - "must_contain": [ - "role: roles/cloudtrace.agent" - ], - "must_not_contain": [ - "role: roles/cloudtrace.user", - "role: roles/cloudtrace.admin", - "role: roles/cloudtrace.editor" - ] -} -``` - -## Budget - -0.08 From 887b92fcf4fac30da769c224bf8ce180d1d64bc9 Mon Sep 17 00:00:00 2001 From: Helle Engebakken Date: Fri, 4 Sep 2026 16:03:14 +0200 Subject: [PATCH 14/14] fix(setup-tracing-java): align Dockerfile guidance with golden path, tighten cloud-logging version check - Merge tracing agent flags into the existing ENTRYPOINT instead of assuming a CMD-based single-jar launch -- Entur's Docker golden path (docker.md) always launches the JVM via ENTRYPOINT, for both the preferred layered-JAR pattern and the single-jar Alpine alternative. - Require entur/cloud-logging v7.1.0+ explicitly (earlier versions don't inject trace/span fields), and add the missing unhappy path: point to logging.md when cloud-logging is missing or too old, and flag the Spring Boot 4.1.x dependency that comes with it. - Tighten the skill's scope to Spring Boot specifically, with an explicit check before proceeding, since the rest of the skill (log correlation) only works for Spring Boot services. - Link the Cloud Profiler critical rule to profiler.md instead of leaving it as a dangling reference. - Update test scenario 30 to match the ENTRYPOINT-based guidance. --- skills/setup-tracing-java/SKILL.md | 49 +++++++++++++------ .../30-tracing-dockerfile-single-cmd.md | 45 ----------------- ...30-tracing-dockerfile-single-entrypoint.md | 46 +++++++++++++++++ 3 files changed, 79 insertions(+), 61 deletions(-) delete mode 100644 tests/scenarios/30-tracing-dockerfile-single-cmd.md create mode 100644 tests/scenarios/30-tracing-dockerfile-single-entrypoint.md diff --git a/skills/setup-tracing-java/SKILL.md b/skills/setup-tracing-java/SKILL.md index b08aa37..2cdccb4 100644 --- a/skills/setup-tracing-java/SKILL.md +++ b/skills/setup-tracing-java/SKILL.md @@ -1,46 +1,58 @@ --- name: setup-tracing-java description: > - Wire distributed tracing into an Entur Kotlin/Java (Spring Boot) service + Wire distributed tracing into an Entur Kotlin/Java Spring Boot service using the OpenTelemetry Java Agent, following the golden path to Cloud Trace. Use when the user says "add tracing", "set up OpenTelemetry", - "instrument for Cloud Trace", or "add distributed tracing" for a Kotlin, - Java, or Spring Boot service -- typically a repo with `build.gradle.kts`. + "instrument for Cloud Trace", or "add distributed tracing" for a Kotlin + or Java service that runs on Spring Boot -- typically a repo with + `build.gradle.kts` and a `spring-boot-starter-*` dependency. Does not + apply to non-Spring-Boot Kotlin/Java services (e.g. plain Ktor, Micronaut, + or CLI apps) -- those need manual OpenTelemetry SDK instrumentation, + which is out of scope for this skill. --- # Set Up Tracing -- Kotlin/Java (Golden Path) Wire distributed tracing into an Entur Kotlin/Java service so every inbound request produces a span in Cloud Trace, correlated with structured logs. This golden path covers **Spring Boot via the OpenTelemetry Java Agent** only -- it does not cover Go, Python, or any other language. Trace spans are exported to a shared host project (`ent-kub-`), not the application's own project -- this is what makes it possible to correlate a trace with the logs from the same request. +**Before starting, confirm the project is actually Spring Boot** (a `spring-boot-starter-*` dependency in `build.gradle.kts`). If it isn't -- e.g. a plain Ktor, Micronaut, or non-web service -- stop and tell the user this skill only covers Spring Boot; manual OpenTelemetry SDK instrumentation is out of scope here. ## Step 1: Instrument the application ### OpenTelemetry Java Agent (default; do not hand-instrument without a specific reason) -Attach the agent via `-javaagent` in a multi-stage Dockerfile -- a temporary Alpine stage downloads the JARs, the distroless final stage only carries the JARs themselves: +Attach the agent via `-javaagent` in the existing Dockerfile: add one temporary stage that downloads the JARs, and merge the flags into the final stage's `ENTRYPOINT` -- do not touch any other existing stages (bundler, builder, layers, etc.) and do not introduce a `CMD`. Entur's Docker golden path ([docker.md](../../guides/reference/docker.md)) always launches the JVM via `ENTRYPOINT`, both in the preferred layered-JAR pattern and the single-jar Alpine alternative, so the agent flags belong in that array, prepended before the existing launch arguments. ```dockerfile -# Dockerfile +# Stage: download OTel JARs (temporary -- never shipped) FROM alpine:3.24 AS otel RUN mkdir /otel && \ wget -q -O /otel/opentelemetry-javaagent.jar \ https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.29.0/opentelemetry-javaagent.jar && \ wget -q -O /otel/gcp-auth-extension.jar \ https://repo1.maven.org/maven2/io/opentelemetry/contrib/opentelemetry-gcp-auth-extension/1.58.0-alpha/opentelemetry-gcp-auth-extension-1.58.0-alpha-shadow.jar +``` + +Then, in the project's existing final (runtime) stage, add `COPY --from=otel /otel /otel` and prepend the agent flags to the existing `ENTRYPOINT`. For the **preferred layered-JAR pattern** (docker.md's default), that looks like: -FROM gcr.io/distroless/java25-debian13:nonroot -WORKDIR /app +```dockerfile +# Final runtime stage -- only the two lines below are new; everything else +# (the COPY --from=layers lines, base image, etc.) is whatever the project already has COPY --from=otel /otel /otel -COPY build/libs/app.jar app.jar -CMD ["-javaagent:/otel/opentelemetry-javaagent.jar", \ - "-Dotel.javaagent.extensions=/otel/gcp-auth-extension.jar", \ - "-Dotel.javaagent.logging=application", \ - "-jar", "/app/app.jar"] +ENTRYPOINT ["java", \ + "-javaagent:/otel/opentelemetry-javaagent.jar", \ + "-Dotel.javaagent.extensions=/otel/gcp-auth-extension.jar", \ + "-Dotel.javaagent.logging=application", \ + "-XX:MaxRAMPercentage=75.0", \ + "org.springframework.boot.loader.launch.JarLauncher"] ``` +For the **single-jar Alpine alternative** (`ENTRYPOINT ["java", "-jar", "app.jar"]`), insert the same three `-javaagent`/`-D` flags immediately before `-jar` in that array instead. + - Both JARs are required: `opentelemetry-javaagent.jar` does the bytecode instrumentation; `gcp-auth-extension` attaches a valid GCP access token (via Application Default Credentials) to outbound OTLP calls -- without it, `telemetry.googleapis.com` rejects the export as unauthenticated. - Use the pinned versions above as-is -- `v2.29.0` (agent) / `1.58.0-alpha` (gcp-auth-extension). Do not spend time checking upstream for a newer release. In Step 4's summary, tell the user which versions were used and point to the `otel` build stage in the Dockerfile as where to bump them later. -- `java25-debian13` is an example tag, not a guarantee -- match the distroless image version to the project's own Java toolchain (`build.gradle.kts`/`.tool-versions`), and confirm the resulting tag actually exists before using it. Flag the tag used in the Step 4 summary as something the user may need to change. +- The base/runtime image (e.g. `java25-debian13`) is whatever the project's Dockerfile already uses -- do not change it. Only add the `otel` stage and edit the final stage's `ENTRYPOINT`. ## Step 2: Set the sampler explicitly @@ -74,7 +86,12 @@ Only create the single env file this way -- do not scaffold `helm//Chart.ya ## Step 3: Correlate logs with traces -For Java/Kotlin Spring Boot services, no manual work is needed. If `entur/cloud-logging` (v7.1.0) together with `spring-boot-starter-gcp-web` is correctly set up, those two handle all of this automatically -- the library injects the fields via Micrometer Tracing, and standard SLF4J logging picks them up on every log line within a traced request. +For Java/Kotlin Spring Boot services, no manual work is needed -- **provided the project is on `entur/cloud-logging` v7.1.0 or later**. Only 7.1.0+ injects the trace/span fields via Micrometer Tracing; earlier versions do not support this correlation. Together with `spring-boot-starter-gcp-web`, that's all that's required for standard SLF4J logging to pick up the trace/span ID on every log line within a traced request. + +Check the `entur/cloud-logging` version pinned in `build.gradle.kts` (or the version catalog): + +- **v7.1.0 or later already present**: nothing to do. +- **Older version, or not present at all**: log/trace correlation will not work until this is fixed, and it's outside what this skill edits. Tell the user their logs won't correlate with traces yet, and point them to [logging.md](../../guides/reference/logging.md) to add or upgrade `entur/cloud-logging` and `spring-boot-starter-gcp-web`. Note that `cloud-logging` 7.x requires Spring Boot 4.1.x -- if the project is on an older Spring Boot line, upgrading `cloud-logging` alone won't be enough; flag that dependency too. ## Step 4: Tell the user what's left @@ -85,7 +102,7 @@ Steps 1-3 are everything this skill can do by editing the repo. Verifying traces 3. Check **Monitoring → Trace → Trace Explorer** in the GCP Console under the shared host project `ent-kub-` (not the application project, traces always land in the host project). Since traces from multiple applications land in the same project, filter Trace Explorer by `service.name` to scope the view to just this service. 4. If no spans show up: trace storage provisions automatically the first time a span is successfully written to the project, and it isn't instant -- give it a few minutes before assuming something is broken. Also double check the sampler env var actually reached the deployed container for that environment (a common miss is setting it in the wrong Helm values file). -Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. Also state the distroless base image tag used and that it may need to change to match the project's Java toolchain. +Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth-extension (`1.58.0-alpha`) versions were pinned as-is, not checked against upstream for something newer, and are set in the Dockerfile's `otel` build stage if the user wants to bump them later. ## Critical Rules @@ -93,4 +110,4 @@ Also state in the summary: the OpenTelemetry Java agent (`v2.29.0`) and gcp-auth - **IAM roles and the required Google Cloud APIs are provisioned automatically** through the common Helm chart. Never add Terraform to enable `cloudtrace.googleapis.com`/`telemetry.googleapis.com` or to grant a trace-related IAM role for this. - **Trace storage auto-provisions** on first successful span write -- never tell the user to manually enable it in the console, and never script it. - **Defaults to the Java Agent.** Only hand-roll manual OpenTelemetry instrumentation if the user gives a specific reason. -- **One `CMD` per Dockerfile.** Merge Cloud Profiler flags into the same `CMD` as the tracing agent flags if present. +- **One JVM launch entrypoint per Dockerfile, and it's `ENTRYPOINT`, not `CMD`** -- Entur's Docker golden path launches the JVM via `ENTRYPOINT` (see [docker.md](../../guides/reference/docker.md)). If the project also uses [Cloud Profiler](../../guides/reference/profiler.md) (its own `-javaagent`/`-D` flags for the profiler agent), merge those into the same `ENTRYPOINT` array as the tracing flags rather than adding a second launch mechanism. diff --git a/tests/scenarios/30-tracing-dockerfile-single-cmd.md b/tests/scenarios/30-tracing-dockerfile-single-cmd.md deleted file mode 100644 index 38a482b..0000000 --- a/tests/scenarios/30-tracing-dockerfile-single-cmd.md +++ /dev/null @@ -1,45 +0,0 @@ -# Scenario: Tracing and Profiler Flags Merge into One CMD - -## Description - -Verifies the agent knows a Dockerfile only honors its last CMD instruction, so -adding the tracing Java Agent to a service that already runs Cloud Profiler must -merge flags into the existing CMD, not append a second one -- the common LLM -instinct when told to "add" a step to an existing file. - -## Prompt - -A service's Dockerfile already has this line to run Cloud Profiler: - -CMD ["-javaagent:/profiler/profiler_java_agent.so", "-jar", "/app/app.jar"] - -Now add OpenTelemetry Java Agent tracing per Entur's golden path. - -Read `skills/setup-tracing-java/SKILL.md` in this repository and answer in `key: value` -format on its own line: - -- number_of_cmd_instructions: -- both_agents_in_same_cmd: -- why: - -## Assertions - -```json -{ - "must_contain": [ - "number_of_cmd_instructions: 1", - "both_agents_in_same_cmd: yes" - ], - "must_not_contain": [ - "number_of_cmd_instructions: 2", - "both_agents_in_same_cmd: no" - ], - "must_match": [ - "last CMD|only.*(last|one) CMD|second CMD.*(disable|override|ignore)" - ] -} -``` - -## Budget - -0.08 diff --git a/tests/scenarios/30-tracing-dockerfile-single-entrypoint.md b/tests/scenarios/30-tracing-dockerfile-single-entrypoint.md new file mode 100644 index 0000000..cb11d90 --- /dev/null +++ b/tests/scenarios/30-tracing-dockerfile-single-entrypoint.md @@ -0,0 +1,46 @@ +# Scenario: Tracing and Profiler Flags Merge into One ENTRYPOINT + +## Description + +Verifies the agent knows a Dockerfile only honors its last ENTRYPOINT instruction, so +adding the tracing Java Agent to a service that already runs Cloud Profiler must +merge flags into the existing ENTRYPOINT, not append a second one or a CMD -- +the common LLM instinct when told to "add" a step to an existing file. + +## Prompt + +A service's Dockerfile already has this line to run Cloud Profiler: + +ENTRYPOINT ["java", "-javaagent:/profiler/profiler_java_agent.so", "-jar", "/app/app.jar"] + +Now add OpenTelemetry Java Agent tracing per Entur's golden path. + +Read `skills/setup-tracing-java/SKILL.md` in this repository and answer in `key: value` +format on its own line: + +- number_of_entrypoint_instructions: +- both_agents_in_same_entrypoint: +- why: + +## Assertions + +```json +{ + "must_contain": [ + "number_of_entrypoint_instructions: 1", + "both_agents_in_same_entrypoint: yes" + ], + "must_not_contain": [ + "number_of_entrypoint_instructions: 2", + "both_agents_in_same_entrypoint: no", + "CMD [" + ], + "must_match": [ + "last ENTRYPOINT|only.*(last|one) ENTRYPOINT|second ENTRYPOINT.*(disable|override|ignore)" + ] +} +``` + +## Budget + +0.08