Conversation
MarcoDN
marked this pull request as draft
September 11, 2026 15:42
jefchien
reviewed
Sep 16, 2026
Contributor
There was a problem hiding this comment.
This is almost entirely the same as https://github.com/aws/amazon-cloudwatch-agent-test/blob/main/test/azure/vm/payloads_test.go. Can we move it into a utility under test/otel_collect/otlpvalidation or something like that instead of replicating the entire file and update both the Azure VM and GCE tests to use it?
Contributor
There was a problem hiding this comment.
This is also practically identical to the AKS otlp_load_generator.sh aside from the aks/gke prefix. It already renders via templatefile(), so could we move that one to a shared location and add one prefix template variable for those? The same prefix is used for validation so consider using the lowercase computeType for the prefix for both generation and validation.
Comment on lines
+121
to
+284
| // validateLogs confirms the OTLP log record landed in the default:otel log group on the stream the | ||
| // agent's log routing is expected to derive for this host. | ||
| func validateLogs() status.TestResult { | ||
| testResult := status.TestResult{Name: "GCE_Logs", Status: status.FAILED} | ||
|
|
||
| // The agent routes OTLP logs to {host.id}/{service.name}, so assert that exact stream: it makes the | ||
| // check prove log routing rather than just delivery, and keeps cost flat as the shared group | ||
| // accumulates a stream per VM. Retries because the stream and events both lag. | ||
| logStream := fmt.Sprintf("%s/%s", env.InstanceId, serviceName) | ||
| // Clean up only on success: the group is shared by every VM run, so drop this run's stream but never | ||
| // the group. On failure the stream is left in place as evidence for whoever debugs the run. | ||
| defer func() { | ||
| if testResult.Status == status.SUCCESSFUL { | ||
| awsservice.DeleteLogStream(otlpLogGroup, logStream) | ||
| } | ||
| }() | ||
| marker := fmt.Sprintf("gce_otlp_log_%s", env.InstanceId) | ||
| const maxRetries = 4 | ||
| const retryInterval = 30 * time.Second | ||
| for attempt := 1; attempt <= maxRetries; attempt++ { | ||
| since := time.Now().Add(-loadWindow - time.Minute) | ||
| until := time.Now() | ||
| log.Printf("[GCE_Logs] attempt %d: checking %s/%s", attempt, otlpLogGroup, logStream) | ||
| err := awsservice.ValidateLogs( | ||
| otlpLogGroup, logStream, &since, &until, | ||
| awsservice.AssertLogsNotEmpty(), | ||
| awsservice.AssertPerLog(awsservice.AssertLogContainsSubstring(marker)), | ||
| ) | ||
| if err == nil { | ||
| testResult.Status = status.SUCCESSFUL | ||
| return testResult | ||
| } | ||
| testResult.Reason = err | ||
| if attempt < maxRetries { | ||
| log.Printf("[GCE_Logs] %v — retrying in %v", testResult.Reason, retryInterval) | ||
| time.Sleep(retryInterval) | ||
| } | ||
| } | ||
| return testResult | ||
| } | ||
|
|
||
| // validateTraces confirms every OTLP span emitted during the load window reached AWS through the | ||
| // X-Ray OTLP endpoint. That endpoint requires Transaction Search (trace segment destination = | ||
| // CloudWatchLogs), which stores 100% of ingested spans in the aws/spans log group; the X-Ray query | ||
| // APIs (GetTraceSummaries/BatchGetTraces) only see the indexed subset (1% by default), so aws/spans | ||
| // is the authoritative surface for OTLP trace delivery. Ingestion lags a few minutes, hence retries. | ||
| func validateTraces(traceIDs []string) status.TestResult { | ||
| testResult := status.TestResult{Name: "GCE_Traces", Status: status.FAILED} | ||
|
|
||
| if len(traceIDs) == 0 { | ||
| testResult.Reason = fmt.Errorf("no trace IDs were generated during the load window") | ||
| return testResult | ||
| } | ||
|
|
||
| quoted := make([]string, len(traceIDs)) | ||
| for i, id := range traceIDs { | ||
| quoted[i] = fmt.Sprintf("%q", id) | ||
| } | ||
| query := fmt.Sprintf("fields traceId | filter traceId in [%s] | dedup traceId", strings.Join(quoted, ", ")) | ||
| log.Printf("[GCE_Traces] expecting %d trace IDs in %s (sample: %s)", len(traceIDs), spansLogGroup, traceIDs[0]) | ||
|
|
||
| const maxRetries = 5 | ||
| const retryInterval = 60 * time.Second | ||
| for attempt := 1; attempt <= maxRetries; attempt++ { | ||
| since := time.Now().Add(-loadWindow - 10*time.Minute) | ||
| rows, err := awsservice.GetLogQueryResults(spansLogGroup, since.Unix(), time.Now().Unix(), query) | ||
| if err != nil { | ||
| testResult.Reason = fmt.Errorf("attempt %d: %s query failed (is Transaction Search enabled in the account?): %w", | ||
| attempt, spansLogGroup, err) | ||
| } else { | ||
| found := make(map[string]bool, len(rows)) | ||
| for _, row := range rows { | ||
| for _, field := range row { | ||
| if aws.ToString(field.Field) == "traceId" { | ||
| found[aws.ToString(field.Value)] = true | ||
| } | ||
| } | ||
| } | ||
| var missing []string | ||
| for _, id := range traceIDs { | ||
| if !found[id] { | ||
| missing = append(missing, id) | ||
| } | ||
| } | ||
| if len(missing) == 0 { | ||
| log.Printf("[GCE_Traces] attempt %d: all %d traces found in %s", attempt, len(traceIDs), spansLogGroup) | ||
| testResult.Status = status.SUCCESSFUL | ||
| return testResult | ||
| } | ||
| testResult.Reason = fmt.Errorf("attempt %d: %d/%d traces missing from %s (first missing: %s)", | ||
| attempt, len(missing), len(traceIDs), spansLogGroup, missing[0]) | ||
| } | ||
| if attempt < maxRetries { | ||
| log.Printf("[GCE_Traces] %v — retrying in %v", testResult.Reason, retryInterval) | ||
| time.Sleep(retryInterval) | ||
| } | ||
| } | ||
| return testResult | ||
| } | ||
|
|
||
| // sendTelemetry pushes OTLP metrics, logs, and traces to the local collector until stop is closed. | ||
| func sendTelemetry(stop <-chan struct{}) { | ||
| ticker := time.NewTicker(10 * time.Second) | ||
| defer ticker.Stop() | ||
| for { | ||
| select { | ||
| case <-stop: | ||
| return | ||
| case <-ticker.C: | ||
| post("/v1/metrics", buildMetricsPayload(env.InstanceId)) | ||
| post("/v1/logs", buildLogsPayload(env.InstanceId)) | ||
| // Only record the trace ID once the collector has accepted the span. Recording it | ||
| // unconditionally would make a single transient POST failure guarantee a validation | ||
| // failure for a trace that was never actually sent. | ||
| payload, traceID := buildTracesPayload(env.InstanceId) | ||
| if post("/v1/traces", payload) { | ||
| recordTraceID(traceID) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // post sends an OTLP payload and reports whether the collector accepted it. | ||
| func post(path string, payload []byte) bool { | ||
| req, err := http.NewRequest("POST", otlpEndpoint+path, bytes.NewReader(payload)) | ||
| if err != nil { | ||
| log.Printf("failed to build OTLP request for %s: %v", path, err) | ||
| return false | ||
| } | ||
| req.Header.Set("Content-Type", "application/json") | ||
| resp, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| log.Printf("failed to POST OTLP to %s: %v", path, err) | ||
| return false | ||
| } | ||
| // Drain before closing so the connection can be reused. | ||
| defer func() { | ||
| _, _ = io.Copy(io.Discard, resp.Body) | ||
| resp.Body.Close() | ||
| }() | ||
| if resp.StatusCode < 200 || resp.StatusCode > 299 { | ||
| log.Printf("OTLP POST to %s returned %s", path, resp.Status) | ||
| return false | ||
| } | ||
| return true | ||
| } | ||
|
|
||
| // filterLogLines returns lines from a multi-line string that contain any of the given substrings (case-insensitive). | ||
| func filterLogLines(text string, substrs ...string) []string { | ||
| var result []string | ||
| for _, line := range strings.Split(text, "\n") { | ||
| lower := strings.ToLower(line) | ||
| for _, s := range substrs { | ||
| if strings.Contains(lower, strings.ToLower(s)) { | ||
| result = append(result, line) | ||
| break | ||
| } | ||
| } | ||
| } | ||
| if len(result) > 50 { | ||
| result = result[len(result)-50:] | ||
| } | ||
| return result | ||
| } |
Contributor
There was a problem hiding this comment.
These functions are the same as the Azure VM test. Consider extracting to a shared utility package.
| # translated YAML. set-env runs before fetch-config so both are set on the first agent start. | ||
| provisioner "remote-exec" { | ||
| inline = [ | ||
| "export PATH=$PATH:/usr/local/go/bin", |
Contributor
There was a problem hiding this comment.
nit: Is this necessary? I see we're installing go via apt-get, so won't it be in /usr/bin instead?
Comment on lines
+23
to
+24
| An existing VPC network and subnetwork must be passed as `gcp_network_name` / | ||
| `gcp_subnetwork_name`; the auto-created `default` network works. |
Contributor
There was a problem hiding this comment.
nit: Seems like we'll want to avoid using the default network if we want to lock it down to only the runner_ip since there are some pre-populated firewall rules that will allow any IP (default-allow-ssh).
https://docs.cloud.google.com/firewall/docs/firewalls#more_rules_default_vpc
The terraform/gcp/gce and terraform/gcp/gke modules provision a GCE instance and a zonal GKE cluster, federate a per-run IAM role to AWS via web identity, install the agent with the default OTel config, generate OTLP load, and run the Go suites validating metrics, logs, and traces in CloudWatch. Adds the GCE and GKE compute types and a README covering the one-time GCP project and AWS account setup.
MarcoDN
force-pushed
the
gcp-integration-tests
branch
from
September 17, 2026 10:21
f69c7ac to
3bfa707
Compare
MarcoDN
marked this pull request as ready for review
September 17, 2026 13:00
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Description of the issue
The agent supports running on GCP hosts (GCE and GKE) with the default OTel config, authenticating to AWS via web-identity federation — but there is no integration test coverage proving the credential chains and end-to-end telemetry delivery (metrics, logs, traces) to CloudWatch from either environment.
Description of changes
Adds two new test suites, each provisioned by terraform (companion workflow jobs to follow in aws/amazon-cloudwatch-agent):
GCE (
terraform/gcp/gce,test/gcp/gce).debover SSH, and starts the agent viaamazon-cloudwatch-agent-ctl -a set-env+-a fetch-config -m auto -s -c default:otel.AssumeRoleWithWebIdentity). Google is a built-in AWS web-identity provider, so the per-run role trustsaccounts.google.comdirectly — no IAM OIDC provider resource — pinning all three Google condition keys::audand:sub(the service account's unique ID) and:oaud(the requested audience).{host.id}/{service.name}stream, so routing is covered rather than just delivery), and traces (the exact generated trace IDs inaws/spansvia Logs Insights).GKE (
terraform/gcp/gke,test/gcp/gke)container.googleapis.com/v1/.../clusters/...path that the terraform resource does not export as an attribute.sts.amazonaws.com); a load-generator Job pushes OTLP metrics/logs/traces to the agent over hostNetwork.test_iddatapoint attribute and assertscloud.platform=gcp_kubernetes_engine, so it isolates the run and covers resource detection rather than echoing back an injected attribute.GCEandGKEcompute types inenvironment/computetype, adds-gkeClusterNameto the environment metadata, and addsterraform/gcp/README.mddocumenting the one-time project/account setup (APIs, CI service-account roles, CI authentication, GitHub configuration, Transaction Search).License
By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
Tests
Both modules verified with complete local
terraform applyruns — the apply runs the suite via provisioners, so its exit code is the test verdict:TestGCEPASS in 362.5s — Metrics 0.55s / Logs 0.67s / Traces 151.27s, all generated trace IDs found inaws/spanson the first attempt.TestGKEMetrics/Logs/Traces all PASS on the first complete run of the module.gofmtandgo vet -tags integrationclean on the new packages;terraform validateandterraform fmt -checkclean on both stacks.Worth noting
us-east-2is load-bearing, not arbitrary.aws/spansis only populated where the X-Ray trace segment destination is CloudWatch Logs, which is a per-region setting; us-west-2 deliberately keeps theXRaydestination for the App Signals suite that validates through the X-Ray query APIs.CWAGENT_ROLE_ARN(the translator only wires the web-identity token file whenrole_arnis non-empty) and carries an inline policy of validation reads because its test runs on the instance under that role. GKE omitsCWAGENT_ROLE_ARNso sigv4auth falls through to the default credential chain and the projected token, and its role has no inline policy because the test runs on the runner under the runner's credentials.accounts.google.comprincipal with per-run conditions; GKE registers a per-run IAM OIDC provider for the cluster issuer.