diff --git a/README.md b/README.md index eee8b33..472a267 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ The hostname/SNI targeting is what makes **one** tool work for both internal - **Inspected path:** when a rule targets by hostname, the proxy reads only the first TLS record to extract the SNI (cleartext — **no MITM, no certificates**), replays those bytes to the upstream, then splices the remainder. +- **Interception path (opt-in):** only when a CA is supplied via + `--tls-ca-cert`/`--tls-ca-key` *and* a matching rule carries `httpStatus`, an + HTTPS connection is terminated so the response can be synthesized. See + [HTTPS response injection](#https-response-injection). Without a CA the proxy + never decrypts anything. ## Fault rules @@ -82,9 +87,79 @@ selectors match — an empty selector means "any". - `hosts` — match the TLS SNI, exact or subdomain (external targeting). - `latency` — Go duration string, added before the upstream connect. - `abort` — reset (RST) the connection. -- `httpStatus` — synthesize this HTTP status (L7, cleartext HTTP). +- `httpStatus` — synthesize this HTTP status (L7). Cleartext HTTP always; HTTPS + only with an interception CA (see below). - `probability` — `[0,1]` chance to apply the fault per connection (`0`/unset = always). +## HTTPS response injection + +By default the proxy never decrypts TLS: it reads the SNI in cleartext and +splices the bytes through. Supplying a CA opts in to terminating **matched** +HTTPS connections so an `httpStatus` fault can be synthesized inside TLS: + +```bash +transparent-proxy \ + --tls-ca-cert /etc/steadybit/intercept-ca.crt \ + --tls-ca-key /etc/steadybit/intercept-ca.key \ + --fault-hosts api.stripe.com --fault-http-status 503 +``` + +`--tls-ca-stdin` reads the same CA as **one PEM stream on stdin** (certificate +and key, either order) instead of from files. This is what an orchestrator +should use: it keeps the key off the command line, off any disk the target could +reach, and it is the only channel that works when the proxy runs inside an +overlay of the orchestrator's filesystem — an overlay does not carry the +orchestrator's submounts, so a key mounted there (a Kubernetes Secret, say) is +not visible by path. + +```bash +cat intercept-ca.crt intercept-ca.key | + transparent-proxy --tls-ca-stdin \ + --fault-hosts api.stripe.com --fault-http-status 503 +``` + +**The caller must close stdin.** The read is capped at 1 MiB and bounded by a +30s deadline, so a writer that never closes fails loudly rather than hanging the +proxy before it installs any rules. The key must not be passphrase-protected. + +The proxy mints a short-lived certificate for the connection's SNI, signed by +that CA, and answers the request itself. **HTTP/1.1 and HTTP/2 are both +supported** — the response is delivered over whichever the client negotiates +via ALPN. + +**The CA is yours, and it need not be a root.** An intermediate issued by your +own PKI works and is the better choice: you keep the root key offline, the +workloads already trust the root, and the proxy presents the intermediate so +the chain still builds. Constrain it further with `nameConstraints` if you want +it usable only for the dependencies under test. + +You generate it, choose how long it lives, and install the trust anchor in the +truststores of the workloads you want to fault. The proxy only signs with +it; it never creates, rotates, or renews a CA. A CA already outside its validity +window is rejected at startup rather than failing every handshake later. + +This is deliberately **one-sided**: the real dependency is never contacted. The +proxy makes no trust decision about the origin's certificate, and a dependency +behind mutual TLS is unaffected. The trade-off is that the response is +fabricated rather than a modified real one. + +**When it does not apply** — the connection is spliced through untouched: + +- no CA configured, or the client sent no SNI; +- the rule carries no `httpStatus`; +- the connection lost the `probability` roll. + +**When the client refuses** — if the workload does not trust the CA (or pins +certificates) it either fails the handshake, or, under TLS 1.3, completes it and +then walks away without sending a request. Both are counted as +`tls_intercept_rejected` and deliberately *not* as a fault, so a non-zero value +is the signal that the CA is missing from the target's truststore rather than a +silent no-op. Only a response actually written counts as an injected fault. + +> Interception requires a key that can impersonate any HTTPS endpoint to +> anything trusting the CA. Treat it as a test/staging capability and keep the +> key restricted. + ## Build & test ```bash diff --git a/internal/fault/fault.go b/internal/fault/fault.go index 7f71aed..8dba25c 100644 --- a/internal/fault/fault.go +++ b/internal/fault/fault.go @@ -43,9 +43,13 @@ type Rule struct { Abort bool // HTTPStatus, if non-zero, makes the proxy synthesize an HTTP response with - // this status code instead of forwarding — an L7 fault that applies only to - // cleartext HTTP (it is ignored on TLS/opaque connections). Selected by the - // Host header, matched with the same semantics as Hosts. + // this status code instead of forwarding — an L7 fault selected by the Host + // header or TLS SNI, matched with the same semantics as Hosts. + // + // It applies to cleartext HTTP always, and to HTTPS only when the proxy was + // given an interception CA (--tls-ca-cert/--tls-ca-key): the connection is + // then terminated with a certificate minted for its SNI. Without a CA, TLS + // connections are spliced through untouched. Opaque L4 is never affected. HTTPStatus int // HTTPBody, if set, replaces the default synthesized response body. diff --git a/internal/interception/rules.go b/internal/interception/rules.go index 4087643..c871115 100644 --- a/internal/interception/rules.go +++ b/internal/interception/rules.go @@ -50,6 +50,21 @@ type Config struct { // reset so they re-establish through the proxy and immediately feel the // fault; set it when only new connections should be affected. SkipFlush bool + // FlushDestinations narrows the connection-pool flush to these destinations + // instead of the whole capture filter. + // + // The capture filter is deliberately broad — typically 0.0.0.0/0 on ports + // 80 and 443 — because the proxy decides what to fault by hostname once it + // has seen the request. The flush cannot do that: it is a stateless iptables + // REJECT and knows only addresses. Applying it to the capture filter would + // therefore reset every established HTTP/HTTPS connection in the target, + // including ones to dependencies the attack never names. Resolving the + // targeted hostnames up front and flushing only those addresses keeps the + // collateral to the dependency actually under test. + // + // Empty keeps the old behaviour and flushes the whole capture filter, which + // is what a CIDR-targeted attack (no hostnames) actually wants. + FlushDestinations []netip.Prefix } func (c Config) mark() uint32 { @@ -129,7 +144,11 @@ func (c Config) AddScript() []string { for _, ex := range excludes { s = append(s, fmt.Sprintf("-A %s -d %s -j RETURN", flush, ex)) } - for _, in := range includes { + flushDsts := includes + if len(c.FlushDestinations) > 0 { + flushDsts = includeV4(c.FlushDestinations) + } + for _, in := range flushDsts { for _, p := range c.Filter.Ports { s = append(s, fmt.Sprintf("-A %s -p tcp -d %s --dport %d -m conntrack --ctstate ESTABLISHED -j REJECT --reject-with tcp-reset", flush, in, p)) } diff --git a/internal/interception/rules_test.go b/internal/interception/rules_test.go index 6e00d06..9703055 100644 --- a/internal/interception/rules_test.go +++ b/internal/interception/rules_test.go @@ -269,3 +269,53 @@ func TestRevert_FailsVerificationWhenRulesRemain(t *testing.T) { t.Fatal("Revert must fail verification while a chain still contains rules") } } + +// The capture filter is deliberately broad (0.0.0.0/0 on 80/443) because the +// proxy picks its victims by hostname. The flush cannot — it is a stateless +// REJECT that knows only addresses — so scoping it to the capture filter would +// reset every established HTTP/HTTPS connection in the target, not just the +// dependency under test. +func Test_flushIsScopedToResolvedDestinations(t *testing.T) { + base := Config{ + ExecutionID: "exec", + ProxyPort: 3128, + Filter: Filter{ + Include: []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, + Ports: []uint16{80, 443}, + }, + } + + // Without resolved destinations the flush covers the whole capture filter — + // what a CIDR-targeted attack wants. + broad := strings.Join(base.AddScript(), "\n") + if !strings.Contains(broad, "-d 0.0.0.0/0 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT") { + t.Fatalf("expected a filter-wide flush without resolved hosts:\n%s", broad) + } + + scoped := base + scoped.FlushDestinations = []netip.Prefix{ + netip.MustParsePrefix("93.184.216.34/32"), + netip.MustParsePrefix("1.2.3.4/32"), + } + got := strings.Join(scoped.AddScript(), "\n") + + for _, want := range []string{ + "-d 93.184.216.34/32 --dport 80 -m conntrack --ctstate ESTABLISHED -j REJECT", + "-d 93.184.216.34/32 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT", + "-d 1.2.3.4/32 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT", + } { + if !strings.Contains(got, want) { + t.Fatalf("missing scoped flush rule %q in:\n%s", want, got) + } + } + // The whole point: nothing else on those ports is reset. + if strings.Contains(got, "-d 0.0.0.0/0 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT") { + t.Fatalf("flush still resets the entire capture filter:\n%s", got) + } + // Capture itself must stay broad — the proxy still needs to see everything + // so it can match by hostname. + if !strings.Contains(got, "-d 0.0.0.0/0 -p tcp -m tcp --dport 443 -j REDIRECT") && + !strings.Contains(got, "0.0.0.0/0") { + t.Fatalf("capture filter was narrowed too:\n%s", got) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0a074d3..612b71d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -26,6 +26,7 @@ type Metrics struct { ConnectionsFaulted atomic.Int64 // connections a fault was actually applied to (once each) LatencyApplied atomic.Int64 // connections a latency fault delayed HTTPResponsesInjected atomic.Int64 // connections given a synthesized HTTP response + TLSInterceptRejected atomic.Int64 // HTTPS interception rejected by the client (CA not trusted / pinning) UpstreamErrors atomic.Int64 // dial failures BytesToUpstream atomic.Int64 BytesToClient atomic.Int64 @@ -63,6 +64,7 @@ type Snapshot struct { ConnectionsFaulted int64 `json:"connections_faulted"` LatencyApplied int64 `json:"latency_applied"` HTTPResponsesInjected int64 `json:"http_responses_injected"` + TLSInterceptRejected int64 `json:"tls_intercept_rejected"` UpstreamErrors int64 `json:"upstream_errors"` BytesToUpstream int64 `json:"bytes_to_upstream"` BytesToClient int64 `json:"bytes_to_client"` @@ -93,6 +95,7 @@ func (m *Metrics) Snapshot() Snapshot { ConnectionsFaulted: m.ConnectionsFaulted.Load(), LatencyApplied: m.LatencyApplied.Load(), HTTPResponsesInjected: m.HTTPResponsesInjected.Load(), + TLSInterceptRejected: m.TLSInterceptRejected.Load(), UpstreamErrors: m.UpstreamErrors.Load(), BytesToUpstream: m.BytesToUpstream.Load(), BytesToClient: m.BytesToClient.Load(), @@ -177,6 +180,19 @@ func (m *Metrics) HTTPInjected() { } } +// TLSRejected records an HTTPS connection on which the client refused the +// injected certificate — either by failing the handshake, or (under TLS 1.3, +// where the server's handshake completes before the client's verdict arrives) +// by abandoning the connection without ever sending a request. A non-zero count +// is the canonical "our CA is not trusted by the target, or the client pins +// certificates" signal. In both cases no response was delivered, so it is +// deliberately not counted as faulted. +func (m *Metrics) TLSRejected() { + if m != nil { + m.TLSInterceptRejected.Add(1) + } +} + // MatchedHost records that a connection carrying the given dependency hostname // matched a rule. FaultedHost records that a fault was actually applied to it // (i.e. it passed the probability roll). Empty hosts are ignored. diff --git a/internal/proxy/http.go b/internal/proxy/http.go index 5d10a9e..732daef 100644 --- a/internal/proxy/http.go +++ b/internal/proxy/http.go @@ -133,20 +133,32 @@ func isHTTPMethodStart(b byte) bool { // Content-Type overrides the default); Content-Length and Connection are always // set by the proxy so they stay correct and the connection closes cleanly. func writeHTTPResponse(c net.Conn, status int, headers map[string]string, body string) error { + // Normalised the same way as the HTTPS path, so one rule produces the same + // response whether the matched dependency happened to be HTTP or HTTPS. 1xx + // is informational and cannot carry a fault; 204/304 must not carry a body. + if status < 200 || status > 599 { + status = http.StatusServiceUnavailable + } + noBody := status == http.StatusNoContent || status == http.StatusNotModified reason := http.StatusText(status) if reason == "" { reason = "Fault Injected" } - if body == "" { + if body == "" && !noBody { body = fmt.Sprintf("%d %s (injected by steadybit transparent-proxy)\n", status, reason) } + if noBody { + body = "" + } h := map[string]string{"Content-Type": "text/plain; charset=utf-8"} for k, v := range headers { h[textproto.CanonicalMIMEHeaderKey(k)] = v } // Proxy-owned headers: keep the framing correct regardless of caller input. - h["Content-Length"] = strconv.Itoa(len(body)) + if !noBody { + h["Content-Length"] = strconv.Itoa(len(body)) + } h["Connection"] = "close" var sb strings.Builder diff --git a/internal/proxy/http_test.go b/internal/proxy/http_test.go index 70077d1..2aba52e 100644 --- a/internal/proxy/http_test.go +++ b/internal/proxy/http_test.go @@ -49,9 +49,9 @@ func Test_writeHTTPResponse_defaults(t *testing.T) { func Test_writeHTTPResponse_customBodyAndHeaders(t *testing.T) { body := `{"error":"nope"}` resp := readSynthesized(t, 429, map[string]string{ - "content-type": "application/json", // lower-case, should be canonicalized + override default - "Retry-After": "30", - "X-Fault": "injected", + "content-type": "application/json", // lower-case, should be canonicalized + override default + "Retry-After": "30", + "X-Fault": "injected", }, body) defer resp.Body.Close() diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 77de879..a0af6f8 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -10,6 +10,7 @@ package proxy import ( "context" + "errors" "log/slog" "net" "net/netip" @@ -18,6 +19,7 @@ import ( "github.com/steadybit/transparent-proxy/internal/fault" "github.com/steadybit/transparent-proxy/internal/metrics" + "github.com/steadybit/transparent-proxy/internal/tlsinject" ) const keepAlivePeriod = 30 * time.Second @@ -63,6 +65,13 @@ type Server struct { // ConnectionsMatched under load is the canonical silent-no-op signal. Metrics *metrics.Metrics + // TLSInject, when non-nil, enables HTTPS response injection: a matched TLS + // connection carrying an L7 status fault is terminated with a certificate + // minted for its SNI, and the synthesized response is written inside TLS. + // Nil (the default) means TLS is never decrypted — HTTPS connections are + // spliced through untouched exactly as before. + TLSInject *tlsinject.CA + // listenPort and localAddrs are captured at Serve time for the self-loop // guard, so a redirected flow resolving back to this proxy (on loopback or // any local interface address) is refused rather than dialed in a storm. @@ -249,15 +258,68 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { } // L7: synthesize an HTTP status response without contacting the upstream. - // Only valid for cleartext HTTP; ignored otherwise. - if proto == protoHTTP && action.HTTPStatus != 0 { - markFaulted() - if err := writeHTTPResponse(client, action.HTTPStatus, action.HTTPHeaders, action.HTTPBody); err != nil { - log.Debug("failed to write injected status", slog.Any("err", err)) + // Cleartext HTTP is written directly. HTTPS is only decrypted when a CA is + // configured and the client offered an SNI to mint a certificate for; + // otherwise the connection falls through and is spliced untouched, which is + // the pre-CA behaviour. + if action.HTTPStatus != 0 { + switch { + case proto == protoHTTP: + markFaulted() + if err := writeHTTPResponse(client, action.HTTPStatus, action.HTTPHeaders, action.HTTPBody); err != nil { + log.Debug("failed to write injected status", slog.Any("err", err)) + } + s.Metrics.HTTPInjected() + log.Info("injected http status", slog.Int("status", action.HTTPStatus)) + return + + case proto == protoTLS && s.TLSInject != nil && identity != "": + // Counted from the delivery callback, not after ServeForged returns: an + // HTTP/2 client keeps the connection pooled for the whole attack, so + // counting on return would report a working fault as "matched but never + // faulted" — the proxy's own silent-no-op signature. + err := s.TLSInject.ServeForged(ctx, client, prefix, tlsinject.Request{ + Response: tlsinject.Response{ + Status: action.HTTPStatus, + Body: action.HTTPBody, + Headers: action.HTTPHeaders, + }, + HandshakeTimeout: s.peekTimeout(), + OnDelivered: func() { + markFaulted() + s.Metrics.HTTPInjected() + log.Info("injected https status", slog.Int("status", action.HTTPStatus)) + }, + }) + + var rejErr *tlsinject.RejectedError + if errors.As(err, &rejErr) { + // The client rejected our certificate, so the fault never applied — + // counted separately from faults, never as one. This is the signal + // that the CA is missing from the workload's truststore. + s.Metrics.TLSRejected() + log.Warn("client rejected the injected certificate; is the CA trusted by the target?", + slog.String("stage", rejErr.Stage), + slog.Any("err", err)) + return + } + if err != nil { + // The connection was taken over and cannot be forwarded now, so it + // ends here either way. Count it so matched still reconciles with the + // outcome counters instead of silently losing a connection. + s.Metrics.Dropped() + if errors.Is(err, tlsinject.ErrNotDelivered) { + // Teardown while the connection was open: expected, not a failure. + log.Debug("interception ended without delivering a response", slog.Any("err", err)) + } else { + log.Warn("failed to serve injected https response", slog.Any("err", err)) + } + return + } + // Success is reported by OnDelivered above, which fires when the + // response is written rather than when the connection ends. + return } - s.Metrics.HTTPInjected() - log.Info("injected http status", slog.Int("status", action.HTTPStatus)) - return } s.forward(ctx, log, client, dst, prefix) diff --git a/internal/proxy/server_tlsinject_test.go b/internal/proxy/server_tlsinject_test.go new file mode 100644 index 0000000..665ed86 --- /dev/null +++ b/internal/proxy/server_tlsinject_test.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package proxy + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "testing" + "time" + + "github.com/steadybit/transparent-proxy/internal/fault" + "github.com/steadybit/transparent-proxy/internal/metrics" + "github.com/steadybit/transparent-proxy/internal/tlsinject" +) + +// upstreamHost matches the name in httptest's built-in certificate, so a +// pass-through connection validates against the real upstream too. +const upstreamHost = "example.com" + +func newInterceptCA(t *testing.T) (*tlsinject.CA, []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Test Intercept CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + ca, err := tlsinject.LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + return ca, certPEM +} + +// startTLSUpstream stands in for the real dependency. +func startTLSUpstream(t *testing.T) (*httptest.Server, netip.AddrPort) { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "real-response") + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse upstream url: %v", err) + } + ap, err := netip.ParseAddrPort(u.Host) + if err != nil { + t.Fatalf("parse upstream addr: %v", err) + } + return srv, ap +} + +// clientVia dials every request at the proxy while presenting upstreamHost as +// the SNI, so the proxy sees exactly what a redirected dependency call looks +// like. The pool trusts both the upstream and (when given) the intercept CA. +func clientVia(t *testing.T, proxyAddr netip.AddrPort, upstream *httptest.Server, interceptCAPEM []byte) *http.Client { + t.Helper() + pool := x509.NewCertPool() + pool.AddCert(upstream.Certificate()) + if len(interceptCAPEM) > 0 && !pool.AppendCertsFromPEM(interceptCAPEM) { + t.Fatal("failed to add intercept CA to pool") + } + tr := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", proxyAddr.String()) + }, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + } + t.Cleanup(tr.CloseIdleConnections) + return &http.Client{Transport: tr, Timeout: 10 * time.Second} +} + +func httpsRule() *fault.Engine { + p := 1.0 + return fault.NewEngine([]fault.Rule{{ + Name: "https-intercept", + Hosts: []string{upstreamHost}, + HTTPStatus: 503, + Probability: &p, + }}) +} + +// waitFor polls until cond holds, so assertions do not race the proxy +// goroutine finishing its bookkeeping. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", msg) +} + +// With a CA configured, an HTTPS dependency call is terminated and answered +// with the forged response instead of reaching the real upstream. +func TestServer_TLSInject_ForgesResponse(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, caPEM := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + client := clientVia(t, proxyAddr, upstream, caPEM) + + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + if strings.Contains(string(body), "real-response") { + t.Fatal("the request reached the real upstream; it should have been forged") + } + if !strings.Contains(string(body), "injected by steadybit") { + t.Fatalf("body = %q", body) + } + + waitFor(t, func() bool { return m.Snapshot().HTTPResponsesInjected == 1 }, "an injected response") + snap := m.Snapshot() + if snap.ConnectionsFaulted != 1 { + t.Fatalf("ConnectionsFaulted = %d, want 1", snap.ConnectionsFaulted) + } + if snap.TLSInterceptRejected != 0 { + t.Fatalf("TLSInterceptRejected = %d, want 0", snap.TLSInterceptRejected) + } + if got := snap.PerHost[upstreamHost]; got.Faulted != 1 { + t.Fatalf("per-host faulted = %d, want 1", got.Faulted) + } +} + +// Regression: HTTP/2 clients pool the connection for the whole attack. The +// fault counters must reflect the injection while that connection is still +// open — otherwise a working attack reports as "matched but never faulted", +// which the platform reads as a silent no-op. +func TestServer_TLSInject_CountsWhileHTTP2ConnectionStaysOpen(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, caPEM := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + + pool := x509.NewCertPool() + pool.AddCert(upstream.Certificate()) + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("failed to add intercept CA to pool") + } + tr := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", proxyAddr.String()) + }, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + } + defer tr.CloseIdleConnections() + + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays pooled", resp.Proto) + } + + // Deliberately do NOT close idle connections first — that is the bug. + waitFor(t, func() bool { return m.Snapshot().ConnectionsFaulted == 1 }, + "the fault to be counted while the h2 connection is still open") + snap := m.Snapshot() + if snap.HTTPResponsesInjected != 1 { + t.Fatalf("HTTPResponsesInjected = %d, want 1", snap.HTTPResponsesInjected) + } + if got := snap.PerHost[upstreamHost]; got.Faulted != 1 { + t.Fatalf("per-host faulted = %d, want 1", got.Faulted) + } +} + +// Without a CA the same rule must leave HTTPS alone — the pre-existing +// behaviour, and the guarantee that enabling the feature is opt-in. +func TestServer_TLSInject_DisabledPassesThrough(t *testing.T) { + upstream, dst := startTLSUpstream(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m}, dst) + client := clientVia(t, proxyAddr, upstream, nil) + + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK || string(body) != "real-response" { + t.Fatalf("got %d %q, want 200 real-response", resp.StatusCode, body) + } + if got := m.Snapshot().ConnectionsFaulted; got != 0 { + t.Fatalf("ConnectionsFaulted = %d, want 0 (no CA configured)", got) + } +} + +// A workload that does not trust the CA is the expected first failure. It must +// be counted as a handshake failure and never as an applied fault, so the +// operator gets a pointed diagnosis instead of a silent no-op. +func TestServer_TLSInject_UntrustedClientIsCounted(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, _ := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + // Trusts the upstream but not the intercept CA. + client := clientVia(t, proxyAddr, upstream, nil) + + if _, err := client.Get("https://" + upstreamHost + "/v1/messages"); err == nil { + t.Fatal("expected the client to reject the injected certificate") + } + + waitFor(t, func() bool { return m.Snapshot().TLSInterceptRejected == 1 }, "a counted handshake failure") + snap := m.Snapshot() + if snap.ConnectionsFaulted != 0 { + t.Fatalf("ConnectionsFaulted = %d, want 0 — the fault never applied", snap.ConnectionsFaulted) + } + if snap.HTTPResponsesInjected != 0 { + t.Fatalf("HTTPResponsesInjected = %d, want 0", snap.HTTPResponsesInjected) + } +} diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go new file mode 100644 index 0000000..31dd399 --- /dev/null +++ b/internal/tlsinject/ca.go @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +// Package tlsinject terminates TLS on an intercepted connection so an L7 fault +// can be synthesized for an HTTPS dependency. +// +// The proxy mints a short-lived leaf certificate for the connection's SNI, +// signed by a certificate authority the customer supplies. That CA is entirely +// theirs: they generate it, choose its validity, and install it in the +// truststores of the workloads they want to fault. The proxy only consumes the +// pair — it never creates, rotates, or expires a CA. With no CA configured the +// proxy never decrypts anything and HTTPS keeps flowing through untouched. +// +// Interception here is deliberately one-sided: the real dependency is never +// contacted. Nothing upstream is dialed, so the proxy makes no trust decision +// about the origin's certificate and a dependency behind mutual TLS is +// unaffected — the client-authenticated handshake to the origin simply never +// happens. The cost is that the response is fabricated rather than a modified +// real one. +package tlsinject + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "os" + "strings" + "sync" + "time" +) + +const ( + // defaultLeafValidity bounds a minted leaf's lifetime when the caller does + // not choose one. It is additionally clamped to the CA's own NotAfter, so a + // leaf never outlives its issuer. + defaultLeafValidity = 24 * time.Hour + // leafBackdate tolerates modest clock skew between the proxy and the client. + leafBackdate = 1 * time.Hour + // leafRenewBefore re-mints a cached leaf this long before it expires, so a + // long-running proxy never serves an expired certificate. + leafRenewBefore = 1 * time.Hour + // maxCachedLeaves bounds the per-SNI cache so traffic to a great many + // hostnames cannot grow it without limit. Beyond the cap certificates are + // still minted, just not retained. + maxCachedLeaves = 1024 +) + +// CertError marks a failure to produce a certificate at all — an expired CA, a +// signing failure, a ClientHello with no SNI. It is our side failing, not the +// client refusing us, and the two must not be conflated: reporting a dead CA as +// "the client rejected the certificate" sends the operator to inspect a +// truststore that is perfectly fine. +type CertError struct{ Err error } + +func (e *CertError) Error() string { return "could not mint a certificate: " + e.Err.Error() } +func (e *CertError) Unwrap() error { return e.Err } + +// CA mints per-SNI leaf certificates from a customer-supplied authority. +// It is safe for concurrent use. +type CA struct { + cert *x509.Certificate + key crypto.Signer + + // leafKey is generated once and shared by every minted leaf, so issuing a + // certificate for a new hostname costs one signature rather than a fresh + // keypair. It never leaves this process. + leafKey *ecdsa.PrivateKey + + // leafValidity is how long a minted leaf is valid for. Shorter values limit + // the window in which a leaf that escaped the proxy would be usable, at the + // cost of re-signing more often. + leafValidity time.Duration + + // chain is every certificate the operator supplied, signing certificate + // first. Presenting all of them lets an intermediate CA be used: the client + // trusts the root it already has, and we hand it the intermediates needed to + // build the path. Signing with an intermediate means the operator never has + // to part with a root key. + chain [][]byte + + mu sync.Mutex + cache map[string]*tls.Certificate +} + +// SetLeafValidity overrides how long minted leaves are valid. Values below the +// renew window are raised to it, since a leaf shorter than that would be +// considered stale the moment it is issued and re-signed on every handshake. +func (c *CA) SetLeafValidity(d time.Duration) { + if d <= 0 { + return + } + if d < 2*leafRenewBefore { + d = 2 * leafRenewBefore + } + c.mu.Lock() + c.leafValidity = d + // Existing leaves were cut to the old window; drop them so the new one applies. + c.cache = map[string]*tls.Certificate{} + c.mu.Unlock() +} + +// LoadCA parses a PEM certificate and matching private key. The certificate +// must be a signing CA; anything else is rejected up front rather than failing +// later on every handshake. +func LoadCA(certPEM, keyPEM []byte) (*CA, error) { + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("invalid CA keypair: %w", err) + } + if len(pair.Certificate) == 0 { + return nil, errors.New("CA certificate is empty") + } + cert, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unparseable CA certificate: %w", err) + } + if !cert.IsCA || cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, errors.New("certificate is not a signing CA: it needs basicConstraints CA:TRUE and keyUsage certSign") + } + signer, ok := pair.PrivateKey.(crypto.Signer) + if !ok { + return nil, errors.New("CA private key does not implement crypto.Signer") + } + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate leaf key: %w", err) + } + return &CA{ + cert: cert, + key: signer, + chain: pair.Certificate, + leafKey: leafKey, + leafValidity: defaultLeafValidity, + cache: map[string]*tls.Certificate{}, + }, nil +} + +// LoadCACombined parses one PEM stream carrying both the CA certificate and its +// private key, in any order. +// +// This is the form the proxy accepts on stdin, which is how an orchestrator +// hands over the key without writing it to a disk the target could reach or +// exposing it on the command line. It also sidesteps a filesystem asymmetry: +// the proxy may run inside an overlay of the orchestrator's root, and an +// overlay does not carry the orchestrator's submounts — so a key mounted there +// (a Kubernetes Secret, say) would simply not be visible by path. +func LoadCACombined(pemBytes []byte) (*CA, error) { + var certPEM, keyPEM []byte + rest := pemBytes + for { + var blk *pem.Block + blk, rest = pem.Decode(rest) + if blk == nil { + break + } + switch { + case blk.Type == "CERTIFICATE": + certPEM = append(certPEM, pem.EncodeToMemory(blk)...) + case strings.Contains(blk.Type, "ENCRYPTED PRIVATE KEY"): + // Accepting it here would fail later inside X509KeyPair with an opaque + // parse error, hiding the actual problem. + return nil, errors.New("the CA key is passphrase-protected; supply an unencrypted PRIVATE KEY block") + case strings.Contains(blk.Type, "PRIVATE KEY"): + keyPEM = append(keyPEM, pem.EncodeToMemory(blk)...) + } + } + if len(certPEM) == 0 { + return nil, errors.New("no CERTIFICATE block in the supplied PEM") + } + if len(keyPEM) == 0 { + return nil, errors.New("no PRIVATE KEY block in the supplied PEM") + } + return LoadCA(certPEM, keyPEM) +} + +// LoadCAFromFiles reads a PEM certificate and key from disk. +func LoadCAFromFiles(certPath, keyPath string) (*CA, error) { + certPEM, err := os.ReadFile(certPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate: %w", err) + } + keyPEM, err := os.ReadFile(keyPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA key: %w", err) + } + return LoadCA(certPEM, keyPEM) +} + +// NotAfter reports when the CA expires, for startup logging. +func (c *CA) NotAfter() time.Time { return c.cert.NotAfter } + +// Expired reports whether the CA is outside its validity window at now. The +// customer owns the CA's lifecycle; this exists only so an unusable CA is +// reported at startup instead of failing every handshake later. +func (c *CA) Expired(now time.Time) bool { + return now.Before(c.cert.NotBefore) || now.After(c.cert.NotAfter) +} + +// ServerTLSConfig returns a config that mints a certificate for whatever SNI +// the client asks for. Both h2 and http/1.1 are advertised so the forged +// response can be delivered over whichever the client negotiates. +func (c *CA) ServerTLSConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hello.ServerName == "" { + // Without SNI there is no name to impersonate, so the handshake is + // aborted. There is no falling back to forwarding at this point: the + // ClientHello has been consumed and TLS records already written. + // Callers avoid reaching here by only intercepting connections whose + // SNI they already read. + return nil, &CertError{Err: errors.New("client sent no SNI")} + } + cert, err := c.leafFor(hello.ServerName) + if err != nil { + return nil, &CertError{Err: err} + } + return cert, nil + }, + } +} + +// leafFor returns a cached certificate for host, minting one on first use and +// re-minting before the cached one expires. Without the expiry check a proxy +// outliving leafValidity would serve an expired certificate for every hostname +// it had ever seen, and every client would reject it — indistinguishable, from +// the operator's side, from the CA not being trusted. +func (c *CA) leafFor(host string) (*tls.Certificate, error) { + now := time.Now() + c.mu.Lock() + if cert, ok := c.cache[host]; ok && fresh(cert, now) { + c.mu.Unlock() + return cert, nil + } + c.mu.Unlock() + + cert, err := c.mint(host) + if err != nil { + return nil, err + } + + c.mu.Lock() + defer c.mu.Unlock() + // Another goroutine may have minted the same host concurrently; prefer the + // stored one so every caller for a host shares a single certificate — unless + // it is the stale one we set out to replace. + if existing, ok := c.cache[host]; ok && fresh(existing, now) { + return existing, nil + } + // Replacing an existing (stale) entry never grows the map, so the cap only + // gates genuinely new hostnames. + if _, replacing := c.cache[host]; replacing || len(c.cache) < maxCachedLeaves { + c.cache[host] = cert + } + return cert, nil +} + +// fresh reports whether a cached leaf is far enough from expiry to keep using. +// +// The renew window is clamped to half the remaining validity: leaves are capped +// at the CA's own NotAfter, so once the CA has less than leafRenewBefore left, +// a fixed window would mark every cached leaf permanently stale and re-sign on +// every single handshake. +func fresh(cert *tls.Certificate, now time.Time) bool { + renew := leafRenewBefore + if remaining := cert.Leaf.NotAfter.Sub(now); remaining/2 < renew { + renew = remaining / 2 + } + return now.Before(cert.Leaf.NotAfter.Add(-renew)) +} + +func (c *CA) mint(host string) (*tls.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("failed to generate serial: %w", err) + } + + now := time.Now() + c.mu.Lock() + validity := c.leafValidity + c.mu.Unlock() + if validity <= 0 { + validity = defaultLeafValidity + } + notAfter := now.Add(validity) + // A leaf must never outlive the CA that signed it. + if notAfter.After(c.cert.NotAfter) { + notAfter = c.cert.NotAfter + } + // The CA expiring mid-run is only caught here — the startup check cannot see + // it. Minting a certificate that is already expired would surface to the + // operator as "the client rejected us", pointing at the truststore instead of + // at the real cause. + if !notAfter.After(now) { + return nil, fmt.Errorf("CA expired at %s; cannot mint a certificate for %q", c.cert.NotAfter.Format(time.RFC3339), host) + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: now.Add(-leafBackdate), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + // Clients validate the SAN, not the CN. An IP literal in SNI is unusual but + // cheap to support correctly. + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, c.cert, c.leafKey.Public(), c.key) + if err != nil { + return nil, fmt.Errorf("failed to sign certificate for %q: %w", host, err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("failed to parse minted certificate: %w", err) + } + // leaf first, then everything the operator supplied, so a client holding + // only the root can still build the path through any intermediates. + chain := make([][]byte, 0, len(c.chain)+1) + chain = append(chain, der) + chain = append(chain, c.chain...) + + return &tls.Certificate{ + Certificate: chain, + PrivateKey: c.leafKey, + Leaf: leaf, + }, nil +} diff --git a/internal/tlsinject/ca_test.go b/internal/tlsinject/ca_test.go new file mode 100644 index 0000000..e2e5d50 --- /dev/null +++ b/internal/tlsinject/ca_test.go @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "strings" + "testing" + "time" +) + +// newTestCAPEM builds a self-signed authority for tests. isCA=false yields a +// leaf-shaped certificate, used to prove non-CA input is rejected. +func newTestCAPEM(t *testing.T, notAfter time.Time, isCA bool) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Test Intercept CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: notAfter, + BasicConstraintsValid: true, + } + if isCA { + tmpl.IsCA = true + tmpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign + } else { + tmpl.KeyUsage = x509.KeyUsageDigitalSignature + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +func mustLoadTestCA(t *testing.T) (*CA, []byte) { + t.Helper() + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(30*24*time.Hour), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + return ca, certPEM +} + +func Test_LoadCA_acceptsSigningCA(t *testing.T) { + ca, _ := mustLoadTestCA(t) + if ca.Expired(time.Now()) { + t.Fatal("freshly issued CA reported as expired") + } + if ca.NotAfter().Before(time.Now()) { + t.Fatal("NotAfter is in the past") + } +} + +func Test_LoadCA_rejectsNonCA(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(time.Hour), false) + _, err := LoadCA(certPEM, keyPEM) + if err == nil { + t.Fatal("expected a non-CA certificate to be rejected") + } + if !strings.Contains(err.Error(), "not a signing CA") { + t.Fatalf("unexpected error: %v", err) + } +} + +func Test_LoadCA_rejectsMismatchedKey(t *testing.T) { + certPEM, _ := newTestCAPEM(t, time.Now().Add(time.Hour), true) + _, otherKey := newTestCAPEM(t, time.Now().Add(time.Hour), true) + if _, err := LoadCA(certPEM, otherKey); err == nil { + t.Fatal("expected a mismatched key to be rejected") + } +} + +func Test_LoadCA_rejectsGarbage(t *testing.T) { + if _, err := LoadCA([]byte("not a pem"), []byte("neither")); err == nil { + t.Fatal("expected garbage input to be rejected") + } +} + +func Test_Expired(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(time.Hour), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + if !ca.Expired(time.Now().Add(2 * time.Hour)) { + t.Fatal("expected a CA to be expired past its NotAfter") + } + if !ca.Expired(time.Now().Add(-2 * time.Hour)) { + t.Fatal("expected a CA to be invalid before its NotBefore") + } +} + +func Test_leafFor_mintsForSNIAndCaches(t *testing.T) { + ca, _ := mustLoadTestCA(t) + + first, err := ca.leafFor("api.anthropic.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if got := first.Leaf.DNSNames; len(got) != 1 || got[0] != "api.anthropic.com" { + t.Fatalf("leaf SAN = %v, want [api.anthropic.com]", got) + } + // The leaf must actually chain to the CA a client would trust. + pool := x509.NewCertPool() + pool.AddCert(ca.cert) + if _, err := first.Leaf.Verify(x509.VerifyOptions{Roots: pool, DNSName: "api.anthropic.com", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + t.Fatalf("minted leaf does not verify against the CA: %v", err) + } + + second, err := ca.leafFor("api.anthropic.com") + if err != nil { + t.Fatalf("leafFor (cached): %v", err) + } + if first != second { + t.Fatal("expected the second call to return the cached certificate") + } + + other, err := ca.leafFor("example.com") + if err != nil { + t.Fatalf("leafFor (other host): %v", err) + } + if other == first { + t.Fatal("expected a distinct certificate per hostname") + } +} + +func Test_leafFor_usesIPSANForIPLiteral(t *testing.T) { + ca, _ := mustLoadTestCA(t) + cert, err := ca.leafFor("10.1.2.3") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if len(cert.Leaf.IPAddresses) != 1 || cert.Leaf.IPAddresses[0].String() != "10.1.2.3" { + t.Fatalf("IP SANs = %v, want [10.1.2.3]", cert.Leaf.IPAddresses) + } + if len(cert.Leaf.DNSNames) != 0 { + t.Fatalf("unexpected DNS SANs on an IP leaf: %v", cert.Leaf.DNSNames) + } +} + +// A leaf must never outlive the authority that signed it, however short-lived +// the customer chose to make their CA. +func Test_leafFor_clampsValidityToCA(t *testing.T) { + caNotAfter := time.Now().Add(time.Hour) // shorter than leafValidity + certPEM, keyPEM := newTestCAPEM(t, caNotAfter, true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + cert, err := ca.leafFor("short.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if cert.Leaf.NotAfter.After(ca.NotAfter()) { + t.Fatalf("leaf NotAfter %s outlives CA NotAfter %s", cert.Leaf.NotAfter, ca.NotAfter()) + } +} + +func Test_ServerTLSConfig_requiresSNI(t *testing.T) { + ca, _ := mustLoadTestCA(t) + cfg := ca.ServerTLSConfig() + + if _, err := cfg.GetCertificate(&tls.ClientHelloInfo{}); err == nil { + t.Fatal("expected a ClientHello without SNI to be refused") + } + if _, err := cfg.GetCertificate(&tls.ClientHelloInfo{ServerName: "api.anthropic.com"}); err != nil { + t.Fatalf("expected a certificate for a named host: %v", err) + } + + // Both protocols must be offered so the forged response can be delivered + // over whichever the client negotiates. + want := map[string]bool{"h2": false, "http/1.1": false} + for _, p := range cfg.NextProtos { + want[p] = true + } + for p, seen := range want { + if !seen { + t.Fatalf("ALPN does not advertise %q (got %v)", p, cfg.NextProtos) + } + } +} + +func Test_LoadCACombined(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(24*time.Hour), true) + + // Order must not matter: an orchestrator concatenates whichever way round. + for _, combined := range [][]byte{ + append(append([]byte{}, certPEM...), keyPEM...), + append(append([]byte{}, keyPEM...), certPEM...), + } { + ca, err := LoadCACombined(combined) + if err != nil { + t.Fatalf("LoadCACombined: %v", err) + } + if _, err := ca.leafFor("api.anthropic.com"); err != nil { + t.Fatalf("minting from a combined PEM failed: %v", err) + } + } + + // A stream missing either half is rejected with a pointed message rather + // than failing later on every handshake. + if _, err := LoadCACombined(certPEM); err == nil || + !strings.Contains(err.Error(), "PRIVATE KEY") { + t.Fatalf("expected a missing-key error, got %v", err) + } + if _, err := LoadCACombined(keyPEM); err == nil || + !strings.Contains(err.Error(), "CERTIFICATE") { + t.Fatalf("expected a missing-certificate error, got %v", err) + } + if _, err := LoadCACombined([]byte("not pem at all")); err == nil { + t.Fatal("expected garbage to be rejected") + } +} + +// A CA that expires mid-run must be reported as our failure, not as the client +// rejecting us — otherwise the operator is sent to inspect a truststore that is +// perfectly fine. This is the exact misdiagnosis the expiry check exists to +// prevent, so it must not be undone by the error classification above it. +func Test_mint_expiredCAIsCertErrorNotRejection(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(2*time.Second), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + // Force the CA past its NotAfter without waiting. + ca.cert.NotAfter = time.Now().Add(-time.Minute) + + _, err = ca.ServerTLSConfig().GetCertificate(&tls.ClientHelloInfo{ServerName: "api.example.com"}) + if err == nil { + t.Fatal("expected minting to fail once the CA has expired") + } + if !isCertError(err) { + t.Fatalf("err = %v, want a *CertError so it is not misreported as a client rejection", err) + } +} + +func Test_ServerTLSConfig_noSNIIsCertError(t *testing.T) { + ca, _ := mustLoadTestCA(t) + _, err := ca.ServerTLSConfig().GetCertificate(&tls.ClientHelloInfo{}) + if !isCertError(err) { + t.Fatalf("err = %v, want a *CertError", err) + } +} + +// With a fixed renew window, a CA with less than that window left would mark +// every cached leaf permanently stale — turning each handshake into a fresh +// signature under the mutex. +func Test_leafFor_cachesEvenWhenCANearExpiry(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(30*time.Minute), true) // < leafRenewBefore + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + first, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + second, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if first != second { + t.Fatal("cache thrashing: re-minted despite a still-valid leaf") + } +} + +func Test_LoadCACombined_rejectsEncryptedKey(t *testing.T) { + certPEM, _ := newTestCAPEM(t, time.Now().Add(time.Hour), true) + enc := pem.EncodeToMemory(&pem.Block{Type: "ENCRYPTED PRIVATE KEY", Bytes: []byte("nope")}) + _, err := LoadCACombined(append(append([]byte{}, certPEM...), enc...)) + if err == nil || !strings.Contains(err.Error(), "passphrase-protected") { + t.Fatalf("err = %v, want a pointed passphrase-protected message", err) + } +} + +// An operator must never have to hand over a root key. Signing with an +// intermediate issued by their own PKI has to work: the client trusts only the +// root it already has, and the proxy supplies the intermediate so the path can +// be built. +func Test_LoadCA_worksWithAnIntermediate(t *testing.T) { + // root (stays offline; only its certificate is trusted by the client) + rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + rootTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Customer Root CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(72 * time.Hour), + IsCA: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, + } + rootDER, err := x509.CreateCertificate(rand.Reader, rootTmpl, rootTmpl, rootKey.Public(), rootKey) + if err != nil { + t.Fatal(err) + } + rootCert, _ := x509.ParseCertificate(rootDER) + + // intermediate — this is what the operator hands the proxy + interKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + interTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "Steadybit Intercept Intermediate"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(48 * time.Hour), + IsCA: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, + } + interDER, err := x509.CreateCertificate(rand.Reader, interTmpl, rootCert, interKey.Public(), rootKey) + if err != nil { + t.Fatal(err) + } + interKeyDER, _ := x509.MarshalECPrivateKey(interKey) + + // The operator supplies intermediate (+ root) and the intermediate's key. + certPEM := append( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: interDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER})...) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: interKeyDER}) + + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("an intermediate CA must be accepted: %v", err) + } + cert, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + + // The client trusts ONLY the root; the chain we present must let it verify. + roots := x509.NewCertPool() + roots.AddCert(rootCert) + inters := x509.NewCertPool() + for _, der := range cert.Certificate[1:] { + c, perr := x509.ParseCertificate(der) + if perr != nil { + t.Fatal(perr) + } + inters.AddCert(c) + } + if _, err := cert.Leaf.Verify(x509.VerifyOptions{ + Roots: roots, Intermediates: inters, DNSName: "api.example.com", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Fatalf("a client trusting only the root could not verify the presented chain: %v", err) + } +} diff --git a/internal/tlsinject/serve.go b/internal/tlsinject/serve.go new file mode 100644 index 0000000..fccf8bd --- /dev/null +++ b/internal/tlsinject/serve.go @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/textproto" + "strconv" + "sync" + "sync/atomic" + "time" +) + +// readHeaderTimeout bounds how long a client may take to send its request head +// after the handshake, so a silent client cannot pin a goroutine indefinitely. +const readHeaderTimeout = 10 * time.Second + +// hopByHop headers are meaningful only to a single transport hop. They are +// stripped from caller-supplied headers because net/http owns framing, and +// because HTTP/2 rejects them outright. +var hopByHop = []string{ + "Connection", "Keep-Alive", "Proxy-Connection", "Transfer-Encoding", + "Te", "Trailer", "Upgrade", +} + +// Response is the forged HTTP response written back to the intercepted client. +// The real dependency is never contacted, so every byte here is synthesized. +type Response struct { + Status int + Body string + Headers map[string]string +} + +// RejectedError reports that the client refused the injected certificate — +// almost always because the CA is absent from the workload's truststore, or +// because the client pins certificates. +// +// It covers two stages, because refusal is not always visible at handshake +// time. Under TLS 1.2 the handshake itself fails. Under TLS 1.3 the server +// completes its handshake before learning the client's verdict, so a rejecting +// client (OpenSSL/curl among them) instead abandons the connection without ever +// sending a request. Both mean the fault was never delivered, so both are +// reported here rather than counted as a successful injection. +type RejectedError struct { + // Stage is "handshake" or "post-handshake". + Stage string + // Err is the underlying failure; nil when the client simply went away + // without sending a request. + Err error +} + +func (e *RejectedError) Error() string { + msg := "client rejected the injected certificate (" + e.Stage + ")" + if e.Err != nil { + return msg + ": " + e.Err.Error() + } + return msg + ": no request was sent" +} +func (e *RejectedError) Unwrap() error { return e.Err } + +// Request is one interception: what to answer with, and how to report it. +type Request struct { + Response Response + HandshakeTimeout time.Duration + // OnDelivered is invoked the moment the first forged response is written, + // at most once per connection. + // + // Delivery must be reported from here rather than inferred from ServeForged + // returning: an HTTP/2 client pools the connection for the whole attack, so + // waiting for the connection to end would leave a fault that is demonstrably + // in effect counted as "matched but never faulted" — the proxy's canonical + // silent-no-op signature. + OnDelivered func() +} + +// ServeForged terminates TLS on conn using a certificate minted for the +// client's SNI, then answers requests with req.Response. clientHello replays +// the bytes already consumed while sniffing the SNI, so the handshake sees the +// original stream; pass nil when nothing was consumed. +// +// It blocks until the connection is finished — for HTTP/1.1 that is one +// request, for HTTP/2 until the client goes away — or until ctx is cancelled, +// which closes the connection so an attack teardown never leaks a goroutine. +// +// A nil return means the connection ended without the client refusing us; it is +// not a claim that anything was delivered. Use OnDelivered for that. +func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, req Request) error { + tc := tls.Server(replayConn(conn, clientHello), c.ServerTLSConfig()) + + hctx, cancel := context.WithTimeout(ctx, req.HandshakeTimeout) + defer cancel() + if err := tc.HandshakeContext(hctx); err != nil { + switch { + case ctx.Err() != nil: + // Teardown closed the connection mid-handshake; not the client's doing, + // and must not be blamed on the truststore. + return nil + case errors.Is(err, context.DeadlineExceeded): + // Our own deadline elapsed. A slow client or a loaded proxy is not a + // rejection, and reporting it as one sends the operator hunting for a + // truststore problem that does not exist. + return fmt.Errorf("tls handshake did not complete within %s: %w", req.HandshakeTimeout, err) + case isCertError(err): + // We could not produce a certificate (expired CA, signing failure, no + // SNI). Blaming the client's truststore here would point the operator at + // the one thing that is not broken. + return err + default: + return &RejectedError{Stage: "handshake", Err: err} + } + } + + // Cancellation must reach a connection parked inside net/http; closing it is + // the only way to unblock the server loop. + stop := make(chan struct{}) + defer close(stop) + go func() { + select { + case <-ctx.Done(): + _ = tc.Close() + case <-stop: + } + }() + + // delivered is the only trustworthy proof the fault landed: a completed + // handshake is not one, because a TLS 1.3 client reports a certificate it + // dislikes only by walking away afterwards. + var delivered atomic.Bool + if err := serveOne(tc, req, &delivered); err != nil { + return err + } + // Claim the flag rather than reading it. An HTTP/2 handler goroutine can + // still be finishing as ServeConn returns, so a plain read could see "not + // delivered", have the caller count a rejection, and then have the straggler + // fire OnDelivered — booking one connection into two mutually exclusive + // buckets. Winning this CAS means nothing was delivered and nothing can be. + if !delivered.CompareAndSwap(false, true) { + return nil // delivered; OnDelivered has fired or is firing + } + if ctx.Err() != nil { + // Torn down mid-connection: not the client's doing. + return ErrNotDelivered + } + return &RejectedError{Stage: "post-handshake"} +} + +// ErrNotDelivered reports a connection that ended without a response and +// without the client refusing us — teardown, essentially. It is returned so the +// caller can account for the connection instead of silently losing it from the +// outcome counters. +var ErrNotDelivered = errors.New("tlsinject: connection ended without delivering a response") + +// isCertError reports whether err came from our own certificate production. +func isCertError(err error) bool { + var ce *CertError + return errors.As(err, &ce) +} + +// serveOne runs net/http over a single already-handshaken connection. Handing +// it to http.Server (rather than writing the response by hand) is what gets +// HTTP/2 support for free: a TLSConfig advertising h2 makes Serve install the +// stdlib's HTTP/2 handler, and net/http then dispatches on the protocol ALPN +// negotiated during the handshake above. +func serveOne(tc *tls.Conn, req Request, delivered *atomic.Bool) error { + ln := newOneShotListener(tc) + srv := &http.Server{ + Handler: req.handler(delivered), + ReadHeaderTimeout: readHeaderTimeout, + // A TLSConfig advertising h2 is what makes Serve install the stdlib's + // HTTP/2 handler. net/http then picks the protocol by type-asserting the + // connection to *tls.Conn and reading the negotiated ALPN — so the + // listener must hand over the *tls.Conn itself, unwrapped. + TLSConfig: &tls.Config{NextProtos: []string{"h2", "http/1.1"}}, + // net/http logs client-side noise (malformed requests, resets) that is + // expected here and would otherwise pollute the proxy's own log stream. + ErrorLog: log.New(io.Discard, "", 0), + // Completion is observed here rather than by wrapping the connection, + // which would hide its *tls.Conn type and silently downgrade h2 to h1. + ConnState: func(_ net.Conn, state http.ConnState) { + if state == http.StateClosed || state == http.StateHijacked { + ln.finish() + } + }, + } + + if err := srv.Serve(ln); err != nil && !errors.Is(err, errServed) { + return err + } + return nil +} + +// handler writes the forged response. It is shared by the HTTP/1.1 and HTTP/2 +// paths, so both produce an identical status, header set and body. +func (q Request) handler(delivered *atomic.Bool) http.Handler { + r := q.Response + status := r.resolvedStatus() + body := r.resolvedBody() + // 204 and 304 must not carry a body; net/http would strip it and the + // advertised Content-Length would be a lie. + if status == http.StatusNoContent || status == http.StatusNotModified { + body = "" + } + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + h := w.Header() + h.Set("Content-Type", "text/plain; charset=utf-8") + for k, v := range r.Headers { + h.Set(textproto.CanonicalMIMEHeaderKey(k), v) + } + for _, k := range hopByHop { + h.Del(k) + } + if body != "" { + h.Set("Content-Length", strconv.Itoa(len(body))) + } + // One forged response per HTTP/1.1 connection: the client must not reuse + // a connection we took over from its real dependency. Connection is + // hop-by-hop and illegal in HTTP/2, where the client owns the lifetime — + // so this is set per protocol rather than by disabling keep-alives, which + // would make the HTTP/2 server send an immediate GOAWAY. + if req.ProtoMajor == 1 { + h.Set("Connection", "close") + } + w.WriteHeader(status) + if body != "" { + _, _ = io.WriteString(w, body) + } + // Report at the point of writing, not when the connection ends: an HTTP/2 + // client holds the connection open for the whole attack. + if delivered.CompareAndSwap(false, true) && q.OnDelivered != nil { + q.OnDelivered() + } + }) +} + +// resolvedStatus rejects 1xx as out of range: net/http treats an informational +// status as non-committing, so a body written after it would silently commit +// 200 instead — the client would see success where a fault was configured. +func (r Response) resolvedStatus() int { + if r.Status < 200 || r.Status > 599 { + return http.StatusServiceUnavailable + } + return r.Status +} + +// resolvedBody mirrors the cleartext injector's default one-liner, so a fault +// reads the same whether it landed on HTTP or HTTPS. +func (r Response) resolvedBody() string { + if r.Body != "" { + return r.Body + } + status := r.resolvedStatus() + reason := http.StatusText(status) + if reason == "" { + reason = "Fault Injected" + } + return fmt.Sprintf("%d %s (injected by steadybit transparent-proxy)\n", status, reason) +} + +// errServed is returned by the one-shot listener once its single connection has +// been fully served; it ends http.Server.Serve normally rather than as a fault. +var errServed = errors.New("tlsinject: connection served") + +// oneShotListener adapts a single accepted connection to net.Listener. The +// second Accept blocks until that connection is finished, so Serve returns only +// once the response has actually been delivered. The connection is handed over +// exactly as given — see the ConnState note in serveOne. +type oneShotListener struct { + mu sync.Mutex + conn net.Conn + addr net.Addr + done chan struct{} + once sync.Once +} + +func newOneShotListener(c net.Conn) *oneShotListener { + return &oneShotListener{conn: c, addr: c.LocalAddr(), done: make(chan struct{})} +} + +func (l *oneShotListener) Accept() (net.Conn, error) { + l.mu.Lock() + c := l.conn + l.conn = nil + l.mu.Unlock() + if c != nil { + return c, nil + } + <-l.done + return nil, errServed +} + +// Close is a no-op: the listener owns no resources of its own, and net/http +// closes it on the way out of Serve while the connection is still being torn +// down. +func (l *oneShotListener) Close() error { return nil } +func (l *oneShotListener) Addr() net.Addr { return l.addr } + +func (l *oneShotListener) finish() { l.once.Do(func() { close(l.done) }) } + +// replayConn prepends already-consumed bytes to a connection's read side, so +// the TLS handshake sees the ClientHello the caller peeked at. +func replayConn(c net.Conn, prefix []byte) net.Conn { + if len(prefix) == 0 { + return c + } + return &prefixedConn{Conn: c, r: io.MultiReader(bytes.NewReader(prefix), c)} +} + +type prefixedConn struct { + net.Conn + r io.Reader +} + +func (c *prefixedConn) Read(p []byte) (int, error) { return c.r.Read(p) } diff --git a/internal/tlsinject/serve_test.go b/internal/tlsinject/serve_test.go new file mode 100644 index 0000000..5291882 --- /dev/null +++ b/internal/tlsinject/serve_test.go @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +const testSNI = "api.anthropic.com" + +// serveForgedOnce accepts a single connection and injects r into it, returning +// the listener address and a channel carrying ServeForged's result. +func serveForgedOnce(t *testing.T, ca *CA, r Response) (string, <-chan error) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(context.Background(), conn, nil, Request{Response: r, HandshakeTimeout: 5 * time.Second}) + }() + return ln.Addr().String(), ch +} + +// clientTrusting builds a client that trusts caPEM and pins the SNI, so the +// request looks like one aimed at the dependency regardless of the dial address. +func clientTrusting(t *testing.T, caPEM []byte, h2 bool) (*http.Client, *http.Transport) { + t.Helper() + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("failed to add test CA to pool") + } + cfg := &tls.Config{RootCAs: pool, ServerName: testSNI, MinVersion: tls.VersionTLS12} + if !h2 { + cfg.NextProtos = []string{"http/1.1"} + } + tr := &http.Transport{TLSClientConfig: cfg, ForceAttemptHTTP2: h2} + t.Cleanup(tr.CloseIdleConnections) + return &http.Client{Transport: tr, Timeout: 10 * time.Second}, tr +} + +func waitServed(t *testing.T, ch <-chan error) error { + t.Helper() + select { + case err := <-ch: + return err + case <-time.After(10 * time.Second): + t.Fatal("ServeForged did not return") + return nil + } +} + +func Test_ServeForged_http1(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{ + Status: 503, + Body: `{"error":"injected"}`, + Headers: map[string]string{"Retry-After": "30", "Content-Type": "application/json"}, + }) + client, _ := clientTrusting(t, caPEM, false) + + resp, err := client.Get("https://" + addr + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + defer resp.Body.Close() + + if resp.Proto != "HTTP/1.1" { + t.Fatalf("proto = %q, want HTTP/1.1", resp.Proto) + } + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != `{"error":"injected"}` { + t.Fatalf("body = %q", body) + } + if got := resp.Header.Get("Retry-After"); got != "30" { + t.Fatalf("Retry-After = %q, want 30", got) + } + // A caller-supplied Content-Type must win over the default. + if got := resp.Header.Get("Content-Type"); got != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", got) + } + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// The dependency this targets speaks HTTP/2, so serving the forged response +// over h2 is the case that matters most in practice. +func Test_ServeForged_http2(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + client, tr := clientTrusting(t, caPEM, true) + + resp, err := client.Get("https://" + addr + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0", resp.Proto) + } + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + if !strings.Contains(string(body), "injected by steadybit") { + t.Fatalf("default body missing marker: %q", body) + } + + // h2 keeps the connection open for further streams; releasing it lets the + // server finish. + tr.CloseIdleConnections() + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// The canonical failure mode: the workload does not trust our CA. It must be +// reported as a HandshakeError so the caller can count it and say so, rather +// than looking like a silent no-op. +func Test_ServeForged_untrustedClientYieldsHandshakeError(t *testing.T) { + ca, _ := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + + tr := &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: x509.NewCertPool(), // trusts nothing + ServerName: testSNI, + MinVersion: tls.VersionTLS12, + }} + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + + if _, err := client.Get("https://" + addr + "/v1/messages"); err == nil { + t.Fatal("expected the client to reject the injected certificate") + } + + err := waitServed(t, ch) + var rej *RejectedError + if !errors.As(err, &rej) { + t.Fatalf("err = %v, want a *RejectedError", err) + } +} + +// Under TLS 1.3 the server's handshake completes before the client reports that +// it dislikes the certificate — the client simply walks away without sending a +// request. Observed with curl/OpenSSL against a real proxy, where it made a +// rejected connection look like a successfully injected fault. A completed +// handshake is therefore not proof of delivery; an actual response is. +func Test_ServeForged_rejectedAfterHandshakeIsNotDelivery(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + conn, err := tls.Dial("tcp", addr, &tls.Config{ + RootCAs: pool, ServerName: testSNI, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatalf("dial: %v", err) + } + // Complete the handshake, then leave without a request — exactly what a + // client that refuses the certificate does. + if err := conn.Handshake(); err != nil { + t.Fatalf("handshake: %v", err) + } + _ = conn.Close() + + err = waitServed(t, ch) + var rej *RejectedError + if !errors.As(err, &rej) { + t.Fatalf("err = %v, want a *RejectedError (nothing was ever delivered)", err) + } + if rej.Stage != "post-handshake" { + t.Fatalf("stage = %q, want post-handshake", rej.Stage) + } +} + +// Regression: an HTTP/2 client pools the connection for the whole attack, so +// delivery must be reported when the response is written — not when the +// connection ends. Reporting it late leaves a demonstrably working fault +// counted as "matched but never faulted", the proxy's silent-no-op signature. +func Test_ServeForged_reportsDeliveryBeforeConnectionCloses(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + delivered := make(chan struct{}, 1) + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(context.Background(), conn, nil, Request{ + Response: Response{Status: 503}, + HandshakeTimeout: 5 * time.Second, + OnDelivered: func() { delivered <- struct{}{} }, + }) + }() + + client, tr := clientTrusting(t, caPEM, true) + resp, err := client.Get("https://" + ln.Addr().String() + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays pooled", resp.Proto) + } + + // The connection is still open here; delivery must already be reported. + select { + case <-delivered: + case <-time.After(5 * time.Second): + t.Fatal("delivery was not reported while the h2 connection was still open") + } + select { + case err := <-ch: + t.Fatalf("ServeForged returned early (%v); the h2 connection should still be open", err) + default: + } + + tr.CloseIdleConnections() + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// Teardown must never be booked as a delivered fault: nothing was written, so +// OnDelivered must not fire. +func Test_ServeForged_cancelDoesNotReportDelivery(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + var deliveries atomic.Int64 + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(ctx, conn, nil, Request{ + Response: Response{Status: 503}, + HandshakeTimeout: 5 * time.Second, + OnDelivered: func() { deliveries.Add(1) }, + }) + }() + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + conn, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ + RootCAs: pool, ServerName: testSNI, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + + cancel() + if err := waitServed(t, ch); err != nil && !errors.Is(err, net.ErrClosed) { + var rej *RejectedError + if errors.As(err, &rej) { + t.Fatalf("teardown was misreported as a client rejection: %v", err) + } + } + if got := deliveries.Load(); got != 0 { + t.Fatalf("OnDelivered fired %d times on teardown; nothing was written", got) + } +} + +func Test_ServeForged_cancelledContextClosesConnection(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(ctx, conn, nil, Request{Response: Response{Status: 503}, HandshakeTimeout: 5 * time.Second}) + }() + + // HTTP/2 holds the connection open after the response, so once a reply has + // been read the server is definitively parked inside net/http — exactly the + // state an attack teardown has to unblock. Driving a real request first also + // removes any race with the handshake still completing. + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, ServerName: testSNI, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + } + defer tr.CloseIdleConnections() + + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + resp, err := client.Get("https://" + ln.Addr().String() + "/") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays open", resp.Proto) + } + + cancel() + if err := waitServed(t, ch); err != nil && !errors.Is(err, net.ErrClosed) { + t.Fatalf("ServeForged after cancel: %v", err) + } +} + +func Test_Response_defaults(t *testing.T) { + // An out-of-range status falls back to 503 rather than producing an invalid + // response. + for _, status := range []int{0, 42, 700} { + r := Response{Status: status} + if got := r.resolvedStatus(); got != http.StatusServiceUnavailable { + t.Fatalf("resolvedStatus(%d) = %d, want 503", status, got) + } + } + if got := (Response{Status: 418}).resolvedStatus(); got != 418 { + t.Fatalf("resolvedStatus(418) = %d", got) + } + + body := Response{Status: 503}.resolvedBody() + if !strings.Contains(body, "503") || !strings.Contains(body, "injected by steadybit") { + t.Fatalf("default body = %q", body) + } + if got := (Response{Status: 503, Body: "custom"}).resolvedBody(); got != "custom" { + t.Fatalf("explicit body = %q", got) + } +} + +// Hop-by-hop headers are illegal in HTTP/2 and are owned by net/http in +// HTTP/1.1; a caller supplying one must not be able to corrupt the response. +func Test_Response_stripsHopByHopHeaders(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{ + Status: 503, + Headers: map[string]string{"Connection": "keep-alive", "Transfer-Encoding": "chunked", "X-Kept": "yes"}, + }) + client, _ := clientTrusting(t, caPEM, false) + + resp, err := client.Get("https://" + addr + "/") + if err != nil { + t.Fatalf("request: %v", err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("X-Kept"); got != "yes" { + t.Fatalf("X-Kept = %q, want yes", got) + } + if got := resp.Header.Get("Transfer-Encoding"); got != "" { + t.Fatalf("Transfer-Encoding leaked through: %q", got) + } + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +func Test_replayConn(t *testing.T) { + // Without a prefix the connection is handed through unwrapped. + a, b := net.Pipe() + defer func() { _ = a.Close() }() + defer func() { _ = b.Close() }() + if got := replayConn(a, nil); got != a { + t.Fatal("expected the original conn when there is no prefix") + } + + // With a prefix the consumed bytes are replayed ahead of the live stream, + // which is what lets the TLS handshake see the peeked ClientHello. + c, d := net.Pipe() + defer func() { _ = c.Close() }() + go func() { + _, _ = d.Write([]byte("world")) + _ = d.Close() + }() + got, err := io.ReadAll(replayConn(c, []byte("hello "))) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "hello world" { + t.Fatalf("read %q, want %q", got, "hello world") + } +} diff --git a/main.go b/main.go index a2b1dbe..eeb731a 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "errors" "flag" "fmt" + "io" "log/slog" "net" "net/http" @@ -37,6 +38,7 @@ import ( "github.com/steadybit/transparent-proxy/internal/preflight" "github.com/steadybit/transparent-proxy/internal/proxy" "github.com/steadybit/transparent-proxy/internal/supervisor" + "github.com/steadybit/transparent-proxy/internal/tlsinject" ) func main() { @@ -60,12 +62,20 @@ func main() { revert = flag.Bool("revert", false, "remove the interception rules for the given --exec-id/--intercept-* and exit (out-of-band teardown, idempotent)") + // HTTPS response injection. The CA is supplied by the customer, who owns + // its validity and installs it in their workloads' truststores; the proxy + // only signs per-SNI leaves with it. Unset = TLS is never decrypted. + tlsCACert = flag.String("tls-ca-cert", "", "PEM CA certificate used to mint per-SNI certificates, enabling HTTPS response injection (requires --tls-ca-key)") + tlsCAKey = flag.String("tls-ca-key", "", "PEM private key matching --tls-ca-cert") + tlsLeafTTL = flag.Duration("tls-leaf-validity", 0, "how long minted per-SNI certificates are valid (0 = built-in default; always clamped to the CA's own expiry)") + tlsCAStdin = flag.Bool("tls-ca-stdin", false, "read the interception CA (certificate and private key, one PEM stream) from stdin instead of from files") + // Single-rule fault flags — a convenience for orchestrators that inject // one fault, avoiding a JSON --config file. Appended to any --config rules. faultLatency = flag.Duration("fault-latency", 0, "single fault: latency added before connecting upstream") faultReset = flag.Bool("fault-reset", false, "single fault: reset (RST) matching connections") - faultStatus = flag.Int("fault-http-status", 0, "single fault: injected HTTP status (L7, cleartext HTTP)") - faultBody = flag.String("fault-http-body", "", "single fault: injected HTTP response body (L7, cleartext HTTP)") + faultStatus = flag.Int("fault-http-status", 0, "single fault: injected HTTP status (L7; cleartext HTTP, plus HTTPS when --tls-ca-cert is set)") + faultBody = flag.String("fault-http-body", "", "single fault: injected HTTP response body (L7; cleartext HTTP, plus HTTPS when --tls-ca-cert is set)") faultProb = flag.Float64("fault-probability", 1, "single fault: probability [0,1] to apply the fault per connection (default 1 = always, 0 = never)") faultHosts = flag.String("fault-hosts", "", "single fault: comma-separated host selectors (SNI/Host)") faultCIDRs = flag.String("fault-cidrs", "", "single fault: comma-separated CIDR selectors") @@ -119,7 +129,7 @@ func main() { // Validate the interception filter and whether self-managed mode is wanted. // The port is filled in after we bind (below); 0 is fine here because this // instance is only used for --revert, where the port is irrelevant. - interceptor, wantIntercept, err := buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), 0, *noFlush) + interceptor, wantIntercept, err := buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), 0, *noFlush, nil) if err != nil { logger.Error("invalid interception configuration", slog.Any("err", err)) os.Exit(2) @@ -141,6 +151,23 @@ func main() { return } + // The CA is loaded only after the --revert branch above: teardown must never + // depend on it. An orchestrator naturally reuses the same argument vector for + // --revert, by which time the CA files may be gone or expired — refusing to + // start there would leave the interception rules installed, breaking the + // guaranteed-cleanup contract. + injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey, *tlsCAStdin) + if err != nil { + logger.Error("invalid TLS interception CA", slog.Any("err", err)) + os.Exit(2) + } + if injector != nil { + injector.SetLeafValidity(*tlsLeafTTL) + srv.TLSInject = injector + logger.Info("HTTPS response injection enabled", + slog.Time("ca_not_after", injector.NotAfter())) + } + // Preflight: refuse to fight an existing mesh proxy. Ports default to the // intercept ports when not given explicitly. preflightSpec := *prePorts @@ -164,7 +191,7 @@ func main() { os.Exit(1) } port := uint16(ln.Addr().(*net.TCPAddr).Port) - interceptor, _, err = buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), port, *noFlush) + interceptor, _, err = buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), port, *noFlush, flushDestinations(rules, logger)) if err != nil { logger.Error("invalid interception configuration", slog.Any("err", err)) os.Exit(2) @@ -190,6 +217,64 @@ func main() { logger.Info("proxy stopped", slog.Any("metrics", m.Snapshot())) } +const ( + // maxCAStdinBytes caps the CA read from stdin; a PEM pair is a few KB. + maxCAStdinBytes = 1 << 20 + // caStdinTimeout bounds that read, so a writer that never closes the pipe + // fails loudly instead of hanging the proxy before it installs anything. + caStdinTimeout = 30 * time.Second +) + +// flushDestinations resolves the hostnames the fault rules target, so the +// connection-pool flush can be narrowed to those addresses. +// +// This runs inside the target's network namespace, so it sees the same routes +// the workload does. It is a snapshot: a dependency behind rotating IPs may +// hold connections to an address that no longer resolves, and those simply are +// not flushed — they still get faulted when they next reconnect. Under-flushing +// is the right side to err on, since over-flushing resets connections to +// dependencies the attack never named. +// +// Returns nil when no rule targets a hostname, which leaves the flush scoped to +// the capture filter — what a CIDR-targeted attack actually wants. +func flushDestinations(rules []fault.Rule, logger *slog.Logger) []netip.Prefix { + var hosts []string + for _, r := range rules { + hosts = append(hosts, r.Hosts...) + } + if len(hosts) == 0 { + return nil + } + + seen := map[netip.Prefix]bool{} + var out []netip.Prefix + for _, h := range hosts { + addrs, err := net.LookupHost(h) + if err != nil { + // Not fatal: the attack still applies to new connections. Warn, + // because the operator asked for existing ones to be reset too. + logger.Warn("could not resolve a targeted dependency; its existing connections will not be reset", + slog.String("host", h), slog.Any("err", err)) + continue + } + for _, a := range addrs { + addr, perr := netip.ParseAddr(a) + if perr != nil || !addr.Is4() { + continue // interception is IPv4-only + } + pfx := netip.PrefixFrom(addr, 32) + if !seen[pfx] { + seen[pfx] = true + out = append(out, pfx) + } + } + } + if len(out) == 0 { + logger.Warn("no targeted dependency resolved to an IPv4 address; existing connections will not be reset") + } + return out +} + func loadRules(path string) ([]fault.Rule, error) { if path == "" { return nil, nil @@ -197,6 +282,58 @@ func loadRules(path string) ([]fault.Rule, error) { return config.Load(path) } +// loadInterceptCA loads the optional HTTPS-interception CA. Returning (nil, nil) +// means the feature is off and TLS is never decrypted. +// +// The customer owns this CA — they generate it, choose how long it lives, and +// install it in the truststores of the workloads they want to fault. The only +// lifecycle judgement made here is refusing one that is already outside its +// validity window, because it would otherwise fail every handshake with a far +// less obvious error. +func loadInterceptCA(certPath, keyPath string, fromStdin bool) (*tlsinject.CA, error) { + var ( + ca *tlsinject.CA + err error + ) + switch { + case fromStdin && (certPath != "" || keyPath != ""): + return nil, errors.New("--tls-ca-stdin cannot be combined with --tls-ca-cert/--tls-ca-key") + case fromStdin: + // Reading the key from stdin keeps it off the command line and off any + // filesystem the target could reach. It is also the only channel that + // works uniformly: the proxy may run inside an overlay of the + // orchestrator's root, which does not carry the orchestrator's submounts, + // so a key mounted there is invisible by path. + // Bounded and deadlined: this read happens before the listener is bound, + // before preflight, and before the --max-duration deadman is armed, so a + // writer that never closes the pipe would hang the proxy forever with no + // rules installed and no way out. + _ = os.Stdin.SetReadDeadline(time.Now().Add(caStdinTimeout)) + pemBytes, rerr := io.ReadAll(io.LimitReader(os.Stdin, maxCAStdinBytes+1)) + _ = os.Stdin.SetReadDeadline(time.Time{}) + if rerr != nil { + return nil, fmt.Errorf("failed to read CA from stdin: %w", rerr) + } + if len(pemBytes) > maxCAStdinBytes { + return nil, fmt.Errorf("CA on stdin exceeds %d bytes", maxCAStdinBytes) + } + ca, err = tlsinject.LoadCACombined(pemBytes) + case certPath == "" && keyPath == "": + return nil, nil + case certPath == "" || keyPath == "": + return nil, errors.New("--tls-ca-cert and --tls-ca-key must be set together") + default: + ca, err = tlsinject.LoadCAFromFiles(certPath, keyPath) + } + if err != nil { + return nil, err + } + if ca.Expired(time.Now()) { + return nil, fmt.Errorf("CA is outside its validity window (not after %s); issue a new one", ca.NotAfter().Format(time.RFC3339)) + } + return ca, nil +} + // stringList is a repeatable string flag (e.g. --fault-http-header used more // than once). type stringList []string @@ -268,7 +405,7 @@ func (a interceptorAdapter) Revert(ctx context.Context) error { return a.cfg.Rev // targets the exact port the proxy bound — there is no pre-allocated port that // another process could steal between allocation and bind. For --revert the // port is irrelevant (chain names derive from the exec-id) and may be 0. -func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyPort uint16, noFlush bool) (supervisor.Interceptor, bool, error) { +func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyPort uint16, noFlush bool, flushDsts []netip.Prefix) (supervisor.Interceptor, bool, error) { if cidrs == "" && ports == "" { return nil, false, nil } @@ -290,10 +427,11 @@ func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyP } cfg := interception.Config{ - ExecutionID: execID, - ProxyPort: proxyPort, - Mark: mark, - SkipFlush: noFlush, + ExecutionID: execID, + ProxyPort: proxyPort, + Mark: mark, + SkipFlush: noFlush, + FlushDestinations: flushDsts, Filter: interception.Filter{ Include: include, Exclude: exclude,