From 5df7201d87e4f4fc0a7396544a632a5b8bfdb505 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Wed, 2 Sep 2026 14:42:26 +0200 Subject: [PATCH 1/8] feat(tlsinject): synthesize HTTP responses for HTTPS dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An httpStatus fault could previously only be applied to cleartext HTTP: TLS connections were matched by SNI but always spliced through untouched, so an HTTPS dependency could be delayed or reset but never made to return a status. Add an opt-in interception path. When a customer-supplied CA is configured and a matching rule carries httpStatus, the connection is terminated with a short-lived certificate minted for its SNI and the response is synthesized inside TLS. The response is served through net/http, which supports HTTP/1.1 and HTTP/2 by ALPN without adding a dependency (the repo stays stdlib-only). The CA belongs to the customer: they generate it, choose its validity, and install it in their workloads' truststores. The proxy only signs per-SNI leaves, clamped to never outlive the CA, and rejects an already-expired CA at startup. Interception is one-sided — the dependency is never dialed — so no upstream trust decision is made and mutual-TLS dependencies are unaffected. Without a CA, without an SNI, or without an httpStatus rule, TLS is spliced through exactly as before. A client that rejects the minted certificate is recorded as tls_handshake_failures and never as an applied fault, so a missing CA in the truststore is diagnosable instead of a silent no-op. --- README.md | 51 ++++- internal/fault/fault.go | 10 +- internal/metrics/metrics.go | 13 ++ internal/proxy/server.go | 58 ++++- internal/proxy/server_tlsinject_test.go | 220 ++++++++++++++++++ internal/tlsinject/ca.go | 208 +++++++++++++++++ internal/tlsinject/ca_test.go | 202 +++++++++++++++++ internal/tlsinject/serve.go | 222 ++++++++++++++++++ internal/tlsinject/serve_test.go | 288 ++++++++++++++++++++++++ main.go | 43 ++++ 10 files changed, 1303 insertions(+), 12 deletions(-) create mode 100644 internal/proxy/server_tlsinject_test.go create mode 100644 internal/tlsinject/ca.go create mode 100644 internal/tlsinject/ca_test.go create mode 100644 internal/tlsinject/serve.go create mode 100644 internal/tlsinject/serve_test.go diff --git a/README.md b/README.md index eee8b33..7fb560e 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ The hostname/SNI targeting is what makes **one** tool work for both internal - **Inspected path:** when a rule targets by hostname, the proxy reads only the first TLS record to extract the SNI (cleartext — **no MITM, no certificates**), replays those bytes to the upstream, then splices the remainder. +- **Interception path (opt-in):** only when a CA is supplied via + `--tls-ca-cert`/`--tls-ca-key` *and* a matching rule carries `httpStatus`, an + HTTPS connection is terminated so the response can be synthesized. See + [HTTPS response injection](#https-response-injection). Without a CA the proxy + never decrypts anything. ## Fault rules @@ -82,9 +87,53 @@ selectors match — an empty selector means "any". - `hosts` — match the TLS SNI, exact or subdomain (external targeting). - `latency` — Go duration string, added before the upstream connect. - `abort` — reset (RST) the connection. -- `httpStatus` — synthesize this HTTP status (L7, cleartext HTTP). +- `httpStatus` — synthesize this HTTP status (L7). Cleartext HTTP always; HTTPS + only with an interception CA (see below). - `probability` — `[0,1]` chance to apply the fault per connection (`0`/unset = always). +## HTTPS response injection + +By default the proxy never decrypts TLS: it reads the SNI in cleartext and +splices the bytes through. Supplying a CA opts in to terminating **matched** +HTTPS connections so an `httpStatus` fault can be synthesized inside TLS: + +```bash +transparent-proxy \ + --tls-ca-cert /etc/steadybit/intercept-ca.crt \ + --tls-ca-key /etc/steadybit/intercept-ca.key \ + --fault-hosts api.stripe.com --fault-http-status 503 +``` + +The proxy mints a short-lived certificate for the connection's SNI, signed by +that CA, and answers the request itself. **HTTP/1.1 and HTTP/2 are both +supported** — the response is delivered over whichever the client negotiates +via ALPN. + +**The CA is yours.** You generate it, choose how long it lives, and install it +in the truststores of the workloads you want to fault. The proxy only signs with +it; it never creates, rotates, or renews a CA. A CA already outside its validity +window is rejected at startup rather than failing every handshake later. + +This is deliberately **one-sided**: the real dependency is never contacted. The +proxy makes no trust decision about the origin's certificate, and a dependency +behind mutual TLS is unaffected. The trade-off is that the response is +fabricated rather than a modified real one. + +**When it does not apply** — the connection is spliced through untouched: + +- no CA configured, or the client sent no SNI; +- the rule carries no `httpStatus`; +- the connection lost the `probability` roll. + +**When the client refuses** — if the workload does not trust the CA (or pins +certificates), the handshake fails. That is counted as `tls_handshake_failures` +and is deliberately *not* counted as a fault, so a non-zero value is the signal +that the CA is missing from the target's truststore rather than a silent no-op. + +> Interception requires a key that can impersonate any HTTPS endpoint to +> anything trusting the CA. Treat it as a test/staging capability and keep the +> key restricted. + ## Build & test ```bash diff --git a/internal/fault/fault.go b/internal/fault/fault.go index 7f71aed..8dba25c 100644 --- a/internal/fault/fault.go +++ b/internal/fault/fault.go @@ -43,9 +43,13 @@ type Rule struct { Abort bool // HTTPStatus, if non-zero, makes the proxy synthesize an HTTP response with - // this status code instead of forwarding — an L7 fault that applies only to - // cleartext HTTP (it is ignored on TLS/opaque connections). Selected by the - // Host header, matched with the same semantics as Hosts. + // this status code instead of forwarding — an L7 fault selected by the Host + // header or TLS SNI, matched with the same semantics as Hosts. + // + // It applies to cleartext HTTP always, and to HTTPS only when the proxy was + // given an interception CA (--tls-ca-cert/--tls-ca-key): the connection is + // then terminated with a certificate minted for its SNI. Without a CA, TLS + // connections are spliced through untouched. Opaque L4 is never affected. HTTPStatus int // HTTPBody, if set, replaces the default synthesized response body. diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0a074d3..d557c8e 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -26,6 +26,7 @@ type Metrics struct { ConnectionsFaulted atomic.Int64 // connections a fault was actually applied to (once each) LatencyApplied atomic.Int64 // connections a latency fault delayed HTTPResponsesInjected atomic.Int64 // connections given a synthesized HTTP response + TLSHandshakeFailures atomic.Int64 // HTTPS interception rejected by the client (CA not trusted / pinning) UpstreamErrors atomic.Int64 // dial failures BytesToUpstream atomic.Int64 BytesToClient atomic.Int64 @@ -63,6 +64,7 @@ type Snapshot struct { ConnectionsFaulted int64 `json:"connections_faulted"` LatencyApplied int64 `json:"latency_applied"` HTTPResponsesInjected int64 `json:"http_responses_injected"` + TLSHandshakeFailures int64 `json:"tls_handshake_failures"` UpstreamErrors int64 `json:"upstream_errors"` BytesToUpstream int64 `json:"bytes_to_upstream"` BytesToClient int64 `json:"bytes_to_client"` @@ -93,6 +95,7 @@ func (m *Metrics) Snapshot() Snapshot { ConnectionsFaulted: m.ConnectionsFaulted.Load(), LatencyApplied: m.LatencyApplied.Load(), HTTPResponsesInjected: m.HTTPResponsesInjected.Load(), + TLSHandshakeFailures: m.TLSHandshakeFailures.Load(), UpstreamErrors: m.UpstreamErrors.Load(), BytesToUpstream: m.BytesToUpstream.Load(), BytesToClient: m.BytesToClient.Load(), @@ -177,6 +180,16 @@ func (m *Metrics) HTTPInjected() { } } +// TLSHandshakeFailed records an HTTPS connection the client refused to complete +// a handshake on. A non-zero count is the canonical "our CA is not trusted by +// the target (or the client pins certificates)" signal — the fault could not be +// applied, so it is deliberately not counted as faulted. +func (m *Metrics) TLSHandshakeFailed() { + if m != nil { + m.TLSHandshakeFailures.Add(1) + } +} + // MatchedHost records that a connection carrying the given dependency hostname // matched a rule. FaultedHost records that a fault was actually applied to it // (i.e. it passed the probability roll). Empty hosts are ignored. diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 77de879..f8bba2b 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -10,6 +10,7 @@ package proxy import ( "context" + "errors" "log/slog" "net" "net/netip" @@ -18,6 +19,7 @@ import ( "github.com/steadybit/transparent-proxy/internal/fault" "github.com/steadybit/transparent-proxy/internal/metrics" + "github.com/steadybit/transparent-proxy/internal/tlsinject" ) const keepAlivePeriod = 30 * time.Second @@ -63,6 +65,13 @@ type Server struct { // ConnectionsMatched under load is the canonical silent-no-op signal. Metrics *metrics.Metrics + // TLSInject, when non-nil, enables HTTPS response injection: a matched TLS + // connection carrying an L7 status fault is terminated with a certificate + // minted for its SNI, and the synthesized response is written inside TLS. + // Nil (the default) means TLS is never decrypted — HTTPS connections are + // spliced through untouched exactly as before. + TLSInject *tlsinject.CA + // listenPort and localAddrs are captured at Serve time for the self-loop // guard, so a redirected flow resolving back to this proxy (on loopback or // any local interface address) is refused rather than dialed in a storm. @@ -249,15 +258,48 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { } // L7: synthesize an HTTP status response without contacting the upstream. - // Only valid for cleartext HTTP; ignored otherwise. - if proto == protoHTTP && action.HTTPStatus != 0 { - markFaulted() - if err := writeHTTPResponse(client, action.HTTPStatus, action.HTTPHeaders, action.HTTPBody); err != nil { - log.Debug("failed to write injected status", slog.Any("err", err)) + // Cleartext HTTP is written directly. HTTPS is only decrypted when a CA is + // configured and the client offered an SNI to mint a certificate for; + // otherwise the connection falls through and is spliced untouched, which is + // the pre-CA behaviour. + if action.HTTPStatus != 0 { + switch { + case proto == protoHTTP: + markFaulted() + if err := writeHTTPResponse(client, action.HTTPStatus, action.HTTPHeaders, action.HTTPBody); err != nil { + log.Debug("failed to write injected status", slog.Any("err", err)) + } + s.Metrics.HTTPInjected() + log.Info("injected http status", slog.Int("status", action.HTTPStatus)) + return + + case proto == protoTLS && s.TLSInject != nil && identity != "": + err := s.TLSInject.ServeForged(ctx, client, prefix, tlsinject.Response{ + Status: action.HTTPStatus, + Body: action.HTTPBody, + Headers: action.HTTPHeaders, + }, s.peekTimeout()) + + var hsErr *tlsinject.HandshakeError + if errors.As(err, &hsErr) { + // The client rejected our certificate, so the fault never applied — + // counted separately from faults, never as one. This is the signal + // that the CA is missing from the workload's truststore. + s.Metrics.TLSHandshakeFailed() + log.Warn("client rejected the injected certificate; is the CA trusted by the target?", + slog.Any("err", err)) + return + } + if err != nil { + log.Debug("failed to serve injected https response", slog.Any("err", err)) + return + } + // Recorded only once the response was actually delivered. + markFaulted() + s.Metrics.HTTPInjected() + log.Info("injected https status", slog.Int("status", action.HTTPStatus)) + return } - s.Metrics.HTTPInjected() - log.Info("injected http status", slog.Int("status", action.HTTPStatus)) - return } s.forward(ctx, log, client, dst, prefix) diff --git a/internal/proxy/server_tlsinject_test.go b/internal/proxy/server_tlsinject_test.go new file mode 100644 index 0000000..8985adb --- /dev/null +++ b/internal/proxy/server_tlsinject_test.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package proxy + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "testing" + "time" + + "github.com/steadybit/transparent-proxy/internal/fault" + "github.com/steadybit/transparent-proxy/internal/metrics" + "github.com/steadybit/transparent-proxy/internal/tlsinject" +) + +// upstreamHost matches the name in httptest's built-in certificate, so a +// pass-through connection validates against the real upstream too. +const upstreamHost = "example.com" + +func newInterceptCA(t *testing.T) (*tlsinject.CA, []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Test Intercept CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + ca, err := tlsinject.LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + return ca, certPEM +} + +// startTLSUpstream stands in for the real dependency. +func startTLSUpstream(t *testing.T) (*httptest.Server, netip.AddrPort) { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "real-response") + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse upstream url: %v", err) + } + ap, err := netip.ParseAddrPort(u.Host) + if err != nil { + t.Fatalf("parse upstream addr: %v", err) + } + return srv, ap +} + +// clientVia dials every request at the proxy while presenting upstreamHost as +// the SNI, so the proxy sees exactly what a redirected dependency call looks +// like. The pool trusts both the upstream and (when given) the intercept CA. +func clientVia(t *testing.T, proxyAddr netip.AddrPort, upstream *httptest.Server, interceptCAPEM []byte) *http.Client { + t.Helper() + pool := x509.NewCertPool() + pool.AddCert(upstream.Certificate()) + if len(interceptCAPEM) > 0 && !pool.AppendCertsFromPEM(interceptCAPEM) { + t.Fatal("failed to add intercept CA to pool") + } + tr := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", proxyAddr.String()) + }, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + } + t.Cleanup(tr.CloseIdleConnections) + return &http.Client{Transport: tr, Timeout: 10 * time.Second} +} + +func httpsRule() *fault.Engine { + p := 1.0 + return fault.NewEngine([]fault.Rule{{ + Name: "https-intercept", + Hosts: []string{upstreamHost}, + HTTPStatus: 503, + Probability: &p, + }}) +} + +// waitFor polls until cond holds, so assertions do not race the proxy +// goroutine finishing its bookkeeping. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", msg) +} + +// With a CA configured, an HTTPS dependency call is terminated and answered +// with the forged response instead of reaching the real upstream. +func TestServer_TLSInject_ForgesResponse(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, caPEM := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + client := clientVia(t, proxyAddr, upstream, caPEM) + + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + if strings.Contains(string(body), "real-response") { + t.Fatal("the request reached the real upstream; it should have been forged") + } + if !strings.Contains(string(body), "injected by steadybit") { + t.Fatalf("body = %q", body) + } + + waitFor(t, func() bool { return m.Snapshot().HTTPResponsesInjected == 1 }, "an injected response") + snap := m.Snapshot() + if snap.ConnectionsFaulted != 1 { + t.Fatalf("ConnectionsFaulted = %d, want 1", snap.ConnectionsFaulted) + } + if snap.TLSHandshakeFailures != 0 { + t.Fatalf("TLSHandshakeFailures = %d, want 0", snap.TLSHandshakeFailures) + } + if got := snap.PerHost[upstreamHost]; got.Faulted != 1 { + t.Fatalf("per-host faulted = %d, want 1", got.Faulted) + } +} + +// Without a CA the same rule must leave HTTPS alone — the pre-existing +// behaviour, and the guarantee that enabling the feature is opt-in. +func TestServer_TLSInject_DisabledPassesThrough(t *testing.T) { + upstream, dst := startTLSUpstream(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m}, dst) + client := clientVia(t, proxyAddr, upstream, nil) + + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK || string(body) != "real-response" { + t.Fatalf("got %d %q, want 200 real-response", resp.StatusCode, body) + } + if got := m.Snapshot().ConnectionsFaulted; got != 0 { + t.Fatalf("ConnectionsFaulted = %d, want 0 (no CA configured)", got) + } +} + +// A workload that does not trust the CA is the expected first failure. It must +// be counted as a handshake failure and never as an applied fault, so the +// operator gets a pointed diagnosis instead of a silent no-op. +func TestServer_TLSInject_UntrustedClientIsCounted(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, _ := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + // Trusts the upstream but not the intercept CA. + client := clientVia(t, proxyAddr, upstream, nil) + + if _, err := client.Get("https://" + upstreamHost + "/v1/messages"); err == nil { + t.Fatal("expected the client to reject the injected certificate") + } + + waitFor(t, func() bool { return m.Snapshot().TLSHandshakeFailures == 1 }, "a counted handshake failure") + snap := m.Snapshot() + if snap.ConnectionsFaulted != 0 { + t.Fatalf("ConnectionsFaulted = %d, want 0 — the fault never applied", snap.ConnectionsFaulted) + } + if snap.HTTPResponsesInjected != 0 { + t.Fatalf("HTTPResponsesInjected = %d, want 0", snap.HTTPResponsesInjected) + } +} diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go new file mode 100644 index 0000000..77a3155 --- /dev/null +++ b/internal/tlsinject/ca.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +// Package tlsinject terminates TLS on an intercepted connection so an L7 fault +// can be synthesized for an HTTPS dependency. +// +// The proxy mints a short-lived leaf certificate for the connection's SNI, +// signed by a certificate authority the customer supplies. That CA is entirely +// theirs: they generate it, choose its validity, and install it in the +// truststores of the workloads they want to fault. The proxy only consumes the +// pair — it never creates, rotates, or expires a CA. With no CA configured the +// proxy never decrypts anything and HTTPS keeps flowing through untouched. +// +// Interception here is deliberately one-sided: the real dependency is never +// contacted. Nothing upstream is dialed, so the proxy makes no trust decision +// about the origin's certificate and a dependency behind mutual TLS is +// unaffected — the client-authenticated handshake to the origin simply never +// happens. The cost is that the response is fabricated rather than a modified +// real one. +package tlsinject + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "math/big" + "net" + "os" + "sync" + "time" +) + +const ( + // leafValidity bounds a minted leaf's lifetime. It is additionally clamped + // to the CA's own NotAfter, so a leaf never outlives its issuer. + leafValidity = 24 * time.Hour + // leafBackdate tolerates modest clock skew between the proxy and the client. + leafBackdate = 1 * time.Hour + // maxCachedLeaves bounds the per-SNI cache so traffic to a great many + // hostnames cannot grow it without limit. Beyond the cap certificates are + // still minted, just not retained. + maxCachedLeaves = 1024 +) + +// CA mints per-SNI leaf certificates from a customer-supplied authority. +// It is safe for concurrent use. +type CA struct { + cert *x509.Certificate + key crypto.Signer + + // leafKey is generated once and shared by every minted leaf, so issuing a + // certificate for a new hostname costs one signature rather than a fresh + // keypair. It never leaves this process. + leafKey *ecdsa.PrivateKey + + mu sync.Mutex + cache map[string]*tls.Certificate +} + +// LoadCA parses a PEM certificate and matching private key. The certificate +// must be a signing CA; anything else is rejected up front rather than failing +// later on every handshake. +func LoadCA(certPEM, keyPEM []byte) (*CA, error) { + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("invalid CA keypair: %w", err) + } + if len(pair.Certificate) == 0 { + return nil, errors.New("CA certificate is empty") + } + cert, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unparseable CA certificate: %w", err) + } + if !cert.IsCA || cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, errors.New("certificate is not a signing CA: it needs basicConstraints CA:TRUE and keyUsage certSign") + } + signer, ok := pair.PrivateKey.(crypto.Signer) + if !ok { + return nil, errors.New("CA private key does not implement crypto.Signer") + } + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate leaf key: %w", err) + } + return &CA{cert: cert, key: signer, leafKey: leafKey, cache: map[string]*tls.Certificate{}}, nil +} + +// LoadCAFromFiles reads a PEM certificate and key from disk. +func LoadCAFromFiles(certPath, keyPath string) (*CA, error) { + certPEM, err := os.ReadFile(certPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate: %w", err) + } + keyPEM, err := os.ReadFile(keyPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA key: %w", err) + } + return LoadCA(certPEM, keyPEM) +} + +// NotAfter reports when the CA expires, for startup logging. +func (c *CA) NotAfter() time.Time { return c.cert.NotAfter } + +// Expired reports whether the CA is outside its validity window at now. The +// customer owns the CA's lifecycle; this exists only so an unusable CA is +// reported at startup instead of failing every handshake later. +func (c *CA) Expired(now time.Time) bool { + return now.Before(c.cert.NotBefore) || now.After(c.cert.NotAfter) +} + +// ServerTLSConfig returns a config that mints a certificate for whatever SNI +// the client asks for. Both h2 and http/1.1 are advertised so the forged +// response can be delivered over whichever the client negotiates. +func (c *CA) ServerTLSConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hello.ServerName == "" { + // Without SNI there is no name to impersonate. Failing here aborts + // the handshake; the caller forwards such connections instead. + return nil, errors.New("client sent no SNI; cannot mint a certificate") + } + return c.leafFor(hello.ServerName) + }, + } +} + +// leafFor returns a cached certificate for host, minting one on first use. +func (c *CA) leafFor(host string) (*tls.Certificate, error) { + c.mu.Lock() + if cert, ok := c.cache[host]; ok { + c.mu.Unlock() + return cert, nil + } + c.mu.Unlock() + + cert, err := c.mint(host) + if err != nil { + return nil, err + } + + c.mu.Lock() + defer c.mu.Unlock() + // Another goroutine may have minted the same host concurrently; prefer the + // stored one so every caller for a host shares a single certificate. + if existing, ok := c.cache[host]; ok { + return existing, nil + } + if len(c.cache) < maxCachedLeaves { + c.cache[host] = cert + } + return cert, nil +} + +func (c *CA) mint(host string) (*tls.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("failed to generate serial: %w", err) + } + + now := time.Now() + notAfter := now.Add(leafValidity) + // A leaf must never outlive the CA that signed it. + if notAfter.After(c.cert.NotAfter) { + notAfter = c.cert.NotAfter + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: now.Add(-leafBackdate), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + // Clients validate the SAN, not the CN. An IP literal in SNI is unusual but + // cheap to support correctly. + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, c.cert, c.leafKey.Public(), c.key) + if err != nil { + return nil, fmt.Errorf("failed to sign certificate for %q: %w", host, err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("failed to parse minted certificate: %w", err) + } + return &tls.Certificate{ + // Send the CA alongside the leaf so clients that trust it by a different + // path can still build the chain. + Certificate: [][]byte{der, c.cert.Raw}, + PrivateKey: c.leafKey, + Leaf: leaf, + }, nil +} diff --git a/internal/tlsinject/ca_test.go b/internal/tlsinject/ca_test.go new file mode 100644 index 0000000..1a7d72f --- /dev/null +++ b/internal/tlsinject/ca_test.go @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "strings" + "testing" + "time" +) + +// newTestCAPEM builds a self-signed authority for tests. isCA=false yields a +// leaf-shaped certificate, used to prove non-CA input is rejected. +func newTestCAPEM(t *testing.T, notAfter time.Time, isCA bool) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Test Intercept CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: notAfter, + BasicConstraintsValid: true, + } + if isCA { + tmpl.IsCA = true + tmpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign + } else { + tmpl.KeyUsage = x509.KeyUsageDigitalSignature + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, key.Public(), key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +func mustLoadTestCA(t *testing.T) (*CA, []byte) { + t.Helper() + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(30*24*time.Hour), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + return ca, certPEM +} + +func Test_LoadCA_acceptsSigningCA(t *testing.T) { + ca, _ := mustLoadTestCA(t) + if ca.Expired(time.Now()) { + t.Fatal("freshly issued CA reported as expired") + } + if ca.NotAfter().Before(time.Now()) { + t.Fatal("NotAfter is in the past") + } +} + +func Test_LoadCA_rejectsNonCA(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(time.Hour), false) + _, err := LoadCA(certPEM, keyPEM) + if err == nil { + t.Fatal("expected a non-CA certificate to be rejected") + } + if !strings.Contains(err.Error(), "not a signing CA") { + t.Fatalf("unexpected error: %v", err) + } +} + +func Test_LoadCA_rejectsMismatchedKey(t *testing.T) { + certPEM, _ := newTestCAPEM(t, time.Now().Add(time.Hour), true) + _, otherKey := newTestCAPEM(t, time.Now().Add(time.Hour), true) + if _, err := LoadCA(certPEM, otherKey); err == nil { + t.Fatal("expected a mismatched key to be rejected") + } +} + +func Test_LoadCA_rejectsGarbage(t *testing.T) { + if _, err := LoadCA([]byte("not a pem"), []byte("neither")); err == nil { + t.Fatal("expected garbage input to be rejected") + } +} + +func Test_Expired(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(time.Hour), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + if !ca.Expired(time.Now().Add(2 * time.Hour)) { + t.Fatal("expected a CA to be expired past its NotAfter") + } + if !ca.Expired(time.Now().Add(-2 * time.Hour)) { + t.Fatal("expected a CA to be invalid before its NotBefore") + } +} + +func Test_leafFor_mintsForSNIAndCaches(t *testing.T) { + ca, _ := mustLoadTestCA(t) + + first, err := ca.leafFor("api.anthropic.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if got := first.Leaf.DNSNames; len(got) != 1 || got[0] != "api.anthropic.com" { + t.Fatalf("leaf SAN = %v, want [api.anthropic.com]", got) + } + // The leaf must actually chain to the CA a client would trust. + pool := x509.NewCertPool() + pool.AddCert(ca.cert) + if _, err := first.Leaf.Verify(x509.VerifyOptions{Roots: pool, DNSName: "api.anthropic.com", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + t.Fatalf("minted leaf does not verify against the CA: %v", err) + } + + second, err := ca.leafFor("api.anthropic.com") + if err != nil { + t.Fatalf("leafFor (cached): %v", err) + } + if first != second { + t.Fatal("expected the second call to return the cached certificate") + } + + other, err := ca.leafFor("example.com") + if err != nil { + t.Fatalf("leafFor (other host): %v", err) + } + if other == first { + t.Fatal("expected a distinct certificate per hostname") + } +} + +func Test_leafFor_usesIPSANForIPLiteral(t *testing.T) { + ca, _ := mustLoadTestCA(t) + cert, err := ca.leafFor("10.1.2.3") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if len(cert.Leaf.IPAddresses) != 1 || cert.Leaf.IPAddresses[0].String() != "10.1.2.3" { + t.Fatalf("IP SANs = %v, want [10.1.2.3]", cert.Leaf.IPAddresses) + } + if len(cert.Leaf.DNSNames) != 0 { + t.Fatalf("unexpected DNS SANs on an IP leaf: %v", cert.Leaf.DNSNames) + } +} + +// A leaf must never outlive the authority that signed it, however short-lived +// the customer chose to make their CA. +func Test_leafFor_clampsValidityToCA(t *testing.T) { + caNotAfter := time.Now().Add(time.Hour) // shorter than leafValidity + certPEM, keyPEM := newTestCAPEM(t, caNotAfter, true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + cert, err := ca.leafFor("short.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if cert.Leaf.NotAfter.After(ca.NotAfter()) { + t.Fatalf("leaf NotAfter %s outlives CA NotAfter %s", cert.Leaf.NotAfter, ca.NotAfter()) + } +} + +func Test_ServerTLSConfig_requiresSNI(t *testing.T) { + ca, _ := mustLoadTestCA(t) + cfg := ca.ServerTLSConfig() + + if _, err := cfg.GetCertificate(&tls.ClientHelloInfo{}); err == nil { + t.Fatal("expected a ClientHello without SNI to be refused") + } + if _, err := cfg.GetCertificate(&tls.ClientHelloInfo{ServerName: "api.anthropic.com"}); err != nil { + t.Fatalf("expected a certificate for a named host: %v", err) + } + + // Both protocols must be offered so the forged response can be delivered + // over whichever the client negotiates. + want := map[string]bool{"h2": false, "http/1.1": false} + for _, p := range cfg.NextProtos { + want[p] = true + } + for p, seen := range want { + if !seen { + t.Fatalf("ALPN does not advertise %q (got %v)", p, cfg.NextProtos) + } + } +} diff --git a/internal/tlsinject/serve.go b/internal/tlsinject/serve.go new file mode 100644 index 0000000..63115f5 --- /dev/null +++ b/internal/tlsinject/serve.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/textproto" + "strconv" + "sync" + "time" +) + +// readHeaderTimeout bounds how long a client may take to send its request head +// after the handshake, so a silent client cannot pin a goroutine indefinitely. +const readHeaderTimeout = 10 * time.Second + +// hopByHop headers are meaningful only to a single transport hop. They are +// stripped from caller-supplied headers because net/http owns framing, and +// because HTTP/2 rejects them outright. +var hopByHop = []string{ + "Connection", "Keep-Alive", "Proxy-Connection", "Transfer-Encoding", + "Te", "Trailer", "Upgrade", +} + +// Response is the forged HTTP response written back to the intercepted client. +// The real dependency is never contacted, so every byte here is synthesized. +type Response struct { + Status int + Body string + Headers map[string]string +} + +// HandshakeError marks a failure to complete the TLS handshake with the client. +// In practice this means the minted certificate was rejected: the CA is not in +// the workload's truststore, or the client pins certificates. It is a distinct +// type so the caller can count it and surface that diagnosis rather than +// reporting a silent no-op. +type HandshakeError struct{ Err error } + +func (e *HandshakeError) Error() string { + return "tls handshake with client failed: " + e.Err.Error() +} +func (e *HandshakeError) Unwrap() error { return e.Err } + +// ServeForged terminates TLS on conn using a certificate minted for the +// client's SNI, then answers the request with r and closes. clientHello replays +// the bytes already consumed while sniffing the SNI, so the handshake sees the +// original stream; pass nil when nothing was consumed. +// +// It blocks until the connection is finished — for HTTP/1.1 that is one +// request, for HTTP/2 until the client goes away — or until ctx is cancelled, +// which closes the connection so an attack teardown never leaks a goroutine. +func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, r Response, handshakeTimeout time.Duration) error { + tc := tls.Server(replayConn(conn, clientHello), c.ServerTLSConfig()) + + hctx, cancel := context.WithTimeout(ctx, handshakeTimeout) + defer cancel() + if err := tc.HandshakeContext(hctx); err != nil { + return &HandshakeError{Err: err} + } + + // Cancellation must reach a connection parked inside net/http; closing it is + // the only way to unblock the server loop. + stop := make(chan struct{}) + defer close(stop) + go func() { + select { + case <-ctx.Done(): + _ = tc.Close() + case <-stop: + } + }() + + return serveOne(tc, r) +} + +// serveOne runs net/http over a single already-handshaken connection. Handing +// it to http.Server (rather than writing the response by hand) is what gets +// HTTP/2 support for free: a TLSConfig advertising h2 makes Serve install the +// stdlib's HTTP/2 handler, and net/http then dispatches on the protocol ALPN +// negotiated during the handshake above. +func serveOne(tc *tls.Conn, r Response) error { + ln := newOneShotListener(tc) + srv := &http.Server{ + Handler: r.handler(), + ReadHeaderTimeout: readHeaderTimeout, + // A TLSConfig advertising h2 is what makes Serve install the stdlib's + // HTTP/2 handler. net/http then picks the protocol by type-asserting the + // connection to *tls.Conn and reading the negotiated ALPN — so the + // listener must hand over the *tls.Conn itself, unwrapped. + TLSConfig: &tls.Config{NextProtos: []string{"h2", "http/1.1"}}, + // net/http logs client-side noise (malformed requests, resets) that is + // expected here and would otherwise pollute the proxy's own log stream. + ErrorLog: log.New(io.Discard, "", 0), + // Completion is observed here rather than by wrapping the connection, + // which would hide its *tls.Conn type and silently downgrade h2 to h1. + ConnState: func(_ net.Conn, state http.ConnState) { + if state == http.StateClosed || state == http.StateHijacked { + ln.finish() + } + }, + } + + if err := srv.Serve(ln); err != nil && !errors.Is(err, errServed) { + return err + } + return nil +} + +// handler writes the forged response. It is shared by the HTTP/1.1 and HTTP/2 +// paths, so both produce an identical status, header set and body. +func (r Response) handler() http.Handler { + body := r.resolvedBody() + status := r.resolvedStatus() + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + h := w.Header() + h.Set("Content-Type", "text/plain; charset=utf-8") + for k, v := range r.Headers { + h.Set(textproto.CanonicalMIMEHeaderKey(k), v) + } + for _, k := range hopByHop { + h.Del(k) + } + h.Set("Content-Length", strconv.Itoa(len(body))) + // One forged response per HTTP/1.1 connection: the client must not reuse + // a connection we took over from its real dependency. Connection is + // hop-by-hop and illegal in HTTP/2, where the client owns the lifetime — + // so this is set per protocol rather than by disabling keep-alives, which + // would make the HTTP/2 server send an immediate GOAWAY. + if req.ProtoMajor == 1 { + h.Set("Connection", "close") + } + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + }) +} + +func (r Response) resolvedStatus() int { + if r.Status < 100 || r.Status > 599 { + return http.StatusServiceUnavailable + } + return r.Status +} + +// resolvedBody mirrors the cleartext injector's default one-liner, so a fault +// reads the same whether it landed on HTTP or HTTPS. +func (r Response) resolvedBody() string { + if r.Body != "" { + return r.Body + } + status := r.resolvedStatus() + reason := http.StatusText(status) + if reason == "" { + reason = "Fault Injected" + } + return fmt.Sprintf("%d %s (injected by steadybit transparent-proxy)\n", status, reason) +} + +// errServed is returned by the one-shot listener once its single connection has +// been fully served; it ends http.Server.Serve normally rather than as a fault. +var errServed = errors.New("tlsinject: connection served") + +// oneShotListener adapts a single accepted connection to net.Listener. The +// second Accept blocks until that connection is finished, so Serve returns only +// once the response has actually been delivered. The connection is handed over +// exactly as given — see the ConnState note in serveOne. +type oneShotListener struct { + mu sync.Mutex + conn net.Conn + addr net.Addr + done chan struct{} + once sync.Once +} + +func newOneShotListener(c net.Conn) *oneShotListener { + return &oneShotListener{conn: c, addr: c.LocalAddr(), done: make(chan struct{})} +} + +func (l *oneShotListener) Accept() (net.Conn, error) { + l.mu.Lock() + c := l.conn + l.conn = nil + l.mu.Unlock() + if c != nil { + return c, nil + } + <-l.done + return nil, errServed +} + +// Close is a no-op: the listener owns no resources of its own, and net/http +// closes it on the way out of Serve while the connection is still being torn +// down. +func (l *oneShotListener) Close() error { return nil } +func (l *oneShotListener) Addr() net.Addr { return l.addr } + +func (l *oneShotListener) finish() { l.once.Do(func() { close(l.done) }) } + +// replayConn prepends already-consumed bytes to a connection's read side, so +// the TLS handshake sees the ClientHello the caller peeked at. +func replayConn(c net.Conn, prefix []byte) net.Conn { + if len(prefix) == 0 { + return c + } + return &prefixedConn{Conn: c, r: io.MultiReader(bytes.NewReader(prefix), c)} +} + +type prefixedConn struct { + net.Conn + r io.Reader +} + +func (c *prefixedConn) Read(p []byte) (int, error) { return c.r.Read(p) } diff --git a/internal/tlsinject/serve_test.go b/internal/tlsinject/serve_test.go new file mode 100644 index 0000000..f551314 --- /dev/null +++ b/internal/tlsinject/serve_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package tlsinject + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" +) + +const testSNI = "api.anthropic.com" + +// serveForgedOnce accepts a single connection and injects r into it, returning +// the listener address and a channel carrying ServeForged's result. +func serveForgedOnce(t *testing.T, ca *CA, r Response) (string, <-chan error) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(context.Background(), conn, nil, r, 5*time.Second) + }() + return ln.Addr().String(), ch +} + +// clientTrusting builds a client that trusts caPEM and pins the SNI, so the +// request looks like one aimed at the dependency regardless of the dial address. +func clientTrusting(t *testing.T, caPEM []byte, h2 bool) (*http.Client, *http.Transport) { + t.Helper() + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("failed to add test CA to pool") + } + cfg := &tls.Config{RootCAs: pool, ServerName: testSNI, MinVersion: tls.VersionTLS12} + if !h2 { + cfg.NextProtos = []string{"http/1.1"} + } + tr := &http.Transport{TLSClientConfig: cfg, ForceAttemptHTTP2: h2} + t.Cleanup(tr.CloseIdleConnections) + return &http.Client{Transport: tr, Timeout: 10 * time.Second}, tr +} + +func waitServed(t *testing.T, ch <-chan error) error { + t.Helper() + select { + case err := <-ch: + return err + case <-time.After(10 * time.Second): + t.Fatal("ServeForged did not return") + return nil + } +} + +func Test_ServeForged_http1(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{ + Status: 503, + Body: `{"error":"injected"}`, + Headers: map[string]string{"Retry-After": "30", "Content-Type": "application/json"}, + }) + client, _ := clientTrusting(t, caPEM, false) + + resp, err := client.Get("https://" + addr + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + defer resp.Body.Close() + + if resp.Proto != "HTTP/1.1" { + t.Fatalf("proto = %q, want HTTP/1.1", resp.Proto) + } + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != `{"error":"injected"}` { + t.Fatalf("body = %q", body) + } + if got := resp.Header.Get("Retry-After"); got != "30" { + t.Fatalf("Retry-After = %q, want 30", got) + } + // A caller-supplied Content-Type must win over the default. + if got := resp.Header.Get("Content-Type"); got != "application/json" { + t.Fatalf("Content-Type = %q, want application/json", got) + } + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// The dependency this targets speaks HTTP/2, so serving the forged response +// over h2 is the case that matters most in practice. +func Test_ServeForged_http2(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + client, tr := clientTrusting(t, caPEM, true) + + resp, err := client.Get("https://" + addr + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0", resp.Proto) + } + if resp.StatusCode != 503 { + t.Fatalf("status = %d, want 503", resp.StatusCode) + } + if !strings.Contains(string(body), "injected by steadybit") { + t.Fatalf("default body missing marker: %q", body) + } + + // h2 keeps the connection open for further streams; releasing it lets the + // server finish. + tr.CloseIdleConnections() + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// The canonical failure mode: the workload does not trust our CA. It must be +// reported as a HandshakeError so the caller can count it and say so, rather +// than looking like a silent no-op. +func Test_ServeForged_untrustedClientYieldsHandshakeError(t *testing.T) { + ca, _ := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + + tr := &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: x509.NewCertPool(), // trusts nothing + ServerName: testSNI, + MinVersion: tls.VersionTLS12, + }} + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + + if _, err := client.Get("https://" + addr + "/v1/messages"); err == nil { + t.Fatal("expected the client to reject the injected certificate") + } + + err := waitServed(t, ch) + var hs *HandshakeError + if !errors.As(err, &hs) { + t.Fatalf("err = %v, want a *HandshakeError", err) + } +} + +func Test_ServeForged_cancelledContextClosesConnection(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(ctx, conn, nil, Response{Status: 503}, 5*time.Second) + }() + + // HTTP/2 holds the connection open after the response, so once a reply has + // been read the server is definitively parked inside net/http — exactly the + // state an attack teardown has to unblock. Driving a real request first also + // removes any race with the handshake still completing. + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, ServerName: testSNI, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + } + defer tr.CloseIdleConnections() + + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + resp, err := client.Get("https://" + ln.Addr().String() + "/") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays open", resp.Proto) + } + + cancel() + if err := waitServed(t, ch); err != nil && !errors.Is(err, net.ErrClosed) { + t.Fatalf("ServeForged after cancel: %v", err) + } +} + +func Test_Response_defaults(t *testing.T) { + // An out-of-range status falls back to 503 rather than producing an invalid + // response. + for _, status := range []int{0, 42, 700} { + r := Response{Status: status} + if got := r.resolvedStatus(); got != http.StatusServiceUnavailable { + t.Fatalf("resolvedStatus(%d) = %d, want 503", status, got) + } + } + if got := (Response{Status: 418}).resolvedStatus(); got != 418 { + t.Fatalf("resolvedStatus(418) = %d", got) + } + + body := Response{Status: 503}.resolvedBody() + if !strings.Contains(body, "503") || !strings.Contains(body, "injected by steadybit") { + t.Fatalf("default body = %q", body) + } + if got := (Response{Status: 503, Body: "custom"}).resolvedBody(); got != "custom" { + t.Fatalf("explicit body = %q", got) + } +} + +// Hop-by-hop headers are illegal in HTTP/2 and are owned by net/http in +// HTTP/1.1; a caller supplying one must not be able to corrupt the response. +func Test_Response_stripsHopByHopHeaders(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{ + Status: 503, + Headers: map[string]string{"Connection": "keep-alive", "Transfer-Encoding": "chunked", "X-Kept": "yes"}, + }) + client, _ := clientTrusting(t, caPEM, false) + + resp, err := client.Get("https://" + addr + "/") + if err != nil { + t.Fatalf("request: %v", err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("X-Kept"); got != "yes" { + t.Fatalf("X-Kept = %q, want yes", got) + } + if got := resp.Header.Get("Transfer-Encoding"); got != "" { + t.Fatalf("Transfer-Encoding leaked through: %q", got) + } + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +func Test_replayConn(t *testing.T) { + // Without a prefix the connection is handed through unwrapped. + a, b := net.Pipe() + defer func() { _ = a.Close() }() + defer func() { _ = b.Close() }() + if got := replayConn(a, nil); got != a { + t.Fatal("expected the original conn when there is no prefix") + } + + // With a prefix the consumed bytes are replayed ahead of the live stream, + // which is what lets the TLS handshake see the peeked ClientHello. + c, d := net.Pipe() + defer func() { _ = c.Close() }() + go func() { + _, _ = d.Write([]byte("world")) + _ = d.Close() + }() + got, err := io.ReadAll(replayConn(c, []byte("hello "))) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "hello world" { + t.Fatalf("read %q, want %q", got, "hello world") + } +} diff --git a/main.go b/main.go index a2b1dbe..7696549 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( "github.com/steadybit/transparent-proxy/internal/preflight" "github.com/steadybit/transparent-proxy/internal/proxy" "github.com/steadybit/transparent-proxy/internal/supervisor" + "github.com/steadybit/transparent-proxy/internal/tlsinject" ) func main() { @@ -60,6 +61,12 @@ func main() { revert = flag.Bool("revert", false, "remove the interception rules for the given --exec-id/--intercept-* and exit (out-of-band teardown, idempotent)") + // HTTPS response injection. The CA is supplied by the customer, who owns + // its validity and installs it in their workloads' truststores; the proxy + // only signs per-SNI leaves with it. Unset = TLS is never decrypted. + tlsCACert = flag.String("tls-ca-cert", "", "PEM CA certificate used to mint per-SNI certificates, enabling HTTPS response injection (requires --tls-ca-key)") + tlsCAKey = flag.String("tls-ca-key", "", "PEM private key matching --tls-ca-cert") + // Single-rule fault flags — a convenience for orchestrators that inject // one fault, avoiding a JSON --config file. Appended to any --config rules. faultLatency = flag.Duration("fault-latency", 0, "single fault: latency added before connecting upstream") @@ -92,6 +99,16 @@ func main() { logger.Info("loaded fault rules", slog.Int("count", len(rules))) } + injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey) + if err != nil { + logger.Error("invalid TLS interception CA", slog.Any("err", err)) + os.Exit(2) + } + if injector != nil { + logger.Info("HTTPS response injection enabled", + slog.Time("ca_not_after", injector.NotAfter())) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -103,6 +120,7 @@ func main() { DialTimeout: *dialTimeout, Mark: uint32(*mark), Metrics: m, + TLSInject: injector, } // The metrics endpoint runs for the whole process lifetime. @@ -197,6 +215,31 @@ func loadRules(path string) ([]fault.Rule, error) { return config.Load(path) } +// loadInterceptCA loads the optional HTTPS-interception CA. Returning (nil, nil) +// means the feature is off and TLS is never decrypted. +// +// The customer owns this CA — they generate it, choose how long it lives, and +// install it in the truststores of the workloads they want to fault. The only +// lifecycle judgement made here is refusing one that is already outside its +// validity window, because it would otherwise fail every handshake with a far +// less obvious error. +func loadInterceptCA(certPath, keyPath string) (*tlsinject.CA, error) { + if certPath == "" && keyPath == "" { + return nil, nil + } + if certPath == "" || keyPath == "" { + return nil, errors.New("--tls-ca-cert and --tls-ca-key must be set together") + } + ca, err := tlsinject.LoadCAFromFiles(certPath, keyPath) + if err != nil { + return nil, err + } + if ca.Expired(time.Now()) { + return nil, fmt.Errorf("CA is outside its validity window (not after %s); issue a new one", ca.NotAfter().Format(time.RFC3339)) + } + return ca, nil +} + // stringList is a repeatable string flag (e.g. --fault-http-header used more // than once). type stringList []string From 5fbfd634616888c3b5fd1d5dbfe4b15241e103fc Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Wed, 2 Sep 2026 14:58:01 +0200 Subject: [PATCH 2/8] fix(tlsinject): count a delivered response, not a completed handshake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Smoke-testing against a real curl/OpenSSL client showed a rejected connection being reported as a successful injection: connections_faulted and http_responses_injected both incremented while the handshake-failure counter stayed at zero, so the CA-not-trusted diagnostic never fired. Two causes. Under TLS 1.3 the server's handshake completes before the client's verdict on the certificate arrives — a client that rejects it does not fail the handshake, it abandons the connection without sending a request. And success was inferred from ServeForged returning, which happens whenever the connection closes, even if net/http never invoked the handler. Delivery is now tracked in the handler itself and is the only thing counted as a fault. HandshakeError becomes RejectedError, carrying the stage so both the TLS 1.2 (handshake) and TLS 1.3 (post-handshake) refusals are reported as what they are. Teardown via a cancelled context is excluded, since that is not the client's doing. The metric is renamed tls_intercept_rejected to match. Verified end to end under real iptables: an untrusted client now yields faulted=0, injected=0, rejected=1, and a trusted one faulted=1, injected=1. --- README.md | 2 +- internal/metrics/metrics.go | 21 +++++---- internal/proxy/server.go | 7 +-- internal/proxy/server_tlsinject_test.go | 6 +-- internal/tlsinject/serve.go | 59 +++++++++++++++++++------ internal/tlsinject/serve_test.go | 40 +++++++++++++++-- 6 files changed, 102 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 7fb560e..c223c0a 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ fabricated rather than a modified real one. - the connection lost the `probability` roll. **When the client refuses** — if the workload does not trust the CA (or pins -certificates), the handshake fails. That is counted as `tls_handshake_failures` +certificates), the handshake fails. That is counted as `tls_intercept_rejected` and is deliberately *not* counted as a fault, so a non-zero value is the signal that the CA is missing from the target's truststore rather than a silent no-op. diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index d557c8e..612b71d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -26,7 +26,7 @@ type Metrics struct { ConnectionsFaulted atomic.Int64 // connections a fault was actually applied to (once each) LatencyApplied atomic.Int64 // connections a latency fault delayed HTTPResponsesInjected atomic.Int64 // connections given a synthesized HTTP response - TLSHandshakeFailures atomic.Int64 // HTTPS interception rejected by the client (CA not trusted / pinning) + TLSInterceptRejected atomic.Int64 // HTTPS interception rejected by the client (CA not trusted / pinning) UpstreamErrors atomic.Int64 // dial failures BytesToUpstream atomic.Int64 BytesToClient atomic.Int64 @@ -64,7 +64,7 @@ type Snapshot struct { ConnectionsFaulted int64 `json:"connections_faulted"` LatencyApplied int64 `json:"latency_applied"` HTTPResponsesInjected int64 `json:"http_responses_injected"` - TLSHandshakeFailures int64 `json:"tls_handshake_failures"` + TLSInterceptRejected int64 `json:"tls_intercept_rejected"` UpstreamErrors int64 `json:"upstream_errors"` BytesToUpstream int64 `json:"bytes_to_upstream"` BytesToClient int64 `json:"bytes_to_client"` @@ -95,7 +95,7 @@ func (m *Metrics) Snapshot() Snapshot { ConnectionsFaulted: m.ConnectionsFaulted.Load(), LatencyApplied: m.LatencyApplied.Load(), HTTPResponsesInjected: m.HTTPResponsesInjected.Load(), - TLSHandshakeFailures: m.TLSHandshakeFailures.Load(), + TLSInterceptRejected: m.TLSInterceptRejected.Load(), UpstreamErrors: m.UpstreamErrors.Load(), BytesToUpstream: m.BytesToUpstream.Load(), BytesToClient: m.BytesToClient.Load(), @@ -180,13 +180,16 @@ func (m *Metrics) HTTPInjected() { } } -// TLSHandshakeFailed records an HTTPS connection the client refused to complete -// a handshake on. A non-zero count is the canonical "our CA is not trusted by -// the target (or the client pins certificates)" signal — the fault could not be -// applied, so it is deliberately not counted as faulted. -func (m *Metrics) TLSHandshakeFailed() { +// TLSRejected records an HTTPS connection on which the client refused the +// injected certificate — either by failing the handshake, or (under TLS 1.3, +// where the server's handshake completes before the client's verdict arrives) +// by abandoning the connection without ever sending a request. A non-zero count +// is the canonical "our CA is not trusted by the target, or the client pins +// certificates" signal. In both cases no response was delivered, so it is +// deliberately not counted as faulted. +func (m *Metrics) TLSRejected() { if m != nil { - m.TLSHandshakeFailures.Add(1) + m.TLSInterceptRejected.Add(1) } } diff --git a/internal/proxy/server.go b/internal/proxy/server.go index f8bba2b..c32fb5b 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -280,13 +280,14 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { Headers: action.HTTPHeaders, }, s.peekTimeout()) - var hsErr *tlsinject.HandshakeError - if errors.As(err, &hsErr) { + var rejErr *tlsinject.RejectedError + if errors.As(err, &rejErr) { // The client rejected our certificate, so the fault never applied — // counted separately from faults, never as one. This is the signal // that the CA is missing from the workload's truststore. - s.Metrics.TLSHandshakeFailed() + s.Metrics.TLSRejected() log.Warn("client rejected the injected certificate; is the CA trusted by the target?", + slog.String("stage", rejErr.Stage), slog.Any("err", err)) return } diff --git a/internal/proxy/server_tlsinject_test.go b/internal/proxy/server_tlsinject_test.go index 8985adb..96f99dd 100644 --- a/internal/proxy/server_tlsinject_test.go +++ b/internal/proxy/server_tlsinject_test.go @@ -161,8 +161,8 @@ func TestServer_TLSInject_ForgesResponse(t *testing.T) { if snap.ConnectionsFaulted != 1 { t.Fatalf("ConnectionsFaulted = %d, want 1", snap.ConnectionsFaulted) } - if snap.TLSHandshakeFailures != 0 { - t.Fatalf("TLSHandshakeFailures = %d, want 0", snap.TLSHandshakeFailures) + if snap.TLSInterceptRejected != 0 { + t.Fatalf("TLSInterceptRejected = %d, want 0", snap.TLSInterceptRejected) } if got := snap.PerHost[upstreamHost]; got.Faulted != 1 { t.Fatalf("per-host faulted = %d, want 1", got.Faulted) @@ -209,7 +209,7 @@ func TestServer_TLSInject_UntrustedClientIsCounted(t *testing.T) { t.Fatal("expected the client to reject the injected certificate") } - waitFor(t, func() bool { return m.Snapshot().TLSHandshakeFailures == 1 }, "a counted handshake failure") + waitFor(t, func() bool { return m.Snapshot().TLSInterceptRejected == 1 }, "a counted handshake failure") snap := m.Snapshot() if snap.ConnectionsFaulted != 0 { t.Fatalf("ConnectionsFaulted = %d, want 0 — the fault never applied", snap.ConnectionsFaulted) diff --git a/internal/tlsinject/serve.go b/internal/tlsinject/serve.go index 63115f5..b902a67 100644 --- a/internal/tlsinject/serve.go +++ b/internal/tlsinject/serve.go @@ -16,6 +16,7 @@ import ( "net/textproto" "strconv" "sync" + "sync/atomic" "time" ) @@ -39,17 +40,32 @@ type Response struct { Headers map[string]string } -// HandshakeError marks a failure to complete the TLS handshake with the client. -// In practice this means the minted certificate was rejected: the CA is not in -// the workload's truststore, or the client pins certificates. It is a distinct -// type so the caller can count it and surface that diagnosis rather than -// reporting a silent no-op. -type HandshakeError struct{ Err error } +// RejectedError reports that the client refused the injected certificate — +// almost always because the CA is absent from the workload's truststore, or +// because the client pins certificates. +// +// It covers two stages, because refusal is not always visible at handshake +// time. Under TLS 1.2 the handshake itself fails. Under TLS 1.3 the server +// completes its handshake before learning the client's verdict, so a rejecting +// client (OpenSSL/curl among them) instead abandons the connection without ever +// sending a request. Both mean the fault was never delivered, so both are +// reported here rather than counted as a successful injection. +type RejectedError struct { + // Stage is "handshake" or "post-handshake". + Stage string + // Err is the underlying failure; nil when the client simply went away + // without sending a request. + Err error +} -func (e *HandshakeError) Error() string { - return "tls handshake with client failed: " + e.Err.Error() +func (e *RejectedError) Error() string { + msg := "client rejected the injected certificate (" + e.Stage + ")" + if e.Err != nil { + return msg + ": " + e.Err.Error() + } + return msg + ": no request was sent" } -func (e *HandshakeError) Unwrap() error { return e.Err } +func (e *RejectedError) Unwrap() error { return e.Err } // ServeForged terminates TLS on conn using a certificate minted for the // client's SNI, then answers the request with r and closes. clientHello replays @@ -65,7 +81,7 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, hctx, cancel := context.WithTimeout(ctx, handshakeTimeout) defer cancel() if err := tc.HandshakeContext(hctx); err != nil { - return &HandshakeError{Err: err} + return &RejectedError{Stage: "handshake", Err: err} } // Cancellation must reach a connection parked inside net/http; closing it is @@ -80,7 +96,21 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, } }() - return serveOne(tc, r) + // delivered is the only trustworthy proof the fault landed: a completed + // handshake is not one, because a TLS 1.3 client reports a certificate it + // dislikes only by walking away afterwards. + var delivered atomic.Bool + if err := serveOne(tc, r, &delivered); err != nil { + return err + } + if delivered.Load() { + return nil + } + if ctx.Err() != nil { + // Torn down mid-connection: not the client's doing. + return nil + } + return &RejectedError{Stage: "post-handshake"} } // serveOne runs net/http over a single already-handshaken connection. Handing @@ -88,10 +118,10 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, // HTTP/2 support for free: a TLSConfig advertising h2 makes Serve install the // stdlib's HTTP/2 handler, and net/http then dispatches on the protocol ALPN // negotiated during the handshake above. -func serveOne(tc *tls.Conn, r Response) error { +func serveOne(tc *tls.Conn, r Response, delivered *atomic.Bool) error { ln := newOneShotListener(tc) srv := &http.Server{ - Handler: r.handler(), + Handler: r.handler(delivered), ReadHeaderTimeout: readHeaderTimeout, // A TLSConfig advertising h2 is what makes Serve install the stdlib's // HTTP/2 handler. net/http then picks the protocol by type-asserting the @@ -118,7 +148,7 @@ func serveOne(tc *tls.Conn, r Response) error { // handler writes the forged response. It is shared by the HTTP/1.1 and HTTP/2 // paths, so both produce an identical status, header set and body. -func (r Response) handler() http.Handler { +func (r Response) handler(delivered *atomic.Bool) http.Handler { body := r.resolvedBody() status := r.resolvedStatus() return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -141,6 +171,7 @@ func (r Response) handler() http.Handler { } w.WriteHeader(status) _, _ = io.WriteString(w, body) + delivered.Store(true) }) } diff --git a/internal/tlsinject/serve_test.go b/internal/tlsinject/serve_test.go index f551314..f3a8b86 100644 --- a/internal/tlsinject/serve_test.go +++ b/internal/tlsinject/serve_test.go @@ -157,9 +157,43 @@ func Test_ServeForged_untrustedClientYieldsHandshakeError(t *testing.T) { } err := waitServed(t, ch) - var hs *HandshakeError - if !errors.As(err, &hs) { - t.Fatalf("err = %v, want a *HandshakeError", err) + var rej *RejectedError + if !errors.As(err, &rej) { + t.Fatalf("err = %v, want a *RejectedError", err) + } +} + +// Under TLS 1.3 the server's handshake completes before the client reports that +// it dislikes the certificate — the client simply walks away without sending a +// request. Observed with curl/OpenSSL against a real proxy, where it made a +// rejected connection look like a successfully injected fault. A completed +// handshake is therefore not proof of delivery; an actual response is. +func Test_ServeForged_rejectedAfterHandshakeIsNotDelivery(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + addr, ch := serveForgedOnce(t, ca, Response{Status: 503}) + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + conn, err := tls.Dial("tcp", addr, &tls.Config{ + RootCAs: pool, ServerName: testSNI, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatalf("dial: %v", err) + } + // Complete the handshake, then leave without a request — exactly what a + // client that refuses the certificate does. + if err := conn.Handshake(); err != nil { + t.Fatalf("handshake: %v", err) + } + _ = conn.Close() + + err = waitServed(t, ch) + var rej *RejectedError + if !errors.As(err, &rej) { + t.Fatalf("err = %v, want a *RejectedError (nothing was ever delivered)", err) + } + if rej.Stage != "post-handshake" { + t.Fatalf("stage = %q, want post-handshake", rej.Stage) } } From a16251aa585830bac82b74fedac928893a04155c Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Wed, 2 Sep 2026 15:13:20 +0200 Subject: [PATCH 3/8] =?UTF-8?q?fix(tlsinject):=20address=20code=20review?= =?UTF-8?q?=20=E2=80=94=20report=20delivery,=20not=20connection=20end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven fixes from review of this branch. The two that matter most: Over HTTP/2 the fault counters stayed at zero for the whole attack. Delivery was inferred from ServeForged returning, but that only happens once the client goes away — and an h2 client (gRPC, pooled SDK clients: exactly the dependencies this targets) holds the connection open for the duration. A working attack therefore reported 'matched but never faulted', which is the proxy's documented silent-no-op signature. Delivery is now reported from the handler via Request.OnDelivered, the moment the response is written. Conversely, teardown was counted as a delivered fault: a cancelled context returned nil, and the caller booked an injected 503 for a connection that got nothing. OnDelivered simply never fires there. Also: a cancelled or timed-out handshake is no longer blamed on the truststore (it was bucketed as 'client rejected the certificate'); cached leaves are re-minted before they expire, and minting fails loudly once the CA has expired mid-run, instead of serving certificates every client rejects; 1xx is treated as out of range, since net/http does not commit an informational status and the body would silently commit 200; 204/304 no longer carry a body or Content-Length; a failed injection increments Dropped so matched still reconciles with the outcome counters; and the CA is loaded after the --revert branch, so teardown can never be blocked by a missing or expired CA. --- README.md | 8 +- internal/proxy/http_test.go | 6 +- internal/proxy/server.go | 32 +++++-- internal/proxy/server_tlsinject_test.go | 48 ++++++++++ internal/tlsinject/ca.go | 35 ++++++-- internal/tlsinject/serve.go | 73 +++++++++++++--- internal/tlsinject/serve_test.go | 111 +++++++++++++++++++++++- main.go | 31 ++++--- 8 files changed, 294 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index c223c0a..0bf25f3 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,11 @@ fabricated rather than a modified real one. - the connection lost the `probability` roll. **When the client refuses** — if the workload does not trust the CA (or pins -certificates), the handshake fails. That is counted as `tls_intercept_rejected` -and is deliberately *not* counted as a fault, so a non-zero value is the signal -that the CA is missing from the target's truststore rather than a silent no-op. +certificates) it either fails the handshake, or, under TLS 1.3, completes it and +then walks away without sending a request. Both are counted as +`tls_intercept_rejected` and deliberately *not* as a fault, so a non-zero value +is the signal that the CA is missing from the target's truststore rather than a +silent no-op. Only a response actually written counts as an injected fault. > Interception requires a key that can impersonate any HTTPS endpoint to > anything trusting the CA. Treat it as a test/staging capability and keep the diff --git a/internal/proxy/http_test.go b/internal/proxy/http_test.go index 70077d1..2aba52e 100644 --- a/internal/proxy/http_test.go +++ b/internal/proxy/http_test.go @@ -49,9 +49,9 @@ func Test_writeHTTPResponse_defaults(t *testing.T) { func Test_writeHTTPResponse_customBodyAndHeaders(t *testing.T) { body := `{"error":"nope"}` resp := readSynthesized(t, 429, map[string]string{ - "content-type": "application/json", // lower-case, should be canonicalized + override default - "Retry-After": "30", - "X-Fault": "injected", + "content-type": "application/json", // lower-case, should be canonicalized + override default + "Retry-After": "30", + "X-Fault": "injected", }, body) defer resp.Body.Close() diff --git a/internal/proxy/server.go b/internal/proxy/server.go index c32fb5b..c23655b 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -274,11 +274,23 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { return case proto == protoTLS && s.TLSInject != nil && identity != "": - err := s.TLSInject.ServeForged(ctx, client, prefix, tlsinject.Response{ - Status: action.HTTPStatus, - Body: action.HTTPBody, - Headers: action.HTTPHeaders, - }, s.peekTimeout()) + // Counted from the delivery callback, not after ServeForged returns: an + // HTTP/2 client keeps the connection pooled for the whole attack, so + // counting on return would report a working fault as "matched but never + // faulted" — the proxy's own silent-no-op signature. + err := s.TLSInject.ServeForged(ctx, client, prefix, tlsinject.Request{ + Response: tlsinject.Response{ + Status: action.HTTPStatus, + Body: action.HTTPBody, + Headers: action.HTTPHeaders, + }, + HandshakeTimeout: s.peekTimeout(), + OnDelivered: func() { + markFaulted() + s.Metrics.HTTPInjected() + log.Info("injected https status", slog.Int("status", action.HTTPStatus)) + }, + }) var rejErr *tlsinject.RejectedError if errors.As(err, &rejErr) { @@ -292,13 +304,15 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { return } if err != nil { + // The connection was taken over and cannot be forwarded now, so it + // ends here. Count it so matched still reconciles with the outcome + // counters instead of silently losing a connection. + s.Metrics.Dropped() log.Debug("failed to serve injected https response", slog.Any("err", err)) return } - // Recorded only once the response was actually delivered. - markFaulted() - s.Metrics.HTTPInjected() - log.Info("injected https status", slog.Int("status", action.HTTPStatus)) + // Success is reported by OnDelivered above, which fires when the + // response is written rather than when the connection ends. return } } diff --git a/internal/proxy/server_tlsinject_test.go b/internal/proxy/server_tlsinject_test.go index 96f99dd..665ed86 100644 --- a/internal/proxy/server_tlsinject_test.go +++ b/internal/proxy/server_tlsinject_test.go @@ -169,6 +169,54 @@ func TestServer_TLSInject_ForgesResponse(t *testing.T) { } } +// Regression: HTTP/2 clients pool the connection for the whole attack. The +// fault counters must reflect the injection while that connection is still +// open — otherwise a working attack reports as "matched but never faulted", +// which the platform reads as a silent no-op. +func TestServer_TLSInject_CountsWhileHTTP2ConnectionStaysOpen(t *testing.T) { + upstream, dst := startTLSUpstream(t) + ca, caPEM := newInterceptCA(t) + m := metrics.New() + + proxyAddr := serveProxy(t, &Server{Faults: httpsRule(), Metrics: m, TLSInject: ca}, dst) + + pool := x509.NewCertPool() + pool.AddCert(upstream.Certificate()) + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("failed to add intercept CA to pool") + } + tr := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", proxyAddr.String()) + }, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + } + defer tr.CloseIdleConnections() + + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + resp, err := client.Get("https://" + upstreamHost + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays pooled", resp.Proto) + } + + // Deliberately do NOT close idle connections first — that is the bug. + waitFor(t, func() bool { return m.Snapshot().ConnectionsFaulted == 1 }, + "the fault to be counted while the h2 connection is still open") + snap := m.Snapshot() + if snap.HTTPResponsesInjected != 1 { + t.Fatalf("HTTPResponsesInjected = %d, want 1", snap.HTTPResponsesInjected) + } + if got := snap.PerHost[upstreamHost]; got.Faulted != 1 { + t.Fatalf("per-host faulted = %d, want 1", got.Faulted) + } +} + // Without a CA the same rule must leave HTTPS alone — the pre-existing // behaviour, and the guarantee that enabling the feature is opt-in. func TestServer_TLSInject_DisabledPassesThrough(t *testing.T) { diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go index 77a3155..9fd32bb 100644 --- a/internal/tlsinject/ca.go +++ b/internal/tlsinject/ca.go @@ -42,6 +42,9 @@ const ( leafValidity = 24 * time.Hour // leafBackdate tolerates modest clock skew between the proxy and the client. leafBackdate = 1 * time.Hour + // leafRenewBefore re-mints a cached leaf this long before it expires, so a + // long-running proxy never serves an expired certificate. + leafRenewBefore = 1 * time.Hour // maxCachedLeaves bounds the per-SNI cache so traffic to a great many // hostnames cannot grow it without limit. Beyond the cap certificates are // still minted, just not retained. @@ -124,8 +127,11 @@ func (c *CA) ServerTLSConfig() *tls.Config { NextProtos: []string{"h2", "http/1.1"}, GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { if hello.ServerName == "" { - // Without SNI there is no name to impersonate. Failing here aborts - // the handshake; the caller forwards such connections instead. + // Without SNI there is no name to impersonate, so the handshake is + // aborted. There is no falling back to forwarding at this point: the + // ClientHello has been consumed and TLS records already written. + // Callers avoid reaching here by only interception connections whose + // SNI they already read. return nil, errors.New("client sent no SNI; cannot mint a certificate") } return c.leafFor(hello.ServerName) @@ -133,10 +139,15 @@ func (c *CA) ServerTLSConfig() *tls.Config { } } -// leafFor returns a cached certificate for host, minting one on first use. +// leafFor returns a cached certificate for host, minting one on first use and +// re-minting before the cached one expires. Without the expiry check a proxy +// outliving leafValidity would serve an expired certificate for every hostname +// it had ever seen, and every client would reject it — indistinguishable, from +// the operator's side, from the CA not being trusted. func (c *CA) leafFor(host string) (*tls.Certificate, error) { + now := time.Now() c.mu.Lock() - if cert, ok := c.cache[host]; ok { + if cert, ok := c.cache[host]; ok && now.Before(cert.Leaf.NotAfter.Add(-leafRenewBefore)) { c.mu.Unlock() return cert, nil } @@ -150,11 +161,14 @@ func (c *CA) leafFor(host string) (*tls.Certificate, error) { c.mu.Lock() defer c.mu.Unlock() // Another goroutine may have minted the same host concurrently; prefer the - // stored one so every caller for a host shares a single certificate. - if existing, ok := c.cache[host]; ok { + // stored one so every caller for a host shares a single certificate — unless + // it is the stale one we set out to replace. + if existing, ok := c.cache[host]; ok && now.Before(existing.Leaf.NotAfter.Add(-leafRenewBefore)) { return existing, nil } - if len(c.cache) < maxCachedLeaves { + // Replacing an existing (stale) entry never grows the map, so the cap only + // gates genuinely new hostnames. + if _, replacing := c.cache[host]; replacing || len(c.cache) < maxCachedLeaves { c.cache[host] = cert } return cert, nil @@ -172,6 +186,13 @@ func (c *CA) mint(host string) (*tls.Certificate, error) { if notAfter.After(c.cert.NotAfter) { notAfter = c.cert.NotAfter } + // The CA expiring mid-run is only caught here — the startup check cannot see + // it. Minting a certificate that is already expired would surface to the + // operator as "the client rejected us", pointing at the truststore instead of + // at the real cause. + if !notAfter.After(now) { + return nil, fmt.Errorf("CA expired at %s; cannot mint a certificate for %q", c.cert.NotAfter.Format(time.RFC3339), host) + } tmpl := &x509.Certificate{ SerialNumber: serial, diff --git a/internal/tlsinject/serve.go b/internal/tlsinject/serve.go index b902a67..4379564 100644 --- a/internal/tlsinject/serve.go +++ b/internal/tlsinject/serve.go @@ -67,21 +67,51 @@ func (e *RejectedError) Error() string { } func (e *RejectedError) Unwrap() error { return e.Err } +// Request is one interception: what to answer with, and how to report it. +type Request struct { + Response Response + HandshakeTimeout time.Duration + // OnDelivered is invoked the moment the first forged response is written, + // at most once per connection. + // + // Delivery must be reported from here rather than inferred from ServeForged + // returning: an HTTP/2 client pools the connection for the whole attack, so + // waiting for the connection to end would leave a fault that is demonstrably + // in effect counted as "matched but never faulted" — the proxy's canonical + // silent-no-op signature. + OnDelivered func() +} + // ServeForged terminates TLS on conn using a certificate minted for the -// client's SNI, then answers the request with r and closes. clientHello replays +// client's SNI, then answers requests with req.Response. clientHello replays // the bytes already consumed while sniffing the SNI, so the handshake sees the // original stream; pass nil when nothing was consumed. // // It blocks until the connection is finished — for HTTP/1.1 that is one // request, for HTTP/2 until the client goes away — or until ctx is cancelled, // which closes the connection so an attack teardown never leaks a goroutine. -func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, r Response, handshakeTimeout time.Duration) error { +// +// A nil return means the connection ended without the client refusing us; it is +// not a claim that anything was delivered. Use OnDelivered for that. +func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, req Request) error { tc := tls.Server(replayConn(conn, clientHello), c.ServerTLSConfig()) - hctx, cancel := context.WithTimeout(ctx, handshakeTimeout) + hctx, cancel := context.WithTimeout(ctx, req.HandshakeTimeout) defer cancel() if err := tc.HandshakeContext(hctx); err != nil { - return &RejectedError{Stage: "handshake", Err: err} + switch { + case ctx.Err() != nil: + // Teardown closed the connection mid-handshake; not the client's doing, + // and must not be blamed on the truststore. + return nil + case errors.Is(err, context.DeadlineExceeded): + // Our own deadline elapsed. A slow client or a loaded proxy is not a + // rejection, and reporting it as one sends the operator hunting for a + // truststore problem that does not exist. + return fmt.Errorf("tls handshake did not complete within %s: %w", req.HandshakeTimeout, err) + default: + return &RejectedError{Stage: "handshake", Err: err} + } } // Cancellation must reach a connection parked inside net/http; closing it is @@ -100,7 +130,7 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, // handshake is not one, because a TLS 1.3 client reports a certificate it // dislikes only by walking away afterwards. var delivered atomic.Bool - if err := serveOne(tc, r, &delivered); err != nil { + if err := serveOne(tc, req, &delivered); err != nil { return err } if delivered.Load() { @@ -118,10 +148,10 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, // HTTP/2 support for free: a TLSConfig advertising h2 makes Serve install the // stdlib's HTTP/2 handler, and net/http then dispatches on the protocol ALPN // negotiated during the handshake above. -func serveOne(tc *tls.Conn, r Response, delivered *atomic.Bool) error { +func serveOne(tc *tls.Conn, req Request, delivered *atomic.Bool) error { ln := newOneShotListener(tc) srv := &http.Server{ - Handler: r.handler(delivered), + Handler: req.handler(delivered), ReadHeaderTimeout: readHeaderTimeout, // A TLSConfig advertising h2 is what makes Serve install the stdlib's // HTTP/2 handler. net/http then picks the protocol by type-asserting the @@ -148,9 +178,15 @@ func serveOne(tc *tls.Conn, r Response, delivered *atomic.Bool) error { // handler writes the forged response. It is shared by the HTTP/1.1 and HTTP/2 // paths, so both produce an identical status, header set and body. -func (r Response) handler(delivered *atomic.Bool) http.Handler { - body := r.resolvedBody() +func (q Request) handler(delivered *atomic.Bool) http.Handler { + r := q.Response status := r.resolvedStatus() + body := r.resolvedBody() + // 204 and 304 must not carry a body; net/http would strip it and the + // advertised Content-Length would be a lie. + if status == http.StatusNoContent || status == http.StatusNotModified { + body = "" + } return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { h := w.Header() h.Set("Content-Type", "text/plain; charset=utf-8") @@ -160,7 +196,9 @@ func (r Response) handler(delivered *atomic.Bool) http.Handler { for _, k := range hopByHop { h.Del(k) } - h.Set("Content-Length", strconv.Itoa(len(body))) + if body != "" { + h.Set("Content-Length", strconv.Itoa(len(body))) + } // One forged response per HTTP/1.1 connection: the client must not reuse // a connection we took over from its real dependency. Connection is // hop-by-hop and illegal in HTTP/2, where the client owns the lifetime — @@ -170,13 +208,22 @@ func (r Response) handler(delivered *atomic.Bool) http.Handler { h.Set("Connection", "close") } w.WriteHeader(status) - _, _ = io.WriteString(w, body) - delivered.Store(true) + if body != "" { + _, _ = io.WriteString(w, body) + } + // Report at the point of writing, not when the connection ends: an HTTP/2 + // client holds the connection open for the whole attack. + if delivered.CompareAndSwap(false, true) && q.OnDelivered != nil { + q.OnDelivered() + } }) } +// resolvedStatus rejects 1xx as out of range: net/http treats an informational +// status as non-committing, so a body written after it would silently commit +// 200 instead — the client would see success where a fault was configured. func (r Response) resolvedStatus() int { - if r.Status < 100 || r.Status > 599 { + if r.Status < 200 || r.Status > 599 { return http.StatusServiceUnavailable } return r.Status diff --git a/internal/tlsinject/serve_test.go b/internal/tlsinject/serve_test.go index f3a8b86..5291882 100644 --- a/internal/tlsinject/serve_test.go +++ b/internal/tlsinject/serve_test.go @@ -12,6 +12,7 @@ import ( "net" "net/http" "strings" + "sync/atomic" "testing" "time" ) @@ -35,7 +36,7 @@ func serveForgedOnce(t *testing.T, ca *CA, r Response) (string, <-chan error) { ch <- aerr return } - ch <- ca.ServeForged(context.Background(), conn, nil, r, 5*time.Second) + ch <- ca.ServeForged(context.Background(), conn, nil, Request{Response: r, HandshakeTimeout: 5 * time.Second}) }() return ln.Addr().String(), ch } @@ -197,6 +198,112 @@ func Test_ServeForged_rejectedAfterHandshakeIsNotDelivery(t *testing.T) { } } +// Regression: an HTTP/2 client pools the connection for the whole attack, so +// delivery must be reported when the response is written — not when the +// connection ends. Reporting it late leaves a demonstrably working fault +// counted as "matched but never faulted", the proxy's silent-no-op signature. +func Test_ServeForged_reportsDeliveryBeforeConnectionCloses(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + delivered := make(chan struct{}, 1) + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(context.Background(), conn, nil, Request{ + Response: Response{Status: 503}, + HandshakeTimeout: 5 * time.Second, + OnDelivered: func() { delivered <- struct{}{} }, + }) + }() + + client, tr := clientTrusting(t, caPEM, true) + resp, err := client.Get("https://" + ln.Addr().String() + "/v1/messages") + if err != nil { + t.Fatalf("request: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.Proto != "HTTP/2.0" { + t.Fatalf("proto = %q, want HTTP/2.0 so the connection stays pooled", resp.Proto) + } + + // The connection is still open here; delivery must already be reported. + select { + case <-delivered: + case <-time.After(5 * time.Second): + t.Fatal("delivery was not reported while the h2 connection was still open") + } + select { + case err := <-ch: + t.Fatalf("ServeForged returned early (%v); the h2 connection should still be open", err) + default: + } + + tr.CloseIdleConnections() + if err := waitServed(t, ch); err != nil { + t.Fatalf("ServeForged: %v", err) + } +} + +// Teardown must never be booked as a delivered fault: nothing was written, so +// OnDelivered must not fire. +func Test_ServeForged_cancelDoesNotReportDelivery(t *testing.T) { + ca, caPEM := mustLoadTestCA(t) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + var deliveries atomic.Int64 + ch := make(chan error, 1) + go func() { + conn, aerr := ln.Accept() + if aerr != nil { + ch <- aerr + return + } + ch <- ca.ServeForged(ctx, conn, nil, Request{ + Response: Response{Status: 503}, + HandshakeTimeout: 5 * time.Second, + OnDelivered: func() { deliveries.Add(1) }, + }) + }() + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + conn, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ + RootCAs: pool, ServerName: testSNI, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12, + }) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + + cancel() + if err := waitServed(t, ch); err != nil && !errors.Is(err, net.ErrClosed) { + var rej *RejectedError + if errors.As(err, &rej) { + t.Fatalf("teardown was misreported as a client rejection: %v", err) + } + } + if got := deliveries.Load(); got != 0 { + t.Fatalf("OnDelivered fired %d times on teardown; nothing was written", got) + } +} + func Test_ServeForged_cancelledContextClosesConnection(t *testing.T) { ca, caPEM := mustLoadTestCA(t) @@ -214,7 +321,7 @@ func Test_ServeForged_cancelledContextClosesConnection(t *testing.T) { ch <- aerr return } - ch <- ca.ServeForged(ctx, conn, nil, Response{Status: 503}, 5*time.Second) + ch <- ca.ServeForged(ctx, conn, nil, Request{Response: Response{Status: 503}, HandshakeTimeout: 5 * time.Second}) }() // HTTP/2 holds the connection open after the response, so once a reply has diff --git a/main.go b/main.go index 7696549..9c7ff58 100644 --- a/main.go +++ b/main.go @@ -71,8 +71,8 @@ func main() { // one fault, avoiding a JSON --config file. Appended to any --config rules. faultLatency = flag.Duration("fault-latency", 0, "single fault: latency added before connecting upstream") faultReset = flag.Bool("fault-reset", false, "single fault: reset (RST) matching connections") - faultStatus = flag.Int("fault-http-status", 0, "single fault: injected HTTP status (L7, cleartext HTTP)") - faultBody = flag.String("fault-http-body", "", "single fault: injected HTTP response body (L7, cleartext HTTP)") + faultStatus = flag.Int("fault-http-status", 0, "single fault: injected HTTP status (L7; cleartext HTTP, plus HTTPS when --tls-ca-cert is set)") + faultBody = flag.String("fault-http-body", "", "single fault: injected HTTP response body (L7; cleartext HTTP, plus HTTPS when --tls-ca-cert is set)") faultProb = flag.Float64("fault-probability", 1, "single fault: probability [0,1] to apply the fault per connection (default 1 = always, 0 = never)") faultHosts = flag.String("fault-hosts", "", "single fault: comma-separated host selectors (SNI/Host)") faultCIDRs = flag.String("fault-cidrs", "", "single fault: comma-separated CIDR selectors") @@ -99,16 +99,6 @@ func main() { logger.Info("loaded fault rules", slog.Int("count", len(rules))) } - injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey) - if err != nil { - logger.Error("invalid TLS interception CA", slog.Any("err", err)) - os.Exit(2) - } - if injector != nil { - logger.Info("HTTPS response injection enabled", - slog.Time("ca_not_after", injector.NotAfter())) - } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -120,7 +110,6 @@ func main() { DialTimeout: *dialTimeout, Mark: uint32(*mark), Metrics: m, - TLSInject: injector, } // The metrics endpoint runs for the whole process lifetime. @@ -159,6 +148,22 @@ func main() { return } + // The CA is loaded only after the --revert branch above: teardown must never + // depend on it. An orchestrator naturally reuses the same argument vector for + // --revert, by which time the CA files may be gone or expired — refusing to + // start there would leave the interception rules installed, breaking the + // guaranteed-cleanup contract. + injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey) + if err != nil { + logger.Error("invalid TLS interception CA", slog.Any("err", err)) + os.Exit(2) + } + if injector != nil { + srv.TLSInject = injector + logger.Info("HTTPS response injection enabled", + slog.Time("ca_not_after", injector.NotAfter())) + } + // Preflight: refuse to fight an existing mesh proxy. Ports default to the // intercept ports when not given explicitly. preflightSpec := *prePorts From 36a5cdc77b51829642d7684f0cb2b9060c44e49d Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Wed, 2 Sep 2026 15:44:43 +0200 Subject: [PATCH 4/8] feat(tlsinject): accept the interception CA on stdin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Passing the CA by file path does not work for the runc backend. That backend runs the proxy in a bundle whose rootfs is an overlay of the orchestrator's "/", and an overlay does not carry the orchestrator's submounts — so a CA mounted from a Kubernetes Secret is invisible by path inside the sidecar. Verified against the real mount options: image-layer files are visible, while the Secret mount point appears as an empty directory. Left as-is this would have failed only at runtime, and failed misleadingly: every handshake would abort and be reported as "the client rejected our certificate", pointing the operator at their truststore rather than at a proxy that never had a usable CA. --tls-ca-stdin reads one PEM stream carrying both halves, in any order. It works identically for both backends, keeps the key off the command line, and never writes it to a filesystem the target could reach. The file flags stay for standalone and manual use, and the two forms are mutually exclusive. Verified under real iptables: the CA loads from stdin, a trusted client gets the forged 503 over HTTP/2, an untrusted one is counted as rejected and not as faulted, and untargeted traffic still passes through. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019wB5XrsrNJjU9MH6yegmTA --- internal/tlsinject/ca.go | 36 +++++++++++++++++++++++++++++++++++ internal/tlsinject/ca_test.go | 32 +++++++++++++++++++++++++++++++ main.go | 36 +++++++++++++++++++++++++++-------- 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go index 9fd32bb..5db0d70 100644 --- a/internal/tlsinject/ca.go +++ b/internal/tlsinject/ca.go @@ -27,11 +27,13 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "math/big" "net" "os" + "strings" "sync" "time" ) @@ -95,6 +97,40 @@ func LoadCA(certPEM, keyPEM []byte) (*CA, error) { return &CA{cert: cert, key: signer, leafKey: leafKey, cache: map[string]*tls.Certificate{}}, nil } +// LoadCACombined parses one PEM stream carrying both the CA certificate and its +// private key, in any order. +// +// This is the form the proxy accepts on stdin, which is how an orchestrator +// hands over the key without writing it to a disk the target could reach or +// exposing it on the command line. It also sidesteps a filesystem asymmetry: +// the proxy may run inside an overlay of the orchestrator's root, and an +// overlay does not carry the orchestrator's submounts — so a key mounted there +// (a Kubernetes Secret, say) would simply not be visible by path. +func LoadCACombined(pemBytes []byte) (*CA, error) { + var certPEM, keyPEM []byte + rest := pemBytes + for { + var blk *pem.Block + blk, rest = pem.Decode(rest) + if blk == nil { + break + } + switch { + case blk.Type == "CERTIFICATE": + certPEM = append(certPEM, pem.EncodeToMemory(blk)...) + case strings.Contains(blk.Type, "PRIVATE KEY"): + keyPEM = append(keyPEM, pem.EncodeToMemory(blk)...) + } + } + if len(certPEM) == 0 { + return nil, errors.New("no CERTIFICATE block in the supplied PEM") + } + if len(keyPEM) == 0 { + return nil, errors.New("no PRIVATE KEY block in the supplied PEM") + } + return LoadCA(certPEM, keyPEM) +} + // LoadCAFromFiles reads a PEM certificate and key from disk. func LoadCAFromFiles(certPath, keyPath string) (*CA, error) { certPEM, err := os.ReadFile(certPath) diff --git a/internal/tlsinject/ca_test.go b/internal/tlsinject/ca_test.go index 1a7d72f..1f7891d 100644 --- a/internal/tlsinject/ca_test.go +++ b/internal/tlsinject/ca_test.go @@ -200,3 +200,35 @@ func Test_ServerTLSConfig_requiresSNI(t *testing.T) { } } } + +func Test_LoadCACombined(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(24*time.Hour), true) + + // Order must not matter: an orchestrator concatenates whichever way round. + for _, combined := range [][]byte{ + append(append([]byte{}, certPEM...), keyPEM...), + append(append([]byte{}, keyPEM...), certPEM...), + } { + ca, err := LoadCACombined(combined) + if err != nil { + t.Fatalf("LoadCACombined: %v", err) + } + if _, err := ca.leafFor("api.anthropic.com"); err != nil { + t.Fatalf("minting from a combined PEM failed: %v", err) + } + } + + // A stream missing either half is rejected with a pointed message rather + // than failing later on every handshake. + if _, err := LoadCACombined(certPEM); err == nil || + !strings.Contains(err.Error(), "PRIVATE KEY") { + t.Fatalf("expected a missing-key error, got %v", err) + } + if _, err := LoadCACombined(keyPEM); err == nil || + !strings.Contains(err.Error(), "CERTIFICATE") { + t.Fatalf("expected a missing-certificate error, got %v", err) + } + if _, err := LoadCACombined([]byte("not pem at all")); err == nil { + t.Fatal("expected garbage to be rejected") + } +} diff --git a/main.go b/main.go index 9c7ff58..2da3986 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "errors" "flag" "fmt" + "io" "log/slog" "net" "net/http" @@ -64,8 +65,9 @@ func main() { // HTTPS response injection. The CA is supplied by the customer, who owns // its validity and installs it in their workloads' truststores; the proxy // only signs per-SNI leaves with it. Unset = TLS is never decrypted. - tlsCACert = flag.String("tls-ca-cert", "", "PEM CA certificate used to mint per-SNI certificates, enabling HTTPS response injection (requires --tls-ca-key)") - tlsCAKey = flag.String("tls-ca-key", "", "PEM private key matching --tls-ca-cert") + tlsCACert = flag.String("tls-ca-cert", "", "PEM CA certificate used to mint per-SNI certificates, enabling HTTPS response injection (requires --tls-ca-key)") + tlsCAKey = flag.String("tls-ca-key", "", "PEM private key matching --tls-ca-cert") + tlsCAStdin = flag.Bool("tls-ca-stdin", false, "read the interception CA (certificate and private key, one PEM stream) from stdin instead of from files") // Single-rule fault flags — a convenience for orchestrators that inject // one fault, avoiding a JSON --config file. Appended to any --config rules. @@ -153,7 +155,7 @@ func main() { // --revert, by which time the CA files may be gone or expired — refusing to // start there would leave the interception rules installed, breaking the // guaranteed-cleanup contract. - injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey) + injector, err := loadInterceptCA(*tlsCACert, *tlsCAKey, *tlsCAStdin) if err != nil { logger.Error("invalid TLS interception CA", slog.Any("err", err)) os.Exit(2) @@ -228,14 +230,32 @@ func loadRules(path string) ([]fault.Rule, error) { // lifecycle judgement made here is refusing one that is already outside its // validity window, because it would otherwise fail every handshake with a far // less obvious error. -func loadInterceptCA(certPath, keyPath string) (*tlsinject.CA, error) { - if certPath == "" && keyPath == "" { +func loadInterceptCA(certPath, keyPath string, fromStdin bool) (*tlsinject.CA, error) { + var ( + ca *tlsinject.CA + err error + ) + switch { + case fromStdin && (certPath != "" || keyPath != ""): + return nil, errors.New("--tls-ca-stdin cannot be combined with --tls-ca-cert/--tls-ca-key") + case fromStdin: + // Reading the key from stdin keeps it off the command line and off any + // filesystem the target could reach. It is also the only channel that + // works uniformly: the proxy may run inside an overlay of the + // orchestrator's root, which does not carry the orchestrator's submounts, + // so a key mounted there is invisible by path. + pemBytes, rerr := io.ReadAll(os.Stdin) + if rerr != nil { + return nil, fmt.Errorf("failed to read CA from stdin: %w", rerr) + } + ca, err = tlsinject.LoadCACombined(pemBytes) + case certPath == "" && keyPath == "": return nil, nil - } - if certPath == "" || keyPath == "" { + case certPath == "" || keyPath == "": return nil, errors.New("--tls-ca-cert and --tls-ca-key must be set together") + default: + ca, err = tlsinject.LoadCAFromFiles(certPath, keyPath) } - ca, err := tlsinject.LoadCAFromFiles(certPath, keyPath) if err != nil { return nil, err } From 5e8b95be6ad251504ae8f2d9182146b5d92ec6c0 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Fri, 4 Sep 2026 10:23:24 +0200 Subject: [PATCH 5/8] fix(tlsinject): stop misreporting our own failures as client rejections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six review findings, most of them variations on one theme: an error that is ours getting reported as the client's. The mid-run CA-expiry check added last commit exists precisely to avoid misdiagnosis, and was then undone by the classification above it — every non-context handshake error became a RejectedError, so an expired CA logged "is the CA trusted by the target?" and sent the operator to inspect a truststore that was fine. Certificate production failures now carry a CertError and are reported as themselves. The same applied to a ClientHello with no SNI. ServeForged returned nil for two non-delivery cases, so those connections were counted in matched and in nothing else; at teardown with many connections in flight, a whole batch could vanish from the outcome counters. Teardown now returns ErrNotDelivered and the caller records it. Delivery was read rather than claimed. An HTTP/2 handler can still be finishing as ServeConn returns, so a plain read could see "not delivered", have the caller count a rejection, and then have the straggler fire OnDelivered — one connection booked into two mutually exclusive buckets. A CAS closes that window. The stdin read was unbounded and untimed, and runs before the listener is bound, before preflight, and before the deadman is armed: a writer that never closed the pipe would hang the proxy forever with nothing installed. Now capped at 1 MiB with a 30s deadline. Also: cached leaves were compared against a fixed renew window while their validity is clamped to the CA's, so a CA with under an hour left made every cached leaf permanently stale and re-signed on every handshake; the window is now clamped to half the remaining validity. A passphrase-protected key is refused with a pointed message instead of an opaque parse failure later. The cleartext path normalises status the same way as the HTTPS path, so one rule no longer produces different bytes depending on the dependency's protocol. And --tls-ca-stdin is documented. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019wB5XrsrNJjU9MH6yegmTA --- README.md | 18 +++++++++++ internal/proxy/http.go | 16 +++++++-- internal/proxy/server.go | 11 +++++-- internal/tlsinject/ca.go | 42 +++++++++++++++++++++--- internal/tlsinject/ca_test.go | 61 +++++++++++++++++++++++++++++++++++ internal/tlsinject/serve.go | 28 ++++++++++++++-- main.go | 19 ++++++++++- 7 files changed, 181 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0bf25f3..496d180 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,24 @@ transparent-proxy \ --fault-hosts api.stripe.com --fault-http-status 503 ``` +`--tls-ca-stdin` reads the same CA as **one PEM stream on stdin** (certificate +and key, either order) instead of from files. This is what an orchestrator +should use: it keeps the key off the command line, off any disk the target could +reach, and it is the only channel that works when the proxy runs inside an +overlay of the orchestrator's filesystem — an overlay does not carry the +orchestrator's submounts, so a key mounted there (a Kubernetes Secret, say) is +not visible by path. + +```bash +cat intercept-ca.crt intercept-ca.key | + transparent-proxy --tls-ca-stdin \ + --fault-hosts api.stripe.com --fault-http-status 503 +``` + +**The caller must close stdin.** The read is capped at 1 MiB and bounded by a +30s deadline, so a writer that never closes fails loudly rather than hanging the +proxy before it installs any rules. The key must not be passphrase-protected. + The proxy mints a short-lived certificate for the connection's SNI, signed by that CA, and answers the request itself. **HTTP/1.1 and HTTP/2 are both supported** — the response is delivered over whichever the client negotiates diff --git a/internal/proxy/http.go b/internal/proxy/http.go index 5d10a9e..732daef 100644 --- a/internal/proxy/http.go +++ b/internal/proxy/http.go @@ -133,20 +133,32 @@ func isHTTPMethodStart(b byte) bool { // Content-Type overrides the default); Content-Length and Connection are always // set by the proxy so they stay correct and the connection closes cleanly. func writeHTTPResponse(c net.Conn, status int, headers map[string]string, body string) error { + // Normalised the same way as the HTTPS path, so one rule produces the same + // response whether the matched dependency happened to be HTTP or HTTPS. 1xx + // is informational and cannot carry a fault; 204/304 must not carry a body. + if status < 200 || status > 599 { + status = http.StatusServiceUnavailable + } + noBody := status == http.StatusNoContent || status == http.StatusNotModified reason := http.StatusText(status) if reason == "" { reason = "Fault Injected" } - if body == "" { + if body == "" && !noBody { body = fmt.Sprintf("%d %s (injected by steadybit transparent-proxy)\n", status, reason) } + if noBody { + body = "" + } h := map[string]string{"Content-Type": "text/plain; charset=utf-8"} for k, v := range headers { h[textproto.CanonicalMIMEHeaderKey(k)] = v } // Proxy-owned headers: keep the framing correct regardless of caller input. - h["Content-Length"] = strconv.Itoa(len(body)) + if !noBody { + h["Content-Length"] = strconv.Itoa(len(body)) + } h["Connection"] = "close" var sb strings.Builder diff --git a/internal/proxy/server.go b/internal/proxy/server.go index c23655b..a0af6f8 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -305,10 +305,15 @@ func (s *Server) handle(ctx context.Context, client *net.TCPConn) { } if err != nil { // The connection was taken over and cannot be forwarded now, so it - // ends here. Count it so matched still reconciles with the outcome - // counters instead of silently losing a connection. + // ends here either way. Count it so matched still reconciles with the + // outcome counters instead of silently losing a connection. s.Metrics.Dropped() - log.Debug("failed to serve injected https response", slog.Any("err", err)) + if errors.Is(err, tlsinject.ErrNotDelivered) { + // Teardown while the connection was open: expected, not a failure. + log.Debug("interception ended without delivering a response", slog.Any("err", err)) + } else { + log.Warn("failed to serve injected https response", slog.Any("err", err)) + } return } // Success is reported by OnDelivered above, which fires when the diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go index 5db0d70..15303e5 100644 --- a/internal/tlsinject/ca.go +++ b/internal/tlsinject/ca.go @@ -53,6 +53,16 @@ const ( maxCachedLeaves = 1024 ) +// CertError marks a failure to produce a certificate at all — an expired CA, a +// signing failure, a ClientHello with no SNI. It is our side failing, not the +// client refusing us, and the two must not be conflated: reporting a dead CA as +// "the client rejected the certificate" sends the operator to inspect a +// truststore that is perfectly fine. +type CertError struct{ Err error } + +func (e *CertError) Error() string { return "could not mint a certificate: " + e.Err.Error() } +func (e *CertError) Unwrap() error { return e.Err } + // CA mints per-SNI leaf certificates from a customer-supplied authority. // It is safe for concurrent use. type CA struct { @@ -118,6 +128,10 @@ func LoadCACombined(pemBytes []byte) (*CA, error) { switch { case blk.Type == "CERTIFICATE": certPEM = append(certPEM, pem.EncodeToMemory(blk)...) + case strings.Contains(blk.Type, "ENCRYPTED PRIVATE KEY"): + // Accepting it here would fail later inside X509KeyPair with an opaque + // parse error, hiding the actual problem. + return nil, errors.New("the CA key is passphrase-protected; supply an unencrypted PRIVATE KEY block") case strings.Contains(blk.Type, "PRIVATE KEY"): keyPEM = append(keyPEM, pem.EncodeToMemory(blk)...) } @@ -166,11 +180,15 @@ func (c *CA) ServerTLSConfig() *tls.Config { // Without SNI there is no name to impersonate, so the handshake is // aborted. There is no falling back to forwarding at this point: the // ClientHello has been consumed and TLS records already written. - // Callers avoid reaching here by only interception connections whose + // Callers avoid reaching here by only intercepting connections whose // SNI they already read. - return nil, errors.New("client sent no SNI; cannot mint a certificate") + return nil, &CertError{Err: errors.New("client sent no SNI")} } - return c.leafFor(hello.ServerName) + cert, err := c.leafFor(hello.ServerName) + if err != nil { + return nil, &CertError{Err: err} + } + return cert, nil }, } } @@ -183,7 +201,7 @@ func (c *CA) ServerTLSConfig() *tls.Config { func (c *CA) leafFor(host string) (*tls.Certificate, error) { now := time.Now() c.mu.Lock() - if cert, ok := c.cache[host]; ok && now.Before(cert.Leaf.NotAfter.Add(-leafRenewBefore)) { + if cert, ok := c.cache[host]; ok && fresh(cert, now) { c.mu.Unlock() return cert, nil } @@ -199,7 +217,7 @@ func (c *CA) leafFor(host string) (*tls.Certificate, error) { // Another goroutine may have minted the same host concurrently; prefer the // stored one so every caller for a host shares a single certificate — unless // it is the stale one we set out to replace. - if existing, ok := c.cache[host]; ok && now.Before(existing.Leaf.NotAfter.Add(-leafRenewBefore)) { + if existing, ok := c.cache[host]; ok && fresh(existing, now) { return existing, nil } // Replacing an existing (stale) entry never grows the map, so the cap only @@ -210,6 +228,20 @@ func (c *CA) leafFor(host string) (*tls.Certificate, error) { return cert, nil } +// fresh reports whether a cached leaf is far enough from expiry to keep using. +// +// The renew window is clamped to half the remaining validity: leaves are capped +// at the CA's own NotAfter, so once the CA has less than leafRenewBefore left, +// a fixed window would mark every cached leaf permanently stale and re-sign on +// every single handshake. +func fresh(cert *tls.Certificate, now time.Time) bool { + renew := leafRenewBefore + if remaining := cert.Leaf.NotAfter.Sub(now); remaining/2 < renew { + renew = remaining / 2 + } + return now.Before(cert.Leaf.NotAfter.Add(-renew)) +} + func (c *CA) mint(host string) (*tls.Certificate, error) { serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) if err != nil { diff --git a/internal/tlsinject/ca_test.go b/internal/tlsinject/ca_test.go index 1f7891d..3581d42 100644 --- a/internal/tlsinject/ca_test.go +++ b/internal/tlsinject/ca_test.go @@ -232,3 +232,64 @@ func Test_LoadCACombined(t *testing.T) { t.Fatal("expected garbage to be rejected") } } + +// A CA that expires mid-run must be reported as our failure, not as the client +// rejecting us — otherwise the operator is sent to inspect a truststore that is +// perfectly fine. This is the exact misdiagnosis the expiry check exists to +// prevent, so it must not be undone by the error classification above it. +func Test_mint_expiredCAIsCertErrorNotRejection(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(2*time.Second), true) + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + // Force the CA past its NotAfter without waiting. + ca.cert.NotAfter = time.Now().Add(-time.Minute) + + _, err = ca.ServerTLSConfig().GetCertificate(&tls.ClientHelloInfo{ServerName: "api.example.com"}) + if err == nil { + t.Fatal("expected minting to fail once the CA has expired") + } + if !isCertError(err) { + t.Fatalf("err = %v, want a *CertError so it is not misreported as a client rejection", err) + } +} + +func Test_ServerTLSConfig_noSNIIsCertError(t *testing.T) { + ca, _ := mustLoadTestCA(t) + _, err := ca.ServerTLSConfig().GetCertificate(&tls.ClientHelloInfo{}) + if !isCertError(err) { + t.Fatalf("err = %v, want a *CertError", err) + } +} + +// With a fixed renew window, a CA with less than that window left would mark +// every cached leaf permanently stale — turning each handshake into a fresh +// signature under the mutex. +func Test_leafFor_cachesEvenWhenCANearExpiry(t *testing.T) { + certPEM, keyPEM := newTestCAPEM(t, time.Now().Add(30*time.Minute), true) // < leafRenewBefore + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + first, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + second, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + if first != second { + t.Fatal("cache thrashing: re-minted despite a still-valid leaf") + } +} + +func Test_LoadCACombined_rejectsEncryptedKey(t *testing.T) { + certPEM, _ := newTestCAPEM(t, time.Now().Add(time.Hour), true) + enc := pem.EncodeToMemory(&pem.Block{Type: "ENCRYPTED PRIVATE KEY", Bytes: []byte("nope")}) + _, err := LoadCACombined(append(append([]byte{}, certPEM...), enc...)) + if err == nil || !strings.Contains(err.Error(), "passphrase-protected") { + t.Fatalf("err = %v, want a pointed passphrase-protected message", err) + } +} diff --git a/internal/tlsinject/serve.go b/internal/tlsinject/serve.go index 4379564..fccf8bd 100644 --- a/internal/tlsinject/serve.go +++ b/internal/tlsinject/serve.go @@ -109,6 +109,11 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, // rejection, and reporting it as one sends the operator hunting for a // truststore problem that does not exist. return fmt.Errorf("tls handshake did not complete within %s: %w", req.HandshakeTimeout, err) + case isCertError(err): + // We could not produce a certificate (expired CA, signing failure, no + // SNI). Blaming the client's truststore here would point the operator at + // the one thing that is not broken. + return err default: return &RejectedError{Stage: "handshake", Err: err} } @@ -133,16 +138,33 @@ func (c *CA) ServeForged(ctx context.Context, conn net.Conn, clientHello []byte, if err := serveOne(tc, req, &delivered); err != nil { return err } - if delivered.Load() { - return nil + // Claim the flag rather than reading it. An HTTP/2 handler goroutine can + // still be finishing as ServeConn returns, so a plain read could see "not + // delivered", have the caller count a rejection, and then have the straggler + // fire OnDelivered — booking one connection into two mutually exclusive + // buckets. Winning this CAS means nothing was delivered and nothing can be. + if !delivered.CompareAndSwap(false, true) { + return nil // delivered; OnDelivered has fired or is firing } if ctx.Err() != nil { // Torn down mid-connection: not the client's doing. - return nil + return ErrNotDelivered } return &RejectedError{Stage: "post-handshake"} } +// ErrNotDelivered reports a connection that ended without a response and +// without the client refusing us — teardown, essentially. It is returned so the +// caller can account for the connection instead of silently losing it from the +// outcome counters. +var ErrNotDelivered = errors.New("tlsinject: connection ended without delivering a response") + +// isCertError reports whether err came from our own certificate production. +func isCertError(err error) bool { + var ce *CertError + return errors.As(err, &ce) +} + // serveOne runs net/http over a single already-handshaken connection. Handing // it to http.Server (rather than writing the response by hand) is what gets // HTTP/2 support for free: a TLSConfig advertising h2 makes Serve install the diff --git a/main.go b/main.go index 2da3986..0c5b211 100644 --- a/main.go +++ b/main.go @@ -215,6 +215,14 @@ func main() { logger.Info("proxy stopped", slog.Any("metrics", m.Snapshot())) } +const ( + // maxCAStdinBytes caps the CA read from stdin; a PEM pair is a few KB. + maxCAStdinBytes = 1 << 20 + // caStdinTimeout bounds that read, so a writer that never closes the pipe + // fails loudly instead of hanging the proxy before it installs anything. + caStdinTimeout = 30 * time.Second +) + func loadRules(path string) ([]fault.Rule, error) { if path == "" { return nil, nil @@ -244,10 +252,19 @@ func loadInterceptCA(certPath, keyPath string, fromStdin bool) (*tlsinject.CA, e // works uniformly: the proxy may run inside an overlay of the // orchestrator's root, which does not carry the orchestrator's submounts, // so a key mounted there is invisible by path. - pemBytes, rerr := io.ReadAll(os.Stdin) + // Bounded and deadlined: this read happens before the listener is bound, + // before preflight, and before the --max-duration deadman is armed, so a + // writer that never closes the pipe would hang the proxy forever with no + // rules installed and no way out. + _ = os.Stdin.SetReadDeadline(time.Now().Add(caStdinTimeout)) + pemBytes, rerr := io.ReadAll(io.LimitReader(os.Stdin, maxCAStdinBytes+1)) + _ = os.Stdin.SetReadDeadline(time.Time{}) if rerr != nil { return nil, fmt.Errorf("failed to read CA from stdin: %w", rerr) } + if len(pemBytes) > maxCAStdinBytes { + return nil, fmt.Errorf("CA on stdin exceeds %d bytes", maxCAStdinBytes) + } ca, err = tlsinject.LoadCACombined(pemBytes) case certPath == "" && keyPath == "": return nil, nil From 129fcfe031e8cf253fa72e20ea0443422f21dc77 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Fri, 4 Sep 2026 13:54:44 +0200 Subject: [PATCH 6/8] feat(tlsinject): make minted leaf validity configurable --tls-leaf-validity sets how long the per-SNI certificates the proxy mints stay valid, so an operator can narrow the window in which a leaf that escaped the proxy would be usable. Unset keeps the previous 24h. Values below twice the renew window are raised to it: a leaf shorter than the window is stale the moment it is issued, which would re-sign on every single handshake. The value is still clamped to the CA's own expiry, so a leaf never outlives its issuer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019wB5XrsrNJjU9MH6yegmTA --- internal/tlsinject/ca.go | 39 ++++++++++++++++++++++++++++++++++----- main.go | 2 ++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go index 15303e5..4cae6df 100644 --- a/internal/tlsinject/ca.go +++ b/internal/tlsinject/ca.go @@ -39,9 +39,10 @@ import ( ) const ( - // leafValidity bounds a minted leaf's lifetime. It is additionally clamped - // to the CA's own NotAfter, so a leaf never outlives its issuer. - leafValidity = 24 * time.Hour + // defaultLeafValidity bounds a minted leaf's lifetime when the caller does + // not choose one. It is additionally clamped to the CA's own NotAfter, so a + // leaf never outlives its issuer. + defaultLeafValidity = 24 * time.Hour // leafBackdate tolerates modest clock skew between the proxy and the client. leafBackdate = 1 * time.Hour // leafRenewBefore re-mints a cached leaf this long before it expires, so a @@ -74,10 +75,32 @@ type CA struct { // keypair. It never leaves this process. leafKey *ecdsa.PrivateKey + // leafValidity is how long a minted leaf is valid for. Shorter values limit + // the window in which a leaf that escaped the proxy would be usable, at the + // cost of re-signing more often. + leafValidity time.Duration + mu sync.Mutex cache map[string]*tls.Certificate } +// SetLeafValidity overrides how long minted leaves are valid. Values below the +// renew window are raised to it, since a leaf shorter than that would be +// considered stale the moment it is issued and re-signed on every handshake. +func (c *CA) SetLeafValidity(d time.Duration) { + if d <= 0 { + return + } + if d < 2*leafRenewBefore { + d = 2 * leafRenewBefore + } + c.mu.Lock() + c.leafValidity = d + // Existing leaves were cut to the old window; drop them so the new one applies. + c.cache = map[string]*tls.Certificate{} + c.mu.Unlock() +} + // LoadCA parses a PEM certificate and matching private key. The certificate // must be a signing CA; anything else is rejected up front rather than failing // later on every handshake. @@ -104,7 +127,7 @@ func LoadCA(certPEM, keyPEM []byte) (*CA, error) { if err != nil { return nil, fmt.Errorf("failed to generate leaf key: %w", err) } - return &CA{cert: cert, key: signer, leafKey: leafKey, cache: map[string]*tls.Certificate{}}, nil + return &CA{cert: cert, key: signer, leafKey: leafKey, leafValidity: defaultLeafValidity, cache: map[string]*tls.Certificate{}}, nil } // LoadCACombined parses one PEM stream carrying both the CA certificate and its @@ -249,7 +272,13 @@ func (c *CA) mint(host string) (*tls.Certificate, error) { } now := time.Now() - notAfter := now.Add(leafValidity) + c.mu.Lock() + validity := c.leafValidity + c.mu.Unlock() + if validity <= 0 { + validity = defaultLeafValidity + } + notAfter := now.Add(validity) // A leaf must never outlive the CA that signed it. if notAfter.After(c.cert.NotAfter) { notAfter = c.cert.NotAfter diff --git a/main.go b/main.go index 0c5b211..dc51ba2 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ func main() { // only signs per-SNI leaves with it. Unset = TLS is never decrypted. tlsCACert = flag.String("tls-ca-cert", "", "PEM CA certificate used to mint per-SNI certificates, enabling HTTPS response injection (requires --tls-ca-key)") tlsCAKey = flag.String("tls-ca-key", "", "PEM private key matching --tls-ca-cert") + tlsLeafTTL = flag.Duration("tls-leaf-validity", 0, "how long minted per-SNI certificates are valid (0 = built-in default; always clamped to the CA's own expiry)") tlsCAStdin = flag.Bool("tls-ca-stdin", false, "read the interception CA (certificate and private key, one PEM stream) from stdin instead of from files") // Single-rule fault flags — a convenience for orchestrators that inject @@ -161,6 +162,7 @@ func main() { os.Exit(2) } if injector != nil { + injector.SetLeafValidity(*tlsLeafTTL) srv.TLSInject = injector logger.Info("HTTPS response injection enabled", slog.Time("ca_not_after", injector.NotAfter())) From c93f4b81eb96371bf426d71c648ff6840f201898 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Fri, 4 Sep 2026 14:04:17 +0200 Subject: [PATCH 7/8] fix(interception): scope the connection-pool flush to the targeted hostnames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capture filter is deliberately broad — typically 0.0.0.0/0 on ports 80 and 443 — because the proxy decides what to fault by hostname once it has seen the request. The flush cannot do that: it is a stateless iptables REJECT that knows only addresses. Scoping it to the capture filter therefore reset every established HTTP/HTTPS connection in the target, including ones to dependencies the attack never named. Starting a slow-dependency attack on one hostname briefly severed everything else the workload was talking to. The targeted hostnames are now resolved up front, inside the target's network namespace so the answers match what the workload sees, and only those addresses are flushed. Capture stays broad, since hostname matching still needs it. This is a snapshot: a dependency behind rotating IPs may hold a connection to an address that no longer resolves, and that one is not reset — it is still faulted when it next reconnects. Under-flushing is the right side to err on. A hostname that fails to resolve warns rather than failing the attack, and an attack that targets CIDRs rather than hostnames keeps the previous filter-wide flush, which is what it actually wants. Verified under real iptables: the flush chain names only the two addresses example.com resolved to, while the REDIRECT chain still captures 0.0.0.0/0. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019wB5XrsrNJjU9MH6yegmTA --- internal/interception/rules.go | 21 +++++++++- internal/interception/rules_test.go | 50 ++++++++++++++++++++++ main.go | 65 +++++++++++++++++++++++++---- 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/internal/interception/rules.go b/internal/interception/rules.go index 4087643..c871115 100644 --- a/internal/interception/rules.go +++ b/internal/interception/rules.go @@ -50,6 +50,21 @@ type Config struct { // reset so they re-establish through the proxy and immediately feel the // fault; set it when only new connections should be affected. SkipFlush bool + // FlushDestinations narrows the connection-pool flush to these destinations + // instead of the whole capture filter. + // + // The capture filter is deliberately broad — typically 0.0.0.0/0 on ports + // 80 and 443 — because the proxy decides what to fault by hostname once it + // has seen the request. The flush cannot do that: it is a stateless iptables + // REJECT and knows only addresses. Applying it to the capture filter would + // therefore reset every established HTTP/HTTPS connection in the target, + // including ones to dependencies the attack never names. Resolving the + // targeted hostnames up front and flushing only those addresses keeps the + // collateral to the dependency actually under test. + // + // Empty keeps the old behaviour and flushes the whole capture filter, which + // is what a CIDR-targeted attack (no hostnames) actually wants. + FlushDestinations []netip.Prefix } func (c Config) mark() uint32 { @@ -129,7 +144,11 @@ func (c Config) AddScript() []string { for _, ex := range excludes { s = append(s, fmt.Sprintf("-A %s -d %s -j RETURN", flush, ex)) } - for _, in := range includes { + flushDsts := includes + if len(c.FlushDestinations) > 0 { + flushDsts = includeV4(c.FlushDestinations) + } + for _, in := range flushDsts { for _, p := range c.Filter.Ports { s = append(s, fmt.Sprintf("-A %s -p tcp -d %s --dport %d -m conntrack --ctstate ESTABLISHED -j REJECT --reject-with tcp-reset", flush, in, p)) } diff --git a/internal/interception/rules_test.go b/internal/interception/rules_test.go index 6e00d06..9703055 100644 --- a/internal/interception/rules_test.go +++ b/internal/interception/rules_test.go @@ -269,3 +269,53 @@ func TestRevert_FailsVerificationWhenRulesRemain(t *testing.T) { t.Fatal("Revert must fail verification while a chain still contains rules") } } + +// The capture filter is deliberately broad (0.0.0.0/0 on 80/443) because the +// proxy picks its victims by hostname. The flush cannot — it is a stateless +// REJECT that knows only addresses — so scoping it to the capture filter would +// reset every established HTTP/HTTPS connection in the target, not just the +// dependency under test. +func Test_flushIsScopedToResolvedDestinations(t *testing.T) { + base := Config{ + ExecutionID: "exec", + ProxyPort: 3128, + Filter: Filter{ + Include: []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, + Ports: []uint16{80, 443}, + }, + } + + // Without resolved destinations the flush covers the whole capture filter — + // what a CIDR-targeted attack wants. + broad := strings.Join(base.AddScript(), "\n") + if !strings.Contains(broad, "-d 0.0.0.0/0 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT") { + t.Fatalf("expected a filter-wide flush without resolved hosts:\n%s", broad) + } + + scoped := base + scoped.FlushDestinations = []netip.Prefix{ + netip.MustParsePrefix("93.184.216.34/32"), + netip.MustParsePrefix("1.2.3.4/32"), + } + got := strings.Join(scoped.AddScript(), "\n") + + for _, want := range []string{ + "-d 93.184.216.34/32 --dport 80 -m conntrack --ctstate ESTABLISHED -j REJECT", + "-d 93.184.216.34/32 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT", + "-d 1.2.3.4/32 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT", + } { + if !strings.Contains(got, want) { + t.Fatalf("missing scoped flush rule %q in:\n%s", want, got) + } + } + // The whole point: nothing else on those ports is reset. + if strings.Contains(got, "-d 0.0.0.0/0 --dport 443 -m conntrack --ctstate ESTABLISHED -j REJECT") { + t.Fatalf("flush still resets the entire capture filter:\n%s", got) + } + // Capture itself must stay broad — the proxy still needs to see everything + // so it can match by hostname. + if !strings.Contains(got, "-d 0.0.0.0/0 -p tcp -m tcp --dport 443 -j REDIRECT") && + !strings.Contains(got, "0.0.0.0/0") { + t.Fatalf("capture filter was narrowed too:\n%s", got) + } +} diff --git a/main.go b/main.go index dc51ba2..eeb731a 100644 --- a/main.go +++ b/main.go @@ -129,7 +129,7 @@ func main() { // Validate the interception filter and whether self-managed mode is wanted. // The port is filled in after we bind (below); 0 is fine here because this // instance is only used for --revert, where the port is irrelevant. - interceptor, wantIntercept, err := buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), 0, *noFlush) + interceptor, wantIntercept, err := buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), 0, *noFlush, nil) if err != nil { logger.Error("invalid interception configuration", slog.Any("err", err)) os.Exit(2) @@ -191,7 +191,7 @@ func main() { os.Exit(1) } port := uint16(ln.Addr().(*net.TCPAddr).Port) - interceptor, _, err = buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), port, *noFlush) + interceptor, _, err = buildInterceptor(*interceptCIDRs, *interceptPorts, *excludeCIDRs, *execID, uint32(*mark), port, *noFlush, flushDestinations(rules, logger)) if err != nil { logger.Error("invalid interception configuration", slog.Any("err", err)) os.Exit(2) @@ -225,6 +225,56 @@ const ( caStdinTimeout = 30 * time.Second ) +// flushDestinations resolves the hostnames the fault rules target, so the +// connection-pool flush can be narrowed to those addresses. +// +// This runs inside the target's network namespace, so it sees the same routes +// the workload does. It is a snapshot: a dependency behind rotating IPs may +// hold connections to an address that no longer resolves, and those simply are +// not flushed — they still get faulted when they next reconnect. Under-flushing +// is the right side to err on, since over-flushing resets connections to +// dependencies the attack never named. +// +// Returns nil when no rule targets a hostname, which leaves the flush scoped to +// the capture filter — what a CIDR-targeted attack actually wants. +func flushDestinations(rules []fault.Rule, logger *slog.Logger) []netip.Prefix { + var hosts []string + for _, r := range rules { + hosts = append(hosts, r.Hosts...) + } + if len(hosts) == 0 { + return nil + } + + seen := map[netip.Prefix]bool{} + var out []netip.Prefix + for _, h := range hosts { + addrs, err := net.LookupHost(h) + if err != nil { + // Not fatal: the attack still applies to new connections. Warn, + // because the operator asked for existing ones to be reset too. + logger.Warn("could not resolve a targeted dependency; its existing connections will not be reset", + slog.String("host", h), slog.Any("err", err)) + continue + } + for _, a := range addrs { + addr, perr := netip.ParseAddr(a) + if perr != nil || !addr.Is4() { + continue // interception is IPv4-only + } + pfx := netip.PrefixFrom(addr, 32) + if !seen[pfx] { + seen[pfx] = true + out = append(out, pfx) + } + } + } + if len(out) == 0 { + logger.Warn("no targeted dependency resolved to an IPv4 address; existing connections will not be reset") + } + return out +} + func loadRules(path string) ([]fault.Rule, error) { if path == "" { return nil, nil @@ -355,7 +405,7 @@ func (a interceptorAdapter) Revert(ctx context.Context) error { return a.cfg.Rev // targets the exact port the proxy bound — there is no pre-allocated port that // another process could steal between allocation and bind. For --revert the // port is irrelevant (chain names derive from the exec-id) and may be 0. -func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyPort uint16, noFlush bool) (supervisor.Interceptor, bool, error) { +func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyPort uint16, noFlush bool, flushDsts []netip.Prefix) (supervisor.Interceptor, bool, error) { if cidrs == "" && ports == "" { return nil, false, nil } @@ -377,10 +427,11 @@ func buildInterceptor(cidrs, ports, excludes, execID string, mark uint32, proxyP } cfg := interception.Config{ - ExecutionID: execID, - ProxyPort: proxyPort, - Mark: mark, - SkipFlush: noFlush, + ExecutionID: execID, + ProxyPort: proxyPort, + Mark: mark, + SkipFlush: noFlush, + FlushDestinations: flushDsts, Filter: interception.Filter{ Include: include, Exclude: exclude, From 38c396938b085fab623962bfdb768717ff5ce981 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Fri, 4 Sep 2026 14:12:51 +0200 Subject: [PATCH 8/8] feat(tlsinject): present the whole supplied chain, so an intermediate CA works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the signing certificate was presented alongside the minted leaf, so a certificate authority that is itself issued by something else left the client unable to build a path. That forced the operator towards handing over a root key, which is the last thing anyone should be asked for. The full supplied bundle is now presented, signing certificate first. An operator can issue a short-lived intermediate from their own PKI, keep the root offline, and hand the proxy only the intermediate and its key — the workloads already trust the root, and the chain resolves. Constraining that intermediate with nameConstraints then bounds what it can impersonate at all. Covered by a test that verifies a client trusting only the root against a leaf minted by an intermediate. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019wB5XrsrNJjU9MH6yegmTA --- README.md | 10 ++++- internal/tlsinject/ca.go | 26 +++++++++++-- internal/tlsinject/ca_test.go | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 496d180..472a267 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,14 @@ that CA, and answers the request itself. **HTTP/1.1 and HTTP/2 are both supported** — the response is delivered over whichever the client negotiates via ALPN. -**The CA is yours.** You generate it, choose how long it lives, and install it -in the truststores of the workloads you want to fault. The proxy only signs with +**The CA is yours, and it need not be a root.** An intermediate issued by your +own PKI works and is the better choice: you keep the root key offline, the +workloads already trust the root, and the proxy presents the intermediate so +the chain still builds. Constrain it further with `nameConstraints` if you want +it usable only for the dependencies under test. + +You generate it, choose how long it lives, and install the trust anchor in the +truststores of the workloads you want to fault. The proxy only signs with it; it never creates, rotates, or renews a CA. A CA already outside its validity window is rejected at startup rather than failing every handshake later. diff --git a/internal/tlsinject/ca.go b/internal/tlsinject/ca.go index 4cae6df..31dd399 100644 --- a/internal/tlsinject/ca.go +++ b/internal/tlsinject/ca.go @@ -80,6 +80,13 @@ type CA struct { // cost of re-signing more often. leafValidity time.Duration + // chain is every certificate the operator supplied, signing certificate + // first. Presenting all of them lets an intermediate CA be used: the client + // trusts the root it already has, and we hand it the intermediates needed to + // build the path. Signing with an intermediate means the operator never has + // to part with a root key. + chain [][]byte + mu sync.Mutex cache map[string]*tls.Certificate } @@ -127,7 +134,14 @@ func LoadCA(certPEM, keyPEM []byte) (*CA, error) { if err != nil { return nil, fmt.Errorf("failed to generate leaf key: %w", err) } - return &CA{cert: cert, key: signer, leafKey: leafKey, leafValidity: defaultLeafValidity, cache: map[string]*tls.Certificate{}}, nil + return &CA{ + cert: cert, + key: signer, + chain: pair.Certificate, + leafKey: leafKey, + leafValidity: defaultLeafValidity, + cache: map[string]*tls.Certificate{}, + }, nil } // LoadCACombined parses one PEM stream carrying both the CA certificate and its @@ -316,10 +330,14 @@ func (c *CA) mint(host string) (*tls.Certificate, error) { if err != nil { return nil, fmt.Errorf("failed to parse minted certificate: %w", err) } + // leaf first, then everything the operator supplied, so a client holding + // only the root can still build the path through any intermediates. + chain := make([][]byte, 0, len(c.chain)+1) + chain = append(chain, der) + chain = append(chain, c.chain...) + return &tls.Certificate{ - // Send the CA alongside the leaf so clients that trust it by a different - // path can still build the chain. - Certificate: [][]byte{der, c.cert.Raw}, + Certificate: chain, PrivateKey: c.leafKey, Leaf: leaf, }, nil diff --git a/internal/tlsinject/ca_test.go b/internal/tlsinject/ca_test.go index 3581d42..e2e5d50 100644 --- a/internal/tlsinject/ca_test.go +++ b/internal/tlsinject/ca_test.go @@ -293,3 +293,74 @@ func Test_LoadCACombined_rejectsEncryptedKey(t *testing.T) { t.Fatalf("err = %v, want a pointed passphrase-protected message", err) } } + +// An operator must never have to hand over a root key. Signing with an +// intermediate issued by their own PKI has to work: the client trusts only the +// root it already has, and the proxy supplies the intermediate so the path can +// be built. +func Test_LoadCA_worksWithAnIntermediate(t *testing.T) { + // root (stays offline; only its certificate is trusted by the client) + rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + rootTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Customer Root CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(72 * time.Hour), + IsCA: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, + } + rootDER, err := x509.CreateCertificate(rand.Reader, rootTmpl, rootTmpl, rootKey.Public(), rootKey) + if err != nil { + t.Fatal(err) + } + rootCert, _ := x509.ParseCertificate(rootDER) + + // intermediate — this is what the operator hands the proxy + interKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + interTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "Steadybit Intercept Intermediate"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(48 * time.Hour), + IsCA: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, + } + interDER, err := x509.CreateCertificate(rand.Reader, interTmpl, rootCert, interKey.Public(), rootKey) + if err != nil { + t.Fatal(err) + } + interKeyDER, _ := x509.MarshalECPrivateKey(interKey) + + // The operator supplies intermediate (+ root) and the intermediate's key. + certPEM := append( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: interDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER})...) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: interKeyDER}) + + ca, err := LoadCA(certPEM, keyPEM) + if err != nil { + t.Fatalf("an intermediate CA must be accepted: %v", err) + } + cert, err := ca.leafFor("api.example.com") + if err != nil { + t.Fatalf("leafFor: %v", err) + } + + // The client trusts ONLY the root; the chain we present must let it verify. + roots := x509.NewCertPool() + roots.AddCert(rootCert) + inters := x509.NewCertPool() + for _, der := range cert.Certificate[1:] { + c, perr := x509.ParseCertificate(der) + if perr != nil { + t.Fatal(perr) + } + inters.AddCert(c) + } + if _, err := cert.Leaf.Verify(x509.VerifyOptions{ + Roots: roots, Intermediates: inters, DNSName: "api.example.com", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Fatalf("a client trusting only the root could not verify the presented chain: %v", err) + } +}