From e125d5d3277a739cc46d80220b55e130cb42d75a Mon Sep 17 00:00:00 2001 From: Lewis Buckley Date: Sun, 13 Sep 2026 23:39:26 +0100 Subject: [PATCH] Give the load balancer the real client IP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare puts the caller's address in True-Client-IP, and for most of our apps the manage_x_forwarded iRule copies it into X-Forwarded-For on the F5. Fizzy's HTTPS virtual server is fastL4 passthrough, so no HTTP iRule can run, and the app tier's kamal-proxy access log — which the !o 4xx report reads — carries an internal address instead. kamal-proxy 0.10.0 added --client-ip-header, which makes the proxy trust a named header for its own access log and for the X-Forwarded-For it sends on. Pass it at fizzy-lb, with --forward-headers so the rewritten value actually reaches the app tier: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does there. That needs the accessory off the `lb` tag, which predates the flag, so pin it to v0.10.0 — a moving tag made what a host runs depend on when it last rebooted anyway. Co-Authored-By: Claude Opus 5 (1M context) --- saas/config/deploy.beta.yml | 7 ++++++- saas/config/deploy.production.yml | 7 ++++++- saas/config/deploy.staging.yml | 7 ++++++- saas/lib/fizzy/saas/true_client_ip.rb | 6 ++++++ saas/script/configure-lb-beta.sh | 11 +++++++++++ saas/script/configure-lb-production.sh | 25 +++++++++++++++++++++++++ saas/script/configure-lb-staging.sh | 17 +++++++++++++++++ 7 files changed, 77 insertions(+), 3 deletions(-) diff --git a/saas/config/deploy.beta.yml b/saas/config/deploy.beta.yml index 7e60f41a40..d7f9dbb2a0 100644 --- a/saas/config/deploy.beta.yml +++ b/saas/config/deploy.beta.yml @@ -86,7 +86,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 host: <%= @lb_host %> labels: otel_role: load-balancer diff --git a/saas/config/deploy.production.yml b/saas/config/deploy.production.yml index d1dd705b0a..d3234fce4b 100644 --- a/saas/config/deploy.production.yml +++ b/saas/config/deploy.production.yml @@ -74,7 +74,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 hosts: - fizzy-lb-101.df-iad-int.37signals.com - fizzy-lb-102.df-iad-int.37signals.com diff --git a/saas/config/deploy.staging.yml b/saas/config/deploy.staging.yml index b45fabcdd1..b90bd96e51 100644 --- a/saas/config/deploy.staging.yml +++ b/saas/config/deploy.staging.yml @@ -71,7 +71,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 hosts: - fizzy-staging-lb-01.sc-chi-int.37signals.com - fizzy-staging-lb-101.df-iad-int.37signals.com diff --git a/saas/lib/fizzy/saas/true_client_ip.rb b/saas/lib/fizzy/saas/true_client_ip.rb index 27c5f07296..0f37887cc6 100644 --- a/saas/lib/fizzy/saas/true_client_ip.rb +++ b/saas/lib/fizzy/saas/true_client_ip.rb @@ -7,6 +7,12 @@ # However, for Fizzy the F5s are configured to do passthrough, so the header value isn't being # copied for us. Let's do that bit of work here, before Rails' RemoteIp middleware. # +# Since kamal-proxy 0.10.0 the load balancer does the same copy for us +# (--client-ip-header=True-Client-IP in saas/script/configure-lb-*.sh), which is what fixes the +# proxy access logs the app tier ships. This middleware is kept because it also covers the app +# when it is reached without going through that load balancer, and because it collapses the +# forwarded chain to the one address we actually trust. +# class TrackTrueClientIp def initialize(app) @app = app diff --git a/saas/script/configure-lb-beta.sh b/saas/script/configure-lb-beta.sh index 1857e05580..b3c54b3268 100755 --- a/saas/script/configure-lb-beta.sh +++ b/saas/script/configure-lb-beta.sh @@ -2,11 +2,22 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # Beta 1: fizzy-beta-lb-101 -> fizzy-beta-app-101 ssh app@fizzy-beta-lb-101.df-iad-int.37signals.com \ docker exec fizzy-load-balancer \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=beta1.fizzy-beta.com \ --target=fizzy-beta-app-101.df-iad-int.37signals.com diff --git a/saas/script/configure-lb-production.sh b/saas/script/configure-lb-production.sh index 94a8a16297..0e0abbbbc1 100755 --- a/saas/script/configure-lb-production.sh +++ b/saas/script/configure-lb-production.sh @@ -2,6 +2,15 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # fizzy-lb-101.df-iad-int.37signals.com # ssh app@fizzy-lb-101.df-iad-int.37signals.com \ @@ -9,6 +18,8 @@ ssh app@fizzy-lb-101.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -23,6 +34,8 @@ ssh app@fizzy-lb-102.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -37,6 +50,8 @@ ssh app@fizzy-lb-01.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -53,6 +68,8 @@ ssh app@fizzy-lb-02.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -69,6 +86,8 @@ ssh app@fizzy-lb-401.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -85,6 +104,8 @@ ssh app@fizzy-lb-402.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -100,6 +121,8 @@ ssh app@fizzy-lb-501.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -116,6 +139,8 @@ ssh app@fizzy-lb-502.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ diff --git a/saas/script/configure-lb-staging.sh b/saas/script/configure-lb-staging.sh index 7093291237..929e8dc435 100755 --- a/saas/script/configure-lb-staging.sh +++ b/saas/script/configure-lb-staging.sh @@ -2,6 +2,15 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # fizzy-staging-lb-01.sc-chi-int.37signals.com # ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ @@ -9,6 +18,8 @@ ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -24,6 +35,8 @@ ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -37,6 +50,8 @@ ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -52,6 +67,8 @@ ssh app@fizzy-staging-lb-501.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \