diff --git a/saas/config/deploy.beta.yml b/saas/config/deploy.beta.yml index 7e60f41a40..d7f9dbb2a0 100644 --- a/saas/config/deploy.beta.yml +++ b/saas/config/deploy.beta.yml @@ -86,7 +86,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 host: <%= @lb_host %> labels: otel_role: load-balancer diff --git a/saas/config/deploy.production.yml b/saas/config/deploy.production.yml index d1dd705b0a..d3234fce4b 100644 --- a/saas/config/deploy.production.yml +++ b/saas/config/deploy.production.yml @@ -74,7 +74,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 hosts: - fizzy-lb-101.df-iad-int.37signals.com - fizzy-lb-102.df-iad-int.37signals.com diff --git a/saas/config/deploy.staging.yml b/saas/config/deploy.staging.yml index b45fabcdd1..b90bd96e51 100644 --- a/saas/config/deploy.staging.yml +++ b/saas/config/deploy.staging.yml @@ -71,7 +71,12 @@ env: accessories: load-balancer: - image: basecamp/kamal-proxy:lb + # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer + # affinity) reached a release. Pinned now for the same reason as hotcell's image in + # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes + # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with + # --client-ip-header, which saas/script/configure-lb-*.sh passes. + image: basecamp/kamal-proxy:v0.10.0 hosts: - fizzy-staging-lb-01.sc-chi-int.37signals.com - fizzy-staging-lb-101.df-iad-int.37signals.com diff --git a/saas/lib/fizzy/saas/true_client_ip.rb b/saas/lib/fizzy/saas/true_client_ip.rb index 27c5f07296..0f37887cc6 100644 --- a/saas/lib/fizzy/saas/true_client_ip.rb +++ b/saas/lib/fizzy/saas/true_client_ip.rb @@ -7,6 +7,12 @@ # However, for Fizzy the F5s are configured to do passthrough, so the header value isn't being # copied for us. Let's do that bit of work here, before Rails' RemoteIp middleware. # +# Since kamal-proxy 0.10.0 the load balancer does the same copy for us +# (--client-ip-header=True-Client-IP in saas/script/configure-lb-*.sh), which is what fixes the +# proxy access logs the app tier ships. This middleware is kept because it also covers the app +# when it is reached without going through that load balancer, and because it collapses the +# forwarded chain to the one address we actually trust. +# class TrackTrueClientIp def initialize(app) @app = app diff --git a/saas/script/configure-lb-beta.sh b/saas/script/configure-lb-beta.sh index 1857e05580..b3c54b3268 100755 --- a/saas/script/configure-lb-beta.sh +++ b/saas/script/configure-lb-beta.sh @@ -2,11 +2,22 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # Beta 1: fizzy-beta-lb-101 -> fizzy-beta-app-101 ssh app@fizzy-beta-lb-101.df-iad-int.37signals.com \ docker exec fizzy-load-balancer \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=beta1.fizzy-beta.com \ --target=fizzy-beta-app-101.df-iad-int.37signals.com diff --git a/saas/script/configure-lb-production.sh b/saas/script/configure-lb-production.sh index 94a8a16297..0e0abbbbc1 100755 --- a/saas/script/configure-lb-production.sh +++ b/saas/script/configure-lb-production.sh @@ -2,6 +2,15 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # fizzy-lb-101.df-iad-int.37signals.com # ssh app@fizzy-lb-101.df-iad-int.37signals.com \ @@ -9,6 +18,8 @@ ssh app@fizzy-lb-101.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -23,6 +34,8 @@ ssh app@fizzy-lb-102.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -37,6 +50,8 @@ ssh app@fizzy-lb-01.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -53,6 +68,8 @@ ssh app@fizzy-lb-02.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -69,6 +86,8 @@ ssh app@fizzy-lb-401.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -85,6 +104,8 @@ ssh app@fizzy-lb-402.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -100,6 +121,8 @@ ssh app@fizzy-lb-501.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -116,6 +139,8 @@ ssh app@fizzy-lb-502.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy.do \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ diff --git a/saas/script/configure-lb-staging.sh b/saas/script/configure-lb-staging.sh index 7093291237..929e8dc435 100755 --- a/saas/script/configure-lb-staging.sh +++ b/saas/script/configure-lb-staging.sh @@ -2,6 +2,15 @@ set -e +# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into +# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is +# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address +# instead (see the "4xx fizzy returns internal IPs" card). +# +# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the +# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy +# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here. + # fizzy-staging-lb-01.sc-chi-int.37signals.com # ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ @@ -9,6 +18,8 @@ ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -24,6 +35,8 @@ ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -37,6 +50,8 @@ ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \ @@ -52,6 +67,8 @@ ssh app@fizzy-staging-lb-501.sjc-int.37signals.com \ kamal-proxy deploy fizzy \ --force \ --tls \ + --client-ip-header=True-Client-IP \ + --forward-headers \ --host=app.fizzy-staging.com \ --writer-affinity-timeout=0 \ --tls-acme-cache-path=/certificates \