Give the load balancer the real client IP

saas/config/deploy.beta.yml

@@ -86,7 +86,12 @@ env:
 
 accessories:
   load-balancer:
-    image: basecamp/kamal-proxy:lb
+    # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer
+    # affinity) reached a release. Pinned now for the same reason as hotcell's image in
+    # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes
+    # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with
+    # --client-ip-header, which saas/script/configure-lb-*.sh passes.
+    image: basecamp/kamal-proxy:v0.10.0
     host: <%= @lb_host %>
     labels:
       otel_role: load-balancer

saas/config/deploy.production.yml

@@ -74,7 +74,12 @@ env:
 
 accessories:
   load-balancer:
-    image: basecamp/kamal-proxy:lb
+    # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer
+    # affinity) reached a release. Pinned now for the same reason as hotcell's image in
+    # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes
+    # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with
+    # --client-ip-header, which saas/script/configure-lb-*.sh passes.
+    image: basecamp/kamal-proxy:v0.10.0
     hosts:
       - fizzy-lb-101.df-iad-int.37signals.com
       - fizzy-lb-102.df-iad-int.37signals.com

saas/config/deploy.staging.yml

@@ -71,7 +71,12 @@ env:
 
 accessories:
   load-balancer:
-    image: basecamp/kamal-proxy:lb
+    # Was `:lb`, a moving tag from before the load-balancer features (read targets, writer
+    # affinity) reached a release. Pinned now for the same reason as hotcell's image in
+    # deploy.yml: a reboot pulls whatever the tag names at that moment, so a moving tag makes
+    # what a host runs depend on when it last rebooted. v0.10.0 is also the first release with
+    # --client-ip-header, which saas/script/configure-lb-*.sh passes.
+    image: basecamp/kamal-proxy:v0.10.0
     hosts:
       - fizzy-staging-lb-01.sc-chi-int.37signals.com
       - fizzy-staging-lb-101.df-iad-int.37signals.com

saas/lib/fizzy/saas/true_client_ip.rb

@@ -7,6 +7,12 @@
 #  However, for Fizzy the F5s are configured to do passthrough, so the header value isn't being
 #  copied for us. Let's do that bit of work here, before Rails' RemoteIp middleware.
 #
+#  Since kamal-proxy 0.10.0 the load balancer does the same copy for us
+#  (--client-ip-header=True-Client-IP in saas/script/configure-lb-*.sh), which is what fixes the
+#  proxy access logs the app tier ships. This middleware is kept because it also covers the app
+#  when it is reached without going through that load balancer, and because it collapses the
+#  forwarded chain to the one address we actually trust.
+#
 class TrackTrueClientIp
   def initialize(app)
     @app = app

saas/script/configure-lb-beta.sh

@@ -2,11 +2,22 @@
 
 set -e
 
+# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into
+# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is
+# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address
+# instead (see the "4xx fizzy returns internal IPs" card).
+#
+# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the
+# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy
+# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here.
+
 # Beta 1: fizzy-beta-lb-101 -> fizzy-beta-app-101
 ssh app@fizzy-beta-lb-101.df-iad-int.37signals.com \
   docker exec fizzy-load-balancer \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=beta1.fizzy-beta.com \
       --target=fizzy-beta-app-101.df-iad-int.37signals.com

saas/script/configure-lb-production.sh

@@ -2,13 +2,24 @@
 
 set -e
 
+# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into
+# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is
+# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address
+# instead (see the "4xx fizzy returns internal IPs" card).
+#
+# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the
+# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy
+# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here.
+
 # fizzy-lb-101.df-iad-int.37signals.com
 #
 ssh app@fizzy-lb-101.df-iad-int.37signals.com \
   docker exec fizzy-load-balancer \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -23,6 +34,8 @@ ssh app@fizzy-lb-102.df-iad-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -37,6 +50,8 @@ ssh app@fizzy-lb-01.sc-chi-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -53,6 +68,8 @@ ssh app@fizzy-lb-02.sc-chi-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -69,6 +86,8 @@ ssh app@fizzy-lb-401.df-ams-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -85,6 +104,8 @@ ssh app@fizzy-lb-402.df-ams-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -100,6 +121,8 @@ ssh app@fizzy-lb-501.sjc-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -116,6 +139,8 @@ ssh app@fizzy-lb-502.sjc-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy.do \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \

saas/script/configure-lb-staging.sh

@@ -2,13 +2,24 @@
 
 set -e
 
+# Cloudflare puts the real client IP in True-Client-IP. Most 37signals apps get that copied into
+# X-Forwarded-For by the manage_x_forwarded iRule on the F5s, but fizzy's HTTPS virtual server is
+# fastL4 passthrough, so no HTTP iRule can run and every tier behind here logs an internal address
+# instead (see the "4xx fizzy returns internal IPs" card).
+#
+# --client-ip-header makes kamal-proxy trust True-Client-IP for its own access log and for the
+# X-Forwarded-For it sends on. --forward-headers is what lets that value reach the app tier's proxy
+# and Rails: X-Forwarded-For is dropped by default when the proxy terminates TLS, as it does here.
+
 # fizzy-staging-lb-01.sc-chi-int.37signals.com
 #
 ssh app@fizzy-staging-lb-01.sc-chi-int.37signals.com \
   docker exec fizzy-load-balancer \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy-staging.com \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -24,6 +35,8 @@ ssh app@fizzy-staging-lb-101.df-iad-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy-staging.com \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -37,6 +50,8 @@ ssh app@fizzy-staging-lb-401.df-ams-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy-staging.com \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \
@@ -52,6 +67,8 @@ ssh app@fizzy-staging-lb-501.sjc-int.37signals.com \
     kamal-proxy deploy fizzy \
       --force \
       --tls \
+      --client-ip-header=True-Client-IP \
+      --forward-headers \
       --host=app.fizzy-staging.com \
       --writer-affinity-timeout=0 \
       --tls-acme-cache-path=/certificates \