Filter secrets from logged query strings and Referer/Location URLs

lib/logtail-rack/http_events.rb

@@ -1,4 +1,5 @@
 require "set"
+require "uri"
 
 require "logtail/config"
 require "logtail/contexts/http"
@@ -16,6 +17,7 @@ module Rack
       # respectively.
       class HTTPEvents < Middleware
         DEFAULT_HTTP_HEADER_FILTERS = ["Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"].freeze
+        DEFAULT_QUERY_STRING_FILTERS = %w[passw secret token _key crypt salt certificate otp ssn cvv cvc].freeze
 
         class << self
           # Allows you to capture the HTTP request body, default is off (false).
@@ -123,14 +125,59 @@ def http_header_filters
             @http_header_filters
           end
 
+          # Filter sensitive query string parameters (such as "?token=secret_token")
+          #
+          # Filtered values will be sent to Better Stack as "[FILTERED]", in the query string of
+          # the request event and in the query of the URLs in the Referer and Location headers.
+          # Like Rails' filter_parameters, a String or Symbol filters every parameter whose
+          # URL-decoded name contains it, ignoring case, and a Regexp every name it matches.
+          #
+          # {DEFAULT_QUERY_STRING_FILTERS} are filtered out of the box. Setting this replaces
+          # the whole list, pass an empty list to log query strings unfiltered.
+          #
+          # @example
+          #   Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + ["code", /\Aapi_/]
+          def query_string_filters=(value)
+            strings = value.select { |filter| filter.is_a?(String) || filter.is_a?(Symbol) }
+            regexps = value.select { |filter| filter.is_a?(Regexp) }
+            if !strings.empty?
+              regexps << Regexp.new(strings.map { |filter| Regexp.escape(filter.to_s) }.join("|"), Regexp::IGNORECASE)
+            end
+
+            @query_string_filters = value
+            @query_string_filter_regexps = regexps
+          end
+
+          # Accessor method for {#query_string_filters=}
+          def query_string_filters
+            @query_string_filters
+          end
+
+          # Whether {#query_string_filters} filter the parameter with this name from a query string
+          def filter_query_string_parameter?(name)
+            begin
+              name = URI.decode_www_form_component(name)
+            rescue ArgumentError
+              # Invalid %-encoding, match the name as it is
+            end
+
+            name = name.scrub
+            @query_string_filter_regexps.any? { |regexp| regexp =~ name }
+          end
+
           def normalize_header_name(name)
             name.to_s.downcase.gsub("-", "_")
           end
         end
 
         self.http_header_filters = DEFAULT_HTTP_HEADER_FILTERS
+        self.query_string_filters = DEFAULT_QUERY_STRING_FILTERS
 
         CONTENT_LENGTH_KEY = 'content-length'.freeze
+        # A query string parameter name and its value, up to the next separator
+        QUERY_STRING_PARAMETER = /([^&;=]+)=([^&;]+)/
+        URL_QUERY = /\?([^#]*)/
+        URL_HEADERS = ["referer", "location"].freeze
 
         def call(env)
           request = Util::Request.new(env)
@@ -191,7 +238,7 @@ def call(env)
                 method: request.request_method,
                 path: request.path,
                 port: request.port,
-                query_string: Util::Encoding.force_utf8_encoding(request.query_string),
+                query_string: filter_query_string(Util::Encoding.force_utf8_encoding(request.query_string)),
                 request_id: request.request_id,
                 scheme: Util::Encoding.force_utf8_encoding(request.scheme),
               )
@@ -279,10 +326,29 @@ def filter_http_headers(headers)
             headers.map do |name, value|
               normalized_name = self.class.normalize_header_name(name)
               is_filtered = self.class.http_header_filters.include?(normalized_name)
+              value = filter_url_query(value) if URL_HEADERS.include?(normalized_name)
               [name, is_filtered ? "[FILTERED]" : value]
             end.to_h
           end
 
+          # Replaces the value of every parameter that {.query_string_filters} match with
+          # "[FILTERED]", leaving the rest of the query string unchanged.
+          def filter_query_string(query_string)
+            return query_string if !query_string.is_a?(String) || self.class.query_string_filters.empty?
+
+            query_string.gsub(QUERY_STRING_PARAMETER) do |parameter|
+              name = Regexp.last_match(1)
+              self.class.filter_query_string_parameter?(name) ? "#{name}=[FILTERED]" : parameter
+            end
+          end
+
+          # Filters the query of the URL in a Referer or Location header.
+          def filter_url_query(url)
+            return url if !url.is_a?(String) || !url.include?("?") || self.class.query_string_filters.empty?
+
+            Util::Encoding.force_utf8_encoding(url).sub(URL_QUERY) { "?#{filter_query_string(Regexp.last_match(1))}" }
+          end
+
           # Rack 3 applications return "content-length", older ones usually "Content-Length".
           def response_content_length(headers)
             _name, value = headers.find { |name, _value| name.to_s.downcase == CONTENT_LENGTH_KEY }

spec/logtail-rack/http_events_spec.rb

@@ -69,6 +69,93 @@
     expect(JSON.parse(request_headers_json)).to eq({"Authorization" => "Bearer secret_token", "Content_Type" => "text/plain"})
   end
 
+  it "filter query string parameters with secrets in their names by default" do
+    expect(described_class::DEFAULT_QUERY_STRING_FILTERS).to eq(%w[passw secret token _key crypt salt certificate otp ssn cvv cvc])
+    expect(described_class.query_string_filters).to eq(described_class::DEFAULT_QUERY_STRING_FILTERS)
+
+    logs = capture_logs { middleware.call request_with_query_string("password=hunter2&page=2&Api_Key=k1&ACCESS_TOKEN=t1&client_secret=s1&q=search") }
+
+    query_string = logs.first["event"]["http_request_received"]["query_string"]
+    expect(query_string).to eq("password=[FILTERED]&page=2&Api_Key=[FILTERED]&ACCESS_TOKEN=[FILTERED]&client_secret=[FILTERED]&q=search")
+  end
+
+  it "leave the rest of the query string byte for byte unchanged" do
+    logs = capture_logs { middleware.call request_with_query_string("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=abc=def&=x&page=3") }
+
+    query_string = logs.first["event"]["http_request_received"]["query_string"]
+    expect(query_string).to eq("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=[FILTERED]&=x&page=3")
+  end
+
+  it "match the URL-decoded full name of nested query string parameters" do
+    logs = capture_logs { middleware.call request_with_query_string("user[password]=p1&user%5Bpassword_confirmation%5D=p2&user[name]=Jane&pass%77ord=p3&%zz=1&%FFtoken=t1") }
+
+    query_string = logs.first["event"]["http_request_received"]["query_string"]
+    expect(query_string).to eq("user[password]=[FILTERED]&user%5Bpassword_confirmation%5D=[FILTERED]&user[name]=Jane&pass%77ord=[FILTERED]&%zz=1&%FFtoken=[FILTERED]")
+  end
+
+  it "filter the query string with custom query_string_filters" do
+    logs = capture_logs do
+      with_query_string_filters(["code", :page]) { middleware.call request_with_query_string("password=hunter2&page=2&Code=c1&q=search") }
+    end
+
+    query_string = logs.first["event"]["http_request_received"]["query_string"]
+    expect(query_string).to eq("password=hunter2&page=[FILTERED]&Code=[FILTERED]&q=search")
+  end
+
+  it "filter the query string with Regexp query_string_filters, ignoring Procs" do
+    filters = [/\Aq\z/, /sig/, ->(_name, value) { value.replace("changed") }]
+
+    logs = capture_logs do
+      with_query_string_filters(filters) { middleware.call request_with_query_string("q=1&query=2&Q=3&x_sig=abc&page=4") }
+    end
+
+    query_string = logs.first["event"]["http_request_received"]["query_string"]
+    expect(query_string).to eq("q=[FILTERED]&query=2&Q=3&x_sig=[FILTERED]&page=4")
+  end
+
+  it "log the query string and URLs unfiltered when query_string_filters is set to an empty list" do
+    app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1" }, []] }
+    request = request_with_query_string("password=hunter2&token=t1", "HTTP_REFERER" => "https://example.com/signup?password=hunter2")
+
+    logs = capture_logs { with_query_string_filters([]) { described_class.new(app).call request } }
+
+    expect(logs.first["event"]["http_request_received"]["query_string"]).to eq("password=hunter2&token=t1")
+    expect(JSON.parse(logs.first["event"]["http_request_received"]["headers_json"])["Referer"]).to eq("https://example.com/signup?password=hunter2")
+    expect(JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"])["Location"]).to eq("https://example.com/next?token=t1")
+  end
+
+  it "filter the query of the URLs in the Referer request header and the Location response header" do
+    app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1&step=2#top", "Content-Type" => "text/plain" }, []] }
+    request = Rack::MockRequest.env_for("https://example.com/test-page", "HTTP_REFERER" => "https://example.com/signup?password=hunter2&ref=ad")
+
+    logs = capture_logs { described_class.new(app).call request }
+
+    request_headers = JSON.parse(logs.first["event"]["http_request_received"]["headers_json"])
+    expect(request_headers["Referer"]).to eq("https://example.com/signup?password=[FILTERED]&ref=ad")
+    response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"])
+    expect(response_headers).to eq({"Location" => "https://example.com/next?token=[FILTERED]&step=2#top", "Content-Type" => "text/plain"})
+  end
+
+  it "filter the query of the URL in a lower-case location response header" do
+    app = ->(env) { [302, { "location" => "/next?client_secret=s1&step=2" }, []] }
+
+    logs = capture_logs { described_class.new(app).call mock_request }
+
+    response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"])
+    expect(response_headers).to eq({"location" => "/next?client_secret=[FILTERED]&step=2"})
+  end
+
+  it "filter the query of the URL in the Location header of the single collapsed event" do
+    app = ->(env) { [302, { "location" => "/next?token=t1&step=2" }, []] }
+    stack = Logtail::Integrations::Rack::HTTPContext.new(described_class.new(app))
+
+    logs = capture_logs { with_collapse_into_single_event { stack.call mock_request } }
+
+    expect(logs.length).to eq(1)
+    response_headers = JSON.parse(logs.first["event"]["http_response_sent"]["headers_json"])
+    expect(response_headers).to eq({"location" => "/next?token=[FILTERED]&step=2"})
+  end
+
   it "log the content length of a Rack 3 response with lower-case header names" do
     app = ->(env) { [200, { "content-type" => "text/plain", "content-length" => "5" }, ["hello"]] }
 
@@ -156,6 +243,19 @@ def with_http_header_filters(headers, &blk)
     Logtail::Integrations::Rack::HTTPEvents.http_header_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_HTTP_HEADER_FILTERS
   end
 
+  def with_query_string_filters(filters, &blk)
+    Logtail::Integrations::Rack::HTTPEvents.query_string_filters = filters
+
+    blk.call
+  ensure
+    Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS
+  end
+
+  # Sets QUERY_STRING directly, as some of the query strings aren't valid URIs
+  def request_with_query_string(query_string, env = {})
+    Rack::MockRequest.env_for("https://example.com/test-page", env).merge("QUERY_STRING" => query_string)
+  end
+
   def with_collapse_into_single_event(&blk)
     Logtail::Integrations::Rack::HTTPEvents.collapse_into_single_event = true