From 1a82cc7fc1da57df9e460bd6d1ef8b8984295b8f Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 1 Oct 2026 18:41:45 +0200 Subject: [PATCH 1/2] Add failing tests for filtering secrets from query strings and URLs HTTPEvents logs the raw query string, so ?password=hunter2&token=... is stored verbatim, and so are the queries of the URLs in the Referer request header and the Location response header. The tests pin a query_string_filters setting that works like Rails' filter_parameters: the default list, a custom list, Regexps, an empty list to disable filtering, nested and URL-encoded names, Referer and Location in either casing, the collapsed event, and the rest of the string left byte for byte unchanged. Co-Authored-By: Claude Opus 5.5 --- spec/logtail-rack/http_events_spec.rb | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/spec/logtail-rack/http_events_spec.rb b/spec/logtail-rack/http_events_spec.rb index 9de15db..3c65994 100755 --- a/spec/logtail-rack/http_events_spec.rb +++ b/spec/logtail-rack/http_events_spec.rb @@ -69,6 +69,93 @@ expect(JSON.parse(request_headers_json)).to eq({"Authorization" => "Bearer secret_token", "Content_Type" => "text/plain"}) end + it "filter query string parameters with secrets in their names by default" do + expect(described_class::DEFAULT_QUERY_STRING_FILTERS).to eq(%w[passw secret token _key crypt salt certificate otp ssn cvv cvc]) + expect(described_class.query_string_filters).to eq(described_class::DEFAULT_QUERY_STRING_FILTERS) + + logs = capture_logs { middleware.call request_with_query_string("password=hunter2&page=2&Api_Key=k1&ACCESS_TOKEN=t1&client_secret=s1&q=search") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("password=[FILTERED]&page=2&Api_Key=[FILTERED]&ACCESS_TOKEN=[FILTERED]&client_secret=[FILTERED]&q=search") + end + + it "leave the rest of the query string byte for byte unchanged" do + logs = capture_logs { middleware.call request_with_query_string("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=abc=def&=x&page=3") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=[FILTERED]&=x&page=3") + end + + it "match the URL-decoded full name of nested query string parameters" do + logs = capture_logs { middleware.call request_with_query_string("user[password]=p1&user%5Bpassword_confirmation%5D=p2&user[name]=Jane&pass%77ord=p3&%zz=1&%FFtoken=t1") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("user[password]=[FILTERED]&user%5Bpassword_confirmation%5D=[FILTERED]&user[name]=Jane&pass%77ord=[FILTERED]&%zz=1&%FFtoken=[FILTERED]") + end + + it "filter the query string with custom query_string_filters" do + logs = capture_logs do + with_query_string_filters(["code", :page]) { middleware.call request_with_query_string("password=hunter2&page=2&Code=c1&q=search") } + end + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("password=hunter2&page=[FILTERED]&Code=[FILTERED]&q=search") + end + + it "filter the query string with Regexp query_string_filters, ignoring Procs" do + filters = [/\Aq\z/, /sig/, ->(_name, value) { value.replace("changed") }] + + logs = capture_logs do + with_query_string_filters(filters) { middleware.call request_with_query_string("q=1&query=2&Q=3&x_sig=abc&page=4") } + end + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("q=[FILTERED]&query=2&Q=3&x_sig=[FILTERED]&page=4") + end + + it "log the query string and URLs unfiltered when query_string_filters is set to an empty list" do + app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1" }, []] } + request = request_with_query_string("password=hunter2&token=t1", "HTTP_REFERER" => "https://example.com/signup?password=hunter2") + + logs = capture_logs { with_query_string_filters([]) { described_class.new(app).call request } } + + expect(logs.first["event"]["http_request_received"]["query_string"]).to eq("password=hunter2&token=t1") + expect(JSON.parse(logs.first["event"]["http_request_received"]["headers_json"])["Referer"]).to eq("https://example.com/signup?password=hunter2") + expect(JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"])["Location"]).to eq("https://example.com/next?token=t1") + end + + it "filter the query of the URLs in the Referer request header and the Location response header" do + app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1&step=2#top", "Content-Type" => "text/plain" }, []] } + request = Rack::MockRequest.env_for("https://example.com/test-page", "HTTP_REFERER" => "https://example.com/signup?password=hunter2&ref=ad") + + logs = capture_logs { described_class.new(app).call request } + + request_headers = JSON.parse(logs.first["event"]["http_request_received"]["headers_json"]) + expect(request_headers["Referer"]).to eq("https://example.com/signup?password=[FILTERED]&ref=ad") + response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"Location" => "https://example.com/next?token=[FILTERED]&step=2#top", "Content-Type" => "text/plain"}) + end + + it "filter the query of the URL in a lower-case location response header" do + app = ->(env) { [302, { "location" => "/next?client_secret=s1&step=2" }, []] } + + logs = capture_logs { described_class.new(app).call mock_request } + + response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"location" => "/next?client_secret=[FILTERED]&step=2"}) + end + + it "filter the query of the URL in the Location header of the single collapsed event" do + app = ->(env) { [302, { "location" => "/next?token=t1&step=2" }, []] } + stack = Logtail::Integrations::Rack::HTTPContext.new(described_class.new(app)) + + logs = capture_logs { with_collapse_into_single_event { stack.call mock_request } } + + expect(logs.length).to eq(1) + response_headers = JSON.parse(logs.first["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"location" => "/next?token=[FILTERED]&step=2"}) + end + it "log the content length of a Rack 3 response with lower-case header names" do app = ->(env) { [200, { "content-type" => "text/plain", "content-length" => "5" }, ["hello"]] } @@ -156,6 +243,19 @@ def with_http_header_filters(headers, &blk) Logtail::Integrations::Rack::HTTPEvents.http_header_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_HTTP_HEADER_FILTERS end + def with_query_string_filters(filters, &blk) + Logtail::Integrations::Rack::HTTPEvents.query_string_filters = filters + + blk.call + ensure + Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + end + + # Sets QUERY_STRING directly, as some of the query strings aren't valid URIs + def request_with_query_string(query_string, env = {}) + Rack::MockRequest.env_for("https://example.com/test-page", env).merge("QUERY_STRING" => query_string) + end + def with_collapse_into_single_event(&blk) Logtail::Integrations::Rack::HTTPEvents.collapse_into_single_event = true From 3d365bff56771f1df51bd2ca989419f74b631f5f Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 1 Oct 2026 18:46:10 +0200 Subject: [PATCH 2/2] Filter secrets from query strings and Referer/Location URLs Adds HTTPEvents.query_string_filters, a class-level list like http_header_filters, defaulting to DEFAULT_QUERY_STRING_FILTERS (the list Rails puts in new apps, minus email). Like Rails' filter_parameters, a String or Symbol filters every parameter whose URL-decoded name contains it, ignoring case, a Regexp every name it matches, and other entries such as Procs are ignored. Nested names are matched as the full decoded name. A matching parameter's value becomes [FILTERED] and the rest of the string stays byte for byte unchanged. The filters apply to the query string of http_request_received and to the query of the URLs in the Referer and Location headers, whatever their casing, so the collapsed event is covered too. An empty list disables the filtering. Co-Authored-By: Claude Opus 5.5 --- lib/logtail-rack/http_events.rb | 68 ++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/lib/logtail-rack/http_events.rb b/lib/logtail-rack/http_events.rb index 6cce46e..bb33996 100755 --- a/lib/logtail-rack/http_events.rb +++ b/lib/logtail-rack/http_events.rb @@ -1,4 +1,5 @@ require "set" +require "uri" require "logtail/config" require "logtail/contexts/http" @@ -16,6 +17,7 @@ module Rack # respectively. class HTTPEvents < Middleware DEFAULT_HTTP_HEADER_FILTERS = ["Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"].freeze + DEFAULT_QUERY_STRING_FILTERS = %w[passw secret token _key crypt salt certificate otp ssn cvv cvc].freeze class << self # Allows you to capture the HTTP request body, default is off (false). @@ -123,14 +125,59 @@ def http_header_filters @http_header_filters end + # Filter sensitive query string parameters (such as "?token=secret_token") + # + # Filtered values will be sent to Better Stack as "[FILTERED]", in the query string of + # the request event and in the query of the URLs in the Referer and Location headers. + # Like Rails' filter_parameters, a String or Symbol filters every parameter whose + # URL-decoded name contains it, ignoring case, and a Regexp every name it matches. + # + # {DEFAULT_QUERY_STRING_FILTERS} are filtered out of the box. Setting this replaces + # the whole list, pass an empty list to log query strings unfiltered. + # + # @example + # Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + ["code", /\Aapi_/] + def query_string_filters=(value) + strings = value.select { |filter| filter.is_a?(String) || filter.is_a?(Symbol) } + regexps = value.select { |filter| filter.is_a?(Regexp) } + if !strings.empty? + regexps << Regexp.new(strings.map { |filter| Regexp.escape(filter.to_s) }.join("|"), Regexp::IGNORECASE) + end + + @query_string_filters = value + @query_string_filter_regexps = regexps + end + + # Accessor method for {#query_string_filters=} + def query_string_filters + @query_string_filters + end + + # Whether {#query_string_filters} filter the parameter with this name from a query string + def filter_query_string_parameter?(name) + begin + name = URI.decode_www_form_component(name) + rescue ArgumentError + # Invalid %-encoding, match the name as it is + end + + name = name.scrub + @query_string_filter_regexps.any? { |regexp| regexp =~ name } + end + def normalize_header_name(name) name.to_s.downcase.gsub("-", "_") end end self.http_header_filters = DEFAULT_HTTP_HEADER_FILTERS + self.query_string_filters = DEFAULT_QUERY_STRING_FILTERS CONTENT_LENGTH_KEY = 'content-length'.freeze + # A query string parameter name and its value, up to the next separator + QUERY_STRING_PARAMETER = /([^&;=]+)=([^&;]+)/ + URL_QUERY = /\?([^#]*)/ + URL_HEADERS = ["referer", "location"].freeze def call(env) request = Util::Request.new(env) @@ -191,7 +238,7 @@ def call(env) method: request.request_method, path: request.path, port: request.port, - query_string: Util::Encoding.force_utf8_encoding(request.query_string), + query_string: filter_query_string(Util::Encoding.force_utf8_encoding(request.query_string)), request_id: request.request_id, scheme: Util::Encoding.force_utf8_encoding(request.scheme), ) @@ -279,10 +326,29 @@ def filter_http_headers(headers) headers.map do |name, value| normalized_name = self.class.normalize_header_name(name) is_filtered = self.class.http_header_filters.include?(normalized_name) + value = filter_url_query(value) if URL_HEADERS.include?(normalized_name) [name, is_filtered ? "[FILTERED]" : value] end.to_h end + # Replaces the value of every parameter that {.query_string_filters} match with + # "[FILTERED]", leaving the rest of the query string unchanged. + def filter_query_string(query_string) + return query_string if !query_string.is_a?(String) || self.class.query_string_filters.empty? + + query_string.gsub(QUERY_STRING_PARAMETER) do |parameter| + name = Regexp.last_match(1) + self.class.filter_query_string_parameter?(name) ? "#{name}=[FILTERED]" : parameter + end + end + + # Filters the query of the URL in a Referer or Location header. + def filter_url_query(url) + return url if !url.is_a?(String) || !url.include?("?") || self.class.query_string_filters.empty? + + Util::Encoding.force_utf8_encoding(url).sub(URL_QUERY) { "?#{filter_query_string(Regexp.last_match(1))}" } + end + # Rack 3 applications return "content-length", older ones usually "Content-Length". def response_content_length(headers) _name, value = headers.find { |name, _value| name.to_s.downcase == CONTENT_LENGTH_KEY }