diff --git a/lib/logtail-rack/http_events.rb b/lib/logtail-rack/http_events.rb index 6cce46e..bb33996 100755 --- a/lib/logtail-rack/http_events.rb +++ b/lib/logtail-rack/http_events.rb @@ -1,4 +1,5 @@ require "set" +require "uri" require "logtail/config" require "logtail/contexts/http" @@ -16,6 +17,7 @@ module Rack # respectively. class HTTPEvents < Middleware DEFAULT_HTTP_HEADER_FILTERS = ["Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"].freeze + DEFAULT_QUERY_STRING_FILTERS = %w[passw secret token _key crypt salt certificate otp ssn cvv cvc].freeze class << self # Allows you to capture the HTTP request body, default is off (false). @@ -123,14 +125,59 @@ def http_header_filters @http_header_filters end + # Filter sensitive query string parameters (such as "?token=secret_token") + # + # Filtered values will be sent to Better Stack as "[FILTERED]", in the query string of + # the request event and in the query of the URLs in the Referer and Location headers. + # Like Rails' filter_parameters, a String or Symbol filters every parameter whose + # URL-decoded name contains it, ignoring case, and a Regexp every name it matches. + # + # {DEFAULT_QUERY_STRING_FILTERS} are filtered out of the box. Setting this replaces + # the whole list, pass an empty list to log query strings unfiltered. + # + # @example + # Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + ["code", /\Aapi_/] + def query_string_filters=(value) + strings = value.select { |filter| filter.is_a?(String) || filter.is_a?(Symbol) } + regexps = value.select { |filter| filter.is_a?(Regexp) } + if !strings.empty? + regexps << Regexp.new(strings.map { |filter| Regexp.escape(filter.to_s) }.join("|"), Regexp::IGNORECASE) + end + + @query_string_filters = value + @query_string_filter_regexps = regexps + end + + # Accessor method for {#query_string_filters=} + def query_string_filters + @query_string_filters + end + + # Whether {#query_string_filters} filter the parameter with this name from a query string + def filter_query_string_parameter?(name) + begin + name = URI.decode_www_form_component(name) + rescue ArgumentError + # Invalid %-encoding, match the name as it is + end + + name = name.scrub + @query_string_filter_regexps.any? { |regexp| regexp =~ name } + end + def normalize_header_name(name) name.to_s.downcase.gsub("-", "_") end end self.http_header_filters = DEFAULT_HTTP_HEADER_FILTERS + self.query_string_filters = DEFAULT_QUERY_STRING_FILTERS CONTENT_LENGTH_KEY = 'content-length'.freeze + # A query string parameter name and its value, up to the next separator + QUERY_STRING_PARAMETER = /([^&;=]+)=([^&;]+)/ + URL_QUERY = /\?([^#]*)/ + URL_HEADERS = ["referer", "location"].freeze def call(env) request = Util::Request.new(env) @@ -191,7 +238,7 @@ def call(env) method: request.request_method, path: request.path, port: request.port, - query_string: Util::Encoding.force_utf8_encoding(request.query_string), + query_string: filter_query_string(Util::Encoding.force_utf8_encoding(request.query_string)), request_id: request.request_id, scheme: Util::Encoding.force_utf8_encoding(request.scheme), ) @@ -279,10 +326,29 @@ def filter_http_headers(headers) headers.map do |name, value| normalized_name = self.class.normalize_header_name(name) is_filtered = self.class.http_header_filters.include?(normalized_name) + value = filter_url_query(value) if URL_HEADERS.include?(normalized_name) [name, is_filtered ? "[FILTERED]" : value] end.to_h end + # Replaces the value of every parameter that {.query_string_filters} match with + # "[FILTERED]", leaving the rest of the query string unchanged. + def filter_query_string(query_string) + return query_string if !query_string.is_a?(String) || self.class.query_string_filters.empty? + + query_string.gsub(QUERY_STRING_PARAMETER) do |parameter| + name = Regexp.last_match(1) + self.class.filter_query_string_parameter?(name) ? "#{name}=[FILTERED]" : parameter + end + end + + # Filters the query of the URL in a Referer or Location header. + def filter_url_query(url) + return url if !url.is_a?(String) || !url.include?("?") || self.class.query_string_filters.empty? + + Util::Encoding.force_utf8_encoding(url).sub(URL_QUERY) { "?#{filter_query_string(Regexp.last_match(1))}" } + end + # Rack 3 applications return "content-length", older ones usually "Content-Length". def response_content_length(headers) _name, value = headers.find { |name, _value| name.to_s.downcase == CONTENT_LENGTH_KEY } diff --git a/spec/logtail-rack/http_events_spec.rb b/spec/logtail-rack/http_events_spec.rb index 9de15db..3c65994 100755 --- a/spec/logtail-rack/http_events_spec.rb +++ b/spec/logtail-rack/http_events_spec.rb @@ -69,6 +69,93 @@ expect(JSON.parse(request_headers_json)).to eq({"Authorization" => "Bearer secret_token", "Content_Type" => "text/plain"}) end + it "filter query string parameters with secrets in their names by default" do + expect(described_class::DEFAULT_QUERY_STRING_FILTERS).to eq(%w[passw secret token _key crypt salt certificate otp ssn cvv cvc]) + expect(described_class.query_string_filters).to eq(described_class::DEFAULT_QUERY_STRING_FILTERS) + + logs = capture_logs { middleware.call request_with_query_string("password=hunter2&page=2&Api_Key=k1&ACCESS_TOKEN=t1&client_secret=s1&q=search") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("password=[FILTERED]&page=2&Api_Key=[FILTERED]&ACCESS_TOKEN=[FILTERED]&client_secret=[FILTERED]&q=search") + end + + it "leave the rest of the query string byte for byte unchanged" do + logs = capture_logs { middleware.call request_with_query_string("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=abc=def&=x&page=3") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("q=caf%C3%A9+au+lait&empty=&flag&a=1;b=2&x=y%26z&list[]=1&list[]=2&token=[FILTERED]&=x&page=3") + end + + it "match the URL-decoded full name of nested query string parameters" do + logs = capture_logs { middleware.call request_with_query_string("user[password]=p1&user%5Bpassword_confirmation%5D=p2&user[name]=Jane&pass%77ord=p3&%zz=1&%FFtoken=t1") } + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("user[password]=[FILTERED]&user%5Bpassword_confirmation%5D=[FILTERED]&user[name]=Jane&pass%77ord=[FILTERED]&%zz=1&%FFtoken=[FILTERED]") + end + + it "filter the query string with custom query_string_filters" do + logs = capture_logs do + with_query_string_filters(["code", :page]) { middleware.call request_with_query_string("password=hunter2&page=2&Code=c1&q=search") } + end + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("password=hunter2&page=[FILTERED]&Code=[FILTERED]&q=search") + end + + it "filter the query string with Regexp query_string_filters, ignoring Procs" do + filters = [/\Aq\z/, /sig/, ->(_name, value) { value.replace("changed") }] + + logs = capture_logs do + with_query_string_filters(filters) { middleware.call request_with_query_string("q=1&query=2&Q=3&x_sig=abc&page=4") } + end + + query_string = logs.first["event"]["http_request_received"]["query_string"] + expect(query_string).to eq("q=[FILTERED]&query=2&Q=3&x_sig=[FILTERED]&page=4") + end + + it "log the query string and URLs unfiltered when query_string_filters is set to an empty list" do + app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1" }, []] } + request = request_with_query_string("password=hunter2&token=t1", "HTTP_REFERER" => "https://example.com/signup?password=hunter2") + + logs = capture_logs { with_query_string_filters([]) { described_class.new(app).call request } } + + expect(logs.first["event"]["http_request_received"]["query_string"]).to eq("password=hunter2&token=t1") + expect(JSON.parse(logs.first["event"]["http_request_received"]["headers_json"])["Referer"]).to eq("https://example.com/signup?password=hunter2") + expect(JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"])["Location"]).to eq("https://example.com/next?token=t1") + end + + it "filter the query of the URLs in the Referer request header and the Location response header" do + app = ->(env) { [302, { "Location" => "https://example.com/next?token=t1&step=2#top", "Content-Type" => "text/plain" }, []] } + request = Rack::MockRequest.env_for("https://example.com/test-page", "HTTP_REFERER" => "https://example.com/signup?password=hunter2&ref=ad") + + logs = capture_logs { described_class.new(app).call request } + + request_headers = JSON.parse(logs.first["event"]["http_request_received"]["headers_json"]) + expect(request_headers["Referer"]).to eq("https://example.com/signup?password=[FILTERED]&ref=ad") + response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"Location" => "https://example.com/next?token=[FILTERED]&step=2#top", "Content-Type" => "text/plain"}) + end + + it "filter the query of the URL in a lower-case location response header" do + app = ->(env) { [302, { "location" => "/next?client_secret=s1&step=2" }, []] } + + logs = capture_logs { described_class.new(app).call mock_request } + + response_headers = JSON.parse(logs.last["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"location" => "/next?client_secret=[FILTERED]&step=2"}) + end + + it "filter the query of the URL in the Location header of the single collapsed event" do + app = ->(env) { [302, { "location" => "/next?token=t1&step=2" }, []] } + stack = Logtail::Integrations::Rack::HTTPContext.new(described_class.new(app)) + + logs = capture_logs { with_collapse_into_single_event { stack.call mock_request } } + + expect(logs.length).to eq(1) + response_headers = JSON.parse(logs.first["event"]["http_response_sent"]["headers_json"]) + expect(response_headers).to eq({"location" => "/next?token=[FILTERED]&step=2"}) + end + it "log the content length of a Rack 3 response with lower-case header names" do app = ->(env) { [200, { "content-type" => "text/plain", "content-length" => "5" }, ["hello"]] } @@ -156,6 +243,19 @@ def with_http_header_filters(headers, &blk) Logtail::Integrations::Rack::HTTPEvents.http_header_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_HTTP_HEADER_FILTERS end + def with_query_string_filters(filters, &blk) + Logtail::Integrations::Rack::HTTPEvents.query_string_filters = filters + + blk.call + ensure + Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + end + + # Sets QUERY_STRING directly, as some of the query strings aren't valid URIs + def request_with_query_string(query_string, env = {}) + Rack::MockRequest.env_for("https://example.com/test-page", env).merge("QUERY_STRING" => query_string) + end + def with_collapse_into_single_event(&blk) Logtail::Integrations::Rack::HTTPEvents.collapse_into_single_event = true