perf: parameterize filter values and cache parsers and regexes

QueryKit.IntegrationTests/Tests/DatabaseFilteringTests.cs

@@ -735,7 +735,7 @@
         var fakePersonTwo = new FakeTestingPersonBuilder().Build();
         await testingServiceScope.InsertAsync(fakePersonOne, fakePersonTwo);
 
-        var input = $"""(id @=* "9edb")""";
+        var input = $"""(id @=* "9edb-a3ec")""";
 
         // Act
         var queryablePeople = testingServiceScope.DbContext().People;
@@ -1391,7 +1391,9 @@
     {
         // Arrange
         var testingServiceScope = new TestingServiceScope();
-        var fakeAuthorOne = new FakeAuthorBuilder().Build();
+        var fakeAuthorOne = new FakeAuthorBuilder()
+            .WithName(Guid.NewGuid().ToString())
+            .Build();
         var fakeRecipeOne = new FakeRecipeBuilder().Build();
         fakeRecipeOne.SetAuthor(fakeAuthorOne);
 
@@ -1472,7 +1474,9 @@
     {
         // Arrange
         var testingServiceScope = new TestingServiceScope();
-        var fakeAuthorOne = new FakeAuthorBuilder().Build();
+        var fakeAuthorOne = new FakeAuthorBuilder()
+            .WithName(Guid.NewGuid().ToString())
+            .Build();
         var fakeRecipeOne = new FakeRecipeBuilder().Build();
         fakeRecipeOne.SetAuthor(fakeAuthorOne);
 
@@ -3939,7 +3943,7 @@
         var config = new QueryKitConfiguration(settings =>
         {
             settings.CaseInsensitiveComparison = CaseInsensitiveMode.Lower;
             settings.Property<TestingPerson>(x => x.Title).HasCaseInsensitiveMode(CaseInsensitiveMode.Upper);
         });
 
         // Act
@@ -4190,7 +4194,7 @@
         var config = new QueryKitConfiguration(settings =>
         {
             settings.CaseInsensitiveComparison = CaseInsensitiveMode.Upper;
             settings.Property<TestingPerson>(x => x.Title).HasCaseInsensitiveMode(CaseInsensitiveMode.Lower);
         });
 
         // Act
@@ -4222,7 +4226,7 @@
         var config = new QueryKitConfiguration(settings =>
         {
             settings.CaseInsensitiveComparison = CaseInsensitiveMode.Lower;
             settings.Property<TestingPerson>(x => x.Title).HasCaseInsensitiveMode(CaseInsensitiveMode.Upper);
         });
 
         // Act

QueryKit.IntegrationTests/Tests/FilterParameterTests.cs

@@ -0,0 +1,62 @@
+namespace QueryKit.IntegrationTests.Tests;
+
+using FluentAssertions;
+using Microsoft.EntityFrameworkCore;
+using SharedTestingHelper.Fakes;
+using WebApiTestProject.Entities;
+
+public class FilterParameterTests() : TestBase
+{
+    [Theory]
+    [InlineData("""Title == "lamb" """, """Title == "chicken" """)]
+    [InlineData("""Title @=* "lamb" """, """Title @=* "chicken" """)]
+    [InlineData("Age > 30", "Age > 18")]
+    [InlineData("Rating > 3.5", "Rating > 4.5")]
+    [InlineData("BirthMonth == 1", "BirthMonth == 2")]
+    [InlineData("Favorite == true", "Favorite == false")]
+    [InlineData("SpecificDate > 2022-07-01T00:00:03Z", "SpecificDate > 2023-01-01T00:00:00Z")]
+    [InlineData("SpecificDateTime > 2022-07-01T00:00:03Z", "SpecificDateTime > 2023-01-01T00:00:00Z")]
+    [InlineData("Date == 2022-07-01", "Date == 2023-01-01")]
+    [InlineData("Time == 00:00:03", "Time == 12:30:00")]
+    [InlineData("""Id == "aa648248-cb69-4217-ac95-d7484795afb2" """, """Id == "bb648248-cb69-4217-ac95-d7484795afb2" """)]
+    [InlineData("""Title ^^ ["lamb", "chicken"]""", """Title ^^ ["beef", "pork", "tofu"]""")]
+    [InlineData("""Title ^^* ["lamb", "chicken"]""", """Title ^^* ["beef", "pork", "tofu"]""")]
+    [InlineData("""Title !^^ ["lamb", "chicken"]""", """Title !^^ ["beef", "pork", "tofu"]""")]
+    [InlineData("(Age + 5) > 30", "(Age + 7) > 18")]
+    public void filters_that_differ_only_in_values_share_one_parameterized_query(string first, string second)
+    {
+        var testingServiceScope = new TestingServiceScope();
+
+        var firstSql = SqlWithoutParameterValues(testingServiceScope, first);
+        var secondSql = SqlWithoutParameterValues(testingServiceScope, second);
+
+        firstSql.Should().Contain("@");
+        firstSql.Should().Be(secondSql);
+    }
+
+    [Fact]
+    public async Task in_list_is_one_array_parameter_and_still_filters()
+    {
+        var testingServiceScope = new TestingServiceScope();
+        var lamb = new FakeTestingPersonBuilder().WithTitle($"lamb {Guid.NewGuid()}").Build();
+        var chicken = new FakeTestingPersonBuilder().WithTitle($"chicken {Guid.NewGuid()}").Build();
+        var beef = new FakeTestingPersonBuilder().WithTitle($"beef {Guid.NewGuid()}").Build();
+        await testingServiceScope.InsertAsync(lamb, chicken, beef);
+
+        var input = $"""Title ^^* ["{lamb.Title!.ToUpper()}", "{chicken.Title}"]""";
+        var query = testingServiceScope.DbContext().People.ApplyQueryKitFilter(input);
+        var people = await query.ToListAsync();
+
+        SqlWithoutParameterValues(query).Should().Contain("= ANY (@");
+        people.Select(x => x.Id).Should().BeEquivalentTo(new[] { lamb.Id, chicken.Id });
+    }
+
+    private static string SqlWithoutParameterValues(TestingServiceScope testingServiceScope, string input)
+        => SqlWithoutParameterValues(testingServiceScope.DbContext().People.ApplyQueryKitFilter(input));
+
+    // ToQueryString() writes each parameter value in a "-- @p='...'" comment line before the SQL.
+    private static string SqlWithoutParameterValues(IQueryable<TestingPerson> query)
+        => string.Join('\n', query.ToQueryString()
+            .Split('\n')
+            .Where(line => !line.StartsWith("--")));
+}

QueryKit.UnitTests/CustomFilterPropertyTests.cs

@@ -17,7 +17,7 @@ public void can_have_child_prop_name_ownsone()
         var value = faker.Lorem.Word();
         var input = $"""PhysicalAddress.State == "{value}" """;
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input);
-        filterExpression.ToString().Should().Be($"""x => (x.PhysicalAddress.State == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.PhysicalAddress.State == "{value}")""");
     }
 
     [Fact]
@@ -32,7 +32,7 @@ public void can_have_custom_child_prop_name_ownsone()
             config.Property<TestingPerson>(x => x.PhysicalAddress.State).HasQueryName("state");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.PhysicalAddress.State == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.PhysicalAddress.State == "{value}")""");
     }
 
     [Fact(Skip = "Will need something like this if i want to support HasConversion in efcore.")]
@@ -42,7 +42,7 @@ public void can_have_child_prop_name_for_efcore_HasConversion()
         var value = faker.Lorem.Word();
         var input = $"""Email.Value == "{value}" """;
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input);
-        filterExpression.ToString().Should().Be($"""x => (x.Email == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Email == "{value}")""");
     }
 
     [Fact(Skip = "Will need something like this if i want to support HasConversion in efcore.")]
@@ -57,7 +57,7 @@ public void can_have_custom_child_prop_name_for_efcore_HasConversion()
             config.Property<TestingPerson>(x => x.Email).HasQueryName("email");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.Email == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Email == "{value}")""");
     }
 
     [Fact]
@@ -72,7 +72,7 @@ public void can_have_custom_prop_name_for_string()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.Title == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Title == "{value}")""");
     }
 
     [Fact]
@@ -87,7 +87,7 @@ public void can_handle_alias_in_value()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.Title == "{value} with special_value")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Title == "{value} with special_value")""");
     }
 
     [Fact]
@@ -102,7 +102,7 @@ public void can_handle_alias_in_value_with_operator_after_it()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.Title == "{value} with special_value @=* a thing")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Title == "{value} with special_value @=* a thing")""");
     }
 
     [Fact]
@@ -119,7 +119,7 @@ public void can_have_custom_prop_name_for_multiple_props()
             config.Property<TestingPerson>(x => x.Id).HasQueryName("identifier");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (x.Id == {guidValue}))""");
+        filterExpression.ToDisplayString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (x.Id == {guidValue}))""");
     }
 
     [Fact]
@@ -135,7 +135,7 @@ public void can_have_custom_prop_name_for_some_props()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (x.Id == {guidValue}))""");
+        filterExpression.ToDisplayString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (x.Id == {guidValue}))""");
     }
 
     [Fact]
@@ -150,7 +150,7 @@ public void can_handle_case_insensitive_custom_props()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("specialtitle");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (x.Title == "{value}")""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (x.Title == "{value}")""");
     }
 
     [Fact]
@@ -167,7 +167,7 @@ public void can_have_custom_prop_excluded_from_filter()
             config.Property<TestingPerson>(x => x.Id).PreventFilter();
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (True == True))""");
+        filterExpression.ToDisplayString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (True == True))""");
     }
 
     [Fact]
@@ -184,7 +184,7 @@ public void can_have_custom_prop_excluded_from_filter_with_custom_propname()
             config.Property<TestingPerson>(x => x.Id).HasQueryName("identifier").PreventFilter();
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (True == True))""");
+        filterExpression.ToDisplayString().Should().Be($"""x => ((x.Title == "{stringValue}") OrElse (True == True))""");
     }
 
     [Fact]
@@ -197,7 +197,7 @@ public void can_have_custom_prop_name_with_in_operator()
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should()
+        filterExpression.ToDisplayString().Should()
             .Be("""x => value(System.Collections.Generic.List`1[System.String]).Contains(x.Title)""");
     }
 
@@ -215,7 +215,7 @@ public void can_have_custom_prop_name_with_regex_special_character_operators(str
             config.Property<TestingPerson>(x => x.Title!).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Contain("x.Title");
+        filterExpression.ToDisplayString().Should().Contain("x.Title");
     }
 
     [Theory]
@@ -232,7 +232,7 @@ public void can_have_custom_prop_name_with_has_operators(string comparisonOperat
             config.Property<Recipe>(x => x.Tags).HasQueryName("special_tags");
         });
         var filterExpression = FilterParser.ParseFilter<Recipe>(input, config);
-        filterExpression.ToString().Should().Contain("x.Tags");
+        filterExpression.ToDisplayString().Should().Contain("x.Tags");
     }
 
     [Fact]
@@ -247,7 +247,7 @@ public void can_have_custom_prop_work_with_collection_filters()
             config.Property<Recipe>(x => x.Title).HasQueryName("special_title");
         });
         var filterExpression = FilterParser.ParseFilter<Recipe>(input, config);
-        filterExpression.ToString().Should().Be(
+        filterExpression.ToDisplayString().Should().Be(
             $"""x => ((x.Title == "{stringValue}") AndAlso x.Ingredients.Select(y => y.Name).Any(z => (z == "flour")))""");
     }
 
@@ -263,7 +263,7 @@ public void can_have_derived_prop_work_with_collection_filters()
             config.DerivedProperty<Recipe>(x => x.Title + x.Directions).HasQueryName("special_title_directions");
         });
         var filterExpression = FilterParser.ParseFilter<Recipe>(input, config);
-        filterExpression.ToString().Should().Be(
+        filterExpression.ToDisplayString().Should().Be(
             $"""x => (((x.Title + x.Directions) == "{stringValue}") AndAlso x.Ingredients.Select(y => y.Name).Any(z => (z == "flour")))""");
     }
 
@@ -280,7 +280,7 @@ public void filter_prevented_props_always_have_true_equals_true_regardless_of_co
             config.Property<TestingPerson>(x => x.Id).PreventFilter();
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be($"""x => (True == True)""");
+        filterExpression.ToDisplayString().Should().Be($"""x => (True == True)""");
     }
 
     [Fact]
@@ -311,6 +311,6 @@ public void can_handle_nonexistent_property()
             config.AllowUnknownProperties = true;
         });
         var filterExpression = FilterParser.ParseFilter<TestingPerson>(input, config);
-        filterExpression.ToString().Should().Be("x => (True == True)");
+        filterExpression.ToDisplayString().Should().Be("x => (True == True)");
     }
 }

QueryKit.UnitTests/ExpressionDisplay.cs

@@ -0,0 +1,51 @@
+namespace QueryKit.UnitTests;
+
+using System.Linq.Expressions;
+
+// QueryKit sends filter values as FilterValue<T> field reads so that EF Core makes SQL parameters.
+// ToString() prints these reads as value(QueryKit.FilterValue`1[...]).Value, so the tests put the
+// values back inline. Date and time values print as constructor calls, the same as before.
+public static class ExpressionDisplay
+{
+    public static string ToDisplayString(this Expression expression)
+        => new InlineFilterValues().Visit(expression)!.ToString();
+
+    private sealed class InlineFilterValues : ExpressionVisitor
+    {
+        protected override Expression VisitMember(MemberExpression node)
+        {
+            if (node.Expression is not ConstantExpression { Value: { } holder }
+                || !holder.GetType().IsGenericType
+                || holder.GetType().GetGenericTypeDefinition() != typeof(FilterValue<>))
+                return base.VisitMember(node);
+
+            var value = ((System.Reflection.FieldInfo)node.Member).GetValue(holder);
+            return Inline(value, node.Type);
+        }
+
+        private static Expression Inline(object? value, Type type)
+        {
+            var underlying = Nullable.GetUnderlyingType(type);
+            var isDateOrTime = value is DateTime or DateTimeOffset or DateOnly or TimeOnly;
+            if (!isDateOrTime)
+                return Expression.Constant(value, type);
+
+            Expression newExpr = value switch
+            {
+                DateTime dt => New<DateTime>(new[] { typeof(long), typeof(DateTimeKind) }, dt.Ticks, dt.Kind),
+                DateTimeOffset dto => New<DateTimeOffset>(new[] { typeof(long), typeof(TimeSpan) }, dto.Ticks, dto.Offset),
+                DateOnly date => New<DateOnly>(new[] { typeof(int), typeof(int), typeof(int) }, date.Year, date.Month, date.Day),
+                TimeOnly time => New<TimeOnly>(new[] { typeof(int), typeof(int), typeof(int), typeof(int), typeof(int) },
+                    time.Hour, time.Minute, time.Second, time.Millisecond, time.Microsecond),
+                _ => throw new InvalidOperationException()
+            };
+
+            return underlying == null
+                ? newExpr
+                : Expression.New(type.GetConstructor(new[] { underlying })!, newExpr);
+        }
+
+        private static NewExpression New<T>(Type[] parameterTypes, params object[] args)
+            => Expression.New(typeof(T).GetConstructor(parameterTypes)!, args.Select(Expression.Constant));
+    }
+}

QueryKit.UnitTests/FilterParserAllocationTests.cs

@@ -0,0 +1,58 @@
+namespace QueryKit.UnitTests;
+
+using Configuration;
+using FluentAssertions;
+using WebApiTestProject.Entities;
+
+// The parsers and the alias regexes are built once, not on each parse. Each budget is about two
+// times the bytes that one parse allocates, and less than the bytes that one parse allocated when
+// each parse built the parsers and the regexes again.
+public class FilterParserAllocationTests
+{
+    private const int Iterations = 50;
+
+    [Fact]
+    public void a_simple_filter_does_not_build_the_parsers_again()
+    {
+        BytesForEachParse("Age > 25", null).Should().BeLessThan(100 * 1024);
+    }
+
+    [Fact]
+    public void property_aliases_do_not_build_a_regex_on_each_parse()
+    {
+        var config = new QueryKitConfiguration(c =>
+        {
+            c.Property<TestingPerson>(x => x.Title).HasQueryName("name");
+            c.Property<TestingPerson>(x => x.Age).HasQueryName("years");
+            c.Property<TestingPerson>(x => x.Rating).HasQueryName("score");
+            c.Property<TestingPerson>(x => x.FirstName).HasQueryName("first");
+            c.Property<TestingPerson>(x => x.Id).HasQueryName("key");
+        });
+
+        BytesForEachParse("""name == "lamb" && years > 25""", config).Should().BeLessThan(250 * 1024);
+    }
+
+    [Fact]
+    public void operator_aliases_do_not_build_a_regex_on_each_parse()
+    {
+        var config = new QueryKitConfiguration(c =>
+        {
+            c.EqualsOperator = "eq";
+            c.GreaterThanOperator = "gt";
+            c.AndOperator = "and";
+        });
+
+        BytesForEachParse("""Title eq "lamb" and Age gt 25""", config).Should().BeLessThan(200 * 1024);
+    }
+
+    private static long BytesForEachParse(string filter, IQueryKitConfiguration? config)
+    {
+        for (var i = 0; i < Iterations; i++)
+            FilterParser.ParseFilter<TestingPerson>(filter, config);
+
+        var before = GC.GetAllocatedBytesForCurrentThread();
+        for (var i = 0; i < Iterations; i++)
+            FilterParser.ParseFilter<TestingPerson>(filter, config);
+        return (GC.GetAllocatedBytesForCurrentThread() - before) / Iterations;
+    }
+}