feat: Mark evaluation events as override-affected

pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs

@@ -30,7 +30,8 @@ public void RecordEvaluationEvent(EventProcessorTypes.EvaluationEvent e) =>
                 TrackEvents = e.TrackEvents,
                 DebugEventsUntilDate = e.DebugEventsUntilDate,
                 SamplingRatio = e.SamplingRatio,
-                ExcludeFromSummaries = e.ExcludeFromSummaries
+                ExcludeFromSummaries = e.ExcludeFromSummaries,
+                OverrideAffected = e.OverrideAffected
             });
 
         public void RecordIdentifyEvent(EventProcessorTypes.IdentifyEvent e) =>

pkgs/sdk/server/src/Internal/Events/EventFactory.cs

@@ -38,15 +38,19 @@ LdValue defaultValue
                 TrackEvents = flag.TrackEvents || isExperiment,
                 DebugEventsUntilDate = flag.DebugEventsUntilDate,
                 SamplingRatio = flag.SamplingRatio,
-                ExcludeFromSummaries = flag.ExcludeFromSummaries
+                ExcludeFromSummaries = flag.ExcludeFromSummaries,
+                // The marking is carried on the reason of every result, whether or not the caller
+                // asked for reasons. The event processor keys on this scalar, not on the reason.
+                OverrideAffected = result.Reason.OverrideAffected
             };
         }
 
         internal EvaluationEvent NewDefaultValueEvaluationEvent(
             FeatureFlag flag,
             Context context,
             LdValue defaultValue,
-            EvaluationErrorKind errorKind
+            EvaluationErrorKind errorKind,
+            bool overrideAffected
             )
         {
             return new EvaluationEvent
@@ -57,9 +61,11 @@ EvaluationErrorKind errorKind
                 FlagVersion = flag.Version,
                 Value = defaultValue,
                 Default = defaultValue,
-                Reason = _withReasons ? EvaluationReason.ErrorReason(errorKind) : (EvaluationReason?)null,
+                Reason = _withReasons ?
+                    EvaluationReason.ErrorReason(errorKind).WithOverrideAffected(overrideAffected) : (EvaluationReason?)null,
                 TrackEvents = flag.TrackEvents,
-                DebugEventsUntilDate = flag.DebugEventsUntilDate
+                DebugEventsUntilDate = flag.DebugEventsUntilDate,
+                OverrideAffected = overrideAffected
             };
         }
 

pkgs/sdk/server/src/LdClient.cs

@@ -566,10 +566,11 @@ public FeatureFlagsState AllFlagsState(Context context, params FlagsStateOption[
                             evalDetail.Value.Type,
                             featureKey);
 
-                        _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent(
-                            featureFlag, context, defaultValue, EvaluationErrorKind.WrongType));
                         // The type mismatch replaces the reason. The evaluation read the same
-                        // definitions, so the new reason keeps the override-affected marking.
+                        // definitions, so the new reason and the event keep the override-affected marking.
+                        _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent(
+                            featureFlag, context, defaultValue, EvaluationErrorKind.WrongType,
+                            evalDetail.Reason.OverrideAffected));
                         return (new EvaluationDetail<T>(defaultValueOfType, null,
                             EvaluationReason.ErrorReason(EvaluationErrorKind.WrongType)
                                 .WithOverrideAffected(evalDetail.Reason.OverrideAffected)), featureFlag);

pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs

@@ -90,6 +90,21 @@ public struct EvaluationEvent
             /// If true the event will not be included in summaries.
             /// </summary>
             public bool ExcludeFromSummaries { get; set; }
+
+            /// <summary>
+            /// True if a flag override affected the evaluation, directly or transitively.
+            /// </summary>
+            /// <remarks>
+            /// <para>
+            /// A marked evaluation produces no individual feature or debug event, whatever the flag's
+            /// configuration requests, and is counted in a separate summary counter that carries the
+            /// <c>overrideAffected</c> marker.
+            /// </para>
+            /// <para>
+            /// Flag overrides are currently experimental and subject to change.
+            /// </para>
+            /// </remarks>
+            public bool OverrideAffected { get; set; }
         }
 
         /// <summary>

pkgs/sdk/server/test/LdClientOverrideEventsTest.cs

@@ -0,0 +1,246 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LaunchDarkly.Sdk.Internal.Events;
+using LaunchDarkly.Sdk.Server.Internal.Model;
+using LaunchDarkly.Sdk.Server.Subsystems;
+using Xunit;
+using Xunit.Abstractions;
+
+using static LaunchDarkly.Sdk.Server.Subsystems.DataStoreTypes;
+using static LaunchDarkly.Sdk.Server.Subsystems.EventProcessorTypes;
+
+namespace LaunchDarkly.Sdk.Server
+{
+    // Analytics event behavior of override-affected evaluations: the evaluation records the client
+    // hands to the event processor carry the marking, and the event output contains no individual
+    // feature or debug event for a marked evaluation while its summary counter carries the marker.
+    public class LdClientOverrideEventsTest : BaseTest
+    {
+        private static readonly Context context = Context.New("userkey");
+
+        // The prerequisite tree used by the marking tests. Every flag requests individual events.
+        //
+        //   top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden)
+        //                           --> plain-flag (LaunchDarkly)
+        //
+        // The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override.
+        private const string TopKey = "top-flag";
+        private const string MidKey = "mid-flag";
+        private const string LeafKey = "leaf-flag";
+        private const string PlainKey = "plain-flag";
+
+        public LdClientOverrideEventsTest(ITestOutputHelper testOutput) : base(testOutput) { }
+
+        private static FeatureFlagBuilder TrackedBoolFlag(string key) =>
+            new FeatureFlagBuilder(key).Version(1).Variations(false, true).OffVariation(0).FallthroughVariation(1)
+                .TrackEvents(true);
+
+        private static FullDataSet<ItemDescriptor> PrerequisiteTreeLaunchDarklyData() =>
+            LdClientOverridesTest.FlagsOnly(
+                TrackedBoolFlag(TopKey).On(true).Prerequisites(new Prerequisite(MidKey, 1), new Prerequisite(PlainKey, 1)).Build(),
+                TrackedBoolFlag(MidKey).On(true).Prerequisites(new Prerequisite(LeafKey, 1)).Build(),
+                TrackedBoolFlag(PlainKey).On(true).Build(),
+                TrackedBoolFlag(LeafKey).On(false).Build());
+
+        private static FullDataSet<ItemDescriptor> PrerequisiteTreeOverrides() =>
+            LdClientOverridesTest.FlagsOnly(TrackedBoolFlag(LeafKey).Version(2).On(true).Build());
+
+        // A null data set means the client can never obtain LaunchDarkly data.
+        private LdClient MakeClient(FullDataSet<ItemDescriptor>? launchDarklyData, TestOverrideSource source,
+            IComponentConfigurer<IEventProcessor> events)
+        {
+            var dataSystem = Components.DataSystem().Custom().Overrides(source);
+            var config = BasicConfig().Events(events);
+            if (launchDarklyData.HasValue)
+            {
+                dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(launchDarklyData.Value));
+                config.StartWaitTime(TimeSpan.FromSeconds(5));
+            }
+            else
+            {
+                dataSystem.Synchronizers(MockComponents.MockDataSourceThatNeverStarts());
+            }
+            return new LdClient(config.DataSystem(dataSystem).Build());
+        }
+
+        private static Dictionary<string, EvaluationEvent> EvaluationRecordsByKey(MockEventProcessor events) =>
+            events.Events.OfType<EvaluationEvent>().ToDictionary(e => e.FlagKey, e => e);
+
+        [Fact]
+        public void OverrideEvaluationEventCarriesTheMarking()
+        {
+            var events = new MockEventProcessor();
+            var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly(
+                LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of(true))));
+            using (var client = MakeClient(null, source, events.AsSingletonFactory<IEventProcessor>()))
+            {
+                Assert.True(client.BoolVariation("overridden-flag", context, false));
+
+                var records = EvaluationRecordsByKey(events);
+                Assert.Single(records);
+                Assert.True(records["overridden-flag"].OverrideAffected);
+                // The marking is carried even though the caller did not ask for reasons.
+                Assert.Null(records["overridden-flag"].Reason);
+            }
+        }
+
+        [Fact]
+        public void PlainEvaluationEventIsNotMarked()
+        {
+            var events = new MockEventProcessor();
+            var ldFlag = new FeatureFlagBuilder("flag").Version(1).OffWithValue(LdValue.Of(true)).Build();
+            using (var client = MakeClient(LdClientOverridesTest.FlagsOnly(ldFlag), new TestOverrideSource(),
+                events.AsSingletonFactory<IEventProcessor>()))
+            {
+                Assert.True(client.BoolVariation("flag", context, false));
+                Assert.False(EvaluationRecordsByKey(events)["flag"].OverrideAffected);
+            }
+        }
+
+        [Fact]
+        public void OverriddenPrerequisiteMarksTheDependentEvaluationRecords()
+        {
+            var events = new MockEventProcessor();
+            var source = new TestOverrideSource(PrerequisiteTreeOverrides());
+            using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events.AsSingletonFactory<IEventProcessor>()))
+            {
+                var detail = client.BoolVariationDetail(TopKey, context, false);
+                Assert.True(detail.Value);
+                Assert.Equal(EvaluationReasonKind.Fallthrough, detail.Reason.Kind);
+                Assert.True(detail.Reason.OverrideAffected);
+
+                var records = EvaluationRecordsByKey(events);
+                Assert.Equal(4, records.Count);
+
+                // The top-level flag came from LaunchDarkly. Its record is marked because a definition
+                // read during its evaluation came from the override store.
+                Assert.True(records[TopKey].OverrideAffected);
+                Assert.Null(records[TopKey].PrerequisiteOf);
+
+                // The intermediate prerequisite also came from LaunchDarkly. Its own subtree read the
+                // override, so its record is marked.
+                Assert.True(records[MidKey].OverrideAffected);
+                Assert.Equal(TopKey, records[MidKey].PrerequisiteOf);
+
+                // The overridden leaf is marked directly, and it is recorded with the override version.
+                Assert.True(records[LeafKey].OverrideAffected);
+                Assert.Equal(MidKey, records[LeafKey].PrerequisiteOf);
+                Assert.Equal(2, records[LeafKey].FlagVersion);
+
+                // The sibling prerequisite read nothing from the override store, so it is not marked.
+                Assert.False(records[PlainKey].OverrideAffected);
+                Assert.Equal(TopKey, records[PlainKey].PrerequisiteOf);
+
+                // Every flag requested individual events. The marking alone decides which records the
+                // event processor keeps out of the individual event stream.
+                Assert.All(records.Values, r => Assert.True(r.TrackEvents));
+            }
+        }
+
+        [Fact]
+        public void WrongTypeEventOfOverriddenFlagIsMarked()
+        {
+            var events = new MockEventProcessor();
+            var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly(
+                LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of("not-a-bool"))));
+            using (var client = MakeClient(null, source, events.AsSingletonFactory<IEventProcessor>()))
+            {
+                var detail = client.BoolVariationDetail("overridden-flag", context, false);
+                Assert.Equal(EvaluationErrorKind.WrongType, detail.Reason.ErrorKind);
+
+                var record = EvaluationRecordsByKey(events)["overridden-flag"];
+                Assert.True(record.OverrideAffected);
+                Assert.Equal(LdValue.Of(false), record.Value);
+                Assert.Null(record.Variation);
+                Assert.True(record.Reason.HasValue);
+                Assert.True(record.Reason.Value.OverrideAffected);
+                Assert.Equal(EvaluationErrorKind.WrongType, record.Reason.Value.ErrorKind);
+            }
+        }
+
+        [Fact]
+        public void OverrideAffectedEvaluationsAppearOnlyInSummaryOutput()
+        {
+            var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents };
+            var events = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1));
+            var source = new TestOverrideSource(PrerequisiteTreeOverrides());
+            using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events))
+            {
+                Assert.True(client.BoolVariation(TopKey, context, false));
+                Assert.True(client.FlushAndWait(TimeSpan.FromSeconds(5)));
+            }
+
+            var payload = LdValue.Parse(sender.RequirePayload().Data);
+            var featureEventKeys = new List<string>();
+            var debugEventKeys = new List<string>();
+            LdValue summary = LdValue.Null;
+            foreach (var e in payload.List)
+            {
+                switch (e.Get("kind").AsString)
+                {
+                    case "feature":
+                        featureEventKeys.Add(e.Get("key").AsString);
+                        break;
+                    case "debug":
+                        debugEventKeys.Add(e.Get("key").AsString);
+                        break;
+                    case "summary":
+                        summary = e;
+                        break;
+                }
+            }
+
+            // Only the unaffected sibling produces an individual event, even though every flag in the
+            // tree has event tracking on.
+            Assert.Equal(new[] { PlainKey }, featureEventKeys);
+            Assert.Empty(debugEventKeys);
+
+            Assert.False(summary.IsNull);
+            var features = summary.Get("features");
+            foreach (var key in new[] { TopKey, MidKey, LeafKey, PlainKey })
+            {
+                var counters = features.Get(key).Get("counters");
+                Assert.True(counters.Count == 1, key + ": expected one counter");
+                var marker = counters.Get(0).Get("overrideAffected");
+                if (key == PlainKey)
+                {
+                    Assert.True(marker.IsNull, key + ": the marker is omitted for an unaffected counter");
+                }
+                else
+                {
+                    Assert.Equal(LdValue.Of(true), marker);
+                }
+            }
+        }
+
+        [Fact]
+        public void DefaultEventProcessorWrapperPassesTheMarkingThrough()
+        {
+            var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents };
+            var processor = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1))
+                .Build(ContextFrom(BasicConfig().Build()));
+            using (processor)
+            {
+                processor.RecordEvaluationEvent(new EvaluationEvent
+                {
+                    Timestamp = UnixMillisecondTime.Now,
+                    Context = context,
+                    FlagKey = "flag",
+                    FlagVersion = 3,
+                    Variation = 0,
+                    Value = LdValue.Of(true),
+                    Default = LdValue.Of(false),
+                    TrackEvents = true,
+                    OverrideAffected = true
+                });
+                Assert.True(processor.FlushAndWait(TimeSpan.FromSeconds(5)));
+            }
+            var payload = LdValue.Parse(sender.RequirePayload().Data);
+            Assert.DoesNotContain(payload.List, e => e.Get("kind").AsString == "feature");
+            var counters = payload.List.First(e => e.Get("kind").AsString == "summary")
+                .Get("features").Get("flag").Get("counters");
+            Assert.Equal(LdValue.Of(true), counters.Get(0).Get("overrideAffected"));
+        }
+    }
+}

pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using LaunchDarkly.Sdk.Json;
 using LaunchDarkly.Sdk.Server.Internal.Model;
+using LaunchDarkly.Sdk.Server.Subsystems;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -57,8 +58,9 @@ public void Vector(int index, string description)
             var overrideDataSet = DataSetFrom(overrides.Get("flags"), overrides.Get("segments"), overrides.Get("flagValues"));
 
             var source = new TestOverrideSource(overrideDataSet);
+            var events = new MockEventProcessor();
             var dataSystem = Components.DataSystem().Custom().Overrides(source);
-            var config = BasicConfig();
+            var config = BasicConfig().Events(events.AsSingletonFactory<IEventProcessor>());
             if (initialized)
             {
                 dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(ldDataSet));
@@ -92,6 +94,19 @@ public void Vector(int index, string description)
                     Assert.Equal(expectedVariation.AsInt, detail.VariationIndex);
                 }
                 AssertReason(expect.Get("reason"), detail.Reason);
+
+                // summaryOverrideAffected is the marking the client hands to the event processor for
+                // this evaluation. The event processor keys individual-event suppression and the summary
+                // counter marker on that scalar, not on the reason.
+                var expectedSummaryMarker = expect.Get("summaryOverrideAffected");
+                if (!expectedSummaryMarker.IsNull)
+                {
+                    var flagKey = evaluate.Get("flagKey").AsString;
+                    var records = events.Events.OfType<EventProcessorTypes.EvaluationEvent>()
+                        .Where(e => e.FlagKey == flagKey).ToList();
+                    Assert.True(records.Count == 1, "expected exactly one evaluation record for the flag");
+                    Assert.Equal(expectedSummaryMarker.AsBool, records[0].OverrideAffected);
+                }
             }
         }