From 9ddeb282ee6375aad56a6bb2f77ab774ea7545f6 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:42:52 -0700 Subject: [PATCH] feat: Mark evaluation events as override-affected EventProcessorTypes.EvaluationEvent gains an OverrideAffected property. The event factory sets it from the evaluation result for every evaluation event and prerequisite record, whether or not the caller asked for reasons, and the event processor wrapper passes it through, so a marked evaluation produces no individual feature or debug event and is counted under a separate summary counter. A wrong-type result keeps the marking on its reason and its event. The OVERRIDE specification's test vectors now also assert the per-evaluation summary contribution. --- .../Events/DefaultEventProcessorWrapper.cs | 3 +- .../src/Internal/Events/EventFactory.cs | 14 +- pkgs/sdk/server/src/LdClient.cs | 7 +- .../src/Subsystems/EventProcessorTypes.cs | 15 ++ .../server/test/LdClientOverrideEventsTest.cs | 246 ++++++++++++++++++ .../test/LdClientOverrideVectorsTest.cs | 17 +- 6 files changed, 293 insertions(+), 9 deletions(-) create mode 100644 pkgs/sdk/server/test/LdClientOverrideEventsTest.cs diff --git a/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs b/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs index 5ffcf7ac..5555183f 100644 --- a/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs +++ b/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs @@ -30,7 +30,8 @@ public void RecordEvaluationEvent(EventProcessorTypes.EvaluationEvent e) => TrackEvents = e.TrackEvents, DebugEventsUntilDate = e.DebugEventsUntilDate, SamplingRatio = e.SamplingRatio, - ExcludeFromSummaries = e.ExcludeFromSummaries + ExcludeFromSummaries = e.ExcludeFromSummaries, + OverrideAffected = e.OverrideAffected }); public void RecordIdentifyEvent(EventProcessorTypes.IdentifyEvent e) => diff --git a/pkgs/sdk/server/src/Internal/Events/EventFactory.cs b/pkgs/sdk/server/src/Internal/Events/EventFactory.cs index c0aead07..e2a35fb1 100644 --- a/pkgs/sdk/server/src/Internal/Events/EventFactory.cs +++ b/pkgs/sdk/server/src/Internal/Events/EventFactory.cs @@ -38,7 +38,10 @@ LdValue defaultValue TrackEvents = flag.TrackEvents || isExperiment, DebugEventsUntilDate = flag.DebugEventsUntilDate, SamplingRatio = flag.SamplingRatio, - ExcludeFromSummaries = flag.ExcludeFromSummaries + ExcludeFromSummaries = flag.ExcludeFromSummaries, + // The marking is carried on the reason of every result, whether or not the caller + // asked for reasons. The event processor keys on this scalar, not on the reason. + OverrideAffected = result.Reason.OverrideAffected }; } @@ -46,7 +49,8 @@ internal EvaluationEvent NewDefaultValueEvaluationEvent( FeatureFlag flag, Context context, LdValue defaultValue, - EvaluationErrorKind errorKind + EvaluationErrorKind errorKind, + bool overrideAffected ) { return new EvaluationEvent @@ -57,9 +61,11 @@ EvaluationErrorKind errorKind FlagVersion = flag.Version, Value = defaultValue, Default = defaultValue, - Reason = _withReasons ? EvaluationReason.ErrorReason(errorKind) : (EvaluationReason?)null, + Reason = _withReasons ? + EvaluationReason.ErrorReason(errorKind).WithOverrideAffected(overrideAffected) : (EvaluationReason?)null, TrackEvents = flag.TrackEvents, - DebugEventsUntilDate = flag.DebugEventsUntilDate + DebugEventsUntilDate = flag.DebugEventsUntilDate, + OverrideAffected = overrideAffected }; } diff --git a/pkgs/sdk/server/src/LdClient.cs b/pkgs/sdk/server/src/LdClient.cs index 6029429b..2402c798 100644 --- a/pkgs/sdk/server/src/LdClient.cs +++ b/pkgs/sdk/server/src/LdClient.cs @@ -566,10 +566,11 @@ public FeatureFlagsState AllFlagsState(Context context, params FlagsStateOption[ evalDetail.Value.Type, featureKey); - _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent( - featureFlag, context, defaultValue, EvaluationErrorKind.WrongType)); // The type mismatch replaces the reason. The evaluation read the same - // definitions, so the new reason keeps the override-affected marking. + // definitions, so the new reason and the event keep the override-affected marking. + _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent( + featureFlag, context, defaultValue, EvaluationErrorKind.WrongType, + evalDetail.Reason.OverrideAffected)); return (new EvaluationDetail(defaultValueOfType, null, EvaluationReason.ErrorReason(EvaluationErrorKind.WrongType) .WithOverrideAffected(evalDetail.Reason.OverrideAffected)), featureFlag); diff --git a/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs b/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs index 3dc44845..cc02a1d3 100644 --- a/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs +++ b/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs @@ -90,6 +90,21 @@ public struct EvaluationEvent /// If true the event will not be included in summaries. /// public bool ExcludeFromSummaries { get; set; } + + /// + /// True if a flag override affected the evaluation, directly or transitively. + /// + /// + /// + /// A marked evaluation produces no individual feature or debug event, whatever the flag's + /// configuration requests, and is counted in a separate summary counter that carries the + /// overrideAffected marker. + /// + /// + /// Flag overrides are currently experimental and subject to change. + /// + /// + public bool OverrideAffected { get; set; } } /// diff --git a/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs b/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs new file mode 100644 index 00000000..c437a7ce --- /dev/null +++ b/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LaunchDarkly.Sdk.Internal.Events; +using LaunchDarkly.Sdk.Server.Internal.Model; +using LaunchDarkly.Sdk.Server.Subsystems; +using Xunit; +using Xunit.Abstractions; + +using static LaunchDarkly.Sdk.Server.Subsystems.DataStoreTypes; +using static LaunchDarkly.Sdk.Server.Subsystems.EventProcessorTypes; + +namespace LaunchDarkly.Sdk.Server +{ + // Analytics event behavior of override-affected evaluations: the evaluation records the client + // hands to the event processor carry the marking, and the event output contains no individual + // feature or debug event for a marked evaluation while its summary counter carries the marker. + public class LdClientOverrideEventsTest : BaseTest + { + private static readonly Context context = Context.New("userkey"); + + // The prerequisite tree used by the marking tests. Every flag requests individual events. + // + // top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + // --> plain-flag (LaunchDarkly) + // + // The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + private const string TopKey = "top-flag"; + private const string MidKey = "mid-flag"; + private const string LeafKey = "leaf-flag"; + private const string PlainKey = "plain-flag"; + + public LdClientOverrideEventsTest(ITestOutputHelper testOutput) : base(testOutput) { } + + private static FeatureFlagBuilder TrackedBoolFlag(string key) => + new FeatureFlagBuilder(key).Version(1).Variations(false, true).OffVariation(0).FallthroughVariation(1) + .TrackEvents(true); + + private static FullDataSet PrerequisiteTreeLaunchDarklyData() => + LdClientOverridesTest.FlagsOnly( + TrackedBoolFlag(TopKey).On(true).Prerequisites(new Prerequisite(MidKey, 1), new Prerequisite(PlainKey, 1)).Build(), + TrackedBoolFlag(MidKey).On(true).Prerequisites(new Prerequisite(LeafKey, 1)).Build(), + TrackedBoolFlag(PlainKey).On(true).Build(), + TrackedBoolFlag(LeafKey).On(false).Build()); + + private static FullDataSet PrerequisiteTreeOverrides() => + LdClientOverridesTest.FlagsOnly(TrackedBoolFlag(LeafKey).Version(2).On(true).Build()); + + // A null data set means the client can never obtain LaunchDarkly data. + private LdClient MakeClient(FullDataSet? launchDarklyData, TestOverrideSource source, + IComponentConfigurer events) + { + var dataSystem = Components.DataSystem().Custom().Overrides(source); + var config = BasicConfig().Events(events); + if (launchDarklyData.HasValue) + { + dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(launchDarklyData.Value)); + config.StartWaitTime(TimeSpan.FromSeconds(5)); + } + else + { + dataSystem.Synchronizers(MockComponents.MockDataSourceThatNeverStarts()); + } + return new LdClient(config.DataSystem(dataSystem).Build()); + } + + private static Dictionary EvaluationRecordsByKey(MockEventProcessor events) => + events.Events.OfType().ToDictionary(e => e.FlagKey, e => e); + + [Fact] + public void OverrideEvaluationEventCarriesTheMarking() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly( + LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of(true)))); + using (var client = MakeClient(null, source, events.AsSingletonFactory())) + { + Assert.True(client.BoolVariation("overridden-flag", context, false)); + + var records = EvaluationRecordsByKey(events); + Assert.Single(records); + Assert.True(records["overridden-flag"].OverrideAffected); + // The marking is carried even though the caller did not ask for reasons. + Assert.Null(records["overridden-flag"].Reason); + } + } + + [Fact] + public void PlainEvaluationEventIsNotMarked() + { + var events = new MockEventProcessor(); + var ldFlag = new FeatureFlagBuilder("flag").Version(1).OffWithValue(LdValue.Of(true)).Build(); + using (var client = MakeClient(LdClientOverridesTest.FlagsOnly(ldFlag), new TestOverrideSource(), + events.AsSingletonFactory())) + { + Assert.True(client.BoolVariation("flag", context, false)); + Assert.False(EvaluationRecordsByKey(events)["flag"].OverrideAffected); + } + } + + [Fact] + public void OverriddenPrerequisiteMarksTheDependentEvaluationRecords() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(PrerequisiteTreeOverrides()); + using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events.AsSingletonFactory())) + { + var detail = client.BoolVariationDetail(TopKey, context, false); + Assert.True(detail.Value); + Assert.Equal(EvaluationReasonKind.Fallthrough, detail.Reason.Kind); + Assert.True(detail.Reason.OverrideAffected); + + var records = EvaluationRecordsByKey(events); + Assert.Equal(4, records.Count); + + // The top-level flag came from LaunchDarkly. Its record is marked because a definition + // read during its evaluation came from the override store. + Assert.True(records[TopKey].OverrideAffected); + Assert.Null(records[TopKey].PrerequisiteOf); + + // The intermediate prerequisite also came from LaunchDarkly. Its own subtree read the + // override, so its record is marked. + Assert.True(records[MidKey].OverrideAffected); + Assert.Equal(TopKey, records[MidKey].PrerequisiteOf); + + // The overridden leaf is marked directly, and it is recorded with the override version. + Assert.True(records[LeafKey].OverrideAffected); + Assert.Equal(MidKey, records[LeafKey].PrerequisiteOf); + Assert.Equal(2, records[LeafKey].FlagVersion); + + // The sibling prerequisite read nothing from the override store, so it is not marked. + Assert.False(records[PlainKey].OverrideAffected); + Assert.Equal(TopKey, records[PlainKey].PrerequisiteOf); + + // Every flag requested individual events. The marking alone decides which records the + // event processor keeps out of the individual event stream. + Assert.All(records.Values, r => Assert.True(r.TrackEvents)); + } + } + + [Fact] + public void WrongTypeEventOfOverriddenFlagIsMarked() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly( + LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of("not-a-bool")))); + using (var client = MakeClient(null, source, events.AsSingletonFactory())) + { + var detail = client.BoolVariationDetail("overridden-flag", context, false); + Assert.Equal(EvaluationErrorKind.WrongType, detail.Reason.ErrorKind); + + var record = EvaluationRecordsByKey(events)["overridden-flag"]; + Assert.True(record.OverrideAffected); + Assert.Equal(LdValue.Of(false), record.Value); + Assert.Null(record.Variation); + Assert.True(record.Reason.HasValue); + Assert.True(record.Reason.Value.OverrideAffected); + Assert.Equal(EvaluationErrorKind.WrongType, record.Reason.Value.ErrorKind); + } + } + + [Fact] + public void OverrideAffectedEvaluationsAppearOnlyInSummaryOutput() + { + var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents }; + var events = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1)); + var source = new TestOverrideSource(PrerequisiteTreeOverrides()); + using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events)) + { + Assert.True(client.BoolVariation(TopKey, context, false)); + Assert.True(client.FlushAndWait(TimeSpan.FromSeconds(5))); + } + + var payload = LdValue.Parse(sender.RequirePayload().Data); + var featureEventKeys = new List(); + var debugEventKeys = new List(); + LdValue summary = LdValue.Null; + foreach (var e in payload.List) + { + switch (e.Get("kind").AsString) + { + case "feature": + featureEventKeys.Add(e.Get("key").AsString); + break; + case "debug": + debugEventKeys.Add(e.Get("key").AsString); + break; + case "summary": + summary = e; + break; + } + } + + // Only the unaffected sibling produces an individual event, even though every flag in the + // tree has event tracking on. + Assert.Equal(new[] { PlainKey }, featureEventKeys); + Assert.Empty(debugEventKeys); + + Assert.False(summary.IsNull); + var features = summary.Get("features"); + foreach (var key in new[] { TopKey, MidKey, LeafKey, PlainKey }) + { + var counters = features.Get(key).Get("counters"); + Assert.True(counters.Count == 1, key + ": expected one counter"); + var marker = counters.Get(0).Get("overrideAffected"); + if (key == PlainKey) + { + Assert.True(marker.IsNull, key + ": the marker is omitted for an unaffected counter"); + } + else + { + Assert.Equal(LdValue.Of(true), marker); + } + } + } + + [Fact] + public void DefaultEventProcessorWrapperPassesTheMarkingThrough() + { + var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents }; + var processor = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1)) + .Build(ContextFrom(BasicConfig().Build())); + using (processor) + { + processor.RecordEvaluationEvent(new EvaluationEvent + { + Timestamp = UnixMillisecondTime.Now, + Context = context, + FlagKey = "flag", + FlagVersion = 3, + Variation = 0, + Value = LdValue.Of(true), + Default = LdValue.Of(false), + TrackEvents = true, + OverrideAffected = true + }); + Assert.True(processor.FlushAndWait(TimeSpan.FromSeconds(5))); + } + var payload = LdValue.Parse(sender.RequirePayload().Data); + Assert.DoesNotContain(payload.List, e => e.Get("kind").AsString == "feature"); + var counters = payload.List.First(e => e.Get("kind").AsString == "summary") + .Get("features").Get("flag").Get("counters"); + Assert.Equal(LdValue.Of(true), counters.Get(0).Get("overrideAffected")); + } + } +} diff --git a/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs b/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs index 62204213..d1873ea1 100644 --- a/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs +++ b/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs @@ -4,6 +4,7 @@ using System.Linq; using LaunchDarkly.Sdk.Json; using LaunchDarkly.Sdk.Server.Internal.Model; +using LaunchDarkly.Sdk.Server.Subsystems; using Xunit; using Xunit.Abstractions; @@ -57,8 +58,9 @@ public void Vector(int index, string description) var overrideDataSet = DataSetFrom(overrides.Get("flags"), overrides.Get("segments"), overrides.Get("flagValues")); var source = new TestOverrideSource(overrideDataSet); + var events = new MockEventProcessor(); var dataSystem = Components.DataSystem().Custom().Overrides(source); - var config = BasicConfig(); + var config = BasicConfig().Events(events.AsSingletonFactory()); if (initialized) { dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(ldDataSet)); @@ -92,6 +94,19 @@ public void Vector(int index, string description) Assert.Equal(expectedVariation.AsInt, detail.VariationIndex); } AssertReason(expect.Get("reason"), detail.Reason); + + // summaryOverrideAffected is the marking the client hands to the event processor for + // this evaluation. The event processor keys individual-event suppression and the summary + // counter marker on that scalar, not on the reason. + var expectedSummaryMarker = expect.Get("summaryOverrideAffected"); + if (!expectedSummaryMarker.IsNull) + { + var flagKey = evaluate.Get("flagKey").AsString; + var records = events.Events.OfType() + .Where(e => e.FlagKey == flagKey).ToList(); + Assert.True(records.Count == 1, "expected exactly one evaluation record for the flag"); + Assert.Equal(expectedSummaryMarker.AsBool, records[0].OverrideAffected); + } } }