diff --git a/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs b/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs index 5ffcf7ac..5555183f 100644 --- a/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs +++ b/pkgs/sdk/server/src/Internal/Events/DefaultEventProcessorWrapper.cs @@ -30,7 +30,8 @@ public void RecordEvaluationEvent(EventProcessorTypes.EvaluationEvent e) => TrackEvents = e.TrackEvents, DebugEventsUntilDate = e.DebugEventsUntilDate, SamplingRatio = e.SamplingRatio, - ExcludeFromSummaries = e.ExcludeFromSummaries + ExcludeFromSummaries = e.ExcludeFromSummaries, + OverrideAffected = e.OverrideAffected }); public void RecordIdentifyEvent(EventProcessorTypes.IdentifyEvent e) => diff --git a/pkgs/sdk/server/src/Internal/Events/EventFactory.cs b/pkgs/sdk/server/src/Internal/Events/EventFactory.cs index c0aead07..e2a35fb1 100644 --- a/pkgs/sdk/server/src/Internal/Events/EventFactory.cs +++ b/pkgs/sdk/server/src/Internal/Events/EventFactory.cs @@ -38,7 +38,10 @@ LdValue defaultValue TrackEvents = flag.TrackEvents || isExperiment, DebugEventsUntilDate = flag.DebugEventsUntilDate, SamplingRatio = flag.SamplingRatio, - ExcludeFromSummaries = flag.ExcludeFromSummaries + ExcludeFromSummaries = flag.ExcludeFromSummaries, + // The marking is carried on the reason of every result, whether or not the caller + // asked for reasons. The event processor keys on this scalar, not on the reason. + OverrideAffected = result.Reason.OverrideAffected }; } @@ -46,7 +49,8 @@ internal EvaluationEvent NewDefaultValueEvaluationEvent( FeatureFlag flag, Context context, LdValue defaultValue, - EvaluationErrorKind errorKind + EvaluationErrorKind errorKind, + bool overrideAffected ) { return new EvaluationEvent @@ -57,9 +61,11 @@ EvaluationErrorKind errorKind FlagVersion = flag.Version, Value = defaultValue, Default = defaultValue, - Reason = _withReasons ? EvaluationReason.ErrorReason(errorKind) : (EvaluationReason?)null, + Reason = _withReasons ? + EvaluationReason.ErrorReason(errorKind).WithOverrideAffected(overrideAffected) : (EvaluationReason?)null, TrackEvents = flag.TrackEvents, - DebugEventsUntilDate = flag.DebugEventsUntilDate + DebugEventsUntilDate = flag.DebugEventsUntilDate, + OverrideAffected = overrideAffected }; } diff --git a/pkgs/sdk/server/src/LdClient.cs b/pkgs/sdk/server/src/LdClient.cs index 6029429b..2402c798 100644 --- a/pkgs/sdk/server/src/LdClient.cs +++ b/pkgs/sdk/server/src/LdClient.cs @@ -566,10 +566,11 @@ public FeatureFlagsState AllFlagsState(Context context, params FlagsStateOption[ evalDetail.Value.Type, featureKey); - _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent( - featureFlag, context, defaultValue, EvaluationErrorKind.WrongType)); // The type mismatch replaces the reason. The evaluation read the same - // definitions, so the new reason keeps the override-affected marking. + // definitions, so the new reason and the event keep the override-affected marking. + _eventProcessor.RecordEvaluationEvent(eventFactory.NewDefaultValueEvaluationEvent( + featureFlag, context, defaultValue, EvaluationErrorKind.WrongType, + evalDetail.Reason.OverrideAffected)); return (new EvaluationDetail(defaultValueOfType, null, EvaluationReason.ErrorReason(EvaluationErrorKind.WrongType) .WithOverrideAffected(evalDetail.Reason.OverrideAffected)), featureFlag); diff --git a/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs b/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs index 3dc44845..cc02a1d3 100644 --- a/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs +++ b/pkgs/sdk/server/src/Subsystems/EventProcessorTypes.cs @@ -90,6 +90,21 @@ public struct EvaluationEvent /// If true the event will not be included in summaries. /// public bool ExcludeFromSummaries { get; set; } + + /// + /// True if a flag override affected the evaluation, directly or transitively. + /// + /// + /// + /// A marked evaluation produces no individual feature or debug event, whatever the flag's + /// configuration requests, and is counted in a separate summary counter that carries the + /// overrideAffected marker. + /// + /// + /// Flag overrides are currently experimental and subject to change. + /// + /// + public bool OverrideAffected { get; set; } } /// diff --git a/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs b/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs new file mode 100644 index 00000000..c437a7ce --- /dev/null +++ b/pkgs/sdk/server/test/LdClientOverrideEventsTest.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LaunchDarkly.Sdk.Internal.Events; +using LaunchDarkly.Sdk.Server.Internal.Model; +using LaunchDarkly.Sdk.Server.Subsystems; +using Xunit; +using Xunit.Abstractions; + +using static LaunchDarkly.Sdk.Server.Subsystems.DataStoreTypes; +using static LaunchDarkly.Sdk.Server.Subsystems.EventProcessorTypes; + +namespace LaunchDarkly.Sdk.Server +{ + // Analytics event behavior of override-affected evaluations: the evaluation records the client + // hands to the event processor carry the marking, and the event output contains no individual + // feature or debug event for a marked evaluation while its summary counter carries the marker. + public class LdClientOverrideEventsTest : BaseTest + { + private static readonly Context context = Context.New("userkey"); + + // The prerequisite tree used by the marking tests. Every flag requests individual events. + // + // top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + // --> plain-flag (LaunchDarkly) + // + // The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + private const string TopKey = "top-flag"; + private const string MidKey = "mid-flag"; + private const string LeafKey = "leaf-flag"; + private const string PlainKey = "plain-flag"; + + public LdClientOverrideEventsTest(ITestOutputHelper testOutput) : base(testOutput) { } + + private static FeatureFlagBuilder TrackedBoolFlag(string key) => + new FeatureFlagBuilder(key).Version(1).Variations(false, true).OffVariation(0).FallthroughVariation(1) + .TrackEvents(true); + + private static FullDataSet PrerequisiteTreeLaunchDarklyData() => + LdClientOverridesTest.FlagsOnly( + TrackedBoolFlag(TopKey).On(true).Prerequisites(new Prerequisite(MidKey, 1), new Prerequisite(PlainKey, 1)).Build(), + TrackedBoolFlag(MidKey).On(true).Prerequisites(new Prerequisite(LeafKey, 1)).Build(), + TrackedBoolFlag(PlainKey).On(true).Build(), + TrackedBoolFlag(LeafKey).On(false).Build()); + + private static FullDataSet PrerequisiteTreeOverrides() => + LdClientOverridesTest.FlagsOnly(TrackedBoolFlag(LeafKey).Version(2).On(true).Build()); + + // A null data set means the client can never obtain LaunchDarkly data. + private LdClient MakeClient(FullDataSet? launchDarklyData, TestOverrideSource source, + IComponentConfigurer events) + { + var dataSystem = Components.DataSystem().Custom().Overrides(source); + var config = BasicConfig().Events(events); + if (launchDarklyData.HasValue) + { + dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(launchDarklyData.Value)); + config.StartWaitTime(TimeSpan.FromSeconds(5)); + } + else + { + dataSystem.Synchronizers(MockComponents.MockDataSourceThatNeverStarts()); + } + return new LdClient(config.DataSystem(dataSystem).Build()); + } + + private static Dictionary EvaluationRecordsByKey(MockEventProcessor events) => + events.Events.OfType().ToDictionary(e => e.FlagKey, e => e); + + [Fact] + public void OverrideEvaluationEventCarriesTheMarking() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly( + LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of(true)))); + using (var client = MakeClient(null, source, events.AsSingletonFactory())) + { + Assert.True(client.BoolVariation("overridden-flag", context, false)); + + var records = EvaluationRecordsByKey(events); + Assert.Single(records); + Assert.True(records["overridden-flag"].OverrideAffected); + // The marking is carried even though the caller did not ask for reasons. + Assert.Null(records["overridden-flag"].Reason); + } + } + + [Fact] + public void PlainEvaluationEventIsNotMarked() + { + var events = new MockEventProcessor(); + var ldFlag = new FeatureFlagBuilder("flag").Version(1).OffWithValue(LdValue.Of(true)).Build(); + using (var client = MakeClient(LdClientOverridesTest.FlagsOnly(ldFlag), new TestOverrideSource(), + events.AsSingletonFactory())) + { + Assert.True(client.BoolVariation("flag", context, false)); + Assert.False(EvaluationRecordsByKey(events)["flag"].OverrideAffected); + } + } + + [Fact] + public void OverriddenPrerequisiteMarksTheDependentEvaluationRecords() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(PrerequisiteTreeOverrides()); + using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events.AsSingletonFactory())) + { + var detail = client.BoolVariationDetail(TopKey, context, false); + Assert.True(detail.Value); + Assert.Equal(EvaluationReasonKind.Fallthrough, detail.Reason.Kind); + Assert.True(detail.Reason.OverrideAffected); + + var records = EvaluationRecordsByKey(events); + Assert.Equal(4, records.Count); + + // The top-level flag came from LaunchDarkly. Its record is marked because a definition + // read during its evaluation came from the override store. + Assert.True(records[TopKey].OverrideAffected); + Assert.Null(records[TopKey].PrerequisiteOf); + + // The intermediate prerequisite also came from LaunchDarkly. Its own subtree read the + // override, so its record is marked. + Assert.True(records[MidKey].OverrideAffected); + Assert.Equal(TopKey, records[MidKey].PrerequisiteOf); + + // The overridden leaf is marked directly, and it is recorded with the override version. + Assert.True(records[LeafKey].OverrideAffected); + Assert.Equal(MidKey, records[LeafKey].PrerequisiteOf); + Assert.Equal(2, records[LeafKey].FlagVersion); + + // The sibling prerequisite read nothing from the override store, so it is not marked. + Assert.False(records[PlainKey].OverrideAffected); + Assert.Equal(TopKey, records[PlainKey].PrerequisiteOf); + + // Every flag requested individual events. The marking alone decides which records the + // event processor keeps out of the individual event stream. + Assert.All(records.Values, r => Assert.True(r.TrackEvents)); + } + } + + [Fact] + public void WrongTypeEventOfOverriddenFlagIsMarked() + { + var events = new MockEventProcessor(); + var source = new TestOverrideSource(LdClientOverridesTest.FlagsOnly( + LdClientOverridesTest.SingleValueFlag("overridden-flag", LdValue.Of("not-a-bool")))); + using (var client = MakeClient(null, source, events.AsSingletonFactory())) + { + var detail = client.BoolVariationDetail("overridden-flag", context, false); + Assert.Equal(EvaluationErrorKind.WrongType, detail.Reason.ErrorKind); + + var record = EvaluationRecordsByKey(events)["overridden-flag"]; + Assert.True(record.OverrideAffected); + Assert.Equal(LdValue.Of(false), record.Value); + Assert.Null(record.Variation); + Assert.True(record.Reason.HasValue); + Assert.True(record.Reason.Value.OverrideAffected); + Assert.Equal(EvaluationErrorKind.WrongType, record.Reason.Value.ErrorKind); + } + } + + [Fact] + public void OverrideAffectedEvaluationsAppearOnlyInSummaryOutput() + { + var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents }; + var events = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1)); + var source = new TestOverrideSource(PrerequisiteTreeOverrides()); + using (var client = MakeClient(PrerequisiteTreeLaunchDarklyData(), source, events)) + { + Assert.True(client.BoolVariation(TopKey, context, false)); + Assert.True(client.FlushAndWait(TimeSpan.FromSeconds(5))); + } + + var payload = LdValue.Parse(sender.RequirePayload().Data); + var featureEventKeys = new List(); + var debugEventKeys = new List(); + LdValue summary = LdValue.Null; + foreach (var e in payload.List) + { + switch (e.Get("kind").AsString) + { + case "feature": + featureEventKeys.Add(e.Get("key").AsString); + break; + case "debug": + debugEventKeys.Add(e.Get("key").AsString); + break; + case "summary": + summary = e; + break; + } + } + + // Only the unaffected sibling produces an individual event, even though every flag in the + // tree has event tracking on. + Assert.Equal(new[] { PlainKey }, featureEventKeys); + Assert.Empty(debugEventKeys); + + Assert.False(summary.IsNull); + var features = summary.Get("features"); + foreach (var key in new[] { TopKey, MidKey, LeafKey, PlainKey }) + { + var counters = features.Get(key).Get("counters"); + Assert.True(counters.Count == 1, key + ": expected one counter"); + var marker = counters.Get(0).Get("overrideAffected"); + if (key == PlainKey) + { + Assert.True(marker.IsNull, key + ": the marker is omitted for an unaffected counter"); + } + else + { + Assert.Equal(LdValue.Of(true), marker); + } + } + } + + [Fact] + public void DefaultEventProcessorWrapperPassesTheMarkingThrough() + { + var sender = new MockEventSender { FilterKind = EventDataKind.AnalyticsEvents }; + var processor = Components.SendEvents().EventSender(sender).FlushInterval(TimeSpan.FromHours(1)) + .Build(ContextFrom(BasicConfig().Build())); + using (processor) + { + processor.RecordEvaluationEvent(new EvaluationEvent + { + Timestamp = UnixMillisecondTime.Now, + Context = context, + FlagKey = "flag", + FlagVersion = 3, + Variation = 0, + Value = LdValue.Of(true), + Default = LdValue.Of(false), + TrackEvents = true, + OverrideAffected = true + }); + Assert.True(processor.FlushAndWait(TimeSpan.FromSeconds(5))); + } + var payload = LdValue.Parse(sender.RequirePayload().Data); + Assert.DoesNotContain(payload.List, e => e.Get("kind").AsString == "feature"); + var counters = payload.List.First(e => e.Get("kind").AsString == "summary") + .Get("features").Get("flag").Get("counters"); + Assert.Equal(LdValue.Of(true), counters.Get(0).Get("overrideAffected")); + } + } +} diff --git a/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs b/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs index 62204213..d1873ea1 100644 --- a/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs +++ b/pkgs/sdk/server/test/LdClientOverrideVectorsTest.cs @@ -4,6 +4,7 @@ using System.Linq; using LaunchDarkly.Sdk.Json; using LaunchDarkly.Sdk.Server.Internal.Model; +using LaunchDarkly.Sdk.Server.Subsystems; using Xunit; using Xunit.Abstractions; @@ -57,8 +58,9 @@ public void Vector(int index, string description) var overrideDataSet = DataSetFrom(overrides.Get("flags"), overrides.Get("segments"), overrides.Get("flagValues")); var source = new TestOverrideSource(overrideDataSet); + var events = new MockEventProcessor(); var dataSystem = Components.DataSystem().Custom().Overrides(source); - var config = BasicConfig(); + var config = BasicConfig().Events(events.AsSingletonFactory()); if (initialized) { dataSystem.Synchronizers(MockComponents.MockDataSourceWithData(ldDataSet)); @@ -92,6 +94,19 @@ public void Vector(int index, string description) Assert.Equal(expectedVariation.AsInt, detail.VariationIndex); } AssertReason(expect.Get("reason"), detail.Reason); + + // summaryOverrideAffected is the marking the client hands to the event processor for + // this evaluation. The event processor keys individual-event suppression and the summary + // counter marker on that scalar, not on the reason. + var expectedSummaryMarker = expect.Get("summaryOverrideAffected"); + if (!expectedSummaryMarker.IsNull) + { + var flagKey = evaluate.Get("flagKey").AsString; + var records = events.Events.OfType() + .Where(e => e.FlagKey == flagKey).ToList(); + Assert.True(records.Count == 1, "expected exactly one evaluation record for the flag"); + Assert.Equal(expectedSummaryMarker.AsBool, records[0].OverrideAffected); + } } }