diff --git a/pkgs/sdk/server/src/Integrations/DataSystemBuilder.cs b/pkgs/sdk/server/src/Integrations/DataSystemBuilder.cs index 56bd3257..16a50b8d 100644 --- a/pkgs/sdk/server/src/Integrations/DataSystemBuilder.cs +++ b/pkgs/sdk/server/src/Integrations/DataSystemBuilder.cs @@ -126,7 +126,7 @@ public DataSystemBuilder PersistentStore(IComponentConfigurer persis /// /// /// - /// the override source, such as the file-based override source; + /// the override source, such as ; /// null removes a previously configured source /// a reference to the builder public DataSystemBuilder Overrides(IComponentConfigurer overrideSource) diff --git a/pkgs/sdk/server/src/Integrations/FileOverrideSourceBuilder.cs b/pkgs/sdk/server/src/Integrations/FileOverrideSourceBuilder.cs new file mode 100644 index 00000000..c2a7ec50 --- /dev/null +++ b/pkgs/sdk/server/src/Integrations/FileOverrideSourceBuilder.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using LaunchDarkly.Sdk.Server.Internal; +using LaunchDarkly.Sdk.Server.Internal.Overrides; +using LaunchDarkly.Sdk.Server.Subsystems; + +namespace LaunchDarkly.Sdk.Server.Integrations +{ + /// + /// A builder for configuring the file-based override source. + /// + /// + /// + /// Obtain an instance with , call + /// to specify the override file(s), and pass the builder to + /// . + /// + /// + /// Flag overrides are currently experimental and subject to change. + /// + /// + /// + public sealed class FileOverrideSourceBuilder : IComponentConfigurer + { + /// + /// The interval at which the source examines the files for changes in polling mode when no interval + /// was specified: one second. The source reads local files rather than contacting a service, so a + /// short interval keeps an override responsive during an incident at negligible cost. + /// + public static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(1); + + /// + /// The shortest allowed polling interval: one second. A configured interval below this is raised to + /// it. The minimum exists only to prevent a tight loop over the file system. + /// + public static readonly TimeSpan MinimumPollInterval = TimeSpan.FromSeconds(1); + + internal readonly List _paths = new List(); + internal FileOverrideTypes.DuplicateKeysHandling _duplicateKeysHandling = FileOverrideTypes.DuplicateKeysHandling.Fail; + internal FileOverrideTypes.ChangeDetection _changeDetection = FileOverrideTypes.ChangeDetection.Polling; + internal TimeSpan _pollInterval = DefaultPollInterval; + internal Func _parser = null; + + internal FileOverrideSourceBuilder() { } + + /// + /// Adds one or more override files, specifying each file path as a string. + /// + /// + /// The order is significant. It determines which file wins under the duplicate keys handling when + /// the same key appears in more than one file. A relative path is resolved against the current + /// working directory when the client is created. A configured file does not need to exist. + /// + /// path(s) to the override file(s) + /// the same builder + public FileOverrideSourceBuilder FilePaths(params string[] paths) + { + _paths.AddRange(paths); + return this; + } + + /// + /// Specifies how to handle the same key appearing in more than one file. + /// + /// + /// The default is . A key defined in two + /// files is most likely a mistake. During an incident, a failed reload that leaves the last good + /// overrides in place and logs the conflict is safer than silently serving one of the two definitions. + /// + /// how duplicate keys should be handled + /// the same builder + public FileOverrideSourceBuilder DuplicateKeysHandling(FileOverrideTypes.DuplicateKeysHandling duplicateKeysHandling) + { + _duplicateKeysHandling = duplicateKeysHandling; + return this; + } + + /// + /// Selects how the source detects file changes. + /// + /// + /// The default is . The two modes are + /// alternatives, so setting one replaces the other. Change detection is always on: an override + /// source that needs a restart to pick up a change would defeat its purpose. + /// + /// the change detection mode + /// the same builder + public FileOverrideSourceBuilder ChangeDetection(FileOverrideTypes.ChangeDetection changeDetection) + { + _changeDetection = changeDetection; + return this; + } + + /// + /// Sets the interval between examinations of the files in polling mode. Watching mode ignores it. + /// + /// + /// The default is . An interval below + /// is raised to the minimum, and a warning is logged when the client is created. + /// + /// the polling interval + /// the same builder + public FileOverrideSourceBuilder PollInterval(TimeSpan pollInterval) + { + _pollInterval = pollInterval; + return this; + } + + /// + /// Specifies an alternate parsing function to use for non-JSON override files, such as YAML. + /// + /// + /// + /// By default, the source parses files as JSON objects. To avoid bringing in additional dependencies + /// that might conflict with application dependencies, the SDK does not import a YAML parser, but you + /// can use this method to supply one. The function takes the file content and returns an + /// object made of the basic types that can be represented in JSON: string, numbers, + /// booleans, List, and Dictionary<string, object>. It should throw an exception + /// if it cannot parse the content. + /// + /// + /// The source still parses a file as JSON if its first non-whitespace character is '{'. If that + /// fails, it uses the parsing function. See + /// for an example using the YamlDotNet package. + /// + /// + /// the parsing function + /// the same builder + public FileOverrideSourceBuilder Parser(Func parseFn) + { + _parser = parseFn; + return this; + } + + /// + /// Called internally by the SDK to create the override source. + /// + /// + /// Invalid configuration is reported the same way as for other components: an exception is thrown, + /// and the client is not created. A configured file that does not exist is not a configuration error. + /// + /// the client context + /// the override source + /// no file paths were specified + /// an option has a value outside its enumeration + public IOverrideSource Build(LdClientContext context) + { + if (_paths.Count == 0) + { + throw new ArgumentException("no file paths were specified for the file-based override source"); + } + var paths = _paths.Select(Path.GetFullPath).ToList(); + + switch (_duplicateKeysHandling) + { + case FileOverrideTypes.DuplicateKeysHandling.Fail: + case FileOverrideTypes.DuplicateKeysHandling.Ignore: + break; + default: + throw new ArgumentOutOfRangeException(nameof(DuplicateKeysHandling), _duplicateKeysHandling, + "unrecognized duplicate keys handling for the file-based override source"); + } + switch (_changeDetection) + { + case FileOverrideTypes.ChangeDetection.Polling: + case FileOverrideTypes.ChangeDetection.Watching: + break; + default: + throw new ArgumentOutOfRangeException(nameof(ChangeDetection), _changeDetection, + "unrecognized change detection mode for the file-based override source"); + } + + var logger = context.Logger.SubLogger(LogNames.OverridesSubLog); + var pollInterval = _pollInterval; + if (_changeDetection == FileOverrideTypes.ChangeDetection.Polling && pollInterval < MinimumPollInterval) + { + logger.Warn("Poll interval {0} is below the minimum; using {1}", pollInterval, MinimumPollInterval); + pollInterval = MinimumPollInterval; + } + + return new FileOverrideSource(paths, _duplicateKeysHandling, _changeDetection, pollInterval, _parser, logger); + } + } +} diff --git a/pkgs/sdk/server/src/Integrations/FileOverrideTypes.cs b/pkgs/sdk/server/src/Integrations/FileOverrideTypes.cs new file mode 100644 index 00000000..be53cc03 --- /dev/null +++ b/pkgs/sdk/server/src/Integrations/FileOverrideTypes.cs @@ -0,0 +1,50 @@ +namespace LaunchDarkly.Sdk.Server.Integrations +{ + /// + /// Types that are used in configuring . + /// + /// + /// Flag overrides are currently experimental and subject to change. + /// + public static class FileOverrideTypes + { + /// + /// Determines what happens when the same flag or segment key appears in more than one override file. + /// + /// + public enum DuplicateKeysHandling + { + /// + /// The reload fails, in the same way as a file that cannot be parsed. The previously loaded + /// overrides stay in effect. This is the default. + /// + Fail, + + /// + /// The entry from the first configured file that defines the key is kept. The others are discarded. + /// + Ignore + } + + /// + /// Selects how the file-based override source learns that a file changed. The two modes are alternatives. + /// + /// + public enum ChangeDetection + { + /// + /// The source examines the files on a fixed interval and reloads when the modification time or + /// the size of a file changes, or a file appears or disappears. Polling works on every file system, + /// including network mounts and directories whose contents are swapped through symbolic links, as + /// Kubernetes does for mounted ConfigMaps. This is the default. + /// + Polling, + + /// + /// The source reloads in response to file system change notifications. It reacts faster than + /// polling. It depends on notifications, which some file systems do not deliver reliably. + /// + Watching + } + } +} diff --git a/pkgs/sdk/server/src/Integrations/FileOverrides.cs b/pkgs/sdk/server/src/Integrations/FileOverrides.cs new file mode 100644 index 00000000..0de74222 --- /dev/null +++ b/pkgs/sdk/server/src/Integrations/FileOverrides.cs @@ -0,0 +1,72 @@ +namespace LaunchDarkly.Sdk.Server.Integrations +{ + /// + /// Integration between the LaunchDarkly SDK and flag overrides read from local files. + /// + /// + /// + /// Flag overrides are currently experimental and subject to change. + /// + /// + /// Overrides are flag and segment definitions that take precedence over data received from + /// LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an + /// incident. An operator can force one or more flags to a known state on a running application, + /// whether or not the application can reach LaunchDarkly. The override stays in effect until the + /// operator removes it. Flags not present in the override data are completely unaffected. + /// + /// + /// The override source is an option of the FDv2 data system. Configure it with + /// : + /// + /// + /// var config = Configuration.Builder("sdk-key") + /// .DataSystem(Components.DataSystem().Default() + /// .Overrides(FileOverrides.Source().FilePaths("/etc/launchdarkly/overrides.json"))) + /// .Build(); + /// + /// + /// An evaluation that an override affects is marked. The marking is direct or transitive: it + /// applies when the evaluated flag, a prerequisite at any depth, or a segment read during the + /// evaluation came from the override files. reports + /// the marking. Marked evaluations appear in analytics summary events only, under separate counters, + /// so LaunchDarkly can distinguish them. They produce no individual evaluation events. + /// + /// + /// + public static class FileOverrides + { + /// + /// Creates a builder for a file-based override source. + /// + /// + /// + /// The source reads flag and segment overrides from one or more local files and reloads them as the + /// files change. The files use the same document format as : a JSON object with + /// optional flags, flagValues, and segments properties. A flagValues + /// entry is expanded into a full flag definition that is off and serves the given value for every + /// context. YAML files are supported when a parser is supplied with + /// . + /// + /// + /// When multiple files are configured, their entries are combined. The configured order determines + /// which file wins under the duplicate keys handling. A reload replaces the entire set of overrides, + /// so removing an entry from the files removes the override. A configured file that does not exist + /// contributes no overrides: the file can be created later, deleting it removes its overrides, and + /// deleting every file removes them all. A file that exists but cannot be read or parsed makes that + /// whole reload fail. The previously loaded overrides stay in effect, the source logs the failure, + /// retries after a short delay, and recovers on its own once the file is readable again. + /// + /// + /// Whenever the set of overrides in effect changes, including at startup, the source logs the + /// overrides in effect and what each configured file supplied, at Info level. + /// + /// + /// By default the source polls the files for changes once per second. See + /// and + /// . + /// + /// + /// a + public static FileOverrideSourceBuilder Source() => new FileOverrideSourceBuilder(); + } +} diff --git a/pkgs/sdk/server/src/Internal/Overrides/FileOverrideSource.cs b/pkgs/sdk/server/src/Internal/Overrides/FileOverrideSource.cs new file mode 100644 index 00000000..2bc01af3 --- /dev/null +++ b/pkgs/sdk/server/src/Internal/Overrides/FileOverrideSource.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Threading; +using LaunchDarkly.Logging; +using LaunchDarkly.Sdk.Internal; +using LaunchDarkly.Sdk.Server.Integrations; +using LaunchDarkly.Sdk.Server.Internal.FileLoading; +using LaunchDarkly.Sdk.Server.Subsystems; + +using static LaunchDarkly.Sdk.Server.Subsystems.DataStoreTypes; + +namespace LaunchDarkly.Sdk.Server.Internal.Overrides +{ + /// + /// The file-based override source. It reads one or more files in the file data document format, + /// supplies the merged result to the override sink as a complete snapshot, and reloads when the + /// files change. + /// + internal sealed class FileOverrideSource : IOverrideSource + { + private readonly IReadOnlyList _paths; + private readonly FileOverrideTypes.DuplicateKeysHandling _duplicateKeysHandling; + private readonly FileOverrideTypes.ChangeDetection _changeDetection; + private readonly TimeSpan _pollInterval; + private readonly Func _parser; + private readonly Logger _log; + + private FileDataReloader _reloader; + private IDisposable _changeDetector; + private int _disposed; + + internal FileOverrideSource( + IReadOnlyList paths, + FileOverrideTypes.DuplicateKeysHandling duplicateKeysHandling, + FileOverrideTypes.ChangeDetection changeDetection, + TimeSpan pollInterval, + Func parser, + Logger log + ) + { + _paths = paths; + _duplicateKeysHandling = duplicateKeysHandling; + _changeDetection = changeDetection; + _pollInterval = pollInterval; + _parser = parser; + _log = log; + } + + // Exposed for tests of the builder. + internal FileOverrideTypes.ChangeDetection ChangeDetection => _changeDetection; + internal TimeSpan PollInterval => _pollInterval; + internal IReadOnlyList Paths => _paths; + internal FileOverrideTypes.DuplicateKeysHandling DuplicateKeysHandling => _duplicateKeysHandling; + internal IDisposable ChangeDetector => _changeDetector; + + /// + /// Performs the initial load synchronously, so overrides present in the files are in effect by + /// the time the client constructor returns, and then starts the change detector. A file that + /// does not exist yet contributes no overrides. A file that cannot be read or parsed is not + /// fatal: the client runs with the last good overrides, the failure is logged, and the retry + /// plus the change signal recover once the file is readable. + /// + public void Start(IOverrideSink sink) + { + _reloader = new FileDataReloader(new FileDataReloaderConfig + { + Paths = _paths, + DuplicateKeysHandling = _duplicateKeysHandling == FileOverrideTypes.DuplicateKeysHandling.Ignore ? + FileDataDuplicateKeysHandling.Ignore : FileDataDuplicateKeysHandling.Fail, + SkipMissingPaths = true, + Logger = _log, + AlternateParser = _parser, + // A value-only entry behaves like a flag that is off and serves its single value. + FlagValueExpander = (key, value) => FileDataParser.MakeOffFlagWithValue(key, value, 0), + Apply = merged => + { + sink.SetOverrides(ToDataSet(merged)); + LogOverridesInEffect(merged); + }, + DebounceDelay = FileDataReloader.DefaultDebounceDelay, + RetryDelay = FileDataReloader.DefaultRetryDelay, + SkipUnchanged = true + }); + _reloader.ReloadNow(); + + switch (_changeDetection) + { + case FileOverrideTypes.ChangeDetection.Watching: + try + { + _changeDetector = new FileDataWatcher(_paths, _reloader.Trigger, _log); + } + catch (Exception e) + { + LogHelpers.LogException(_log, "Unable to watch override files", e); + } + break; + default: + _changeDetector = new FileDataPoller(_paths, _pollInterval, _reloader.Trigger, _log); + break; + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + _changeDetector?.Dispose(); + _reloader?.Dispose(); + } + + private static FullDataSet ToDataSet(FileDataMergeResult merged) => + new FullDataSet(ImmutableList.Create( + new KeyValuePair>(DataModel.Features, + new KeyedItems(merged.Flags)), + new KeyValuePair>(DataModel.Segments, + new KeyedItems(merged.Segments)))); + + // Reports the overrides now in effect and the file each came from. The reloader applies a + // snapshot only when the content changed, so this logs each change once. + private void LogOverridesInEffect(FileDataMergeResult merged) + { + var details = new List(merged.Files.Count); + foreach (var file in merged.Files) + { + if (!file.Present) + { + details.Add(file.Path + ": absent"); + } + else if (file.Flags == 0 && file.Segments == 0) + { + details.Add(file.Path + ": no entries"); + } + else + { + details.Add(file.Path + ": " + CountsText(file.Flags, file.Segments)); + } + } + var joinedDetails = string.Join("; ", details); + if (merged.Flags.Count == 0 && merged.Segments.Count == 0) + { + _log.Info("Flag overrides: none in effect ({0})", joinedDetails); + return; + } + _log.Info("Flag overrides in effect: {0} ({1})", CountsText(merged.Flags.Count, merged.Segments.Count), + joinedDetails); + } + + // Formats flag and segment counts, for example "2 flags, 1 segment". + internal static string CountsText(int flags, int segments) + { + var parts = new List(2); + if (flags > 0) + { + parts.Add(Pluralize(flags, "flag")); + } + if (segments > 0) + { + parts.Add(Pluralize(segments, "segment")); + } + return string.Join(", ", parts); + } + + private static string Pluralize(int count, string noun) => + count == 1 ? "1 " + noun : count + " " + noun + "s"; + } +} diff --git a/pkgs/sdk/server/src/Subsystems/IOverrideSource.cs b/pkgs/sdk/server/src/Subsystems/IOverrideSource.cs index 6839d36d..f7ee52d6 100644 --- a/pkgs/sdk/server/src/Subsystems/IOverrideSource.cs +++ b/pkgs/sdk/server/src/Subsystems/IOverrideSource.cs @@ -22,7 +22,8 @@ namespace LaunchDarkly.Sdk.Server.Subsystems /// /// The SDK constructs the source from the given to /// , - /// starts it when the client starts, and disposes it when the client is disposed. + /// starts it when the client starts, and disposes it when the client is disposed. The built-in + /// file-based source is . /// /// /// Flag overrides are currently experimental and subject to change. diff --git a/pkgs/sdk/server/test/Integrations/FileOverrideSourceBuilderTest.cs b/pkgs/sdk/server/test/Integrations/FileOverrideSourceBuilderTest.cs new file mode 100644 index 00000000..2a83873f --- /dev/null +++ b/pkgs/sdk/server/test/Integrations/FileOverrideSourceBuilderTest.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.IO; +using LaunchDarkly.Logging; +using LaunchDarkly.Sdk.Server.Internal.Overrides; +using LaunchDarkly.TestHelpers; +using Xunit; +using Xunit.Abstractions; + +namespace LaunchDarkly.Sdk.Server.Integrations +{ + public class FileOverrideSourceBuilderTest : BaseTest + { + private readonly BuilderBehavior.InternalStateTester _tester = + BuilderBehavior.For(FileOverrides.Source); + + public FileOverrideSourceBuilderTest(ITestOutputHelper testOutput) : base(testOutput) { } + + private FileOverrideSource BuildSource(FileOverrideSourceBuilder builder) => + Assert.IsType(builder.Build(BasicContext)); + + [Fact] + public void FilePaths() + { + Assert.Empty(FileOverrides.Source()._paths); + + var b = FileOverrides.Source(); + b.FilePaths("path1"); + b.FilePaths("path2", "path3"); + Assert.Equal(new List { "path1", "path2", "path3" }, b._paths); + } + + [Fact] + public void DuplicateKeysHandling() + { + var prop = _tester.Property(b => b._duplicateKeysHandling, (b, v) => b.DuplicateKeysHandling(v)); + prop.AssertDefault(FileOverrideTypes.DuplicateKeysHandling.Fail); + prop.AssertCanSet(FileOverrideTypes.DuplicateKeysHandling.Ignore); + } + + [Fact] + public void ChangeDetection() + { + var prop = _tester.Property(b => b._changeDetection, (b, v) => b.ChangeDetection(v)); + prop.AssertDefault(FileOverrideTypes.ChangeDetection.Polling); + prop.AssertCanSet(FileOverrideTypes.ChangeDetection.Watching); + } + + [Fact] + public void PollInterval() + { + var prop = _tester.Property(b => b._pollInterval, (b, v) => b.PollInterval(v)); + prop.AssertDefault(FileOverrideSourceBuilder.DefaultPollInterval); + prop.AssertCanSet(TimeSpan.FromSeconds(5)); + Assert.Equal(TimeSpan.FromSeconds(1), FileOverrideSourceBuilder.DefaultPollInterval); + Assert.Equal(TimeSpan.FromSeconds(1), FileOverrideSourceBuilder.MinimumPollInterval); + } + + [Fact] + public void Parser() + { + var prop = _tester.Property(b => b._parser, (b, v) => b.Parser(v)); + prop.AssertDefault(null); + Func p = s => s; + prop.AssertCanSet(p); + } + + [Fact] + public void BuildRequiresAtLeastOneFilePath() + { + var e = Assert.Throws(() => FileOverrides.Source().Build(BasicContext)); + Assert.Contains("no file paths", e.Message); + } + + [Fact] + public void BuildPassesOptionsToTheSource() + { + var source = BuildSource(FileOverrides.Source().FilePaths("/a.json", "/b.json") + .DuplicateKeysHandling(FileOverrideTypes.DuplicateKeysHandling.Ignore) + .ChangeDetection(FileOverrideTypes.ChangeDetection.Watching) + .PollInterval(TimeSpan.FromSeconds(3))); + + Assert.Equal(new[] { Path.GetFullPath("/a.json"), Path.GetFullPath("/b.json") }, source.Paths); + Assert.Equal(FileOverrideTypes.DuplicateKeysHandling.Ignore, source.DuplicateKeysHandling); + Assert.Equal(FileOverrideTypes.ChangeDetection.Watching, source.ChangeDetection); + Assert.Equal(TimeSpan.FromSeconds(3), source.PollInterval); + } + + [Fact] + public void BuildResolvesRelativePathsAgainstTheCurrentDirectory() + { + var source = BuildSource(FileOverrides.Source().FilePaths("relative/overrides.json")); + Assert.Equal(new[] { Path.GetFullPath("relative/overrides.json") }, source.Paths); + Assert.True(Path.IsPathRooted(source.Paths[0])); + } + + [Fact] + public void BuildDefaultsToPollingEverySecond() + { + var source = BuildSource(FileOverrides.Source().FilePaths("/a.json")); + Assert.Equal(FileOverrideTypes.ChangeDetection.Polling, source.ChangeDetection); + Assert.Equal(FileOverrideSourceBuilder.DefaultPollInterval, source.PollInterval); + } + + [Fact] + public void BuildRaisesAPollIntervalBelowTheMinimumAndWarns() + { + var source = BuildSource(FileOverrides.Source().FilePaths("/a.json").PollInterval(TimeSpan.FromMilliseconds(1))); + Assert.Equal(FileOverrideSourceBuilder.MinimumPollInterval, source.PollInterval); + AssertLogMessageRegex(true, LogLevel.Warn, "below the minimum"); + } + + [Fact] + public void BuildKeepsAPollIntervalAtOrAboveTheMinimum() + { + var source = BuildSource(FileOverrides.Source().FilePaths("/a.json").PollInterval(TimeSpan.FromSeconds(2))); + Assert.Equal(TimeSpan.FromSeconds(2), source.PollInterval); + AssertLogMessageRegex(false, LogLevel.Warn, "below the minimum"); + } + + [Fact] + public void BuildDoesNotWarnAboutThePollIntervalInWatchingMode() + { + var source = BuildSource(FileOverrides.Source().FilePaths("/a.json") + .ChangeDetection(FileOverrideTypes.ChangeDetection.Watching).PollInterval(TimeSpan.FromMilliseconds(1))); + Assert.Equal(FileOverrideTypes.ChangeDetection.Watching, source.ChangeDetection); + AssertLogMessageRegex(false, LogLevel.Warn, "below the minimum"); + } + + [Fact] + public void BuildRejectsAnUnknownDuplicateKeysHandling() + { + Assert.Throws(() => FileOverrides.Source().FilePaths("/a.json") + .DuplicateKeysHandling((FileOverrideTypes.DuplicateKeysHandling)99).Build(BasicContext)); + } + + [Fact] + public void BuildRejectsAnUnknownChangeDetection() + { + Assert.Throws(() => FileOverrides.Source().FilePaths("/a.json") + .ChangeDetection((FileOverrideTypes.ChangeDetection)99).Build(BasicContext)); + } + + [Fact] + public void BuildDoesNotRequireTheFilesToExist() + { + var source = BuildSource(FileOverrides.Source().FilePaths(Path.Combine(Path.GetTempPath(), "no-such-overrides.json"))); + Assert.NotNull(source); + } + } +} diff --git a/pkgs/sdk/server/test/Internal/Overrides/FileOverrideSourceTest.cs b/pkgs/sdk/server/test/Internal/Overrides/FileOverrideSourceTest.cs new file mode 100644 index 00000000..a462743e --- /dev/null +++ b/pkgs/sdk/server/test/Internal/Overrides/FileOverrideSourceTest.cs @@ -0,0 +1,387 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using LaunchDarkly.Logging; +using LaunchDarkly.Sdk.Server.Integrations; +using LaunchDarkly.Sdk.Server.Internal.Evaluation; +using LaunchDarkly.Sdk.Server.Internal.FileLoading; +using LaunchDarkly.Sdk.Server.Internal.Model; +using LaunchDarkly.Sdk.Server.Subsystems; +using LaunchDarkly.TestHelpers; +using Xunit; +using Xunit.Abstractions; +using YamlDotNet.Serialization; + +using static LaunchDarkly.Sdk.Server.Subsystems.DataStoreTypes; + +namespace LaunchDarkly.Sdk.Server.Internal.Overrides +{ + public class FileOverrideSourceTest : BaseTest, IDisposable + { + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10); + private static readonly TimeSpan ShortPollInterval = TimeSpan.FromMilliseconds(50); + + private readonly TempDirectory _dir = TempDirectory.Create(); + private readonly EventSink> _snapshots = new EventSink>(); + private readonly CapturingSink _sink; + private FileOverrideSource _source; + + public FileOverrideSourceTest(ITestOutputHelper testOutput) : base(testOutput) + { + _sink = new CapturingSink(_snapshots); + } + + public void Dispose() + { + _source?.Dispose(); + _dir.Dispose(); + } + + private class CapturingSink : IOverrideSink + { + private readonly EventSink> _snapshots; + public CapturingSink(EventSink> snapshots) { _snapshots = snapshots; } + public void SetOverrides(FullDataSet data) => _snapshots.Enqueue(data); + } + + private FileOverrideSource StartSource( + IEnumerable paths, + FileOverrideTypes.DuplicateKeysHandling duplicateKeysHandling = FileOverrideTypes.DuplicateKeysHandling.Fail, + FileOverrideTypes.ChangeDetection changeDetection = FileOverrideTypes.ChangeDetection.Polling, + TimeSpan? pollInterval = null, + Func parser = null + ) + { + _source = new FileOverrideSource(paths.ToList(), duplicateKeysHandling, changeDetection, + pollInterval ?? ShortPollInterval, parser, TestLogger); + _source.Start(_sink); + return _source; + } + + private static void Write(string path, string content) => File.WriteAllText(path, content); + + private FullDataSet RequireSnapshot() => _snapshots.ExpectValue(TestTimeout); + + // The initial load happens inside Start, so its snapshot is already queued. + private FullDataSet RequireInitialSnapshot() => _snapshots.ExpectValue(TimeSpan.Zero); + + private void RequireNoSnapshot(TimeSpan duration) => _snapshots.ExpectNoValue(duration); + + private static Dictionary FlagsByKey(FullDataSet snapshot) => + snapshot.Data.Where(kv => kv.Key == DataModel.Features).SelectMany(kv => kv.Value.Items) + .ToDictionary(kv => kv.Key, kv => Assert.IsType(kv.Value.Item)); + + private static Dictionary SegmentsByKey(FullDataSet snapshot) => + snapshot.Data.Where(kv => kv.Key == DataModel.Segments).SelectMany(kv => kv.Value.Items) + .ToDictionary(kv => kv.Key, kv => Assert.IsType(kv.Value.Item)); + + // Waits for an Info log line that contains every substring. The source logs after it hands the + // snapshot to the sink, so a test that has just received a snapshot may run ahead of the line. + private void RequireInfoLine(params string[] substrings) + { + var deadline = DateTime.UtcNow + TestTimeout; + while (DateTime.UtcNow < deadline) + { + if (LogCapture.GetMessages().Any(m => m.Level == LogLevel.Info && substrings.All(s => m.Text.Contains(s)))) + { + return; + } + Thread.Sleep(10); + } + Assert.True(false, "timed out waiting for an Info line containing " + string.Join(" and ", substrings) + + "\nlog output:\n" + LogCapture); + } + + [Fact] + public void LoadsInitialDataSynchronously() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"": {""flag1"": true}, ""flags"": {""flag2"": {""key"": ""flag2"", ""version"": 3, ""on"": false}}, + ""segments"": {""seg1"": {""key"": ""seg1"", ""version"": 4}}}"); + + StartSource(new[] { path }); + + var snapshot = RequireInitialSnapshot(); + var flags = FlagsByKey(snapshot); + Assert.Equal(2, flags.Count); + Assert.Equal(3, flags["flag2"].Version); + Assert.Equal(4, SegmentsByKey(snapshot)["seg1"].Version); + + // The flag-value entry was expanded into a full flag definition that is off and serves its + // single value for every context. + var expanded = flags["flag1"]; + Assert.False(expanded.IsOverride, "the source supplies plain definitions; the SDK marks them"); + Assert.Equal(new[] { LdValue.Of(true) }, expanded.Variations); + Assert.False(expanded.On); + Assert.Equal(0, expanded.OffVariation); + var result = EvaluatorTestUtil.BasicEvaluator.Evaluate(expanded, Context.New("anyone")); + Assert.Equal(LdValue.Of(true), result.Result.Value); + Assert.Equal(0, result.Result.VariationIndex); + Assert.Equal(EvaluationReasonKind.Off, result.Result.Reason.Kind); + } + + [Fact] + public void LoadsYamlWithAParser() + { + var path = _dir.PathOf("overrides.yaml"); + Write(path, "flagValues:\n flag1: true\n"); + var yaml = new DeserializerBuilder().WithAttemptingUnquotedStringTypeDeserialization().Build(); + + StartSource(new[] { path }, parser: s => yaml.Deserialize(s)); + + var flags = FlagsByKey(RequireInitialSnapshot()); + Assert.Single(flags); + Assert.Equal(new[] { LdValue.Of(true) }, flags["flag1"].Variations); + } + + [Fact] + public void YamlWithoutAParserFailsTheLoad() + { + var path = _dir.PathOf("overrides.yaml"); + Write(path, "flagValues:\n flag1: true\n"); + + StartSource(new[] { path }); + + RequireNoSnapshot(TimeSpan.FromMilliseconds(200)); + AssertLogMessageRegex(true, LogLevel.Error, "Unable to load flags: error parsing file"); + } + + [Fact] + public void MergesFilesInConfiguredOrder() + { + var first = _dir.PathOf("first.json"); + var second = _dir.PathOf("second.json"); + Write(first, @"{""flags"": {""flag1"": {""key"": ""flag1"", ""version"": 1}}}"); + Write(second, @"{""flags"": {""flag1"": {""key"": ""flag1"", ""version"": 2}}}"); + + StartSource(new[] { first, second }, FileOverrideTypes.DuplicateKeysHandling.Ignore); + + var flags = FlagsByKey(RequireInitialSnapshot()); + Assert.Single(flags); + Assert.Equal(1, flags["flag1"].Version); + } + + [Fact] + public void DuplicateKeysFailByDefault() + { + var first = _dir.PathOf("first.json"); + var second = _dir.PathOf("second.json"); + Write(first, @"{""flags"": {""flag1"": {""key"": ""flag1"", ""version"": 1}}}"); + Write(second, @"{""flags"": {""flag1"": {""key"": ""flag1"", ""version"": 2}}}"); + + StartSource(new[] { first, second }); + + RequireNoSnapshot(TimeSpan.FromMilliseconds(200)); + AssertLogMessageRegex(true, LogLevel.Error, "is specified by multiple files"); + } + + [Fact] + public void StartsWithAMissingFileAndPicksItUpWhenItAppears() + { + var path = _dir.PathOf("not-yet.json"); + + StartSource(new[] { path }); + + // A missing file contributes no overrides. The initial snapshot is empty. + Assert.Empty(FlagsByKey(RequireInitialSnapshot())); + AssertLogMessageRegex(false, LogLevel.Error, ".*"); + AssertLogMessageRegex(false, LogLevel.Warn, ".*"); + + // Once the file appears, the change signal picks it up unprompted. + Write(path, @"{""flagValues"": {""flag1"": true}}"); + Assert.Single(FlagsByKey(RequireSnapshot())); + } + + [Fact] + public void MissingFileContributesNoEntries() + { + var first = _dir.PathOf("first.json"); + var second = _dir.PathOf("second.json"); + Write(first, @"{""flagValues"": {""from-first"": true}}"); + + // Step 1: one configured file exists and one does not. The existing file applies. + StartSource(new[] { first, second }); + var flags = FlagsByKey(RequireInitialSnapshot()); + Assert.Equal(new[] { "from-first" }, flags.Keys); + + // Step 2: the second file appears. Both apply. + Write(second, @"{""flagValues"": {""from-second"": true}}"); + flags = FlagsByKey(RequireSnapshot()); + Assert.Equal(2, flags.Count); + + // Step 3: the second file is deleted. Its overrides are removed. + File.Delete(second); + flags = FlagsByKey(RequireSnapshot()); + Assert.Equal(new[] { "from-first" }, flags.Keys); + + // Step 4: the last file is deleted. The layer is cleared. + File.Delete(first); + Assert.Empty(FlagsByKey(RequireSnapshot())); + } + + [Fact] + public void LogsOverridesInEffectOnEachChange() + { + var first = _dir.PathOf("first.json"); + var second = _dir.PathOf("second.json"); + Write(first, @"{""flagValues"": {""flag1"": true, ""flag2"": false}, ""segments"": {""seg"": {""key"": ""seg"", ""version"": 1}}}"); + + // Step 1: at startup, one file supplies entries and the other is absent. + StartSource(new[] { first, second }); + RequireInitialSnapshot(); + RequireInfoLine("Flag overrides in effect: 2 flags, 1 segment", first + ": 2 flags, 1 segment", second + ": absent"); + + // Step 2: the absent file appears with one entry. + Write(second, @"{""flagValues"": {""flag3"": true}}"); + RequireSnapshot(); + RequireInfoLine("Flag overrides in effect: 3 flags, 1 segment", second + ": 1 flag"); + + // Step 3: both files are deleted. Nothing is in effect. + File.Delete(first); + File.Delete(second); + RequireInfoLine("Flag overrides: none in effect", first + ": absent", second + ": absent"); + } + + [Fact] + public void LogsNoneInEffectAtStartupWithoutFiles() + { + var path = _dir.PathOf("overrides.json"); + StartSource(new[] { path }); + RequireInitialSnapshot(); + RequireInfoLine("Flag overrides: none in effect", path + ": absent"); + } + + [Fact] + public void LogsAFileWithNoEntries() + { + var path = _dir.PathOf("overrides.json"); + Write(path, "{}"); + StartSource(new[] { path }); + RequireInitialSnapshot(); + RequireInfoLine("Flag overrides: none in effect", path + ": no entries"); + } + + [Fact] + public void CountsTextPluralizes() + { + Assert.Equal("1 flag", FileOverrideSource.CountsText(1, 0)); + Assert.Equal("2 flags, 1 segment", FileOverrideSource.CountsText(2, 1)); + Assert.Equal("3 segments", FileOverrideSource.CountsText(0, 3)); + Assert.Equal("", FileOverrideSource.CountsText(0, 0)); + } + + [Fact] + public void WatchingModeIsQuietWhenTheFileIsAbsent() + { + var path = _dir.PathOf("overrides.json"); + + StartSource(new[] { path }, changeDetection: FileOverrideTypes.ChangeDetection.Watching); + RequireInitialSnapshot(); + + Thread.Sleep(300); + Assert.DoesNotContain(LogCapture.GetMessages(), m => m.Level == LogLevel.Error || m.Level == LogLevel.Warn); + + Write(path, @"{""flagValues"": {""flag1"": true}}"); + Assert.Single(FlagsByKey(RequireSnapshot())); + } + + [Fact] + public void WatchingModeReloadsOnChange() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"": {""flag1"": true}}"); + + var source = StartSource(new[] { path }, changeDetection: FileOverrideTypes.ChangeDetection.Watching); + Assert.IsType(source.ChangeDetector); + RequireInitialSnapshot(); + + Write(path, @"{""flagValues"": {""flag1"": true, ""flag2"": false}}"); + Assert.Equal(2, FlagsByKey(RequireSnapshot()).Count); + + // Removing entries removes them from the snapshot. A reload is a full replacement. + Write(path, "{}"); + Assert.Empty(FlagsByKey(RequireSnapshot())); + } + + [Fact] + public void PollingModeReloadsOnChange() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"": {""flag1"": true}}"); + + var source = StartSource(new[] { path }, changeDetection: FileOverrideTypes.ChangeDetection.Polling); + Assert.IsType(source.ChangeDetector); + RequireInitialSnapshot(); + + Write(path, @"{""flagValues"": {""flag1"": true, ""flag2"": false}}"); + // Make the rewrite observable through (modification time, size) whatever the timestamp granularity. + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddSeconds(2)); + Assert.Equal(2, FlagsByKey(RequireSnapshot()).Count); + } + + [Fact] + public void RetainsLastGoodDataAcrossAMalformedEdit() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"": {""flag1"": true}}"); + + StartSource(new[] { path }, changeDetection: FileOverrideTypes.ChangeDetection.Watching); + RequireInitialSnapshot(); + + // A malformed edit produces no snapshot. The previously applied overrides stay in effect + // because the sink is never called. The failure is logged. + Write(path, @"{""flagValues"""); + RequireNoSnapshot(TimeSpan.FromMilliseconds(300)); + AssertLogMessageRegex(true, LogLevel.Error, "Unable to load flags: error parsing file"); + + // Fixing the file recovers, through the change notification or the failure retry. + Write(path, @"{""flagValues"": {""flag1"": false}}"); + var flags = FlagsByKey(RequireSnapshot()); + Assert.Equal(new[] { LdValue.Of(false) }, flags["flag1"].Variations); + } + + [Fact] + public void MalformedInitialFileLeavesTheLayerEmptyUntilFixed() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"""); + + StartSource(new[] { path }); + + RequireNoSnapshot(TimeSpan.FromMilliseconds(100)); + AssertLogMessageRegex(true, LogLevel.Error, "Unable to load flags"); + + Write(path, @"{""flagValues"": {""flag1"": true}}"); + Assert.Single(FlagsByKey(RequireSnapshot())); + } + + [Fact] + public void WatchingModeWithAMissingDirectoryLogsAndStillLoads() + { + var path = Path.Combine(_dir.PathOf("no-such-directory"), "overrides.json"); + + StartSource(new[] { path }, changeDetection: FileOverrideTypes.ChangeDetection.Watching); + + Assert.Empty(FlagsByKey(RequireInitialSnapshot())); + AssertLogMessageRegex(true, LogLevel.Error, "Unable to watch override files"); + } + + [Fact] + public void DisposeIsIdempotentAndStopsReloads() + { + var path = _dir.PathOf("overrides.json"); + Write(path, @"{""flagValues"": {""flag1"": true}}"); + + var source = StartSource(new[] { path }); + RequireInitialSnapshot(); + + source.Dispose(); + source.Dispose(); + + Write(path, @"{""flagValues"": {""flag1"": false}}"); + RequireNoSnapshot(TimeSpan.FromMilliseconds(300)); + } + } +} diff --git a/pkgs/sdk/server/test/LdClientFileOverridesTest.cs b/pkgs/sdk/server/test/LdClientFileOverridesTest.cs new file mode 100644 index 00000000..4a494e21 --- /dev/null +++ b/pkgs/sdk/server/test/LdClientFileOverridesTest.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using System.Threading; +using LaunchDarkly.Sdk.Server.Integrations; +using Xunit; +using Xunit.Abstractions; + +namespace LaunchDarkly.Sdk.Server +{ + // Drives the file-based override source through a running client. An operator writes, edits, and + // empties an override file. Evaluations follow without any client restart, even though the client + // never obtains data from LaunchDarkly. + public class LdClientFileOverridesTest : BaseTest, IDisposable + { + private static readonly TimeSpan ReloadTimeout = TimeSpan.FromSeconds(10); + private static readonly Context context = Context.New("userkey"); + + private readonly TempDirectory _dir = TempDirectory.Create(); + + public LdClientFileOverridesTest(ITestOutputHelper testOutput) : base(testOutput) { } + + public void Dispose() => _dir.Dispose(); + + private static void AssertEventually(Func condition, string description) + { + var deadline = DateTime.UtcNow + ReloadTimeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + Thread.Sleep(50); + } + Assert.True(condition(), "timed out waiting for " + description); + } + + [Theory] + [InlineData(FileOverrideTypes.ChangeDetection.Polling)] + [InlineData(FileOverrideTypes.ChangeDetection.Watching)] + public void FileOverridesEndToEnd(FileOverrideTypes.ChangeDetection changeDetection) + { + var path = _dir.PathOf("overrides.json"); + File.WriteAllText(path, "{}"); + + var config = BasicConfig() + .DataSystem(Components.DataSystem().Custom() + .Synchronizers(MockComponents.MockDataSourceThatNeverStarts()) + .Overrides(FileOverrides.Source().FilePaths(path).ChangeDetection(changeDetection))) + .Build(); + using (var client = new LdClient(config)) + { + Assert.False(client.Initialized); + + // Not initialized and no override present: the default is served. + var detail = client.BoolVariationDetail("overridden-flag", context, false); + Assert.False(detail.Value); + Assert.Equal(EvaluationErrorKind.ClientNotReady, detail.Reason.ErrorKind); + + // An operator adds an override. The running client picks it up. + File.WriteAllText(path, @"{""flagValues"": {""overridden-flag"": true}}"); + AssertEventually(() => client.BoolVariation("overridden-flag", context, false), "the override to take effect"); + + // The override changes value. + File.WriteAllText(path, @"{""flagValues"": {""overridden-flag"": false}}"); + AssertEventually(() => + { + var d = client.BoolVariationDetail("overridden-flag", context, true); + return d.Reason.OverrideAffected && !d.Value; + }, "the changed override to take effect"); + + // The override is removed. The not-initialized short-circuit returns. + File.WriteAllText(path, "{}"); + AssertEventually(() => + client.BoolVariationDetail("overridden-flag", context, false).Reason.ErrorKind == EvaluationErrorKind.ClientNotReady, + "the removed override to stop taking effect"); + } + } + + [Fact] + public void OverridesPresentAtStartupTakeEffectFromTheFirstEvaluation() + { + var path = _dir.PathOf("overrides.json"); + File.WriteAllText(path, @"{""flagValues"": {""overridden-flag"": ""override-value""}}"); + + var config = BasicConfig() + .DataSystem(Components.DataSystem().Custom() + .Synchronizers(MockComponents.MockDataSourceThatNeverStarts()) + .Overrides(FileOverrides.Source().FilePaths(path))) + .Build(); + using (var client = new LdClient(config)) + { + var detail = client.StringVariationDetail("overridden-flag", context, "default"); + Assert.Equal("override-value", detail.Value); + Assert.True(detail.Reason.OverrideAffected); + Assert.Equal(EvaluationReasonKind.Off, detail.Reason.Kind); + } + } + } +}