feat: Add the override store, overlay, source interfaces, and data system wiring

pkgs/sdk/server/src/Integrations/DataSystemBuilder.cs

@@ -20,7 +20,9 @@ public sealed class DataSystemBuilder
         private IComponentConfigurer<IDataStore> _persistentStore;
 
         private DataSystemConfiguration.DataStoreMode _persistentDataStoreMode;
-
+
+        private IComponentConfigurer<IOverrideSource> _overrideSource;
+
         /// <summary>
         /// Add one or more initializers to the builder.
         /// To replace initializers, please refer to <see cref="DataSystemBuilder.ReplaceInitializers"/>.
@@ -101,6 +103,38 @@ public DataSystemBuilder PersistentStore(IComponentConfigurer<IDataStore> persis
             return this;
         }
 
+        /// <summary>
+        /// Configures an override source. Flag overrides are currently experimental and subject to change.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// The source supplies flag and segment definitions that take precedence over data received from
+        /// LaunchDarkly on a per-key basis. Overrides let an operator force one or more flags to a known
+        /// state on a running client, whether or not the client can reach LaunchDarkly. Flags not present
+        /// in the override data are unaffected.
+        /// </para>
+        /// <para>
+        /// The override source is not a data source. It has no effect on the client's initialization status
+        /// or data source status. Configuring it changes nothing until the source actually supplies an
+        /// override. At most one override source can be configured. A later call replaces the earlier one.
+        /// </para>
+        /// <example>
+        /// <code>
+        /// var config = Configuration.Builder("my-sdk-key")
+        ///   .DataSystem(Components.DataSystem().Default()
+        ///     .Overrides(FileOverrides.Source().FilePaths("/etc/launchdarkly/overrides.json")));
+        /// </code>
+        /// </example>
+        /// </remarks>
+        /// <param name="overrideSource">the override source, such as the file-based override source;
+        /// null removes a previously configured source</param>
+        /// <returns>a reference to the builder</returns>
+        public DataSystemBuilder Overrides(IComponentConfigurer<IOverrideSource> overrideSource)
+        {
+            _overrideSource = overrideSource;
+            return this;
+        }
+
         internal DataSystemConfiguration Build()
         {
             // This function should remain internal.
@@ -110,7 +144,8 @@ internal DataSystemConfiguration Build()
                 _synchronizers.ToImmutableList(),
                 _fdV1FallbackSynchronizer,
                 _persistentStore,
-                _persistentDataStoreMode);
+                _persistentDataStoreMode,
+                _overrideSource);
         }
     }
 }

pkgs/sdk/server/src/Internal/DataSources/DataSourceUpdatesImpl.cs

@@ -197,7 +197,25 @@ internal async Task<bool> WaitForAsync(DataSourceState desiredState, TimeSpan ti
 
         #region Private methods
 
-        private bool HasFlagChangeListeners() => FlagChanged != null;
+        internal bool HasFlagChangeListeners() => FlagChanged != null;
+
+        /// <summary>
+        /// Sends a flag change event for each of the given flag keys. The override layer uses this to
+        /// report the flags that an override update affected, through the same path as a change from
+        /// LaunchDarkly.
+        /// </summary>
+        internal void SendFlagChangeEvents(IEnumerable<string> flagKeys)
+        {
+            var copyOfHandlers = FlagChanged;
+            if (copyOfHandlers == null)
+            {
+                return;
+            }
+            foreach (var key in flagKeys)
+            {
+                _taskExecutor.ScheduleEvent(new FlagChangeEvent(key), copyOfHandlers);
+            }
+        }
 
         private void SendChangeEvents(IEnumerable<KindAndKey> affectedItems)
         {

pkgs/sdk/server/src/Internal/DataSystem/FDv1DataSystem.cs

@@ -43,6 +43,11 @@ public Task<bool> Start()
 
         public bool Initialized => _dataSource.Initialized;
 
+        // Overrides are an option of the FDv2 data system only.
+        public bool OverridesConfigured => false;
+
+        public bool HasOverride(DataStoreTypes.DataKind kind, string key) => false;
+
         public IFlagChanged FlagChanged { get; }
         public IDataSourceStatusProvider DataSourceStatusProvider { get; }
         public IDataStoreStatusProvider DataStoreStatusProvider { get; }

pkgs/sdk/server/src/Internal/DataSystem/FDv2DataSystem.cs

@@ -7,6 +7,7 @@
 using LaunchDarkly.Sdk.Server.Internal.DataSources;
 using LaunchDarkly.Sdk.Server.Internal.DataStores;
 using LaunchDarkly.Sdk.Server.Internal.FDv2DataSources;
+using LaunchDarkly.Sdk.Server.Internal.Overrides;
 using LaunchDarkly.Sdk.Server.Subsystems;
 
 namespace LaunchDarkly.Sdk.Server.Internal.DataSystem
@@ -16,16 +17,44 @@ internal class FDv2DataSystem : IDataSystem, IDisposable
         private readonly WriteThroughStore _store;
         private readonly IDataSource _dataSource;
         private readonly DataSourceUpdatesImpl _dataSourceUpdates;
+        // The override source, its layer, and its sink are null when no override source is configured.
+        private readonly IOverrideSource _overrideSource;
+        private readonly OverrideLayer _overrideLayer;
+        private readonly OverrideSink _overrideSink;
         private bool _disposed;
 
         #region IDataSystem implementation
 
         public IReadOnlyStore Store { get; }
 
-        public Task<bool> Start() => _dataSource.Start();
+        public Task<bool> Start()
+        {
+            // The override source starts before the data sources, and its initial load completes
+            // synchronously, so overrides present when the client starts are in effect before the
+            // client evaluates anything. The source has no effect on initialization. A source that
+            // fails to start fails the client constructor; the data system is disposed first, so
+            // nothing it created is left running.
+            try
+            {
+                _overrideSource?.Start(_overrideSink);
+            }
+            catch (Exception)
+            {
+                Dispose();
+                throw;
+            }
+            return _dataSource.Start();
+        }
 
         public bool Initialized => _dataSource.Initialized;
 
+        public bool OverridesConfigured => _overrideSource != null;
+
+        // No entry has a null key, so a null key is not held. The evaluation of a null key then
+        // takes the same path as without an override source.
+        public bool HasOverride(DataStoreTypes.DataKind kind, string key) =>
+            key != null && _overrideLayer != null && _overrideLayer.Get(kind, key).HasValue;
+
         public IFlagChanged FlagChanged { get; }
         public IDataSourceStatusProvider DataSourceStatusProvider { get; }
         public IDataStoreStatusProvider DataStoreStatusProvider { get; }
@@ -34,10 +63,14 @@ internal class FDv2DataSystem : IDataSystem, IDisposable
 
         private FDv2DataSystem(
             WriteThroughStore store,
+            IReadOnlyStore readOnlyStore,
             IDataSource dataSource,
             IDataSourceStatusProvider dataSourceStatusProvider,
             IDataStoreStatusProvider dataStoreStatusProvider,
-            DataSourceUpdatesImpl dataStoreUpdates
+            DataSourceUpdatesImpl dataStoreUpdates,
+            IOverrideSource overrideSource,
+            OverrideLayer overrideLayer,
+            OverrideSink overrideSink
         )
         {
             _store = store;
@@ -46,7 +79,10 @@ DataSourceUpdatesImpl dataStoreUpdates
             DataSourceStatusProvider = dataSourceStatusProvider;
             FlagChanged = new FlagChangedFacade(dataStoreUpdates);
             _dataSourceUpdates = dataStoreUpdates;
-            Store = new ReadonlyStoreFacade(store);
+            Store = readOnlyStore;
+            _overrideSource = overrideSource;
+            _overrideLayer = overrideLayer;
+            _overrideSink = overrideSink;
         }
 
         public static FDv2DataSystem Create(Logger logger, Configuration configuration, LdClientContext clientContext,
@@ -98,8 +134,40 @@ public static FDv2DataSystem Create(Logger logger, Configuration configuration,
 
             var dataSourceStatusProvider = new DataSourceStatusProviderImpl(dataSourceUpdates);
 
-            return new FDv2DataSystem(writeThroughStore, compositeDataSource, dataSourceStatusProvider,
-                dataStoreStatusProvider, dataSourceUpdates);
+            // The override layer sits at the store read boundary. Every read the client performs
+            // goes through the overlay, so evaluation, prerequisite and segment resolution, and
+            // the all-flags state see override entries in preference to LaunchDarkly data. The
+            // overlay never touches the data system's own writes or its initialization status.
+            IReadOnlyStore readOnlyStore = new ReadonlyStoreFacade(writeThroughStore);
+            IOverrideSource overrideSource = null;
+            OverrideLayer overrideLayer = null;
+            OverrideSink overrideSink = null;
+            if (dataSystemConfiguration.OverrideSource != null && !configuration.Offline)
+            {
+                // A configuration error in the source is reported like any other invalid component
+                // configuration: the exception propagates out of the client constructor. The
+                // components created so far are disposed first, so a failed constructor leaves
+                // nothing running.
+                try
+                {
+                    overrideSource = dataSystemConfiguration.OverrideSource.Build(clientContext);
+                }
+                catch (Exception)
+                {
+                    compositeDataSource.Dispose();
+                    dataSourceUpdates.Dispose();
+                    writeThroughStore.Dispose();
+                    throw;
+                }
+                overrideLayer = new OverrideLayer();
+                overrideSink = new OverrideSink(overrideLayer, readOnlyStore,
+                    dataSourceUpdates.SendFlagChangeEvents, dataSourceUpdates.HasFlagChangeListeners,
+                    logger.SubLogger(LogNames.OverridesSubLog));
+                readOnlyStore = new OverrideOverlay(readOnlyStore, overrideLayer);
+            }
+
+            return new FDv2DataSystem(writeThroughStore, readOnlyStore, compositeDataSource, dataSourceStatusProvider,
+                dataStoreStatusProvider, dataSourceUpdates, overrideSource, overrideLayer, overrideSink);
         }
 
         private static Func<IComponentConfigurer<IDataSource>, SourceFactory> FactoryWithContext(
@@ -125,6 +193,7 @@ private void Dispose(bool disposing)
             if (_disposed) return;
             if (disposing)
             {
+                _overrideSource?.Dispose();
                 _dataSource.Dispose();
                 _store.Dispose();
                 _dataSourceUpdates.Dispose();

pkgs/sdk/server/src/Internal/DataSystem/IDataSystem.cs

@@ -27,6 +27,18 @@ internal interface IDataSystem
         Task<bool> Start();
         bool Initialized { get; }
 
+        /// <summary>
+        /// True if the data system was built with an override source. When true, the store applies the
+        /// override layer, and the client consults it before its not-initialized short-circuit.
+        /// </summary>
+        bool OverridesConfigured { get; }
+
+        /// <summary>
+        /// True if the override layer holds an entry of the kind with the key. Always false when no
+        /// override source is configured.
+        /// </summary>
+        bool HasOverride(DataStoreTypes.DataKind kind, string key);
+
         IFlagChanged FlagChanged { get; }
 
         IDataSourceStatusProvider DataSourceStatusProvider { get; }

pkgs/sdk/server/src/Internal/LogNames.cs

@@ -18,5 +18,7 @@ internal static class LogNames
         internal const string EventsSubLog = "Events";
 
         internal const string HooksSubLog = "Hooks";
+
+        internal const string OverridesSubLog = "Overrides";
     }
 }