From a3d6e05e17e6490b80a51e27fc5d296a798091e6 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:22:11 -0700 Subject: [PATCH 1/9] Add option for specify per-edge delay for pubnet data --- src/App/Program.fs | 11 ++ src/FSLibrary.Tests/Tests.fs | 6 +- src/FSLibrary/MaxTPSTest.fs | 10 +- src/FSLibrary/MinBlockTimeTest.fs | 7 +- src/FSLibrary/StellarCoreSet.fs | 5 + src/FSLibrary/StellarMissionContext.fs | 1 + src/FSLibrary/StellarNetworkData.fs | 168 ++++++++++++++---- src/FSLibrary/StellarNetworkDelays.fs | 95 +++++++--- .../sample-network-data-delay.json | 77 ++++++++ 9 files changed, 314 insertions(+), 66 deletions(-) create mode 100644 src/FSLibrary/json-type-samples/sample-network-data-delay.json diff --git a/src/App/Program.fs b/src/App/Program.fs index 0dd5125a..6ec33a44 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -73,6 +73,7 @@ type MissionOptions apiRateLimit: int, httpProxyReplicas: int, pubnetData: string option, + pubnetDataDelay: bool, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -318,6 +319,12 @@ type MissionOptions [] member self.PubnetData = pubnetData + [] + member self.PubnetDataDelay = pubnetDataDelay + [] member self.FlatQuorum = flatQuorum @@ -743,6 +750,9 @@ let main argv = 0 | :? MissionOptions as mission -> + if mission.PubnetData.IsNone && mission.PubnetDataDelay then + failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -848,6 +858,7 @@ let main argv = apiRateLimit = mission.ApiRateLimit httpProxyReplicas = mission.HttpProxyReplicas pubnetData = mission.PubnetData + pubnetDataDelay = mission.PubnetDataDelay flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 7a3c4f73..1e14bacf 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -71,6 +71,7 @@ let ctx : MissionContext = apiRateLimit = 10 httpProxyReplicas = 2 pubnetData = None + pubnetDataDelay = false flatQuorum = None tier1Keys = None maxConnections = None @@ -496,7 +497,10 @@ type Tests(output: ITestOutputHelper) = let Chennai = { lat = 13.08784; lon = 80.27847 } let dns1 = PeerDnsName "www.foo.com" let dns2 = PeerDnsName "www.bar.com" - let cmd = getNetworkDelayCommands Ashburn [| (Beauharnois, dns1); (Chennai, dns2) |] None + + let cmd = + getNetworkDelayCommands (getPeerDelays Ashburn [| (Beauharnois, dns1); (Chennai, dns2) |]) None + let cmdStr = cmd.ToString() Assert.Contains(dns1.StringName, cmdStr) diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index b967a9ff..3cbf162b 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -165,9 +165,15 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") allNodes let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes + + let loadGenNodes = + if List.isEmpty loadGenNodes then + // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes - // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes // Assign pre-generated transaction information to each load generator node. diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 571cb427..af00717a 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -337,8 +337,13 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: None } let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes + let loadGenNodes = + if List.isEmpty loadGenNodes then + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes diff --git a/src/FSLibrary/StellarCoreSet.fs b/src/FSLibrary/StellarCoreSet.fs index e1c1c3ca..a241cb8e 100644 --- a/src/FSLibrary/StellarCoreSet.fs +++ b/src/FSLibrary/StellarCoreSet.fs @@ -192,6 +192,7 @@ type QuorumSetConfiguration = type CoreSetOptions = { nodeCount: int nodeLocs: GeoLoc list option + edgeDelays: Map option dbType: DBType emptyDirType: EmptyDirType syncStartupDelay: int option @@ -211,6 +212,8 @@ type CoreSetOptions = validate: bool homeDomain: string option tier1: bool option + // This should only be set in the pubnet data with delay case + generatesLoad: bool option catchupMode: CatchupMode image: string initialization: CoreSetInitialization @@ -235,6 +238,7 @@ type CoreSetOptions = static member GetDefault(image: string) = { nodeCount = 3 nodeLocs = None + edgeDelays = None dbType = Sqlite emptyDirType = MemoryBackedEmptyDir syncStartupDelay = Some(5) @@ -254,6 +258,7 @@ type CoreSetOptions = validate = true homeDomain = Some "stellar.org" tier1 = None + generatesLoad = None catchupMode = CatchupComplete image = image initialization = CoreSetInitialization.Default diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 828048ff..72af8f34 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -81,6 +81,7 @@ type MissionContext = apiRateLimit: int httpProxyReplicas: int pubnetData: string option + pubnetDataDelay: bool flatQuorum: bool option tier1Keys: string option maxConnections: int option diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index 61f3d6b1..6fb10f1b 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -20,7 +20,42 @@ let PubnetLatestHistoryArchiveState = let TestnetLatestHistoryArchiveState = "http://history.stellar.org/prd/core-testnet/core_testnet_001/.well-known/stellar-history.json" -type PubnetNode = JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> +type PubnetNodeJSON = + JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> + +type PubnetNodeDelayJSON = + JsonProvider<"json-type-samples/sample-network-data-delay.json", SampleIsList=false, ResolutionFolder=cwd> + +type PubnetNode = + { PublicKey: string + Peers: string array + GeneratesLoad: bool option + RadarHomeDomain: string option + RadarIsValidating: bool option + RadarGeoData: {| Latitude: decimal; Longitude: decimal |} option + RadarName: string option } + + static member ofJSON(node: PubnetNodeJSON.Root) : PubnetNode = + { PublicKey = node.PublicKey + Peers = node.Peers + GeneratesLoad = None + RadarHomeDomain = node.RadarHomeDomain + RadarIsValidating = node.RadarIsValidating + RadarGeoData = + match node.RadarGeoData with + | Some geoData -> Some {| Latitude = geoData.Latitude; Longitude = geoData.Longitude |} + | None -> None + RadarName = node.RadarName } + + static member ofJSONDelay(node: PubnetNodeDelayJSON.Root) : PubnetNode = + { PublicKey = node.PublicKey + Peers = node.Peers |> Array.map (fun p -> p.Key) + GeneratesLoad = node.GeneratesLoad + RadarHomeDomain = node.RadarHomeDomain + RadarIsValidating = node.RadarIsValidating + RadarGeoData = None + RadarName = node.RadarName } + type Tier1PublicKey = JsonProvider<"json-type-samples/sample-keys.json", SampleIsList=false, ResolutionFolder=cwd> // Adjacency map for peers @@ -147,8 +182,8 @@ let locations = // Each edge connecting a and b is represented as (a, b) if a < b, and (b, a) otherwise. // This makes sense as the graph is undirected, and it also makes it easier to handle a set of edges. -let extractEdges (graph: PubnetNode.Root array) : (string * string) array = - let getEdgesFromNode (node: PubnetNode.Root) : (string * string) array = +let extractEdges (graph: PubnetNode array) : (string * string) array = + let getEdgesFromNode (node: PubnetNode) : (string * string) array = node.Peers |> Array.filter (fun peer -> peer < node.PublicKey) // This filter ensures that we add each edge exactly once. |> Array.map (fun peer -> (peer, node.PublicKey)) @@ -260,7 +295,7 @@ let private pruneAdjacencyMap (maxConnections: int) (noPrune: Set) (m: P // then we pick a random edge (a, b), remove (a, b) and add (a, u) and (u, b). // We continue this process until u has a desired degree. let addEdges - (graph: PubnetNode.Root array) + (graph: PubnetNode array) (newNodes: string array) (tier1KeySet: Set) (random: System.Random) @@ -334,14 +369,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin if context.pubnetData.IsNone then failwith "pubnet simulation requires --pubnet-data=" - let allPubnetNodes : PubnetNode.Root array = PubnetNode.Load(context.pubnetData.Value) - // A Random object with a fixed seed. let random = System.Random context.randomSeed let newTier1Nodes = [ for i in 1 .. context.tier1OrgsToAdd * tier1OrgSize -> - PubnetNode.Parse( + PubnetNodeJSON.Parse( sprintf """ [{ "publicKey": "%s", "radar_homeDomain": "home.domain.%d" }] """ (KeyPair.Random().Address) @@ -351,19 +384,19 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let newNonTier1Nodes = [ for i in 1 .. context.nonTier1NodesToAdd -> - PubnetNode.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] ] + PubnetNodeJSON.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] ] |> Array.ofList let tier1KeySet : Set = if context.tier1Keys.IsSome then - let newTier1Keys = Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newTier1Nodes in + let newTier1Keys = Array.map (fun (n: PubnetNodeJSON.Root) -> n.PublicKey) newTier1Nodes in Tier1PublicKey.Load(context.tier1Keys.Value) |> Array.map (fun n -> n.PublicKey) |> Array.append newTier1Keys |> Set.ofArray else - PubnetNode.Load(context.pubnetData.Value) + PubnetNodeJSON.Load(context.pubnetData.Value) // Any node with a home domain is considered tier1. |> Array.filter (fun n -> n.RadarHomeDomain.IsSome) |> Array.map (fun n -> n.PublicKey) @@ -376,7 +409,40 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin Array.append newTier1Nodes newNonTier1Nodes |> Array.sortBy (fun _ -> random.Next()) - let allPubnetNodes = allPubnetNodes |> Array.append newNodes + let (allPubnetNodes: PubnetNode array, edgeDelays: Map option) = + if context.pubnetDataDelay then + if newNodes.Length > 0 then + failwith "--pubnet-data-delay cannot be used with --tier1-orgs-to-add or --non-tier1-nodes-to-add" + + let nodes = PubnetNodeDelayJSON.Load context.pubnetData.Value + + let edgeDelays = + nodes + |> Array.map (fun n -> n.Peers |> Array.map (fun p -> (n.PublicKey, p.Key), p.OwdMs)) + |> Array.concat + |> Map.ofArray + // Check that edgeDelays is connection-wise symmetric (but allow asymmetric delays) + for key1, key2 in edgeDelays.Keys do + if not (edgeDelays.ContainsKey(key2, key1)) then + failwithf + "Edge delay data is not symmetric: (%s, %s) is present but (%s, %s) is not" + key1 + key2 + key2 + key1 + + let nodes = nodes |> Array.map PubnetNode.ofJSONDelay + // Check that all load-generating nodes have a home domain. + for n in nodes do + if n.GeneratesLoad = Some true && n.RadarHomeDomain.IsNone then + failwithf "Load generator node %s does not have a home domain" n.PublicKey + + nodes, Some edgeDelays + else + PubnetNodeJSON.Load context.pubnetData.Value + |> Array.append newNodes + |> Array.map PubnetNode.ofJSON, + None // For each pubkey in the pubnet, we map it to an actual KeyPair (with a private // key) to use in the simulation. It's important to keep these straight! The keys @@ -384,7 +450,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // throughout the rest of this function, as strings called "pubkey", but should not // appear in the final CoreSets we're building. let mutable pubnetKeyToSimKey : Map = - Array.map (fun (n: PubnetNode.Root) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes + Array.map (fun (n: PubnetNode) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes |> Map.ofArray // Not every pubkey used in the qsets is represented in the base set of pubnet nodes, @@ -401,8 +467,14 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // This is because `networkSizeLimit` may be smaller than the degree of some new node // and cause an issue to the scaling algorithm. let adjacencyMap = - addEdges allPubnetNodes (Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newNodes) tier1KeySet random - |> if context.fullyConnectTier1 then fullyConnectTier1 tier1KeySet else id + addEdges allPubnetNodes (Array.map (fun (n: PubnetNodeJSON.Root) -> n.PublicKey) newNodes) tier1KeySet random + |> if context.fullyConnectTier1 then + if context.pubnetDataDelay then + failwith "--pubnet-data-delay cannot be used with --fully-connect-tier1" + else + fullyConnectTier1 tier1KeySet + else + id |> match context.maxConnections with | Some maxConnections -> // Prune map to ensure that no node has more than @@ -420,8 +492,8 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgNodes, miscNodes = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) - |> Array.partition (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) + |> Array.partition (fun (n: PubnetNode) -> n.RadarHomeDomain.IsSome) // We then trim down the set of misc nodes so that they fit within simulation // size limit passed. If we can't even fit the org nodes, we fail here. @@ -439,18 +511,27 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let allPubnetNodes = Array.append orgNodes miscNodes let _ = assert ((Array.length allPubnetNodes) <= context.networkSizeLimit) - let allPubnetNodeKeys = - Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) allPubnetNodes - |> Set.ofArray + let allPubnetNodeKeys = Array.map (fun (n: PubnetNode) -> n.PublicKey) allPubnetNodes |> Set.ofArray LogInfo "SimulatePubnet will run with %d nodes" (Array.length allPubnetNodes) + let edgeDelays : Map option = + match edgeDelays with + | None -> None + | Some edgeDelays -> + edgeDelays + |> Map.toSeq + |> Seq.filter (fun ((a, b), _) -> Set.contains a allPubnetNodeKeys && Set.contains b allPubnetNodeKeys) + |> Seq.map (fun ((a, b), delay) -> ((getSimKey a).PublicKey, (getSimKey b).PublicKey), delay) + |> Map.ofSeq + |> Some + // We then group the org nodes by their home domains. The domain names are drawn // from the HomeDomains of the public network but with periods replaced with dashes, // and lowercased, so for example keybase.io turns into keybase-io. - let groupedOrgNodes : (HomeDomainName * PubnetNode.Root array) array = + let groupedOrgNodes : (HomeDomainName * PubnetNode array) array = Array.groupBy - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let domain = n.RadarHomeDomain.Value // We turn 'www.stellar.org' into 'stellar' // and 'stellar.blockdaemon.com' into 'blockdaemon' @@ -478,12 +559,22 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin HomeDomainName lowercase) orgNodes + // Check that load generators each exist in size-1 orgs + for hdn, nodes in groupedOrgNodes do + let loadGenerators = nodes |> Array.filter (fun n -> n.GeneratesLoad = Some true) + + if loadGenerators.Length > 0 && nodes.Length > 1 then + failwithf + "Load generator node(s) found in org %s with %d nodes. Each node that generates load should have its own home domain" + hdn.StringName + nodes.Length + // Then build a map from accountID to HomeDomainName and index-within-domain // for each org node. let orgNodeHomeDomains : Map = Array.collect - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> - Array.mapi (fun (i: int) (n: PubnetNode.Root) -> (n.PublicKey, (hdn, i))) nodes) + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> + Array.mapi (fun (i: int) (n: PubnetNode) -> (n.PublicKey, (hdn, i))) nodes) groupedOrgNodes |> Map.ofArray @@ -538,14 +629,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let defaultQuorum : QuorumSetSpec = let tier1Nodes = allPubnetNodes - |> Array.filter - (fun (n: PubnetNode.Root) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) let tier1NodesGroupedByHomeDomain : (string array) array = tier1Nodes - |> Array.groupBy (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.Value) - |> Array.map - (fun (_, nodes: PubnetNode.Root []) -> Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) nodes) + |> Array.groupBy (fun (n: PubnetNode) -> n.RadarHomeDomain.Value) + |> Array.map (fun (_, nodes: PubnetNode []) -> Array.map (fun (n: PubnetNode) -> n.PublicKey) nodes) let orgToExplicitQSet (org: string array) : ExplicitQuorumSet = { thresholdPercent = Some(51) // Simple majority @@ -555,7 +644,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let tier1Orgs = tier1Nodes |> Array.map - (fun (n: PubnetNode.Root) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) + (fun (n: PubnetNode) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) |> Set.ofArray let flatQset = @@ -596,9 +685,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // as long as the random function is persistent. let geoLocations : GeoLoc array = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> n.RadarGeoData.IsSome) + |> Array.filter (fun (n: PubnetNode) -> n.RadarGeoData.IsSome) |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> { lat = float n.RadarGeoData.Value.Latitude lon = float n.RadarGeoData.Value.Longitude }) |> Seq.ofArray @@ -611,7 +700,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // The assignment is deterministic as it depends on the public key of the // node. This ensures that geolocations persist across runs, even if the // total number of nodes changes via the *-orgs-to-add flags. - let getGeoLocOrDefault (n: PubnetNode.Root) : GeoLoc = + let getGeoLocOrDefault (n: PubnetNode) : GeoLoc = match n.RadarGeoData with | Some geoData -> { lat = float geoData.Latitude; lon = float geoData.Longitude } | None -> @@ -636,7 +725,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin allPubnetNodes |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let key = getSimPubKey n.PublicKey let peers = @@ -659,7 +748,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // Given a node, returns a tuple where the first element is a boolean // indicating whether the node is a validator, and the second element is an // appropriate quorum set configuration for that node. - let computeQset (n: PubnetNode.Root) = + let computeQset (n: PubnetNode) = let tier1 = Set.contains n.PublicKey tier1KeySet let hdn = homeDomainNameForKey n.PublicKey @@ -746,7 +835,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let miscCoreSets : CoreSet array = Array.mapi - (fun (_: int) (n: PubnetNode.Root) -> + (fun (_: int) (n: PubnetNode) -> let hdn = homeDomainNameForKey n.PublicKey let keys = [| getSimKey n.PublicKey |] let tier1 = Set.contains n.PublicKey tier1KeySet @@ -760,7 +849,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = None nodeLocs = Some [ getGeoLocOrDefault n ] + edgeDelays = edgeDelays preferredPeersMap = Some(keysToPreferredPeersMap keys) } let shouldWaitForConsensus = manualclose @@ -770,11 +861,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgCoreSets : CoreSet array = Array.map - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> assert (nodes.Length <> 0) let nodeList = List.ofArray nodes - let keys = Array.map (fun (n: PubnetNode.Root) -> getSimKey n.PublicKey) nodes + let keys = Array.map (fun (n: PubnetNode) -> getSimKey n.PublicKey) nodes let tier1 = Set.contains nodes.[0].PublicKey tier1KeySet + let generatesLoad = nodes.[0].GeneratesLoad let validate, qset = mergeQSets (List.map computeQset nodeList) @@ -785,7 +877,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = generatesLoad nodeLocs = Some(List.map getGeoLocOrDefault nodeList) + edgeDelays = edgeDelays preferredPeersMap = Some(keysToPreferredPeersMap keys) } let shouldWaitForConsensus = manualclose diff --git a/src/FSLibrary/StellarNetworkDelays.fs b/src/FSLibrary/StellarNetworkDelays.fs index c934f669..468f70e5 100644 --- a/src/FSLibrary/StellarNetworkDelays.fs +++ b/src/FSLibrary/StellarNetworkDelays.fs @@ -55,7 +55,12 @@ let networkPingInMs (loc1: GeoLoc) (loc2: GeoLoc) : double = // A ping is a round trip, so double one-way delay. 2.0 * (networkDelayInMs loc1 loc2) -let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) array) (delay: int option) : ShCmd = +let getPeerDelays (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) array) : (int * PeerDnsName) array = + // Get the one way delays from loc1 to the locationss in locsAndNames + locsAndNames + |> Array.map (fun (loc2, name) -> int (networkDelayInMs loc1 loc2), name) + +let getNetworkDelayCommands (delaysAndNames: (int * PeerDnsName) array) (delay: int option) : ShCmd = // Traffic shaping happens using the 'tc' command on linux. This is a // complicated command. We build up the commands in pieces. @@ -264,19 +269,19 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) let perPeerResolveCmds : ShCmd array = Array.mapi - (fun (i: int) (loc2: GeoLoc, peer: PeerDnsName) -> + (fun (i: int) (_, peer: PeerDnsName) -> let classNo = 1 + i [| resolveName classNo peer |]) - locsAndNames + delaysAndNames |> Array.concat let perPeerCmds : ShCmd array = Array.mapi - (fun (i: int) (loc2: GeoLoc, peer: PeerDnsName) -> + (fun (i: int) (edgeDelay: int, peer: PeerDnsName) -> let msDelay = match delay with | Some d -> d - | None -> int (networkDelayInMs loc1 loc2) + | None -> edgeDelay let classNo = 1 + i @@ -284,7 +289,7 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) addFilter classNo addNetemQdisc classNo msDelay addFqLeaf classNo |]) - locsAndNames + delaysAndNames |> Array.concat let seq = @@ -302,7 +307,6 @@ let getNetworkDelayCommands (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) ShSeq seq - type NetworkCfg with member self.LocAndDnsName (cs: CoreSet) (i: int) : (GeoLoc * PeerDnsName) Option = @@ -319,27 +323,68 @@ type NetworkCfg with match self.missionContext.installNetworkDelay with | Some true -> let atLeastOneLocation = Map.exists (fun _ cs -> cs.options.nodeLocs.IsSome) self.coreSets - - if atLeastOneLocation then - true - else - // If there's _some_ geo info, we can extrapolate. - // However, if there's _no_ geo info, we can't really do anything. - // Don't install network delay if your topology has no geo info. - failwith "Network delays can't be installed if no geo info provided." + let haveDelays = Map.forall (fun _ cs -> cs.options.edgeDelays.IsSome) self.coreSets + + match self.missionContext.pubnetDataDelay with + | true -> + if haveDelays then + true + else + failwith "Per-edge network delays can't be installed if edge data is missing" + | false -> + if atLeastOneLocation then + true + else + // If there's _some_ geo info, we can extrapolate. + // However, if there's _no_ geo info, we can't really do anything. + // Don't install network delay if your topology has no geo info. + failwith "Network delays can't be installed if no geo info provided." | _ -> false member self.NetworkDelayScript (cs: CoreSet) (i: int) : ShCmd = - match cs.options.nodeLocs with - | None -> ShCmd.True() - | Some (locs) -> - let selfLoc = locs.[i] - - let otherLocsAndNames : (GeoLoc * PeerDnsName) array = + match self.missionContext.pubnetDataDelay with + | true -> + let otherDelaysAndNames : (int * PeerDnsName) array = match cs.options.preferredPeersMap with | Some (otherMap) -> let otherKeys = Array.ofList otherMap.[cs.keys.[i].PublicKey] - Array.choose self.LocAndDnsNameForKey otherKeys - | None -> Array.choose id (self.MapAllPeers self.LocAndDnsName) - - getNetworkDelayCommands selfLoc otherLocsAndNames self.missionContext.flatNetworkDelay + let otherKeysSet = Set.ofArray otherKeys + + let names = + self.MapAllPeers + (fun cs i -> + let pk = cs.keys.[i].PublicKey + if otherKeysSet.Contains pk then Some(pk, self.PeerDnsName cs i) else None) + |> Array.choose id + |> Map.ofArray + + let delays = cs.options.edgeDelays.Value + let selfKey = cs.keys.[i].PublicKey + + otherKeys + |> Array.map (fun peerKey -> delays.[selfKey, peerKey], names.[peerKey]) + | None -> + if self.missionContext.flatNetworkDelay.IsNone then + failwith + "Failed to construct network delay script: no preferred peers map or flat network delay" + else + [||] + + getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay + | false -> + match cs.options.nodeLocs with + | None -> ShCmd.True() + | Some (locs) -> + let selfLoc = locs.[i] + + let otherDelaysAndNames : (int * PeerDnsName) array = + let locsAndNames = + match cs.options.preferredPeersMap with + | Some (otherMap) -> + let otherKeys = Array.ofList otherMap.[cs.keys.[i].PublicKey] + Array.choose self.LocAndDnsNameForKey otherKeys + | None -> Array.choose id (self.MapAllPeers self.LocAndDnsName) + + locsAndNames |> getPeerDelays selfLoc + + getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay diff --git a/src/FSLibrary/json-type-samples/sample-network-data-delay.json b/src/FSLibrary/json-type-samples/sample-network-data-delay.json new file mode 100644 index 00000000..becb7256 --- /dev/null +++ b/src/FSLibrary/json-type-samples/sample-network-data-delay.json @@ -0,0 +1,77 @@ +[ + { + "isTier1": true, + "numTotalInboundPeers": 2, + "numTotalOutboundPeers": 2, + "peers": [ + {"key": "GBOOLF6VITJWPTYYTWGJST35XHFUCTPTOJGZBPA7FLNINX6YD3AQHUBL", "owdMs": 3}, + {"key": "GDUBKMXWLSQNK33XQNRPSDEYODROPPRM4XLJGXFEZPWCLKIOFOTTCX3B", "owdMs": 10} + ], + "publicKey": "GCP7LQUW5UXXDNQI4ENXTUULZOIEIR6UDRC5S3VN6ZGKMOZ6TL3YB6IS", + "radar_homeDomain": "www.someone.org", + "radar_index": 1, + "radar_ip": "192.168.1.1", + "radar_isValidating": true, + "radar_isp": "Some Datacenter Inc.", + "radar_name": "SomeoneNet-1", + "radar_organizationId": "764efa883dda1e11db47671c4a3bbd9e", + "radar_quorumSet": { + "hashKey": "MzI3MzUyODNjODk2NzQ2YmQyMzNjOThkNDZkMDM4OThkOGJmNDdjZiAgLQo=", + "innerQuorumSets": [ + { + "hashKey": "OTI1NzIxZjllYzZiYjE1MGVjODQ0YWJjN2MwNjVkZDI5ODY0NjhiZiAgLQo=", + "innerQuorumSets": [], + "threshold": 2, + "validators": [ + "GDMSQ7KDJKDVQCVDMZ7VW2FXDUZLHILTEA7JEE7TBFADYG6PNJSABZTM", + "GA4LI76TTHQDK3QL6DAUCOUMBB3UHDU3IWT3HAVNZ36HRWQQS2CMPVB5", + "GBBAF6QXXYXG2RC2ZWSIX6UR3LQRDLFNCRFJ226L5DMFK3O3AEVFOCP6" + ] + }, + { + "hashKey": "NTVjYTYyODZlM2U0ZjRmYmE1ZDA0NDgzMzNmYTk5ZmM1YTQwNGE3MyAgLQo=", + "innerQuorumSets": [], + "threshold": 2, + "validators": [ + "GDVIDS5TYLQJ5THYGB6ULNJJMTUIZYROPZOGZGLVFO336KBGYSJCYA4N", + "GA7SDMIG3CNKEE6CP45IJSK3VOATMIZIXGHVNC2HRQACWC3KHD7EA5S3", + "GB37UNY2X5X4DSOUVUAXPXTOVKWF4NHTXTTPW2FCVHGNSGYLPQKBUBXW" + ] + } + ], + "threshold": 1, + "validators": [] + }, + "version": "v13.1.0" + }, + { + "numTotalInboundPeers": 1, + "numTotalOutboundPeers": 1, + "peers": [ + {"key": "GDQHFKIEF3NPKZVDC2WO5E5P54H3UG3QU2AZZCDINVF4THK5BZV6E4AK", "owdMs": 5} + ], + "publicKey": "GBDRGD7ZSIBXYLRHNO2LRI2BLOLOLD4THAMDCMCVK3JX2GFE4KQWABUE", + "radar_index": 0.5, + "radar_ip": "192.168.1.1", + "radar_isValidating": false, + "radar_isp": "Some ISP", + "radar_quorumSet": { + "innerQuorumSets": [], + "threshold": 9999999, + "validators": ["GBN5UP5XG2NKBGSBXGV632XFUQYYU62XYWR47GH7KQZUQDTVT7ROGK5C", + "GDQHFKIEF3NPKZVDC2WO5E5P54H3UG3QU2AZZCDINVF4THK5BZV6E4AK" + ] + }, + "generatesLoad": false, + "version": "stellar-core 15.0.0 (8d6e6d74cfae65943211bdcc5ba083976d525a04)" + }, + { + "peers": [ + {"key": "GBN5UP5XG2NKBGSBXGV632XFUQYYU62XYWR47GH7KQZUQDTVT7ROGK5C", "owdMs": 9} + ], + "radar_homeDomain": "horizon", + "generatesLoad": true, + "publicKey": "GATTOMMXJ4TITUGIP4MQ635Z6FJYK57WOWGDQ7AM6DUWYBM5TUJ2BLFV", + "version": "v15.0.0" + } +] From ad4e6d8a493c481a9f9153264c4398c7ca7d83d8 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:26:53 -0700 Subject: [PATCH 2/9] Add flag for enabling e2e metric --- src/App/Program.fs | 11 +++++++++++ src/FSLibrary.Tests/Tests.fs | 1 + src/FSLibrary/StellarCoreCfg.fs | 6 ++++++ src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 19 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 6ec33a44..0bde8ab3 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -74,6 +74,7 @@ type MissionOptions httpProxyReplicas: int, pubnetData: string option, pubnetDataDelay: bool, + measureE2eLatency: bool, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -325,6 +326,12 @@ type MissionOptions Default = false)>] member self.PubnetDataDelay = pubnetDataDelay + [] + member self.MeasureE2eLatency = measureE2eLatency + [] member self.FlatQuorum = flatQuorum @@ -753,6 +760,9 @@ let main argv = if mission.PubnetData.IsNone && mission.PubnetDataDelay then failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" + if mission.MeasureE2eLatency && not mission.PubnetDataDelay then + failwith "Error: --measure-e2e-latency requires --pubnet-data-delay to be set" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -859,6 +869,7 @@ let main argv = httpProxyReplicas = mission.HttpProxyReplicas pubnetData = mission.PubnetData pubnetDataDelay = mission.PubnetDataDelay + measureE2eLatency = mission.MeasureE2eLatency flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 1e14bacf..964f9314 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -72,6 +72,7 @@ let ctx : MissionContext = httpProxyReplicas = 2 pubnetData = None pubnetDataDelay = false + measureE2eLatency = false flatQuorum = None tier1Keys = None maxConnections = None diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 190fea2b..1a563fd3 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -166,6 +166,7 @@ type StellarCoreCfg = automaticMaintenanceCount: int accelerateTime: bool generateLoad: bool + measureE2eLatency: bool updateSorobanCosts: bool option manualClose: bool invariantChecks: InvariantChecksSpec @@ -317,6 +318,9 @@ type StellarCoreCfg = t.Add("ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING", self.accelerateTime) |> ignore t.Add("ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING", self.generateLoad) |> ignore + if self.measureE2eLatency && self.network.missionContext.measureE2eLatency then + t.Add("LOADGEN_MEASURE_TX_LATENCY_FOR_TESTING", true) |> ignore + if self.updateSorobanCosts.IsSome then t.Add("UPDATE_SOROBAN_COSTS_DURING_PROTOCOL_UPGRADE_FOR_TESTING", self.updateSorobanCosts.Value) |> ignore @@ -653,6 +657,7 @@ type NetworkCfg with automaticMaintenanceCount = if opts.performMaintenance then 50000 else 0 accelerateTime = opts.accelerateTime generateLoad = true + measureE2eLatency = opts.generatesLoad = Some true updateSorobanCosts = opts.updateSorobanCosts manualClose = false invariantChecks = opts.invariantChecks @@ -698,6 +703,7 @@ type NetworkCfg with automaticMaintenanceCount = if c.options.performMaintenance then 50000 else 0 accelerateTime = c.options.accelerateTime generateLoad = true + measureE2eLatency = c.options.generatesLoad = Some true updateSorobanCosts = c.options.updateSorobanCosts manualClose = false invariantChecks = c.options.invariantChecks diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 72af8f34..a6bfc5e1 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -82,6 +82,7 @@ type MissionContext = httpProxyReplicas: int pubnetData: string option pubnetDataDelay: bool + measureE2eLatency: bool flatQuorum: bool option tier1Keys: string option maxConnections: int option From 282127d1b09aafbe0df1f70c3420e655ca3f2ce4 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:27:37 -0700 Subject: [PATCH 3/9] Add flag for setting PEER_AUTHENTICATION_TIMEOUT --- src/App/Program.fs | 7 +++++++ src/FSLibrary.Tests/Tests.fs | 1 + src/FSLibrary/StellarCoreCfg.fs | 4 ++++ src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 13 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 0bde8ab3..1eb7079c 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -75,6 +75,7 @@ type MissionOptions pubnetData: string option, pubnetDataDelay: bool, measureE2eLatency: bool, + peerAuthenticationTimeout: int option, flatQuorum: bool option, tier1Keys: string option, maxConnections: int option, @@ -332,6 +333,11 @@ type MissionOptions Default = false)>] member self.MeasureE2eLatency = measureE2eLatency + [] + member self.PeerAuthenticationTimeout = peerAuthenticationTimeout + [] member self.FlatQuorum = flatQuorum @@ -870,6 +876,7 @@ let main argv = pubnetData = mission.PubnetData pubnetDataDelay = mission.PubnetDataDelay measureE2eLatency = mission.MeasureE2eLatency + peerAuthenticationTimeout = mission.PeerAuthenticationTimeout flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys maxConnections = mission.MaxConnections diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 964f9314..c85cab85 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -73,6 +73,7 @@ let ctx : MissionContext = pubnetData = None pubnetDataDelay = false measureE2eLatency = false + peerAuthenticationTimeout = None flatQuorum = None tier1Keys = None maxConnections = None diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 1a563fd3..7e4b033a 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -369,6 +369,10 @@ type StellarCoreCfg = t.Add("MAX_ADDITIONAL_PEER_CONNECTIONS", self.targetPeerConnections * 3) |> ignore + match self.network.missionContext.peerAuthenticationTimeout with + | Some timeout -> t.Add("PEER_AUTHENTICATION_TIMEOUT", timeout) |> ignore + | None -> () + t.Add("QUORUM_INTERSECTION_CHECKER", false) |> ignore t.Add("MANUAL_CLOSE", self.manualClose) |> ignore diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index a6bfc5e1..5772cfd4 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -83,6 +83,7 @@ type MissionContext = pubnetData: string option pubnetDataDelay: bool measureE2eLatency: bool + peerAuthenticationTimeout: int option flatQuorum: bool option tier1Keys: string option maxConnections: int option From d951b166c44ba4530478bec157184f97110482d0 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Fri, 26 Jun 2026 10:12:48 -0700 Subject: [PATCH 4/9] Address copilot feedback --- src/FSLibrary/StellarNetworkDelays.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FSLibrary/StellarNetworkDelays.fs b/src/FSLibrary/StellarNetworkDelays.fs index 468f70e5..2e41a7e9 100644 --- a/src/FSLibrary/StellarNetworkDelays.fs +++ b/src/FSLibrary/StellarNetworkDelays.fs @@ -56,7 +56,7 @@ let networkPingInMs (loc1: GeoLoc) (loc2: GeoLoc) : double = 2.0 * (networkDelayInMs loc1 loc2) let getPeerDelays (loc1: GeoLoc) (locsAndNames: (GeoLoc * PeerDnsName) array) : (int * PeerDnsName) array = - // Get the one way delays from loc1 to the locationss in locsAndNames + // Get the one way delays from loc1 to the locations in locsAndNames locsAndNames |> Array.map (fun (loc2, name) -> int (networkDelayInMs loc1 loc2), name) @@ -368,7 +368,7 @@ type NetworkCfg with failwith "Failed to construct network delay script: no preferred peers map or flat network delay" else - [||] + self.MapAllPeers (fun cs i -> 0, self.PeerDnsName cs i) getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay | false -> From 62f6b288c5d790229166f5e24915bb0d532433c9 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Fri, 26 Jun 2026 10:16:54 -0700 Subject: [PATCH 5/9] format --- src/FSLibrary/StellarNetworkDelays.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FSLibrary/StellarNetworkDelays.fs b/src/FSLibrary/StellarNetworkDelays.fs index 2e41a7e9..c524be39 100644 --- a/src/FSLibrary/StellarNetworkDelays.fs +++ b/src/FSLibrary/StellarNetworkDelays.fs @@ -368,7 +368,7 @@ type NetworkCfg with failwith "Failed to construct network delay script: no preferred peers map or flat network delay" else - self.MapAllPeers (fun cs i -> 0, self.PeerDnsName cs i) + self.MapAllPeers(fun cs i -> 0, self.PeerDnsName cs i) getNetworkDelayCommands otherDelaysAndNames self.missionContext.flatNetworkDelay | false -> From c75ec3d1566542a9db1e04fdb09947e47b1c7213 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:17:17 -0700 Subject: [PATCH 6/9] Replace JSON generatesLoad with --loadgen-keys CLI arg --- src/App/Program.fs | 8 +++ src/FSLibrary.Tests/Tests.fs | 1 + src/FSLibrary/MaxTPSTest.fs | 2 +- src/FSLibrary/MinBlockTimeTest.fs | 2 +- src/FSLibrary/StellarCoreCfg.fs | 4 +- src/FSLibrary/StellarCoreSet.fs | 5 +- src/FSLibrary/StellarMissionContext.fs | 1 + src/FSLibrary/StellarNetworkData.fs | 61 +++++++++++++++---- .../sample-network-data-delay.json | 2 - 9 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 1eb7079c..f3098178 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -78,6 +78,7 @@ type MissionOptions peerAuthenticationTimeout: int option, flatQuorum: bool option, tier1Keys: string option, + loadgenKeys: string option, maxConnections: int option, fullyConnectTier1: bool, byteCountValues: seq, @@ -344,6 +345,9 @@ type MissionOptions [] member self.Tier1Keys = tier1Keys + [] + member self.LoadgenKeys = loadgenKeys + [] @@ -769,6 +773,9 @@ let main argv = if mission.MeasureE2eLatency && not mission.PubnetDataDelay then failwith "Error: --measure-e2e-latency requires --pubnet-data-delay to be set" + if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then + failwith "Error: --measure-e2e-latency requires --loadgen-keys" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -879,6 +886,7 @@ let main argv = peerAuthenticationTimeout = mission.PeerAuthenticationTimeout flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys + loadgenKeys = mission.LoadgenKeys maxConnections = mission.MaxConnections fullyConnectTier1 = mission.FullyConnectTier1 byteCountDistribution = diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index c85cab85..805a993a 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -76,6 +76,7 @@ let ctx : MissionContext = peerAuthenticationTimeout = None flatQuorum = None tier1Keys = None + loadgenKeys = None maxConnections = None fullyConnectTier1 = false peerReadingCapacity = None diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index 3cbf162b..19bc5c3d 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -165,7 +165,7 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") allNodes let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes - let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad) allNodes let loadGenNodes = if List.isEmpty loadGenNodes then diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index af00717a..b8547e5b 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -337,7 +337,7 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: None } let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes - let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad) allNodes let loadGenNodes = if List.isEmpty loadGenNodes then diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 7e4b033a..b15d9f6a 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -661,7 +661,7 @@ type NetworkCfg with automaticMaintenanceCount = if opts.performMaintenance then 50000 else 0 accelerateTime = opts.accelerateTime generateLoad = true - measureE2eLatency = opts.generatesLoad = Some true + measureE2eLatency = opts.generatesLoad updateSorobanCosts = opts.updateSorobanCosts manualClose = false invariantChecks = opts.invariantChecks @@ -707,7 +707,7 @@ type NetworkCfg with automaticMaintenanceCount = if c.options.performMaintenance then 50000 else 0 accelerateTime = c.options.accelerateTime generateLoad = true - measureE2eLatency = c.options.generatesLoad = Some true + measureE2eLatency = c.options.generatesLoad updateSorobanCosts = c.options.updateSorobanCosts manualClose = false invariantChecks = c.options.invariantChecks diff --git a/src/FSLibrary/StellarCoreSet.fs b/src/FSLibrary/StellarCoreSet.fs index a241cb8e..8333137f 100644 --- a/src/FSLibrary/StellarCoreSet.fs +++ b/src/FSLibrary/StellarCoreSet.fs @@ -212,8 +212,7 @@ type CoreSetOptions = validate: bool homeDomain: string option tier1: bool option - // This should only be set in the pubnet data with delay case - generatesLoad: bool option + generatesLoad: bool catchupMode: CatchupMode image: string initialization: CoreSetInitialization @@ -258,7 +257,7 @@ type CoreSetOptions = validate = true homeDomain = Some "stellar.org" tier1 = None - generatesLoad = None + generatesLoad = false catchupMode = CatchupComplete image = image initialization = CoreSetInitialization.Default diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 5772cfd4..24e2d676 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -86,6 +86,7 @@ type MissionContext = peerAuthenticationTimeout: int option flatQuorum: bool option tier1Keys: string option + loadgenKeys: string option maxConnections: int option fullyConnectTier1: bool byteCountDistribution: ((int * int) list) diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index 6fb10f1b..aec44daa 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -29,7 +29,6 @@ type PubnetNodeDelayJSON = type PubnetNode = { PublicKey: string Peers: string array - GeneratesLoad: bool option RadarHomeDomain: string option RadarIsValidating: bool option RadarGeoData: {| Latitude: decimal; Longitude: decimal |} option @@ -38,7 +37,6 @@ type PubnetNode = static member ofJSON(node: PubnetNodeJSON.Root) : PubnetNode = { PublicKey = node.PublicKey Peers = node.Peers - GeneratesLoad = None RadarHomeDomain = node.RadarHomeDomain RadarIsValidating = node.RadarIsValidating RadarGeoData = @@ -50,7 +48,6 @@ type PubnetNode = static member ofJSONDelay(node: PubnetNodeDelayJSON.Root) : PubnetNode = { PublicKey = node.PublicKey Peers = node.Peers |> Array.map (fun p -> p.Key) - GeneratesLoad = node.GeneratesLoad RadarHomeDomain = node.RadarHomeDomain RadarIsValidating = node.RadarIsValidating RadarGeoData = None @@ -402,6 +399,19 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin |> Array.map (fun n -> n.PublicKey) |> Set.ofArray + let loadgenKeySet : Set = + if context.loadgenKeys.IsSome then + Tier1PublicKey.Load(context.loadgenKeys.Value) + |> Array.map (fun n -> n.PublicKey) + |> Set.ofArray + else + Set.empty + + // Check that there is no overlap between tier1 and loadgen keys + let overlap = Set.intersect tier1KeySet loadgenKeySet + + if not (Set.isEmpty overlap) then + failwithf "Overlap between tier1 and loadgen keys: %A" overlap // Shuffle the nodes to ensure that the order will // not affect the outcome of the scaling algorithm. @@ -432,10 +442,6 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin key1 let nodes = nodes |> Array.map PubnetNode.ofJSONDelay - // Check that all load-generating nodes have a home domain. - for n in nodes do - if n.GeneratesLoad = Some true && n.RadarHomeDomain.IsNone then - failwithf "Load generator node %s does not have a home domain" n.PublicKey nodes, Some edgeDelays else @@ -444,6 +450,16 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin |> Array.map PubnetNode.ofJSON, None + // Ensure that each load-generator has a home domain + let allPubnetNodes : PubnetNode array = + allPubnetNodes + |> Array.map + (fun n -> + if Set.contains n.PublicKey loadgenKeySet && n.RadarHomeDomain.IsNone then + { n with RadarHomeDomain = Some "loadgennode" } + else + n) + // For each pubkey in the pubnet, we map it to an actual KeyPair (with a private // key) to use in the simulation. It's important to keep these straight! The keys // in the pubnet are _not_ used in the simulation. They are used as node identities @@ -513,6 +529,18 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let allPubnetNodeKeys = Array.map (fun (n: PubnetNode) -> n.PublicKey) allPubnetNodes |> Set.ofArray + // Check if we removed any tier1 or loadgen nodes. + let keptNodes : Set = + allPubnetNodeKeys + |> Seq.filter (fun (n: string) -> Set.contains n tier1KeySet || Set.contains n loadgenKeySet) + |> Set.ofSeq + + for removedNode in Set.difference tier1KeySet keptNodes do + LogWarn "Removed tier1 node %s from simulation" removedNode + + for removedNode in Set.difference loadgenKeySet keptNodes do + LogWarn "Removed loadgen node %s from simulation" removedNode + LogInfo "SimulatePubnet will run with %d nodes" (Array.length allPubnetNodes) let edgeDelays : Map option = @@ -529,6 +557,11 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // We then group the org nodes by their home domains. The domain names are drawn // from the HomeDomains of the public network but with periods replaced with dashes, // and lowercased, so for example keybase.io turns into keybase-io. + // Additionally, each load generator needs to exist in a size-1 org (because + // the downstream only generates load on one node per CoreSet). So, we add a + // loadgen-%d suffix to each load generator. + let loadgenNumber : Map = loadgenKeySet |> Seq.mapi (fun i key -> key, i) |> Map.ofSeq + let groupedOrgNodes : (HomeDomainName * PubnetNode array) array = Array.groupBy (fun (n: PubnetNode) -> @@ -548,20 +581,22 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // or not tier 1, we split them into separate orgs by appending // "-non-tier1" to the home domain of any non-tier1 node in an // org. - let withTierInfo = - if not (Set.contains n.PublicKey tier1KeySet) then + let withNodeInfo = + if Set.contains n.PublicKey loadgenKeySet then + cleanOrgName + "-loadgen-" + loadgenNumber.[n.PublicKey].ToString() + elif not (Set.contains n.PublicKey tier1KeySet) then cleanOrgName + "-non-tier1" else cleanOrgName - let lowercase = withTierInfo.ToLower() + let lowercase = withNodeInfo.ToLower() HomeDomainName lowercase) orgNodes // Check that load generators each exist in size-1 orgs for hdn, nodes in groupedOrgNodes do - let loadGenerators = nodes |> Array.filter (fun n -> n.GeneratesLoad = Some true) + let loadGenerators = nodes |> Array.filter (fun n -> Set.contains n.PublicKey loadgenKeySet) if loadGenerators.Length > 0 && nodes.Length > 1 then failwithf @@ -849,7 +884,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None - generatesLoad = None + generatesLoad = false nodeLocs = Some [ getGeoLocOrDefault n ] edgeDelays = edgeDelays preferredPeersMap = Some(keysToPreferredPeersMap keys) } @@ -866,7 +901,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let nodeList = List.ofArray nodes let keys = Array.map (fun (n: PubnetNode) -> getSimKey n.PublicKey) nodes let tier1 = Set.contains nodes.[0].PublicKey tier1KeySet - let generatesLoad = nodes.[0].GeneratesLoad + let generatesLoad = Set.contains nodes.[0].PublicKey loadgenKeySet let validate, qset = mergeQSets (List.map computeQset nodeList) diff --git a/src/FSLibrary/json-type-samples/sample-network-data-delay.json b/src/FSLibrary/json-type-samples/sample-network-data-delay.json index becb7256..df75d445 100644 --- a/src/FSLibrary/json-type-samples/sample-network-data-delay.json +++ b/src/FSLibrary/json-type-samples/sample-network-data-delay.json @@ -62,7 +62,6 @@ "GDQHFKIEF3NPKZVDC2WO5E5P54H3UG3QU2AZZCDINVF4THK5BZV6E4AK" ] }, - "generatesLoad": false, "version": "stellar-core 15.0.0 (8d6e6d74cfae65943211bdcc5ba083976d525a04)" }, { @@ -70,7 +69,6 @@ {"key": "GBN5UP5XG2NKBGSBXGV632XFUQYYU62XYWR47GH7KQZUQDTVT7ROGK5C", "owdMs": 9} ], "radar_homeDomain": "horizon", - "generatesLoad": true, "publicKey": "GATTOMMXJ4TITUGIP4MQ635Z6FJYK57WOWGDQ7AM6DUWYBM5TUJ2BLFV", "version": "v15.0.0" } From 4b86894b1d236c90827fec9f251c13058c1341e5 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:23:55 -0700 Subject: [PATCH 7/9] Allow specifying --loadgen-keys on non-delay style pubnet data --- src/App/Program.fs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index f3098178..0f6eb437 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -770,9 +770,6 @@ let main argv = if mission.PubnetData.IsNone && mission.PubnetDataDelay then failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" - if mission.MeasureE2eLatency && not mission.PubnetDataDelay then - failwith "Error: --measure-e2e-latency requires --pubnet-data-delay to be set" - if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then failwith "Error: --measure-e2e-latency requires --loadgen-keys" From 30ae4e00e4c8d58a00ce26d70374a64c431e723d Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:56:20 -0700 Subject: [PATCH 8/9] Add check that pubnet data is set when specifying loadgen keys --- src/App/Program.fs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 0f6eb437..2b28c154 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -770,6 +770,9 @@ let main argv = if mission.PubnetData.IsNone && mission.PubnetDataDelay then failwith "Error: --pubnet-data-delay requires --pubnet-data to be set" + if mission.LoadgenKeys.IsSome && mission.PubnetData.IsNone then + failwith "Error: --loadgen-keys requires --pubnet-data to be set" + if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then failwith "Error: --measure-e2e-latency requires --loadgen-keys" From 003e5f4e22514ec37c701a9e914012f6284d9761 Mon Sep 17 00:00:00 2001 From: Daniel Rebelsky <4641927+drebelsky@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:40:29 -0700 Subject: [PATCH 9/9] Update flag name --- src/FSLibrary/StellarCoreCfg.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index b15d9f6a..154508ff 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -319,7 +319,7 @@ type StellarCoreCfg = t.Add("ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING", self.generateLoad) |> ignore if self.measureE2eLatency && self.network.missionContext.measureE2eLatency then - t.Add("LOADGEN_MEASURE_TX_LATENCY_FOR_TESTING", true) |> ignore + t.Add("LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING", true) |> ignore if self.updateSorobanCosts.IsSome then t.Add("UPDATE_SOROBAN_COSTS_DURING_PROTOCOL_UPGRADE_FOR_TESTING", self.updateSorobanCosts.Value)