From 8a2ea4cf21f885969aca2fe9bc592317e473abea Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 16:54:19 -0400 Subject: [PATCH 01/24] Move TestParseBlockSizeMatchesBytes into external_test.go The test covers ParsedBlock's Size and Bytes, which live in external.go, so it belongs beside them rather than in the instance tests. The body is unchanged. --- external_test.go | 115 +++++++++++++++++++++++++++++++++++++++++++++++ instance_test.go | 101 ----------------------------------------- 2 files changed, 115 insertions(+), 101 deletions(-) create mode 100644 external_test.go diff --git a/external_test.go b/external_test.go new file mode 100644 index 00000000..6af93437 --- /dev/null +++ b/external_test.go @@ -0,0 +1,115 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "sync" + "testing" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/stretchr/testify/require" +) + +func TestParseBlockSizeMatchesBytes(t *testing.T) { + // Case 1: Bytes() first, Size() second, size returns the cached length. + pb := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 7, + TS: time.UnixMilli(8), + Payload: []byte("payload"), + }, + }, + } + bytes := pb.Bytes() + require.Equal(t, len(bytes), pb.Size()) + + // Case 2: Size() first on a non serialized block. it will + // compute the size and match a later Byte() call. + pb2 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 9, + TS: time.UnixMilli(10), + Payload: []byte("other payload"), + }, + }, + } + size := pb2.Size() + require.NotZero(t, size) + bytes2 := pb2.Bytes() + require.Equal(t, len(bytes2), size) + + // case 3: concurrent Size() calls on a block that was never serialized. + // the goroutines rase to compute the size, the lock must make this + // safe and every call must return the correct value + + pb3 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 11, + TS: time.UnixMilli(12), + Payload: []byte("concurrent"), + }, + }, + } + var wg sync.WaitGroup + sizes := make([]int, 4) + for i := range sizes { + wg.Add(1) + go func() { + defer wg.Done() + sizes[i] = pb3.Size() + }() + } + wg.Wait() + bytes3 := pb3.Bytes() + for _, size := range sizes { + require.Equal(t, len(bytes3), size) + } +} diff --git a/instance_test.go b/instance_test.go index c70cd9b8..3e1ef991 100644 --- a/instance_test.go +++ b/instance_test.go @@ -443,107 +443,6 @@ func TestInstanceRestartAcrossEpochs(t *testing.T) { waitForNumBlocks(t, storage, storage.NumBlocks()+2) } -func TestParseBlockSizeMatchesBytes(t *testing.T) { - // Case 1: Bytes() first, Size() second, size returns the cached length. - pb := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 7, - TS: time.UnixMilli(8), - Payload: []byte("payload"), - }, - }, - } - bytes := pb.Bytes() - require.Equal(t, len(bytes), pb.Size()) - - // Case 2: Size() first on a non serialized block. it will - // compute the size and match a later Byte() call. - pb2 := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 9, - TS: time.UnixMilli(10), - Payload: []byte("other payload"), - }, - }, - } - size := pb2.Size() - require.NotZero(t, size) - bytes2 := pb2.Bytes() - require.Equal(t, len(bytes2), size) - - // case 3: cincurrent Size() calls on a block that was never serialized. - // the goroutines rase to compute the size, the lock must make this - // safe and every call must return the correct value - - pb3 := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 11, - TS: time.UnixMilli(12), - Payload: []byte("concurrent"), - }, - }, - } - var wg sync.WaitGroup - sizes := make([]int, 4) - for i := range sizes { - wg.Add(1) - go func() { - defer wg.Done() - sizes[i] = pb3.Size() - }() - } - wg.Wait() - bytes3 := pb3.Bytes() - for _, size := range sizes { - require.Equal(t, len(bytes3), size) - } -} - // TestInstanceZeroBlockUsesLastNonSimplexPChainHeight asserts that the first ever Simplex block // references the P-chain height of the last non-Simplex block. func TestInstanceZeroBlockUsesLastNonSimplexPChainHeight(t *testing.T) { From 86c4be9b8ce1ad9f7cb2e1eb8cf992613c7fad99 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 16:54:29 -0400 Subject: [PATCH 02/24] Delete the instance tests Every one of them is rewritten against the network harness that follows, so they come out wholesale rather than being edited in place. The file is left holding only the helpers. --- instance_test.go | 758 ----------------------------------------------- 1 file changed, 758 deletions(-) diff --git a/instance_test.go b/instance_test.go index 3e1ef991..bf2497eb 100644 --- a/instance_test.go +++ b/instance_test.go @@ -29,764 +29,6 @@ import ( "go.uber.org/zap/zapcore" ) -func TestInstanceMixedNodeType(t *testing.T) { - t.Skip("skipping until test instance refactor") - - // One node is a validator at genesis, the other is a non-validator. - // After some blocks, the second (non-validator) node also becomes a validator. - // The test ensures that the second node tracks the chain while the first node expands the chain - // in the first epoch, and that both nodes move to the second epoch and then both are used for consensus together. - const ( - basePChainHeight = uint64(1) - epochChangePChainHeight = uint64(100) - ) - - var id [20]byte - rand.Read(id[:]) - firstNodeID := common.NodeID(id[:]) - - // The peer that joins the validator set in the last epoch. Its ID is chosen - // to differ from the (random) node under test. - var peerID [20]byte - rand.Read(peerID[:]) - secondNodeID := common.NodeID(peerID[:]) - - // Epoch 1 is single-validator - // The last epoch is expanded to two validators. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - epochChangePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, - {NodeID: peerID, BLSKey: []byte{0xbb}, Weight: 2}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // Create the storage for the instances and append the genesis block to each - storage := newStorageWithGenesis(t, genesisBlock) - storage2 := newStorageWithGenesis(t, genesisBlock) - - // Create the instances and register them to the network - firstInstance := newInstance(t, firstNodeID, storage, net, pChain, cops, genesisBlock) - secondInstance := newInstance(t, secondNodeID, storage2, net, pChain, cops, genesisBlock) - net.register(firstNodeID, firstInstance) - net.register(secondNodeID, secondInstance) - - /// Start the instances - require.NoError(t, firstInstance.Start(t.Context())) - require.NoError(t, secondInstance.Start(t.Context())) - t.Cleanup(firstInstance.Stop) - t.Cleanup(secondInstance.Stop) - - // Epoch 1: wait until the node has committed a series of normal blocks on its own. - const epoch1Target = uint64(5) // genesis(0) + zero block(1) + 3 normal blocks - waitForNumBlocks(t, storage, epoch1Target) - waitForNumBlocks(t, storage2, epoch1Target) - - // The validator set in force is the one introduced by the most recent block - // that carries a BlockValidationDescriptor (the zero block in epoch 1). - require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage)) - require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage2)) - - // Trigger the epoch change: the validator set changes at epochChangePChainHeight, - // growing from one validator to two. - pChain.advanceTo(epochChangePChainHeight) - approval := &common.ValidatorSetApproval{ - NodeID: peerID, - PChainHeight: epochChangePChainHeight, - AuxInfoDigest: sha256.Sum256(nil), - Signature: []byte{1, 2, 3}, - } - - // The node seals the epoch once it has a quorum of approvals of the new - // (two-validator) set. With two validators the node's self-approval is no longer - // a quorum and the peer is not running yet, so waitForSealingBlock injects the - // peer's approval on each poll until the sealing block is committed. - // TODO: Implement this capability in production so we won't need to inject approvals in tests. - sealingBlockSeq := waitForSealingBlock(t, firstInstance, approval, storage.NumBlocks()) - waitForNumBlocks(t, storage2, sealingBlockSeq) // Ensure the new validator has replicated the sealing block. - - // With both validators live, the two-validator epoch commits more blocks. - const epoch2Extra = uint64(3) - waitForNumBlocks(t, storage, sealingBlockSeq+epoch2Extra) - - // Confirm the second epoch has the second validator in the sealing block - require.Equal(t, secondInstance.Config.ID, latestValidatorID(t, storage)) -} - -// emptyVoteRecorder wraps a Broadcaster and signals the first time an empty vote is broadcast. -type emptyVoteRecorder struct { - Broadcaster - got chan struct{} -} - -func (r *emptyVoteRecorder) Broadcast(msg *common.Message) { - if msg.EmptyVoteMessage != nil { - select { - case r.got <- struct{}{}: - default: - } - } - r.Broadcaster.Broadcast(msg) -} - -func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { - const basePChainHeight = uint64(1) - - // Two validators, but only one is instantiated. Our node has the smaller ID so it sorts to - // index 0 and is a non-leader for round 1 (LeaderForRound picks index 1%2). The other - // validator is the round leader but is never created, so no block is ever proposed. - var ourID, leaderID [20]byte - ourID[0], leaderID[0] = 0x01, 0x02 - ourNode := common.NodeID(ourID[:]) - - require.NotEqual(t, ourNode, simplex.LeaderForRound([]common.NodeID{ourNode, leaderID[:]}, 1)) // ensure the leader is not our node - - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: ourID, BLSKey: []byte{0xaa}, Weight: 1}, - {NodeID: leaderID, BLSKey: []byte{0xbb}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - // A paused VM never has a pending block, so its WaitForPendingBlock blocks until its context - // is cancelled: the MSM must decide to build on its own for the round to make progress. - vm := newTestVM() - vm.pause() - - inst := newInstanceWithVM(t, ourNode, storage, net, pChain, cops, genesisBlock, vm) - - // Capture the empty vote the node broadcasts once it gives up waiting for the leader. - recorder := &emptyVoteRecorder{Broadcaster: inst.Config.Broadcaster, got: make(chan struct{}, 1)} - inst.Config.Broadcaster = recorder - - require.NoError(t, inst.Start(t.Context())) - t.Cleanup(inst.Stop) - - select { - case <-recorder.got: - case <-time.After(10 * time.Second): - require.FailNow(t, "node never broadcast an empty vote, so the Epoch did not drive the MSM's WaitForPendingBlock") - } -} - -func TestInstanceNonValidatorBootstraps(t *testing.T) { - t.Skip("skipping until test instance refactor") - - // One node is a validator and progresses the chain by building blocks, - // and its weight changes while the chain progresses in 3 different P-chain epoch heights. - // Then, we add another node which is a non-validator. - // The node should bootstrap the chain but without shutting down the non-validator instance. - // Later on, the non-validator becomes a validator. - const ( - basePChainHeight = uint64(1) - secondEpochP = uint64(100) - thirdEpochP = uint64(200) - joinEpochP = uint64(300) - ) - - var id [20]byte - rand.Read(id[:]) - validatorNodeID := common.NodeID(id[:]) - - // The node that joins later, first as a non-validator and eventually as a validator. - var nv [20]byte - rand.Read(nv[:]) - nonValidatorNodeID := common.NodeID(nv[:]) - - // The lone validator's weight changes at three different P-chain heights, sealing an - // epoch on each change. Because it remains the sole validator throughout, its own - // approval is a quorum and every epoch seals without any other node's participation. - // The last checkpoint (joinEpochP) grows the set to two validators, admitting the peer. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - secondEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, - }, - thirdEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, - }, - joinEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, - {NodeID: nv, BLSKey: []byte{0xbb}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // Both storages start with only the genesis block. - storage := newStorageWithGenesis(t, genesisBlock) - storage2 := newStorageWithGenesis(t, genesisBlock) - - validatorInstance := newInstance(t, validatorNodeID, storage, net, pChain, cops, genesisBlock) - nonValidatorInstance := newInstance(t, nonValidatorNodeID, storage2, net, pChain, cops, genesisBlock) - - // transitioned is closed when the node starts a Simplex epoch, i.e. becomes a validator. - // The node only ever starts an epoch here as part of its non-validator -> validator - // transition. - transitioned := make(chan struct{}) - nonValidatorInstance.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { - if strings.Contains(entry.Message, "Starting Simplex Epoch") { - select { - case <-transitioned: - default: - close(transitioned) - } - } - return nil - }) - - // Only the validator is running at first; it builds and seals the chain on its own. - net.register(validatorNodeID, validatorInstance) - require.NoError(t, validatorInstance.Start(t.Context())) - t.Cleanup(validatorInstance.Stop) - - // Epoch 1: wait until the validator has committed a series of blocks on its own. - waitForNumBlocks(t, storage, 5) // genesis(0) + zero block(1) + a few normal blocks - - // Drive two more epoch transitions by changing the validator's weight. Each change seals - // an epoch (and produces a sealing block) without any other node, since the validator's - // own approval is a quorum of the single-node set. The counts below include the zero block, - // which carries a block validation descriptor as well. - pChain.advanceTo(secondEpochP) - waitForSealingBlockCount(t, storage, 2) - - pChain.advanceTo(thirdEpochP) - waitForSealingBlockCount(t, storage, 3) - - // Let the third epoch grow a few normal blocks before the non validator joins, so bootstrap has to - // replicate past the sealing blocks and into ordinary blocks. - waitForNumBlocks(t, storage, storage.NumBlocks()+3) - - // The new node joins as a non-validator (it is absent from the validator set at the current - // P-chain tip) and bootstraps the chain from the validator. - net.register(nonValidatorNodeID, nonValidatorInstance) - require.NoError(t, nonValidatorInstance.Start(t.Context())) - t.Cleanup(nonValidatorInstance.Stop) - - // The non-validator replicates every sealed epoch and stays a non-validator throughout. - bootstrapTarget := storage.NumBlocks() - waitForNumBlocks(t, storage2, bootstrapTarget) - - // It replicated through the sealed epochs without becoming a validator. - select { - case <-transitioned: - t.Fatal("non-validator transitioned to validator before joining the set") - default: - } - - // Now grow the validator set to include the peer at the P-chain tip. - pChain.advanceTo(joinEpochP) - approval := &common.ValidatorSetApproval{ - NodeID: nv, - PChainHeight: joinEpochP, - AuxInfoDigest: sha256.Sum256(nil), - Signature: []byte{1, 2, 3}, - } - - // With two validators the validator's self-approval is no longer a quorum and the peer is - // still a non-validator, so we inject the peer's approval until the sealing block commits. - // TODO: Implement this capability in production so we won't need to inject approvals in tests. - sealingBlockSeq := waitForSealingBlock(t, validatorInstance, approval, storage.NumBlocks()) - waitForNumBlocks(t, storage2, sealingBlockSeq) - - // Once the non-validator replicates the sealing block that admits it, it detects that it is - // now a validator at the tip and transitions from non-validator to validator. - select { - case <-transitioned: - case <-time.After(20 * time.Second): - t.Fatal("non-validator did not transition to validator") - } - - // The newly promoted validator now participates in extending the chain. - require.Equal(t, nonValidatorInstance.Config.ID, latestValidatorID(t, storage)) - - // With both validators live, the two-validator epoch keeps committing blocks, and both - // nodes replicate them together. This confirms the promoted node contributes to consensus - // rather than merely tracking the chain. - const twoValidatorExtra = uint64(3) - extendedTarget := sealingBlockSeq + twoValidatorExtra - waitForNumBlocks(t, storage, extendedTarget) - waitForNumBlocks(t, storage2, extendedTarget) -} - -func TestInstanceRestartAcrossEpochs(t *testing.T) { - t.Skip("skipping until test instance refactor") - - // Restart a single validator at three different points in its lifecycle so that, - // on each (re)start, constructEpochAndValidatorSet takes a different branch of - // its switch: - // - // - Cold boot, ledger holds only the genesis (non-Simplex) block -> "genesis" branch. - // - Restart when the tip is a sealing block -> "sealing block at tip" branch. - // - Restart mid-epoch, when the tip is an ordinary Simplex block -> "sealing block in storage" branch. - // - const ( - basePChainHeight = uint64(1) - epochChangePChainHeight = uint64(100) - ) - - var id [20]byte - rand.Read(id[:]) - nodeID := common.NodeID(id[:]) - - // The lone validator's weight changes at epochChangePChainHeight, which seals the first - // epoch. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - epochChangePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - vm := newTestVM() - - const ( - logEpochFromGenesis = "Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)" - logEpochFromSealingTip = "Determined epoch and validator set from sealing block at tip" - logEpochFromSealingStorage = "Determined epoch and validator set from sealing block in storage" - ) - - // lastEpochBranch holds the full debug message constructEpochAndValidatorSet - // logs, identifying which branch of its switch the latest (re)start took. It is - // written synchronously during Start, but also from the epoch-change goroutine, - // so an atomic guards it. - var lastEpochBranch atomic.Pointer[string] - - // start (re)creates an instance over the same storage/network/VM. The log - // interceptor, installed before Start, records which branch startup took. - start := func() *Instance { - inst := newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, vm) - inst.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { - switch entry.Message { - case logEpochFromGenesis, logEpochFromSealingTip, logEpochFromSealingStorage: - msg := entry.Message - lastEpochBranch.Store(&msg) - } - return nil - }) - net.register(nodeID, inst) - require.NoError(t, inst.Start(t.Context())) - return inst - } - - // Pause block production before the node even starts: only protocol blocks (the - // zero block, the epoch transition and its sealing block) get built, and the - // chain stops at the sealing block since no ordinary block can be built on top. - vm.pause() - - // --- Case 1: cold boot, ledger holds only the genesis block. --- - inst := start() - require.Equal(t, logEpochFromGenesis, *lastEpochBranch.Load()) - - // --- Case 2: restart when the tip is a sealing block. --- - // Change the validator's weight to seal the first epoch. - // countSealingBlocks == 2: the zero block plus that epoch's sealing block. With the VM - // paused, the sealing block stays the tip because no ordinary block can be built on top. - pChain.advanceTo(epochChangePChainHeight) - waitForSealingBlockCount(t, storage, 2) - requireTipIsSealing(t, storage, true) - - inst.Stop() - inst = start() - require.Equal(t, logEpochFromSealingTip, *lastEpochBranch.Load()) - - // --- Case 3: restart mid-epoch, tip is an ordinary Simplex block. --- - // Resume production; the node extends the new epoch with ordinary blocks. - vm.resume() - waitForNumBlocks(t, storage, storage.NumBlocks()+3) - requireTipIsSealing(t, storage, false) - - inst.Stop() - inst = start() - t.Cleanup(inst.Stop) - require.Equal(t, logEpochFromSealingStorage, *lastEpochBranch.Load()) - - // The restarted node keeps extending the chain. - waitForNumBlocks(t, storage, storage.NumBlocks()+2) -} - -// TestInstanceZeroBlockUsesLastNonSimplexPChainHeight asserts that the first ever Simplex block -// references the P-chain height of the last non-Simplex block. -func TestInstanceZeroBlockUsesLastNonSimplexPChainHeight(t *testing.T) { - t.Skip("skipping until test instance refactor") - const basePChainHeight = uint64(7) - - var id [20]byte - rand.Read(id[:]) - nodeID := common.NodeID(id[:]) - - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - inst := newInstance(t, nodeID, storage, net, pChain, cops, genesisBlock) - net.register(nodeID, inst) - require.NoError(t, inst.Start(t.Context())) - t.Cleanup(inst.Stop) - - waitForNumBlocks(t, storage, 2) // genesis(0) + the zero block(1) - - zeroBlock, ok := storage.blockAt(1) - require.True(t, ok) - require.Equal(t, metadata.BlockTypeZero, zeroBlock.Type()) - require.Equal(t, basePChainHeight, zeroBlock.Metadata.PChainHeight) - require.Equal(t, basePChainHeight, zeroBlock.Metadata.SimplexEpochInfo.PChainReferenceHeight) -} - -func TestInstanceDoubleStartFails(t *testing.T) { - const basePChainHeight = uint64(1) - - var id [20]byte - rand.Read(id[:]) - nodeID := common.NodeID(id[:]) - - // Single-validator set including this node, so Start brings up a validator epoch. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - inst := newInstance(t, nodeID, storage, net, pChain, cops, genesisBlock) - - require.NoError(t, inst.Start(t.Context())) - t.Cleanup(inst.Stop) - - require.ErrorIs(t, inst.Start(t.Context()), errAlreadyStarted) -} - -func TestNonValidatorSkipsMSMVerification(t *testing.T) { - t.Skip("skipping until test instance refactor") - - // This test proves that a non-validator doesn't use the MSM to verify blocks. - // It does so by forcing a non-validator ti commit a block whose MSM state machine - // transition is invalid. - - const basePChainHeight = uint64(1) - - var id [20]byte - rand.Read(id[:]) - validatorNodeID := common.NodeID(id[:]) - - // The node under test. It is absent from the validator set, so it comes up as a non-validator. - var nv [20]byte - rand.Read(nv[:]) - nonValidatorNodeID := common.NodeID(nv[:]) - - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // The lone validator builds a chain on its own and then shuts down, so that from here on the - // only source of blocks is this test. - storage := newStorageWithGenesis(t, genesisBlock) - validatorInstance := newInstance(t, validatorNodeID, storage, net, pChain, cops, genesisBlock) - net.register(validatorNodeID, validatorInstance) - require.NoError(t, validatorInstance.Start(t.Context())) - t.Cleanup(validatorInstance.Stop) - - waitForNumBlocks(t, storage, 7) - validatorInstance.Stop() - require.True(t, validatorInstance.isStopped()) - - replicatedSeq := storage.NumBlocks() - 1 - replicated, ok := storage.blockAt(replicatedSeq) - require.True(t, ok) - parent, ok := storage.blockAt(replicatedSeq - 1) - require.True(t, ok) - - // The non-validator holds the chain up to, but not including, that last block, and is wired to - // a network of its own where nobody answers: the replication response we hand it below is the - // only way it can ever learn about the block. - nonValidatorStorage := storage.cloneBelow(replicatedSeq) - require.Equal(t, replicatedSeq, nonValidatorStorage.NumBlocks()) - _, ok = nonValidatorStorage.blockAt(replicatedSeq) - require.False(t, ok, "the non-validator already has the block it is meant to replicate") - - nonValidatorInstance := newInstance(t, nonValidatorNodeID, nonValidatorStorage, newInMemNetwork(t), pChain, cops, genesisBlock) - require.NoError(t, nonValidatorInstance.Start(t.Context())) - t.Cleanup(nonValidatorInstance.Stop) - - // The MSM that built the chain - the validator has stopped, so nothing else uses it - accepts - // the block, so we know the block is valid. - msm := validatorInstance.msm - - // The block we feed the non-validator is that same block with a single defect: a timestamp - // that precedes its parent's. Everything else - round, sequence, epoch info, P-chain height, - // inner block - is left alone, so the only thing wrong with it is its state machine - // transition. - tampered := replicated.Clone() - tampered.Metadata.Timestamp = parent.Metadata.Timestamp - 1 - - // Wire the MSM that built the chain to the tampered block, so we can prove that the MSM would have rejected it. - tamperedBlock := &ParsedBlock{StateMachineBlock: tampered.Clone(), msm: msm} - _, err := tamperedBlock.Verify(context.Background()) - require.ErrorContains(t, err, "proposed timestamp is before parent block's timestamp") - - // Changing the timestamp made it a block the validator never built: no sequence of its ledger - // holds it. - for seq := uint64(0); seq < storage.NumBlocks(); seq++ { - stored, ok := storage.blockAt(seq) - require.True(t, ok) - require.NotEqual(t, tampered.Digest(), stored.Digest(), "the validator has the tampered block at seq %d", seq) - } - - // Send precisely that block: the finalization we hand over is a quorum on its digest, not on - // the digest of the block the validator built. - block := &ParsedBlock{StateMachineBlock: tampered.Clone()} - finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: 1}, block, []common.NodeID{validatorNodeID}) - require.Equal(t, common.Digest(tampered.Digest()), finalization.Finalization.Digest) - require.NotEqual(t, common.Digest(replicated.Digest()), finalization.Finalization.Digest) - - require.NoError(t, nonValidatorInstance.HandleMessage(&common.Message{ - ReplicationResponse: &common.ReplicationResponse{ - Data: []common.QuorumRound{{Block: block, Finalization: &finalization}}, - }, - }, validatorNodeID)) - - // It commits the block its state machine would have rejected... - waitForNumBlocks(t, nonValidatorStorage, replicatedSeq+1) - committed, ok := nonValidatorStorage.blockAt(replicatedSeq) - require.True(t, ok) - require.Equal(t, tampered.Digest(), committed.Digest()) - - // ... so the two ledgers are the same height but disagree on their last block: the - // non-validator committed a block that exists nowhere in the validator's storage. - require.Equal(t, storage.NumBlocks(), nonValidatorStorage.NumBlocks()) - require.NotEqual(t, replicated.Digest(), committed.Digest()) -} - -func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { - t.Skip("skipping until test instance refactor") - - // This test ensures that validators that are lagging behind do not use the MSM - // to verify blocks they replicate through the replication path, as they have a QC. - // We check once for a notarized block and once for a finalized block. - - for _, tt := range []struct { - name string - // quorumRound wraps the replicated block with a QC. - quorumRound func(t *testing.T, logger common.Logger, block *ParsedBlock, signers []common.NodeID) common.QuorumRound - // requireReplicated asserts the lagging node replicated the block. - requireReplicated func(t *testing.T, storage *MockStorage, block metadata.StateMachineBlock) - }{ - { - name: "notarization", - quorumRound: func(t *testing.T, logger common.Logger, block *ParsedBlock, signers []common.NodeID) common.QuorumRound { - notarization, err := testutil.NewNotarization(logger, &testutil.TestSignatureAggregator{N: len(signers)}, block, signers) - require.NoError(t, err) - return common.QuorumRound{Block: block, Notarization: ¬arization} - }, - // A notarized block is not committed but notarized: the node persists the - // notarization to its WAL, which it only reaches after the block verified. - requireReplicated: func(t *testing.T, storage *MockStorage, block metadata.StateMachineBlock) { - round := block.Metadata.SimplexProtocolMetadata.Round - require.Eventually(t, func() bool { - return storage.containsNotarization(round) - }, 20*time.Second, 100*time.Millisecond, "no notarization for round %d was persisted to the WAL", round) - require.Equal(t, block.Metadata.SimplexProtocolMetadata.Seq, storage.NumBlocks(), "a notarized block should not have been committed") - }, - }, - { - name: "finalization", - quorumRound: func(t *testing.T, _ common.Logger, block *ParsedBlock, signers []common.NodeID) common.QuorumRound { - finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(signers)}, block, signers) - return common.QuorumRound{Block: block, Finalization: &finalization} - }, - // A finalized block is committed. - requireReplicated: func(t *testing.T, storage *MockStorage, block metadata.StateMachineBlock) { - seq := block.Metadata.SimplexProtocolMetadata.Seq - waitForNumBlocks(t, storage, seq+1) - committed, ok := storage.blockAt(seq) - require.True(t, ok) - require.Equal(t, block.Digest(), committed.Digest()) - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - const basePChainHeight = uint64(1) - - var first, second [20]byte - rand.Read(first[:]) - rand.Read(second[:]) - firstNodeID := common.NodeID(first[:]) - secondNodeID := common.NodeID(second[:]) - - // Two validators, so a quorum is both of them: once one of them is down, the other - // cannot commit a block, nor even empty notarize a round, on its own. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: first, BLSKey: []byte{0xaa}, Weight: 1}, - {NodeID: second, BLSKey: []byte{0xbb}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // The two validators build a chain together and then both shut down, so that from - // here on the only source of blocks is this test. - storage := newStorageWithGenesis(t, genesisBlock) - storage2 := newStorageWithGenesis(t, genesisBlock) - firstInstance := newInstance(t, firstNodeID, storage, net, pChain, cops, genesisBlock) - secondInstance := newInstance(t, secondNodeID, storage2, net, pChain, cops, genesisBlock) - net.register(firstNodeID, firstInstance) - net.register(secondNodeID, secondInstance) - require.NoError(t, firstInstance.Start(t.Context())) - require.NoError(t, secondInstance.Start(t.Context())) - t.Cleanup(firstInstance.Stop) - t.Cleanup(secondInstance.Stop) - - waitForNumBlocks(t, storage, 7) - firstInstance.Stop() - secondInstance.Stop() - require.True(t, firstInstance.isStopped()) - require.True(t, secondInstance.isStopped()) - - replicatedSeq := storage.NumBlocks() - 1 - replicated, ok := storage.blockAt(replicatedSeq) - require.True(t, ok) - parent, ok := storage.blockAt(replicatedSeq - 1) - require.True(t, ok) - - // The node restored at the parent below sits at the round following the parent's, and - // only processes a replicated round it has reached, so the block it is missing must be - // the one that directly follows its parent's round. - require.Equal(t, parent.Metadata.SimplexProtocolMetadata.Round+1, replicated.Metadata.SimplexProtocolMetadata.Round, - "the chain grew an empty round before its last block") - - // The MSM that built the chain - the validator has stopped, so nothing else uses it - - // accepts the block, so we know the block is valid. - msm := firstInstance.msm - - // The block we hand the node is that same block with a single defect: a timestamp preceding its parent's. - // Everything else - round, sequence, epoch info, P-chain height, inner block - is left alone, - // so the only thing wrong with it is its state machine transition, which is what the state - // machine rejects and what verifying only the inner block - what a node replicating a - // block does - accepts. - tampered := replicated.Clone() - tampered.Metadata.Timestamp = parent.Metadata.Timestamp - 1 - - // The MSM rejects the tampered block, so we know the block is invalid. - tamperedBlock := &ParsedBlock{StateMachineBlock: tampered.Clone(), msm: msm} - _, err := tamperedBlock.Verify(context.Background()) - require.ErrorContains(t, err, "proposed timestamp is before parent block's timestamp") - _, err = tamperedBlock.Verify(context.Background(), common.OnlyVMVerifyOpt) - require.NoError(t, err) - - // Changing the timestamp made it a block neither validator ever built: no sequence of - // either ledger holds it. - for seq := uint64(0); seq < storage.NumBlocks(); seq++ { - for _, ledger := range []*MockStorage{storage, storage2} { - stored, ok := ledger.blockAt(seq) - require.True(t, ok) - require.NotEqual(t, tampered.Digest(), stored.Digest(), "a validator has the tampered block at seq %d", seq) - } - } - - // The first validator comes back up lagging the chain by that block, with a paused VM - // and on a network of its own where nobody answers. It can therefore neither build - // the block nor replicate it legitimately, and since its peer is down it cannot reach - // a quorum to empty notarize either: it sits at exactly the round of the block it is - // missing until we hand it one. - laggingStorage := storage.cloneBelow(replicatedSeq) - require.Equal(t, replicatedSeq, laggingStorage.NumBlocks()) - _, ok = laggingStorage.blockAt(replicatedSeq) - require.False(t, ok, "the lagging validator already has the block it is meant to replicate") - - vm := newTestVM() - vm.pause() - laggingInstance := newInstanceWithVM(t, firstNodeID, laggingStorage, newInMemNetwork(t), pChain, cops, genesisBlock, vm) - require.NoError(t, laggingInstance.Start(t.Context())) - t.Cleanup(laggingInstance.Stop) - - // Send precisely that block: the quorum certificate we hand over is on its digest, not - // on the digest of the block the validators built. - block := &ParsedBlock{StateMachineBlock: tampered.Clone()} - quorumRound := tt.quorumRound(t, laggingInstance.Config.Logger, block, []common.NodeID{firstNodeID, secondNodeID}) - require.Equal(t, common.Digest(tampered.Digest()), quorumRound.Block.BlockHeader().Digest) - require.NotEqual(t, common.Digest(replicated.Digest()), quorumRound.Block.BlockHeader().Digest) - - require.NoError(t, laggingInstance.HandleMessage(&common.Message{ - ReplicationResponse: &common.ReplicationResponse{Data: []common.QuorumRound{quorumRound}}, - }, secondNodeID)) - - tt.requireReplicated(t, laggingStorage, tampered) - }) - } -} - // requireTipIsSealing asserts whether the last block in storage is a sealing block. func requireTipIsSealing(t *testing.T, storage *MockStorage, want bool) { t.Helper() From 5372de51c1133709f0b7873813f92a35fd237782 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 16:55:09 -0400 Subject: [PATCH 03/24] Delete the test helpers the rewrite does not use The per-test instance constructors, the storage polling helpers, testVM, the inMemNetwork message router and its networkSender all go away with the tests that used them. Trims the imports that went unused with them. --- instance_test.go | 404 ----------------------------------------------- 1 file changed, 404 deletions(-) diff --git a/instance_test.go b/instance_test.go index bf2497eb..3584d5ac 100644 --- a/instance_test.go +++ b/instance_test.go @@ -4,164 +4,25 @@ package simplex import ( - "bytes" "context" - "crypto/rand" "crypto/sha256" "encoding/asn1" "encoding/binary" "fmt" "sort" - "strings" "sync" - "sync/atomic" "testing" "time" "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" - "github.com/ava-labs/simplex/simplex" "github.com/ava-labs/simplex/testutil" "github.com/ava-labs/simplex/wal" "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" ) -// requireTipIsSealing asserts whether the last block in storage is a sealing block. -func requireTipIsSealing(t *testing.T, storage *MockStorage, want bool) { - t.Helper() - num := storage.NumBlocks() - require.Positive(t, num) - block, ok := storage.blockAt(num - 1) - require.True(t, ok) - require.Equal(t, want, block.SealingBlockInfo() != nil) -} - -// countSealingBlocks returns the number of sealing blocks (blocks carrying a -// BlockValidationDescriptor) currently in storage. -func countSealingBlocks(t *testing.T, storage *MockStorage) int { - t.Helper() - count := 0 - num := storage.NumBlocks() - for seq := uint64(0); seq < num; seq++ { - block, ok := storage.blockAt(seq) - if !ok { - continue - } - if block.SealingBlockInfo() != nil { - count++ - } - } - return count -} - -// waitForSealingBlockCount waits until storage holds at least target sealing blocks. -func waitForSealingBlockCount(t *testing.T, storage *MockStorage, target int) { - t.Helper() - require.Eventually(t, func() bool { - return countSealingBlocks(t, storage) >= target - }, 20*time.Second, 100*time.Millisecond) -} - -// newStorageWithGenesis returns storage holding only the genesis block, the ledger every node -// here starts from. -func newStorageWithGenesis(t *testing.T, genesisBlock *testInnerBlock) *MockStorage { - t.Helper() - storage := NewMockStorage(t) - genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} - require.NoError(t, storage.Index(context.Background(), genesis, common.Finalization{})) - return storage -} - -// newInstance builds an Instance sharing the common test dependencies but with its own ID, -// storage and VM. -func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { - return newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, newTestVM()) -} - -// newInstanceWithVM is like newInstance but uses a caller-supplied VM, so a test -// can share one controllable VM across restarts of the same node. -func newInstanceWithVM(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock, vm *testVM) *Instance { - comm := &networkSender{net: net, self: nodeID} - config := Config{ - Logger: testutil.MakeLogger(t, int(nodeID[0])), - ID: nodeID, - VM: vm, - Storage: storage, - ICMETransition: vm.ComputeICMEpoch, - Sender: comm, - Broadcaster: comm, - PlatformChain: pChain, - CryptoOps: cops, - LastNonSimplexInnerBlock: genesisBlock, - WalCreator: storage.CreateWAL, - ParameterConfig: ParameterConfig{ - MaxNetworkDelay: 500 * time.Millisecond, - MaxRoundWindow: 100, - WALMaxSizeBytes: 1024, - }, - } - return NewInstance(config) -} - -func latestValidatorID(t *testing.T, storage *MockStorage) common.NodeID { - t.Helper() - num := storage.NumBlocks() - // Iterate backwards and find the latest sealing block (a block with a block validation descriptor) - for seq := int64(num) - 1; seq >= 0; seq-- { - block, ok := storage.blockAt(uint64(seq)) - if !ok { - continue - } - if info := block.SealingBlockInfo(); info != nil { - return info.ValidatorSet[len(info.ValidatorSet)-1].Id - } - } - t.Fatalf("no block with a BlockValidationDescriptor found in storage") - return nil -} - -// waitForNumBlocks waits until the given storage has at least targetHeight blocks. -func waitForNumBlocks(t *testing.T, storage *MockStorage, targetHeight uint64) { - t.Helper() - require.Eventually(t, func() bool { - return storage.NumBlocks() >= targetHeight - }, 20*time.Second, 100*time.Millisecond, "storage did not commit %d blocks in time", targetHeight) -} - -// waitForSealingBlock waits until a sealing block (a block carrying a BlockValidationDescriptor with the new weight) -// is committed at or after fromSeq. It periodically injects approvals into the given instance. -// Returns the seq of the sealing block. -func waitForSealingBlock(t *testing.T, inst *Instance, approval *common.ValidatorSetApproval, fromSeq uint64) uint64 { - t.Helper() - var result uint64 - storage := inst.Config.Storage.(*MockStorage) - require.Eventually(t, func() bool { - inst.lock.Lock() - msm := inst.msm - inst.lock.Unlock() - if msm != nil { - msm.HandleApproval(approval, 1) - } - - num := storage.NumBlocks() - for seq := fromSeq; seq < num; seq++ { - block, ok := storage.blockAt(seq) - if !ok { - continue - } - if block.SealingBlockInfo() != nil { - result = seq - return true - } - } - return false - }, 20*time.Second, 100*time.Millisecond) - return result -} - type testInnerBlock struct { Height_ uint64 TS time.Time @@ -193,73 +54,6 @@ func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { return b, nil } -type testVM struct { - nextHeight atomic.Uint64 - // When paused, the VM behaves as a chain with no pending transactions: - // WaitForPendingBlock and BuildBlock block until their context expires, so the - // epoch stops producing ordinary blocks. The epoch-transition and sealing - // machinery, which builds its block once the inner build times out, still runs — - // so pausing before an epoch change leaves the sealing block at the tip with - // nothing built on top. Lets a test pin the chain tip without touching storage. - paused atomic.Bool -} - -func newTestVM() *testVM { - vm := &testVM{} - vm.nextHeight.Store(1) // the genesis inner block is height 0 - return vm -} - -func (vm *testVM) pause() { vm.paused.Store(true) } -func (vm *testVM) resume() { vm.paused.Store(false) } - -func (vm *testVM) BuildBlock(ctx context.Context, _ uint64) (avalanchego.VMBlock, error) { - if vm.paused.Load() { - <-ctx.Done() // let the caller's impatient build time out - return nil, ctx.Err() - } - h := vm.nextHeight.Add(1) - 1 - payload := make([]byte, 8) - binary.BigEndian.PutUint64(payload, h) - return &testInnerBlock{Height_: h, TS: time.Now(), Payload: payload}, nil -} - -func (vm *testVM) WaitForPendingBlock(ctx context.Context) { - if vm.paused.Load() { - <-ctx.Done() // no pending block while paused - return - } - select { - case <-ctx.Done(): - case <-time.After(100 * time.Millisecond): - } -} - -func (vm *testVM) ParseBlock(_ context.Context, b []byte) (avalanchego.VMBlock, error) { - return parseTestInnerBlock(b) -} - -func (vm *testVM) ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo { - // ACP-181-style transition (mirrors the msm test helper). - var zero metadata.ICMEpochInfo - if input.ParentEpoch == zero { - return metadata.ICMEpochInfo{ - PChainEpochHeight: input.ParentPChainHeight, - EpochNumber: 1, - EpochStartTime: uint64(input.ParentTimestamp.Unix()), - } - } - endTime := time.Unix(int64(input.ParentEpoch.EpochStartTime), 0).Add(time.Second) - if input.ParentTimestamp.Before(endTime) { - return input.ParentEpoch - } - return metadata.ICMEpochInfo{ - PChainEpochHeight: input.ParentPChainHeight, - EpochNumber: input.ParentEpoch.EpochNumber + 1, - EpochStartTime: uint64(input.ParentTimestamp.Unix()), - } -} - type testPlatformChain struct { baseHeight uint64 validatorSetAtHeight map[uint64]metadata.NodeBLSMappings // height --> validator set @@ -465,22 +259,6 @@ func (m *MockStorage) CreateWAL() (wal.DeletableWAL, error) { return w, nil } -// cloneBelow returns a storage holding every block of m below seq - the block at seq itself, and -// anything after it, is left out - so that a test can bring up a node that lags the chain by them. -func (m *MockStorage) cloneBelow(seq uint64) *MockStorage { - clone := NewMockStorage(m.t) - for cloned := uint64(0); cloned < seq; cloned++ { - m.snapLock.Lock() - stored, ok := m.blocks[cloned] - m.snapLock.Unlock() - require.True(m.t, ok) - - block := &ParsedBlock{StateMachineBlock: m.parseStored(stored.rawBlock)} - require.NoError(m.t, clone.Index(context.Background(), block, stored.fin)) - } - return clone -} - // containsNotarization reports whether any WAL this storage handed out holds a notarization for // the given round. func (m *MockStorage) containsNotarization(round uint64) bool { @@ -496,125 +274,6 @@ func (m *MockStorage) containsNotarization(round uint64) bool { return false } -// --------------------------------------------------------------------------- -// inMemNetwork: Routes messages between Instances. -// Delivery happens on a per-node goroutine rather than inline in Send, -// due to locking. -// --------------------------------------------------------------------------- - -type netMsg struct { - from common.NodeID - msg *common.Message -} - -type netNode struct { - inst *Instance - // in is a buffered inbox drained by the delivery goroutine. The channel itself - // signals that work is available, so no separate wake signal is needed. Sends - // never block (see enqueue); on the rare chance the buffer fills, a dropped - // message costs at most an empty round the epoch recovers from. - in chan netMsg - done chan struct{} - stopped chan struct{} -} - -type inMemNetwork struct { - t *testing.T - lock sync.Mutex - nodes map[string]*netNode -} - -func newInMemNetwork(t *testing.T) *inMemNetwork { - return &inMemNetwork{t: t, nodes: make(map[string]*netNode)} -} - -// register wires inst into the network and starts delivering messages to it. -// Messages that arrive before the epoch exists are dropped by the instance's -// nil-epoch guard, which at worst costs a few empty rounds the epoch recovers from. -func (n *inMemNetwork) register(id common.NodeID, inst *Instance) { - node := &netNode{ - inst: inst, - in: make(chan netMsg, 1024), - done: make(chan struct{}), - stopped: make(chan struct{}), - } - n.lock.Lock() - defer n.lock.Unlock() - - // If an instance was previously registered under this id (e.g. a restart replacing - // the node), stop its delivery goroutine before swapping in the new one. - old := n.nodes[string(id)] - if old != nil { - close(old.done) - <-old.stopped - } - - n.nodes[string(id)] = node - - go n.deliver(node) -} - -func (n *inMemNetwork) stop() { - n.lock.Lock() - nodes := make([]*netNode, 0, len(n.nodes)) - for _, node := range n.nodes { - nodes = append(nodes, node) - } - n.nodes = make(map[string]*netNode) - n.lock.Unlock() - for _, node := range nodes { - close(node.done) - <-node.stopped - } -} - -// registeredIDs returns the nodes currently wired into the network. Broadcasters go through it -// rather than reading the map directly, since nodes register while others are already running. -func (n *inMemNetwork) registeredIDs() []common.NodeID { - n.lock.Lock() - defer n.lock.Unlock() - ids := make([]common.NodeID, 0, len(n.nodes)) - for _, node := range n.nodes { - ids = append(ids, node.inst.Config.ID) - } - return ids -} - -func (n *inMemNetwork) enqueue(dest common.NodeID, m netMsg) { - n.lock.Lock() - node := n.nodes[string(dest)] - n.lock.Unlock() - if node == nil { - // Destination not registered; drop. This only happens before an instance - // is registered, never mid-run. - return - } - select { - case node.in <- m: - default: - // Never block the sender (Send runs under the epoch lock). A dropped message - // costs at most an empty round the epoch recovers from. - } -} - -func (n *inMemNetwork) deliver(node *netNode) { - defer close(node.stopped) - for { - select { - case <-node.done: - return - case m := <-node.in: - n.dispatch(node.inst, m) - } - } -} - -func (n *inMemNetwork) dispatch(inst *Instance, m netMsg) { - if err := inst.HandleMessage(m.msg, m.from); err != nil { - n.t.Logf("HandleMessage from %x failed: %v", m.from, err) - } -} - // toRawBlock re-encodes a verified block into the wire RawBlock the receiving // instance parses in HandleBlockMessage. func toRawBlock(t *testing.T, vb common.VerifiedBlock) *metadata.RawBlock { @@ -640,69 +299,6 @@ func reparseBlock(t *testing.T, vb common.VerifiedBlock) *ParsedBlock { } } -type networkSender struct { - net *inMemNetwork - self common.NodeID -} - -func (s *networkSender) Broadcast(msg *common.Message) { - for _, dest := range s.net.registeredIDs() { - s.Send(msg, dest) - } -} - -func (s *networkSender) Send(msg *common.Message, dest common.NodeID) { - if bytes.Equal(s.self, dest) { - // Do not send to myself - return - } - m := s.createIngressMessage(msg) - s.net.enqueue(dest, m) -} - -// CreateIngressMessage translates a message into the form the receiving instance expects on the wire. -// For example, a VerifiedBlockMessage is re-encoded as a BlockMessage with a RawBlock. -// A VerifiedReplicationResponse is re-encoded as a ReplicationResponse with independent copies of the carried blocks. -func (s *networkSender) createIngressMessage(msg *common.Message) netMsg { - m := netMsg{from: s.self} - switch { - case msg.VerifiedBlockMessage != nil: - m.msg = &common.Message{ - BlockMessage: &common.BlockMessage{ - Vote: msg.VerifiedBlockMessage.Vote, - Block: reparseBlock(s.net.t, msg.VerifiedBlockMessage.VerifiedBlock), - }, - } - case msg.VerifiedReplicationResponse != nil: - m.msg = &common.Message{ReplicationResponse: toReplicationResponse(s.net.t, msg.VerifiedReplicationResponse)} - default: - m.msg = msg - } - return m -} - -// toReplicationResponse translates a VerifiedReplicationResponse (the sender's -// internal form) into the ReplicationResponse a receiver handles on the wire, -// mirroring testutil.TestComm. Each carried block is reconstructed as an -// independent copy so the delivery goroutine never touches the sender's live -// block object (whose canoto digest cache the sender keeps mutating). -func toReplicationResponse(t *testing.T, vrr *common.VerifiedReplicationResponse) *common.ReplicationResponse { - data := make([]common.QuorumRound, 0, len(vrr.Data)) - for _, vqr := range vrr.Data { - data = append(data, verifiedQuorumRoundToQuorumRound(t, vqr)) - } - resp := &common.ReplicationResponse{Data: data} - if vrr.LatestRound != nil { - qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestRound) - resp.LatestRound = &qr - } - if vrr.LatestFinalizedSeq != nil { - qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestFinalizedSeq) - resp.LatestSeq = &qr - } - return resp -} - func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRound) common.QuorumRound { qr := common.QuorumRound{ Notarization: vqr.Notarization, From ea08f41d947c06041feb8328a9abf89347ba575c Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 16:55:14 -0400 Subject: [PATCH 04/24] Rename instance_test.go to instance_helpers_test.go The file now holds only helpers. The new instance tests land in a fresh instance_test.go later in the series. --- instance_test.go => instance_helpers_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename instance_test.go => instance_helpers_test.go (100%) diff --git a/instance_test.go b/instance_helpers_test.go similarity index 100% rename from instance_test.go rename to instance_helpers_test.go From f8e9a0359376464cd67ad855a52718cc6b22cc72 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 16:59:54 -0400 Subject: [PATCH 05/24] Update the shared instance test helpers MockStorage parses stored blocks through an injected testInnerBlockDeserializer instead of the package level parseTestInnerBlock, and hands its WAL bookkeeping to a standalone walCreator. testPlatformChain wakes WaitForProgress waiters over a channel rather than a sync.Cond, looks up validator sets by exact height, and lets a test install a set at a height. reparseBlock inlines toRawBlock. --- instance_helpers_test.go | 210 ++++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 101 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 3584d5ac..28a46f72 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -9,7 +9,6 @@ import ( "encoding/asn1" "encoding/binary" "fmt" - "sort" "sync" "testing" "time" @@ -19,7 +18,6 @@ import ( metadata "github.com/ava-labs/simplex/msm" "github.com/ava-labs/simplex/testutil" "github.com/ava-labs/simplex/wal" - "github.com/stretchr/testify/require" ) @@ -46,7 +44,9 @@ func (b *testInnerBlock) Height() uint64 { return b.Height func (b *testInnerBlock) Timestamp() time.Time { return b.TS } func (b *testInnerBlock) Verify(context.Context, uint64) error { return nil } -func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { +type testInnerBlockDeserializer struct{} + +func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte) (avalanchego.VMBlock, error) { b := &testInnerBlock{} b.Height_ = binary.BigEndian.Uint64(buff[0:8]) b.TS = time.UnixMilli(int64(binary.BigEndian.Uint64(buff[8:16]))) @@ -55,28 +55,30 @@ func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { } type testPlatformChain struct { - baseHeight uint64 - validatorSetAtHeight map[uint64]metadata.NodeBLSMappings // height --> validator set - lock sync.Mutex - cond *sync.Cond - height uint64 -} - -func newTestPlatformChain(baseHeight uint64, validatorSetsAtHeight map[uint64]metadata.NodeBLSMappings) *testPlatformChain { - pc := &testPlatformChain{ - baseHeight: baseHeight, - validatorSetAtHeight: validatorSetsAtHeight, - height: baseHeight, + genesisHeight uint64 // genesis height is the height of the pchain the genesis validator set lives + + // lock guards the sets, which the running instances read while a test installs new ones. + lock sync.Mutex + // validatorSetAtHeight maps a P-chain height to the validator set in force from it on. + validatorSetAtHeight map[uint64]metadata.NodeBLSMappings + + height uint64 + // heightChanged is closed and replaced on every advanceHeight, waking waiters + // so they re-check the height. + heightChanged chan struct{} +} + +// newTestPChain returns a P-chain holding only the genesis validator set, in force from +// genesisPChainHeight on. +func newTestPChain(genesisSet metadata.NodeBLSMappings) *testPlatformChain { + return &testPlatformChain{ + genesisHeight: genesisPChainHeight, + validatorSetAtHeight: map[uint64]metadata.NodeBLSMappings{ + genesisPChainHeight: genesisSet, + }, + height: genesisPChainHeight, + heightChanged: make(chan struct{}), } - pc.cond = sync.NewCond(&pc.lock) - return pc -} - -func (pc *testPlatformChain) advanceTo(h uint64) { - pc.lock.Lock() - defer pc.lock.Unlock() - pc.height = h - pc.cond.Broadcast() // wake any WaitForProgress waiters } func (pc *testPlatformChain) currentHeight() uint64 { @@ -85,32 +87,22 @@ func (pc *testPlatformChain) currentHeight() uint64 { return pc.height } -func (pc *testPlatformChain) validatorSet(height uint64) metadata.NodeBLSMappings { - heights := make([]uint64, 0, len(pc.validatorSetAtHeight)) - for h := range pc.validatorSetAtHeight { - heights = append(heights, h) - } - sort.Slice(heights, func(i, j int) bool { return heights[i] < heights[j] }) +func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { + pc.lock.Lock() + defer pc.lock.Unlock() - var lastCheckpoint uint64 - for _, h := range heights { - if h > height { - break - } - lastCheckpoint = h + set, ok := pc.validatorSetAtHeight[height] + if !ok { + return nil, fmt.Errorf("no validator set at %d", height) } - // Return a copy instead of the original slice so the reference won't be used in other goroutines concurrently. - // Since we allocate a nil slice, a new underlying array is allocated and the copy is safe to use concurrently. - src := pc.validatorSetAtHeight[lastCheckpoint] - return append(metadata.NodeBLSMappings(nil), src...) -} - -func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { - return pc.validatorSet(height), nil + return set, nil } func (pc *testPlatformChain) GenesisValidatorSet() metadata.NodeBLSMappings { - return pc.validatorSet(pc.baseHeight) + pc.lock.Lock() + defer pc.lock.Unlock() + + return pc.validatorSetAtHeight[pc.genesisHeight] } func (pc *testPlatformChain) GetMinimumHeight() uint64 { @@ -121,32 +113,48 @@ func (pc *testPlatformChain) GetCurrentHeight() uint64 { return pc.currentHeight() } +// WaitForProgress blocks until the context is cancelled or the P-chain height +// has increased past pChainHeight. func (pc *testPlatformChain) WaitForProgress(ctx context.Context, pChainHeight uint64) error { - stop := pc.signalWhenContextFinished(ctx) - defer stop() + for { + pc.lock.Lock() + if pc.height > pChainHeight { + pc.lock.Unlock() + return nil + } + ch := pc.heightChanged + pc.lock.Unlock() - pc.lock.Lock() - defer pc.lock.Unlock() - for pc.height == pChainHeight { - if err := ctx.Err(); err != nil { - return err + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() } - pc.cond.Wait() } - return nil } -func (pc *testPlatformChain) signalWhenContextFinished(ctx context.Context) func() bool { - stop := context.AfterFunc(ctx, func() { - pc.lock.Lock() - defer pc.lock.Unlock() - pc.cond.Broadcast() - }) - return stop +func (pc *testPlatformChain) setValidatorSetAt(height uint64, validatorSet metadata.NodeBLSMappings) { + pc.lock.Lock() + defer pc.lock.Unlock() + + pc.validatorSetAtHeight[height] = validatorSet +} + +// advanceHeight bumps the P-chain height and wakes every WaitForProgress waiter. +func (pc *testPlatformChain) advanceHeight(height uint64) { + pc.lock.Lock() + defer pc.lock.Unlock() + + if height <= pc.height { + panic("smaller height") + } + pc.height = height + close(pc.heightChanged) + pc.heightChanged = make(chan struct{}) } func (pc *testPlatformChain) LastNonSimplexBlockPChainHeight() uint64 { - return pc.baseHeight + return pc.genesisHeight } type testCryptoOps struct{} @@ -184,10 +192,10 @@ func (c *testCryptoOps) DeserializeQuorumCertificate(bytes []byte) (common.Quoru type MockStorage struct { t *testing.T *testutil.InMemStorage + bd *testInnerBlockDeserializer - snapLock sync.Mutex - blocks map[uint64]storedBlock - wals []*testutil.TestWAL + blocksLock sync.Mutex + blocks map[uint64]storedBlock } type storedBlock struct { @@ -195,11 +203,12 @@ type storedBlock struct { fin common.Finalization } -func NewMockStorage(t *testing.T) *MockStorage { +func NewMockStorage(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { return &MockStorage{ t: t, InMemStorage: testutil.NewInMemStorage(), blocks: make(map[uint64]storedBlock), + bd: bd, } } @@ -207,9 +216,9 @@ func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, cer // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. encoded := block.Bytes() seq := m.NumBlocks() - m.snapLock.Lock() + m.blocksLock.Lock() m.blocks[seq] = storedBlock{rawBlock: encoded, fin: certificate} - m.snapLock.Unlock() + m.blocksLock.Unlock() return m.InMemStorage.Index(ctx, block, certificate) } @@ -230,9 +239,9 @@ func (m *MockStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common. // the instance's live block objects (whose canoto digest cache the instance keeps // mutating). func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { - m.snapLock.Lock() + m.blocksLock.Lock() sb, ok := m.blocks[seq] - m.snapLock.Unlock() + m.blocksLock.Unlock() if !ok { return metadata.StateMachineBlock{}, false } @@ -244,58 +253,57 @@ func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { require.NoError(m.t, raw.UnmarshalCanoto(encoded)) var inner avalanchego.VMBlock if len(raw.InnerBlockBytes) > 0 { - parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) + parsed, err := m.bd.ParseBlock(context.Background(), raw.InnerBlockBytes) require.NoError(m.t, err) inner = parsed } return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} } -func (m *MockStorage) CreateWAL() (wal.DeletableWAL, error) { - w := testutil.NewTestWAL(m.t) - m.snapLock.Lock() - m.wals = append(m.wals, w) - m.snapLock.Unlock() - return w, nil +type walCreator struct { + t *testing.T + + lock sync.Mutex + wals []*testutil.TestWAL +} + +func (w *walCreator) createWAL() (wal.DeletableWAL, error) { + tw := testutil.NewTestWAL(w.t) + w.lock.Lock() + w.wals = append(w.wals, tw) + w.lock.Unlock() + return tw, nil } -// containsNotarization reports whether any WAL this storage handed out holds a notarization for -// the given round. -func (m *MockStorage) containsNotarization(round uint64) bool { - m.snapLock.Lock() - wals := append([]*testutil.TestWAL(nil), m.wals...) - m.snapLock.Unlock() +// containsNotarization reports whether any WAL this creator handed out holds a +// notarization for the given round. +func (w *walCreator) containsNotarization(round uint64) bool { + w.lock.Lock() + defer w.lock.Unlock() - for _, w := range wals { - if w.ContainsNotarization(round) { + for _, tw := range w.wals { + if tw.ContainsNotarization(round) { return true } } return false } -// toRawBlock re-encodes a verified block into the wire RawBlock the receiving -// instance parses in HandleBlockMessage. -func toRawBlock(t *testing.T, vb common.VerifiedBlock) *metadata.RawBlock { - bytes := vb.Bytes() - raw := &metadata.RawBlock{} - require.NoError(t, raw.UnmarshalCanoto(bytes)) - return raw -} - -// reparseBlock reconstructs an independent *ParsedBlock from a verified block's -// wire bytes. Each call yields a fresh object sharing no pointers with the -// sender's live block, so that remaining references to the sender's block don't race with the receiver. +// reparseBlock rebuilds an independent ParsedBlock from a verified block's wire bytes. +// The zero block has no inner block, so its inner bytes stay empty. func reparseBlock(t *testing.T, vb common.VerifiedBlock) *ParsedBlock { - raw := toRawBlock(t, vb) + var rawBlock metadata.RawBlock + require.NoError(t, rawBlock.UnmarshalCanoto(vb.Bytes())) + var inner avalanchego.VMBlock - if len(raw.InnerBlockBytes) > 0 { - parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) + if len(rawBlock.InnerBlockBytes) > 0 { + bd := &testInnerBlockDeserializer{} + parsed, err := bd.ParseBlock(context.Background(), rawBlock.InnerBlockBytes) require.NoError(t, err) inner = parsed } return &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata}, + StateMachineBlock: metadata.StateMachineBlock{InnerBlock: inner, Metadata: rawBlock.Metadata}, } } From 8fcc0895e6e3651b5de678429f1267a6cb7876dc Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 17:01:15 -0400 Subject: [PATCH 06/24] Add the network harness helpers A network drives a set of nodes over an instanceComm that hands each message straight to the destination instance, and steps the chain one block at a time through a blockBuilderVM gated by a controlled block builder. Nodes start from a storage seeded with genesis, or from newChainStorage when a test needs the epoch defining block at the tip. --- instance_helpers_test.go | 427 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 427 insertions(+) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 28a46f72..b677dcb2 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -4,6 +4,7 @@ package simplex import ( + "bytes" "context" "crypto/sha256" "encoding/asn1" @@ -16,9 +17,11 @@ import ( "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/simplex" "github.com/ava-labs/simplex/testutil" "github.com/ava-labs/simplex/wal" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type testInnerBlock struct { @@ -54,6 +57,15 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte return b, nil } +const genesisPChainHeight uint64 = 0 + +var genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} +var paramConfig = ParameterConfig{ + MaxNetworkDelay: 500 * time.Millisecond, + MaxRoundWindow: 100, + WALMaxSizeBytes: 1024, +} + type testPlatformChain struct { genesisHeight uint64 // genesis height is the height of the pchain the genesis validator set lives @@ -212,6 +224,19 @@ func NewMockStorage(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { } } +func NewMockStorageWithGenesis(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { + s := &MockStorage{ + t: t, + InMemStorage: testutil.NewInMemStorage(), + blocks: make(map[uint64]storedBlock), + bd: bd, + } + + genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} + require.NoError(t, s.Index(context.Background(), genesis, common.Finalization{})) + return s +} + func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. encoded := block.Bytes() @@ -260,6 +285,34 @@ func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} } +// newChainStorage builds and indexes the minimum chain a node can start from: genesis plus +// epoch 1's defining block, which carries the descriptor naming the epoch's validator set. +// It returns the storage and the epoch-defining block at its tip. +func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*MockStorage, metadata.StateMachineBlock) { + storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) + genesis, ok := storage.blockAt(0) + require.True(t, ok) + + epochBlock := metadata.StateMachineBlock{ + InnerBlock: &testInnerBlock{Height_: 1, TS: time.Now(), Payload: []byte("epoch")}, + Metadata: metadata.StateMachineMetadata{ + Timestamp: uint64(time.Now().UnixMilli()), + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 1, Seq: 1, Prev: common.Digest(genesis.Digest())}, + SimplexEpochInfo: metadata.SimplexEpochInfo{ + EpochNumber: 1, + BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: validators}, + }, + }, + }, + } + + block := &ParsedBlock{StateMachineBlock: epochBlock.Clone()} + finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(validators)}, block, validators.NodeIDs()) + require.NoError(t, storage.Index(context.Background(), block, finalization)) + return storage, epochBlock +} + type walCreator struct { t *testing.T @@ -318,3 +371,377 @@ func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRou } return qr } + +type instanceComm struct { + c *network + // id is the node this comm belongs to, reported as the sender of every message it sends. + id common.NodeID +} + +func newInstanceComm(c *network, id common.NodeID) *instanceComm { + return &instanceComm{c: c, id: id} +} + +func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { + // loop through all nodes in chain directly send to handle message directly via but do it in a separate go routine + for _, n := range c.c.nodesSnapshot() { + if !bytes.Equal(n.id, destination) { + continue + } + + go func(dst *Instance) { + require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", destination) + require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) + }(n.inst) + return + } +} + +func (c *instanceComm) Broadcast(msg *common.Message) { + // send to every node in the chain but ourselves, each on its own go routine + for _, n := range c.c.nodesSnapshot() { + if bytes.Equal(n.id, c.id) { + continue + } + + go func(dst *Instance) { + require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", n.id) + require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) + }(n.inst) + } +} + +// translateOutgoingToIncomingMessage converts the verified message types an instance +// sends into the wire types a receiver handles, like testutil.TestComm. Each carried +// block is re-parsed into a fresh ParsedBlock because HandleMessage mutates the block +// it receives, so recipients cannot share the sender's live block. +func translateOutgoingToIncomingMessage(t *testing.T, msg *common.Message) *common.Message { + switch { + case msg.VerifiedBlockMessage != nil: + return &common.Message{ + BlockMessage: &common.BlockMessage{ + Vote: msg.VerifiedBlockMessage.Vote, + Block: reparseBlock(t, msg.VerifiedBlockMessage.VerifiedBlock), + }, + } + case msg.VerifiedReplicationResponse != nil: + vrr := msg.VerifiedReplicationResponse + data := make([]common.QuorumRound, 0, len(vrr.Data)) + for _, vqr := range vrr.Data { + data = append(data, verifiedQuorumRoundToQuorumRound(t, vqr)) + } + resp := &common.ReplicationResponse{Data: data} + if vrr.LatestRound != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestRound) + resp.LatestRound = &qr + } + if vrr.LatestFinalizedSeq != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestFinalizedSeq) + resp.LatestSeq = &qr + } + return &common.Message{ReplicationResponse: resp} + default: + return msg + } +} + +// pendingBlockSignal broadcasts to every waiter by closing the current channel and +// replacing it with a fresh one for the next generation of waiters. +type pendingBlockSignal struct { + lock sync.Mutex + ch chan struct{} +} + +func newPendingBlockSignal() *pendingBlockSignal { + return &pendingBlockSignal{ch: make(chan struct{})} +} + +// wait returns when the signal is broadcast or ctx is cancelled. +func (s *pendingBlockSignal) wait(ctx context.Context) { + s.lock.Lock() + ch := s.ch + s.lock.Unlock() + + select { + case <-ch: + case <-ctx.Done(): + } +} + +// broadcast wakes every current waiter. +func (s *pendingBlockSignal) broadcast() { + s.lock.Lock() + close(s.ch) + s.ch = make(chan struct{}) + s.lock.Unlock() +} + +// blockBuilderVM builds an inner block only when the test triggers one on the block builder, so +// the chain grows one block per index call. +type blockBuilderVM struct { + bb *testutil.TestControlledBlockBuilder + storage *MockStorage + pending *pendingBlockSignal +} + +func newBlockBuilderVM(bb *testutil.TestControlledBlockBuilder, storage *MockStorage, pending *pendingBlockSignal) *blockBuilderVM { + return &blockBuilderVM{bb: bb, storage: storage, pending: pending} +} + +func (vm *blockBuilderVM) BuildBlock(ctx context.Context, pChainHeight uint64) (avalanchego.VMBlock, error) { + // The builder gates when a block is built; the block it returns is not an inner block, so + // it is thrown away. + if _, ok := vm.bb.BuildBlock(ctx, common.ProtocolMetadata{}, common.Blacklist{}); !ok { + return nil, ctx.Err() + } + + // the inner height is the seq of the block being built, which is how many blocks the node + // has committed so far + height := vm.storage.NumBlocks() + payload := make([]byte, 8) + binary.BigEndian.PutUint64(payload, height) + return &testInnerBlock{Height_: height, TS: time.Now(), Payload: payload}, nil +} + +func (vm *blockBuilderVM) ParseBlock(ctx context.Context, bytes []byte) (avalanchego.VMBlock, error) { + return vm.storage.bd.ParseBlock(ctx, bytes) +} + +// WaitForPendingBlock returns when index broadcasts that a block is being created, +// or when ctx is cancelled. +func (vm *blockBuilderVM) WaitForPendingBlock(ctx context.Context) { + vm.pending.wait(ctx) +} + +// noopICMTransition keeps every block in the same ICM epoch, so an epoch only ever changes +// because the validator set did. +func noopICMTransition(_ metadata.ICMEpochInput) metadata.ICMEpochInfo { + return metadata.ICMEpochInfo{} +} + +type node struct { + t *testing.T + id common.NodeID + vm *blockBuilderVM + inst *Instance + storage *MockStorage + comm *instanceComm + wals *walCreator +} + +type network struct { + t *testing.T + + pChain *testPlatformChain + seq uint64 + epoch uint64 + + // pending wakes every VM blocked in WaitForPendingBlock when index creates a block. + pending *pendingBlockSignal + + validatorSets map[uint64]common.Nodes // epoch -> sorted validators + + // lock guards nodes, which comm goroutines read while addNode appends. + lock sync.Mutex + nodes []node +} + +func (n *network) nodesSnapshot() []node { + n.lock.Lock() + defer n.lock.Unlock() + return append([]node(nil), n.nodes...) +} + +func newNetwork(t *testing.T, pChain *testPlatformChain) *network { + validatorSets := make(map[uint64]common.Nodes) + genesisNodes := pChain.GenesisValidatorSet().Nodes() + common.SortNodes(genesisNodes) + validatorSets[1] = genesisNodes + + return &network{ + t: t, + pChain: pChain, + pending: newPendingBlockSignal(), + validatorSets: validatorSets, + + // Genesis at seq 0. Then first simplex block is built automatically + // without a build block notification + seq: 2, + epoch: 1, + } +} + +// nodeConfig holds optional overrides for a node added to the network. +type nodeConfig struct { + // storage the node starts from; defaults to a fresh storage holding only genesis. + storage *MockStorage + // wals are pre-existing WALs the instance restores on start. + wals []wal.DeletableWAL +} + +// addNode adds a node to the network and blocks until it catches up with the latest tip +func (n *network) addNode(id common.NodeID) *node { + node := n.addNodeWithConfig(id, nodeConfig{}) + node.storage.WaitForBlockCommit(n.seq - 1) + return node +} + +// addNodeWithStorage adds a node that starts from the given storage. +func (n *network) addNodeWithStorage(id common.NodeID, storage *MockStorage) *node { + return n.addNodeWithConfig(id, nodeConfig{storage: storage}) +} + +// addNodeWithConfig adds a node built from the given config. +func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { + storage := cfg.storage + if storage == nil { + storage = NewMockStorageWithGenesis(n.t, &testInnerBlockDeserializer{}) + } + + // ensure a unique id; snapshot because nodes may be added concurrently + for _, node := range n.nodesSnapshot() { + require.NotEqual(n.t, node.id, id) + } + + comm := newInstanceComm(n, id) + + vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(n.t), storage, n.pending) + wc := &walCreator{t: n.t} + instance := NewInstance(Config{ + LastNonSimplexInnerBlock: genesisBlock, + ParameterConfig: paramConfig, + PlatformChain: n.pChain, + Broadcaster: comm, + Sender: comm, + CryptoOps: &testCryptoOps{}, + WalCreator: wc.createWAL, + Storage: storage, + // the first byte of the node id labels the node's log records + Logger: testutil.MakeLogger(n.t, int(id[0])), + WALs: cfg.wals, + VM: vm, + ICMETransition: noopICMTransition, + ID: id, + }) + + node := node{ + t: n.t, + id: id, + storage: storage, + comm: comm, + vm: vm, + inst: instance, + wals: wc, + } + + n.lock.Lock() + n.nodes = append(n.nodes, node) + n.lock.Unlock() + + ctx, cancel := context.WithCancel(context.Background()) + n.t.Cleanup(cancel) + + require.NoError(n.t, node.inst.Start(ctx)) + n.t.Cleanup(node.inst.Stop) + + instance.Config.Logger.Debug("Added a node to the test network", zap.Uint64("Seq", n.seq), zap.Uint64("num block", node.storage.NumBlocks())) + return &node +} + +// acceptNewBlock blocks until every node has accepted a newly indexed block. +func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { + nodes, ok := n.validatorSets[n.epoch] + require.True(n.t, ok, fmt.Sprintf("epoch is not set epoch: %d. trying to index seq: %d", n.epoch, n.seq)) + + // no nodes have indexed this sequence yet + for _, node := range n.nodes { + node.storage.EnsureNoBlockCommit(n.t, n.seq) + } + + leaderID := simplex.LeaderForRound(nodes.NodeIDs(), n.seq) + for _, node := range n.nodes { + if bytes.Equal(node.id, leaderID) { + node.vm.bb.TriggerNewBlock() + } + } + + // wake every VM blocked in WaitForPendingBlock + n.pending.broadcast() + + var block common.VerifiedBlock + var finalization common.Finalization + for _, node := range n.nodes { + committedBlock := node.storage.WaitForBlockCommit(n.seq) + if block == nil { + _, fin, err := node.storage.Retrieve(n.seq) + require.NoError(n.t, err) + finalization = fin + block = committedBlock + } else { + require.Equal(n.t, block.Bytes(), committedBlock.Bytes()) + } + } + + require.Equal(n.t, block.BlockHeader().Seq, n.seq) + n.seq++ + + // check if its a sealing + if block.SealingBlockInfo() != nil { + n.epoch = n.seq + newValidatorSet := block.SealingBlockInfo().ValidatorSet + common.SortNodes(newValidatorSet) + n.validatorSets[n.epoch] = newValidatorSet + } + + return block, finalization +} + +// waitUntilSealingBlock waits until every node commits the block at the current seq, +// repeating until that block is a sealing block. It then advances the network into +// the new epoch and returns the sealing block. +// This is useful for when we are transitioning epochs because blocks will be built impatiently +// without a notification from the mempool. +func (n *network) waitUntilSealingBlock() common.VerifiedBlock { + for { + var block common.VerifiedBlock + for _, node := range n.nodes { + committedBlock := node.storage.WaitForBlockCommit(n.seq) + if block == nil { + block = committedBlock + } else { + require.Equal(n.t, block.Bytes(), committedBlock.Bytes()) + } + } + + require.Equal(n.t, block.BlockHeader().Seq, n.seq) + n.seq++ + + if block.SealingBlockInfo() == nil { + continue + } + + // add the validator set to the networks memory for block building + n.epoch = n.seq + newValidatorSet := block.SealingBlockInfo().ValidatorSet + common.SortNodes(newValidatorSet) + n.validatorSets[n.epoch] = newValidatorSet + return block + } +} + +// newBLSMapping creates a mapping with a nodeID, BLSKey and Weight with a given [id]. +// id is passed as an int for consistent logs between runs. +func newBLSMapping(id int) metadata.NodeBLSMapping { + avaID := [20]byte{byte(id)} + + return metadata.NodeBLSMapping{ + NodeID: avalanchego.NodeID(avaID), + BLSKey: []byte{avaID[0], byte(id + 1)}, + Weight: 1, + } +} + +// assertExpectedNodeIds asserts the validator set contains exactly the expected node IDs. +func assertExpectedNodeIds(t *testing.T, validatorSet []common.NodeID, expected []common.NodeID) { + require.ElementsMatch(t, expected, validatorSet) +} From e25483adfba404f881ac368aef3e6cdca31cbed3 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 17:01:22 -0400 Subject: [PATCH 07/24] Add the instance tests Each test builds a network and steps it a block at a time. Covers the epoch transition paths the old tests skipped: a non validator that becomes a validator, a validator dropped from the set, a node offline across a transition, and a validator that misses an epoch entirely. --- instance_test.go | 459 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 instance_test.go diff --git a/instance_test.go b/instance_test.go new file mode 100644 index 00000000..becb793a --- /dev/null +++ b/instance_test.go @@ -0,0 +1,459 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "sync" + "testing" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/simplex" + "github.com/ava-labs/simplex/testutil" + "github.com/stretchr/testify/require" +) + +// TestValidatorIndexes tests that a validator indexes and accepts a new block sent by the network +// It is the only validator, so it will build and finalize its own block. +func TestValidatorIndexes(t *testing.T) { + validator := newBLSMapping(1) + + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) + + chain.acceptNewBlock() +} + +// emptyVoteRecorder signals the first empty vote broadcast and drops all other traffic. +type emptyVoteRecorder struct { + got chan struct{} +} + +func (r *emptyVoteRecorder) Broadcast(msg *common.Message) { + if msg.EmptyVoteMessage != nil { + select { + case r.got <- struct{}{}: + default: + } + } +} + +func (r *emptyVoteRecorder) Send(*common.Message, common.NodeID) {} + +// TestEpochInvokesMSMWaitForPendingBlock verifies the epoch drives the MSM's WaitForPendingBlock: +// a non-leader whose VM never has a pending block must still broadcast an empty vote. +// The round leader is never instantiated, so no proposal ever arrives. +func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { + ourValidator := newBLSMapping(1) + leader := newBLSMapping(2) + genesisSet := []metadata.NodeBLSMapping{ourValidator, leader} + + pChain := newTestPChain(genesisSet) + nodes := pChain.GenesisValidatorSet().Nodes() + common.SortNodes(nodes) + require.NotEqual(t, common.NodeID(ourValidator.NodeID[:]), simplex.LeaderForRound(nodes.NodeIDs(), 1)) + + storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) + // the VM blocks in WaitForPendingBlock until a block is indexed, which never happens here + vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(t), storage, newPendingBlockSignal()) + recorder := &emptyVoteRecorder{got: make(chan struct{}, 1)} + wc := &walCreator{t: t} + + inst := NewInstance(Config{ + LastNonSimplexInnerBlock: genesisBlock, + ParameterConfig: paramConfig, + PlatformChain: pChain, + Broadcaster: recorder, + Sender: recorder, + CryptoOps: &testCryptoOps{}, + WalCreator: wc.createWAL, + Storage: storage, + Logger: testutil.MakeLogger(t, 1), + VM: vm, + ICMETransition: noopICMTransition, + ID: ourValidator.NodeID[:], + }) + + require.NoError(t, inst.Start(t.Context())) + t.Cleanup(inst.Stop) + + select { + case <-recorder.got: + case <-time.After(10 * time.Second): + require.FailNow(t, "node never broadcast an empty vote, so the Epoch did not drive the MSM's WaitForPendingBlock") + } +} + +// TestNonValidatorSyncs that a non-validator syncs the chain when added to the network. +func TestNonValidatorSyncs(t *testing.T) { + validator := newBLSMapping(1) + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]) + + network.acceptNewBlock() + + nonValidator := newBLSMapping(2) + network.addNode(nonValidator.NodeID[:]) +} + +// TestNonValidator_BecomesValidator tests that an upcoming validator becomes a validator +// when an epoch change they are following is sealed. The non-validator must contribute it's approval in +// order to do so. +func TestNonValidator_BecomesValidator(t *testing.T) { + validator := newBLSMapping(1) + + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]) + + network.acceptNewBlock() + + // The non-validator node syncs the accepted blocks and then contributes to the next blocks + upcomingValidator := newBLSMapping(2) + network.addNode(upcomingValidator.NodeID[:]) + + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator, upcomingValidator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) + + sealingBlock := network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + + // a normal block should be signed by both the validators now + _, finalization := network.acceptNewBlock() + assertExpectedNodeIds(t, finalization.QC.Signers(), newValidatorSet.NodeIDs()) +} + +// TestValidator_ValidatorSetNotChanged tests that a P-chain height increase +// that does not have a unique validator set, does not create a new epoch +func TestValidator_ValidatorSetNotChanged(t *testing.T) { + validator := newBLSMapping(1) + + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]) + + firstBlock, _ := network.acceptNewBlock() + + // initiate an epoch change + pChain.setValidatorSetAt(10, []metadata.NodeBLSMapping{validator}) + pChain.advanceHeight(10) + + // potential time to propose blocks (if any) + time.Sleep(3 * time.Second) + + secondBlock, _ := network.acceptNewBlock() + require.Equal(t, uint64(1), secondBlock.BlockHeader().Epoch) + require.Equal(t, firstBlock.BlockHeader().Seq+1, secondBlock.BlockHeader().Seq) +} + +// TestValidator_ValidatorSetDecreased tests that an epoch with two validators +// is reduced to one, when the pchain height notes a validator is leaving. +func TestValidator_ValidatorSetDecreased(t *testing.T) { + validator := newBLSMapping(1) + leavingValidator := newBLSMapping(2) + + genesisSet := []metadata.NodeBLSMapping{validator, leavingValidator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + wg := sync.WaitGroup{} + + wg.Go(func() { + // add node is a synchronous call. + network.addNode(validator.NodeID[:]) + }) + + network.addNode(leavingValidator.NodeID[:]) + + // all nodes have synced the first every simplex block + wg.Wait() + + block, _ := network.acceptNewBlock() + require.Equal(t, uint64(2), block.BlockHeader().Round) + + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) + + sealing := network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealing.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) +} + +// TestInstance_OfflineDuringTransition asserts an epoch transition completes while a +// current validator is offline. The online quorum batches approvals and finalizes the +// sealing block, and the new epoch finalizes blocks without the offline node. +func TestInstance_OfflineDuringTransition(t *testing.T) { + v1 := newBLSMapping(1) + v2 := newBLSMapping(2) + v3 := newBLSMapping(3) + offline := newBLSMapping(4) + + genesisSet := []metadata.NodeBLSMapping{v1, v2, v3, offline} + nodeIDs := metadata.NodeBLSMappings(genesisSet).NodeIDs() + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + + v1Storage, _ := newChainStorage(t, genesisSet) + v2Storage, _ := newChainStorage(t, genesisSet) + v3Storage, _ := newChainStorage(t, genesisSet) + + // addNode blocks until the node syncs, which needs a quorum online, + // so the first nodes are added concurrently + network.addNodeWithStorage(v1.NodeID[:], v1Storage) + network.addNodeWithStorage(v2.NodeID[:], v2Storage) + network.addNodeWithStorage(v3.NodeID[:], v3Storage) + + // using seq should be fine since we have no empty blocks + leader := simplex.LeaderForRound(pChain.GenesisValidatorSet().NodeIDs(), network.seq) + for !leader.Equals(offline.NodeID[:]) { + network.acceptNewBlock() + leader = simplex.LeaderForRound(nodeIDs, network.seq) + } + + // initiate an epoch change when the offline node is the leader + newValidatorSet := []metadata.NodeBLSMapping{v1, v2, v3} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) + + network.waitUntilSealingBlock() +} + +// TestNonValidator_StaysNonValidator ensures that a non-validator does not restart when it is processing +// previous epoch changes. +func TestNonValidator_StaysNonValidator(t *testing.T) { + targetNode := newBLSMapping(42) + + // case 1: epoch change is not highest and we are NOT in the validator1 set + // case 1: epoch change is not highest, and we are in the validator1 set + // case 1: epoch change is highest, and we are in the validator1 set + // case 1: epoch change is highest, and we are NOT in the validator1 set + validator1 := newBLSMapping(1) + + genesisSet := []metadata.NodeBLSMapping{validator1} + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + network.addNode(validator1.NodeID[:]) + + validator2 := newBLSMapping(2) + validator3 := newBLSMapping(3) + validator4 := newBLSMapping(4) + network.addNode(validator2.NodeID[:]) + network.addNode(validator3.NodeID[:]) + network.addNode(validator4.NodeID[:]) + + // we should have a quorum without the target node to create this epoch change + targetNodeNotInMiddleEpoch := metadata.NodeBLSMappings{validator1, validator2, validator3} + targetNodeInMiddleEpoch := metadata.NodeBLSMappings{validator1, validator2, validator3, targetNode} + targetNodeInHighestEpoch := metadata.NodeBLSMappings{validator1, validator2, validator3, validator4, targetNode} + + // set the pchain heights + pChain.setValidatorSetAt(10, targetNodeNotInMiddleEpoch) + pChain.setValidatorSetAt(20, targetNodeInMiddleEpoch) + pChain.setValidatorSetAt(30, targetNodeInHighestEpoch) + + pChain.advanceHeight(10) + sealingBlock := network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeNotInMiddleEpoch.NodeIDs()) + + pChain.advanceHeight(20) + sealingBlock = network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeInMiddleEpoch.NodeIDs()) + + // Occasionally, this will offset the proposer of the transition block to be the offline target node + // Therefore, if we don't have a mechanism to skip offline leaders during the transition phase, this test will hang + network.acceptNewBlock() + + pChain.advanceHeight(30) + sealingBlock = network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeInHighestEpoch.NodeIDs()) + + // the target node should join now + network.addNode(targetNode.NodeID[:]) +} + +// TestInstanceValidatorSkipsAnEpoch tests that a validator stops and starts being a validator +// It boots up as a non-validator then syncs to the highest epoch where it is a validator, +// then it is no longer a validator, and finally it is +func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { + validator := newBLSMapping(1) + + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]) + + // The non-validator node syncs the accepted blocks and then contributes to the next blocks + onOffValidator := newBLSMapping(2) + network.addNode(onOffValidator.NodeID[:]) + + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator, onOffValidator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) + + sealingBlock := network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + + newValidatorSet = metadata.NodeBLSMappings{validator} + pChain.setValidatorSetAt(20, newValidatorSet) + pChain.advanceHeight(20) + sealingBlock = network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + + // accept a new block to ensure both nodes are still syncing the chain + network.acceptNewBlock() + + // initiate the final epoch change + newValidatorSet = metadata.NodeBLSMappings{validator, onOffValidator} + pChain.setValidatorSetAt(30, newValidatorSet) + pChain.advanceHeight(30) + + sealingBlock = network.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) +} + +func TestInstanceDoubleStartFails(t *testing.T) { + validator := newBLSMapping(1) + genesisSet := []metadata.NodeBLSMapping{validator} + + pChain := newTestPChain(genesisSet) + network := newNetwork(t, pChain) + node := network.addNode(validator.NodeID[:]) + require.ErrorIs(t, node.inst.Start(t.Context()), errAlreadyStarted) +} + +// TestNonValidatorSkipsMSMVerification proves that a non-validator does not use the MSM to +// verify blocks: it commits a finalized block whose state machine transition is invalid. +func TestNonValidatorSkipsMSMVerification(t *testing.T) { + validator := newBLSMapping(1) + genesisValidatorSet := []metadata.NodeBLSMapping{validator} + pChain := newTestPChain(genesisValidatorSet) + + // The non-validator holds genesis plus epoch 1's defining block, and lives on a network + // of its own, so the replication response below is the only way it can learn a block. + storage, parent := newChainStorage(t, genesisValidatorSet) + nonValidator := newBLSMapping(2) + nonValidatorNode := newNetwork(t, pChain).addNodeWithStorage(nonValidator.NodeID[:], storage) + + // A block whose only defect is its state machine transition: its timestamp precedes its + // parent's. + invalid := metadata.StateMachineBlock{ + InnerBlock: &testInnerBlock{Height_: 2, TS: time.Now(), Payload: []byte("invalid")}, + Metadata: metadata.StateMachineMetadata{ + Timestamp: parent.Metadata.Timestamp - 1, + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 2, Seq: 2, Prev: common.Digest(parent.Digest())}, + }, + } + + // Hand the non-validator that block, finalized over its digest. + block := &ParsedBlock{StateMachineBlock: invalid.Clone()} + finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: 1}, block, []common.NodeID{validator.NodeID[:]}) + require.NoError(t, nonValidatorNode.inst.HandleMessage(&common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + Data: []common.QuorumRound{{Block: block, Finalization: &finalization}}, + }, + }, validator.NodeID[:])) + + // It commits the block its state machine would have rejected. + storage.WaitForBlockCommit(2) + committed, ok := storage.blockAt(2) + require.True(t, ok) + require.Equal(t, invalid.Digest(), committed.Digest()) +} + +// TestValidatorSkipsMSMVerificationWhenReplicating proves that a lagging validator does not +// use the MSM to verify blocks it replicates, as they carry a QC. It checks a notarized and +// a finalized block. +func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { + for _, tt := range []struct { + name string + // quorumRound wraps the replicated block with a QC. + quorumRound func(t *testing.T, block *ParsedBlock, signers []common.NodeID) common.QuorumRound + // requireReplicated asserts the lagging node replicated the block. + requireReplicated func(t *testing.T, laggingNode *node, block metadata.StateMachineBlock) + }{ + { + name: "notarization", + quorumRound: func(t *testing.T, block *ParsedBlock, signers []common.NodeID) common.QuorumRound { + notarization, err := testutil.NewNotarization(testutil.MakeLogger(t, 0), &testutil.TestSignatureAggregator{N: len(signers)}, block, signers) + require.NoError(t, err) + return common.QuorumRound{Block: block, Notarization: ¬arization} + }, + // A notarized block is not committed but notarized: the node persists the + // notarization to its WAL, which it only reaches after the block verified. + requireReplicated: func(t *testing.T, laggingNode *node, block metadata.StateMachineBlock) { + round := block.Metadata.SimplexProtocolMetadata.Round + require.Eventually(t, func() bool { + return laggingNode.wals.containsNotarization(round) + }, 10*time.Second, 10*time.Millisecond, "no notarization for round %d was persisted to the WAL", round) + require.Equal(t, block.Metadata.SimplexProtocolMetadata.Seq, laggingNode.storage.NumBlocks(), "a notarized block should not have been committed") + }, + }, + { + name: "finalization", + quorumRound: func(t *testing.T, block *ParsedBlock, signers []common.NodeID) common.QuorumRound { + finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(signers)}, block, signers) + return common.QuorumRound{Block: block, Finalization: &finalization} + }, + // A finalized block is committed. + requireReplicated: func(t *testing.T, laggingNode *node, block metadata.StateMachineBlock) { + seq := block.Metadata.SimplexProtocolMetadata.Seq + laggingNode.storage.WaitForBlockCommit(seq) + committed, ok := laggingNode.storage.blockAt(seq) + require.True(t, ok) + require.Equal(t, block.Digest(), committed.Digest()) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Two validators, so a quorum is both of them: with its peer absent, the lagging + // validator can neither build a block nor empty notarize a round on its own. + lagging := newBLSMapping(1) + peer := newBLSMapping(2) + validators := metadata.NodeBLSMappings{lagging, peer} + pChain := newTestPChain(validators) + + // The lagging validator holds genesis plus epoch 1's defining block, and lives on + // a network of its own, so the replication response below is the only way it can + // learn the block at the round it sits on. + storage, parent := newChainStorage(t, validators) + laggingNode := newNetwork(t, pChain).addNodeWithStorage(lagging.NodeID[:], storage) + + // A block whose only defect is its state machine transition: its timestamp + // precedes its parent's. + invalid := metadata.StateMachineBlock{ + InnerBlock: &testInnerBlock{Height_: 2, TS: time.Now(), Payload: []byte("invalid")}, + Metadata: metadata.StateMachineMetadata{ + Timestamp: parent.Metadata.Timestamp - 1, + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 2, Seq: 2, Prev: common.Digest(parent.Digest())}, + }, + } + + // Hand the lagging validator that block wrapped in a QC. + block := &ParsedBlock{StateMachineBlock: invalid.Clone()} + quorumRound := tt.quorumRound(t, block, validators.NodeIDs()) + require.NoError(t, laggingNode.inst.HandleMessage(&common.Message{ + ReplicationResponse: &common.ReplicationResponse{Data: []common.QuorumRound{quorumRound}}, + }, peer.NodeID[:])) + + // It replicates the block its state machine would have rejected. + tt.requireReplicated(t, laggingNode, invalid) + }) + } +} From b1c88c8da802dc1850ae5b021741d0cb6812481d Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 17:01:22 -0400 Subject: [PATCH 08/24] Update the remaining callers of the instance test helpers adapters_test.go builds its node through the network harness instead of wiring an instance by hand, util_test.go follows the newTestPChain rename, and the instance logs the validator set it is notified of on an epoch change. --- adapters_test.go | 60 +++++++++++------------------------------------- instance.go | 1 + util_test.go | 2 +- 3 files changed, 15 insertions(+), 48 deletions(-) diff --git a/adapters_test.go b/adapters_test.go index b889432d..24ff5c0d 100644 --- a/adapters_test.go +++ b/adapters_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" "github.com/ava-labs/simplex/testutil" @@ -39,7 +38,7 @@ func newTestParsedBlock(num uint64, payload string) *ParsedBlock { // and a verified but not yet indexed block at seq 5. A zero digest matches on // seq alone, a non-zero digest must match the block's digest exactly. func TestCachedStorageRetrieve(t *testing.T) { - cs := NewCachedStorage(NewMockStorage(t)) + cs := NewCachedStorage(NewMockStorage(t, &testInnerBlockDeserializer{})) indexedBlock := newTestParsedBlock(0, "indexed") require.NoError(t, cs.Index(t.Context(), indexedBlock, common.Finalization{})) @@ -114,7 +113,7 @@ func TestCachedStorageRetrieve(t *testing.T) { // a zero-digest Retrieve of that seq returns the finalized block with its // finalization, even when a verified fork at the same seq was cached. func TestCachedStorageIndexEvictsSameSeqFork(t *testing.T) { - cs := NewCachedStorage(NewMockStorage(t)) + cs := NewCachedStorage(NewMockStorage(t, &testInnerBlockDeserializer{})) require.NoError(t, cs.Index(t.Context(), newTestParsedBlock(0, "genesis"), common.Finalization{})) equivocatedBlock := &cachedBlock{ @@ -137,27 +136,13 @@ func TestCachedStorageIndexEvictsSameSeqFork(t *testing.T) { // startup ends up in the instance's CachedStorage, retrievable by seq before it // is finalized and indexed. func TestCachedStoragePopulatedByWal(t *testing.T) { - const basePChainHeight = uint64(1) - - // Four equal-weight validators; the node under test is the first. - numNodes := 4 - validatorSet := make(metadata.NodeBLSMappings, numNodes) - for i := range numNodes { - validatorSet[i] = metadata.NodeBLSMapping{NodeID: avalanchego.NodeID{byte(i + 1)}, BLSKey: []byte{byte(i + 1)}, Weight: 1} + // Four equal-weight validators; only the first runs, so no quorum forms + // and the restored block stays unfinalized. + validatorSet := make(metadata.NodeBLSMappings, 4) + for i := range validatorSet { + validatorSet[i] = newBLSMapping(i + 1) } - pChain := newTestPlatformChain(basePChainHeight, map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: validatorSet, - }) - - vm := newTestVM() - vm.pause() - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - storage := newStorageWithGenesis(t, genesisBlock) nodeIDs := validatorSet.Nodes().NodeIDs() - comm := testutil.NewNoopComm(nodeIDs) - logger := testutil.MakeLogger(t, 1) - testWAL := testutil.NewTestWAL(t) // The first Simplex block on top of the genesis block. genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} @@ -165,44 +150,25 @@ func TestCachedStoragePopulatedByWal(t *testing.T) { block.Metadata.SimplexProtocolMetadata.Epoch = 1 block.Metadata.SimplexProtocolMetadata.Prev = genesis.BlockHeader().Digest + testWAL := testutil.NewTestWAL(t) blockRecord, err := common.BlockRecord(block.BlockHeader(), block.Bytes()) require.NoError(t, err) - - // write block record to wal require.NoError(t, testWAL.Append(blockRecord)) // notarize the block so restoring the WAL keeps it as the round in progress + cops := &testCryptoOps{} quorum := common.Quorum(len(nodeIDs)) - notarizationRecord, err := testutil.NewNotarizationRecord(logger, cops.CreateSignatureAggregator(validatorSet.Nodes()), block, nodeIDs[:quorum]) + notarizationRecord, err := testutil.NewNotarizationRecord(testutil.MakeLogger(t, 1), cops.CreateSignatureAggregator(validatorSet.Nodes()), block, nodeIDs[:quorum]) require.NoError(t, err) require.NoError(t, testWAL.Append(notarizationRecord)) - config := Config{ - Logger: logger, - ID: nodeIDs[0], - VM: vm, - Storage: storage, - Sender: comm, - Broadcaster: comm, - PlatformChain: pChain, - CryptoOps: cops, - LastNonSimplexInnerBlock: genesisBlock, - WalCreator: storage.CreateWAL, - ParameterConfig: ParameterConfig{ - MaxNetworkDelay: 500 * time.Millisecond, - MaxRoundWindow: 100, - WALMaxSizeBytes: 1024, - }, - WALs: []wal.DeletableWAL{testWAL}, - } - instance := NewInstance(config) - require.NoError(t, instance.Start(t.Context())) - t.Cleanup(instance.Stop) + chain := newNetwork(t, newTestPChain(validatorSet)) + node := chain.addNodeWithConfig(nodeIDs[0], nodeConfig{wals: []wal.DeletableWAL{testWAL}}) // The restored block is verified asynchronously and not indexed, so poll until // a seq-only lookup serves it from the cache. require.Eventually(t, func() bool { - got, fin, err := instance.cs.Retrieve(1, common.Digest{}) + got, fin, err := node.inst.cs.Retrieve(1, common.Digest{}) if err != nil || fin != nil { return false } diff --git a/instance.go b/instance.go index 448c4eda..d8421a55 100644 --- a/instance.go +++ b/instance.go @@ -209,6 +209,7 @@ func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { } func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes) { + i.Config.Logger.Debug("Notifying the instance of an epoch change", zap.Uint64("Epoch", epoch), zap.Stringers("Validators", validators.NodeIDs())) ec := epochChange{ epoch: epoch, validators: validators, diff --git a/util_test.go b/util_test.go index 498f268f..6d63f086 100644 --- a/util_test.go +++ b/util_test.go @@ -78,7 +78,7 @@ func epochTestConfig(t *testing.T, storage *stubStorage, genesisSet metadata.Nod } return &Config{ Storage: storage, - PlatformChain: newTestPlatformChain(0, map[uint64]metadata.NodeBLSMappings{0: genesisSet}), + PlatformChain: newTestPChain(genesisSet), LastNonSimplexInnerBlock: &testInnerBlock{Height_: lastNonSimplexHeight}, Logger: testutil.MakeLogger(t, 1), } From f2d56835b971f49098055796133063d494cb12e2 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 19:51:52 -0400 Subject: [PATCH 09/24] merge conflicts --- adapters_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/adapters_test.go b/adapters_test.go index 24ff5c0d..f591f718 100644 --- a/adapters_test.go +++ b/adapters_test.go @@ -180,22 +180,21 @@ func TestCachedStoragePopulatedByWal(t *testing.T) { // own proposal is inserted into the CachedStorage, retrievable by seq and digest before // it is finalized and indexed. func TestCachedStoragePopulatedBySelfBuiltBlock(t *testing.T) { - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - cs := NewCachedStorage(newStorageWithGenesis(t, genesisBlock)) + storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) + cs := NewCachedStorage(storage) msm, err := metadata.NewStateMachine(&metadata.Config{ Logger: testutil.MakeLogger(t, 1), GetBlock: cs.RetrieveBlock, LastNonSimplexInnerBlock: genesisBlock, - GenesisValidatorSet: metadata.NodeBLSMappings{ - {NodeID: avalanchego.NodeID{1}, BLSKey: []byte{1}, Weight: 1}, - }, - AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, + GenesisValidatorSet: metadata.NodeBLSMappings{newBLSMapping(1)}, + AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, }) require.NoError(t, err) cs.msm = msm - bw := newBlockBuilderWaiter(msm, cs, newTestVM()) + vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(t), storage, newPendingBlockSignal()) + bw := newBlockBuilderWaiter(msm, cs, vm) // Build a block on top of genesis genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} From c06829f4658d1177c8718b7fb032a595a42bb5c0 Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 19:55:13 -0400 Subject: [PATCH 10/24] comments --- instance_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instance_test.go b/instance_test.go index becb793a..6dd5c64c 100644 --- a/instance_test.go +++ b/instance_test.go @@ -239,9 +239,9 @@ func TestNonValidator_StaysNonValidator(t *testing.T) { targetNode := newBLSMapping(42) // case 1: epoch change is not highest and we are NOT in the validator1 set - // case 1: epoch change is not highest, and we are in the validator1 set - // case 1: epoch change is highest, and we are in the validator1 set - // case 1: epoch change is highest, and we are NOT in the validator1 set + // case 2: epoch change is not highest, and we are in the validator1 set + // case 3: epoch change is highest, and we are in the validator1 set + // case 4: epoch change is highest, and we are NOT in the validator1 set validator1 := newBLSMapping(1) genesisSet := []metadata.NodeBLSMapping{validator1} From 672f3db84edcabc7392be294a40bd550f527e21c Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 1 Sep 2026 20:43:23 -0400 Subject: [PATCH 11/24] consistent time --- instance_helpers_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index b677dcb2..f3721593 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -60,6 +60,10 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte const genesisPChainHeight uint64 = 0 var genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} + +// epochBlockTime fixes the timestamp of the epoch-defining block +// this ensures a consistent block digest +var epochBlockTime = genesisBlock.TS.Add(time.Millisecond) var paramConfig = ParameterConfig{ MaxNetworkDelay: 500 * time.Millisecond, MaxRoundWindow: 100, @@ -294,9 +298,9 @@ func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*MockSt require.True(t, ok) epochBlock := metadata.StateMachineBlock{ - InnerBlock: &testInnerBlock{Height_: 1, TS: time.Now(), Payload: []byte("epoch")}, + InnerBlock: &testInnerBlock{Height_: 1, TS: epochBlockTime, Payload: []byte("epoch")}, Metadata: metadata.StateMachineMetadata{ - Timestamp: uint64(time.Now().UnixMilli()), + Timestamp: uint64(epochBlockTime.UnixMilli()), SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 1, Seq: 1, Prev: common.Digest(genesis.Digest())}, SimplexEpochInfo: metadata.SimplexEpochInfo{ EpochNumber: 1, From 9c4e363c61ed6367baf69432456471c8bb36d07b Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 12:12:50 -0400 Subject: [PATCH 12/24] speedup test, and ensure block notification is not dropped with offline leader --- adapters_test.go | 2 +- instance_helpers_test.go | 108 ++++++++++++++++++++++++--------------- instance_test.go | 2 +- 3 files changed, 68 insertions(+), 44 deletions(-) diff --git a/adapters_test.go b/adapters_test.go index f591f718..a1a9c8e6 100644 --- a/adapters_test.go +++ b/adapters_test.go @@ -193,7 +193,7 @@ func TestCachedStoragePopulatedBySelfBuiltBlock(t *testing.T) { require.NoError(t, err) cs.msm = msm - vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(t), storage, newPendingBlockSignal()) + vm := newBlockBuilderVM(storage, newPendingBlockSignal()) bw := newBlockBuilderWaiter(msm, cs, vm) // Build a block on top of genesis diff --git a/instance_helpers_test.go b/instance_helpers_test.go index f3721593..3e54e256 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -17,7 +17,6 @@ import ( "github.com/ava-labs/simplex/avalanchego" "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" - "github.com/ava-labs/simplex/simplex" "github.com/ava-labs/simplex/testutil" "github.com/ava-labs/simplex/wal" "github.com/stretchr/testify/require" @@ -65,7 +64,7 @@ var genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), // this ensures a consistent block digest var epochBlockTime = genesisBlock.TS.Add(time.Millisecond) var paramConfig = ParameterConfig{ - MaxNetworkDelay: 500 * time.Millisecond, + MaxNetworkDelay: 200 * time.Millisecond, MaxRoundWindow: 100, WALMaxSizeBytes: 1024, } @@ -449,53 +448,85 @@ func translateOutgoingToIncomingMessage(t *testing.T, msg *common.Message) *comm } } -// pendingBlockSignal broadcasts to every waiter by closing the current channel and -// replacing it with a fresh one for the next generation of waiters. +// pendingBlockSignal is the network's shared mempool. It stays pending until a block builder consumes it, +// so an offline leader only delays the block: rounds are empty notarized until an online leader claims it. type pendingBlockSignal struct { - lock sync.Mutex - ch chan struct{} + lock sync.Mutex + pending bool + ch chan struct{} } func newPendingBlockSignal() *pendingBlockSignal { return &pendingBlockSignal{ch: make(chan struct{})} } -// wait returns when the signal is broadcast or ctx is cancelled. -func (s *pendingBlockSignal) wait(ctx context.Context) { +// pendingBlockSignal is the network's shared mempool. It stays pending until a block builder consumes it, +// so calling addPendingBlock will always produce a block. +// If the current leader is offline, rounds are empty notarized until an online leader claims it. +func (s *pendingBlockSignal) addPendingBlock() { s.lock.Lock() - ch := s.ch - s.lock.Unlock() + defer s.lock.Unlock() + + s.pending = true + close(s.ch) + s.ch = make(chan struct{}) +} + +// wait returns once a block is pending or ctx is cancelled. It does not +// consume the block for block building. +func (s *pendingBlockSignal) wait(ctx context.Context) { + for { + s.lock.Lock() + pending, ch := s.pending, s.ch + s.lock.Unlock() + + if pending { + return + } - select { - case <-ch: - case <-ctx.Done(): + select { + case <-ch: + case <-ctx.Done(): + return + } } } -// broadcast wakes every current waiter. -func (s *pendingBlockSignal) broadcast() { - s.lock.Lock() - close(s.ch) - s.ch = make(chan struct{}) - s.lock.Unlock() +// consume waits for a pending block and claims it, reporting false if ctx is cancelled first. +// called from the block builder. +func (s *pendingBlockSignal) consume(ctx context.Context) bool { + for { + s.lock.Lock() + ch := s.ch + if s.pending { + s.pending = false + s.lock.Unlock() + return true + } + s.lock.Unlock() + + select { + case <-ch: + case <-ctx.Done(): + return false + } + } } -// blockBuilderVM builds an inner block only when the test triggers one on the block builder, so -// the chain grows one block per index call. +// blockBuilderVM builds an inner block only when the test arms a pending block, so the +// chain grows one block per index call. type blockBuilderVM struct { - bb *testutil.TestControlledBlockBuilder storage *MockStorage pending *pendingBlockSignal } -func newBlockBuilderVM(bb *testutil.TestControlledBlockBuilder, storage *MockStorage, pending *pendingBlockSignal) *blockBuilderVM { - return &blockBuilderVM{bb: bb, storage: storage, pending: pending} +func newBlockBuilderVM(storage *MockStorage, pending *pendingBlockSignal) *blockBuilderVM { + return &blockBuilderVM{storage: storage, pending: pending} } func (vm *blockBuilderVM) BuildBlock(ctx context.Context, pChainHeight uint64) (avalanchego.VMBlock, error) { - // The builder gates when a block is built; the block it returns is not an inner block, so - // it is thrown away. - if _, ok := vm.bb.BuildBlock(ctx, common.ProtocolMetadata{}, common.Blacklist{}); !ok { + // Claiming the pending block is what gates block building. + if !vm.pending.consume(ctx) { return nil, ctx.Err() } @@ -511,8 +542,7 @@ func (vm *blockBuilderVM) ParseBlock(ctx context.Context, bytes []byte) (avalanc return vm.storage.bd.ParseBlock(ctx, bytes) } -// WaitForPendingBlock returns when index broadcasts that a block is being created, -// or when ctx is cancelled. +// WaitForPendingBlock returns while a block is pending, or when ctx is cancelled. func (vm *blockBuilderVM) WaitForPendingBlock(ctx context.Context) { vm.pending.wait(ctx) } @@ -540,7 +570,7 @@ type network struct { seq uint64 epoch uint64 - // pending wakes every VM blocked in WaitForPendingBlock when index creates a block. + // pending holds the block the network has been asked to build, claimable by any leader. pending *pendingBlockSignal validatorSets map[uint64]common.Nodes // epoch -> sorted validators @@ -609,7 +639,7 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { comm := newInstanceComm(n, id) - vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(n.t), storage, n.pending) + vm := newBlockBuilderVM(storage, n.pending) wc := &walCreator{t: n.t} instance := NewInstance(Config{ LastNonSimplexInnerBlock: genesisBlock, @@ -654,23 +684,17 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { // acceptNewBlock blocks until every node has accepted a newly indexed block. func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { - nodes, ok := n.validatorSets[n.epoch] + _, ok := n.validatorSets[n.epoch] require.True(n.t, ok, fmt.Sprintf("epoch is not set epoch: %d. trying to index seq: %d", n.epoch, n.seq)) // no nodes have indexed this sequence yet for _, node := range n.nodes { - node.storage.EnsureNoBlockCommit(n.t, n.seq) - } - - leaderID := simplex.LeaderForRound(nodes.NodeIDs(), n.seq) - for _, node := range n.nodes { - if bytes.Equal(node.id, leaderID) { - node.vm.bb.TriggerNewBlock() - } + _, _, err := node.storage.Retrieve(n.seq) + require.ErrorIs(n.t, err, common.ErrBlockNotFound) } - // wake every VM blocked in WaitForPendingBlock - n.pending.broadcast() + // Adds a pending block to the network, to be built by the next online node. + n.pending.addPendingBlock() var block common.VerifiedBlock var finalization common.Finalization diff --git a/instance_test.go b/instance_test.go index 6dd5c64c..fcf2d150 100644 --- a/instance_test.go +++ b/instance_test.go @@ -60,7 +60,7 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) // the VM blocks in WaitForPendingBlock until a block is indexed, which never happens here - vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(t), storage, newPendingBlockSignal()) + vm := newBlockBuilderVM(storage, newPendingBlockSignal()) recorder := &emptyVoteRecorder{got: make(chan struct{}, 1)} wc := &walCreator{t: t} From 289da307c69197b851a46ef193578742780c2478 Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 12:29:39 -0400 Subject: [PATCH 13/24] add validator set interpolation --- instance_helpers_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 3e54e256..8adcc065 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -102,14 +102,33 @@ func (pc *testPlatformChain) currentHeight() uint64 { return pc.height } +// GetValidatorSet grabs the validator set at `height`. If one does not directly exist at that height, +// it returns validator set at the first height less than `height`. func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { pc.lock.Lock() defer pc.lock.Unlock() set, ok := pc.validatorSetAtHeight[height] + if ok { + return set, nil + } + + var nextLargestHeight uint64 + for h, _ := range pc.validatorSetAtHeight { + if h >= height { + continue + } + + if h > nextLargestHeight { + nextLargestHeight = h + } + } + + set, ok = pc.validatorSetAtHeight[nextLargestHeight] if !ok { return nil, fmt.Errorf("no validator set at %d", height) } + return set, nil } From 808edeebf617143072338f08c0874f4dd114840d Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 13:15:00 -0400 Subject: [PATCH 14/24] use var group --- instance_helpers_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 8adcc065..aa606957 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -56,9 +56,10 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte return b, nil } -const genesisPChainHeight uint64 = 0 - -var genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} +var ( + genesisPChainHeight uint64 = 0 + genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} +) // epochBlockTime fixes the timestamp of the epoch-defining block // this ensures a consistent block digest From b00b420bce11768ad2e719a80bc6e82bf85aef28 Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 14:03:19 -0400 Subject: [PATCH 15/24] wait until validators are running --- instance_helpers_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index aa606957..bd4217f9 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -583,6 +583,14 @@ type node struct { wals *walCreator } +// role reports whether the node is running a validator rather than a non-validator. +func (n *node) role() (isValidator bool) { + n.inst.lock.Lock() + defer n.inst.lock.Unlock() + + return n.inst.e != nil +} + type network struct { t *testing.T @@ -702,6 +710,23 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { return &node } +// waitUntilValidatorsRunning blocks until every node that should be validator in the current epoch +// is actually a validator. This ensures that acceptNewBlock, waits for any nodes that may be transitioning from +// a non-validator actually here about the new proposal. +func (n *network) waitUntilValidatorsReady() { + validators, ok := n.validatorSets[n.epoch] + require.True(n.t, ok, fmt.Sprintf("epoch is not set. epoch: %d", n.epoch)) + + for _, node := range n.nodesSnapshot() { + if common.NodeIDs(validators.NodeIDs()).IndexOf(node.id) < 0 { + continue + } + + require.Eventually(n.t, node.role, time.Minute, time.Millisecond, + "node %x never started running a validator", node.id) + } +} + // acceptNewBlock blocks until every node has accepted a newly indexed block. func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { _, ok := n.validatorSets[n.epoch] @@ -713,6 +738,8 @@ func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { require.ErrorIs(n.t, err, common.ErrBlockNotFound) } + n.waitUntilValidatorsReady() + // Adds a pending block to the network, to be built by the next online node. n.pending.addPendingBlock() From 1d6f03d187b306f3afa68c93a3a2e22f23932571 Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 14:06:53 -0400 Subject: [PATCH 16/24] lint --- instance_helpers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index bd4217f9..64bf6df5 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -115,7 +115,7 @@ func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMap } var nextLargestHeight uint64 - for h, _ := range pc.validatorSetAtHeight { + for h := range pc.validatorSetAtHeight { if h >= height { continue } From 92406560e3d9e10e87423c5c12e3dd7b5dc6b03d Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 18:50:17 -0400 Subject: [PATCH 17/24] review comments --- adapters_test.go | 4 +- instance_helpers_test.go | 9 +-- instance_test.go | 142 +++++++++++++++++++-------------------- 3 files changed, 77 insertions(+), 78 deletions(-) diff --git a/adapters_test.go b/adapters_test.go index a1a9c8e6..fe5d4194 100644 --- a/adapters_test.go +++ b/adapters_test.go @@ -140,7 +140,7 @@ func TestCachedStoragePopulatedByWal(t *testing.T) { // and the restored block stays unfinalized. validatorSet := make(metadata.NodeBLSMappings, 4) for i := range validatorSet { - validatorSet[i] = newBLSMapping(i + 1) + validatorSet[i] = newNodeMapping(i + 1) } nodeIDs := validatorSet.Nodes().NodeIDs() @@ -187,7 +187,7 @@ func TestCachedStoragePopulatedBySelfBuiltBlock(t *testing.T) { Logger: testutil.MakeLogger(t, 1), GetBlock: cs.RetrieveBlock, LastNonSimplexInnerBlock: genesisBlock, - GenesisValidatorSet: metadata.NodeBLSMappings{newBLSMapping(1)}, + GenesisValidatorSet: metadata.NodeBLSMappings{newNodeMapping(1)}, AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, }) require.NoError(t, err) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 64bf6df5..efb1a0f3 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -776,7 +776,7 @@ func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { // the new epoch and returns the sealing block. // This is useful for when we are transitioning epochs because blocks will be built impatiently // without a notification from the mempool. -func (n *network) waitUntilSealingBlock() common.VerifiedBlock { +func (n *network) waitUntilSealingBlock(expectedValidatorSet common.Nodes) common.VerifiedBlock { for { var block common.VerifiedBlock for _, node := range n.nodes { @@ -798,15 +798,16 @@ func (n *network) waitUntilSealingBlock() common.VerifiedBlock { // add the validator set to the networks memory for block building n.epoch = n.seq newValidatorSet := block.SealingBlockInfo().ValidatorSet + assertExpectedNodeIds(n.t, expectedValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) common.SortNodes(newValidatorSet) n.validatorSets[n.epoch] = newValidatorSet return block } } -// newBLSMapping creates a mapping with a nodeID, BLSKey and Weight with a given [id]. +// newNodeMapping creates a mapping with a nodeID, BLSKey and Weight with a given [id]. // id is passed as an int for consistent logs between runs. -func newBLSMapping(id int) metadata.NodeBLSMapping { +func newNodeMapping(id int) metadata.NodeBLSMapping { avaID := [20]byte{byte(id)} return metadata.NodeBLSMapping{ @@ -817,6 +818,6 @@ func newBLSMapping(id int) metadata.NodeBLSMapping { } // assertExpectedNodeIds asserts the validator set contains exactly the expected node IDs. -func assertExpectedNodeIds(t *testing.T, validatorSet []common.NodeID, expected []common.NodeID) { +func assertExpectedNodeIds(t *testing.T, validatorSet common.NodeIDs, expected common.NodeIDs) { require.ElementsMatch(t, expected, validatorSet) } diff --git a/instance_test.go b/instance_test.go index fcf2d150..87922dfe 100644 --- a/instance_test.go +++ b/instance_test.go @@ -4,7 +4,6 @@ package simplex import ( - "sync" "testing" "time" @@ -18,7 +17,7 @@ import ( // TestValidatorIndexes tests that a validator indexes and accepts a new block sent by the network // It is the only validator, so it will build and finalize its own block. func TestValidatorIndexes(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} @@ -45,12 +44,12 @@ func (r *emptyVoteRecorder) Broadcast(msg *common.Message) { func (r *emptyVoteRecorder) Send(*common.Message, common.NodeID) {} -// TestEpochInvokesMSMWaitForPendingBlock verifies the epoch drives the MSM's WaitForPendingBlock: -// a non-leader whose VM never has a pending block must still broadcast an empty vote. -// The round leader is never instantiated, so no proposal ever arrives. func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { - ourValidator := newBLSMapping(1) - leader := newBLSMapping(2) + // Two validators, but only one is instantiated. Our node has the smaller ID so it sorts to + // index 0 and is a non-leader for round 1 (LeaderForRound picks index 1%2). The other + // validator is the round leader but is never created, so no block is ever proposed. + ourValidator := newNodeMapping(1) + leader := newNodeMapping(2) genesisSet := []metadata.NodeBLSMapping{ourValidator, leader} pChain := newTestPChain(genesisSet) @@ -59,7 +58,7 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { require.NotEqual(t, common.NodeID(ourValidator.NodeID[:]), simplex.LeaderForRound(nodes.NodeIDs(), 1)) storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) - // the VM blocks in WaitForPendingBlock until a block is indexed, which never happens here + vm := newBlockBuilderVM(storage, newPendingBlockSignal()) recorder := &emptyVoteRecorder{got: make(chan struct{}, 1)} wc := &walCreator{t: t} @@ -91,7 +90,7 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { // TestNonValidatorSyncs that a non-validator syncs the chain when added to the network. func TestNonValidatorSyncs(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisSet) @@ -100,15 +99,16 @@ func TestNonValidatorSyncs(t *testing.T) { network.acceptNewBlock() - nonValidator := newBLSMapping(2) + nonValidator := newNodeMapping(2) network.addNode(nonValidator.NodeID[:]) + network.acceptNewBlock() } -// TestNonValidator_BecomesValidator tests that an upcoming validator becomes a validator +// TestNonValidatorBecomesValidator tests that an upcoming validator becomes a validator // when an epoch change they are following is sealed. The non-validator must contribute it's approval in // order to do so. -func TestNonValidator_BecomesValidator(t *testing.T) { - validator := newBLSMapping(1) +func TestNonValidatorBecomesValidator(t *testing.T) { + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} @@ -119,7 +119,7 @@ func TestNonValidator_BecomesValidator(t *testing.T) { network.acceptNewBlock() // The non-validator node syncs the accepted blocks and then contributes to the next blocks - upcomingValidator := newBLSMapping(2) + upcomingValidator := newNodeMapping(2) network.addNode(upcomingValidator.NodeID[:]) // initiate an epoch change @@ -127,8 +127,7 @@ func TestNonValidator_BecomesValidator(t *testing.T) { pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) - sealingBlock := network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + network.waitUntilSealingBlock(newValidatorSet.Nodes()) // a normal block should be signed by both the validators now _, finalization := network.acceptNewBlock() @@ -138,13 +137,13 @@ func TestNonValidator_BecomesValidator(t *testing.T) { // TestValidator_ValidatorSetNotChanged tests that a P-chain height increase // that does not have a unique validator set, does not create a new epoch func TestValidator_ValidatorSetNotChanged(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]) + node := network.addNode(validator.NodeID[:]) firstBlock, _ := network.acceptNewBlock() @@ -152,35 +151,35 @@ func TestValidator_ValidatorSetNotChanged(t *testing.T) { pChain.setValidatorSetAt(10, []metadata.NodeBLSMapping{validator}) pChain.advanceHeight(10) + blockCount := node.storage.NumBlocks() // potential time to propose blocks (if any) - time.Sleep(3 * time.Second) + require.Never(t, func() bool { + return node.storage.NumBlocks() != blockCount + }, time.Second, 100*time.Millisecond, "no new block should have been proposed") secondBlock, _ := network.acceptNewBlock() require.Equal(t, uint64(1), secondBlock.BlockHeader().Epoch) require.Equal(t, firstBlock.BlockHeader().Seq+1, secondBlock.BlockHeader().Seq) } -// TestValidator_ValidatorSetDecreased tests that an epoch with two validators +// TestValidatorValidatorSetDecreased tests that an epoch with two validators // is reduced to one, when the pchain height notes a validator is leaving. -func TestValidator_ValidatorSetDecreased(t *testing.T) { - validator := newBLSMapping(1) - leavingValidator := newBLSMapping(2) +func TestValidatorValidatorSetDecreased(t *testing.T) { + validator := newNodeMapping(1) + leavingValidator := newNodeMapping(2) genesisSet := []metadata.NodeBLSMapping{validator, leavingValidator} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - wg := sync.WaitGroup{} - wg.Go(func() { - // add node is a synchronous call. - network.addNode(validator.NodeID[:]) - }) - - network.addNode(leavingValidator.NodeID[:]) + validatorStorage, _ := newChainStorage(t, genesisSet) + leavingValidatorStorage, _ := newChainStorage(t, genesisSet) - // all nodes have synced the first every simplex block - wg.Wait() + // starting from storage holding the first simplex block avoids + // blocking on a sync that needs a quorum online + network.addNodeWithStorage(validator.NodeID[:], validatorStorage) + network.addNodeWithStorage(leavingValidator.NodeID[:], leavingValidatorStorage) block, _ := network.acceptNewBlock() require.Equal(t, uint64(2), block.BlockHeader().Round) @@ -190,18 +189,17 @@ func TestValidator_ValidatorSetDecreased(t *testing.T) { pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) - sealing := network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealing.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + network.waitUntilSealingBlock(newValidatorSet.Nodes()) } -// TestInstance_OfflineDuringTransition asserts an epoch transition completes while a +// TestInstanceOfflineDuringTransition asserts an epoch transition completes while a // current validator is offline. The online quorum batches approvals and finalizes the // sealing block, and the new epoch finalizes blocks without the offline node. -func TestInstance_OfflineDuringTransition(t *testing.T) { - v1 := newBLSMapping(1) - v2 := newBLSMapping(2) - v3 := newBLSMapping(3) - offline := newBLSMapping(4) +func TestInstanceOfflineDuringTransition(t *testing.T) { + v1 := newNodeMapping(1) + v2 := newNodeMapping(2) + v3 := newNodeMapping(3) + offline := newNodeMapping(4) genesisSet := []metadata.NodeBLSMapping{v1, v2, v3, offline} nodeIDs := metadata.NodeBLSMappings(genesisSet).NodeIDs() @@ -212,9 +210,7 @@ func TestInstance_OfflineDuringTransition(t *testing.T) { v2Storage, _ := newChainStorage(t, genesisSet) v3Storage, _ := newChainStorage(t, genesisSet) - // addNode blocks until the node syncs, which needs a quorum online, - // so the first nodes are added concurrently - network.addNodeWithStorage(v1.NodeID[:], v1Storage) + node1 := network.addNodeWithStorage(v1.NodeID[:], v1Storage) network.addNodeWithStorage(v2.NodeID[:], v2Storage) network.addNodeWithStorage(v3.NodeID[:], v3Storage) @@ -222,7 +218,7 @@ func TestInstance_OfflineDuringTransition(t *testing.T) { leader := simplex.LeaderForRound(pChain.GenesisValidatorSet().NodeIDs(), network.seq) for !leader.Equals(offline.NodeID[:]) { network.acceptNewBlock() - leader = simplex.LeaderForRound(nodeIDs, network.seq) + leader = simplex.LeaderForRound(nodeIDs, node1.inst.e.Metadata().Round) } // initiate an epoch change when the offline node is the leader @@ -230,28 +226,27 @@ func TestInstance_OfflineDuringTransition(t *testing.T) { pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) - network.waitUntilSealingBlock() + network.waitUntilSealingBlock(metadata.NodeBLSMappings(newValidatorSet).Nodes()) } -// TestNonValidator_StaysNonValidator ensures that a non-validator does not restart when it is processing -// previous epoch changes. -func TestNonValidator_StaysNonValidator(t *testing.T) { - targetNode := newBLSMapping(42) +// TestNonValidatorStaysNonValidator ensures that a non-validator advances through different epoch changes. +func TestNonValidatorStaysNonValidator(t *testing.T) { + targetNode := newNodeMapping(42) // case 1: epoch change is not highest and we are NOT in the validator1 set // case 2: epoch change is not highest, and we are in the validator1 set // case 3: epoch change is highest, and we are in the validator1 set // case 4: epoch change is highest, and we are NOT in the validator1 set - validator1 := newBLSMapping(1) + validator1 := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator1} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) network.addNode(validator1.NodeID[:]) - validator2 := newBLSMapping(2) - validator3 := newBLSMapping(3) - validator4 := newBLSMapping(4) + validator2 := newNodeMapping(2) + validator3 := newNodeMapping(3) + validator4 := newNodeMapping(4) network.addNode(validator2.NodeID[:]) network.addNode(validator3.NodeID[:]) network.addNode(validator4.NodeID[:]) @@ -267,30 +262,36 @@ func TestNonValidator_StaysNonValidator(t *testing.T) { pChain.setValidatorSetAt(30, targetNodeInHighestEpoch) pChain.advanceHeight(10) - sealingBlock := network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeNotInMiddleEpoch.NodeIDs()) + network.waitUntilSealingBlock(targetNodeNotInMiddleEpoch.Nodes()) pChain.advanceHeight(20) - sealingBlock = network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeInMiddleEpoch.NodeIDs()) + network.waitUntilSealingBlock(targetNodeInMiddleEpoch.Nodes()) // Occasionally, this will offset the proposer of the transition block to be the offline target node // Therefore, if we don't have a mechanism to skip offline leaders during the transition phase, this test will hang network.acceptNewBlock() pChain.advanceHeight(30) - sealingBlock = network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), targetNodeInHighestEpoch.NodeIDs()) + network.waitUntilSealingBlock(targetNodeInHighestEpoch.Nodes()) // the target node should join now network.addNode(targetNode.NodeID[:]) + + // the target node must sign within a bounded number of blocks, otherwise it never rejoined + const maxBlocksUntilTargetSigns = 10 + var signed bool + for i := 0; i < maxBlocksUntilTargetSigns && !signed; i++ { + _, finalization := network.acceptNewBlock() + signed = common.NodeIDs(finalization.QC.Signers()).IndexOf(targetNode.NodeID[:]) != -1 + } + require.True(t, signed, "target node never signed a finalization within %d blocks", maxBlocksUntilTargetSigns) } // TestInstanceValidatorSkipsAnEpoch tests that a validator stops and starts being a validator // It boots up as a non-validator then syncs to the highest epoch where it is a validator, // then it is no longer a validator, and finally it is func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} @@ -299,7 +300,7 @@ func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { network.addNode(validator.NodeID[:]) // The non-validator node syncs the accepted blocks and then contributes to the next blocks - onOffValidator := newBLSMapping(2) + onOffValidator := newNodeMapping(2) network.addNode(onOffValidator.NodeID[:]) // initiate an epoch change @@ -307,14 +308,12 @@ func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) - sealingBlock := network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + network.waitUntilSealingBlock(newValidatorSet.Nodes()) newValidatorSet = metadata.NodeBLSMappings{validator} pChain.setValidatorSetAt(20, newValidatorSet) pChain.advanceHeight(20) - sealingBlock = network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + network.waitUntilSealingBlock(newValidatorSet.Nodes()) // accept a new block to ensure both nodes are still syncing the chain network.acceptNewBlock() @@ -324,12 +323,11 @@ func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { pChain.setValidatorSetAt(30, newValidatorSet) pChain.advanceHeight(30) - sealingBlock = network.waitUntilSealingBlock() - assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet.NodeIDs(), newValidatorSet.NodeIDs()) + network.waitUntilSealingBlock(newValidatorSet.Nodes()) } func TestInstanceDoubleStartFails(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisSet) @@ -341,14 +339,14 @@ func TestInstanceDoubleStartFails(t *testing.T) { // TestNonValidatorSkipsMSMVerification proves that a non-validator does not use the MSM to // verify blocks: it commits a finalized block whose state machine transition is invalid. func TestNonValidatorSkipsMSMVerification(t *testing.T) { - validator := newBLSMapping(1) + validator := newNodeMapping(1) genesisValidatorSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisValidatorSet) // The non-validator holds genesis plus epoch 1's defining block, and lives on a network // of its own, so the replication response below is the only way it can learn a block. storage, parent := newChainStorage(t, genesisValidatorSet) - nonValidator := newBLSMapping(2) + nonValidator := newNodeMapping(2) nonValidatorNode := newNetwork(t, pChain).addNodeWithStorage(nonValidator.NodeID[:], storage) // A block whose only defect is its state machine transition: its timestamp precedes its @@ -424,8 +422,8 @@ func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Two validators, so a quorum is both of them: with its peer absent, the lagging // validator can neither build a block nor empty notarize a round on its own. - lagging := newBLSMapping(1) - peer := newBLSMapping(2) + lagging := newNodeMapping(1) + peer := newNodeMapping(2) validators := metadata.NodeBLSMappings{lagging, peer} pChain := newTestPChain(validators) From 4d6dd6fb0e6e271819b212aec63859aef78a922f Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 2 Sep 2026 18:59:21 -0400 Subject: [PATCH 18/24] added check --- instance_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/instance_test.go b/instance_test.go index 87922dfe..5e810abd 100644 --- a/instance_test.go +++ b/instance_test.go @@ -189,7 +189,10 @@ func TestValidatorValidatorSetDecreased(t *testing.T) { pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) - network.waitUntilSealingBlock(newValidatorSet.Nodes()) + sealing := network.waitUntilSealingBlock(newValidatorSet.Nodes()) + + block, _ = network.acceptNewBlock() + require.Equal(t, sealing.BlockHeader().Seq+1, block.BlockHeader().Seq) } // TestInstanceOfflineDuringTransition asserts an epoch transition completes while a From 42e5881b65e13b2bd482df1fd11b875fac57fa37 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 13:52:55 -0400 Subject: [PATCH 19/24] queue instead of spawn go-routine --- instance_helpers_test.go | 84 ++++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index efb1a0f3..ec62ee95 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -395,42 +395,101 @@ func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRou return qr } +type inflightMessage struct { + to *Instance + from common.NodeID + msg *common.Message +} + type instanceComm struct { c *network // id is the node this comm belongs to, reported as the sender of every message it sends. id common.NodeID + + queue chan inflightMessage + + // closed is shut by stop, releasing run and any pending enqueue. + closed chan struct{} + closeOnce sync.Once + wg sync.WaitGroup } +const maxInFlightMessages = 10000 + func newInstanceComm(c *network, id common.NodeID) *instanceComm { - return &instanceComm{c: c, id: id} + return &instanceComm{ + c: c, + id: id, + queue: make(chan inflightMessage, maxInFlightMessages), + closed: make(chan struct{}), + } +} + +// start launches the delivery loop. It must be called before the comm sends anything. +func (i *instanceComm) start() { + i.wg.Add(1) + i.wg.Go(func() { + defer i.wg.Done() + i.run() + }) +} + +// run sends queued messages one at a time until stop is called. +func (i *instanceComm) run() { + for { + select { + case <-i.closed: + return + case m := <-i.queue: + require.NoError(i.c.t, m.to.HandleMessage(m.msg, m.from)) + } + } +} + +// stop stops message sending and waits for the in-flight message to finish. +func (i *instanceComm) stop() { + i.closeOnce.Do(func() { close(i.closed) }) + i.wg.Wait() +} + +// enqueue adds a message to be sent +func (i *instanceComm) enqueue(m inflightMessage) { + select { + case i.queue <- m: + default: + i.c.t.Errorf("node %x dropped a message, queue is full at %d", i.id, maxInFlightMessages) + } } func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { - // loop through all nodes in chain directly send to handle message directly via but do it in a separate go routine for _, n := range c.c.nodesSnapshot() { if !bytes.Equal(n.id, destination) { continue } - go func(dst *Instance) { - require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", destination) - require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) - }(n.inst) + require.NotNil(c.c.t, n.inst, "node %x was sent a message before it was created", destination) + c.enqueue(inflightMessage{ + to: n.inst, + from: c.id, + msg: translateOutgoingToIncomingMessage(c.c.t, msg), + }) return } } func (c *instanceComm) Broadcast(msg *common.Message) { - // send to every node in the chain but ourselves, each on its own go routine + // every node in the network but ourselves, each with its own re-parsed copy for _, n := range c.c.nodesSnapshot() { if bytes.Equal(n.id, c.id) { continue } - go func(dst *Instance) { - require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", n.id) - require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) - }(n.inst) + require.NotNil(c.c.t, n.inst, "node %x was sent a message before it was created", n.id) + c.enqueue(inflightMessage{ + to: n.inst, + from: c.id, + msg: translateOutgoingToIncomingMessage(c.c.t, msg), + }) } } @@ -703,6 +762,9 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { ctx, cancel := context.WithCancel(context.Background()) n.t.Cleanup(cancel) + comm.start() + n.t.Cleanup(comm.stop) + require.NoError(n.t, node.inst.Start(ctx)) n.t.Cleanup(node.inst.Stop) From 1f3d9a9c11181cd29efb042580a6d0f30acaa31d Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 14:11:07 -0400 Subject: [PATCH 20/24] remove mock storage --- adapters_test.go | 8 +-- instance_helpers_test.go | 102 +++++++++------------------------------ instance_test.go | 10 ++-- 3 files changed, 33 insertions(+), 87 deletions(-) diff --git a/adapters_test.go b/adapters_test.go index fe5d4194..ee11f79f 100644 --- a/adapters_test.go +++ b/adapters_test.go @@ -38,7 +38,7 @@ func newTestParsedBlock(num uint64, payload string) *ParsedBlock { // and a verified but not yet indexed block at seq 5. A zero digest matches on // seq alone, a non-zero digest must match the block's digest exactly. func TestCachedStorageRetrieve(t *testing.T) { - cs := NewCachedStorage(NewMockStorage(t, &testInnerBlockDeserializer{})) + cs := NewCachedStorage(newTestStorage()) indexedBlock := newTestParsedBlock(0, "indexed") require.NoError(t, cs.Index(t.Context(), indexedBlock, common.Finalization{})) @@ -101,7 +101,7 @@ func TestCachedStorageRetrieve(t *testing.T) { return } require.NoError(t, err) - require.Equal(t, tt.wantBlock, got) + require.Equal(t, common.Digest(tt.wantBlock.Digest()), got.BlockHeader().Digest) if tt.wantBlock == verifiedBlock { require.Nil(t, fin) } @@ -113,7 +113,7 @@ func TestCachedStorageRetrieve(t *testing.T) { // a zero-digest Retrieve of that seq returns the finalized block with its // finalization, even when a verified fork at the same seq was cached. func TestCachedStorageIndexEvictsSameSeqFork(t *testing.T) { - cs := NewCachedStorage(NewMockStorage(t, &testInnerBlockDeserializer{})) + cs := NewCachedStorage(newTestStorage()) require.NoError(t, cs.Index(t.Context(), newTestParsedBlock(0, "genesis"), common.Finalization{})) equivocatedBlock := &cachedBlock{ @@ -180,7 +180,7 @@ func TestCachedStoragePopulatedByWal(t *testing.T) { // own proposal is inserted into the CachedStorage, retrievable by seq and digest before // it is finalized and indexed. func TestCachedStoragePopulatedBySelfBuiltBlock(t *testing.T) { - storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) + storage := newTestStorageWithGenesis(t) cs := NewCachedStorage(storage) msm, err := metadata.NewStateMachine(&metadata.Config{ diff --git a/instance_helpers_test.go b/instance_helpers_test.go index ec62ee95..95841b24 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -224,97 +224,42 @@ func (c *testCryptoOps) DeserializeQuorumCertificate(bytes []byte) (common.Quoru return testutil.TestQC(qc), nil } -type MockStorage struct { - t *testing.T +// testStorage serves reads as independent StateMachineBlock copies, so tests never touch +// the instance's live block objects, whose canoto caches the instance keeps mutating. +type testStorage struct { *testutil.InMemStorage - bd *testInnerBlockDeserializer - - blocksLock sync.Mutex - blocks map[uint64]storedBlock } -type storedBlock struct { - rawBlock []byte - fin common.Finalization -} - -func NewMockStorage(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { - return &MockStorage{ - t: t, - InMemStorage: testutil.NewInMemStorage(), - blocks: make(map[uint64]storedBlock), - bd: bd, - } +func newTestStorage() *testStorage { + return &testStorage{InMemStorage: testutil.NewInMemStorage()} } -func NewMockStorageWithGenesis(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { - s := &MockStorage{ - t: t, - InMemStorage: testutil.NewInMemStorage(), - blocks: make(map[uint64]storedBlock), - bd: bd, - } - +func newTestStorageWithGenesis(t *testing.T) *testStorage { + s := newTestStorage() genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} require.NoError(t, s.Index(context.Background(), genesis, common.Finalization{})) return s } -func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { - // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. - encoded := block.Bytes() - seq := m.NumBlocks() - m.blocksLock.Lock() - m.blocks[seq] = storedBlock{rawBlock: encoded, fin: certificate} - m.blocksLock.Unlock() - return m.InMemStorage.Index(ctx, block, certificate) -} - -func (m *MockStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { - _, f, err := m.Retrieve(seq) +func (m *testStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { + block, fin, err := m.Retrieve(seq) if err != nil { return metadata.StateMachineBlock{}, nil, err } - sb, ok := m.blockAt(seq) + parsed, ok := block.(*ParsedBlock) if !ok { - return metadata.StateMachineBlock{}, nil, fmt.Errorf("no snapshot for seq %d", seq) - } - return sb, &f, nil -} - -// blockAt reconstructs an independent copy of the block at seq from its -// stored bytes. Test-only readers use it instead of GetBlock so they never touch -// the instance's live block objects (whose canoto digest cache the instance keeps -// mutating). -func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { - m.blocksLock.Lock() - sb, ok := m.blocks[seq] - m.blocksLock.Unlock() - if !ok { - return metadata.StateMachineBlock{}, false - } - return m.parseStored(sb.rawBlock), true -} - -func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { - raw := &metadata.RawBlock{} - require.NoError(m.t, raw.UnmarshalCanoto(encoded)) - var inner avalanchego.VMBlock - if len(raw.InnerBlockBytes) > 0 { - parsed, err := m.bd.ParseBlock(context.Background(), raw.InnerBlockBytes) - require.NoError(m.t, err) - inner = parsed + return metadata.StateMachineBlock{}, nil, fmt.Errorf("expected *ParsedBlock at seq %d, got %T", seq, block) } - return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} + return parsed.Clone(), &fin, nil } // newChainStorage builds and indexes the minimum chain a node can start from: genesis plus // epoch 1's defining block, which carries the descriptor naming the epoch's validator set. // It returns the storage and the epoch-defining block at its tip. -func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*MockStorage, metadata.StateMachineBlock) { - storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) - genesis, ok := storage.blockAt(0) - require.True(t, ok) +func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*testStorage, metadata.StateMachineBlock) { + storage := newTestStorageWithGenesis(t) + genesis, _, err := storage.GetBlock(0) + require.NoError(t, err) epochBlock := metadata.StateMachineBlock{ InnerBlock: &testInnerBlock{Height_: 1, TS: epochBlockTime, Payload: []byte("epoch")}, @@ -595,11 +540,12 @@ func (s *pendingBlockSignal) consume(ctx context.Context) bool { // blockBuilderVM builds an inner block only when the test arms a pending block, so the // chain grows one block per index call. type blockBuilderVM struct { - storage *MockStorage + storage *testStorage pending *pendingBlockSignal + bd testInnerBlockDeserializer } -func newBlockBuilderVM(storage *MockStorage, pending *pendingBlockSignal) *blockBuilderVM { +func newBlockBuilderVM(storage *testStorage, pending *pendingBlockSignal) *blockBuilderVM { return &blockBuilderVM{storage: storage, pending: pending} } @@ -618,7 +564,7 @@ func (vm *blockBuilderVM) BuildBlock(ctx context.Context, pChainHeight uint64) ( } func (vm *blockBuilderVM) ParseBlock(ctx context.Context, bytes []byte) (avalanchego.VMBlock, error) { - return vm.storage.bd.ParseBlock(ctx, bytes) + return vm.bd.ParseBlock(ctx, bytes) } // WaitForPendingBlock returns while a block is pending, or when ctx is cancelled. @@ -637,7 +583,7 @@ type node struct { id common.NodeID vm *blockBuilderVM inst *Instance - storage *MockStorage + storage *testStorage comm *instanceComm wals *walCreator } @@ -695,7 +641,7 @@ func newNetwork(t *testing.T, pChain *testPlatformChain) *network { // nodeConfig holds optional overrides for a node added to the network. type nodeConfig struct { // storage the node starts from; defaults to a fresh storage holding only genesis. - storage *MockStorage + storage *testStorage // wals are pre-existing WALs the instance restores on start. wals []wal.DeletableWAL } @@ -708,7 +654,7 @@ func (n *network) addNode(id common.NodeID) *node { } // addNodeWithStorage adds a node that starts from the given storage. -func (n *network) addNodeWithStorage(id common.NodeID, storage *MockStorage) *node { +func (n *network) addNodeWithStorage(id common.NodeID, storage *testStorage) *node { return n.addNodeWithConfig(id, nodeConfig{storage: storage}) } @@ -716,7 +662,7 @@ func (n *network) addNodeWithStorage(id common.NodeID, storage *MockStorage) *no func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { storage := cfg.storage if storage == nil { - storage = NewMockStorageWithGenesis(n.t, &testInnerBlockDeserializer{}) + storage = newTestStorageWithGenesis(n.t) } // ensure a unique id; snapshot because nodes may be added concurrently diff --git a/instance_test.go b/instance_test.go index 5e810abd..c3e2ca92 100644 --- a/instance_test.go +++ b/instance_test.go @@ -57,7 +57,7 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { common.SortNodes(nodes) require.NotEqual(t, common.NodeID(ourValidator.NodeID[:]), simplex.LeaderForRound(nodes.NodeIDs(), 1)) - storage := NewMockStorageWithGenesis(t, &testInnerBlockDeserializer{}) + storage := newTestStorageWithGenesis(t) vm := newBlockBuilderVM(storage, newPendingBlockSignal()) recorder := &emptyVoteRecorder{got: make(chan struct{}, 1)} @@ -373,8 +373,8 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { // It commits the block its state machine would have rejected. storage.WaitForBlockCommit(2) - committed, ok := storage.blockAt(2) - require.True(t, ok) + committed, _, err := storage.GetBlock(2) + require.NoError(t, err) require.Equal(t, invalid.Digest(), committed.Digest()) } @@ -416,8 +416,8 @@ func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { requireReplicated: func(t *testing.T, laggingNode *node, block metadata.StateMachineBlock) { seq := block.Metadata.SimplexProtocolMetadata.Seq laggingNode.storage.WaitForBlockCommit(seq) - committed, ok := laggingNode.storage.blockAt(seq) - require.True(t, ok) + committed, _, err := laggingNode.storage.GetBlock(seq) + require.NoError(t, err) require.Equal(t, block.Digest(), committed.Digest()) }, }, From 854e2155ff0b2637e62ba591d09636154a9de6b8 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 16:24:53 -0400 Subject: [PATCH 21/24] start all instance tests from zero block --- instance_helpers_test.go | 127 ++++++++++++++++++++------------------- instance_test.go | 71 +++++++++++----------- 2 files changed, 101 insertions(+), 97 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 95841b24..ab2a32cf 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -253,34 +253,6 @@ func (m *testStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common. return parsed.Clone(), &fin, nil } -// newChainStorage builds and indexes the minimum chain a node can start from: genesis plus -// epoch 1's defining block, which carries the descriptor naming the epoch's validator set. -// It returns the storage and the epoch-defining block at its tip. -func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*testStorage, metadata.StateMachineBlock) { - storage := newTestStorageWithGenesis(t) - genesis, _, err := storage.GetBlock(0) - require.NoError(t, err) - - epochBlock := metadata.StateMachineBlock{ - InnerBlock: &testInnerBlock{Height_: 1, TS: epochBlockTime, Payload: []byte("epoch")}, - Metadata: metadata.StateMachineMetadata{ - Timestamp: uint64(epochBlockTime.UnixMilli()), - SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 1, Seq: 1, Prev: common.Digest(genesis.Digest())}, - SimplexEpochInfo: metadata.SimplexEpochInfo{ - EpochNumber: 1, - BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ - AggregatedMembership: metadata.AggregatedMembership{Members: validators}, - }, - }, - }, - } - - block := &ParsedBlock{StateMachineBlock: epochBlock.Clone()} - finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(validators)}, block, validators.NodeIDs()) - require.NoError(t, storage.Index(context.Background(), block, finalization)) - return storage, epochBlock -} - type walCreator struct { t *testing.T @@ -347,7 +319,7 @@ type inflightMessage struct { } type instanceComm struct { - c *network + n *network // id is the node this comm belongs to, reported as the sender of every message it sends. id common.NodeID @@ -361,16 +333,16 @@ type instanceComm struct { const maxInFlightMessages = 10000 -func newInstanceComm(c *network, id common.NodeID) *instanceComm { +func newInstanceComm(n *network, id common.NodeID) *instanceComm { return &instanceComm{ - c: c, + n: n, id: id, queue: make(chan inflightMessage, maxInFlightMessages), closed: make(chan struct{}), } } -// start launches the delivery loop. It must be called before the comm sends anything. +// start allows messages to be processed func (i *instanceComm) start() { i.wg.Add(1) i.wg.Go(func() { @@ -386,7 +358,7 @@ func (i *instanceComm) run() { case <-i.closed: return case m := <-i.queue: - require.NoError(i.c.t, m.to.HandleMessage(m.msg, m.from)) + require.NoError(i.n.t, m.to.HandleMessage(m.msg, m.from)) } } } @@ -402,21 +374,21 @@ func (i *instanceComm) enqueue(m inflightMessage) { select { case i.queue <- m: default: - i.c.t.Errorf("node %x dropped a message, queue is full at %d", i.id, maxInFlightMessages) + i.n.t.Errorf("node %x dropped a message, queue is full at %d", i.id, maxInFlightMessages) } } func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { - for _, n := range c.c.nodesSnapshot() { + for _, n := range c.n.nodesSnapshot() { if !bytes.Equal(n.id, destination) { continue } - require.NotNil(c.c.t, n.inst, "node %x was sent a message before it was created", destination) + require.NotNil(c.n.t, n.inst, "node %x was sent a message before it was created", destination) c.enqueue(inflightMessage{ to: n.inst, from: c.id, - msg: translateOutgoingToIncomingMessage(c.c.t, msg), + msg: translateOutgoingToIncomingMessage(c.n.t, msg), }) return } @@ -424,16 +396,16 @@ func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { func (c *instanceComm) Broadcast(msg *common.Message) { // every node in the network but ourselves, each with its own re-parsed copy - for _, n := range c.c.nodesSnapshot() { + for _, n := range c.n.nodesSnapshot() { if bytes.Equal(n.id, c.id) { continue } - require.NotNil(c.c.t, n.inst, "node %x was sent a message before it was created", n.id) + require.NotNil(c.n.t, n.inst, "node %x was sent a message before it was created", n.id) c.enqueue(inflightMessage{ to: n.inst, from: c.id, - msg: translateOutgoingToIncomingMessage(c.c.t, msg), + msg: translateOutgoingToIncomingMessage(c.n.t, msg), }) } } @@ -580,6 +552,7 @@ func noopICMTransition(_ metadata.ICMEpochInput) metadata.ICMEpochInfo { type node struct { t *testing.T + net *network id common.NodeID vm *blockBuilderVM inst *Instance @@ -588,6 +561,30 @@ type node struct { wals *walCreator } +// start starts the node's instance and blocks until it commits the network's latest tip. +func (n *node) start() *node { + ctx, cancel := context.WithCancel(context.Background()) + n.t.Cleanup(cancel) + + n.comm.start() + require.NoError(n.t, n.inst.Start(ctx)) + n.t.Cleanup(n.stop) + + return n +} + +// stop stops the instance, then drains the messages the node was sending. +func (n *node) stop() { + n.inst.Stop() + n.comm.stop() +} + +// sync syncs a node by waiting for the commit of the latest sequence. +func (n *node) sync() *node { + n.storage.WaitForBlockCommit(n.net.seq - 1) + return n +} + // role reports whether the node is running a validator rather than a non-validator. func (n *node) role() (isValidator bool) { n.inst.lock.Lock() @@ -596,6 +593,8 @@ func (n *node) role() (isValidator bool) { return n.inst.e != nil } +const firstEverEpoch uint64 = 1 + type network struct { t *testing.T @@ -623,7 +622,7 @@ func newNetwork(t *testing.T, pChain *testPlatformChain) *network { validatorSets := make(map[uint64]common.Nodes) genesisNodes := pChain.GenesisValidatorSet().Nodes() common.SortNodes(genesisNodes) - validatorSets[1] = genesisNodes + validatorSets[firstEverEpoch] = genesisNodes return &network{ t: t, @@ -634,7 +633,7 @@ func newNetwork(t *testing.T, pChain *testPlatformChain) *network { // Genesis at seq 0. Then first simplex block is built automatically // without a build block notification seq: 2, - epoch: 1, + epoch: firstEverEpoch, } } @@ -646,19 +645,12 @@ type nodeConfig struct { wals []wal.DeletableWAL } -// addNode adds a node to the network and blocks until it catches up with the latest tip +// addNode creates a node in the network without starting its instance. Call node.start to run it. func (n *network) addNode(id common.NodeID) *node { - node := n.addNodeWithConfig(id, nodeConfig{}) - node.storage.WaitForBlockCommit(n.seq - 1) - return node -} - -// addNodeWithStorage adds a node that starts from the given storage. -func (n *network) addNodeWithStorage(id common.NodeID, storage *testStorage) *node { - return n.addNodeWithConfig(id, nodeConfig{storage: storage}) + return n.addNodeWithConfig(id, nodeConfig{}) } -// addNodeWithConfig adds a node built from the given config. +// addNodeWithConfig creates a node built from the given config, leaving it stopped. func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { storage := cfg.storage if storage == nil { @@ -693,6 +685,7 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { node := node{ t: n.t, + net: n, id: id, storage: storage, comm: comm, @@ -705,16 +698,7 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { n.nodes = append(n.nodes, node) n.lock.Unlock() - ctx, cancel := context.WithCancel(context.Background()) - n.t.Cleanup(cancel) - - comm.start() - n.t.Cleanup(comm.stop) - - require.NoError(n.t, node.inst.Start(ctx)) - n.t.Cleanup(node.inst.Stop) - - instance.Config.Logger.Debug("Added a node to the test network", zap.Uint64("Seq", n.seq), zap.Uint64("num block", node.storage.NumBlocks())) + instance.Config.Logger.Debug("Created a node in the test network", zap.Uint64("Seq", n.seq), zap.Uint64("num block", node.storage.NumBlocks())) return &node } @@ -735,6 +719,25 @@ func (n *network) waitUntilValidatorsReady() { } } +// sync blocks until every node has committed the block at the network's tip, requiring +// that they all committed the same block and that it sits at n.seq - 1. +func (n *network) sync() { + tip := n.seq - 1 + + var block common.VerifiedBlock + for _, node := range n.nodesSnapshot() { + committedBlock := node.storage.WaitForBlockCommit(tip) + if block == nil { + block = committedBlock + continue + } + + require.Equal(n.t, block.Bytes(), committedBlock.Bytes()) + } + + require.Equal(n.t, tip, block.BlockHeader().Seq) +} + // acceptNewBlock blocks until every node has accepted a newly indexed block. func (n *network) acceptNewBlock() (common.VerifiedBlock, common.Finalization) { _, ok := n.validatorSets[n.epoch] diff --git a/instance_test.go b/instance_test.go index c3e2ca92..44389302 100644 --- a/instance_test.go +++ b/instance_test.go @@ -95,12 +95,12 @@ func TestNonValidatorSyncs(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]) + network.addNode(validator.NodeID[:]).start() network.acceptNewBlock() nonValidator := newNodeMapping(2) - network.addNode(nonValidator.NodeID[:]) + network.addNode(nonValidator.NodeID[:]).start() network.acceptNewBlock() } @@ -165,27 +165,24 @@ func TestValidator_ValidatorSetNotChanged(t *testing.T) { // TestValidatorValidatorSetDecreased tests that an epoch with two validators // is reduced to one, when the pchain height notes a validator is leaving. func TestValidatorValidatorSetDecreased(t *testing.T) { - validator := newNodeMapping(1) - leavingValidator := newNodeMapping(2) + validatorMapping := newNodeMapping(1) + leavingValidatorMapping := newNodeMapping(2) - genesisSet := []metadata.NodeBLSMapping{validator, leavingValidator} + genesisSet := []metadata.NodeBLSMapping{validatorMapping, leavingValidatorMapping} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - validatorStorage, _ := newChainStorage(t, genesisSet) - leavingValidatorStorage, _ := newChainStorage(t, genesisSet) - // starting from storage holding the first simplex block avoids // blocking on a sync that needs a quorum online - network.addNodeWithStorage(validator.NodeID[:], validatorStorage) - network.addNodeWithStorage(leavingValidator.NodeID[:], leavingValidatorStorage) + network.addNode(validatorMapping.NodeID[:]).start() + network.addNode(leavingValidatorMapping.NodeID[:]).start().sync() block, _ := network.acceptNewBlock() - require.Equal(t, uint64(2), block.BlockHeader().Round) + require.Equal(t, uint64(2), block.BlockHeader().Seq) // initiate an epoch change - newValidatorSet := metadata.NodeBLSMappings{validator} + newValidatorSet := metadata.NodeBLSMappings{validatorMapping} pChain.setValidatorSetAt(10, newValidatorSet) pChain.advanceHeight(10) @@ -209,13 +206,10 @@ func TestInstanceOfflineDuringTransition(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - v1Storage, _ := newChainStorage(t, genesisSet) - v2Storage, _ := newChainStorage(t, genesisSet) - v3Storage, _ := newChainStorage(t, genesisSet) - - node1 := network.addNodeWithStorage(v1.NodeID[:], v1Storage) - network.addNodeWithStorage(v2.NodeID[:], v2Storage) - network.addNodeWithStorage(v3.NodeID[:], v3Storage) + node1 := network.addNode(v1.NodeID[:]).start() + network.addNode(v2.NodeID[:]).start() + network.addNode(v3.NodeID[:]).start() + network.sync() // using seq should be fine since we have no empty blocks leader := simplex.LeaderForRound(pChain.GenesisValidatorSet().NodeIDs(), network.seq) @@ -245,14 +239,14 @@ func TestNonValidatorStaysNonValidator(t *testing.T) { genesisSet := []metadata.NodeBLSMapping{validator1} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator1.NodeID[:]) + network.addNode(validator1.NodeID[:]).start() validator2 := newNodeMapping(2) validator3 := newNodeMapping(3) validator4 := newNodeMapping(4) - network.addNode(validator2.NodeID[:]) - network.addNode(validator3.NodeID[:]) - network.addNode(validator4.NodeID[:]) + network.addNode(validator2.NodeID[:]).start().sync() + network.addNode(validator3.NodeID[:]).start().sync() + network.addNode(validator4.NodeID[:]).start().sync() // we should have a quorum without the target node to create this epoch change targetNodeNotInMiddleEpoch := metadata.NodeBLSMappings{validator1, validator2, validator3} @@ -278,7 +272,7 @@ func TestNonValidatorStaysNonValidator(t *testing.T) { network.waitUntilSealingBlock(targetNodeInHighestEpoch.Nodes()) // the target node should join now - network.addNode(targetNode.NodeID[:]) + network.addNode(targetNode.NodeID[:]).start().sync() // the target node must sign within a bounded number of blocks, otherwise it never rejoined const maxBlocksUntilTargetSigns = 10 @@ -300,11 +294,11 @@ func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]) + network.addNode(validator.NodeID[:]).start().sync() // The non-validator node syncs the accepted blocks and then contributes to the next blocks onOffValidator := newNodeMapping(2) - network.addNode(onOffValidator.NodeID[:]) + network.addNode(onOffValidator.NodeID[:]).start().sync() // initiate an epoch change newValidatorSet := metadata.NodeBLSMappings{validator, onOffValidator} @@ -335,7 +329,7 @@ func TestInstanceDoubleStartFails(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - node := network.addNode(validator.NodeID[:]) + node := network.addNode(validator.NodeID[:]).start() require.ErrorIs(t, node.inst.Start(t.Context()), errAlreadyStarted) } @@ -346,11 +340,13 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { genesisValidatorSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisValidatorSet) - // The non-validator holds genesis plus epoch 1's defining block, and lives on a network - // of its own, so the replication response below is the only way it can learn a block. - storage, parent := newChainStorage(t, genesisValidatorSet) nonValidator := newNodeMapping(2) - nonValidatorNode := newNetwork(t, pChain).addNodeWithStorage(nonValidator.NodeID[:], storage) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]).start().sync() + nonValidatorNode := network.addNode(nonValidator.NodeID[:]).start().sync() + + parent, _, err := nonValidatorNode.storage.GetBlock(1) + require.NoError(t, err) // A block whose only defect is its state machine transition: its timestamp precedes its // parent's. @@ -372,8 +368,8 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { }, validator.NodeID[:])) // It commits the block its state machine would have rejected. - storage.WaitForBlockCommit(2) - committed, _, err := storage.GetBlock(2) + nonValidatorNode.storage.WaitForBlockCommit(2) + committed, _, err := nonValidatorNode.storage.GetBlock(2) require.NoError(t, err) require.Equal(t, invalid.Digest(), committed.Digest()) } @@ -433,9 +429,14 @@ func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { // The lagging validator holds genesis plus epoch 1's defining block, and lives on // a network of its own, so the replication response below is the only way it can // learn the block at the round it sits on. - storage, parent := newChainStorage(t, validators) - laggingNode := newNetwork(t, pChain).addNodeWithStorage(lagging.NodeID[:], storage) + network := newNetwork(t, pChain) + network.addNode(peer.NodeID[:]).start() + laggingNode := network.addNode(lagging.NodeID[:]).start() + network.sync() + + parent, _, err := laggingNode.storage.GetBlock(1) + require.NoError(t, err) // A block whose only defect is its state machine transition: its timestamp // precedes its parent's. invalid := metadata.StateMachineBlock{ From 81af48c18ac0dceffdaf767860721ff0acf02d26 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 16:54:08 -0400 Subject: [PATCH 22/24] lint --- instance_helpers_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index ab2a32cf..71203b06 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -61,9 +61,6 @@ var ( genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} ) -// epochBlockTime fixes the timestamp of the epoch-defining block -// this ensures a consistent block digest -var epochBlockTime = genesisBlock.TS.Add(time.Millisecond) var paramConfig = ParameterConfig{ MaxNetworkDelay: 200 * time.Millisecond, MaxRoundWindow: 100, From 64185857c881f00c110930b2b5d9097f3649b432 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 17:29:16 -0400 Subject: [PATCH 23/24] oops, missed a test --- instance_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instance_test.go b/instance_test.go index 44389302..f91e2380 100644 --- a/instance_test.go +++ b/instance_test.go @@ -22,10 +22,10 @@ func TestValidatorIndexes(t *testing.T) { genesisSet := []metadata.NodeBLSMapping{validator} pChain := newTestPChain(genesisSet) - chain := newNetwork(t, pChain) - chain.addNode(validator.NodeID[:]) + network := newNetwork(t, pChain) + network.addNode(validator.NodeID[:]).start().sync() - chain.acceptNewBlock() + network.acceptNewBlock() } // emptyVoteRecorder signals the first empty vote broadcast and drops all other traffic. From 48563fc2d5da03fbc2ca41dbca66af109d2a6be2 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 3 Sep 2026 18:26:29 -0400 Subject: [PATCH 24/24] remove start method, only keep sync --- instance_helpers_test.go | 24 +++++++++------------ instance_test.go | 46 ++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 71203b06..c9f81cb6 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -558,18 +558,6 @@ type node struct { wals *walCreator } -// start starts the node's instance and blocks until it commits the network's latest tip. -func (n *node) start() *node { - ctx, cancel := context.WithCancel(context.Background()) - n.t.Cleanup(cancel) - - n.comm.start() - require.NoError(n.t, n.inst.Start(ctx)) - n.t.Cleanup(n.stop) - - return n -} - // stop stops the instance, then drains the messages the node was sending. func (n *node) stop() { n.inst.Stop() @@ -642,12 +630,12 @@ type nodeConfig struct { wals []wal.DeletableWAL } -// addNode creates a node in the network without starting its instance. Call node.start to run it. +// addNode creates and starts a node in the network. func (n *network) addNode(id common.NodeID) *node { return n.addNodeWithConfig(id, nodeConfig{}) } -// addNodeWithConfig creates a node built from the given config, leaving it stopped. +// addNodeWithConfig creates and starts a node built from the given config. func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { storage := cfg.storage if storage == nil { @@ -696,6 +684,14 @@ func (n *network) addNodeWithConfig(id common.NodeID, cfg nodeConfig) *node { n.lock.Unlock() instance.Config.Logger.Debug("Created a node in the test network", zap.Uint64("Seq", n.seq), zap.Uint64("num block", node.storage.NumBlocks())) + + ctx, cancel := context.WithCancel(context.Background()) + n.t.Cleanup(cancel) + + comm.start() + require.NoError(n.t, instance.Start(ctx)) + n.t.Cleanup(node.stop) + return &node } diff --git a/instance_test.go b/instance_test.go index f91e2380..7336ee88 100644 --- a/instance_test.go +++ b/instance_test.go @@ -23,7 +23,7 @@ func TestValidatorIndexes(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]).start().sync() + network.addNode(validator.NodeID[:]).sync() network.acceptNewBlock() } @@ -95,12 +95,12 @@ func TestNonValidatorSyncs(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]).start() + network.addNode(validator.NodeID[:]) network.acceptNewBlock() nonValidator := newNodeMapping(2) - network.addNode(nonValidator.NodeID[:]).start() + network.addNode(nonValidator.NodeID[:]) network.acceptNewBlock() } @@ -114,13 +114,13 @@ func TestNonValidatorBecomesValidator(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]) + network.addNode(validator.NodeID[:]).sync() network.acceptNewBlock() // The non-validator node syncs the accepted blocks and then contributes to the next blocks upcomingValidator := newNodeMapping(2) - network.addNode(upcomingValidator.NodeID[:]) + network.addNode(upcomingValidator.NodeID[:]).sync() // initiate an epoch change newValidatorSet := metadata.NodeBLSMappings{validator, upcomingValidator} @@ -143,7 +143,7 @@ func TestValidator_ValidatorSetNotChanged(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - node := network.addNode(validator.NodeID[:]) + node := network.addNode(validator.NodeID[:]).sync() firstBlock, _ := network.acceptNewBlock() @@ -175,8 +175,8 @@ func TestValidatorValidatorSetDecreased(t *testing.T) { // starting from storage holding the first simplex block avoids // blocking on a sync that needs a quorum online - network.addNode(validatorMapping.NodeID[:]).start() - network.addNode(leavingValidatorMapping.NodeID[:]).start().sync() + network.addNode(validatorMapping.NodeID[:]) + network.addNode(leavingValidatorMapping.NodeID[:]).sync() block, _ := network.acceptNewBlock() require.Equal(t, uint64(2), block.BlockHeader().Seq) @@ -206,9 +206,9 @@ func TestInstanceOfflineDuringTransition(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - node1 := network.addNode(v1.NodeID[:]).start() - network.addNode(v2.NodeID[:]).start() - network.addNode(v3.NodeID[:]).start() + node1 := network.addNode(v1.NodeID[:]) + network.addNode(v2.NodeID[:]) + network.addNode(v3.NodeID[:]) network.sync() // using seq should be fine since we have no empty blocks @@ -239,14 +239,14 @@ func TestNonValidatorStaysNonValidator(t *testing.T) { genesisSet := []metadata.NodeBLSMapping{validator1} pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator1.NodeID[:]).start() + network.addNode(validator1.NodeID[:]) validator2 := newNodeMapping(2) validator3 := newNodeMapping(3) validator4 := newNodeMapping(4) - network.addNode(validator2.NodeID[:]).start().sync() - network.addNode(validator3.NodeID[:]).start().sync() - network.addNode(validator4.NodeID[:]).start().sync() + network.addNode(validator2.NodeID[:]).sync() + network.addNode(validator3.NodeID[:]).sync() + network.addNode(validator4.NodeID[:]).sync() // we should have a quorum without the target node to create this epoch change targetNodeNotInMiddleEpoch := metadata.NodeBLSMappings{validator1, validator2, validator3} @@ -272,7 +272,7 @@ func TestNonValidatorStaysNonValidator(t *testing.T) { network.waitUntilSealingBlock(targetNodeInHighestEpoch.Nodes()) // the target node should join now - network.addNode(targetNode.NodeID[:]).start().sync() + network.addNode(targetNode.NodeID[:]).sync() // the target node must sign within a bounded number of blocks, otherwise it never rejoined const maxBlocksUntilTargetSigns = 10 @@ -294,11 +294,11 @@ func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]).start().sync() + network.addNode(validator.NodeID[:]).sync() // The non-validator node syncs the accepted blocks and then contributes to the next blocks onOffValidator := newNodeMapping(2) - network.addNode(onOffValidator.NodeID[:]).start().sync() + network.addNode(onOffValidator.NodeID[:]).sync() // initiate an epoch change newValidatorSet := metadata.NodeBLSMappings{validator, onOffValidator} @@ -329,7 +329,7 @@ func TestInstanceDoubleStartFails(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - node := network.addNode(validator.NodeID[:]).start() + node := network.addNode(validator.NodeID[:]) require.ErrorIs(t, node.inst.Start(t.Context()), errAlreadyStarted) } @@ -342,8 +342,8 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { nonValidator := newNodeMapping(2) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]).start().sync() - nonValidatorNode := network.addNode(nonValidator.NodeID[:]).start().sync() + network.addNode(validator.NodeID[:]).sync() + nonValidatorNode := network.addNode(nonValidator.NodeID[:]).sync() parent, _, err := nonValidatorNode.storage.GetBlock(1) require.NoError(t, err) @@ -431,8 +431,8 @@ func TestValidatorSkipsMSMVerificationWhenReplicating(t *testing.T) { // learn the block at the round it sits on. network := newNetwork(t, pChain) - network.addNode(peer.NodeID[:]).start() - laggingNode := network.addNode(lagging.NodeID[:]).start() + network.addNode(peer.NodeID[:]) + laggingNode := network.addNode(lagging.NodeID[:]) network.sync() parent, _, err := laggingNode.storage.GetBlock(1)