Introduce a fake clock to F3 (#473)

* Introduce a fake clock to F3 Signed-off-by: Jakub Sztandera <[email protected]> * use mock clock in tests Signed-off-by: Jakub Sztandera <[email protected]> * Fix tests no idea why some of them are still slow-ish, they seem to send a lot of messages Signed-off-by: Jakub Sztandera <[email protected]> * more test fixes Signed-off-by: Jakub Sztandera <[email protected]> * Add docs Signed-off-by: Jakub Sztandera <[email protected]> * switch to filecoin-project/go-clock --------- Signed-off-by: Jakub Sztandera <[email protected]> Co-authored-by: Steven Allen <[email protected]>
filecoin-project · Jul 15, 2024 · 7ecbfc2 · 7ecbfc2
1 parent 253ee25
commit 7ecbfc2
Show file tree

Hide file tree

Showing 19 changed files with 151 additions and 115 deletions.
diff --git a/certexchange/polling/common.go b/certexchange/polling/common.go
@@ -1,9 +1,7 @@
 package polling
 
 import (
-	"github.com/benbjohnson/clock"
 	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("f3/certexchange")
-var clk clock.Clock = clock.New()
diff --git a/certexchange/polling/common_test.go b/certexchange/polling/common_test.go
@@ -9,20 +9,12 @@ import (
 	"github.com/filecoin-project/go-f3/sim"
 	"github.com/filecoin-project/go-f3/sim/signing"
 
-	"github.com/benbjohnson/clock"
 	"github.com/stretchr/testify/require"
 )
 
 // The network name used in tests.
 const TestNetworkName gpbft.NetworkName = "testnet"
 
-// The clock used in tests. Time doesn't pass in tests unless you add time to this clock.
-var MockClock = clock.NewMock()
-
-func init() {
-	clk = MockClock
-}
-
 func MakeCertificate(t *testing.T, rng *rand.Rand, tsg *sim.TipSetGenerator, backend signing.Backend, base *gpbft.TipSet, instance uint64, powerTable, nextPowerTable gpbft.PowerEntries) *certs.FinalityCertificate {
 	chainLen := rng.Intn(23) + 1
 	chain, err := gpbft.NewChain(*base)

diff --git a/certexchange/polling/peerTracker.go b/certexchange/polling/peerTracker.go
@@ -7,6 +7,7 @@ import (
 	"slices"
 	"time"
 
+	"github.com/filecoin-project/go-f3/internal/clock"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
@@ -65,9 +66,10 @@ type backoffRecord struct {
 	delayUntil int
 }
 
-func newPeerTracker() *peerTracker {
+func newPeerTracker(clk clock.Clock) *peerTracker {
 	return &peerTracker{
 		peers: make(map[peer.ID]*peerRecord),
+		clock: clk,
 	}
 }
 
@@ -76,6 +78,8 @@ type peerTracker struct {
 	active                     []peer.ID
 	backoff                    backoffHeap
 	lastHitRound, currentRound int
+
+	clock clock.Clock
 }
 
 func (r *peerRecord) Cmp(other *peerRecord) int {
@@ -198,7 +202,7 @@ func (r *peerRecord) hitRate() (float64, int) {
 func (t *peerTracker) getOrCreate(p peer.ID) *peerRecord {
 	r, ok := t.peers[p]
 	if !ok {
-		now := clk.Now()
+		now := t.clock.Now()
 		r = &peerRecord{id: p, lastSeen: now}
 		t.peers[p] = r
 	}
@@ -249,7 +253,7 @@ func (t *peerTracker) reactivate(p peer.ID) {
 }
 
 func (t *peerTracker) peerSeen(p peer.ID) {
-	now := clk.Now()
+	now := t.clock.Now()
 	if r, ok := t.peers[p]; !ok {
 		t.peers[p] = &peerRecord{id: p, state: peerActive, lastSeen: now}
 		t.active = append(t.active, p)

diff --git a/certexchange/polling/peerTracker_test.go b/certexchange/polling/peerTracker_test.go
@@ -3,6 +3,7 @@ package polling
 import (
 	"testing"
 
+	"github.com/filecoin-project/go-f3/internal/clock"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/test"
 	"github.com/stretchr/testify/require"
@@ -105,7 +106,7 @@ func TestPeerRecordExponentialBackoff(t *testing.T) {
 }
 
 func TestPeerTracker(t *testing.T) {
-	pt := newPeerTracker()
+	pt := newPeerTracker(clock.NewMock())
 
 	var peers []peer.ID
 	discoverPeers := func(count int) {

diff --git a/certexchange/polling/poller.go b/certexchange/polling/poller.go
@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/go-f3/certs"
 	"github.com/filecoin-project/go-f3/certstore"
 	"github.com/filecoin-project/go-f3/gpbft"
+	"github.com/filecoin-project/go-f3/internal/clock"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
@@ -20,6 +21,7 @@ type Poller struct {
 	SignatureVerifier gpbft.Verifier
 	PowerTable        gpbft.PowerEntries
 	NextInstance      uint64
+	clock             clock.Clock
 }
 
 // NewPoller constructs a new certificate poller and initializes it from the passed certificate store.
@@ -38,6 +40,7 @@ func NewPoller(ctx context.Context, client *certexchange.Client, store *certstor
 		SignatureVerifier: verifier,
 		NextInstance:      nextInstance,
 		PowerTable:        pt,
+		clock:             clock.GetClock(ctx),
 	}, nil
 }
 
@@ -99,13 +102,13 @@ func (p *Poller) Poll(ctx context.Context, peer peer.ID) (*PollResult, error) {
 			return nil, err
 		}
 
-		start := clk.Now()
+		start := p.clock.Now()
 		resp, ch, err := p.Request(ctx, peer, &certexchange.Request{
 			FirstInstance:     p.NextInstance,
 			Limit:             maxRequestLength,
 			IncludePowerTable: false,
 		})
-		res.Latency = clk.Since(start)
+		res.Latency = p.clock.Since(start)
 		if err != nil {
 			res.Status = PollFailed
 			res.Error = err

diff --git a/certexchange/polling/subscriber.go b/certexchange/polling/subscriber.go
@@ -11,6 +11,7 @@ import (
 	"github.com/filecoin-project/go-f3/certexchange"
 	"github.com/filecoin-project/go-f3/certstore"
 	"github.com/filecoin-project/go-f3/gpbft"
+	"github.com/filecoin-project/go-f3/internal/clock"
 )
 
 const maxRequestLength = 256
@@ -28,6 +29,7 @@ type Subscriber struct {
 	peerTracker *peerTracker
 	poller      *Poller
 	discoverCh  <-chan peer.ID
+	clock       clock.Clock
 
 	wg   sync.WaitGroup
 	stop context.CancelFunc
@@ -36,10 +38,11 @@ type Subscriber struct {
 func (s *Subscriber) Start(startCtx context.Context) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	s.stop = cancel
+	s.clock = clock.GetClock(startCtx)
 
 	var err error
 
-	s.peerTracker = newPeerTracker()
+	s.peerTracker = newPeerTracker(s.clock)
 	s.poller, err = NewPoller(startCtx, &s.Client, s.Store, s.SignatureVerifier)
 	if err != nil {
 		return err
@@ -81,7 +84,7 @@ func (s *Subscriber) Stop(stopCtx context.Context) error {
 }
 
 func (s *Subscriber) run(ctx context.Context) error {
-	timer := clk.Timer(s.InitialPollInterval)
+	timer := s.clock.Timer(s.InitialPollInterval)
 	defer timer.Stop()
 
 	predictor := newPredictor(
@@ -114,7 +117,7 @@ func (s *Subscriber) run(ctx context.Context) error {
 
 			nextInterval := predictor.update(progress)
 			nextPollTime := pollTime.Add(nextInterval)
-			delay := max(clk.Until(nextPollTime), 0)
+			delay := max(s.clock.Until(nextPollTime), 0)
 			log.Infof("predicted interval is %s (waiting %s)", nextInterval, delay)
 			timer.Reset(delay)
 		case <-ctx.Done():

diff --git a/certexchange/polling/subscriber_test.go b/certexchange/polling/subscriber_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/filecoin-project/go-f3/certexchange"
 	"github.com/filecoin-project/go-f3/certexchange/polling"
 	"github.com/filecoin-project/go-f3/certstore"
+	"github.com/filecoin-project/go-f3/internal/clock"
 	"github.com/filecoin-project/go-f3/sim/signing"
 
 	"github.com/ipfs/go-datastore"
@@ -25,6 +26,7 @@ func TestSubscriber(t *testing.T) {
 	cg := polling.MakeCertificates(t, rng, backend)
 
 	ctx, cancel := context.WithCancel(context.Background())
+	ctx, clk := clock.WithMockClock(ctx)
 	defer cancel()
 
 	mocknet := mocknetwork.New()
@@ -94,14 +96,14 @@ func TestSubscriber(t *testing.T) {
 			}
 		}
 
-		polling.MockClock.Add(waitTime)
+		clk.Add(waitTime)
 
 		require.Eventually(t, func() bool {
 			latest := clientCs.Latest()
 			if latest != nil && latest.GPBFTInstance == uint64(i-1) {
 				return true
 			}
-			polling.MockClock.WaitForAllTimers()
+			clk.WaitForAllTimers()
 			return false
 		}, 10*time.Second, time.Millisecond)
 

diff --git a/cmd/f3/manifest.go b/cmd/f3/manifest.go
@@ -183,7 +183,7 @@ var manifestServeCmd = cli.Command{
 			return fmt.Errorf("initialzing pubsub: %w", err)
 		}
 
-		sender, err := manifest.NewManifestSender(host, pubSub, currentManifest, c.Duration("publishInterval"))
+		sender, err := manifest.NewManifestSender(c.Context, host, pubSub, currentManifest, c.Duration("publishInterval"))
 		if err != nil {
 			return fmt.Errorf("initialzing manifest sender: %w", err)
 		}

diff --git a/cmd/f3/run.go b/cmd/f3/run.go
@@ -113,7 +113,7 @@ var runCmd = cli.Command{
 		id := c.Uint64("id")
 		signingBackend.Allow(int(id))
 
-		ec := ec.NewFakeEC(1, m.BootstrapEpoch, m.ECPeriod, initialPowerTable, true)
+		ec := ec.NewFakeEC(ctx, 1, m.BootstrapEpoch, m.ECPeriod, initialPowerTable, true)
 
 		module, err := f3.New(ctx, mprovider, ds, h, ps, signingBackend, ec)
 		if err != nil {

diff --git a/ec/fake_ec.go b/ec/fake_ec.go
@@ -10,12 +10,14 @@ import (
 	"golang.org/x/crypto/blake2b"
 
 	"github.com/filecoin-project/go-f3/gpbft"
+	"github.com/filecoin-project/go-f3/internal/clock"
 	mbase "github.com/multiformats/go-multibase"
 )
 
 var _ Backend = (*FakeEC)(nil)
 
 type FakeEC struct {
+	clock             clock.Clock
 	useTime           bool
 	seed              []byte
 	initialPowerTable gpbft.PowerEntries
@@ -64,14 +66,16 @@ func (ts *Tipset) String() string {
 	return res
 }
 
-func NewFakeEC(seed uint64, bootstrapEpoch int64, ecPeriod time.Duration, initialPowerTable gpbft.PowerEntries, useTime bool) *FakeEC {
+func NewFakeEC(ctx context.Context, seed uint64, bootstrapEpoch int64, ecPeriod time.Duration, initialPowerTable gpbft.PowerEntries, useTime bool) *FakeEC {
+	clk := clock.GetClock(ctx)
 	return &FakeEC{
+		clock:             clk,
 		useTime:           useTime,
 		seed:              binary.BigEndian.AppendUint64(nil, seed),
 		initialPowerTable: initialPowerTable,
 
 		ecPeriod: ecPeriod,
-		ecStart:  time.Now().Add(-time.Duration(bootstrapEpoch) * ecPeriod),
+		ecStart:  clk.Now().Add(-time.Duration(bootstrapEpoch) * ecPeriod),
 	}
 }
 
@@ -112,7 +116,7 @@ func (ec *FakeEC) genTipset(epoch int64) *Tipset {
 }
 
 func (ec *FakeEC) currentEpoch() int64 {
-	return int64(time.Since(ec.ecStart) / ec.ecPeriod)
+	return int64(ec.clock.Since(ec.ecStart) / ec.ecPeriod)
 }
 
 // GetTipsetByHeight should return a tipset or nil/empty byte array if it does not exists

diff --git a/ec/powerdelta_test.go b/ec/powerdelta_test.go
@@ -29,7 +29,7 @@ var powerTableC = gpbft.PowerEntries{
 }
 
 func TestReplacePowerTable(t *testing.T) {
-	backend := ec.NewFakeEC(0, 0, 0, powerTableA, false)
+	backend := ec.NewFakeEC(context.Background(), 0, 0, 0, powerTableA, false)
 	modifiedBackend := ec.WithModifiedPower(backend, powerTableB, true)
 
 	head, err := modifiedBackend.GetHead(context.Background())
@@ -42,7 +42,7 @@ func TestReplacePowerTable(t *testing.T) {
 }
 
 func TestModifyPowerTable(t *testing.T) {
-	backend := ec.NewFakeEC(0, 0, 0, powerTableA, false)
+	backend := ec.NewFakeEC(context.Background(), 0, 0, 0, powerTableA, false)
 	modifiedBackend := ec.WithModifiedPower(backend, powerTableB, false)
 
 	head, err := modifiedBackend.GetHead(context.Background())
@@ -54,7 +54,7 @@ func TestModifyPowerTable(t *testing.T) {
 }
 
 func TestBypassModifiedPowerTable(t *testing.T) {
-	backend := ec.NewFakeEC(0, 0, 0, powerTableA, false)
+	backend := ec.NewFakeEC(context.Background(), 0, 0, 0, powerTableA, false)
 	modifiedBackend := ec.WithModifiedPower(backend, nil, false)
 	require.Equal(t, backend, modifiedBackend)
 }
diff --git a/f3.go b/f3.go
@@ -12,12 +12,13 @@ import (
 	"github.com/filecoin-project/go-f3/certstore"
 	"github.com/filecoin-project/go-f3/ec"
 	"github.com/filecoin-project/go-f3/gpbft"
+	"github.com/filecoin-project/go-f3/internal/clock"
 	"github.com/filecoin-project/go-f3/manifest"
 
+	"github.com/ipfs/go-datastore"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/host"
 
-	"github.com/ipfs/go-datastore"
 	"go.uber.org/multierr"
 	"golang.org/x/sync/errgroup"
 )
@@ -32,6 +33,7 @@ type F3 struct {
 	ds     datastore.Datastore
 	ec     ec.Backend
 	pubsub *pubsub.PubSub
+	clock  clock.Clock
 
 	runningCtx context.Context
 	cancelCtx  context.CancelFunc
@@ -60,6 +62,7 @@ func New(_ctx context.Context, manifest manifest.ManifestProvider, ds datastore.
 		ds:               ds,
 		ec:               ec,
 		pubsub:           ps,
+		clock:            clock.GetClock(runningCtx),
 		runningCtx:       runningCtx,
 		cancelCtx:        cancel,
 		errgrp:           errgrp,
@@ -138,7 +141,7 @@ func (m *F3) computeBootstrapDelay(manifest *manifest.Manifest) (time.Duration,
 	}
 	epochDelay := manifest.BootstrapEpoch - currentEpoch
 	start := ts.Timestamp().Add(time.Duration(epochDelay) * manifest.ECPeriod)
-	delay := time.Until(start)
+	delay := m.clock.Until(start)
 	// Add additional delay to skip over null epochs. That way we wait the full 900 epochs.
 	if delay <= 0 {
 		delay = manifest.ECPeriod + delay%manifest.ECPeriod
@@ -187,7 +190,7 @@ func (m *F3) Start(startCtx context.Context) (_err error) {
 			}
 		}()
 
-		manifestChangeTimer := time.NewTimer(initialDelay)
+		manifestChangeTimer := m.clock.Timer(initialDelay)
 		if pendingManifest == nil && !manifestChangeTimer.Stop() {
 			<-manifestChangeTimer.C
 		}

diff --git a/go.mod b/go.mod
@@ -4,10 +4,10 @@ go 1.21
 
 require (
 	github.com/Kubuxu/go-broadcast v0.0.0-20240621161059-1a8c90734cd6
-	github.com/benbjohnson/clock v1.3.5
 	github.com/drand/kyber v1.3.1
 	github.com/drand/kyber-bls12381 v0.3.1
 	github.com/filecoin-project/go-bitfield v0.2.4
+	github.com/filecoin-project/go-clock v0.1.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-leveldb v0.5.0
@@ -27,6 +27,7 @@ require (
 )
 
 require (
+	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect

diff --git a/go.sum b/go.sum
@@ -57,6 +57,8 @@ github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/
 github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk=
 github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
+github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU=
+github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=