From 0b04da359afbe32a8640cb600c0f05a09277e387 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 26 Nov 2025 16:13:23 +0100 Subject: [PATCH 01/41] wip --- apps/evm/cmd/rollback.go | 2 +- apps/testapp/cmd/rollback.go | 2 +- block/components.go | 8 +-- block/internal/common/broadcaster_mock.go | 57 +++++++++++++++++++ block/internal/common/event.go | 3 + block/internal/common/expected_interfaces.go | 1 + block/internal/executing/executor.go | 6 +- .../internal/executing/executor_lazy_test.go | 4 +- .../internal/executing/executor_logic_test.go | 4 +- .../executing/executor_restart_test.go | 8 +-- block/internal/executing/executor_test.go | 48 +--------------- block/internal/submitting/da_submitter.go | 12 ++++ .../da_submitter_integration_test.go | 8 ++- .../submitting/da_submitter_mocks_test.go | 2 +- .../internal/submitting/da_submitter_test.go | 2 + block/internal/submitting/submitter_test.go | 6 +- block/internal/syncing/p2p_handler.go | 12 ++-- block/internal/syncing/p2p_handler_test.go | 20 ++++--- block/internal/syncing/syncer.go | 42 +++++++++++++- block/internal/syncing/syncer_backoff_test.go | 14 ++--- .../internal/syncing/syncer_benchmark_test.go | 2 +- block/internal/syncing/syncer_test.go | 10 ++-- pkg/rpc/client/client_test.go | 9 +-- pkg/rpc/server/server.go | 6 +- pkg/rpc/server/server_test.go | 11 ++-- pkg/sync/sync_service.go | 8 ++- pkg/sync/sync_service_test.go | 5 +- types/signed_header.go | 42 ++++++++++++++ types/utils.go | 8 +-- 29 files changed, 242 insertions(+), 120 deletions(-) diff --git a/apps/evm/cmd/rollback.go b/apps/evm/cmd/rollback.go index 5859b55b27..4a75f9a726 100644 --- a/apps/evm/cmd/rollback.go +++ b/apps/evm/cmd/rollback.go @@ -70,7 +70,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*types.SignedHeader]( + headerStore, err := goheaderstore.NewStore[*types.SignedHeaderWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), diff --git a/apps/testapp/cmd/rollback.go b/apps/testapp/cmd/rollback.go index 6d79a1ef13..761ec207d5 100644 --- a/apps/testapp/cmd/rollback.go +++ b/apps/testapp/cmd/rollback.go @@ -76,7 +76,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*types.SignedHeader]( + headerStore, err := goheaderstore.NewStore[*types.SignedHeaderWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), diff --git a/block/components.go b/block/components.go index 546cda62c3..43fd950f47 100644 --- a/block/components.go +++ b/block/components.go @@ -132,7 +132,7 @@ func NewSyncComponents( store store.Store, exec coreexecutor.Executor, da coreda.DA, - headerStore common.Broadcaster[*types.SignedHeader], + headerStore common.Broadcaster[*types.SignedHeaderWithDAHint], dataStore common.Broadcaster[*types.Data], logger zerolog.Logger, metrics *Metrics, @@ -165,7 +165,7 @@ func NewSyncComponents( ) // Create submitter for sync nodes (no signer, only DA inclusion processing) - daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger) + daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, nil) // todo (Alex): use a noop submitter := submitting.NewSubmitter( store, exec, @@ -198,7 +198,7 @@ func NewAggregatorComponents( sequencer coresequencer.Sequencer, da coreda.DA, signer signer.Signer, - headerBroadcaster common.Broadcaster[*types.SignedHeader], + headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint], dataBroadcaster common.Broadcaster[*types.Data], logger zerolog.Logger, metrics *Metrics, @@ -247,7 +247,7 @@ func NewAggregatorComponents( // Create DA client and submitter for aggregator nodes (with signer for submission) daClient := NewDAClient(da, config, logger) - daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger) + daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerBroadcaster) submitter := submitting.NewSubmitter( store, exec, diff --git a/block/internal/common/broadcaster_mock.go b/block/internal/common/broadcaster_mock.go index 2983478078..7b40164328 100644 --- a/block/internal/common/broadcaster_mock.go +++ b/block/internal/common/broadcaster_mock.go @@ -160,3 +160,60 @@ func (_c *MockBroadcaster_WriteToStoreAndBroadcast_Call[H]) RunAndReturn(run fun _c.Call.Return(run) return _c } + +// XXX provides a mock function for the type MockBroadcaster +func (_mock *MockBroadcaster[H]) XXX(ctx context.Context, headerOrData H) error { + ret := _mock.Called(ctx, headerOrData) + + if len(ret) == 0 { + panic("no return value specified for XXX") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, H) error); ok { + r0 = returnFunc(ctx, headerOrData) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockBroadcaster_XXX_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'XXX' +type MockBroadcaster_XXX_Call[H header.Header[H]] struct { + *mock.Call +} + +// XXX is a helper method to define mock.On call +// - ctx context.Context +// - headerOrData H +func (_e *MockBroadcaster_Expecter[H]) XXX(ctx interface{}, headerOrData interface{}) *MockBroadcaster_XXX_Call[H] { + return &MockBroadcaster_XXX_Call[H]{Call: _e.mock.On("XXX", ctx, headerOrData)} +} + +func (_c *MockBroadcaster_XXX_Call[H]) Run(run func(ctx context.Context, headerOrData H)) *MockBroadcaster_XXX_Call[H] { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 H + if args[1] != nil { + arg1 = args[1].(H) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockBroadcaster_XXX_Call[H]) Return(err error) *MockBroadcaster_XXX_Call[H] { + _c.Call.Return(err) + return _c +} + +func (_c *MockBroadcaster_XXX_Call[H]) RunAndReturn(run func(ctx context.Context, headerOrData H) error) *MockBroadcaster_XXX_Call[H] { + _c.Call.Return(run) + return _c +} diff --git a/block/internal/common/event.go b/block/internal/common/event.go index 69d0300f9f..5b016c246b 100644 --- a/block/internal/common/event.go +++ b/block/internal/common/event.go @@ -20,4 +20,7 @@ type DAHeightEvent struct { DaHeight uint64 // Source indicates where this event originated from (DA or P2P) Source EventSource + + // Optional DA height hint from P2P + DaHeightHint uint64 } diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index 8f36af6240..6015f19963 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -12,4 +12,5 @@ import ( type Broadcaster[H header.Header[H]] interface { WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error Store() header.Store[H] + XXX(ctx context.Context, headerOrData H) error } diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index be969b1a75..d09d01eab1 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -37,7 +37,7 @@ type Executor struct { metrics *common.Metrics // Broadcasting - headerBroadcaster common.Broadcaster[*types.SignedHeader] + headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint] dataBroadcaster common.Broadcaster[*types.Data] // Configuration @@ -76,7 +76,7 @@ func NewExecutor( metrics *common.Metrics, config config.Config, genesis genesis.Genesis, - headerBroadcaster common.Broadcaster[*types.SignedHeader], + headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint], dataBroadcaster common.Broadcaster[*types.Data], logger zerolog.Logger, options common.BlockOptions, @@ -420,7 +420,7 @@ func (e *Executor) produceBlock() error { // broadcast header and data to P2P network g, ctx := errgroup.WithContext(e.ctx) - g.Go(func() error { return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, header) }) + g.Go(func() error { return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, &types.SignedHeaderWithDAHint{SignedHeader: header}) }) g.Go(func() error { return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, data) }) if err := g.Wait(); err != nil { e.logger.Error().Err(err).Msg("failed to broadcast header and/data") diff --git a/block/internal/executing/executor_lazy_test.go b/block/internal/executing/executor_lazy_test.go index b72f0a856b..25b784b29d 100644 --- a/block/internal/executing/executor_lazy_test.go +++ b/block/internal/executing/executor_lazy_test.go @@ -47,7 +47,7 @@ func TestLazyMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -157,7 +157,7 @@ func TestRegularMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_logic_test.go b/block/internal/executing/executor_logic_test.go index 9aa79d0c43..c615e458cc 100644 --- a/block/internal/executing/executor_logic_test.go +++ b/block/internal/executing/executor_logic_test.go @@ -69,7 +69,7 @@ func TestProduceBlock_EmptyBatch_SetsEmptyDataHash(t *testing.T) { mockSeq := testmocks.NewMockSequencer(t) // Broadcasters are required by produceBlock; use generated mocks - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -156,7 +156,7 @@ func TestPendingLimit_SkipsProduction(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_restart_test.go b/block/internal/executing/executor_restart_test.go index 3f0e8b500c..b9a13f68b8 100644 --- a/block/internal/executing/executor_restart_test.go +++ b/block/internal/executing/executor_restart_test.go @@ -47,7 +47,7 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create first executor instance mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb1 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db1 := common.NewMockBroadcaster[*types.Data](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -166,7 +166,7 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create second executor instance (restart scenario) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb2 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db2 := common.NewMockBroadcaster[*types.Data](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -264,7 +264,7 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create first executor and produce one block mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb1 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db1 := common.NewMockBroadcaster[*types.Data](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -316,7 +316,7 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create second executor (restart) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb2 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db2 := common.NewMockBroadcaster[*types.Data](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_test.go b/block/internal/executing/executor_test.go index e310c6d40d..76a7d21748 100644 --- a/block/internal/executing/executor_test.go +++ b/block/internal/executing/executor_test.go @@ -1,7 +1,6 @@ package executing import ( - "context" "testing" "time" @@ -40,7 +39,7 @@ func TestExecutor_BroadcasterIntegration(t *testing.T) { } // Create mock broadcasters - headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeader](t) + headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) dataBroadcaster := common.NewMockBroadcaster[*types.Data](t) // Create executor with broadcasters @@ -120,48 +119,3 @@ func TestExecutor_NilBroadcasters(t *testing.T) { assert.Equal(t, cacheManager, executor.cache) assert.Equal(t, gen, executor.genesis) } - -func TestExecutor_BroadcastFlow(t *testing.T) { - // This test demonstrates how the broadcast flow works - // when an Executor produces a block - - // Create mock broadcasters - headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeader](t) - dataBroadcaster := common.NewMockBroadcaster[*types.Data](t) - - // Create sample data that would be broadcast - sampleHeader := &types.SignedHeader{ - Header: types.Header{ - BaseHeader: types.BaseHeader{ - ChainID: "test-chain", - Height: 1, - Time: uint64(time.Now().UnixNano()), - }, - }, - } - - sampleData := &types.Data{ - Metadata: &types.Metadata{ - ChainID: "test-chain", - Height: 1, - Time: uint64(time.Now().UnixNano()), - }, - Txs: []types.Tx{}, - } - - // Test broadcast calls - ctx := context.Background() - - // Set up expectations - headerBroadcaster.EXPECT().WriteToStoreAndBroadcast(ctx, sampleHeader).Return(nil).Once() - dataBroadcaster.EXPECT().WriteToStoreAndBroadcast(ctx, sampleData).Return(nil).Once() - - // Simulate what happens in produceBlock() after block creation - err := headerBroadcaster.WriteToStoreAndBroadcast(ctx, sampleHeader) - require.NoError(t, err) - - err = dataBroadcaster.WriteToStoreAndBroadcast(ctx, sampleData) - require.NoError(t, err) - - // Verify expectations were met (automatically checked by testify mock on cleanup) -} diff --git a/block/internal/submitting/da_submitter.go b/block/internal/submitting/da_submitter.go index 8cf741dcd9..5123429bf2 100644 --- a/block/internal/submitting/da_submitter.go +++ b/block/internal/submitting/da_submitter.go @@ -93,6 +93,10 @@ func clamp(v, min, max time.Duration) time.Duration { return v } +type xxxer interface { + XXX(ctx context.Context, header *types.SignedHeaderWithDAHint) error +} + // DASubmitter handles DA submission operations type DASubmitter struct { client da.Client @@ -101,6 +105,7 @@ type DASubmitter struct { options common.BlockOptions logger zerolog.Logger metrics *common.Metrics + xxxer xxxer // address selector for multi-account support addressSelector pkgda.AddressSelector @@ -114,6 +119,7 @@ func NewDASubmitter( options common.BlockOptions, metrics *common.Metrics, logger zerolog.Logger, + xxxer xxxer, ) *DASubmitter { daSubmitterLogger := logger.With().Str("component", "da_submitter").Logger() @@ -146,6 +152,7 @@ func NewDASubmitter( metrics: metrics, logger: daSubmitterLogger, addressSelector: addressSelector, + xxxer: xxxer, } } @@ -187,6 +194,11 @@ func (s *DASubmitter) SubmitHeaders(ctx context.Context, cache cache.Manager) er func(submitted []*types.SignedHeader, res *coreda.ResultSubmit) { for _, header := range submitted { cache.SetHeaderDAIncluded(header.Hash().String(), res.Height, header.Height()) + payload := &types.SignedHeaderWithDAHint{SignedHeader: header, DAHeightHint: res.Height} + if err := s.xxxer.XXX(ctx, payload); err != nil { + s.logger.Error().Err(err).Msg("failed to update header in p2p store") + // ignoring error here, since we don't want to block the block submission' + } } if l := len(submitted); l > 0 { lastHeight := submitted[l-1].Height() diff --git a/block/internal/submitting/da_submitter_integration_test.go b/block/internal/submitting/da_submitter_integration_test.go index 5b768e1a51..854b0950d2 100644 --- a/block/internal/submitting/da_submitter_integration_test.go +++ b/block/internal/submitting/da_submitter_integration_test.go @@ -93,7 +93,7 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSubmitter := NewDASubmitter(daClient, cfg, gen, common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop()) + daSubmitter := NewDASubmitter(daClient, cfg, gen, common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), noopXXXer{}) // Submit headers and data require.NoError(t, daSubmitter.SubmitHeaders(context.Background(), cm)) @@ -110,3 +110,9 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( assert.True(t, ok) } + +type noopXXXer struct{} + +func (n noopXXXer) XXX(ctx context.Context, header *types.SignedHeaderWithDAHint) error { + return nil +} diff --git a/block/internal/submitting/da_submitter_mocks_test.go b/block/internal/submitting/da_submitter_mocks_test.go index b215b0cf2f..1716b59929 100644 --- a/block/internal/submitting/da_submitter_mocks_test.go +++ b/block/internal/submitting/da_submitter_mocks_test.go @@ -36,7 +36,7 @@ func newTestSubmitter(mockDA *mocks.MockDA, override func(*config.Config)) *DASu Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - return NewDASubmitter(daClient, cfg, genesis.Genesis{} /*options=*/, common.BlockOptions{}, common.NopMetrics(), zerolog.Nop()) + return NewDASubmitter(daClient, cfg, genesis.Genesis{} /*options=*/, common.BlockOptions{}, common.NopMetrics(), zerolog.Nop(), nil) } // marshal helper for simple items diff --git a/block/internal/submitting/da_submitter_test.go b/block/internal/submitting/da_submitter_test.go index 214ab98db4..f33aaab21f 100644 --- a/block/internal/submitting/da_submitter_test.go +++ b/block/internal/submitting/da_submitter_test.go @@ -65,6 +65,7 @@ func setupDASubmitterTest(t *testing.T) (*DASubmitter, store.Store, cache.Manage common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), + noopXXXer{}, ) return daSubmitter, st, cm, dummyDA, gen @@ -115,6 +116,7 @@ func TestNewDASubmitterSetsVisualizerWhenEnabled(t *testing.T) { common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), + nil, ) require.NotNil(t, server.GetDAVisualizationServer()) diff --git a/block/internal/submitting/submitter_test.go b/block/internal/submitting/submitter_test.go index c1df11bf51..f317e0bdec 100644 --- a/block/internal/submitting/submitter_test.go +++ b/block/internal/submitting/submitter_test.go @@ -168,7 +168,7 @@ func TestSubmitter_setSequencerHeightToDAHeight(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop()) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) s := NewSubmitter(mockStore, nil, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) s.ctx = ctx @@ -253,7 +253,7 @@ func TestSubmitter_processDAInclusionLoop_advances(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop()) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) s := NewSubmitter(st, exec, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) // prepare two consecutive blocks in store with DA included in cache @@ -444,7 +444,7 @@ func TestSubmitter_CacheClearedOnHeightInclusion(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop()) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) s := NewSubmitter(st, exec, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) // Create test blocks diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index d8c10bc4c3..a5d4b53f26 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -27,7 +27,7 @@ type p2pHandler interface { // The handler maintains a processedHeight to track the highest block that has been // successfully validated and sent to the syncer, preventing duplicate processing. type P2PHandler struct { - headerStore goheader.Store[*types.SignedHeader] + headerStore goheader.Store[*types.SignedHeaderWithDAHint] dataStore goheader.Store[*types.Data] cache cache.CacheManager genesis genesis.Genesis @@ -38,7 +38,7 @@ type P2PHandler struct { // NewP2PHandler creates a new P2P handler. func NewP2PHandler( - headerStore goheader.Store[*types.SignedHeader], + headerStore goheader.Store[*types.SignedHeaderWithDAHint], dataStore goheader.Store[*types.Data], cache cache.CacheManager, genesis genesis.Genesis, @@ -104,10 +104,10 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC // further header validation (signature) is done in validateBlock. // we need to be sure that the previous block n-1 was executed before validating block n event := common.DAHeightEvent{ - Header: header, - Data: data, - DaHeight: 0, - Source: common.SourceP2P, + Header: header.SignedHeader, + Data: data, + Source: common.SourceP2P, + DaHeightHint: header.DAHeightHint, } select { diff --git a/block/internal/syncing/p2p_handler_test.go b/block/internal/syncing/p2p_handler_test.go index dfab41faae..aacd54d4cc 100644 --- a/block/internal/syncing/p2p_handler_test.go +++ b/block/internal/syncing/p2p_handler_test.go @@ -57,7 +57,7 @@ func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer [ // P2PTestData aggregates dependencies used by P2P handler tests. type P2PTestData struct { Handler *P2PHandler - HeaderStore *extmocks.MockStore[*types.SignedHeader] + HeaderStore *extmocks.MockStore[*types.SignedHeaderWithDAHint] DataStore *extmocks.MockStore[*types.Data] Cache cache.CacheManager Genesis genesis.Genesis @@ -73,7 +73,7 @@ func setupP2P(t *testing.T) *P2PTestData { gen := genesis.Genesis{ChainID: "p2p-test", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: proposerAddr} - headerStoreMock := extmocks.NewMockStore[*types.SignedHeader](t) + headerStoreMock := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) dataStoreMock := extmocks.NewMockStore[*types.Data](t) cfg := config.Config{ @@ -136,8 +136,8 @@ func TestP2PHandler_ProcessHeight_EmitsEventWhenHeaderAndDataPresent(t *testing. sig, err := p.Signer.Sign(bz) require.NoError(t, err) header.Signature = sig - - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(header, nil).Once() + payload := &types.SignedHeaderWithDAHint{SignedHeader: header} + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(payload, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, nil).Once() ch := make(chan common.DAHeightEvent, 1) @@ -163,7 +163,8 @@ func TestP2PHandler_ProcessHeight_SkipsWhenDataMissing(t *testing.T) { require.NoError(t, err) header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(header, nil).Once() + payload := &types.SignedHeaderWithDAHint{SignedHeader: header} + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(payload, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, errors.New("missing")).Once() ch := make(chan common.DAHeightEvent, 1) @@ -198,7 +199,8 @@ func TestP2PHandler_ProcessHeight_SkipsOnProposerMismatch(t *testing.T) { header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 11, badAddr, pub, signer) header.DataHash = common.DataHashForEmptyTxs - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(header, nil).Once() + payload := &types.SignedHeaderWithDAHint{SignedHeader: header} + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(payload, nil).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 11, ch) @@ -233,7 +235,8 @@ func TestP2PHandler_ProcessedHeightSkipsPreviouslyHandledBlocks(t *testing.T) { require.NoError(t, err) header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(header, nil).Once() + payload := &types.SignedHeaderWithDAHint{SignedHeader: header} + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(payload, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, nil).Once() require.NoError(t, p.Handler.ProcessHeight(ctx, 6, ch)) @@ -256,7 +259,8 @@ func TestP2PHandler_SetProcessedHeightPreventsDuplicates(t *testing.T) { require.NoError(t, err) header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(header, nil).Once() + payload := &types.SignedHeaderWithDAHint{SignedHeader: header} + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(payload, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, nil).Once() ch := make(chan common.DAHeightEvent, 1) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index ee69edea7f..a40e30fbd0 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -49,7 +49,7 @@ type Syncer struct { daRetrieverHeight *atomic.Uint64 // P2P stores - headerStore common.Broadcaster[*types.SignedHeader] + headerStore common.Broadcaster[*types.SignedHeaderWithDAHint] dataStore common.Broadcaster[*types.Data] // Channels for coordination @@ -81,7 +81,7 @@ func NewSyncer( metrics *common.Metrics, config config.Config, genesis genesis.Genesis, - headerStore common.Broadcaster[*types.SignedHeader], + headerStore common.Broadcaster[*types.SignedHeaderWithDAHint], dataStore common.Broadcaster[*types.Data], logger zerolog.Logger, options common.BlockOptions, @@ -432,6 +432,41 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { return } + // If this is a P2P event with a DA height hint, trigger targeted DA retrieval + // This allows us to fetch the block directly from the specified DA height instead of sequential scanning + if event.Source == common.SourceP2P && event.DaHeightHint != 0 { + if _, exists := s.cache.GetHeaderDAIncluded(event.Header.Hash().String()); !exists { + s.logger.Debug(). + Uint64("height", height). + Uint64("da_height_hint", event.DaHeightHint). + Msg("P2P event with DA height hint, triggering targeted DA retrieval") + + // Trigger targeted DA retrieval in background + go func() { + targetEvents, err := s.daRetriever.RetrieveFromDA(s.ctx, event.DaHeightHint) + if err != nil { + s.logger.Debug(). + Err(err). + Uint64("da_height", event.DaHeightHint). + Msg("targeted DA retrieval failed (hint may be incorrect or DA not yet available)") + // Not a critical error - the sequential DA worker will eventually find it + return + } + + // Process retrieved events from the targeted DA height + for _, daEvent := range targetEvents { + select { + case s.heightInCh <- daEvent: + case <-s.ctx.Done(): + return + default: + s.cache.SetPendingEvent(daEvent.Header.Height(), &daEvent) + } + } + }() + } + } + // Last data must be got from store if the event comes from DA and the data hash is empty. // When if the event comes from P2P, the sequencer and then all the full nodes contains the data. if event.Source == common.SourceDA && bytes.Equal(event.Header.DataHash, common.DataHashForEmptyTxs) && currentHeight > 0 { @@ -469,7 +504,8 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { g.Go(func() error { // broadcast header locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - return s.headerStore.WriteToStoreAndBroadcast(ctx, event.Header, pubsub.WithLocalPublication(true)) + payload := &types.SignedHeaderWithDAHint{SignedHeader: event.Header, DAHeightHint: event.DaHeightHint} + return s.headerStore.WriteToStoreAndBroadcast(ctx, payload, pubsub.WithLocalPublication(true)) }) g.Go(func() error { // broadcast data locally only — prevents spamming the p2p network with old height notifications, diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 65f2586966..970dd0cc5c 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -77,13 +77,13 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeader](t) + headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() syncer.headerStore = headerStore @@ -173,13 +173,13 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeader](t) + headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() syncer.headerStore = headerStore @@ -263,13 +263,13 @@ func TestSyncer_BackoffBehaviorIntegration(t *testing.T) { syncer.p2pHandler = p2pHandler // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeader](t) + headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() syncer.headerStore = headerStore @@ -350,7 +350,7 @@ func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), + common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index e2b6f6e51f..29f8e86854 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -153,7 +153,7 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay mockP2P := newMockp2pHandler(b) // not used directly in this benchmark path mockP2P.On("SetProcessedHeight", mock.Anything).Return().Maybe() s.p2pHandler = mockP2P - headerP2PStore := common.NewMockBroadcaster[*types.SignedHeader](b) + headerP2PStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](b) s.headerStore = headerP2PStore dataP2PStore := common.NewMockBroadcaster[*types.Data](b) s.dataStore = dataP2PStore diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 5c16da4435..3fa92e4d26 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -123,7 +123,7 @@ func TestSyncer_validateBlock_DataHashMismatch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), + common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -174,7 +174,7 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), + common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -228,7 +228,7 @@ func TestSequentialBlockSync(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), + common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -340,13 +340,13 @@ func TestSyncLoopPersistState(t *testing.T) { dummyExec := execution.NewDummyExecutor() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - mockP2PHeaderStore := common.NewMockBroadcaster[*types.SignedHeader](t) + mockP2PHeaderStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) mockP2PHeaderStore.EXPECT().Store().Return(mockHeaderStore).Maybe() mockP2PDataStore := common.NewMockBroadcaster[*types.Data](t) diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index 4b2b82e1b3..e517d5128b 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -28,10 +28,11 @@ import ( func setupTestServer( t *testing.T, mockStore *mocks.MockStore, - headerStore goheader.Store[*types.SignedHeader], + headerStore goheader.Store[*types.SignedHeaderWithDAHint], dataStore goheader.Store[*types.Data], mockP2P *mocks.MockP2PRPC, ) (*httptest.Server, *Client) { + t.Helper() mux := http.NewServeMux() logger := zerolog.Nop() @@ -105,13 +106,13 @@ func TestClientGetMetadata(t *testing.T) { func TestClientGetP2PStoreInfo(t *testing.T) { mockStore := mocks.NewMockStore(t) mockP2P := mocks.NewMockP2PRPC(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeader](t) + headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) dataStore := headerstoremocks.NewMockStore[*types.Data](t) now := time.Now().UTC() - headerHead := testSignedHeader(10, now) - headerTail := testSignedHeader(5, now.Add(-time.Minute)) + headerHead := &types.SignedHeaderWithDAHint{SignedHeader: testSignedHeader(10, now)} + headerTail := &types.SignedHeaderWithDAHint{SignedHeader: testSignedHeader(5, now.Add(-time.Minute))} headerStore.On("Height").Return(uint64(10)) headerStore.On("Head", mock.Anything).Return(headerHead, nil) headerStore.On("Tail", mock.Anything).Return(headerTail, nil) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index e0abed2de0..6fb9ee8362 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -34,7 +34,7 @@ var _ rpc.StoreServiceHandler = (*StoreServer)(nil) // StoreServer implements the StoreService defined in the proto file type StoreServer struct { store store.Store - headerStore goheader.Store[*types.SignedHeader] + headerStore goheader.Store[*types.SignedHeaderWithDAHint] dataStore goheader.Store[*types.Data] logger zerolog.Logger } @@ -42,7 +42,7 @@ type StoreServer struct { // NewStoreServer creates a new StoreServer instance func NewStoreServer( store store.Store, - headerStore goheader.Store[*types.SignedHeader], + headerStore goheader.Store[*types.SignedHeaderWithDAHint], dataStore goheader.Store[*types.Data], logger zerolog.Logger, ) *StoreServer { @@ -370,7 +370,7 @@ func (p *P2PServer) GetNetInfo( // NewServiceHandler creates a new HTTP handler for Store, P2P and Config services func NewServiceHandler( store store.Store, - headerStore goheader.Store[*types.SignedHeader], + headerStore goheader.Store[*types.SignedHeaderWithDAHint], dataStore goheader.Store[*types.Data], peerManager p2p.P2PRPC, proposerAddress []byte, diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 32e9b0ebec..befecd910f 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -325,7 +325,7 @@ func TestGetGenesisDaHeight_InvalidLength(t *testing.T) { func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns snapshots for configured stores", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeader](t) + headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) dataStore := headerstoremocks.NewMockStore[*types.Data](t) logger := zerolog.Nop() server := NewStoreServer(mockStore, headerStore, dataStore, logger) @@ -354,10 +354,10 @@ func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns error when a store edge fails", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeader](t) + headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) logger := zerolog.Nop() headerStore.On("Height").Return(uint64(0)) - headerStore.On("Head", mock.Anything).Return((*types.SignedHeader)(nil), fmt.Errorf("boom")) + headerStore.On("Head", mock.Anything).Return((*types.SignedHeaderWithDAHint)(nil), fmt.Errorf("boom")) server := NewStoreServer(mockStore, headerStore, nil, logger) resp, err := server.GetP2PStoreInfo(context.Background(), connect.NewRequest(&emptypb.Empty{})) @@ -627,8 +627,8 @@ func TestHealthReadyEndpoint(t *testing.T) { }) } -func makeTestSignedHeader(height uint64, ts time.Time) *types.SignedHeader { - return &types.SignedHeader{ +func makeTestSignedHeader(height uint64, ts time.Time) *types.SignedHeaderWithDAHint { + return &types.SignedHeaderWithDAHint{SignedHeader: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -639,6 +639,7 @@ func makeTestSignedHeader(height uint64, ts time.Time) *types.SignedHeader { DataHash: []byte{0x02}, AppHash: []byte{0x03}, }, + }, } } diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 6a17a42a85..f4c63c33a8 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -62,7 +62,7 @@ type SyncService[H header.Header[H]] struct { type DataSyncService = SyncService[*types.Data] // HeaderSyncService is the P2P Sync Service for headers. -type HeaderSyncService = SyncService[*types.SignedHeader] +type HeaderSyncService = SyncService[*types.SignedHeaderWithDAHint] // NewDataSyncService returns a new DataSyncService. func NewDataSyncService( @@ -83,7 +83,7 @@ func NewHeaderSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*HeaderSyncService, error) { - return newSyncService[*types.SignedHeader](store, headerSync, conf, genesis, p2p, logger) + return newSyncService[*types.SignedHeaderWithDAHint](store, headerSync, conf, genesis, p2p, logger) } func newSyncService[H header.Header[H]]( @@ -174,6 +174,10 @@ func (syncService *SyncService[H]) WriteToStoreAndBroadcast(ctx context.Context, return nil } +func (s *SyncService[H]) XXX(ctx context.Context, headerOrData H) error { + return s.store.Append(ctx, headerOrData) +} + // Start is a part of Service interface. func (syncService *SyncService[H]) Start(ctx context.Context) error { // setup P2P infrastructure, but don't start Subscriber yet. diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 93603752a7..aeaeda18b8 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -167,7 +167,7 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) } -func nextHeader(t *testing.T, previousHeader *types.SignedHeader, chainID string, noopSigner signer.Signer) *types.SignedHeader { +func nextHeader(t *testing.T, previousHeader *types.SignedHeaderWithDAHint, chainID string, noopSigner signer.Signer) *types.SignedHeaderWithDAHint { newSignedHeader := &types.SignedHeader{ Header: types.GetRandomNextHeader(previousHeader.Header, chainID), Signer: previousHeader.Signer, @@ -178,8 +178,7 @@ func nextHeader(t *testing.T, previousHeader *types.SignedHeader, chainID string require.NoError(t, err) newSignedHeader.Signature = signature require.NoError(t, newSignedHeader.Validate()) - previousHeader = newSignedHeader - return previousHeader + return &types.SignedHeaderWithDAHint{SignedHeader: newSignedHeader} } func bytesN(r *rand.Rand, n int) []byte { diff --git a/types/signed_header.go b/types/signed_header.go index ffacbc847a..d0b322321d 100644 --- a/types/signed_header.go +++ b/types/signed_header.go @@ -3,6 +3,7 @@ package types import ( "bytes" "context" + "encoding/binary" "errors" "fmt" @@ -14,6 +15,47 @@ var ( ErrLastHeaderHashMismatch = errors.New("last header hash mismatch") ) +var _ header.Header[*SignedHeaderWithDAHint] = &SignedHeaderWithDAHint{} + +type SignedHeaderWithDAHint struct { + *SignedHeader + DAHeightHint uint64 +} + +func (s *SignedHeaderWithDAHint) New() *SignedHeaderWithDAHint { + return &SignedHeaderWithDAHint{SignedHeader: &SignedHeader{}} +} +func (sh *SignedHeaderWithDAHint) Verify(untrstH *SignedHeaderWithDAHint) error { + return sh.SignedHeader.Verify(untrstH.SignedHeader) +} + +func (s *SignedHeaderWithDAHint) Zero() bool { + return s == nil +} + +func (s *SignedHeaderWithDAHint) IsZero() bool { + return s == nil +} + +func (s *SignedHeaderWithDAHint) MarshalBinary() ([]byte, error) { + bz, err := s.SignedHeader.MarshalBinary() + if err != nil { + return nil, err + } + out := make([]byte, 8+len(bz)) + binary.BigEndian.PutUint64(out, s.DAHeightHint) + copy(out[8:], bz) + return out, nil +} + +func (s *SignedHeaderWithDAHint) UnmarshalBinary(data []byte) error { + if len(data) < 8 { + return fmt.Errorf("invalid length: %d", len(data)) + } + s.DAHeightHint = binary.BigEndian.Uint64(data) + return s.SignedHeader.UnmarshalBinary(data[8:]) +} + var _ header.Header[*SignedHeader] = &SignedHeader{} // SignedHeader combines Header and its signature. diff --git a/types/utils.go b/types/utils.go index d8c2527521..b0e28c80a8 100644 --- a/types/utils.go +++ b/types/utils.go @@ -78,7 +78,7 @@ func GenerateRandomBlockCustomWithAppHash(config *BlockConfig, chainID string, a Time: uint64(signedHeader.Time().UnixNano()), } - return signedHeader, data, config.PrivKey + return signedHeader.SignedHeader, data, config.PrivKey } // GenerateRandomBlockCustom returns a block with random data and the given height, transactions, privateKey and proposer address. @@ -150,11 +150,11 @@ func GetRandomSignedHeader(chainID string) (*SignedHeader, crypto.PrivKey, error if err != nil { return nil, nil, err } - return signedHeader, pk, nil + return signedHeader.SignedHeader, pk, nil } // GetRandomSignedHeaderCustom creates a signed header based on the provided HeaderConfig. -func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedHeader, error) { +func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedHeaderWithDAHint, error) { pk, err := config.Signer.GetPublic() if err != nil { return nil, err @@ -183,7 +183,7 @@ func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedH return nil, err } signedHeader.Signature = signature - return signedHeader, nil + return &SignedHeaderWithDAHint{SignedHeader: signedHeader}, nil } // GetRandomNextSignedHeader returns a signed header with random data and height of +1 from From 044903f852b300d8a2a9239c97627554ef1787be Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 27 Nov 2025 15:33:04 +0100 Subject: [PATCH 02/41] x --- block/components.go | 21 +-- block/internal/common/broadcaster_mock.go | 134 ++++++++++-------- block/internal/common/event.go | 4 +- block/internal/common/expected_interfaces.go | 53 ++++++- block/internal/executing/executor.go | 10 +- block/internal/submitting/da_submitter.go | 63 ++++---- .../da_submitter_integration_test.go | 2 +- block/internal/syncing/p2p_handler.go | 24 ++-- block/internal/syncing/syncer.go | 109 ++++++++------ pkg/rpc/server/server.go | 6 +- pkg/sync/sync_service.go | 36 +++-- pkg/sync/sync_service_test.go | 6 +- types/da_hint_container.go | 74 ++++++++++ types/signed_header.go | 43 +----- types/utils.go | 8 +- 15 files changed, 374 insertions(+), 219 deletions(-) create mode 100644 types/da_hint_container.go diff --git a/block/components.go b/block/components.go index 43fd950f47..4b5316eab4 100644 --- a/block/components.go +++ b/block/components.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/evstack/ev-node/pkg/sync" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" @@ -132,8 +133,8 @@ func NewSyncComponents( store store.Store, exec coreexecutor.Executor, da coreda.DA, - headerStore common.Broadcaster[*types.SignedHeaderWithDAHint], - dataStore common.Broadcaster[*types.Data], + headerStore *sync.HeaderSyncService, + dataStore *sync.DataSyncService, logger zerolog.Logger, metrics *Metrics, blockOpts BlockOptions, @@ -157,15 +158,15 @@ func NewSyncComponents( metrics, config, genesis, - headerStore, - dataStore, + common.NewDecorator[*types.SignedHeader](headerStore), + common.NewDecorator[*types.Data](dataStore), logger, blockOpts, errorCh, ) // Create submitter for sync nodes (no signer, only DA inclusion processing) - daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, nil) // todo (Alex): use a noop + daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerStore, dataStore) submitter := submitting.NewSubmitter( store, exec, @@ -198,8 +199,8 @@ func NewAggregatorComponents( sequencer coresequencer.Sequencer, da coreda.DA, signer signer.Signer, - headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint], - dataBroadcaster common.Broadcaster[*types.Data], + headerBroadcaster *sync.HeaderSyncService, + dataBroadcaster *sync.DataSyncService, logger zerolog.Logger, metrics *Metrics, blockOpts BlockOptions, @@ -222,8 +223,8 @@ func NewAggregatorComponents( metrics, config, genesis, - headerBroadcaster, - dataBroadcaster, + common.NewDecorator[*types.SignedHeader](headerBroadcaster), + common.NewDecorator[*types.Data](dataBroadcaster), logger, blockOpts, errorCh, @@ -247,7 +248,7 @@ func NewAggregatorComponents( // Create DA client and submitter for aggregator nodes (with signer for submission) daClient := NewDAClient(da, config, logger) - daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerBroadcaster) + daSubmitter := submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerBroadcaster, dataBroadcaster) submitter := submitting.NewSubmitter( store, exec, diff --git a/block/internal/common/broadcaster_mock.go b/block/internal/common/broadcaster_mock.go index 7b40164328..e761aa624a 100644 --- a/block/internal/common/broadcaster_mock.go +++ b/block/internal/common/broadcaster_mock.go @@ -8,6 +8,7 @@ import ( "context" "github.com/celestiaorg/go-header" + "github.com/evstack/ev-node/types" "github.com/libp2p/go-libp2p-pubsub" mock "github.com/stretchr/testify/mock" ) @@ -39,6 +40,82 @@ func (_m *MockBroadcaster[H]) EXPECT() *MockBroadcaster_Expecter[H] { return &MockBroadcaster_Expecter[H]{mock: &_m.Mock} } +// AppendDAHint provides a mock function for the type MockBroadcaster +func (_mock *MockBroadcaster[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { + // types.Hash + _va := make([]interface{}, len(hashes)) + for _i := range hashes { + _va[_i] = hashes[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, daHeight) + _ca = append(_ca, _va...) + ret := _mock.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for AppendDAHint") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64, ...types.Hash) error); ok { + r0 = returnFunc(ctx, daHeight, hashes...) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockBroadcaster_AppendDAHint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendDAHint' +type MockBroadcaster_AppendDAHint_Call[H header.Header[H]] struct { + *mock.Call +} + +// AppendDAHint is a helper method to define mock.On call +// - ctx context.Context +// - daHeight uint64 +// - hashes ...types.Hash +func (_e *MockBroadcaster_Expecter[H]) AppendDAHint(ctx interface{}, daHeight interface{}, hashes ...interface{}) *MockBroadcaster_AppendDAHint_Call[H] { + return &MockBroadcaster_AppendDAHint_Call[H]{Call: _e.mock.On("AppendDAHint", + append([]interface{}{ctx, daHeight}, hashes...)...)} +} + +func (_c *MockBroadcaster_AppendDAHint_Call[H]) Run(run func(ctx context.Context, daHeight uint64, hashes ...types.Hash)) *MockBroadcaster_AppendDAHint_Call[H] { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uint64 + if args[1] != nil { + arg1 = args[1].(uint64) + } + var arg2 []types.Hash + variadicArgs := make([]types.Hash, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(types.Hash) + } + } + arg2 = variadicArgs + run( + arg0, + arg1, + arg2..., + ) + }) + return _c +} + +func (_c *MockBroadcaster_AppendDAHint_Call[H]) Return(err error) *MockBroadcaster_AppendDAHint_Call[H] { + _c.Call.Return(err) + return _c +} + +func (_c *MockBroadcaster_AppendDAHint_Call[H]) RunAndReturn(run func(ctx context.Context, daHeight uint64, hashes ...types.Hash) error) *MockBroadcaster_AppendDAHint_Call[H] { + _c.Call.Return(run) + return _c +} + // Store provides a mock function for the type MockBroadcaster func (_mock *MockBroadcaster[H]) Store() header.Store[H] { ret := _mock.Called() @@ -160,60 +237,3 @@ func (_c *MockBroadcaster_WriteToStoreAndBroadcast_Call[H]) RunAndReturn(run fun _c.Call.Return(run) return _c } - -// XXX provides a mock function for the type MockBroadcaster -func (_mock *MockBroadcaster[H]) XXX(ctx context.Context, headerOrData H) error { - ret := _mock.Called(ctx, headerOrData) - - if len(ret) == 0 { - panic("no return value specified for XXX") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, H) error); ok { - r0 = returnFunc(ctx, headerOrData) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockBroadcaster_XXX_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'XXX' -type MockBroadcaster_XXX_Call[H header.Header[H]] struct { - *mock.Call -} - -// XXX is a helper method to define mock.On call -// - ctx context.Context -// - headerOrData H -func (_e *MockBroadcaster_Expecter[H]) XXX(ctx interface{}, headerOrData interface{}) *MockBroadcaster_XXX_Call[H] { - return &MockBroadcaster_XXX_Call[H]{Call: _e.mock.On("XXX", ctx, headerOrData)} -} - -func (_c *MockBroadcaster_XXX_Call[H]) Run(run func(ctx context.Context, headerOrData H)) *MockBroadcaster_XXX_Call[H] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 H - if args[1] != nil { - arg1 = args[1].(H) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *MockBroadcaster_XXX_Call[H]) Return(err error) *MockBroadcaster_XXX_Call[H] { - _c.Call.Return(err) - return _c -} - -func (_c *MockBroadcaster_XXX_Call[H]) RunAndReturn(run func(ctx context.Context, headerOrData H) error) *MockBroadcaster_XXX_Call[H] { - _c.Call.Return(run) - return _c -} diff --git a/block/internal/common/event.go b/block/internal/common/event.go index 5b016c246b..f02a181de8 100644 --- a/block/internal/common/event.go +++ b/block/internal/common/event.go @@ -21,6 +21,6 @@ type DAHeightEvent struct { // Source indicates where this event originated from (DA or P2P) Source EventSource - // Optional DA height hint from P2P - DaHeightHint uint64 + // Optional DA height hints from P2P. first is the DA height hint for the header, second is the DA height hint for the data + DaHeightHints [2]uint64 } diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index 6015f19963..59a0991670 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -3,14 +3,63 @@ package common import ( "context" + "github.com/evstack/ev-node/types" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/celestiaorg/go-header" ) -// broadcaster interface for P2P broadcasting +type ( + HeaderP2PBroadcaster = Decorator[*types.SignedHeader] + DataP2PBroadcaster = Decorator[*types.Data] +) + +// Broadcaster interface for P2P broadcasting type Broadcaster[H header.Header[H]] interface { WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error Store() header.Store[H] - XXX(ctx context.Context, headerOrData H) error + AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error +} + +// Decorator to access the the payload type without the container +type Decorator[H header.Header[H]] struct { + nested Broadcaster[*types.DAHeightHintContainer[H]] +} + +func NewDecorator[H header.Header[H]](nested Broadcaster[*types.DAHeightHintContainer[H]]) Decorator[H] { + return Decorator[H]{nested: nested} +} + +func (d Decorator[H]) WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error { + return d.nested.WriteToStoreAndBroadcast(ctx, &types.DAHeightHintContainer[H]{Entry: payload}, opts...) +} + +func (d Decorator[H]) Store() HeightStore[H] { + return HeightStoreImpl[H]{store: d.nested.Store()} +} +func (d Decorator[H]) XStore() header.Store[*types.DAHeightHintContainer[H]] { + return d.nested.Store() +} + +func (d Decorator[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { + return d.nested.AppendDAHint(ctx, daHeight, hashes...) +} + +// HeightStore is a subset of goheader.Store +type HeightStore[H header.Header[H]] interface { + GetByHeight(context.Context, uint64) (H, error) +} + +type HeightStoreImpl[H header.Header[H]] struct { + store header.Store[*types.DAHeightHintContainer[H]] +} + +func (s HeightStoreImpl[H]) GetByHeight(ctx context.Context, height uint64) (H, error) { + var zero H + v, err := s.store.GetByHeight(ctx, height) + if err != nil { + return zero, err + } + return v.Entry, nil + } diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index d09d01eab1..7f300aefd9 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -37,8 +37,8 @@ type Executor struct { metrics *common.Metrics // Broadcasting - headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint] - dataBroadcaster common.Broadcaster[*types.Data] + headerBroadcaster common.HeaderP2PBroadcaster + dataBroadcaster common.DataP2PBroadcaster // Configuration config config.Config @@ -76,8 +76,8 @@ func NewExecutor( metrics *common.Metrics, config config.Config, genesis genesis.Genesis, - headerBroadcaster common.Broadcaster[*types.SignedHeaderWithDAHint], - dataBroadcaster common.Broadcaster[*types.Data], + headerBroadcaster common.HeaderP2PBroadcaster, + dataBroadcaster common.DataP2PBroadcaster, logger zerolog.Logger, options common.BlockOptions, errorCh chan<- error, @@ -420,7 +420,7 @@ func (e *Executor) produceBlock() error { // broadcast header and data to P2P network g, ctx := errgroup.WithContext(e.ctx) - g.Go(func() error { return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, &types.SignedHeaderWithDAHint{SignedHeader: header}) }) + g.Go(func() error { return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, header) }) g.Go(func() error { return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, data) }) if err := g.Wait(); err != nil { e.logger.Error().Err(err).Msg("failed to broadcast header and/data") diff --git a/block/internal/submitting/da_submitter.go b/block/internal/submitting/da_submitter.go index 5123429bf2..408ab0a2b7 100644 --- a/block/internal/submitting/da_submitter.go +++ b/block/internal/submitting/da_submitter.go @@ -93,19 +93,20 @@ func clamp(v, min, max time.Duration) time.Duration { return v } -type xxxer interface { - XXX(ctx context.Context, header *types.SignedHeaderWithDAHint) error +type DAHintAppender interface { + AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error } // DASubmitter handles DA submission operations type DASubmitter struct { - client da.Client - config config.Config - genesis genesis.Genesis - options common.BlockOptions - logger zerolog.Logger - metrics *common.Metrics - xxxer xxxer + client da.Client + config config.Config + genesis genesis.Genesis + options common.BlockOptions + logger zerolog.Logger + metrics *common.Metrics + headerDAHintAppender DAHintAppender + dataDAHintAppender DAHintAppender // address selector for multi-account support addressSelector pkgda.AddressSelector @@ -119,7 +120,8 @@ func NewDASubmitter( options common.BlockOptions, metrics *common.Metrics, logger zerolog.Logger, - xxxer xxxer, + headerDAHintAppender DAHintAppender, + dataDAHintAppender DAHintAppender, ) *DASubmitter { daSubmitterLogger := logger.With().Str("component", "da_submitter").Logger() @@ -145,14 +147,15 @@ func NewDASubmitter( } return &DASubmitter{ - client: client, - config: config, - genesis: genesis, - options: options, - metrics: metrics, - logger: daSubmitterLogger, - addressSelector: addressSelector, - xxxer: xxxer, + client: client, + config: config, + genesis: genesis, + options: options, + metrics: metrics, + logger: daSubmitterLogger, + addressSelector: addressSelector, + headerDAHintAppender: headerDAHintAppender, + dataDAHintAppender: dataDAHintAppender, } } @@ -192,13 +195,15 @@ func (s *DASubmitter) SubmitHeaders(ctx context.Context, cache cache.Manager) er return proto.Marshal(headerPb) }, func(submitted []*types.SignedHeader, res *coreda.ResultSubmit) { - for _, header := range submitted { - cache.SetHeaderDAIncluded(header.Hash().String(), res.Height, header.Height()) - payload := &types.SignedHeaderWithDAHint{SignedHeader: header, DAHeightHint: res.Height} - if err := s.xxxer.XXX(ctx, payload); err != nil { - s.logger.Error().Err(err).Msg("failed to update header in p2p store") - // ignoring error here, since we don't want to block the block submission' - } + hashes := make([]types.Hash, len(submitted)) + for i, header := range submitted { + headerHash := header.Hash() + cache.SetHeaderDAIncluded(headerHash.String(), res.Height, header.Height()) + hashes[i] = headerHash + } + if err := s.headerDAHintAppender.AppendDAHint(ctx, res.Height, hashes...); err != nil { + s.logger.Error().Err(err).Msg("failed to append da height hint in header p2p store") + // ignoring error here, since we don't want to block the block submission' } if l := len(submitted); l > 0 { lastHeight := submitted[l-1].Height() @@ -240,8 +245,14 @@ func (s *DASubmitter) SubmitData(ctx context.Context, cache cache.Manager, signe return signedData.MarshalBinary() }, func(submitted []*types.SignedData, res *coreda.ResultSubmit) { - for _, sd := range submitted { + hashes := make([]types.Hash, len(submitted)) + for i, sd := range submitted { cache.SetDataDAIncluded(sd.Data.DACommitment().String(), res.Height, sd.Height()) + hashes[i] = sd.Hash() + } + if err := s.dataDAHintAppender.AppendDAHint(ctx, res.Height, hashes...); err != nil { + s.logger.Error().Err(err).Msg("failed to append da height hint in data p2p store") + // ignoring error here, since we don't want to block the block submission' } if l := len(submitted); l > 0 { lastHeight := submitted[l-1].Height() diff --git a/block/internal/submitting/da_submitter_integration_test.go b/block/internal/submitting/da_submitter_integration_test.go index 854b0950d2..ac35927a04 100644 --- a/block/internal/submitting/da_submitter_integration_test.go +++ b/block/internal/submitting/da_submitter_integration_test.go @@ -113,6 +113,6 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( type noopXXXer struct{} -func (n noopXXXer) XXX(ctx context.Context, header *types.SignedHeaderWithDAHint) error { +func (n noopXXXer) AppendDAHint(ctx context.Context, header *types.SignedHeaderWithDAHint) error { return nil } diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index a5d4b53f26..ef3c19d4ec 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -6,7 +6,6 @@ import ( "fmt" "sync/atomic" - goheader "github.com/celestiaorg/go-header" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" @@ -27,8 +26,8 @@ type p2pHandler interface { // The handler maintains a processedHeight to track the highest block that has been // successfully validated and sent to the syncer, preventing duplicate processing. type P2PHandler struct { - headerStore goheader.Store[*types.SignedHeaderWithDAHint] - dataStore goheader.Store[*types.Data] + headerStore common.HeightStore[*types.SignedHeaderWithDAHint] + dataStore common.HeightStore[*types.DataWithDAHint] cache cache.CacheManager genesis genesis.Genesis logger zerolog.Logger @@ -38,8 +37,8 @@ type P2PHandler struct { // NewP2PHandler creates a new P2P handler. func NewP2PHandler( - headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.Data], + headerStore common.HeightStore[*types.SignedHeaderWithDAHint], + dataStore common.HeightStore[*types.DataWithDAHint], cache cache.CacheManager, genesis genesis.Genesis, logger zerolog.Logger, @@ -74,26 +73,27 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC return nil } - header, err := h.headerStore.GetByHeight(ctx, height) + headerTuple, err := h.headerStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("header unavailable in store") } return err } + header := headerTuple.Entry if err := h.assertExpectedProposer(header.ProposerAddress); err != nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("invalid header from P2P") return err } - data, err := h.dataStore.GetByHeight(ctx, height) + dataTuple, err := h.dataStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("data unavailable in store") } return err } - + data := dataTuple.Entry dataCommitment := data.DACommitment() if !bytes.Equal(header.DataHash[:], dataCommitment[:]) { err := fmt.Errorf("data hash mismatch: header %x, data %x", header.DataHash, dataCommitment) @@ -104,10 +104,10 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC // further header validation (signature) is done in validateBlock. // we need to be sure that the previous block n-1 was executed before validating block n event := common.DAHeightEvent{ - Header: header.SignedHeader, - Data: data, - Source: common.SourceP2P, - DaHeightHint: header.DAHeightHint, + Header: header, + Data: data, + Source: common.SourceP2P, + DaHeightHints: [2]uint64{headerTuple.DAHeightHint, dataTuple.DAHeightHint}, } select { diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index a40e30fbd0..ff42eccd37 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -10,20 +10,18 @@ import ( "sync/atomic" "time" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/rs/zerolog" - "golang.org/x/sync/errgroup" - - coreda "github.com/evstack/ev-node/core/da" - coreexecutor "github.com/evstack/ev-node/core/execution" - "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" "github.com/evstack/ev-node/block/internal/da" + coreda "github.com/evstack/ev-node/core/da" + coreexecutor "github.com/evstack/ev-node/core/execution" "github.com/evstack/ev-node/pkg/config" "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/store" "github.com/evstack/ev-node/types" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" ) // Syncer handles block synchronization from DA and P2P sources. @@ -49,8 +47,8 @@ type Syncer struct { daRetrieverHeight *atomic.Uint64 // P2P stores - headerStore common.Broadcaster[*types.SignedHeaderWithDAHint] - dataStore common.Broadcaster[*types.Data] + headerStore common.HeaderP2PBroadcaster + dataStore common.DataP2PBroadcaster // Channels for coordination heightInCh chan common.DAHeightEvent @@ -81,8 +79,8 @@ func NewSyncer( metrics *common.Metrics, config config.Config, genesis genesis.Genesis, - headerStore common.Broadcaster[*types.SignedHeaderWithDAHint], - dataStore common.Broadcaster[*types.Data], + headerStore common.HeaderP2PBroadcaster, + dataStore common.DataP2PBroadcaster, logger zerolog.Logger, options common.BlockOptions, errorCh chan<- error, @@ -116,7 +114,7 @@ func (s *Syncer) Start(ctx context.Context) error { // Initialize handlers s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger) - s.p2pHandler = NewP2PHandler(s.headerStore.Store(), s.dataStore.Store(), s.cache, s.genesis, s.logger) + s.p2pHandler = NewP2PHandler(s.headerStore.XStore(), s.dataStore.XStore(), s.cache, s.genesis, s.logger) if currentHeight, err := s.store.Height(s.ctx); err != nil { s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") } else { @@ -434,36 +432,64 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { // If this is a P2P event with a DA height hint, trigger targeted DA retrieval // This allows us to fetch the block directly from the specified DA height instead of sequential scanning - if event.Source == common.SourceP2P && event.DaHeightHint != 0 { - if _, exists := s.cache.GetHeaderDAIncluded(event.Header.Hash().String()); !exists { - s.logger.Debug(). - Uint64("height", height). - Uint64("da_height_hint", event.DaHeightHint). - Msg("P2P event with DA height hint, triggering targeted DA retrieval") - - // Trigger targeted DA retrieval in background - go func() { - targetEvents, err := s.daRetriever.RetrieveFromDA(s.ctx, event.DaHeightHint) - if err != nil { - s.logger.Debug(). - Err(err). - Uint64("da_height", event.DaHeightHint). - Msg("targeted DA retrieval failed (hint may be incorrect or DA not yet available)") - // Not a critical error - the sequential DA worker will eventually find it - return - } - - // Process retrieved events from the targeted DA height - for _, daEvent := range targetEvents { - select { - case s.heightInCh <- daEvent: - case <-s.ctx.Done(): + if event.Source == common.SourceP2P { + var daHeightHints []uint64 + switch { + case event.DaHeightHints == [2]uint64{0, 0}: + // empty, nothing to do + case event.DaHeightHints[0] == 0: + // check only data + if _, exists := s.cache.GetDataDAIncluded(event.Data.Hash().String()); !exists { + daHeightHints = []uint64{event.DaHeightHints[1]} + } + case event.DaHeightHints[1] == 0: + // check only header + if _, exists := s.cache.GetHeaderDAIncluded(event.Header.Hash().String()); !exists { + daHeightHints = []uint64{event.DaHeightHints[0]} + } + default: + // check both + if _, exists := s.cache.GetDataDAIncluded(event.Data.Hash().String()); !exists { + daHeightHints = []uint64{event.DaHeightHints[1]} + } + if _, exists := s.cache.GetDataDAIncluded(event.Data.Hash().String()); !exists { + daHeightHints = append(daHeightHints, event.DaHeightHints[1]) + } + if len(daHeightHints) == 2 && daHeightHints[0] == daHeightHints[1] { + daHeightHints = daHeightHints[0:1] + } + } + if len(daHeightHints) > 0 { + for _, daHeightHint := range daHeightHints { + s.logger.Debug(). + Uint64("height", height). + Uint64("da_height_hint", daHeightHint). + Msg("P2P event with DA height hint, triggering targeted DA retrieval") + + // Trigger targeted DA retrieval in background + go func() { + targetEvents, err := s.daRetriever.RetrieveFromDA(s.ctx, daHeightHint) + if err != nil { + s.logger.Debug(). + Err(err). + Uint64("da_height", daHeightHint). + Msg("targeted DA retrieval failed (hint may be incorrect or DA not yet available)") + // Not a critical error - the sequential DA worker will eventually find it return - default: - s.cache.SetPendingEvent(daEvent.Header.Height(), &daEvent) } - } - }() + + // Process retrieved events from the targeted DA height + for _, daEvent := range targetEvents { + select { + case s.heightInCh <- daEvent: + case <-s.ctx.Done(): + return + default: + s.cache.SetPendingEvent(daEvent.Header.Height(), &daEvent) + } + } + }() + } } } @@ -504,8 +530,7 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { g.Go(func() error { // broadcast header locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - payload := &types.SignedHeaderWithDAHint{SignedHeader: event.Header, DAHeightHint: event.DaHeightHint} - return s.headerStore.WriteToStoreAndBroadcast(ctx, payload, pubsub.WithLocalPublication(true)) + return s.headerStore.WriteToStoreAndBroadcast(ctx, event.Header, pubsub.WithLocalPublication(true)) }) g.Go(func() error { // broadcast data locally only — prevents spamming the p2p network with old height notifications, diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 6fb9ee8362..0a8546d3ae 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -35,7 +35,7 @@ var _ rpc.StoreServiceHandler = (*StoreServer)(nil) type StoreServer struct { store store.Store headerStore goheader.Store[*types.SignedHeaderWithDAHint] - dataStore goheader.Store[*types.Data] + dataStore goheader.Store[*types.DataWithDAHint] logger zerolog.Logger } @@ -43,7 +43,7 @@ type StoreServer struct { func NewStoreServer( store store.Store, headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.Data], + dataStore goheader.Store[*types.DataWithDAHint], logger zerolog.Logger, ) *StoreServer { return &StoreServer{ @@ -371,7 +371,7 @@ func (p *P2PServer) GetNetInfo( func NewServiceHandler( store store.Store, headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.Data], + dataStore goheader.Store[*types.DataWithDAHint], peerManager p2p.P2PRPC, proposerAddress []byte, logger zerolog.Logger, diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index f4c63c33a8..3ca112e983 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -36,10 +36,21 @@ const ( // TODO: when we add pruning we can remove this const ninetyNineYears = 99 * 365 * 24 * time.Hour +type EntityWithDAHint[H any] interface { + header.Header[H] + SetDAHint(daHeight uint64) +} + +// DataSyncService is the P2P Sync Service for blocks. +type DataSyncService = SyncService[*types.DataWithDAHint] + +// HeaderSyncService is the P2P Sync Service for headers. +type HeaderSyncService = SyncService[*types.SignedHeaderWithDAHint] + // SyncService is the P2P Sync Service for blocks and headers. // // Uses the go-header library for handling all P2P logic. -type SyncService[H header.Header[H]] struct { +type SyncService[H EntityWithDAHint[H]] struct { conf config.Config logger zerolog.Logger syncType syncType @@ -58,12 +69,6 @@ type SyncService[H header.Header[H]] struct { storeInitialized atomic.Bool } -// DataSyncService is the P2P Sync Service for blocks. -type DataSyncService = SyncService[*types.Data] - -// HeaderSyncService is the P2P Sync Service for headers. -type HeaderSyncService = SyncService[*types.SignedHeaderWithDAHint] - // NewDataSyncService returns a new DataSyncService. func NewDataSyncService( store ds.Batching, @@ -72,7 +77,7 @@ func NewDataSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*DataSyncService, error) { - return newSyncService[*types.Data](store, dataSync, conf, genesis, p2p, logger) + return newSyncService[*types.DataWithDAHint](store, dataSync, conf, genesis, p2p, logger) } // NewHeaderSyncService returns a new HeaderSyncService. @@ -86,7 +91,7 @@ func NewHeaderSyncService( return newSyncService[*types.SignedHeaderWithDAHint](store, headerSync, conf, genesis, p2p, logger) } -func newSyncService[H header.Header[H]]( +func newSyncService[H EntityWithDAHint[H]]( store ds.Batching, syncType syncType, conf config.Config, @@ -174,8 +179,17 @@ func (syncService *SyncService[H]) WriteToStoreAndBroadcast(ctx context.Context, return nil } -func (s *SyncService[H]) XXX(ctx context.Context, headerOrData H) error { - return s.store.Append(ctx, headerOrData) +func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { + entries := make([]H, 0, len(hashes)) + for _, h := range hashes { + v, err := s.store.Get(ctx, h) + if err != nil && !errors.Is(err, header.ErrNotFound) { + return err + } + v.SetDAHint(daHeight) + entries = append(entries, v) + } + return s.store.Append(ctx, entries...) } // Start is a part of Service interface. diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index aeaeda18b8..b8063e874b 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -169,8 +169,8 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { func nextHeader(t *testing.T, previousHeader *types.SignedHeaderWithDAHint, chainID string, noopSigner signer.Signer) *types.SignedHeaderWithDAHint { newSignedHeader := &types.SignedHeader{ - Header: types.GetRandomNextHeader(previousHeader.Header, chainID), - Signer: previousHeader.Signer, + Header: types.GetRandomNextHeader(previousHeader.Entry.Header, chainID), + Signer: previousHeader.Entry.Signer, } b, err := newSignedHeader.Header.MarshalBinary() require.NoError(t, err) @@ -178,7 +178,7 @@ func nextHeader(t *testing.T, previousHeader *types.SignedHeaderWithDAHint, chai require.NoError(t, err) newSignedHeader.Signature = signature require.NoError(t, newSignedHeader.Validate()) - return &types.SignedHeaderWithDAHint{SignedHeader: newSignedHeader} + return &types.SignedHeaderWithDAHint{Entry: newSignedHeader} } func bytesN(r *rand.Rand, n int) []byte { diff --git a/types/da_hint_container.go b/types/da_hint_container.go new file mode 100644 index 0000000000..9e905c6296 --- /dev/null +++ b/types/da_hint_container.go @@ -0,0 +1,74 @@ +package types + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/celestiaorg/go-header" +) + +type DAHeightHintContainer[H header.Header[H]] struct { + Entry H + DAHeightHint uint64 +} + +func (s *DAHeightHintContainer[H]) ChainID() string { + return s.Entry.ChainID() +} + +func (s *DAHeightHintContainer[H]) Hash() header.Hash { + return s.Entry.Hash() +} + +func (s *DAHeightHintContainer[H]) Height() uint64 { + return s.Entry.Height() +} + +func (s *DAHeightHintContainer[H]) LastHeader() header.Hash { + return s.Entry.LastHeader() +} + +func (s *DAHeightHintContainer[H]) Time() time.Time { + return s.Entry.Time() +} + +func (s *DAHeightHintContainer[H]) Validate() error { + return s.Entry.Validate() +} + +func (s *DAHeightHintContainer[H]) New() *DAHeightHintContainer[H] { + var empty H + return &DAHeightHintContainer[H]{Entry: empty.New()} +} + +func (sh *DAHeightHintContainer[H]) Verify(untrstH *DAHeightHintContainer[H]) error { + return sh.Entry.Verify(untrstH.Entry) +} + +func (s *DAHeightHintContainer[H]) SetDAHint(daHeight uint64) { + s.DAHeightHint = daHeight +} + +func (s *DAHeightHintContainer[H]) IsZero() bool { + return s == nil +} + +func (s *DAHeightHintContainer[H]) MarshalBinary() ([]byte, error) { + bz, err := s.Entry.MarshalBinary() + if err != nil { + return nil, err + } + out := make([]byte, 8+len(bz)) + binary.BigEndian.PutUint64(out, s.DAHeightHint) + copy(out[8:], bz) + return out, nil +} + +func (s *DAHeightHintContainer[H]) UnmarshalBinary(data []byte) error { + if len(data) < 8 { + return fmt.Errorf("invalid length: %d", len(data)) + } + s.DAHeightHint = binary.BigEndian.Uint64(data) + return s.Entry.UnmarshalBinary(data[8:]) +} diff --git a/types/signed_header.go b/types/signed_header.go index d0b322321d..c63ecce3ac 100644 --- a/types/signed_header.go +++ b/types/signed_header.go @@ -3,7 +3,6 @@ package types import ( "bytes" "context" - "encoding/binary" "errors" "fmt" @@ -15,46 +14,8 @@ var ( ErrLastHeaderHashMismatch = errors.New("last header hash mismatch") ) -var _ header.Header[*SignedHeaderWithDAHint] = &SignedHeaderWithDAHint{} - -type SignedHeaderWithDAHint struct { - *SignedHeader - DAHeightHint uint64 -} - -func (s *SignedHeaderWithDAHint) New() *SignedHeaderWithDAHint { - return &SignedHeaderWithDAHint{SignedHeader: &SignedHeader{}} -} -func (sh *SignedHeaderWithDAHint) Verify(untrstH *SignedHeaderWithDAHint) error { - return sh.SignedHeader.Verify(untrstH.SignedHeader) -} - -func (s *SignedHeaderWithDAHint) Zero() bool { - return s == nil -} - -func (s *SignedHeaderWithDAHint) IsZero() bool { - return s == nil -} - -func (s *SignedHeaderWithDAHint) MarshalBinary() ([]byte, error) { - bz, err := s.SignedHeader.MarshalBinary() - if err != nil { - return nil, err - } - out := make([]byte, 8+len(bz)) - binary.BigEndian.PutUint64(out, s.DAHeightHint) - copy(out[8:], bz) - return out, nil -} - -func (s *SignedHeaderWithDAHint) UnmarshalBinary(data []byte) error { - if len(data) < 8 { - return fmt.Errorf("invalid length: %d", len(data)) - } - s.DAHeightHint = binary.BigEndian.Uint64(data) - return s.SignedHeader.UnmarshalBinary(data[8:]) -} +type SignedHeaderWithDAHint = DAHeightHintContainer[*SignedHeader] +type DataWithDAHint = DAHeightHintContainer[*Data] var _ header.Header[*SignedHeader] = &SignedHeader{} diff --git a/types/utils.go b/types/utils.go index b0e28c80a8..fe59fb20dc 100644 --- a/types/utils.go +++ b/types/utils.go @@ -68,7 +68,7 @@ func GenerateRandomBlockCustomWithAppHash(config *BlockConfig, chainID string, a } if config.ProposerAddr != nil { - signedHeader.ProposerAddress = config.ProposerAddr + signedHeader.Entry.ProposerAddress = config.ProposerAddr } data.Metadata = &Metadata{ @@ -78,7 +78,7 @@ func GenerateRandomBlockCustomWithAppHash(config *BlockConfig, chainID string, a Time: uint64(signedHeader.Time().UnixNano()), } - return signedHeader.SignedHeader, data, config.PrivKey + return signedHeader.Entry, data, config.PrivKey } // GenerateRandomBlockCustom returns a block with random data and the given height, transactions, privateKey and proposer address. @@ -150,7 +150,7 @@ func GetRandomSignedHeader(chainID string) (*SignedHeader, crypto.PrivKey, error if err != nil { return nil, nil, err } - return signedHeader.SignedHeader, pk, nil + return signedHeader.Entry, pk, nil } // GetRandomSignedHeaderCustom creates a signed header based on the provided HeaderConfig. @@ -183,7 +183,7 @@ func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedH return nil, err } signedHeader.Signature = signature - return &SignedHeaderWithDAHint{SignedHeader: signedHeader}, nil + return &SignedHeaderWithDAHint{Entry: signedHeader}, nil } // GetRandomNextSignedHeader returns a signed header with random data and height of +1 from From 2c9a2128381ab5abb3a5c4e560e8517fb9ff3f26 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 28 Nov 2025 09:20:20 +0100 Subject: [PATCH 03/41] Review feedback --- block/internal/syncing/syncer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index ff42eccd37..b73b8f572d 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -449,8 +449,8 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { } default: // check both - if _, exists := s.cache.GetDataDAIncluded(event.Data.Hash().String()); !exists { - daHeightHints = []uint64{event.DaHeightHints[1]} + if _, exists := s.cache.GetHeaderDAIncluded(event.Header.Hash().String()); !exists { + daHeightHints = []uint64{event.DaHeightHints[0]} } if _, exists := s.cache.GetDataDAIncluded(event.Data.Hash().String()); !exists { daHeightHints = append(daHeightHints, event.DaHeightHints[1]) From e2b0520986a5af78591ffe07dc726b8880e0d50d Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 28 Nov 2025 13:37:01 +0100 Subject: [PATCH 04/41] Encapsulate hint in sync package --- .mockery.yaml | 5 + apps/evm/cmd/rollback.go | 6 +- apps/testapp/cmd/rollback.go | 9 +- block/components.go | 9 +- block/internal/common/broadcaster_mock.go | 66 +++++++--- block/internal/common/expected_interfaces.go | 72 ++++------- .../internal/executing/executor_lazy_test.go | 4 +- .../internal/executing/executor_logic_test.go | 4 +- .../executing/executor_restart_test.go | 8 +- block/internal/executing/executor_test.go | 2 +- .../da_submitter_integration_test.go | 6 +- .../submitting/da_submitter_mocks_test.go | 2 +- .../internal/submitting/da_submitter_test.go | 6 +- block/internal/submitting/submitter_test.go | 6 +- block/internal/syncing/height_store_mock.go | 113 ++++++++++++++++++ block/internal/syncing/p2p_handler.go | 24 ++-- block/internal/syncing/p2p_handler_test.go | 35 +++--- block/internal/syncing/syncer.go | 2 +- block/internal/syncing/syncer_backoff_test.go | 32 +---- .../internal/syncing/syncer_benchmark_test.go | 2 +- block/internal/syncing/syncer_test.go | 13 +- node/full.go | 2 +- pkg/rpc/client/client_test.go | 17 +-- pkg/rpc/server/server.go | 13 +- pkg/rpc/server/server_test.go | 18 +-- {types => pkg/sync}/da_hint_container.go | 9 +- pkg/sync/sync_service.go | 88 ++++++++------ pkg/sync/sync_service_test.go | 8 +- types/signed_header.go | 3 - types/utils.go | 10 +- 30 files changed, 356 insertions(+), 238 deletions(-) create mode 100644 block/internal/syncing/height_store_mock.go rename {types => pkg/sync}/da_hint_container.go (86%) diff --git a/.mockery.yaml b/.mockery.yaml index 8f139231cb..77930d950b 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -58,6 +58,11 @@ packages: dir: ./block/internal/syncing pkgname: syncing filename: syncer_mock.go + HeightStore: + config: + dir: ./block/internal/syncing + pkgname: syncing + filename: height_store_mock.go github.com/evstack/ev-node/block/internal/common: interfaces: Broadcaster: diff --git a/apps/evm/cmd/rollback.go b/apps/evm/cmd/rollback.go index 4a75f9a726..bb34d6a06a 100644 --- a/apps/evm/cmd/rollback.go +++ b/apps/evm/cmd/rollback.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" kt "github.com/ipfs/go-datastore/keytransform" "github.com/spf13/cobra" @@ -13,7 +14,6 @@ import ( "github.com/evstack/ev-node/node" rollcmd "github.com/evstack/ev-node/pkg/cmd" "github.com/evstack/ev-node/pkg/store" - "github.com/evstack/ev-node/types" ) // NewRollbackCmd creates a command to rollback ev-node state by one height. @@ -70,7 +70,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*types.SignedHeaderWithDAHint]( + headerStore, err := goheaderstore.NewStore[*sync.SignedHeaderWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), @@ -79,7 +79,7 @@ func NewRollbackCmd() *cobra.Command { return err } - dataStore, err := goheaderstore.NewStore[*types.Data]( + dataStore, err := goheaderstore.NewStore[*sync.DataWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("dataSync"), goheaderstore.WithMetrics(), diff --git a/apps/testapp/cmd/rollback.go b/apps/testapp/cmd/rollback.go index 761ec207d5..42be93cf18 100644 --- a/apps/testapp/cmd/rollback.go +++ b/apps/testapp/cmd/rollback.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" + goheaderstore "github.com/celestiaorg/go-header/store" kvexecutor "github.com/evstack/ev-node/apps/testapp/kv" "github.com/evstack/ev-node/node" rollcmd "github.com/evstack/ev-node/pkg/cmd" "github.com/evstack/ev-node/pkg/store" - "github.com/evstack/ev-node/types" - - goheaderstore "github.com/celestiaorg/go-header/store" + "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" kt "github.com/ipfs/go-datastore/keytransform" "github.com/spf13/cobra" @@ -76,7 +75,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*types.SignedHeaderWithDAHint]( + headerStore, err := goheaderstore.NewStore[*sync.SignedHeaderWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), @@ -85,7 +84,7 @@ func NewRollbackCmd() *cobra.Command { return err } - dataStore, err := goheaderstore.NewStore[*types.Data]( + dataStore, err := goheaderstore.NewStore[*sync.DataWithDAHint]( evolveDB, goheaderstore.WithStorePrefix("dataSync"), goheaderstore.WithMetrics(), diff --git a/block/components.go b/block/components.go index 4b5316eab4..298a97e379 100644 --- a/block/components.go +++ b/block/components.go @@ -9,7 +9,6 @@ import ( "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" - "github.com/evstack/ev-node/block/internal/common" "github.com/evstack/ev-node/block/internal/executing" "github.com/evstack/ev-node/block/internal/reaping" "github.com/evstack/ev-node/block/internal/submitting" @@ -158,8 +157,8 @@ func NewSyncComponents( metrics, config, genesis, - common.NewDecorator[*types.SignedHeader](headerStore), - common.NewDecorator[*types.Data](dataStore), + headerStore, + dataStore, logger, blockOpts, errorCh, @@ -223,8 +222,8 @@ func NewAggregatorComponents( metrics, config, genesis, - common.NewDecorator[*types.SignedHeader](headerBroadcaster), - common.NewDecorator[*types.Data](dataBroadcaster), + headerBroadcaster, + dataBroadcaster, logger, blockOpts, errorCh, diff --git a/block/internal/common/broadcaster_mock.go b/block/internal/common/broadcaster_mock.go index e761aa624a..39d748f5cb 100644 --- a/block/internal/common/broadcaster_mock.go +++ b/block/internal/common/broadcaster_mock.go @@ -116,48 +116,76 @@ func (_c *MockBroadcaster_AppendDAHint_Call[H]) RunAndReturn(run func(ctx contex return _c } -// Store provides a mock function for the type MockBroadcaster -func (_mock *MockBroadcaster[H]) Store() header.Store[H] { - ret := _mock.Called() +// GetByHeight provides a mock function for the type MockBroadcaster +func (_mock *MockBroadcaster[H]) GetByHeight(ctx context.Context, height uint64) (H, uint64, error) { + ret := _mock.Called(ctx, height) if len(ret) == 0 { - panic("no return value specified for Store") + panic("no return value specified for GetByHeight") } - var r0 header.Store[H] - if returnFunc, ok := ret.Get(0).(func() header.Store[H]); ok { - r0 = returnFunc() + var r0 H + var r1 uint64 + var r2 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) (H, uint64, error)); ok { + return returnFunc(ctx, height) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) H); ok { + r0 = returnFunc(ctx, height) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(header.Store[H]) + r0 = ret.Get(0).(H) } } - return r0 + if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) uint64); ok { + r1 = returnFunc(ctx, height) + } else { + r1 = ret.Get(1).(uint64) + } + if returnFunc, ok := ret.Get(2).(func(context.Context, uint64) error); ok { + r2 = returnFunc(ctx, height) + } else { + r2 = ret.Error(2) + } + return r0, r1, r2 } -// MockBroadcaster_Store_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Store' -type MockBroadcaster_Store_Call[H header.Header[H]] struct { +// MockBroadcaster_GetByHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByHeight' +type MockBroadcaster_GetByHeight_Call[H header.Header[H]] struct { *mock.Call } -// Store is a helper method to define mock.On call -func (_e *MockBroadcaster_Expecter[H]) Store() *MockBroadcaster_Store_Call[H] { - return &MockBroadcaster_Store_Call[H]{Call: _e.mock.On("Store")} +// GetByHeight is a helper method to define mock.On call +// - ctx context.Context +// - height uint64 +func (_e *MockBroadcaster_Expecter[H]) GetByHeight(ctx interface{}, height interface{}) *MockBroadcaster_GetByHeight_Call[H] { + return &MockBroadcaster_GetByHeight_Call[H]{Call: _e.mock.On("GetByHeight", ctx, height)} } -func (_c *MockBroadcaster_Store_Call[H]) Run(run func()) *MockBroadcaster_Store_Call[H] { +func (_c *MockBroadcaster_GetByHeight_Call[H]) Run(run func(ctx context.Context, height uint64)) *MockBroadcaster_GetByHeight_Call[H] { _c.Call.Run(func(args mock.Arguments) { - run() + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uint64 + if args[1] != nil { + arg1 = args[1].(uint64) + } + run( + arg0, + arg1, + ) }) return _c } -func (_c *MockBroadcaster_Store_Call[H]) Return(store header.Store[H]) *MockBroadcaster_Store_Call[H] { - _c.Call.Return(store) +func (_c *MockBroadcaster_GetByHeight_Call[H]) Return(v H, v1 uint64, err error) *MockBroadcaster_GetByHeight_Call[H] { + _c.Call.Return(v, v1, err) return _c } -func (_c *MockBroadcaster_Store_Call[H]) RunAndReturn(run func() header.Store[H]) *MockBroadcaster_Store_Call[H] { +func (_c *MockBroadcaster_GetByHeight_Call[H]) RunAndReturn(run func(ctx context.Context, height uint64) (H, uint64, error)) *MockBroadcaster_GetByHeight_Call[H] { _c.Call.Return(run) return _c } diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index 59a0991670..c880a5b543 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -10,56 +10,34 @@ import ( ) type ( - HeaderP2PBroadcaster = Decorator[*types.SignedHeader] - DataP2PBroadcaster = Decorator[*types.Data] + HeaderP2PBroadcaster = Broadcaster[*types.SignedHeader] + DataP2PBroadcaster = Broadcaster[*types.Data] ) // Broadcaster interface for P2P broadcasting type Broadcaster[H header.Header[H]] interface { WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error - Store() header.Store[H] AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error -} - -// Decorator to access the the payload type without the container -type Decorator[H header.Header[H]] struct { - nested Broadcaster[*types.DAHeightHintContainer[H]] -} - -func NewDecorator[H header.Header[H]](nested Broadcaster[*types.DAHeightHintContainer[H]]) Decorator[H] { - return Decorator[H]{nested: nested} -} - -func (d Decorator[H]) WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error { - return d.nested.WriteToStoreAndBroadcast(ctx, &types.DAHeightHintContainer[H]{Entry: payload}, opts...) -} - -func (d Decorator[H]) Store() HeightStore[H] { - return HeightStoreImpl[H]{store: d.nested.Store()} -} -func (d Decorator[H]) XStore() header.Store[*types.DAHeightHintContainer[H]] { - return d.nested.Store() -} - -func (d Decorator[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { - return d.nested.AppendDAHint(ctx, daHeight, hashes...) -} - -// HeightStore is a subset of goheader.Store -type HeightStore[H header.Header[H]] interface { - GetByHeight(context.Context, uint64) (H, error) -} - -type HeightStoreImpl[H header.Header[H]] struct { - store header.Store[*types.DAHeightHintContainer[H]] -} - -func (s HeightStoreImpl[H]) GetByHeight(ctx context.Context, height uint64) (H, error) { - var zero H - v, err := s.store.GetByHeight(ctx, height) - if err != nil { - return zero, err - } - return v.Entry, nil - -} + GetByHeight(ctx context.Context, height uint64) (H, uint64, error) +} + +// +//// Decorator to access the payload type without the container +//type Decorator[H header.Header[H]] struct { +// nested Broadcaster[*sync.DAHeightHintContainer[H]] +//} +// +//func NewDecorator[H header.Header[H]](nested Broadcaster[*sync.DAHeightHintContainer[H]]) Decorator[H] { +// return Decorator[H]{nested: nested} +//} +// +//func (d Decorator[H]) WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error { +// return d.nested.WriteToStoreAndBroadcast(ctx, &sync.DAHeightHintContainer[H]{Entry: payload}, opts...) +//} +// +//func (d Decorator[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { +// return d.nested.AppendDAHint(ctx, daHeight, hashes...) +//} +//func (d Decorator[H]) GetByHeight(ctx context.Context, height uint64) (H, error) { +// panic("not implemented") +//} diff --git a/block/internal/executing/executor_lazy_test.go b/block/internal/executing/executor_lazy_test.go index 25b784b29d..b72f0a856b 100644 --- a/block/internal/executing/executor_lazy_test.go +++ b/block/internal/executing/executor_lazy_test.go @@ -47,7 +47,7 @@ func TestLazyMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb := common.NewMockBroadcaster[*types.SignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -157,7 +157,7 @@ func TestRegularMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb := common.NewMockBroadcaster[*types.SignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_logic_test.go b/block/internal/executing/executor_logic_test.go index c615e458cc..9aa79d0c43 100644 --- a/block/internal/executing/executor_logic_test.go +++ b/block/internal/executing/executor_logic_test.go @@ -69,7 +69,7 @@ func TestProduceBlock_EmptyBatch_SetsEmptyDataHash(t *testing.T) { mockSeq := testmocks.NewMockSequencer(t) // Broadcasters are required by produceBlock; use generated mocks - hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb := common.NewMockBroadcaster[*types.SignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -156,7 +156,7 @@ func TestPendingLimit_SkipsProduction(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb := common.NewMockBroadcaster[*types.SignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db := common.NewMockBroadcaster[*types.Data](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_restart_test.go b/block/internal/executing/executor_restart_test.go index b9a13f68b8..3f0e8b500c 100644 --- a/block/internal/executing/executor_restart_test.go +++ b/block/internal/executing/executor_restart_test.go @@ -47,7 +47,7 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create first executor instance mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db1 := common.NewMockBroadcaster[*types.Data](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -166,7 +166,7 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create second executor instance (restart scenario) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db2 := common.NewMockBroadcaster[*types.Data](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -264,7 +264,7 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create first executor and produce one block mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db1 := common.NewMockBroadcaster[*types.Data](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() @@ -316,7 +316,7 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create second executor (restart) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() db2 := common.NewMockBroadcaster[*types.Data](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() diff --git a/block/internal/executing/executor_test.go b/block/internal/executing/executor_test.go index 76a7d21748..7ef0f64e21 100644 --- a/block/internal/executing/executor_test.go +++ b/block/internal/executing/executor_test.go @@ -39,7 +39,7 @@ func TestExecutor_BroadcasterIntegration(t *testing.T) { } // Create mock broadcasters - headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) + headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeader](t) dataBroadcaster := common.NewMockBroadcaster[*types.Data](t) // Create executor with broadcasters diff --git a/block/internal/submitting/da_submitter_integration_test.go b/block/internal/submitting/da_submitter_integration_test.go index ac35927a04..943abbff71 100644 --- a/block/internal/submitting/da_submitter_integration_test.go +++ b/block/internal/submitting/da_submitter_integration_test.go @@ -93,7 +93,7 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSubmitter := NewDASubmitter(daClient, cfg, gen, common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), noopXXXer{}) + daSubmitter := NewDASubmitter(daClient, cfg, gen, common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), noopDAHintAppender{}, noopDAHintAppender{}) // Submit headers and data require.NoError(t, daSubmitter.SubmitHeaders(context.Background(), cm)) @@ -111,8 +111,8 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( } -type noopXXXer struct{} +type noopDAHintAppender struct{} -func (n noopXXXer) AppendDAHint(ctx context.Context, header *types.SignedHeaderWithDAHint) error { +func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error { return nil } diff --git a/block/internal/submitting/da_submitter_mocks_test.go b/block/internal/submitting/da_submitter_mocks_test.go index 1716b59929..045cf821e6 100644 --- a/block/internal/submitting/da_submitter_mocks_test.go +++ b/block/internal/submitting/da_submitter_mocks_test.go @@ -36,7 +36,7 @@ func newTestSubmitter(mockDA *mocks.MockDA, override func(*config.Config)) *DASu Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - return NewDASubmitter(daClient, cfg, genesis.Genesis{} /*options=*/, common.BlockOptions{}, common.NopMetrics(), zerolog.Nop(), nil) + return NewDASubmitter(daClient, cfg, genesis.Genesis{} /*options=*/, common.BlockOptions{}, common.NopMetrics(), zerolog.Nop(), nil, nil) } // marshal helper for simple items diff --git a/block/internal/submitting/da_submitter_test.go b/block/internal/submitting/da_submitter_test.go index f33aaab21f..02f06f0573 100644 --- a/block/internal/submitting/da_submitter_test.go +++ b/block/internal/submitting/da_submitter_test.go @@ -65,7 +65,8 @@ func setupDASubmitterTest(t *testing.T) (*DASubmitter, store.Store, cache.Manage common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), - noopXXXer{}, + noopDAHintAppender{}, + noopDAHintAppender{}, ) return daSubmitter, st, cm, dummyDA, gen @@ -116,7 +117,8 @@ func TestNewDASubmitterSetsVisualizerWhenEnabled(t *testing.T) { common.DefaultBlockOptions(), common.NopMetrics(), zerolog.Nop(), - nil, + noopDAHintAppender{}, + noopDAHintAppender{}, ) require.NotNil(t, server.GetDAVisualizationServer()) diff --git a/block/internal/submitting/submitter_test.go b/block/internal/submitting/submitter_test.go index f317e0bdec..dc4c8e142e 100644 --- a/block/internal/submitting/submitter_test.go +++ b/block/internal/submitting/submitter_test.go @@ -168,7 +168,7 @@ func TestSubmitter_setSequencerHeightToDAHeight(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil, nil) s := NewSubmitter(mockStore, nil, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) s.ctx = ctx @@ -253,7 +253,7 @@ func TestSubmitter_processDAInclusionLoop_advances(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil, nil) s := NewSubmitter(st, exec, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) // prepare two consecutive blocks in store with DA included in cache @@ -444,7 +444,7 @@ func TestSubmitter_CacheClearedOnHeightInclusion(t *testing.T) { Namespace: cfg.DA.Namespace, DataNamespace: cfg.DA.DataNamespace, }) - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil, nil) s := NewSubmitter(st, exec, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, zerolog.Nop(), nil) // Create test blocks diff --git a/block/internal/syncing/height_store_mock.go b/block/internal/syncing/height_store_mock.go new file mode 100644 index 0000000000..b4857accfa --- /dev/null +++ b/block/internal/syncing/height_store_mock.go @@ -0,0 +1,113 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package syncing + +import ( + "context" + + "github.com/celestiaorg/go-header" + mock "github.com/stretchr/testify/mock" +) + +// NewMockHeightStore creates a new instance of MockHeightStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockHeightStore[H header.Header[H]](t interface { + mock.TestingT + Cleanup(func()) +}) *MockHeightStore[H] { + mock := &MockHeightStore[H]{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockHeightStore is an autogenerated mock type for the HeightStore type +type MockHeightStore[H header.Header[H]] struct { + mock.Mock +} + +type MockHeightStore_Expecter[H header.Header[H]] struct { + mock *mock.Mock +} + +func (_m *MockHeightStore[H]) EXPECT() *MockHeightStore_Expecter[H] { + return &MockHeightStore_Expecter[H]{mock: &_m.Mock} +} + +// GetByHeight provides a mock function for the type MockHeightStore +func (_mock *MockHeightStore[H]) GetByHeight(ctx context.Context, height uint64) (H, uint64, error) { + ret := _mock.Called(ctx, height) + + if len(ret) == 0 { + panic("no return value specified for GetByHeight") + } + + var r0 H + var r1 uint64 + var r2 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) (H, uint64, error)); ok { + return returnFunc(ctx, height) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) H); ok { + r0 = returnFunc(ctx, height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(H) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) uint64); ok { + r1 = returnFunc(ctx, height) + } else { + r1 = ret.Get(1).(uint64) + } + if returnFunc, ok := ret.Get(2).(func(context.Context, uint64) error); ok { + r2 = returnFunc(ctx, height) + } else { + r2 = ret.Error(2) + } + return r0, r1, r2 +} + +// MockHeightStore_GetByHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByHeight' +type MockHeightStore_GetByHeight_Call[H header.Header[H]] struct { + *mock.Call +} + +// GetByHeight is a helper method to define mock.On call +// - ctx context.Context +// - height uint64 +func (_e *MockHeightStore_Expecter[H]) GetByHeight(ctx interface{}, height interface{}) *MockHeightStore_GetByHeight_Call[H] { + return &MockHeightStore_GetByHeight_Call[H]{Call: _e.mock.On("GetByHeight", ctx, height)} +} + +func (_c *MockHeightStore_GetByHeight_Call[H]) Run(run func(ctx context.Context, height uint64)) *MockHeightStore_GetByHeight_Call[H] { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uint64 + if args[1] != nil { + arg1 = args[1].(uint64) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockHeightStore_GetByHeight_Call[H]) Return(v H, v1 uint64, err error) *MockHeightStore_GetByHeight_Call[H] { + _c.Call.Return(v, v1, err) + return _c +} + +func (_c *MockHeightStore_GetByHeight_Call[H]) RunAndReturn(run func(ctx context.Context, height uint64) (H, uint64, error)) *MockHeightStore_GetByHeight_Call[H] { + _c.Call.Return(run) + return _c +} diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index ef3c19d4ec..18c3e6c634 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -6,12 +6,13 @@ import ( "fmt" "sync/atomic" + "github.com/celestiaorg/go-header" + "github.com/evstack/ev-node/types" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" "github.com/evstack/ev-node/pkg/genesis" - "github.com/evstack/ev-node/types" ) type p2pHandler interface { @@ -19,6 +20,11 @@ type p2pHandler interface { SetProcessedHeight(height uint64) } +// HeightStore is a subset of goheader.Store +type HeightStore[H header.Header[H]] interface { + GetByHeight(ctx context.Context, height uint64) (H, uint64, error) +} + // P2PHandler coordinates block retrieval from P2P stores for the syncer. // It waits for both header and data to be available at a given height, // validates their consistency, and emits events to the syncer for processing. @@ -26,8 +32,8 @@ type p2pHandler interface { // The handler maintains a processedHeight to track the highest block that has been // successfully validated and sent to the syncer, preventing duplicate processing. type P2PHandler struct { - headerStore common.HeightStore[*types.SignedHeaderWithDAHint] - dataStore common.HeightStore[*types.DataWithDAHint] + headerStore HeightStore[*types.SignedHeader] + dataStore HeightStore[*types.Data] cache cache.CacheManager genesis genesis.Genesis logger zerolog.Logger @@ -37,8 +43,8 @@ type P2PHandler struct { // NewP2PHandler creates a new P2P handler. func NewP2PHandler( - headerStore common.HeightStore[*types.SignedHeaderWithDAHint], - dataStore common.HeightStore[*types.DataWithDAHint], + headerStore HeightStore[*types.SignedHeader], + dataStore HeightStore[*types.Data], cache cache.CacheManager, genesis genesis.Genesis, logger zerolog.Logger, @@ -73,27 +79,25 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC return nil } - headerTuple, err := h.headerStore.GetByHeight(ctx, height) + header, headerDAHint, err := h.headerStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("header unavailable in store") } return err } - header := headerTuple.Entry if err := h.assertExpectedProposer(header.ProposerAddress); err != nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("invalid header from P2P") return err } - dataTuple, err := h.dataStore.GetByHeight(ctx, height) + data, dataDAHint, err := h.dataStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("data unavailable in store") } return err } - data := dataTuple.Entry dataCommitment := data.DACommitment() if !bytes.Equal(header.DataHash[:], dataCommitment[:]) { err := fmt.Errorf("data hash mismatch: header %x, data %x", header.DataHash, dataCommitment) @@ -107,7 +111,7 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC Header: header, Data: data, Source: common.SourceP2P, - DaHeightHints: [2]uint64{headerTuple.DAHeightHint, dataTuple.DAHeightHint}, + DaHeightHints: [2]uint64{headerDAHint, dataDAHint}, } select { diff --git a/block/internal/syncing/p2p_handler_test.go b/block/internal/syncing/p2p_handler_test.go index aacd54d4cc..5970900474 100644 --- a/block/internal/syncing/p2p_handler_test.go +++ b/block/internal/syncing/p2p_handler_test.go @@ -18,7 +18,6 @@ import ( "github.com/evstack/ev-node/pkg/genesis" signerpkg "github.com/evstack/ev-node/pkg/signer" "github.com/evstack/ev-node/pkg/signer/noop" - extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" ) @@ -57,8 +56,8 @@ func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer [ // P2PTestData aggregates dependencies used by P2P handler tests. type P2PTestData struct { Handler *P2PHandler - HeaderStore *extmocks.MockStore[*types.SignedHeaderWithDAHint] - DataStore *extmocks.MockStore[*types.Data] + HeaderStore *MockHeightStore[*types.SignedHeader] + DataStore *MockHeightStore[*types.Data] Cache cache.CacheManager Genesis genesis.Genesis ProposerAddr []byte @@ -73,8 +72,8 @@ func setupP2P(t *testing.T) *P2PTestData { gen := genesis.Genesis{ChainID: "p2p-test", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: proposerAddr} - headerStoreMock := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) - dataStoreMock := extmocks.NewMockStore[*types.Data](t) + headerStoreMock := NewMockHeightStore[*types.SignedHeader](t) + dataStoreMock := NewMockHeightStore[*types.Data](t) cfg := config.Config{ RootDir: t.TempDir(), @@ -136,9 +135,9 @@ func TestP2PHandler_ProcessHeight_EmitsEventWhenHeaderAndDataPresent(t *testing. sig, err := p.Signer.Sign(bz) require.NoError(t, err) header.Signature = sig - payload := &types.SignedHeaderWithDAHint{SignedHeader: header} - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(payload, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, nil).Once() + + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(header, 0, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, 0, nil).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 5, ch) @@ -163,9 +162,8 @@ func TestP2PHandler_ProcessHeight_SkipsWhenDataMissing(t *testing.T) { require.NoError(t, err) header.Signature = sig - payload := &types.SignedHeaderWithDAHint{SignedHeader: header} - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(payload, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, errors.New("missing")).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(header, 0, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, 0, errors.New("missing")).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 7, ch) @@ -178,7 +176,7 @@ func TestP2PHandler_ProcessHeight_SkipsWhenHeaderMissing(t *testing.T) { p := setupP2P(t) ctx := context.Background() - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(9)).Return(nil, errors.New("missing")).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(9)).Return(nil, 0, errors.New("missing")).Once() ch := make(chan common.DAHeightEvent, 1) err := p.Handler.ProcessHeight(ctx, 9, ch) @@ -199,8 +197,7 @@ func TestP2PHandler_ProcessHeight_SkipsOnProposerMismatch(t *testing.T) { header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 11, badAddr, pub, signer) header.DataHash = common.DataHashForEmptyTxs - payload := &types.SignedHeaderWithDAHint{SignedHeader: header} - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(payload, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(header, 0, nil).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 11, ch) @@ -235,9 +232,8 @@ func TestP2PHandler_ProcessedHeightSkipsPreviouslyHandledBlocks(t *testing.T) { require.NoError(t, err) header.Signature = sig - payload := &types.SignedHeaderWithDAHint{SignedHeader: header} - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(payload, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(header, 0, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, 0, nil).Once() require.NoError(t, p.Handler.ProcessHeight(ctx, 6, ch)) @@ -259,9 +255,8 @@ func TestP2PHandler_SetProcessedHeightPreventsDuplicates(t *testing.T) { require.NoError(t, err) header.Signature = sig - payload := &types.SignedHeaderWithDAHint{SignedHeader: header} - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(payload, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(header, 0, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, 0, nil).Once() ch := make(chan common.DAHeightEvent, 1) require.NoError(t, p.Handler.ProcessHeight(ctx, 8, ch)) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index b73b8f572d..c9fba02067 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -114,7 +114,7 @@ func (s *Syncer) Start(ctx context.Context) error { // Initialize handlers s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger) - s.p2pHandler = NewP2PHandler(s.headerStore.XStore(), s.dataStore.XStore(), s.cache, s.genesis, s.logger) + s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) if currentHeight, err := s.store.Height(s.ctx); err != nil { s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") } else { diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 970dd0cc5c..ed9cd4c407 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -77,20 +77,12 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) - headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() - syncer.headerStore = headerStore - - dataStore := common.NewMockBroadcaster[*types.Data](t) - dataStore.EXPECT().Store().Return(mockDataStore).Maybe() - syncer.dataStore = dataStore - var callTimes []time.Time callCount := 0 @@ -173,20 +165,12 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) - headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() - syncer.headerStore = headerStore - - dataStore := common.NewMockBroadcaster[*types.Data](t) - dataStore.EXPECT().Store().Return(mockDataStore).Maybe() - syncer.dataStore = dataStore - var callTimes []time.Time // First call - error (should trigger backoff) @@ -263,20 +247,12 @@ func TestSyncer_BackoffBehaviorIntegration(t *testing.T) { syncer.p2pHandler = p2pHandler // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - headerStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) - headerStore.EXPECT().Store().Return(mockHeaderStore).Maybe() - syncer.headerStore = headerStore - - dataStore := common.NewMockBroadcaster[*types.Data](t) - dataStore.EXPECT().Store().Return(mockDataStore).Maybe() - syncer.dataStore = dataStore - var callTimes []time.Time p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() @@ -350,7 +326,7 @@ func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), + common.NewMockBroadcaster[*types.SignedHeader](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 29f8e86854..e2b6f6e51f 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -153,7 +153,7 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay mockP2P := newMockp2pHandler(b) // not used directly in this benchmark path mockP2P.On("SetProcessedHeight", mock.Anything).Return().Maybe() s.p2pHandler = mockP2P - headerP2PStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](b) + headerP2PStore := common.NewMockBroadcaster[*types.SignedHeader](b) s.headerStore = headerP2PStore dataP2PStore := common.NewMockBroadcaster[*types.Data](b) s.dataStore = dataP2PStore diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 3fa92e4d26..36d0a92599 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -123,7 +123,7 @@ func TestSyncer_validateBlock_DataHashMismatch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), + common.NewMockBroadcaster[*types.SignedHeader](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -174,7 +174,7 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), + common.NewMockBroadcaster[*types.SignedHeader](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -228,7 +228,7 @@ func TestSequentialBlockSync(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t), + common.NewMockBroadcaster[*types.SignedHeader](t), common.NewMockBroadcaster[*types.Data](t), zerolog.Nop(), common.DefaultBlockOptions(), @@ -340,17 +340,14 @@ func TestSyncLoopPersistState(t *testing.T) { dummyExec := execution.NewDummyExecutor() // Create mock stores for P2P - mockHeaderStore := extmocks.NewMockStore[*types.SignedHeaderWithDAHint](t) + mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - mockP2PHeaderStore := common.NewMockBroadcaster[*types.SignedHeaderWithDAHint](t) - mockP2PHeaderStore.EXPECT().Store().Return(mockHeaderStore).Maybe() - + mockP2PHeaderStore := common.NewMockBroadcaster[*types.SignedHeader](t) mockP2PDataStore := common.NewMockBroadcaster[*types.Data](t) - mockP2PDataStore.EXPECT().Store().Return(mockDataStore).Maybe() errorCh := make(chan error, 1) syncerInst1 := NewSyncer( diff --git a/node/full.go b/node/full.go index 6d03a87c04..69be1d35b8 100644 --- a/node/full.go +++ b/node/full.go @@ -31,7 +31,7 @@ import ( evsync "github.com/evstack/ev-node/pkg/sync" ) -// prefixes used in KV store to separate rollkit data from execution environment data (if the same data base is reused) +// EvPrefix used in KV store to separate rollkit data from execution environment data (if the same data base is reused) var EvPrefix = "0" const ( diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index e517d5128b..05c8df4d08 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -8,6 +8,7 @@ import ( "time" goheader "github.com/celestiaorg/go-header" + "github.com/evstack/ev-node/pkg/sync" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" @@ -28,8 +29,8 @@ import ( func setupTestServer( t *testing.T, mockStore *mocks.MockStore, - headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.Data], + headerStore goheader.Store[*sync.SignedHeaderWithDAHint], + dataStore goheader.Store[*sync.DataWithDAHint], mockP2P *mocks.MockP2PRPC, ) (*httptest.Server, *Client) { t.Helper() @@ -106,19 +107,19 @@ func TestClientGetMetadata(t *testing.T) { func TestClientGetP2PStoreInfo(t *testing.T) { mockStore := mocks.NewMockStore(t) mockP2P := mocks.NewMockP2PRPC(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) - dataStore := headerstoremocks.NewMockStore[*types.Data](t) + headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) + dataStore := headerstoremocks.NewMockStore[*sync.DataWithDAHint](t) now := time.Now().UTC() - headerHead := &types.SignedHeaderWithDAHint{SignedHeader: testSignedHeader(10, now)} - headerTail := &types.SignedHeaderWithDAHint{SignedHeader: testSignedHeader(5, now.Add(-time.Minute))} + headerHead := &sync.SignedHeaderWithDAHint{Entry: testSignedHeader(10, now)} + headerTail := &sync.SignedHeaderWithDAHint{Entry: testSignedHeader(5, now.Add(-time.Minute))} headerStore.On("Height").Return(uint64(10)) headerStore.On("Head", mock.Anything).Return(headerHead, nil) headerStore.On("Tail", mock.Anything).Return(headerTail, nil) - dataHead := testData(8, now.Add(-30*time.Second)) - dataTail := testData(4, now.Add(-2*time.Minute)) + dataHead := &sync.DataWithDAHint{Entry: testData(8, now.Add(-30*time.Second))} + dataTail := &sync.DataWithDAHint{Entry: testData(4, now.Add(-2*time.Minute))} dataStore.On("Height").Return(uint64(8)) dataStore.On("Head", mock.Anything).Return(dataHead, nil) dataStore.On("Tail", mock.Anything).Return(dataTail, nil) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 0a8546d3ae..f113b52fb8 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -14,6 +14,7 @@ import ( "connectrpc.com/grpcreflect" goheader "github.com/celestiaorg/go-header" coreda "github.com/evstack/ev-node/core/da" + "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" "github.com/rs/zerolog" "golang.org/x/net/http2" @@ -34,16 +35,16 @@ var _ rpc.StoreServiceHandler = (*StoreServer)(nil) // StoreServer implements the StoreService defined in the proto file type StoreServer struct { store store.Store - headerStore goheader.Store[*types.SignedHeaderWithDAHint] - dataStore goheader.Store[*types.DataWithDAHint] + headerStore goheader.Store[*sync.SignedHeaderWithDAHint] + dataStore goheader.Store[*sync.DataWithDAHint] logger zerolog.Logger } // NewStoreServer creates a new StoreServer instance func NewStoreServer( store store.Store, - headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.DataWithDAHint], + headerStore goheader.Store[*sync.SignedHeaderWithDAHint], + dataStore goheader.Store[*sync.DataWithDAHint], logger zerolog.Logger, ) *StoreServer { return &StoreServer{ @@ -370,8 +371,8 @@ func (p *P2PServer) GetNetInfo( // NewServiceHandler creates a new HTTP handler for Store, P2P and Config services func NewServiceHandler( store store.Store, - headerStore goheader.Store[*types.SignedHeaderWithDAHint], - dataStore goheader.Store[*types.DataWithDAHint], + headerStore goheader.Store[*sync.SignedHeaderWithDAHint], + dataStore goheader.Store[*sync.DataWithDAHint], peerManager p2p.P2PRPC, proposerAddress []byte, logger zerolog.Logger, diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index befecd910f..3a4dbd89bd 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -11,6 +11,7 @@ import ( "time" "connectrpc.com/connect" + "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -325,8 +326,8 @@ func TestGetGenesisDaHeight_InvalidLength(t *testing.T) { func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns snapshots for configured stores", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) - dataStore := headerstoremocks.NewMockStore[*types.Data](t) + headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) + dataStore := headerstoremocks.NewMockStore[*sync.DataWithDAHint](t) logger := zerolog.Nop() server := NewStoreServer(mockStore, headerStore, dataStore, logger) @@ -354,10 +355,10 @@ func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns error when a store edge fails", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*types.SignedHeaderWithDAHint](t) + headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) logger := zerolog.Nop() headerStore.On("Height").Return(uint64(0)) - headerStore.On("Head", mock.Anything).Return((*types.SignedHeaderWithDAHint)(nil), fmt.Errorf("boom")) + headerStore.On("Head", mock.Anything).Return((*sync.SignedHeaderWithDAHint)(nil), fmt.Errorf("boom")) server := NewStoreServer(mockStore, headerStore, nil, logger) resp, err := server.GetP2PStoreInfo(context.Background(), connect.NewRequest(&emptypb.Empty{})) @@ -627,8 +628,8 @@ func TestHealthReadyEndpoint(t *testing.T) { }) } -func makeTestSignedHeader(height uint64, ts time.Time) *types.SignedHeaderWithDAHint { - return &types.SignedHeaderWithDAHint{SignedHeader: &types.SignedHeader{ +func makeTestSignedHeader(height uint64, ts time.Time) *sync.SignedHeaderWithDAHint { + return &sync.SignedHeaderWithDAHint{Entry: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -643,12 +644,13 @@ func makeTestSignedHeader(height uint64, ts time.Time) *types.SignedHeaderWithDA } } -func makeTestData(height uint64, ts time.Time) *types.Data { - return &types.Data{ +func makeTestData(height uint64, ts time.Time) *sync.DataWithDAHint { + return &sync.DataWithDAHint{Entry: &types.Data{ Metadata: &types.Metadata{ ChainID: "test-chain", Height: height, Time: uint64(ts.UnixNano()), }, + }, } } diff --git a/types/da_hint_container.go b/pkg/sync/da_hint_container.go similarity index 86% rename from types/da_hint_container.go rename to pkg/sync/da_hint_container.go index 9e905c6296..5d904d885d 100644 --- a/types/da_hint_container.go +++ b/pkg/sync/da_hint_container.go @@ -1,4 +1,4 @@ -package types +package sync import ( "encoding/binary" @@ -6,8 +6,12 @@ import ( "time" "github.com/celestiaorg/go-header" + "github.com/evstack/ev-node/types" ) +type SignedHeaderWithDAHint = DAHeightHintContainer[*types.SignedHeader] +type DataWithDAHint = DAHeightHintContainer[*types.Data] + type DAHeightHintContainer[H header.Header[H]] struct { Entry H DAHeightHint uint64 @@ -49,6 +53,9 @@ func (sh *DAHeightHintContainer[H]) Verify(untrstH *DAHeightHintContainer[H]) er func (s *DAHeightHintContainer[H]) SetDAHint(daHeight uint64) { s.DAHeightHint = daHeight } +func (s *DAHeightHintContainer[H]) DAHint() uint64 { + return s.DAHeightHint +} func (s *DAHeightHintContainer[H]) IsZero() bool { return s == nil diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 3ca112e983..9a45ab4c42 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -39,18 +39,19 @@ const ninetyNineYears = 99 * 365 * 24 * time.Hour type EntityWithDAHint[H any] interface { header.Header[H] SetDAHint(daHeight uint64) + DAHint() uint64 } -// DataSyncService is the P2P Sync Service for blocks. -type DataSyncService = SyncService[*types.DataWithDAHint] - // HeaderSyncService is the P2P Sync Service for headers. -type HeaderSyncService = SyncService[*types.SignedHeaderWithDAHint] +type HeaderSyncService = SyncService[*types.SignedHeader] + +// DataSyncService is the P2P Sync Service for blocks. +type DataSyncService = SyncService[*types.Data] // SyncService is the P2P Sync Service for blocks and headers. // // Uses the go-header library for handling all P2P logic. -type SyncService[H EntityWithDAHint[H]] struct { +type SyncService[V header.Header[V]] struct { conf config.Config logger zerolog.Logger syncType syncType @@ -59,13 +60,13 @@ type SyncService[H EntityWithDAHint[H]] struct { p2p *p2p.Client - ex *goheaderp2p.Exchange[H] - sub *goheaderp2p.Subscriber[H] - p2pServer *goheaderp2p.ExchangeServer[H] - store *goheaderstore.Store[H] - syncer *goheadersync.Syncer[H] + ex *goheaderp2p.Exchange[*DAHeightHintContainer[V]] + sub *goheaderp2p.Subscriber[*DAHeightHintContainer[V]] + p2pServer *goheaderp2p.ExchangeServer[*DAHeightHintContainer[V]] + store *goheaderstore.Store[*DAHeightHintContainer[V]] + syncer *goheadersync.Syncer[*DAHeightHintContainer[V]] syncerStatus *SyncerStatus - topicSubscription header.Subscription[H] + topicSubscription header.Subscription[*DAHeightHintContainer[V]] storeInitialized atomic.Bool } @@ -77,7 +78,7 @@ func NewDataSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*DataSyncService, error) { - return newSyncService[*types.DataWithDAHint](store, dataSync, conf, genesis, p2p, logger) + return newSyncService[*types.Data](store, dataSync, conf, genesis, p2p, logger) } // NewHeaderSyncService returns a new HeaderSyncService. @@ -88,22 +89,22 @@ func NewHeaderSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*HeaderSyncService, error) { - return newSyncService[*types.SignedHeaderWithDAHint](store, headerSync, conf, genesis, p2p, logger) + return newSyncService[*types.SignedHeader](store, headerSync, conf, genesis, p2p, logger) } -func newSyncService[H EntityWithDAHint[H]]( +func newSyncService[V header.Header[V]]( store ds.Batching, syncType syncType, conf config.Config, genesis genesis.Genesis, p2p *p2p.Client, logger zerolog.Logger, -) (*SyncService[H], error) { +) (*SyncService[V], error) { if p2p == nil { return nil, errors.New("p2p client cannot be nil") } - ss, err := goheaderstore.NewStore[H]( + ss, err := goheaderstore.NewStore[*DAHeightHintContainer[V]]( store, goheaderstore.WithStorePrefix(string(syncType)), goheaderstore.WithMetrics(), @@ -112,7 +113,7 @@ func newSyncService[H EntityWithDAHint[H]]( return nil, fmt.Errorf("failed to initialize the %s store: %w", syncType, err) } - svc := &SyncService[H]{ + svc := &SyncService[V]{ conf: conf, genesis: genesis, p2p: p2p, @@ -126,21 +127,22 @@ func newSyncService[H EntityWithDAHint[H]]( } // Store returns the store of the SyncService -func (syncService *SyncService[H]) Store() header.Store[H] { +func (syncService *SyncService[V]) Store() header.Store[*DAHeightHintContainer[V]] { return syncService.store } // WriteToStoreAndBroadcast initializes store if needed and broadcasts provided header or block. // Note: Only returns an error in case store can't be initialized. Logs error if there's one while broadcasting. -func (syncService *SyncService[H]) WriteToStoreAndBroadcast(ctx context.Context, headerOrData H, opts ...pubsub.PubOpt) error { +func (syncService *SyncService[V]) WriteToStoreAndBroadcast(ctx context.Context, payload V, opts ...pubsub.PubOpt) error { if syncService.genesis.InitialHeight == 0 { return fmt.Errorf("invalid initial height; cannot be zero") } - if headerOrData.IsZero() { + if payload.IsZero() { return fmt.Errorf("empty header/data cannot write to store or broadcast") } + headerOrData := &DAHeightHintContainer[V]{Entry: payload} storeInitialized := false if syncService.storeInitialized.CompareAndSwap(false, true) { var err error @@ -179,11 +181,14 @@ func (syncService *SyncService[H]) WriteToStoreAndBroadcast(ctx context.Context, return nil } -func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { - entries := make([]H, 0, len(hashes)) +func (s *SyncService[V]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { + entries := make([]*DAHeightHintContainer[V], 0, len(hashes)) for _, h := range hashes { v, err := s.store.Get(ctx, h) - if err != nil && !errors.Is(err, header.ErrNotFound) { + if err != nil { + if errors.Is(err, header.ErrNotFound) { + continue + } return err } v.SetDAHint(daHeight) @@ -192,8 +197,17 @@ func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, hash return s.store.Append(ctx, entries...) } +func (s *SyncService[V]) GetByHeight(ctx context.Context, height uint64) (V, uint64, error) { + c, err := s.store.GetByHeight(ctx, height) + if err != nil { + var zero V + return zero, 0, err + } + return c.Entry, c.DAHint(), nil +} + // Start is a part of Service interface. -func (syncService *SyncService[H]) Start(ctx context.Context) error { +func (syncService *SyncService[V]) Start(ctx context.Context) error { // setup P2P infrastructure, but don't start Subscriber yet. peerIDs, err := syncService.setupP2PInfrastructure(ctx) if err != nil { @@ -201,7 +215,7 @@ func (syncService *SyncService[H]) Start(ctx context.Context) error { } // create syncer, must be before initFromP2PWithRetry which calls startSyncer. - if syncService.syncer, err = newSyncer( + if syncService.syncer, err = newSyncer[V]( syncService.ex, syncService.store, syncService.sub, @@ -244,7 +258,7 @@ func (syncService *SyncService[H]) startSyncer(ctx context.Context) error { // initStore initializes the store with the given initial header. // it is a no-op if the store is already initialized. // Returns true when the store was initialized by this call. -func (syncService *SyncService[H]) initStore(ctx context.Context, initial H) (bool, error) { +func (syncService *SyncService[V]) initStore(ctx context.Context, initial *DAHeightHintContainer[V]) (bool, error) { if initial.IsZero() { return false, errors.New("failed to initialize the store") } @@ -268,12 +282,12 @@ func (syncService *SyncService[H]) initStore(ctx context.Context, initial H) (bo // setupP2PInfrastructure sets up the P2P infrastructure (Exchange, ExchangeServer, Store) // but does not start the Subscriber. Returns peer IDs for later use. -func (syncService *SyncService[H]) setupP2PInfrastructure(ctx context.Context) ([]peer.ID, error) { +func (syncService *SyncService[V]) setupP2PInfrastructure(ctx context.Context) ([]peer.ID, error) { ps := syncService.p2p.PubSub() var err error // Create subscriber but DON'T start it yet - syncService.sub, err = goheaderp2p.NewSubscriber[H]( + syncService.sub, err = goheaderp2p.NewSubscriber[*DAHeightHintContainer[V]]( ps, pubsub.DefaultMsgIdFn, goheaderp2p.WithSubscriberNetworkID(syncService.getChainID()), @@ -302,7 +316,7 @@ func (syncService *SyncService[H]) setupP2PInfrastructure(ctx context.Context) ( peerIDs := syncService.getPeerIDs() - if syncService.ex, err = newP2PExchange[H](syncService.p2p.Host(), peerIDs, networkID, syncService.genesis.ChainID, syncService.p2p.ConnectionGater()); err != nil { + if syncService.ex, err = newP2PExchange[*DAHeightHintContainer[V]](syncService.p2p.Host(), peerIDs, networkID, syncService.genesis.ChainID, syncService.p2p.ConnectionGater()); err != nil { return nil, fmt.Errorf("error while creating exchange: %w", err) } if err := syncService.ex.Start(ctx); err != nil { @@ -330,14 +344,14 @@ func (syncService *SyncService[H]) startSubscriber(ctx context.Context) error { // It inspects the local store to determine the first height to request: // - when the store already contains items, it reuses the latest height as the starting point; // - otherwise, it falls back to the configured genesis height. -func (syncService *SyncService[H]) initFromP2PWithRetry(ctx context.Context, peerIDs []peer.ID) error { +func (syncService *SyncService[V]) initFromP2PWithRetry(ctx context.Context, peerIDs []peer.ID) error { if len(peerIDs) == 0 { return nil } tryInit := func(ctx context.Context) (bool, error) { var ( - trusted H + trusted *DAHeightHintContainer[V] err error heightToQuery uint64 ) @@ -401,7 +415,7 @@ func (syncService *SyncService[H]) initFromP2PWithRetry(ctx context.Context, pee // Stop is a part of Service interface. // // `store` is closed last because it's used by other services. -func (syncService *SyncService[H]) Stop(ctx context.Context) error { +func (syncService *SyncService[V]) Stop(ctx context.Context) error { // unsubscribe from topic first so that sub.Stop() does not fail syncService.topicSubscription.Cancel() err := errors.Join( @@ -447,17 +461,17 @@ func newP2PExchange[H header.Header[H]]( // newSyncer constructs new Syncer for headers/blocks. func newSyncer[H header.Header[H]]( - ex header.Exchange[H], - store header.Store[H], - sub header.Subscriber[H], + ex header.Exchange[*DAHeightHintContainer[H]], + store header.Store[*DAHeightHintContainer[H]], + sub header.Subscriber[*DAHeightHintContainer[H]], opts []goheadersync.Option, -) (*goheadersync.Syncer[H], error) { +) (*goheadersync.Syncer[*DAHeightHintContainer[H]], error) { opts = append(opts, goheadersync.WithMetrics(), goheadersync.WithPruningWindow(ninetyNineYears), goheadersync.WithTrustingPeriod(ninetyNineYears), ) - return goheadersync.NewSyncer(ex, store, sub, opts...) + return goheadersync.NewSyncer[*DAHeightHintContainer[H]](ex, store, sub, opts...) } func (syncService *SyncService[H]) getNetworkID(network string) string { diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index b8063e874b..99d6ed8a0e 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -167,10 +167,10 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) } -func nextHeader(t *testing.T, previousHeader *types.SignedHeaderWithDAHint, chainID string, noopSigner signer.Signer) *types.SignedHeaderWithDAHint { +func nextHeader(t *testing.T, previousHeader *types.SignedHeader, chainID string, noopSigner signer.Signer) *types.SignedHeader { newSignedHeader := &types.SignedHeader{ - Header: types.GetRandomNextHeader(previousHeader.Entry.Header, chainID), - Signer: previousHeader.Entry.Signer, + Header: types.GetRandomNextHeader(previousHeader.Header, chainID), + Signer: previousHeader.Signer, } b, err := newSignedHeader.Header.MarshalBinary() require.NoError(t, err) @@ -178,7 +178,7 @@ func nextHeader(t *testing.T, previousHeader *types.SignedHeaderWithDAHint, chai require.NoError(t, err) newSignedHeader.Signature = signature require.NoError(t, newSignedHeader.Validate()) - return &types.SignedHeaderWithDAHint{Entry: newSignedHeader} + return newSignedHeader } func bytesN(r *rand.Rand, n int) []byte { diff --git a/types/signed_header.go b/types/signed_header.go index c63ecce3ac..ffacbc847a 100644 --- a/types/signed_header.go +++ b/types/signed_header.go @@ -14,9 +14,6 @@ var ( ErrLastHeaderHashMismatch = errors.New("last header hash mismatch") ) -type SignedHeaderWithDAHint = DAHeightHintContainer[*SignedHeader] -type DataWithDAHint = DAHeightHintContainer[*Data] - var _ header.Header[*SignedHeader] = &SignedHeader{} // SignedHeader combines Header and its signature. diff --git a/types/utils.go b/types/utils.go index fe59fb20dc..d8c2527521 100644 --- a/types/utils.go +++ b/types/utils.go @@ -68,7 +68,7 @@ func GenerateRandomBlockCustomWithAppHash(config *BlockConfig, chainID string, a } if config.ProposerAddr != nil { - signedHeader.Entry.ProposerAddress = config.ProposerAddr + signedHeader.ProposerAddress = config.ProposerAddr } data.Metadata = &Metadata{ @@ -78,7 +78,7 @@ func GenerateRandomBlockCustomWithAppHash(config *BlockConfig, chainID string, a Time: uint64(signedHeader.Time().UnixNano()), } - return signedHeader.Entry, data, config.PrivKey + return signedHeader, data, config.PrivKey } // GenerateRandomBlockCustom returns a block with random data and the given height, transactions, privateKey and proposer address. @@ -150,11 +150,11 @@ func GetRandomSignedHeader(chainID string) (*SignedHeader, crypto.PrivKey, error if err != nil { return nil, nil, err } - return signedHeader.Entry, pk, nil + return signedHeader, pk, nil } // GetRandomSignedHeaderCustom creates a signed header based on the provided HeaderConfig. -func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedHeaderWithDAHint, error) { +func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedHeader, error) { pk, err := config.Signer.GetPublic() if err != nil { return nil, err @@ -183,7 +183,7 @@ func GetRandomSignedHeaderCustom(config *HeaderConfig, chainID string) (*SignedH return nil, err } signedHeader.Signature = signature - return &SignedHeaderWithDAHint{Entry: signedHeader}, nil + return signedHeader, nil } // GetRandomNextSignedHeader returns a signed header with random data and height of +1 from From aaacbde53b5105734baa668aa5b6e403eb701594 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 28 Nov 2025 17:20:08 +0100 Subject: [PATCH 05/41] Async DA pull --- block/internal/common/expected_interfaces.go | 21 --- block/internal/syncing/async_da_retriever.go | 111 +++++++++++ .../syncing/async_da_retriever_test.go | 141 ++++++++++++++ block/internal/syncing/syncer.go | 34 ++-- block/internal/syncing/syncer_test.go | 76 ++++++++ pkg/sync/sync_service_test.go | 175 ++++++++++++++++++ 6 files changed, 514 insertions(+), 44 deletions(-) create mode 100644 block/internal/syncing/async_da_retriever.go create mode 100644 block/internal/syncing/async_da_retriever_test.go diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index c880a5b543..0eeef6ab31 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -20,24 +20,3 @@ type Broadcaster[H header.Header[H]] interface { AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error GetByHeight(ctx context.Context, height uint64) (H, uint64, error) } - -// -//// Decorator to access the payload type without the container -//type Decorator[H header.Header[H]] struct { -// nested Broadcaster[*sync.DAHeightHintContainer[H]] -//} -// -//func NewDecorator[H header.Header[H]](nested Broadcaster[*sync.DAHeightHintContainer[H]]) Decorator[H] { -// return Decorator[H]{nested: nested} -//} -// -//func (d Decorator[H]) WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error { -// return d.nested.WriteToStoreAndBroadcast(ctx, &sync.DAHeightHintContainer[H]{Entry: payload}, opts...) -//} -// -//func (d Decorator[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { -// return d.nested.AppendDAHint(ctx, daHeight, hashes...) -//} -//func (d Decorator[H]) GetByHeight(ctx context.Context, height uint64) (H, error) { -// panic("not implemented") -//} diff --git a/block/internal/syncing/async_da_retriever.go b/block/internal/syncing/async_da_retriever.go new file mode 100644 index 0000000000..7c79a37512 --- /dev/null +++ b/block/internal/syncing/async_da_retriever.go @@ -0,0 +1,111 @@ +package syncing + +import ( + "context" + "sync" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/rs/zerolog" +) + +// AsyncDARetriever handles concurrent DA retrieval operations. +type AsyncDARetriever struct { + retriever DARetriever + resultCh chan<- common.DAHeightEvent + workCh chan uint64 + inFlight map[uint64]struct{} + mu sync.Mutex + logger zerolog.Logger + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc +} + +// NewAsyncDARetriever creates a new AsyncDARetriever. +func NewAsyncDARetriever( + retriever DARetriever, + resultCh chan<- common.DAHeightEvent, + logger zerolog.Logger, +) *AsyncDARetriever { + return &AsyncDARetriever{ + retriever: retriever, + resultCh: resultCh, + workCh: make(chan uint64, 100), // Buffer size 100 + inFlight: make(map[uint64]struct{}), + logger: logger.With().Str("component", "async_da_retriever").Logger(), + } +} + +// Start starts the worker pool. +func (r *AsyncDARetriever) Start(ctx context.Context) { + r.ctx, r.cancel = context.WithCancel(ctx) + // Start 5 workers + for i := 0; i < 5; i++ { + r.wg.Add(1) + go r.worker() + } + r.logger.Info().Msg("AsyncDARetriever started") +} + +// Stop stops the worker pool. +func (r *AsyncDARetriever) Stop() { + if r.cancel != nil { + r.cancel() + } + r.wg.Wait() + r.logger.Info().Msg("AsyncDARetriever stopped") +} + +// RequestRetrieval requests a DA retrieval for the given height. +// It is non-blocking and idempotent. +func (r *AsyncDARetriever) RequestRetrieval(height uint64) { + r.mu.Lock() + defer r.mu.Unlock() + + if _, exists := r.inFlight[height]; exists { + return + } + + select { + case r.workCh <- height: + r.inFlight[height] = struct{}{} + r.logger.Debug().Uint64("height", height).Msg("queued DA retrieval request") + default: + r.logger.Debug().Uint64("height", height).Msg("DA retrieval worker pool full, dropping request") + } +} + +func (r *AsyncDARetriever) worker() { + defer r.wg.Done() + + for { + select { + case <-r.ctx.Done(): + return + case height := <-r.workCh: + r.processRetrieval(height) + } + } +} + +func (r *AsyncDARetriever) processRetrieval(height uint64) { + defer func() { + r.mu.Lock() + delete(r.inFlight, height) + r.mu.Unlock() + }() + + events, err := r.retriever.RetrieveFromDA(r.ctx, height) + if err != nil { + r.logger.Debug().Err(err).Uint64("height", height).Msg("async DA retrieval failed") + return + } + + for _, event := range events { + select { + case r.resultCh <- event: + case <-r.ctx.Done(): + return + } + } +} diff --git a/block/internal/syncing/async_da_retriever_test.go b/block/internal/syncing/async_da_retriever_test.go new file mode 100644 index 0000000000..bce1267a0f --- /dev/null +++ b/block/internal/syncing/async_da_retriever_test.go @@ -0,0 +1,141 @@ +package syncing + +import ( + "context" + "testing" + "time" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { + logger := zerolog.Nop() + mockRetriever := NewMockDARetriever(t) + resultCh := make(chan common.DAHeightEvent, 10) + + asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + asyncRetriever.Start(ctx) + defer asyncRetriever.Stop() + + // 1. Test successful retrieval + height1 := uint64(100) + mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height1).Return([]common.DAHeightEvent{{DaHeight: height1}}, nil).Once() + + asyncRetriever.RequestRetrieval(height1) + + select { + case event := <-resultCh: + assert.Equal(t, height1, event.DaHeight) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for result") + } + + // 2. Test deduplication (idempotency) + // We'll block the retriever to simulate a slow request, then send multiple requests for the same height + height2 := uint64(200) + + // Create a channel to signal when the mock is called + calledCh := make(chan struct{}) + // Create a channel to unblock the mock + unblockCh := make(chan struct{}) + + mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height2).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { + close(calledCh) + <-unblockCh + return []common.DAHeightEvent{{DaHeight: h}}, nil + }).Once() // Should be called only once despite multiple requests + + // Send first request + asyncRetriever.RequestRetrieval(height2) + + // Wait for the worker to pick it up + select { + case <-calledCh: + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for retriever call") + } + + // Send duplicate requests while the first one is still in flight + asyncRetriever.RequestRetrieval(height2) + asyncRetriever.RequestRetrieval(height2) + + // Unblock the worker + close(unblockCh) + + // We should receive exactly one result + select { + case event := <-resultCh: + assert.Equal(t, height2, event.DaHeight) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for result") + } + + // Ensure no more results come through + select { + case <-resultCh: + t.Fatal("received duplicate result") + default: + } +} + +func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) { + logger := zerolog.Nop() + mockRetriever := NewMockDARetriever(t) + resultCh := make(chan common.DAHeightEvent, 100) + + asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + asyncRetriever.Start(ctx) + defer asyncRetriever.Stop() + + // We have 5 workers. We'll block them all. + unblockCh := make(chan struct{}) + + // Expect 5 calls that block + for i := 0; i < 5; i++ { + h := uint64(1000 + i) + mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, h).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { + <-unblockCh + return []common.DAHeightEvent{{DaHeight: h}}, nil + }).Once() + asyncRetriever.RequestRetrieval(h) + } + + // Give workers time to pick up tasks + time.Sleep(100 * time.Millisecond) + + // Now send a 6th request. It should be queued but not processed yet. + height6 := uint64(1005) + processed6 := make(chan struct{}) + mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height6).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { + close(processed6) + return []common.DAHeightEvent{{DaHeight: h}}, nil + }).Once() + + asyncRetriever.RequestRetrieval(height6) + + // Ensure 6th request is NOT processed yet + select { + case <-processed6: + t.Fatal("6th request processed too early") + default: + } + + // Unblock workers + close(unblockCh) + + // Now 6th request should be processed + select { + case <-processed6: + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for 6th request") + } +} diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index c9fba02067..501d2aac06 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -68,6 +68,9 @@ type Syncer struct { // P2P wait coordination p2pWaitState atomic.Value // stores p2pWaitState + + // Async DA retriever + asyncDARetriever *AsyncDARetriever } // NewSyncer creates a new block syncer @@ -114,6 +117,9 @@ func (s *Syncer) Start(ctx context.Context) error { // Initialize handlers s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger) + s.asyncDARetriever = NewAsyncDARetriever(s.daRetriever, s.heightInCh, s.logger) + s.asyncDARetriever.Start(s.ctx) + s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) if currentHeight, err := s.store.Height(s.ctx); err != nil { s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") @@ -144,6 +150,9 @@ func (s *Syncer) Stop() error { if s.cancel != nil { s.cancel() } + if s.asyncDARetriever != nil { + s.asyncDARetriever.Stop() + } s.cancelP2PWait(0) s.wg.Wait() s.logger.Info().Msg("syncer stopped") @@ -466,29 +475,8 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { Uint64("da_height_hint", daHeightHint). Msg("P2P event with DA height hint, triggering targeted DA retrieval") - // Trigger targeted DA retrieval in background - go func() { - targetEvents, err := s.daRetriever.RetrieveFromDA(s.ctx, daHeightHint) - if err != nil { - s.logger.Debug(). - Err(err). - Uint64("da_height", daHeightHint). - Msg("targeted DA retrieval failed (hint may be incorrect or DA not yet available)") - // Not a critical error - the sequential DA worker will eventually find it - return - } - - // Process retrieved events from the targeted DA height - for _, daEvent := range targetEvents { - select { - case s.heightInCh <- daEvent: - case <-s.ctx.Done(): - return - default: - s.cache.SetPendingEvent(daEvent.Header.Height(), &daEvent) - } - } - }() + // Trigger targeted DA retrieval in background via worker pool + s.asyncDARetriever.RequestRetrieval(daHeightHint) } } } diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 36d0a92599..adacd15efc 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -697,3 +697,79 @@ func TestSyncer_getHighestStoredDAHeight(t *testing.T) { highestDA = syncer.getHighestStoredDAHeight() assert.Equal(t, uint64(200), highestDA, "should return highest DA height from most recent included height") } + +func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { + ds := dssync.MutexWrap(datastore.NewMapDatastore()) + st := store.New(ds) + cm, err := cache.NewCacheManager(config.DefaultConfig(), zerolog.Nop()) + require.NoError(t, err) + + addr, _, _ := buildSyncTestSigner(t) + cfg := config.DefaultConfig() + gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} + + mockExec := testmocks.NewMockExecutor(t) + mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), uint64(1024), nil).Once() + + s := NewSyncer( + st, + mockExec, + nil, + cm, + common.NopMetrics(), + cfg, + gen, + common.NewMockBroadcaster[*types.SignedHeader](t), + common.NewMockBroadcaster[*types.Data](t), + zerolog.Nop(), + common.DefaultBlockOptions(), + make(chan error, 1), + ) + require.NoError(t, s.initializeState()) + s.ctx = context.Background() + + // Mock AsyncDARetriever + mockRetriever := NewMockDARetriever(t) + asyncRetriever := NewAsyncDARetriever(mockRetriever, s.heightInCh, zerolog.Nop()) + // We don't start the async retriever to avoid race conditions in test, + // we just want to verify RequestRetrieval queues the request. + // However, RequestRetrieval writes to a channel, so we need a consumer or a buffered channel. + // The workCh is buffered (100), so we are good. + s.asyncDARetriever = asyncRetriever + + // Create event with DA height hint + evt := common.DAHeightEvent{ + Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: "c", Height: 2}}}, + Data: &types.Data{Metadata: &types.Metadata{ChainID: "c", Height: 2}}, + Source: common.SourceP2P, + DaHeightHints: [2]uint64{100, 100}, + } + + // Current height is 0 (from init), event height is 2. + // processHeightEvent checks: + // 1. height <= currentHeight (2 <= 0 -> false) + // 2. height != currentHeight+1 (2 != 1 -> true) -> stores as pending event + + // We need to simulate height 1 being processed first so height 2 is "next" + // OR we can just test that it DOES NOT trigger DA retrieval if it's pending. + // Wait, the logic for DA retrieval is BEFORE the "next block" check? + // Let's check syncer.go... + // Yes, "If this is a P2P event with a DA height hint, trigger targeted DA retrieval" block is AFTER "If this is not the next block in sequence... return" + + // So we need to be at height 1 to process height 2. + // Let's set the store height to 1. + batch, err := st.NewBatch(context.Background()) + require.NoError(t, err) + require.NoError(t, batch.SetHeight(1)) + require.NoError(t, batch.Commit()) + + s.processHeightEvent(&evt) + + // Verify that the request was queued in the async retriever + select { + case h := <-asyncRetriever.workCh: + assert.Equal(t, uint64(100), h) + default: + t.Fatal("expected DA retrieval request to be queued") + } +} diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 99d6ed8a0e..b0e244a95f 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -167,6 +167,181 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) } +func TestDAHintStorageHeader(t *testing.T) { + mainKV := sync.MutexWrap(datastore.NewMapDatastore()) + pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) + require.NoError(t, err) + noopSigner, err := noop.NewNoopSigner(pk) + require.NoError(t, err) + rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only + mn := mocknet.New() + + chainId := "test-chain-id" + + proposerAddr := []byte("test") + genesisDoc := genesispkg.Genesis{ + ChainID: chainId, + StartTime: time.Now(), + InitialHeight: 1, + ProposerAddress: proposerAddr, + } + conf := config.DefaultConfig() + conf.RootDir = t.TempDir() + nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) + require.NoError(t, err) + logger := zerolog.Nop() + priv := nodeKey.PrivKey + p2pHost, err := mn.AddPeer(priv, nil) + require.NoError(t, err) + + p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + + headerSvc, err := NewHeaderSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + require.NoError(t, headerSvc.Start(ctx)) + + headerConfig := types.HeaderConfig{ + Height: genesisDoc.InitialHeight, + DataHash: bytesN(rnd, 32), + AppHash: bytesN(rnd, 32), + Signer: noopSigner, + } + signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, genesisDoc.ChainID) + require.NoError(t, err) + require.NoError(t, signedHeader.Validate()) + + require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, signedHeader)) + + daHeight := uint64(100) + require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Hash())) + + h, hint, err := headerSvc.GetByHeight(ctx, signedHeader.Height()) + require.NoError(t, err) + require.Equal(t, signedHeader.Hash(), h.Hash()) + require.Equal(t, daHeight, hint) + + _ = p2pClient.Close() + _ = headerSvc.Stop(ctx) + cancel() + + // Restart + h2, err := mn.AddPeer(priv, nil) + require.NoError(t, err) + p2pClient, err = p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), h2) + require.NoError(t, err) + + ctx, cancel = context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + t.Cleanup(func() { _ = p2pClient.Close() }) + + headerSvc, err = NewHeaderSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + require.NoError(t, headerSvc.Start(ctx)) + t.Cleanup(func() { _ = headerSvc.Stop(context.Background()) }) + + h, hint, err = headerSvc.GetByHeight(ctx, signedHeader.Height()) + require.NoError(t, err) + require.Equal(t, signedHeader.Hash(), h.Hash()) + require.Equal(t, daHeight, hint) +} + +func TestDAHintStorageData(t *testing.T) { + mainKV := sync.MutexWrap(datastore.NewMapDatastore()) + pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) + require.NoError(t, err) + noopSigner, err := noop.NewNoopSigner(pk) + require.NoError(t, err) + rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only + mn := mocknet.New() + + chainId := "test-chain-id" + + proposerAddr := []byte("test") + genesisDoc := genesispkg.Genesis{ + ChainID: chainId, + StartTime: time.Now(), + InitialHeight: 1, + ProposerAddress: proposerAddr, + } + conf := config.DefaultConfig() + conf.RootDir = t.TempDir() + nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) + require.NoError(t, err) + logger := zerolog.Nop() + priv := nodeKey.PrivKey + p2pHost, err := mn.AddPeer(priv, nil) + require.NoError(t, err) + + p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + + dataSvc, err := NewDataSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + require.NoError(t, dataSvc.Start(ctx)) + + // Need a valid header height for data metadata + headerConfig := types.HeaderConfig{ + Height: genesisDoc.InitialHeight, + DataHash: bytesN(rnd, 32), + AppHash: bytesN(rnd, 32), + Signer: noopSigner, + } + signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, genesisDoc.ChainID) + require.NoError(t, err) + + data := types.Data{ + Txs: types.Txs{[]byte("tx1")}, + Metadata: &types.Metadata{ + Height: signedHeader.Height(), + }, + } + + require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &data)) + + daHeight := uint64(100) + require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Hash())) + + d, hint, err := dataSvc.GetByHeight(ctx, signedHeader.Height()) + require.NoError(t, err) + require.Equal(t, data.Hash(), d.Hash()) + require.Equal(t, daHeight, hint) + + _ = p2pClient.Close() + _ = dataSvc.Stop(ctx) + cancel() + + // Restart + h2, err := mn.AddPeer(priv, nil) + require.NoError(t, err) + p2pClient, err = p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), h2) + require.NoError(t, err) + + ctx, cancel = context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + t.Cleanup(func() { _ = p2pClient.Close() }) + + dataSvc, err = NewDataSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + require.NoError(t, dataSvc.Start(ctx)) + t.Cleanup(func() { _ = dataSvc.Stop(context.Background()) }) + + d, hint, err = dataSvc.GetByHeight(ctx, signedHeader.Height()) + require.NoError(t, err) + require.Equal(t, data.Hash(), d.Hash()) + require.Equal(t, daHeight, hint) +} + func nextHeader(t *testing.T, previousHeader *types.SignedHeader, chainID string, noopSigner signer.Signer) *types.SignedHeader { newSignedHeader := &types.SignedHeader{ Header: types.GetRandomNextHeader(previousHeader.Header, chainID), From b9b5f5b930a4cdf259031516584e9211aca5d1b3 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 1 Dec 2025 09:45:27 +0100 Subject: [PATCH 06/41] Minor cleanup --- block/internal/syncing/async_da_retriever_test.go | 4 ++-- block/internal/syncing/syncer_test.go | 12 ++++++------ test/e2e/go.sum | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/block/internal/syncing/async_da_retriever_test.go b/block/internal/syncing/async_da_retriever_test.go index bce1267a0f..dfaecc922e 100644 --- a/block/internal/syncing/async_da_retriever_test.go +++ b/block/internal/syncing/async_da_retriever_test.go @@ -39,7 +39,7 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { // 2. Test deduplication (idempotency) // We'll block the retriever to simulate a slow request, then send multiple requests for the same height height2 := uint64(200) - + // Create a channel to signal when the mock is called calledCh := make(chan struct{}) // Create a channel to unblock the mock @@ -98,7 +98,7 @@ func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) { // We have 5 workers. We'll block them all. unblockCh := make(chan struct{}) - + // Expect 5 calls that block for i := 0; i < 5; i++ { h := uint64(1000 + i) diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index adacd15efc..5132b37ba7 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -731,7 +731,7 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { // Mock AsyncDARetriever mockRetriever := NewMockDARetriever(t) asyncRetriever := NewAsyncDARetriever(mockRetriever, s.heightInCh, zerolog.Nop()) - // We don't start the async retriever to avoid race conditions in test, + // We don't start the async retriever to avoid race conditions in test, // we just want to verify RequestRetrieval queues the request. // However, RequestRetrieval writes to a channel, so we need a consumer or a buffered channel. // The workCh is buffered (100), so we are good. @@ -739,9 +739,9 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { // Create event with DA height hint evt := common.DAHeightEvent{ - Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: "c", Height: 2}}}, - Data: &types.Data{Metadata: &types.Metadata{ChainID: "c", Height: 2}}, - Source: common.SourceP2P, + Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: "c", Height: 2}}}, + Data: &types.Data{Metadata: &types.Metadata{ChainID: "c", Height: 2}}, + Source: common.SourceP2P, DaHeightHints: [2]uint64{100, 100}, } @@ -749,13 +749,13 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { // processHeightEvent checks: // 1. height <= currentHeight (2 <= 0 -> false) // 2. height != currentHeight+1 (2 != 1 -> true) -> stores as pending event - + // We need to simulate height 1 being processed first so height 2 is "next" // OR we can just test that it DOES NOT trigger DA retrieval if it's pending. // Wait, the logic for DA retrieval is BEFORE the "next block" check? // Let's check syncer.go... // Yes, "If this is a P2P event with a DA height hint, trigger targeted DA retrieval" block is AFTER "If this is not the next block in sequence... return" - + // So we need to be at height 1 to process height 2. // Let's set the store height to 1. batch, err := st.NewBatch(context.Background()) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 4373442258..50914ffe49 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -72,6 +72,8 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/ github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= github.com/celestiaorg/go-header v0.7.4 h1:kQx3bVvKV+H2etxRi4IUuby5VQydBONx3giHFXDcZ/o= github.com/celestiaorg/go-header v0.7.4/go.mod h1:eX9iTSPthVEAlEDLux40ZT/olXPGhpxHd+mEzJeDhd0= +github.com/celestiaorg/go-libp2p-messenger v0.2.2 h1:osoUfqjss7vWTIZrrDSy953RjQz+ps/vBFE7bychLEc= +github.com/celestiaorg/go-libp2p-messenger v0.2.2/go.mod h1:oTCRV5TfdO7V/k6nkx7QjQzGrWuJbupv+0o1cgnY2i4= github.com/celestiaorg/go-square/v3 v3.0.2 h1:eSQOgNII8inK9IhiBZ+6GADQeWbRq4HYY72BOgcduA4= github.com/celestiaorg/go-square/v3 v3.0.2/go.mod h1:oFReMLsSDMRs82ICFEeFQFCqNvwdsbIM1BzCcb0f7dM= github.com/celestiaorg/tastora v0.8.0 h1:+FWAIsP2onwwqPTGzBLIBtx8B1h9sImdx4msv2N4DsI= From c40b96b2ef0e3ea0237b4df38fa230e22190c077 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 15 Dec 2025 10:52:10 +0100 Subject: [PATCH 07/41] Indipendent types for p2p store --- apps/evm/cmd/rollback.go | 6 +- apps/testapp/cmd/rollback.go | 6 +- block/internal/common/expected_interfaces.go | 4 +- block/internal/executing/executor.go | 8 +- .../internal/executing/executor_lazy_test.go | 8 +- .../internal/executing/executor_logic_test.go | 8 +- .../executing/executor_restart_test.go | 16 +- block/internal/executing/executor_test.go | 4 +- block/internal/syncing/p2p_handler.go | 14 +- block/internal/syncing/p2p_handler_test.go | 20 +- block/internal/syncing/syncer.go | 4 +- block/internal/syncing/syncer_backoff_test.go | 4 +- .../internal/syncing/syncer_benchmark_test.go | 4 +- .../syncing/syncer_forced_inclusion_test.go | 28 +-- block/internal/syncing/syncer_test.go | 20 +- pkg/rpc/client/client_test.go | 53 +++--- pkg/rpc/server/server.go | 13 +- pkg/rpc/server/server_test.go | 47 ++--- pkg/sync/da_hint_container.go | 21 +-- pkg/sync/sync_service.go | 52 ++--- pkg/sync/sync_service_test.go | 12 +- proto/evnode/v1/evnode.proto | 15 ++ types/binary_compatibility_test.go | 72 +++++++ types/p2p_data.go | 80 ++++++++ types/p2p_signed_header.go | 85 +++++++++ types/pb/evnode/v1/evnode.pb.go | 178 ++++++++++++++++-- 26 files changed, 586 insertions(+), 196 deletions(-) create mode 100644 types/binary_compatibility_test.go create mode 100644 types/p2p_data.go create mode 100644 types/p2p_signed_header.go diff --git a/apps/evm/cmd/rollback.go b/apps/evm/cmd/rollback.go index a8f3ed645c..25a75d8bd8 100644 --- a/apps/evm/cmd/rollback.go +++ b/apps/evm/cmd/rollback.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/evstack/ev-node/pkg/sync" + "github.com/evstack/ev-node/types" ds "github.com/ipfs/go-datastore" kt "github.com/ipfs/go-datastore/keytransform" "github.com/spf13/cobra" @@ -70,7 +70,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*sync.SignedHeaderWithDAHint]( + headerStore, err := goheaderstore.NewStore[*types.P2PSignedHeader]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), @@ -79,7 +79,7 @@ func NewRollbackCmd() *cobra.Command { return err } - dataStore, err := goheaderstore.NewStore[*sync.DataWithDAHint]( + dataStore, err := goheaderstore.NewStore[*types.P2PData]( evolveDB, goheaderstore.WithStorePrefix("dataSync"), goheaderstore.WithMetrics(), diff --git a/apps/testapp/cmd/rollback.go b/apps/testapp/cmd/rollback.go index 22ee216b9e..76e860e598 100644 --- a/apps/testapp/cmd/rollback.go +++ b/apps/testapp/cmd/rollback.go @@ -10,7 +10,7 @@ import ( "github.com/evstack/ev-node/node" rollcmd "github.com/evstack/ev-node/pkg/cmd" "github.com/evstack/ev-node/pkg/store" - "github.com/evstack/ev-node/pkg/sync" + "github.com/evstack/ev-node/types" ds "github.com/ipfs/go-datastore" kt "github.com/ipfs/go-datastore/keytransform" "github.com/spf13/cobra" @@ -75,7 +75,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback ev-node goheader state - headerStore, err := goheaderstore.NewStore[*sync.SignedHeaderWithDAHint]( + headerStore, err := goheaderstore.NewStore[*types.P2PSignedHeader]( evolveDB, goheaderstore.WithStorePrefix("headerSync"), goheaderstore.WithMetrics(), @@ -84,7 +84,7 @@ func NewRollbackCmd() *cobra.Command { return err } - dataStore, err := goheaderstore.NewStore[*sync.DataWithDAHint]( + dataStore, err := goheaderstore.NewStore[*types.P2PData]( evolveDB, goheaderstore.WithStorePrefix("dataSync"), goheaderstore.WithMetrics(), diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index 0eeef6ab31..64464538a3 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -10,8 +10,8 @@ import ( ) type ( - HeaderP2PBroadcaster = Broadcaster[*types.SignedHeader] - DataP2PBroadcaster = Broadcaster[*types.Data] + HeaderP2PBroadcaster = Broadcaster[*types.P2PSignedHeader] + DataP2PBroadcaster = Broadcaster[*types.P2PData] ) // Broadcaster interface for P2P broadcasting diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 42609de568..212d6c5728 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -439,8 +439,12 @@ func (e *Executor) produceBlock() error { // broadcast header and data to P2P network g, ctx := errgroup.WithContext(e.ctx) - g.Go(func() error { return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, header) }) - g.Go(func() error { return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, data) }) + g.Go(func() error { + return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *header}) + }) + g.Go(func() error { + return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: *data}) + }) if err := g.Wait(); err != nil { e.logger.Error().Err(err).Msg("failed to broadcast header and/data") // don't fail block production on broadcast error diff --git a/block/internal/executing/executor_lazy_test.go b/block/internal/executing/executor_lazy_test.go index a11cf6a1c2..0821454d3f 100644 --- a/block/internal/executing/executor_lazy_test.go +++ b/block/internal/executing/executor_lazy_test.go @@ -47,9 +47,9 @@ func TestLazyMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db := common.NewMockBroadcaster[*types.Data](t) + db := common.NewMockBroadcaster[*types.P2PData](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec, err := NewExecutor( @@ -162,9 +162,9 @@ func TestRegularMode_ProduceBlockLogic(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db := common.NewMockBroadcaster[*types.Data](t) + db := common.NewMockBroadcaster[*types.P2PData](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec, err := NewExecutor( diff --git a/block/internal/executing/executor_logic_test.go b/block/internal/executing/executor_logic_test.go index 6029186e86..2f9b29721a 100644 --- a/block/internal/executing/executor_logic_test.go +++ b/block/internal/executing/executor_logic_test.go @@ -69,9 +69,9 @@ func TestProduceBlock_EmptyBatch_SetsEmptyDataHash(t *testing.T) { mockSeq := testmocks.NewMockSequencer(t) // Broadcasters are required by produceBlock; use generated mocks - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db := common.NewMockBroadcaster[*types.Data](t) + db := common.NewMockBroadcaster[*types.P2PData](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec, err := NewExecutor( @@ -159,9 +159,9 @@ func TestPendingLimit_SkipsProduction(t *testing.T) { mockExec := testmocks.NewMockExecutor(t) mockSeq := testmocks.NewMockSequencer(t) - hb := common.NewMockBroadcaster[*types.SignedHeader](t) + hb := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db := common.NewMockBroadcaster[*types.Data](t) + db := common.NewMockBroadcaster[*types.P2PData](t) db.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec, err := NewExecutor( diff --git a/block/internal/executing/executor_restart_test.go b/block/internal/executing/executor_restart_test.go index 14daccddcc..8b68a1c651 100644 --- a/block/internal/executing/executor_restart_test.go +++ b/block/internal/executing/executor_restart_test.go @@ -47,9 +47,9 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create first executor instance mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb1 := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db1 := common.NewMockBroadcaster[*types.Data](t) + db1 := common.NewMockBroadcaster[*types.P2PData](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec1, err := NewExecutor( @@ -169,9 +169,9 @@ func TestExecutor_RestartUsesPendingHeader(t *testing.T) { // Create second executor instance (restart scenario) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb2 := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db2 := common.NewMockBroadcaster[*types.Data](t) + db2 := common.NewMockBroadcaster[*types.P2PData](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec2, err := NewExecutor( @@ -270,9 +270,9 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create first executor and produce one block mockExec1 := testmocks.NewMockExecutor(t) mockSeq1 := testmocks.NewMockSequencer(t) - hb1 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb1 := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db1 := common.NewMockBroadcaster[*types.Data](t) + db1 := common.NewMockBroadcaster[*types.P2PData](t) db1.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec1, err := NewExecutor( @@ -325,9 +325,9 @@ func TestExecutor_RestartNoPendingHeader(t *testing.T) { // Create second executor (restart) mockExec2 := testmocks.NewMockExecutor(t) mockSeq2 := testmocks.NewMockSequencer(t) - hb2 := common.NewMockBroadcaster[*types.SignedHeader](t) + hb2 := common.NewMockBroadcaster[*types.P2PSignedHeader](t) hb2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() - db2 := common.NewMockBroadcaster[*types.Data](t) + db2 := common.NewMockBroadcaster[*types.P2PData](t) db2.EXPECT().WriteToStoreAndBroadcast(mock.Anything, mock.Anything).Return(nil).Maybe() exec2, err := NewExecutor( diff --git a/block/internal/executing/executor_test.go b/block/internal/executing/executor_test.go index 7ef0f64e21..7e62d09170 100644 --- a/block/internal/executing/executor_test.go +++ b/block/internal/executing/executor_test.go @@ -39,8 +39,8 @@ func TestExecutor_BroadcasterIntegration(t *testing.T) { } // Create mock broadcasters - headerBroadcaster := common.NewMockBroadcaster[*types.SignedHeader](t) - dataBroadcaster := common.NewMockBroadcaster[*types.Data](t) + headerBroadcaster := common.NewMockBroadcaster[*types.P2PSignedHeader](t) + dataBroadcaster := common.NewMockBroadcaster[*types.P2PData](t) // Create executor with broadcasters executor, err := NewExecutor( diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index 18c3e6c634..72f942432d 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -32,8 +32,8 @@ type HeightStore[H header.Header[H]] interface { // The handler maintains a processedHeight to track the highest block that has been // successfully validated and sent to the syncer, preventing duplicate processing. type P2PHandler struct { - headerStore HeightStore[*types.SignedHeader] - dataStore HeightStore[*types.Data] + headerStore HeightStore[*types.P2PSignedHeader] + dataStore HeightStore[*types.P2PData] cache cache.CacheManager genesis genesis.Genesis logger zerolog.Logger @@ -43,8 +43,8 @@ type P2PHandler struct { // NewP2PHandler creates a new P2P handler. func NewP2PHandler( - headerStore HeightStore[*types.SignedHeader], - dataStore HeightStore[*types.Data], + headerStore HeightStore[*types.P2PSignedHeader], + dataStore HeightStore[*types.P2PData], cache cache.CacheManager, genesis genesis.Genesis, logger zerolog.Logger, @@ -79,25 +79,27 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC return nil } - header, headerDAHint, err := h.headerStore.GetByHeight(ctx, height) + p2pHeader, headerDAHint, err := h.headerStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("header unavailable in store") } return err } + header := &p2pHeader.SignedHeader if err := h.assertExpectedProposer(header.ProposerAddress); err != nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("invalid header from P2P") return err } - data, dataDAHint, err := h.dataStore.GetByHeight(ctx, height) + p2pData, dataDAHint, err := h.dataStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("data unavailable in store") } return err } + data := &p2pData.Data dataCommitment := data.DACommitment() if !bytes.Equal(header.DataHash[:], dataCommitment[:]) { err := fmt.Errorf("data hash mismatch: header %x, data %x", header.DataHash, dataCommitment) diff --git a/block/internal/syncing/p2p_handler_test.go b/block/internal/syncing/p2p_handler_test.go index 5970900474..b0d2dbdabc 100644 --- a/block/internal/syncing/p2p_handler_test.go +++ b/block/internal/syncing/p2p_handler_test.go @@ -36,7 +36,7 @@ func buildTestSigner(t *testing.T) ([]byte, crypto.PubKey, signerpkg.Signer) { } // p2pMakeSignedHeader creates a minimally valid SignedHeader for P2P tests. -func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer []byte, pub crypto.PubKey, signer signerpkg.Signer) *types.SignedHeader { +func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer []byte, pub crypto.PubKey, signer signerpkg.Signer) *types.P2PSignedHeader { t.Helper() hdr := &types.SignedHeader{ Header: types.Header{ @@ -50,14 +50,14 @@ func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer [ sig, err := signer.Sign(bz) require.NoError(t, err, "failed to sign header bytes") hdr.Signature = sig - return hdr + return &types.P2PSignedHeader{SignedHeader: *hdr} } // P2PTestData aggregates dependencies used by P2P handler tests. type P2PTestData struct { Handler *P2PHandler - HeaderStore *MockHeightStore[*types.SignedHeader] - DataStore *MockHeightStore[*types.Data] + HeaderStore *MockHeightStore[*types.P2PSignedHeader] + DataStore *MockHeightStore[*types.P2PData] Cache cache.CacheManager Genesis genesis.Genesis ProposerAddr []byte @@ -72,8 +72,8 @@ func setupP2P(t *testing.T) *P2PTestData { gen := genesis.Genesis{ChainID: "p2p-test", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: proposerAddr} - headerStoreMock := NewMockHeightStore[*types.SignedHeader](t) - dataStoreMock := NewMockHeightStore[*types.Data](t) + headerStoreMock := NewMockHeightStore[*types.P2PSignedHeader](t) + dataStoreMock := NewMockHeightStore[*types.P2PData](t) cfg := config.Config{ RootDir: t.TempDir(), @@ -128,7 +128,7 @@ func TestP2PHandler_ProcessHeight_EmitsEventWhenHeaderAndDataPresent(t *testing. require.Equal(t, string(p.Genesis.ProposerAddress), string(p.ProposerAddr)) header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 5, p.ProposerAddr, p.ProposerPub, p.Signer) - data := makeData(p.Genesis.ChainID, 5, 1) + data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 5, 1)} header.DataHash = data.DACommitment() bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) @@ -154,7 +154,7 @@ func TestP2PHandler_ProcessHeight_SkipsWhenDataMissing(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 7, p.ProposerAddr, p.ProposerPub, p.Signer) - data := makeData(p.Genesis.ChainID, 7, 1) + data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 7, 1)} header.DataHash = data.DACommitment() bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) @@ -224,7 +224,7 @@ func TestP2PHandler_ProcessedHeightSkipsPreviouslyHandledBlocks(t *testing.T) { // Height 6 should be fetched normally. header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 6, p.ProposerAddr, p.ProposerPub, p.Signer) - data := makeData(p.Genesis.ChainID, 6, 1) + data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 6, 1)} header.DataHash = data.DACommitment() bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) @@ -247,7 +247,7 @@ func TestP2PHandler_SetProcessedHeightPreventsDuplicates(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 8, p.ProposerAddr, p.ProposerPub, p.Signer) - data := makeData(p.Genesis.ChainID, 8, 0) + data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 8, 0)} header.DataHash = data.DACommitment() bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 0e8b61b640..0e9eaf334e 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -535,12 +535,12 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { g.Go(func() error { // broadcast header locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - return s.headerStore.WriteToStoreAndBroadcast(ctx, event.Header, pubsub.WithLocalPublication(true)) + return s.headerStore.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *event.Header}, pubsub.WithLocalPublication(true)) }) g.Go(func() error { // broadcast data locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - return s.dataStore.WriteToStoreAndBroadcast(ctx, event.Data, pubsub.WithLocalPublication(true)) + return s.dataStore.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: *event.Data}, pubsub.WithLocalPublication(true)) }) if err := g.Wait(); err != nil { s.logger.Error().Err(err).Msg("failed to append event header and/or data to p2p store") diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index ed9cd4c407..dcb2323b59 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -326,8 +326,8 @@ func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index e2b6f6e51f..a1f0a00314 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -153,9 +153,9 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay mockP2P := newMockp2pHandler(b) // not used directly in this benchmark path mockP2P.On("SetProcessedHeight", mock.Anything).Return().Maybe() s.p2pHandler = mockP2P - headerP2PStore := common.NewMockBroadcaster[*types.SignedHeader](b) + headerP2PStore := common.NewMockBroadcaster[*types.P2PSignedHeader](b) s.headerStore = headerP2PStore - dataP2PStore := common.NewMockBroadcaster[*types.Data](b) + dataP2PStore := common.NewMockBroadcaster[*types.P2PData](b) s.dataStore = dataP2PStore return &benchFixture{s: s, st: st, cm: cm, cancel: cancel} } diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index 741432eb28..072d2ed448 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -67,8 +67,8 @@ func TestVerifyForcedInclusionTxs_AllTransactionsIncluded(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -152,8 +152,8 @@ func TestVerifyForcedInclusionTxs_MissingTransactions(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -253,8 +253,8 @@ func TestVerifyForcedInclusionTxs_PartiallyIncluded(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -356,8 +356,8 @@ func TestVerifyForcedInclusionTxs_NoForcedTransactions(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -430,8 +430,8 @@ func TestVerifyForcedInclusionTxs_NamespaceNotConfigured(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -499,8 +499,8 @@ func TestVerifyForcedInclusionTxs_DeferralWithinEpoch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -646,8 +646,8 @@ func TestVerifyForcedInclusionTxs_MaliciousAfterEpochEnd(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 5132b37ba7..f7386d7a5f 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -123,8 +123,8 @@ func TestSyncer_validateBlock_DataHashMismatch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -174,8 +174,8 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), errChan, @@ -228,8 +228,8 @@ func TestSequentialBlockSync(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), errChan, @@ -346,8 +346,8 @@ func TestSyncLoopPersistState(t *testing.T) { mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - mockP2PHeaderStore := common.NewMockBroadcaster[*types.SignedHeader](t) - mockP2PDataStore := common.NewMockBroadcaster[*types.Data](t) + mockP2PHeaderStore := common.NewMockBroadcaster[*types.P2PSignedHeader](t) + mockP2PDataStore := common.NewMockBroadcaster[*types.P2PData](t) errorCh := make(chan error, 1) syncerInst1 := NewSyncer( @@ -719,8 +719,8 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index 05c8df4d08..fd3d1f2dc0 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -8,7 +8,6 @@ import ( "time" goheader "github.com/celestiaorg/go-header" - "github.com/evstack/ev-node/pkg/sync" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" @@ -29,8 +28,8 @@ import ( func setupTestServer( t *testing.T, mockStore *mocks.MockStore, - headerStore goheader.Store[*sync.SignedHeaderWithDAHint], - dataStore goheader.Store[*sync.DataWithDAHint], + headerStore goheader.Store[*types.P2PSignedHeader], + dataStore goheader.Store[*types.P2PData], mockP2P *mocks.MockP2PRPC, ) (*httptest.Server, *Client) { t.Helper() @@ -107,19 +106,19 @@ func TestClientGetMetadata(t *testing.T) { func TestClientGetP2PStoreInfo(t *testing.T) { mockStore := mocks.NewMockStore(t) mockP2P := mocks.NewMockP2PRPC(t) - headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) - dataStore := headerstoremocks.NewMockStore[*sync.DataWithDAHint](t) + headerStore := headerstoremocks.NewMockStore[*types.P2PSignedHeader](t) + dataStore := headerstoremocks.NewMockStore[*types.P2PData](t) now := time.Now().UTC() - headerHead := &sync.SignedHeaderWithDAHint{Entry: testSignedHeader(10, now)} - headerTail := &sync.SignedHeaderWithDAHint{Entry: testSignedHeader(5, now.Add(-time.Minute))} + headerHead := testSignedHeader(10, now) + headerTail := testSignedHeader(5, now.Add(-time.Minute)) headerStore.On("Height").Return(uint64(10)) headerStore.On("Head", mock.Anything).Return(headerHead, nil) headerStore.On("Tail", mock.Anything).Return(headerTail, nil) - dataHead := &sync.DataWithDAHint{Entry: testData(8, now.Add(-30*time.Second))} - dataTail := &sync.DataWithDAHint{Entry: testData(4, now.Add(-2*time.Minute))} + dataHead := testData(8, now.Add(-30*time.Second)) + dataTail := testData(4, now.Add(-2*time.Minute)) dataStore.On("Height").Return(uint64(8)) dataStore.On("Head", mock.Anything).Return(dataHead, nil) dataStore.On("Tail", mock.Anything).Return(dataTail, nil) @@ -252,27 +251,31 @@ func TestClientGetNamespace(t *testing.T) { require.NotEmpty(t, namespaceResp.DataNamespace) } -func testSignedHeader(height uint64, ts time.Time) *types.SignedHeader { - return &types.SignedHeader{ - Header: types.Header{ - BaseHeader: types.BaseHeader{ - Height: height, - Time: uint64(ts.UnixNano()), - ChainID: "test-chain", +func testSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { + return &types.P2PSignedHeader{ + SignedHeader: types.SignedHeader{ + Header: types.Header{ + BaseHeader: types.BaseHeader{ + Height: height, + Time: uint64(ts.UnixNano()), + ChainID: "test-chain", + }, + ProposerAddress: []byte{0x01}, + DataHash: []byte{0x02}, + AppHash: []byte{0x03}, }, - ProposerAddress: []byte{0x01}, - DataHash: []byte{0x02}, - AppHash: []byte{0x03}, }, } } -func testData(height uint64, ts time.Time) *types.Data { - return &types.Data{ - Metadata: &types.Metadata{ - ChainID: "test-chain", - Height: height, - Time: uint64(ts.UnixNano()), +func testData(height uint64, ts time.Time) *types.P2PData { + return &types.P2PData{ + Data: types.Data{ + Metadata: &types.Metadata{ + ChainID: "test-chain", + Height: height, + Time: uint64(ts.UnixNano()), + }, }, } } diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index f113b52fb8..27048c3010 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -14,7 +14,6 @@ import ( "connectrpc.com/grpcreflect" goheader "github.com/celestiaorg/go-header" coreda "github.com/evstack/ev-node/core/da" - "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" "github.com/rs/zerolog" "golang.org/x/net/http2" @@ -35,16 +34,16 @@ var _ rpc.StoreServiceHandler = (*StoreServer)(nil) // StoreServer implements the StoreService defined in the proto file type StoreServer struct { store store.Store - headerStore goheader.Store[*sync.SignedHeaderWithDAHint] - dataStore goheader.Store[*sync.DataWithDAHint] + headerStore goheader.Store[*types.P2PSignedHeader] + dataStore goheader.Store[*types.P2PData] logger zerolog.Logger } // NewStoreServer creates a new StoreServer instance func NewStoreServer( store store.Store, - headerStore goheader.Store[*sync.SignedHeaderWithDAHint], - dataStore goheader.Store[*sync.DataWithDAHint], + headerStore goheader.Store[*types.P2PSignedHeader], + dataStore goheader.Store[*types.P2PData], logger zerolog.Logger, ) *StoreServer { return &StoreServer{ @@ -371,8 +370,8 @@ func (p *P2PServer) GetNetInfo( // NewServiceHandler creates a new HTTP handler for Store, P2P and Config services func NewServiceHandler( store store.Store, - headerStore goheader.Store[*sync.SignedHeaderWithDAHint], - dataStore goheader.Store[*sync.DataWithDAHint], + headerStore goheader.Store[*types.P2PSignedHeader], + dataStore goheader.Store[*types.P2PData], peerManager p2p.P2PRPC, proposerAddress []byte, logger zerolog.Logger, diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 3a4dbd89bd..842adea560 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -11,7 +11,6 @@ import ( "time" "connectrpc.com/connect" - "github.com/evstack/ev-node/pkg/sync" ds "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -326,8 +325,8 @@ func TestGetGenesisDaHeight_InvalidLength(t *testing.T) { func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns snapshots for configured stores", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) - dataStore := headerstoremocks.NewMockStore[*sync.DataWithDAHint](t) + headerStore := headerstoremocks.NewMockStore[*types.P2PSignedHeader](t) + dataStore := headerstoremocks.NewMockStore[*types.P2PData](t) logger := zerolog.Nop() server := NewStoreServer(mockStore, headerStore, dataStore, logger) @@ -355,10 +354,10 @@ func TestGetP2PStoreInfo(t *testing.T) { t.Run("returns error when a store edge fails", func(t *testing.T) { mockStore := mocks.NewMockStore(t) - headerStore := headerstoremocks.NewMockStore[*sync.SignedHeaderWithDAHint](t) + headerStore := headerstoremocks.NewMockStore[*types.P2PSignedHeader](t) logger := zerolog.Nop() headerStore.On("Height").Return(uint64(0)) - headerStore.On("Head", mock.Anything).Return((*sync.SignedHeaderWithDAHint)(nil), fmt.Errorf("boom")) + headerStore.On("Head", mock.Anything).Return((*types.P2PSignedHeader)(nil), fmt.Errorf("boom")) server := NewStoreServer(mockStore, headerStore, nil, logger) resp, err := server.GetP2PStoreInfo(context.Background(), connect.NewRequest(&emptypb.Empty{})) @@ -628,29 +627,31 @@ func TestHealthReadyEndpoint(t *testing.T) { }) } -func makeTestSignedHeader(height uint64, ts time.Time) *sync.SignedHeaderWithDAHint { - return &sync.SignedHeaderWithDAHint{Entry: &types.SignedHeader{ - Header: types.Header{ - BaseHeader: types.BaseHeader{ - Height: height, - Time: uint64(ts.UnixNano()), - ChainID: "test-chain", +func makeTestSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { + return &types.P2PSignedHeader{ + SignedHeader: types.SignedHeader{ + Header: types.Header{ + BaseHeader: types.BaseHeader{ + Height: height, + Time: uint64(ts.UnixNano()), + ChainID: "test-chain", + }, + ProposerAddress: []byte{0x01}, + DataHash: []byte{0x02}, + AppHash: []byte{0x03}, }, - ProposerAddress: []byte{0x01}, - DataHash: []byte{0x02}, - AppHash: []byte{0x03}, }, - }, } } -func makeTestData(height uint64, ts time.Time) *sync.DataWithDAHint { - return &sync.DataWithDAHint{Entry: &types.Data{ - Metadata: &types.Metadata{ - ChainID: "test-chain", - Height: height, - Time: uint64(ts.UnixNano()), +func makeTestData(height uint64, ts time.Time) *types.P2PData { + return &types.P2PData{ + Data: types.Data{ + Metadata: &types.Metadata{ + ChainID: "test-chain", + Height: height, + Time: uint64(ts.UnixNano()), + }, }, - }, } } diff --git a/pkg/sync/da_hint_container.go b/pkg/sync/da_hint_container.go index 5d904d885d..dc52fcb953 100644 --- a/pkg/sync/da_hint_container.go +++ b/pkg/sync/da_hint_container.go @@ -1,17 +1,11 @@ package sync import ( - "encoding/binary" - "fmt" "time" "github.com/celestiaorg/go-header" - "github.com/evstack/ev-node/types" ) -type SignedHeaderWithDAHint = DAHeightHintContainer[*types.SignedHeader] -type DataWithDAHint = DAHeightHintContainer[*types.Data] - type DAHeightHintContainer[H header.Header[H]] struct { Entry H DAHeightHint uint64 @@ -62,20 +56,9 @@ func (s *DAHeightHintContainer[H]) IsZero() bool { } func (s *DAHeightHintContainer[H]) MarshalBinary() ([]byte, error) { - bz, err := s.Entry.MarshalBinary() - if err != nil { - return nil, err - } - out := make([]byte, 8+len(bz)) - binary.BigEndian.PutUint64(out, s.DAHeightHint) - copy(out[8:], bz) - return out, nil + return s.Entry.MarshalBinary() } func (s *DAHeightHintContainer[H]) UnmarshalBinary(data []byte) error { - if len(data) < 8 { - return fmt.Errorf("invalid length: %d", len(data)) - } - s.DAHeightHint = binary.BigEndian.Uint64(data) - return s.Entry.UnmarshalBinary(data[8:]) + return s.Entry.UnmarshalBinary(data) } diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 22d7c3c904..c59ab2048a 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -43,15 +43,15 @@ type EntityWithDAHint[H any] interface { } // HeaderSyncService is the P2P Sync Service for headers. -type HeaderSyncService = SyncService[*types.SignedHeader] +type HeaderSyncService = SyncService[*types.P2PSignedHeader] // DataSyncService is the P2P Sync Service for blocks. -type DataSyncService = SyncService[*types.Data] +type DataSyncService = SyncService[*types.P2PData] // SyncService is the P2P Sync Service for blocks and headers. // // Uses the go-header library for handling all P2P logic. -type SyncService[V header.Header[V]] struct { +type SyncService[V EntityWithDAHint[V]] struct { conf config.Config logger zerolog.Logger syncType syncType @@ -60,13 +60,13 @@ type SyncService[V header.Header[V]] struct { p2p *p2p.Client - ex *goheaderp2p.Exchange[*DAHeightHintContainer[V]] - sub *goheaderp2p.Subscriber[*DAHeightHintContainer[V]] - p2pServer *goheaderp2p.ExchangeServer[*DAHeightHintContainer[V]] - store *goheaderstore.Store[*DAHeightHintContainer[V]] - syncer *goheadersync.Syncer[*DAHeightHintContainer[V]] + ex *goheaderp2p.Exchange[V] + sub *goheaderp2p.Subscriber[V] + p2pServer *goheaderp2p.ExchangeServer[V] + store *goheaderstore.Store[V] + syncer *goheadersync.Syncer[V] syncerStatus *SyncerStatus - topicSubscription header.Subscription[*DAHeightHintContainer[V]] + topicSubscription header.Subscription[V] storeInitialized atomic.Bool } @@ -78,7 +78,7 @@ func NewDataSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*DataSyncService, error) { - return newSyncService[*types.Data](store, dataSync, conf, genesis, p2p, logger) + return newSyncService[*types.P2PData](store, dataSync, conf, genesis, p2p, logger) } // NewHeaderSyncService returns a new HeaderSyncService. @@ -89,10 +89,10 @@ func NewHeaderSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*HeaderSyncService, error) { - return newSyncService[*types.SignedHeader](store, headerSync, conf, genesis, p2p, logger) + return newSyncService[*types.P2PSignedHeader](store, headerSync, conf, genesis, p2p, logger) } -func newSyncService[V header.Header[V]]( +func newSyncService[V EntityWithDAHint[V]]( store ds.Batching, syncType syncType, conf config.Config, @@ -104,7 +104,7 @@ func newSyncService[V header.Header[V]]( return nil, errors.New("p2p client cannot be nil") } - ss, err := goheaderstore.NewStore[*DAHeightHintContainer[V]]( + ss, err := goheaderstore.NewStore[V]( store, goheaderstore.WithStorePrefix(string(syncType)), goheaderstore.WithMetrics(), @@ -127,7 +127,7 @@ func newSyncService[V header.Header[V]]( } // Store returns the store of the SyncService -func (syncService *SyncService[V]) Store() header.Store[*DAHeightHintContainer[V]] { +func (syncService *SyncService[V]) Store() header.Store[V] { return syncService.store } @@ -142,7 +142,7 @@ func (syncService *SyncService[V]) WriteToStoreAndBroadcast(ctx context.Context, return fmt.Errorf("empty header/data cannot write to store or broadcast") } - headerOrData := &DAHeightHintContainer[V]{Entry: payload} + headerOrData := payload storeInitialized := false if syncService.storeInitialized.CompareAndSwap(false, true) { var err error @@ -182,7 +182,7 @@ func (syncService *SyncService[V]) WriteToStoreAndBroadcast(ctx context.Context, } func (s *SyncService[V]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { - entries := make([]*DAHeightHintContainer[V], 0, len(hashes)) + entries := make([]V, 0, len(hashes)) for _, h := range hashes { v, err := s.store.Get(ctx, h) if err != nil { @@ -203,7 +203,7 @@ func (s *SyncService[V]) GetByHeight(ctx context.Context, height uint64) (V, uin var zero V return zero, 0, err } - return c.Entry, c.DAHint(), nil + return c, c.DAHint(), nil } // Start is a part of Service interface. @@ -258,7 +258,7 @@ func (syncService *SyncService[H]) startSyncer(ctx context.Context) error { // initStore initializes the store with the given initial header. // it is a no-op if the store is already initialized. // Returns true when the store was initialized by this call. -func (syncService *SyncService[V]) initStore(ctx context.Context, initial *DAHeightHintContainer[V]) (bool, error) { +func (syncService *SyncService[V]) initStore(ctx context.Context, initial V) (bool, error) { if initial.IsZero() { return false, errors.New("failed to initialize the store") } @@ -292,7 +292,7 @@ func (syncService *SyncService[V]) setupP2PInfrastructure(ctx context.Context) ( networkID := syncService.getNetworkID(chainID) // Create subscriber but DON'T start it yet - syncService.sub, err = goheaderp2p.NewSubscriber[*DAHeightHintContainer[V]]( + syncService.sub, err = goheaderp2p.NewSubscriber[V]( ps, pubsub.DefaultMsgIdFn, goheaderp2p.WithSubscriberNetworkID(networkID), @@ -315,7 +315,7 @@ func (syncService *SyncService[V]) setupP2PInfrastructure(ctx context.Context) ( peerIDs := syncService.getPeerIDs() - if syncService.ex, err = newP2PExchange[*DAHeightHintContainer[V]](syncService.p2p.Host(), peerIDs, networkID, syncService.genesis.ChainID, syncService.p2p.ConnectionGater()); err != nil { + if syncService.ex, err = newP2PExchange[V](syncService.p2p.Host(), peerIDs, networkID, syncService.genesis.ChainID, syncService.p2p.ConnectionGater()); err != nil { return nil, fmt.Errorf("error while creating exchange: %w", err) } if err := syncService.ex.Start(ctx); err != nil { @@ -350,7 +350,7 @@ func (syncService *SyncService[V]) initFromP2PWithRetry(ctx context.Context, pee tryInit := func(ctx context.Context) (bool, error) { var ( - trusted *DAHeightHintContainer[V] + trusted V err error heightToQuery uint64 ) @@ -460,17 +460,17 @@ func newP2PExchange[H header.Header[H]]( // newSyncer constructs new Syncer for headers/blocks. func newSyncer[H header.Header[H]]( - ex header.Exchange[*DAHeightHintContainer[H]], - store header.Store[*DAHeightHintContainer[H]], - sub header.Subscriber[*DAHeightHintContainer[H]], + ex header.Exchange[H], + store header.Store[H], + sub header.Subscriber[H], opts []goheadersync.Option, -) (*goheadersync.Syncer[*DAHeightHintContainer[H]], error) { +) (*goheadersync.Syncer[H], error) { opts = append(opts, goheadersync.WithMetrics(), goheadersync.WithPruningWindow(ninetyNineYears), goheadersync.WithTrustingPeriod(ninetyNineYears), ) - return goheadersync.NewSyncer[*DAHeightHintContainer[H]](ex, store, sub, opts...) + return goheadersync.NewSyncer[H](ex, store, sub, opts...) } func (syncService *SyncService[H]) getNetworkID(network string) string { diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index b0e244a95f..048d19d6cd 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -73,12 +73,12 @@ func TestHeaderSyncServiceRestart(t *testing.T) { signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, genesisDoc.ChainID) require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) for i := genesisDoc.InitialHeight + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) } // then stop and restart service @@ -109,7 +109,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { for i := signedHeader.Height() + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) } cancel() } @@ -164,7 +164,7 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, signedHeader)) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) } func TestDAHintStorageHeader(t *testing.T) { @@ -215,7 +215,7 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, signedHeader)) + require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) daHeight := uint64(100) require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Hash())) @@ -306,7 +306,7 @@ func TestDAHintStorageData(t *testing.T) { }, } - require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &data)) + require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: data})) daHeight := uint64(100) require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Hash())) diff --git a/proto/evnode/v1/evnode.proto b/proto/evnode/v1/evnode.proto index 1cb3e23ea1..8bd7d13a25 100644 --- a/proto/evnode/v1/evnode.proto +++ b/proto/evnode/v1/evnode.proto @@ -95,3 +95,18 @@ message Vote { // Validator address bytes validator_address = 5; } + +// P2PSignedHeader +message P2PSignedHeader { + Header header = 1; + bytes signature = 2; + Signer signer = 3; + optional uint64 da_height_hint = 4; +} + +// P2PData +message P2PData { + Metadata metadata = 1; + repeated bytes txs = 2; + optional uint64 da_height_hint = 3; +} diff --git a/types/binary_compatibility_test.go b/types/binary_compatibility_test.go new file mode 100644 index 0000000000..a8e8253d4e --- /dev/null +++ b/types/binary_compatibility_test.go @@ -0,0 +1,72 @@ +package types + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSignedHeaderBinaryCompatibility(t *testing.T) { + signedHeader, _, err := GetRandomSignedHeader("chain-id") + require.NoError(t, err) + bytes, err := signedHeader.MarshalBinary() + require.NoError(t, err) + + var p2pHeader P2PSignedHeader + err = p2pHeader.UnmarshalBinary(bytes) + require.NoError(t, err) + + assert.Equal(t, signedHeader.Header, p2pHeader.Header) + assert.Equal(t, signedHeader.Signature, p2pHeader.Signature) + assert.Equal(t, signedHeader.Signer, p2pHeader.Signer) + assert.Zero(t, p2pHeader.DAHeightHint) + + p2pHeader.DAHeightHint = 100 + p2pBytes, err := p2pHeader.MarshalBinary() + require.NoError(t, err) + + var decodedSignedHeader SignedHeader + err = decodedSignedHeader.UnmarshalBinary(p2pBytes) + require.NoError(t, err) + assert.Equal(t, signedHeader.Header, decodedSignedHeader.Header) + assert.Equal(t, signedHeader.Signature, decodedSignedHeader.Signature) + assert.Equal(t, signedHeader.Signer, decodedSignedHeader.Signer) +} + +func TestDataBinaryCompatibility(t *testing.T) { + data := &Data{ + Metadata: &Metadata{ + ChainID: "chain-id", + Height: 10, + Time: uint64(time.Now().UnixNano()), + LastDataHash: []byte("last-hash"), + }, + Txs: Txs{ + []byte("tx1"), + []byte("tx2"), + }, + } + bytes, err := data.MarshalBinary() + require.NoError(t, err) + + var p2pData P2PData + err = p2pData.UnmarshalBinary(bytes) + require.NoError(t, err) + + assert.Equal(t, data.Metadata, p2pData.Metadata) + assert.Equal(t, data.Txs, p2pData.Txs) + assert.Zero(t, p2pData.DAHeightHint) + + p2pData.DAHeightHint = 200 + + p2pBytes, err := p2pData.MarshalBinary() + require.NoError(t, err) + + var decodedData Data + err = decodedData.UnmarshalBinary(p2pBytes) + require.NoError(t, err) + assert.Equal(t, data.Metadata, decodedData.Metadata) + assert.Equal(t, data.Txs, decodedData.Txs) +} diff --git a/types/p2p_data.go b/types/p2p_data.go new file mode 100644 index 0000000000..d57671a081 --- /dev/null +++ b/types/p2p_data.go @@ -0,0 +1,80 @@ +package types + +import ( + "errors" + + "github.com/celestiaorg/go-header" + "google.golang.org/protobuf/proto" + + pb "github.com/evstack/ev-node/types/pb/evnode/v1" +) + +var _ header.Header[*P2PData] = &P2PData{} + +type P2PData struct { + Data + DAHeightHint uint64 +} + +func (d *P2PData) New() *P2PData { + return new(P2PData) +} + +func (d *P2PData) IsZero() bool { + return d == nil || d.Data.IsZero() +} + +func (d *P2PData) Verify(untrstD *P2PData) error { + return d.Data.Verify(&untrstD.Data) +} + +func (d *P2PData) SetDAHint(daHeight uint64) { + d.DAHeightHint = daHeight +} + +func (d *P2PData) DAHint() uint64 { + return d.DAHeightHint +} + +func (d *P2PData) MarshalBinary() ([]byte, error) { + msg, err := d.ToProto() + if err != nil { + return nil, err + } + return proto.Marshal(msg) +} + +func (d *P2PData) UnmarshalBinary(data []byte) error { + var pData pb.P2PData + if err := proto.Unmarshal(data, &pData); err != nil { + return err + } + return d.FromProto(&pData) +} + +func (d *P2PData) ToProto() (*pb.P2PData, error) { + pData := d.Data.ToProto() + return &pb.P2PData{ + Metadata: pData.Metadata, + Txs: pData.Txs, + DaHeightHint: &d.DAHeightHint, + }, nil +} + +func (d *P2PData) FromProto(other *pb.P2PData) error { + if other == nil { + return errors.New("P2PData is nil") + } + + pData := &pb.Data{ + Metadata: other.Metadata, + Txs: other.Txs, + } + if err := d.Data.FromProto(pData); err != nil { + return err + } + if other.DaHeightHint != nil { + d.DAHeightHint = *other.DaHeightHint + } + return nil +} diff --git a/types/p2p_signed_header.go b/types/p2p_signed_header.go new file mode 100644 index 0000000000..fe1ce807ba --- /dev/null +++ b/types/p2p_signed_header.go @@ -0,0 +1,85 @@ +package types + +import ( + "errors" + + "github.com/celestiaorg/go-header" + "google.golang.org/protobuf/proto" + + pb "github.com/evstack/ev-node/types/pb/evnode/v1" +) + +var _ header.Header[*P2PSignedHeader] = &P2PSignedHeader{} + +type P2PSignedHeader struct { + SignedHeader + DAHeightHint uint64 +} + +func (sh *P2PSignedHeader) New() *P2PSignedHeader { + return new(P2PSignedHeader) +} + +func (sh *P2PSignedHeader) IsZero() bool { + return sh == nil || sh.SignedHeader.IsZero() +} + +func (sh *P2PSignedHeader) Verify(untrstH *P2PSignedHeader) error { + return sh.SignedHeader.Verify(&untrstH.SignedHeader) +} + +func (sh *P2PSignedHeader) SetDAHint(daHeight uint64) { + sh.DAHeightHint = daHeight +} + +func (sh *P2PSignedHeader) DAHint() uint64 { + return sh.DAHeightHint +} + +func (sh *P2PSignedHeader) MarshalBinary() ([]byte, error) { + msg, err := sh.ToProto() + if err != nil { + return nil, err + } + return proto.Marshal(msg) +} + +func (sh *P2PSignedHeader) UnmarshalBinary(data []byte) error { + var pHeader pb.P2PSignedHeader + if err := proto.Unmarshal(data, &pHeader); err != nil { + return err + } + return sh.FromProto(&pHeader) +} + +func (sh *P2PSignedHeader) ToProto() (*pb.P2PSignedHeader, error) { + psh, err := sh.SignedHeader.ToProto() + if err != nil { + return nil, err + } + return &pb.P2PSignedHeader{ + Header: psh.Header, + Signature: psh.Signature, + Signer: psh.Signer, + DaHeightHint: &sh.DAHeightHint, + }, nil +} + +func (sh *P2PSignedHeader) FromProto(other *pb.P2PSignedHeader) error { + if other == nil { + return errors.New("P2PSignedHeader is nil") + } + // Reconstruct SignedHeader + psh := &pb.SignedHeader{ + Header: other.Header, + Signature: other.Signature, + Signer: other.Signer, + } + if err := sh.SignedHeader.FromProto(psh); err != nil { + return err + } + if other.DaHeightHint != nil { + sh.DAHeightHint = *other.DaHeightHint + } + return nil +} diff --git a/types/pb/evnode/v1/evnode.pb.go b/types/pb/evnode/v1/evnode.pb.go index 7c532c7c7e..775e35e992 100644 --- a/types/pb/evnode/v1/evnode.pb.go +++ b/types/pb/evnode/v1/evnode.pb.go @@ -585,6 +585,134 @@ func (x *Vote) GetValidatorAddress() []byte { return nil } +type P2PSignedHeader struct { + state protoimpl.MessageState `protogen:"open.v1"` + Header *Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + Signature []byte `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` + Signer *Signer `protobuf:"bytes,3,opt,name=signer,proto3" json:"signer,omitempty"` + DaHeightHint *uint64 `protobuf:"varint,4,opt,name=da_height_hint,json=daHeightHint,proto3,oneof" json:"da_height_hint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *P2PSignedHeader) Reset() { + *x = P2PSignedHeader{} + mi := &file_evnode_v1_evnode_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *P2PSignedHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*P2PSignedHeader) ProtoMessage() {} + +func (x *P2PSignedHeader) ProtoReflect() protoreflect.Message { + mi := &file_evnode_v1_evnode_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use P2PSignedHeader.ProtoReflect.Descriptor instead. +func (*P2PSignedHeader) Descriptor() ([]byte, []int) { + return file_evnode_v1_evnode_proto_rawDescGZIP(), []int{8} +} + +func (x *P2PSignedHeader) GetHeader() *Header { + if x != nil { + return x.Header + } + return nil +} + +func (x *P2PSignedHeader) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +func (x *P2PSignedHeader) GetSigner() *Signer { + if x != nil { + return x.Signer + } + return nil +} + +func (x *P2PSignedHeader) GetDaHeightHint() uint64 { + if x != nil && x.DaHeightHint != nil { + return *x.DaHeightHint + } + return 0 +} + +type P2PData struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Txs [][]byte `protobuf:"bytes,2,rep,name=txs,proto3" json:"txs,omitempty"` + DaHeightHint *uint64 `protobuf:"varint,3,opt,name=da_height_hint,json=daHeightHint,proto3,oneof" json:"da_height_hint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *P2PData) Reset() { + *x = P2PData{} + mi := &file_evnode_v1_evnode_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *P2PData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*P2PData) ProtoMessage() {} + +func (x *P2PData) ProtoReflect() protoreflect.Message { + mi := &file_evnode_v1_evnode_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use P2PData.ProtoReflect.Descriptor instead. +func (*P2PData) Descriptor() ([]byte, []int) { + return file_evnode_v1_evnode_proto_rawDescGZIP(), []int{9} +} + +func (x *P2PData) GetMetadata() *Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *P2PData) GetTxs() [][]byte { + if x != nil { + return x.Txs + } + return nil +} + +func (x *P2PData) GetDaHeightHint() uint64 { + if x != nil && x.DaHeightHint != nil { + return *x.DaHeightHint + } + return 0 +} + var File_evnode_v1_evnode_proto protoreflect.FileDescriptor const file_evnode_v1_evnode_proto_rawDesc = "" + @@ -630,7 +758,18 @@ const file_evnode_v1_evnode_proto_rawDesc = "" + "\x06height\x18\x02 \x01(\x04R\x06height\x128\n" + "\ttimestamp\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12\"\n" + "\rblock_id_hash\x18\x04 \x01(\fR\vblockIdHash\x12+\n" + - "\x11validator_address\x18\x05 \x01(\fR\x10validatorAddressB/Z-github.com/evstack/ev-node/types/pb/evnode/v1b\x06proto3" + "\x11validator_address\x18\x05 \x01(\fR\x10validatorAddress\"\xc3\x01\n" + + "\x0fP2PSignedHeader\x12)\n" + + "\x06header\x18\x01 \x01(\v2\x11.evnode.v1.HeaderR\x06header\x12\x1c\n" + + "\tsignature\x18\x02 \x01(\fR\tsignature\x12)\n" + + "\x06signer\x18\x03 \x01(\v2\x11.evnode.v1.SignerR\x06signer\x12)\n" + + "\x0eda_height_hint\x18\x04 \x01(\x04H\x00R\fdaHeightHint\x88\x01\x01B\x11\n" + + "\x0f_da_height_hint\"\x8a\x01\n" + + "\aP2PData\x12/\n" + + "\bmetadata\x18\x01 \x01(\v2\x13.evnode.v1.MetadataR\bmetadata\x12\x10\n" + + "\x03txs\x18\x02 \x03(\fR\x03txs\x12)\n" + + "\x0eda_height_hint\x18\x03 \x01(\x04H\x00R\fdaHeightHint\x88\x01\x01B\x11\n" + + "\x0f_da_height_hintB/Z-github.com/evstack/ev-node/types/pb/evnode/v1b\x06proto3" var ( file_evnode_v1_evnode_proto_rawDescOnce sync.Once @@ -644,7 +783,7 @@ func file_evnode_v1_evnode_proto_rawDescGZIP() []byte { return file_evnode_v1_evnode_proto_rawDescData } -var file_evnode_v1_evnode_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_evnode_v1_evnode_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_evnode_v1_evnode_proto_goTypes = []any{ (*Version)(nil), // 0: evnode.v1.Version (*Header)(nil), // 1: evnode.v1.Header @@ -654,21 +793,26 @@ var file_evnode_v1_evnode_proto_goTypes = []any{ (*Data)(nil), // 5: evnode.v1.Data (*SignedData)(nil), // 6: evnode.v1.SignedData (*Vote)(nil), // 7: evnode.v1.Vote - (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*P2PSignedHeader)(nil), // 8: evnode.v1.P2PSignedHeader + (*P2PData)(nil), // 9: evnode.v1.P2PData + (*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp } var file_evnode_v1_evnode_proto_depIdxs = []int32{ - 0, // 0: evnode.v1.Header.version:type_name -> evnode.v1.Version - 1, // 1: evnode.v1.SignedHeader.header:type_name -> evnode.v1.Header - 3, // 2: evnode.v1.SignedHeader.signer:type_name -> evnode.v1.Signer - 4, // 3: evnode.v1.Data.metadata:type_name -> evnode.v1.Metadata - 5, // 4: evnode.v1.SignedData.data:type_name -> evnode.v1.Data - 3, // 5: evnode.v1.SignedData.signer:type_name -> evnode.v1.Signer - 8, // 6: evnode.v1.Vote.timestamp:type_name -> google.protobuf.Timestamp - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 0, // 0: evnode.v1.Header.version:type_name -> evnode.v1.Version + 1, // 1: evnode.v1.SignedHeader.header:type_name -> evnode.v1.Header + 3, // 2: evnode.v1.SignedHeader.signer:type_name -> evnode.v1.Signer + 4, // 3: evnode.v1.Data.metadata:type_name -> evnode.v1.Metadata + 5, // 4: evnode.v1.SignedData.data:type_name -> evnode.v1.Data + 3, // 5: evnode.v1.SignedData.signer:type_name -> evnode.v1.Signer + 10, // 6: evnode.v1.Vote.timestamp:type_name -> google.protobuf.Timestamp + 1, // 7: evnode.v1.P2PSignedHeader.header:type_name -> evnode.v1.Header + 3, // 8: evnode.v1.P2PSignedHeader.signer:type_name -> evnode.v1.Signer + 4, // 9: evnode.v1.P2PData.metadata:type_name -> evnode.v1.Metadata + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_evnode_v1_evnode_proto_init() } @@ -676,13 +820,15 @@ func file_evnode_v1_evnode_proto_init() { if File_evnode_v1_evnode_proto != nil { return } + file_evnode_v1_evnode_proto_msgTypes[8].OneofWrappers = []any{} + file_evnode_v1_evnode_proto_msgTypes[9].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_evnode_v1_evnode_proto_rawDesc), len(file_evnode_v1_evnode_proto_rawDesc)), NumEnums: 0, - NumMessages: 8, + NumMessages: 10, NumExtensions: 0, NumServices: 0, }, From 56c278f17c60357dfae9876d7888464999ede7be Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 15 Dec 2025 12:03:47 +0100 Subject: [PATCH 08/41] Merge updates --- block/components.go | 1 - block/internal/syncing/syncer_forced_inclusion_test.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/block/components.go b/block/components.go index 54a70e6e8a..6a2f1c2d7e 100644 --- a/block/components.go +++ b/block/components.go @@ -20,7 +20,6 @@ import ( "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/signer" "github.com/evstack/ev-node/pkg/store" - "github.com/evstack/ev-node/types" ) // Components represents the block-related components diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index 2b6448819f..a3bb206354 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -1092,8 +1092,8 @@ func TestVerifyForcedInclusionTxs_SmoothingExceedsEpoch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.SignedHeader](t), - common.NewMockBroadcaster[*types.Data](t), + common.NewMockBroadcaster[*types.P2PSignedHeader](t), + common.NewMockBroadcaster[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), From a5851907cb04852031cbed5708649e1314c0d1dd Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 15 Dec 2025 15:10:15 +0100 Subject: [PATCH 09/41] Bump sonic version --- test/e2e/go.mod | 7 ++++--- test/e2e/go.sum | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 5011f11387..b217d251cd 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -51,15 +51,16 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect - github.com/bytedance/sonic v1.13.2 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.14.2 // indirect + github.com/bytedance/sonic/loader v0.4.0 // indirect github.com/celestiaorg/go-header v0.7.4 // indirect github.com/celestiaorg/go-square/merkle v0.0.0-20240627094109-7d01436067a3 // indirect github.com/celestiaorg/nmt v0.24.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/cloudwego/base64x v0.1.5 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect github.com/cockroachdb/errors v1.12.0 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index f291834660..83a2b7a74d 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -122,11 +122,17 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/ github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= +github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= +github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/celestiaorg/go-header v0.7.4 h1:kQx3bVvKV+H2etxRi4IUuby5VQydBONx3giHFXDcZ/o= github.com/celestiaorg/go-header v0.7.4/go.mod h1:eX9iTSPthVEAlEDLux40ZT/olXPGhpxHd+mEzJeDhd0= @@ -162,6 +168,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -1011,6 +1019,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= From d09c8ab342e53da7b10394a2409ac5cfb7469325 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 15 Dec 2025 15:31:36 +0100 Subject: [PATCH 10/41] Make tidy-all --- test/e2e/go.sum | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 83a2b7a74d..03603f3c36 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -124,13 +124,8 @@ github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28 github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= -github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -166,11 +161,8 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -614,10 +606,8 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= @@ -1423,7 +1413,6 @@ lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 4ecf0a003b615d2486e46bef85327076e4d5a529 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 19 Dec 2025 17:00:08 +0100 Subject: [PATCH 11/41] Use envelope for p2p store --- block/internal/executing/executor.go | 4 +- block/internal/syncing/p2p_handler.go | 4 +- block/internal/syncing/p2p_handler_test.go | 36 ++--- block/internal/syncing/syncer.go | 4 +- docs/guides/migrating-to-ev-abci.md | 6 +- pkg/rpc/client/client_test.go | 4 +- pkg/rpc/server/server_test.go | 4 +- pkg/sync/da_hint_container.go | 64 --------- pkg/sync/sync_service_test.go | 12 +- types/binary_compatibility_test.go | 14 +- types/p2p_data.go | 80 ----------- types/p2p_envelope.go | 153 +++++++++++++++++++++ types/p2p_envelope_test.go | 85 ++++++++++++ types/p2p_signed_header.go | 85 ------------ 14 files changed, 282 insertions(+), 273 deletions(-) delete mode 100644 pkg/sync/da_hint_container.go delete mode 100644 types/p2p_data.go create mode 100644 types/p2p_envelope.go create mode 100644 types/p2p_envelope_test.go delete mode 100644 types/p2p_signed_header.go diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 3424df67c1..be32c37b77 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -432,10 +432,10 @@ func (e *Executor) produceBlock() error { // broadcast header and data to P2P network g, ctx := errgroup.WithContext(e.ctx) g.Go(func() error { - return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *header}) + return e.headerBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: header}) }) g.Go(func() error { - return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: *data}) + return e.dataBroadcaster.WriteToStoreAndBroadcast(ctx, &types.P2PData{Message: data}) }) if err := g.Wait(); err != nil { e.logger.Error().Err(err).Msg("failed to broadcast header and/data") diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index 72f942432d..86fc95dee1 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -86,7 +86,7 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC } return err } - header := &p2pHeader.SignedHeader + header := p2pHeader.Message if err := h.assertExpectedProposer(header.ProposerAddress); err != nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("invalid header from P2P") return err @@ -99,7 +99,7 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC } return err } - data := &p2pData.Data + data := p2pData.Message dataCommitment := data.DACommitment() if !bytes.Equal(header.DataHash[:], dataCommitment[:]) { err := fmt.Errorf("data hash mismatch: header %x, data %x", header.DataHash, dataCommitment) diff --git a/block/internal/syncing/p2p_handler_test.go b/block/internal/syncing/p2p_handler_test.go index b0d2dbdabc..0e0604944b 100644 --- a/block/internal/syncing/p2p_handler_test.go +++ b/block/internal/syncing/p2p_handler_test.go @@ -50,7 +50,7 @@ func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer [ sig, err := signer.Sign(bz) require.NoError(t, err, "failed to sign header bytes") hdr.Signature = sig - return &types.P2PSignedHeader{SignedHeader: *hdr} + return &types.P2PSignedHeader{Message: hdr} } // P2PTestData aggregates dependencies used by P2P handler tests. @@ -128,13 +128,13 @@ func TestP2PHandler_ProcessHeight_EmitsEventWhenHeaderAndDataPresent(t *testing. require.Equal(t, string(p.Genesis.ProposerAddress), string(p.ProposerAddr)) header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 5, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 5, 1)} - header.DataHash = data.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) + data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 5, 1)} + header.Message.DataHash = data.Message.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Signature = sig + header.Message.Signature = sig p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(header, 0, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, 0, nil).Once() @@ -154,13 +154,13 @@ func TestP2PHandler_ProcessHeight_SkipsWhenDataMissing(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 7, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 7, 1)} - header.DataHash = data.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) + data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 7, 1)} + header.Message.DataHash = data.Message.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Signature = sig + header.Message.Signature = sig p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(header, 0, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, 0, errors.New("missing")).Once() @@ -195,7 +195,7 @@ func TestP2PHandler_ProcessHeight_SkipsOnProposerMismatch(t *testing.T) { require.NotEqual(t, string(p.Genesis.ProposerAddress), string(badAddr)) header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 11, badAddr, pub, signer) - header.DataHash = common.DataHashForEmptyTxs + header.Message.DataHash = common.DataHashForEmptyTxs p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(header, 0, nil).Once() @@ -224,13 +224,13 @@ func TestP2PHandler_ProcessedHeightSkipsPreviouslyHandledBlocks(t *testing.T) { // Height 6 should be fetched normally. header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 6, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 6, 1)} - header.DataHash = data.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) + data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 6, 1)} + header.Message.DataHash = data.Message.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Signature = sig + header.Message.Signature = sig p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(header, 0, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, 0, nil).Once() @@ -247,13 +247,13 @@ func TestP2PHandler_SetProcessedHeightPreventsDuplicates(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 8, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Data: *makeData(p.Genesis.ChainID, 8, 0)} - header.DataHash = data.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) + data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 8, 0)} + header.Message.DataHash = data.Message.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Signature = sig + header.Message.Signature = sig p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(header, 0, nil).Once() p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, 0, nil).Once() diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 33c151b0e0..0b419dc17a 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -586,12 +586,12 @@ func (s *Syncer) processHeightEvent(event *common.DAHeightEvent) { g.Go(func() error { // broadcast header locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - return s.headerStore.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *event.Header}, pubsub.WithLocalPublication(true)) + return s.headerStore.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: event.Header}, pubsub.WithLocalPublication(true)) }) g.Go(func() error { // broadcast data locally only — prevents spamming the p2p network with old height notifications, // allowing the syncer to update its target and fill missing blocks - return s.dataStore.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: *event.Data}, pubsub.WithLocalPublication(true)) + return s.dataStore.WriteToStoreAndBroadcast(ctx, &types.P2PData{Message: event.Data}, pubsub.WithLocalPublication(true)) }) if err := g.Wait(); err != nil { s.logger.Error().Err(err).Msg("failed to append event header and/or data to p2p store") diff --git a/docs/guides/migrating-to-ev-abci.md b/docs/guides/migrating-to-ev-abci.md index f49ba6df6f..eb6abcd9e0 100644 --- a/docs/guides/migrating-to-ev-abci.md +++ b/docs/guides/migrating-to-ev-abci.md @@ -41,9 +41,9 @@ import ( ) ``` -2. Add the migration manager keeper to your app struct -3. Register the module in your module manager -4. Configure the migration manager in your app initialization +1. Add the migration manager keeper to your app struct +2. Register the module in your module manager +3. Configure the migration manager in your app initialization ### Step 2: Replace Staking Module with Wrapper diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index fd3d1f2dc0..3841156635 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -253,7 +253,7 @@ func TestClientGetNamespace(t *testing.T) { func testSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { return &types.P2PSignedHeader{ - SignedHeader: types.SignedHeader{ + Message: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -270,7 +270,7 @@ func testSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { func testData(height uint64, ts time.Time) *types.P2PData { return &types.P2PData{ - Data: types.Data{ + Message: &types.Data{ Metadata: &types.Metadata{ ChainID: "test-chain", Height: height, diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 842adea560..42a9812479 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -629,7 +629,7 @@ func TestHealthReadyEndpoint(t *testing.T) { func makeTestSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { return &types.P2PSignedHeader{ - SignedHeader: types.SignedHeader{ + Message: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -646,7 +646,7 @@ func makeTestSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { func makeTestData(height uint64, ts time.Time) *types.P2PData { return &types.P2PData{ - Data: types.Data{ + Message: &types.Data{ Metadata: &types.Metadata{ ChainID: "test-chain", Height: height, diff --git a/pkg/sync/da_hint_container.go b/pkg/sync/da_hint_container.go deleted file mode 100644 index dc52fcb953..0000000000 --- a/pkg/sync/da_hint_container.go +++ /dev/null @@ -1,64 +0,0 @@ -package sync - -import ( - "time" - - "github.com/celestiaorg/go-header" -) - -type DAHeightHintContainer[H header.Header[H]] struct { - Entry H - DAHeightHint uint64 -} - -func (s *DAHeightHintContainer[H]) ChainID() string { - return s.Entry.ChainID() -} - -func (s *DAHeightHintContainer[H]) Hash() header.Hash { - return s.Entry.Hash() -} - -func (s *DAHeightHintContainer[H]) Height() uint64 { - return s.Entry.Height() -} - -func (s *DAHeightHintContainer[H]) LastHeader() header.Hash { - return s.Entry.LastHeader() -} - -func (s *DAHeightHintContainer[H]) Time() time.Time { - return s.Entry.Time() -} - -func (s *DAHeightHintContainer[H]) Validate() error { - return s.Entry.Validate() -} - -func (s *DAHeightHintContainer[H]) New() *DAHeightHintContainer[H] { - var empty H - return &DAHeightHintContainer[H]{Entry: empty.New()} -} - -func (sh *DAHeightHintContainer[H]) Verify(untrstH *DAHeightHintContainer[H]) error { - return sh.Entry.Verify(untrstH.Entry) -} - -func (s *DAHeightHintContainer[H]) SetDAHint(daHeight uint64) { - s.DAHeightHint = daHeight -} -func (s *DAHeightHintContainer[H]) DAHint() uint64 { - return s.DAHeightHint -} - -func (s *DAHeightHintContainer[H]) IsZero() bool { - return s == nil -} - -func (s *DAHeightHintContainer[H]) MarshalBinary() ([]byte, error) { - return s.Entry.MarshalBinary() -} - -func (s *DAHeightHintContainer[H]) UnmarshalBinary(data []byte) error { - return s.Entry.UnmarshalBinary(data) -} diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 048d19d6cd..d8d5cd57ee 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -73,12 +73,12 @@ func TestHeaderSyncServiceRestart(t *testing.T) { signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, genesisDoc.ChainID) require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) for i := genesisDoc.InitialHeight + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) } // then stop and restart service @@ -109,7 +109,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { for i := signedHeader.Height() + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) } cancel() } @@ -164,7 +164,7 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) } func TestDAHintStorageHeader(t *testing.T) { @@ -215,7 +215,7 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: *signedHeader})) + require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) daHeight := uint64(100) require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Hash())) @@ -306,7 +306,7 @@ func TestDAHintStorageData(t *testing.T) { }, } - require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: data})) + require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Message: &data})) daHeight := uint64(100) require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Hash())) diff --git a/types/binary_compatibility_test.go b/types/binary_compatibility_test.go index a8e8253d4e..86f4cf1e9e 100644 --- a/types/binary_compatibility_test.go +++ b/types/binary_compatibility_test.go @@ -14,13 +14,13 @@ func TestSignedHeaderBinaryCompatibility(t *testing.T) { bytes, err := signedHeader.MarshalBinary() require.NoError(t, err) - var p2pHeader P2PSignedHeader + p2pHeader := (&P2PSignedHeader{}).New() err = p2pHeader.UnmarshalBinary(bytes) require.NoError(t, err) - assert.Equal(t, signedHeader.Header, p2pHeader.Header) - assert.Equal(t, signedHeader.Signature, p2pHeader.Signature) - assert.Equal(t, signedHeader.Signer, p2pHeader.Signer) + assert.Equal(t, signedHeader.Header, p2pHeader.Message.Header) + assert.Equal(t, signedHeader.Signature, p2pHeader.Message.Signature) + assert.Equal(t, signedHeader.Signer, p2pHeader.Message.Signer) assert.Zero(t, p2pHeader.DAHeightHint) p2pHeader.DAHeightHint = 100 @@ -51,12 +51,12 @@ func TestDataBinaryCompatibility(t *testing.T) { bytes, err := data.MarshalBinary() require.NoError(t, err) - var p2pData P2PData + p2pData := (&P2PData{}).New() err = p2pData.UnmarshalBinary(bytes) require.NoError(t, err) - assert.Equal(t, data.Metadata, p2pData.Metadata) - assert.Equal(t, data.Txs, p2pData.Txs) + assert.Equal(t, data.Metadata, p2pData.Message.Metadata) + assert.Equal(t, data.Txs, p2pData.Message.Txs) assert.Zero(t, p2pData.DAHeightHint) p2pData.DAHeightHint = 200 diff --git a/types/p2p_data.go b/types/p2p_data.go deleted file mode 100644 index d57671a081..0000000000 --- a/types/p2p_data.go +++ /dev/null @@ -1,80 +0,0 @@ -package types - -import ( - "errors" - - "github.com/celestiaorg/go-header" - "google.golang.org/protobuf/proto" - - pb "github.com/evstack/ev-node/types/pb/evnode/v1" -) - -var _ header.Header[*P2PData] = &P2PData{} - -type P2PData struct { - Data - DAHeightHint uint64 -} - -func (d *P2PData) New() *P2PData { - return new(P2PData) -} - -func (d *P2PData) IsZero() bool { - return d == nil || d.Data.IsZero() -} - -func (d *P2PData) Verify(untrstD *P2PData) error { - return d.Data.Verify(&untrstD.Data) -} - -func (d *P2PData) SetDAHint(daHeight uint64) { - d.DAHeightHint = daHeight -} - -func (d *P2PData) DAHint() uint64 { - return d.DAHeightHint -} - -func (d *P2PData) MarshalBinary() ([]byte, error) { - msg, err := d.ToProto() - if err != nil { - return nil, err - } - return proto.Marshal(msg) -} - -func (d *P2PData) UnmarshalBinary(data []byte) error { - var pData pb.P2PData - if err := proto.Unmarshal(data, &pData); err != nil { - return err - } - return d.FromProto(&pData) -} - -func (d *P2PData) ToProto() (*pb.P2PData, error) { - pData := d.Data.ToProto() - return &pb.P2PData{ - Metadata: pData.Metadata, - Txs: pData.Txs, - DaHeightHint: &d.DAHeightHint, - }, nil -} - -func (d *P2PData) FromProto(other *pb.P2PData) error { - if other == nil { - return errors.New("P2PData is nil") - } - - pData := &pb.Data{ - Metadata: other.Metadata, - Txs: other.Txs, - } - if err := d.Data.FromProto(pData); err != nil { - return err - } - if other.DaHeightHint != nil { - d.DAHeightHint = *other.DaHeightHint - } - return nil -} diff --git a/types/p2p_envelope.go b/types/p2p_envelope.go new file mode 100644 index 0000000000..eb76e9f896 --- /dev/null +++ b/types/p2p_envelope.go @@ -0,0 +1,153 @@ +package types + +import ( + "fmt" + "time" + + "github.com/celestiaorg/go-header" + "google.golang.org/protobuf/proto" + + pb "github.com/evstack/ev-node/types/pb/evnode/v1" +) + +type ( + P2PSignedHeader = P2PEnvelope[*SignedHeader] + P2PData = P2PEnvelope[*Data] +) + +var ( + _ header.Header[*P2PData] = &P2PData{} + _ header.Header[*P2PSignedHeader] = &P2PSignedHeader{} +) + +// P2PEnvelope is a generic envelope for P2P messages that includes a DA height hint. +type P2PEnvelope[H header.Header[H]] struct { + Message H + DAHeightHint uint64 +} + +// New creates a new P2PEnvelope. +func (e *P2PEnvelope[H]) New() *P2PEnvelope[H] { + var empty H + return &P2PEnvelope[H]{Message: empty.New()} +} + +// IsZero checks if the envelope or its message is zero. +func (e *P2PEnvelope[H]) IsZero() bool { + return e == nil || e.Message.IsZero() +} + +// SetDAHint sets the DA height hint. +func (e *P2PEnvelope[H]) SetDAHint(daHeight uint64) { + e.DAHeightHint = daHeight +} + +// DAHint returns the DA height hint. +func (e *P2PEnvelope[H]) DAHint() uint64 { + return e.DAHeightHint +} + +// Verify verifies the envelope message against an untrusted envelope. +func (e *P2PEnvelope[H]) Verify(untrst *P2PEnvelope[H]) error { + return e.Message.Verify(untrst.Message) +} + +// ChainID returns the ChainID of the message. +func (e *P2PEnvelope[H]) ChainID() string { + return e.Message.ChainID() +} + +// Height returns the Height of the message. +func (e *P2PEnvelope[H]) Height() uint64 { + return e.Message.Height() +} + +// LastHeader returns the LastHeader hash of the message. +func (e *P2PEnvelope[H]) LastHeader() Hash { + return e.Message.LastHeader() +} + +// Time returns the Time of the message. +func (e *P2PEnvelope[H]) Time() time.Time { + return e.Message.Time() +} + +// Hash returns the hash of the message. +func (e *P2PEnvelope[H]) Hash() Hash { + return e.Message.Hash() +} + +// Validate performs basic validation on the message. +func (e *P2PEnvelope[H]) Validate() error { + return e.Message.Validate() +} + +// MarshalBinary marshals the envelope to binary. +func (e *P2PEnvelope[H]) MarshalBinary() ([]byte, error) { + var mirrorPb proto.Message + + switch msg := any(e.Message).(type) { + case *Data: + pData := msg.ToProto() + mirrorPb = &pb.P2PData{ + Metadata: pData.Metadata, + Txs: pData.Txs, + DaHeightHint: &e.DAHeightHint, + } + case *SignedHeader: + psh, err := msg.ToProto() + if err != nil { + return nil, err + } + mirrorPb = &pb.P2PSignedHeader{ + Header: psh.Header, + Signature: psh.Signature, + Signer: psh.Signer, + DaHeightHint: &e.DAHeightHint, + } + default: + return nil, fmt.Errorf("unsupported type for toProto: %T", msg) + } + return proto.Marshal(mirrorPb) +} + +// UnmarshalBinary unmarshals the envelope from binary. +func (e *P2PEnvelope[H]) UnmarshalBinary(data []byte) error { + switch target := any(e.Message).(type) { + case *Data: + var pData pb.P2PData + if err := proto.Unmarshal(data, &pData); err != nil { + return err + } + mirrorData := &pb.Data{ + Metadata: pData.Metadata, + Txs: pData.Txs, + } + if err := target.FromProto(mirrorData); err != nil { + return err + } + if pData.DaHeightHint != nil { + e.DAHeightHint = *pData.DaHeightHint + } + return nil + case *SignedHeader: + var pHeader pb.P2PSignedHeader + if err := proto.Unmarshal(data, &pHeader); err != nil { + return err + } + psh := &pb.SignedHeader{ + Header: pHeader.Header, + Signature: pHeader.Signature, + Signer: pHeader.Signer, + } + if err := target.FromProto(psh); err != nil { + return err + } + if pHeader.DaHeightHint != nil { + e.DAHeightHint = *pHeader.DaHeightHint + } + return nil + default: + return fmt.Errorf("unsupported type for UnmarshalBinary: %T", target) + } +} diff --git a/types/p2p_envelope_test.go b/types/p2p_envelope_test.go new file mode 100644 index 0000000000..f0949f66b8 --- /dev/null +++ b/types/p2p_envelope_test.go @@ -0,0 +1,85 @@ +package types + +import ( + "crypto/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { + // Create a P2PData envelope + data := &Data{ + Metadata: &Metadata{ + ChainID: "test-chain", + Height: 10, + Time: uint64(time.Now().UnixNano()), + }, + Txs: nil, + } + envelope := &P2PData{ + Message: data, + DAHeightHint: 100, + } + + // Marshaling + bytes, err := envelope.MarshalBinary() + require.NoError(t, err) + assert.NotEmpty(t, bytes) + + // Unmarshaling + newEnvelope := (&P2PData{}).New() + err = newEnvelope.UnmarshalBinary(bytes) + require.NoError(t, err) + assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) + assert.Equal(t, envelope.Message.Height(), newEnvelope.Message.Height()) + assert.Equal(t, envelope.Message.ChainID(), newEnvelope.Message.ChainID()) +} + +func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { + // Create a SignedHeader + // Minimal valid SignedHeader + header := &SignedHeader{ + Header: Header{ + BaseHeader: BaseHeader{ + ChainID: "test-chain", + Height: 5, + Time: uint64(time.Now().UnixNano()), + }, + Version: Version{ + Block: 1, + App: 2, + }, + DataHash: make([]byte, 32), + }, + Signature: make([]byte, 64), + Signer: Signer{ + // PubKey can be nil for basic marshal check + Address: make([]byte, 20), + }, + } + _, _ = rand.Read(header.DataHash) + _, _ = rand.Read(header.Signature) + _, _ = rand.Read(header.Signer.Address) + + envelope := &P2PSignedHeader{ + Message: header, + DAHeightHint: 200, + } + + // Marshaling + bytes, err := envelope.MarshalBinary() + require.NoError(t, err) + assert.NotEmpty(t, bytes) + + // Unmarshaling + newEnvelope := (&P2PSignedHeader{}).New() + err = newEnvelope.UnmarshalBinary(bytes) + require.NoError(t, err) + assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) + assert.Equal(t, envelope.Message.Height(), newEnvelope.Message.Height()) + assert.Equal(t, envelope.Message.ChainID(), newEnvelope.Message.ChainID()) + // Deep comparison of structs if needed +} diff --git a/types/p2p_signed_header.go b/types/p2p_signed_header.go deleted file mode 100644 index fe1ce807ba..0000000000 --- a/types/p2p_signed_header.go +++ /dev/null @@ -1,85 +0,0 @@ -package types - -import ( - "errors" - - "github.com/celestiaorg/go-header" - "google.golang.org/protobuf/proto" - - pb "github.com/evstack/ev-node/types/pb/evnode/v1" -) - -var _ header.Header[*P2PSignedHeader] = &P2PSignedHeader{} - -type P2PSignedHeader struct { - SignedHeader - DAHeightHint uint64 -} - -func (sh *P2PSignedHeader) New() *P2PSignedHeader { - return new(P2PSignedHeader) -} - -func (sh *P2PSignedHeader) IsZero() bool { - return sh == nil || sh.SignedHeader.IsZero() -} - -func (sh *P2PSignedHeader) Verify(untrstH *P2PSignedHeader) error { - return sh.SignedHeader.Verify(&untrstH.SignedHeader) -} - -func (sh *P2PSignedHeader) SetDAHint(daHeight uint64) { - sh.DAHeightHint = daHeight -} - -func (sh *P2PSignedHeader) DAHint() uint64 { - return sh.DAHeightHint -} - -func (sh *P2PSignedHeader) MarshalBinary() ([]byte, error) { - msg, err := sh.ToProto() - if err != nil { - return nil, err - } - return proto.Marshal(msg) -} - -func (sh *P2PSignedHeader) UnmarshalBinary(data []byte) error { - var pHeader pb.P2PSignedHeader - if err := proto.Unmarshal(data, &pHeader); err != nil { - return err - } - return sh.FromProto(&pHeader) -} - -func (sh *P2PSignedHeader) ToProto() (*pb.P2PSignedHeader, error) { - psh, err := sh.SignedHeader.ToProto() - if err != nil { - return nil, err - } - return &pb.P2PSignedHeader{ - Header: psh.Header, - Signature: psh.Signature, - Signer: psh.Signer, - DaHeightHint: &sh.DAHeightHint, - }, nil -} - -func (sh *P2PSignedHeader) FromProto(other *pb.P2PSignedHeader) error { - if other == nil { - return errors.New("P2PSignedHeader is nil") - } - // Reconstruct SignedHeader - psh := &pb.SignedHeader{ - Header: other.Header, - Signature: other.Signature, - Signer: other.Signer, - } - if err := sh.SignedHeader.FromProto(psh); err != nil { - return err - } - if other.DaHeightHint != nil { - sh.DAHeightHint = *other.DaHeightHint - } - return nil -} From 7abfecc9be722697d380573ae878c29c106264cc Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 19 Dec 2025 17:30:09 +0100 Subject: [PATCH 12/41] Minor cleanup --- types/binary_compatibility_test.go | 72 -------------------- types/p2p_envelope_test.go | 103 +++++++++++++++++++++++------ 2 files changed, 81 insertions(+), 94 deletions(-) delete mode 100644 types/binary_compatibility_test.go diff --git a/types/binary_compatibility_test.go b/types/binary_compatibility_test.go deleted file mode 100644 index 86f4cf1e9e..0000000000 --- a/types/binary_compatibility_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package types - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSignedHeaderBinaryCompatibility(t *testing.T) { - signedHeader, _, err := GetRandomSignedHeader("chain-id") - require.NoError(t, err) - bytes, err := signedHeader.MarshalBinary() - require.NoError(t, err) - - p2pHeader := (&P2PSignedHeader{}).New() - err = p2pHeader.UnmarshalBinary(bytes) - require.NoError(t, err) - - assert.Equal(t, signedHeader.Header, p2pHeader.Message.Header) - assert.Equal(t, signedHeader.Signature, p2pHeader.Message.Signature) - assert.Equal(t, signedHeader.Signer, p2pHeader.Message.Signer) - assert.Zero(t, p2pHeader.DAHeightHint) - - p2pHeader.DAHeightHint = 100 - p2pBytes, err := p2pHeader.MarshalBinary() - require.NoError(t, err) - - var decodedSignedHeader SignedHeader - err = decodedSignedHeader.UnmarshalBinary(p2pBytes) - require.NoError(t, err) - assert.Equal(t, signedHeader.Header, decodedSignedHeader.Header) - assert.Equal(t, signedHeader.Signature, decodedSignedHeader.Signature) - assert.Equal(t, signedHeader.Signer, decodedSignedHeader.Signer) -} - -func TestDataBinaryCompatibility(t *testing.T) { - data := &Data{ - Metadata: &Metadata{ - ChainID: "chain-id", - Height: 10, - Time: uint64(time.Now().UnixNano()), - LastDataHash: []byte("last-hash"), - }, - Txs: Txs{ - []byte("tx1"), - []byte("tx2"), - }, - } - bytes, err := data.MarshalBinary() - require.NoError(t, err) - - p2pData := (&P2PData{}).New() - err = p2pData.UnmarshalBinary(bytes) - require.NoError(t, err) - - assert.Equal(t, data.Metadata, p2pData.Message.Metadata) - assert.Equal(t, data.Txs, p2pData.Message.Txs) - assert.Zero(t, p2pData.DAHeightHint) - - p2pData.DAHeightHint = 200 - - p2pBytes, err := p2pData.MarshalBinary() - require.NoError(t, err) - - var decodedData Data - err = decodedData.UnmarshalBinary(p2pBytes) - require.NoError(t, err) - assert.Equal(t, data.Metadata, decodedData.Metadata) - assert.Equal(t, data.Txs, decodedData.Txs) -} diff --git a/types/p2p_envelope_test.go b/types/p2p_envelope_test.go index f0949f66b8..9433d5a1aa 100644 --- a/types/p2p_envelope_test.go +++ b/types/p2p_envelope_test.go @@ -1,7 +1,7 @@ package types import ( - "crypto/rand" + "bytes" "testing" "time" @@ -13,11 +13,12 @@ func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { // Create a P2PData envelope data := &Data{ Metadata: &Metadata{ - ChainID: "test-chain", - Height: 10, - Time: uint64(time.Now().UnixNano()), + ChainID: "test-chain", + Height: 10, + Time: uint64(time.Now().UnixNano()), + LastDataHash: bytes.Repeat([]byte{0x1}, 32), }, - Txs: nil, + Txs: Txs{[]byte{0x1}, []byte{0x2}}, } envelope := &P2PData{ Message: data, @@ -36,11 +37,11 @@ func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) assert.Equal(t, envelope.Message.Height(), newEnvelope.Message.Height()) assert.Equal(t, envelope.Message.ChainID(), newEnvelope.Message.ChainID()) + assert.Equal(t, envelope.Message.LastDataHash, newEnvelope.Message.LastDataHash) + assert.Equal(t, envelope.Message.Txs, newEnvelope.Message.Txs) } func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { - // Create a SignedHeader - // Minimal valid SignedHeader header := &SignedHeader{ Header: Header{ BaseHeader: BaseHeader{ @@ -52,17 +53,14 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { Block: 1, App: 2, }, - DataHash: make([]byte, 32), - }, - Signature: make([]byte, 64), - Signer: Signer{ - // PubKey can be nil for basic marshal check - Address: make([]byte, 20), + LastHeaderHash: GetRandomBytes(32), + DataHash: GetRandomBytes(32), + AppHash: GetRandomBytes(32), + ProposerAddress: GetRandomBytes(32), + ValidatorHash: GetRandomBytes(32), }, + // Signature and Signer are transient } - _, _ = rand.Read(header.DataHash) - _, _ = rand.Read(header.Signature) - _, _ = rand.Read(header.Signer.Address) envelope := &P2PSignedHeader{ Message: header, @@ -70,16 +68,77 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { } // Marshaling - bytes, err := envelope.MarshalBinary() + bz, err := envelope.MarshalBinary() require.NoError(t, err) - assert.NotEmpty(t, bytes) + assert.NotEmpty(t, bz) // Unmarshaling newEnvelope := (&P2PSignedHeader{}).New() - err = newEnvelope.UnmarshalBinary(bytes) + err = newEnvelope.UnmarshalBinary(bz) require.NoError(t, err) assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) - assert.Equal(t, envelope.Message.Height(), newEnvelope.Message.Height()) - assert.Equal(t, envelope.Message.ChainID(), newEnvelope.Message.ChainID()) - // Deep comparison of structs if needed + assert.Equal(t, envelope, newEnvelope) +} + +func TestSignedHeaderBinaryCompatibility(t *testing.T) { + signedHeader, _, err := GetRandomSignedHeader("chain-id") + require.NoError(t, err) + bytes, err := signedHeader.MarshalBinary() + require.NoError(t, err) + + p2pHeader := (&P2PSignedHeader{}).New() + err = p2pHeader.UnmarshalBinary(bytes) + require.NoError(t, err) + + assert.Equal(t, signedHeader.Header, p2pHeader.Message.Header) + assert.Equal(t, signedHeader.Signature, p2pHeader.Message.Signature) + assert.Equal(t, signedHeader.Signer, p2pHeader.Message.Signer) + assert.Zero(t, p2pHeader.DAHeightHint) + + p2pHeader.DAHeightHint = 100 + p2pBytes, err := p2pHeader.MarshalBinary() + require.NoError(t, err) + + var decodedSignedHeader SignedHeader + err = decodedSignedHeader.UnmarshalBinary(p2pBytes) + require.NoError(t, err) + assert.Equal(t, signedHeader.Header, decodedSignedHeader.Header) + assert.Equal(t, signedHeader.Signature, decodedSignedHeader.Signature) + assert.Equal(t, signedHeader.Signer, decodedSignedHeader.Signer) +} + +func TestDataBinaryCompatibility(t *testing.T) { + data := &Data{ + Metadata: &Metadata{ + ChainID: "chain-id", + Height: 10, + Time: uint64(time.Now().UnixNano()), + LastDataHash: []byte("last-hash"), + }, + Txs: Txs{ + []byte("tx1"), + []byte("tx2"), + }, + } + bytes, err := data.MarshalBinary() + require.NoError(t, err) + + p2pData := (&P2PData{}).New() + err = p2pData.UnmarshalBinary(bytes) + require.NoError(t, err) + + assert.Equal(t, data.Metadata, p2pData.Message.Metadata) + assert.Equal(t, data.Txs, p2pData.Message.Txs) + assert.Zero(t, p2pData.DAHeightHint) + + p2pData.DAHeightHint = 200 + + p2pBytes, err := p2pData.MarshalBinary() + require.NoError(t, err) + + var decodedData Data + err = decodedData.UnmarshalBinary(p2pBytes) + require.NoError(t, err) + assert.Equal(t, data.Metadata, decodedData.Metadata) + assert.Equal(t, data.Txs, decodedData.Txs) } From e59384851eddee74c86fc50ea508d280a9eeb62d Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Fri, 19 Dec 2025 17:46:37 +0100 Subject: [PATCH 13/41] Better test data (cherry picked from commit ad3e21b79e8e57e3f17c9e4d16868bff3c18b9f8) --- types/p2p_envelope_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/types/p2p_envelope_test.go b/types/p2p_envelope_test.go index 9433d5a1aa..3dc2127fed 100644 --- a/types/p2p_envelope_test.go +++ b/types/p2p_envelope_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -42,6 +43,9 @@ func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { } func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { + _, pubKey, err := crypto.GenerateEd25519Key(nil) + require.NoError(t, err) + header := &SignedHeader{ Header: Header{ BaseHeader: BaseHeader{ @@ -59,7 +63,11 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { ProposerAddress: GetRandomBytes(32), ValidatorHash: GetRandomBytes(32), }, - // Signature and Signer are transient + Signature: GetRandomBytes(64), + Signer: Signer{ + PubKey: pubKey, + Address: GetRandomBytes(20), + }, } envelope := &P2PSignedHeader{ @@ -77,6 +85,7 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { err = newEnvelope.UnmarshalBinary(bz) require.NoError(t, err) assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) + assert.Equal(t, envelope.Message.Signer, newEnvelope.Message.Signer) assert.Equal(t, envelope, newEnvelope) } From 544c0b9aa4383c87a0dcb5f9cf5f0d0c5531ff88 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Thu, 8 Jan 2026 10:55:53 +0100 Subject: [PATCH 14/41] Linter --- block/internal/submitting/submitter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/internal/submitting/submitter_test.go b/block/internal/submitting/submitter_test.go index f7e4436ef3..3e18213851 100644 --- a/block/internal/submitting/submitter_test.go +++ b/block/internal/submitting/submitter_test.go @@ -251,7 +251,7 @@ func TestSubmitter_processDAInclusionLoop_advances(t *testing.T) { daClient.On("GetDataNamespace").Return([]byte(cfg.DA.DataNamespace)).Maybe() daClient.On("GetForcedInclusionNamespace").Return([]byte(nil)).Maybe() daClient.On("HasForcedInclusionNamespace").Return(false).Maybe() - daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil ,nil) + daSub := NewDASubmitter(daClient, cfg, genesis.Genesis{}, common.BlockOptions{}, metrics, zerolog.Nop(), nil, nil) s := NewSubmitter(st, exec, cm, metrics, cfg, genesis.Genesis{}, daSub, nil, nil, zerolog.Nop(), nil) // prepare two consecutive blocks in store with DA included in cache From 570ac068d201a0496f67fcb029b8e6077abac063 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 19 Jan 2026 09:55:38 +0100 Subject: [PATCH 15/41] Resolve merge conflicts --- block/internal/syncing/syncer.go | 2 +- block/internal/syncing/syncer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 375de64b31..4dddc1c874 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -207,7 +207,7 @@ func (s *Syncer) Start(ctx context.Context) error { s.asyncDARetriever = NewAsyncDARetriever(s.daRetriever, s.heightInCh, s.logger) s.asyncDARetriever.Start(s.ctx) s.fiRetriever = da.NewForcedInclusionRetriever(s.daClient, s.logger, s.config, s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) - s.p2pHandler = NewP2PHandler(s.headerStore.Store(), s.dataStore.Store(), s.cache, s.genesis, s.logger) + s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) if currentHeight, err := s.store.Height(s.ctx); err != nil { s.logger.Error().Err(err).Msg("failed to set initial processed height for p2p handler") } else { diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 4b1b6476b5..7c213375d1 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -769,7 +769,7 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { require.NoError(t, batch.SetHeight(1)) require.NoError(t, batch.Commit()) - s.processHeightEvent(&evt) + s.processHeightEvent(t.Context(), &evt) // Verify that the request was queued in the async retriever select { From 4907c925d8742d1d02b3a67064aab2f07085a79c Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 19 Jan 2026 10:47:54 +0100 Subject: [PATCH 16/41] Tidy all --- test/e2e/go.mod | 1 + test/e2e/go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 5a8da6b705..c8dbf09715 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -54,6 +54,7 @@ require ( github.com/bytedance/sonic v1.14.2 // indirect github.com/bytedance/sonic/loader v0.4.0 // indirect github.com/celestiaorg/go-header v0.7.4 // indirect + github.com/celestiaorg/go-libp2p-messenger v0.2.2 // indirect github.com/celestiaorg/go-square/merkle v0.0.0-20240627094109-7d01436067a3 // indirect github.com/celestiaorg/nmt v0.24.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 916a4ed2a0..53c7a7227f 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -129,6 +129,8 @@ github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCc github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/celestiaorg/go-header v0.7.4 h1:kQx3bVvKV+H2etxRi4IUuby5VQydBONx3giHFXDcZ/o= github.com/celestiaorg/go-header v0.7.4/go.mod h1:eX9iTSPthVEAlEDLux40ZT/olXPGhpxHd+mEzJeDhd0= +github.com/celestiaorg/go-libp2p-messenger v0.2.2 h1:osoUfqjss7vWTIZrrDSy953RjQz+ps/vBFE7bychLEc= +github.com/celestiaorg/go-libp2p-messenger v0.2.2/go.mod h1:oTCRV5TfdO7V/k6nkx7QjQzGrWuJbupv+0o1cgnY2i4= github.com/celestiaorg/go-square/merkle v0.0.0-20240627094109-7d01436067a3 h1:wP84mtwOCVNOTfS3zErICjxKLnh74Z1uf+tdrlSFjVM= github.com/celestiaorg/go-square/merkle v0.0.0-20240627094109-7d01436067a3/go.mod h1:86qIYnEhmn/hfW+xvw98NOI3zGaDEB3x8JGjYo2FqLs= github.com/celestiaorg/go-square/v3 v3.0.2 h1:eSQOgNII8inK9IhiBZ+6GADQeWbRq4HYY72BOgcduA4= From 66b6db8f8e2f98668014d16615b1e1f07467ce45 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Wed, 21 Jan 2026 12:11:00 +0100 Subject: [PATCH 17/41] Integrate changes --- pkg/store/store.go | 2 +- pkg/store/types.go | 6 + pkg/sync/sync_service.go | 102 +++++++++--- pkg/sync/sync_service_test.go | 286 ++++++++++++++++++++++++++++++++-- 4 files changed, 365 insertions(+), 31 deletions(-) diff --git a/pkg/store/store.go b/pkg/store/store.go index 972b94e0e4..b76678c37d 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -152,7 +152,7 @@ func (s *DefaultStore) GetStateAtHeight(ctx context.Context, height uint64) (typ blob, err := s.db.Get(ctx, ds.NewKey(getStateAtHeightKey(height))) if err != nil { if errors.Is(err, ds.ErrNotFound) { - return types.State{}, fmt.Errorf("no state found at height %d", height) + return types.State{}, fmt.Errorf("get state at height %d: %w", height, ErrNotFound) } return types.State{}, fmt.Errorf("failed to retrieve state at height %d: %w", height, err) } diff --git a/pkg/store/types.go b/pkg/store/types.go index bf1cb6ced8..8a01f7c467 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -2,12 +2,18 @@ package store import ( "context" + "errors" ds "github.com/ipfs/go-datastore" "github.com/evstack/ev-node/types" ) +var ( + // ErrNotFound is returned when the entry is not found in the store. + ErrNotFound = errors.New("not found") +) + // Batch provides atomic operations for the store type Batch interface { // SaveBlockData atomically saves the block header, data, and signature diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 92aab279b4..8744f7d50e 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -84,23 +84,53 @@ func NewDataSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*DataSyncService, error) { - var getter GetterFunc[*types.Data] - var getterByHeight GetterByHeightFunc[*types.Data] - var rangeGetter RangeGetterFunc[*types.Data] + var getter GetterFunc[*types.P2PData] + var getterByHeight GetterByHeightFunc[*types.P2PData] + var rangeGetter RangeGetterFunc[*types.P2PData] if daStore != nil { - getter = func(ctx context.Context, hash header.Hash) (*types.Data, error) { + getter = func(ctx context.Context, hash header.Hash) (*types.P2PData, error) { _, d, err := daStore.GetBlockByHash(ctx, hash) - return d, err + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, d.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PData{Message: d, DAHeightHint: 0}, nil + } + return &types.P2PData{Message: d, DAHeightHint: state.DAHeight}, nil } - getterByHeight = func(ctx context.Context, height uint64) (*types.Data, error) { + getterByHeight = func(ctx context.Context, height uint64) (*types.P2PData, error) { _, d, err := daStore.GetBlockData(ctx, height) - return d, err + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, d.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PData{Message: d, DAHeightHint: 0}, nil + } + return &types.P2PData{Message: d, DAHeightHint: state.DAHeight}, nil } - rangeGetter = func(ctx context.Context, from, to uint64) ([]*types.Data, uint64, error) { - return getContiguousRange(ctx, from, to, func(ctx context.Context, h uint64) (*types.Data, error) { + rangeGetter = func(ctx context.Context, from, to uint64) ([]*types.P2PData, uint64, error) { + return getContiguousRange(ctx, from, to, func(ctx context.Context, h uint64) (*types.P2PData, error) { _, d, err := daStore.GetBlockData(ctx, h) - return d, err + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, d.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PData{Message: d, DAHeightHint: 0}, nil + } + return &types.P2PData{Message: d, DAHeightHint: state.DAHeight}, nil }) } } @@ -116,20 +146,54 @@ func NewHeaderSyncService( p2p *p2p.Client, logger zerolog.Logger, ) (*HeaderSyncService, error) { - var getter GetterFunc[*types.SignedHeader] - var getterByHeight GetterByHeightFunc[*types.SignedHeader] - var rangeGetter RangeGetterFunc[*types.SignedHeader] + var getter GetterFunc[*types.P2PSignedHeader] + var getterByHeight GetterByHeightFunc[*types.P2PSignedHeader] + var rangeGetter RangeGetterFunc[*types.P2PSignedHeader] if daStore != nil { - getter = func(ctx context.Context, hash header.Hash) (*types.SignedHeader, error) { + getter = func(ctx context.Context, hash header.Hash) (*types.P2PSignedHeader, error) { h, _, err := daStore.GetBlockByHash(ctx, hash) - return h, err + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, h.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PSignedHeader{Message: h, DAHeightHint: 0}, nil + } + return &types.P2PSignedHeader{Message: h, DAHeightHint: state.DAHeight}, nil } - getterByHeight = func(ctx context.Context, height uint64) (*types.SignedHeader, error) { - return daStore.GetHeader(ctx, height) + getterByHeight = func(ctx context.Context, height uint64) (*types.P2PSignedHeader, error) { + h, err := daStore.GetHeader(ctx, height) + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, h.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PSignedHeader{Message: h, DAHeightHint: 0}, nil + } + return &types.P2PSignedHeader{Message: h, DAHeightHint: state.DAHeight}, nil } - rangeGetter = func(ctx context.Context, from, to uint64) ([]*types.SignedHeader, uint64, error) { - return getContiguousRange(ctx, from, to, daStore.GetHeader) + rangeGetter = func(ctx context.Context, from, to uint64) ([]*types.P2PSignedHeader, uint64, error) { + return getContiguousRange(ctx, from, to, func(ctx context.Context, h uint64) (*types.P2PSignedHeader, error) { + sh, err := daStore.GetHeader(ctx, h) + if err != nil { + return nil, err + } + state, err := daStore.GetStateAtHeight(ctx, sh.Height()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + return &types.P2PSignedHeader{Message: sh, DAHeightHint: 0}, nil + } + return &types.P2PSignedHeader{Message: sh, DAHeightHint: state.DAHeight}, nil + }) } } return newSyncService[*types.P2PSignedHeader](dsStore, getter, getterByHeight, rangeGetter, headerSync, conf, genesis, p2p, logger) diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 86c76a9957..91f864aef7 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -14,17 +14,283 @@ import ( "github.com/evstack/ev-node/pkg/p2p/key" "github.com/evstack/ev-node/pkg/signer" "github.com/evstack/ev-node/pkg/signer/noop" + "github.com/evstack/ev-node/pkg/store" "github.com/evstack/ev-node/types" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/crypto" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - - "github.com/evstack/ev-node/pkg/store" ) +type mockStore struct { + mock.Mock +} + +func (m *mockStore) Height(ctx context.Context) (uint64, error) { + args := m.Called(ctx) + return args.Get(0).(uint64), args.Error(1) +} + +func (m *mockStore) GetBlockData(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) { + args := m.Called(ctx, height) + return args.Get(0).(*types.SignedHeader), args.Get(1).(*types.Data), args.Error(2) +} + +func (m *mockStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { + args := m.Called(ctx, hash) + return args.Get(0).(*types.SignedHeader), args.Get(1).(*types.Data), args.Error(2) +} + +func (m *mockStore) GetHeader(ctx context.Context, height uint64) (*types.SignedHeader, error) { + args := m.Called(ctx, height) + return args.Get(0).(*types.SignedHeader), args.Error(1) +} + +func (m *mockStore) GetStateAtHeight(ctx context.Context, height uint64) (types.State, error) { + args := m.Called(ctx, height) + return args.Get(0).(types.State), args.Error(1) +} + +func (m *mockStore) GetSignature(ctx context.Context, height uint64) (*types.Signature, error) { + args := m.Called(ctx, height) + return args.Get(0).(*types.Signature), args.Error(1) +} + +func (m *mockStore) GetSignatureByHash(ctx context.Context, hash []byte) (*types.Signature, error) { + args := m.Called(ctx, hash) + return args.Get(0).(*types.Signature), args.Error(1) +} + +func (m *mockStore) GetState(ctx context.Context) (types.State, error) { + args := m.Called(ctx) + return args.Get(0).(types.State), args.Error(1) +} + +func (m *mockStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { + args := m.Called(ctx, key) + return args.Get(0).([]byte), args.Error(1) +} + +func (m *mockStore) SetMetadata(ctx context.Context, key string, value []byte) error { + args := m.Called(ctx, key, value) + return args.Error(0) +} + +func (m *mockStore) Close() error { + args := m.Called() + return args.Error(0) +} + +func (m *mockStore) NewBatch(ctx context.Context) (store.Batch, error) { + args := m.Called(ctx) + return args.Get(0).(store.Batch), args.Error(1) +} + +func (m *mockStore) Rollback(ctx context.Context, height uint64, aggregator bool) error { + args := m.Called(ctx, height, aggregator) + return args.Error(0) +} + +func TestDAHintFromDAStore(t *testing.T) { + mainKV := sync.MutexWrap(datastore.NewMapDatastore()) + daStore := new(mockStore) + + pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) + require.NoError(t, err) + noopSigner, err := noop.NewNoopSigner(pk) + require.NoError(t, err) + rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only + mn := mocknet.New() + + chainId := "test-chain-id" + genesisDoc := genesispkg.Genesis{ + ChainID: chainId, + StartTime: time.Now(), + InitialHeight: 1, + ProposerAddress: []byte("test"), + } + conf := config.DefaultConfig() + conf.RootDir = t.TempDir() + nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) + require.NoError(t, err) + logger := zerolog.Nop() + + p2pHost, err := mn.AddPeer(nodeKey.PrivKey, nil) + require.NoError(t, err) + p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + + // Prepare data + height := uint64(1) + daHeight := uint64(100) + headerConfig := types.HeaderConfig{ + Height: height, + DataHash: bytesN(rnd, 32), + AppHash: bytesN(rnd, 32), + Signer: noopSigner, + } + signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, chainId) + require.NoError(t, err) + data := &types.Data{ + Metadata: &types.Metadata{ + ChainID: chainId, + Height: height, + }, + Txs: types.Txs{[]byte("tx1")}, + } + headerHash := signedHeader.Hash() + dataHash := data.Hash() + + state := types.State{ + ChainID: chainId, + LastBlockHeight: height, + DAHeight: daHeight, + } + + daStore.On("GetHeader", mock.Anything, height).Return(signedHeader, nil) + daStore.On("GetBlockByHash", mock.Anything, []byte(headerHash)).Return(signedHeader, data, nil) + daStore.On("GetBlockByHash", mock.Anything, []byte(dataHash)).Return(signedHeader, data, nil) + daStore.On("GetBlockData", mock.Anything, height).Return(signedHeader, data, nil) + daStore.On("GetStateAtHeight", mock.Anything, height).Return(state, nil) + + t.Run("header sync service", func(t *testing.T) { + headerSvc, err := NewHeaderSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + + h, err := headerSvc.getterByHeight(ctx, height) + require.NoError(t, err) + assert.Equal(t, daHeight, h.DAHint()) + assert.Equal(t, headerHash, h.Hash()) + + h2, err := headerSvc.getter(ctx, headerHash) + require.NoError(t, err) + assert.Equal(t, daHeight, h2.DAHint()) + + hRange, next, err := headerSvc.rangeGetter(ctx, height, height+1) + require.NoError(t, err) + assert.Equal(t, height+1, next) + require.Len(t, hRange, 1) + assert.Equal(t, daHeight, hRange[0].DAHint()) + }) + + t.Run("data sync service", func(t *testing.T) { + dataSvc, err := NewDataSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + + d, err := dataSvc.getterByHeight(ctx, height) + require.NoError(t, err) + assert.Equal(t, daHeight, d.DAHint()) + assert.Equal(t, dataHash, d.Hash()) + + d2, err := dataSvc.getter(ctx, dataHash) + require.NoError(t, err) + assert.Equal(t, daHeight, d2.DAHint()) + + dRange, next, err := dataSvc.rangeGetter(ctx, height, height+1) + require.NoError(t, err) + assert.Equal(t, height+1, next) + require.Len(t, dRange, 1) + assert.Equal(t, daHeight, dRange[0].DAHint()) + }) +} + +func TestDAHintFromDAStoreResilience(t *testing.T) { + mainKV := sync.MutexWrap(datastore.NewMapDatastore()) + daStore := new(mockStore) + + pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) + require.NoError(t, err) + noopSigner, err := noop.NewNoopSigner(pk) + require.NoError(t, err) + rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only + mn := mocknet.New() + + chainId := "test-chain-id" + genesisDoc := genesispkg.Genesis{ + ChainID: chainId, + StartTime: time.Now(), + InitialHeight: 1, + ProposerAddress: []byte("test"), + } + conf := config.DefaultConfig() + conf.RootDir = t.TempDir() + nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) + require.NoError(t, err) + logger := zerolog.Nop() + + p2pHost, err := mn.AddPeer(nodeKey.PrivKey, nil) + require.NoError(t, err) + p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + require.NoError(t, p2pClient.Start(ctx)) + + // Prepare data + height := uint64(1) + headerConfig := types.HeaderConfig{ + Height: height, + DataHash: bytesN(rnd, 32), + AppHash: bytesN(rnd, 32), + Signer: noopSigner, + } + signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, chainId) + require.NoError(t, err) + data := &types.Data{ + Metadata: &types.Metadata{ + ChainID: chainId, + Height: height, + }, + Txs: types.Txs{[]byte("tx1")}, + } + headerHash := signedHeader.Hash() + dataHash := data.Hash() + + daStore.On("GetHeader", mock.Anything, height).Return(signedHeader, nil) + daStore.On("GetBlockByHash", mock.Anything, []byte(headerHash)).Return(signedHeader, data, nil) + daStore.On("GetBlockByHash", mock.Anything, []byte(dataHash)).Return(signedHeader, data, nil) + daStore.On("GetBlockData", mock.Anything, height).Return(signedHeader, data, nil) + // Return "not found" error for state + daStore.On("GetStateAtHeight", mock.Anything, height).Return(types.State{}, store.ErrNotFound) + + t.Run("header sync service resilience", func(t *testing.T) { + headerSvc, err := NewHeaderSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + + h, err := headerSvc.getterByHeight(ctx, height) + require.NoError(t, err) + assert.Equal(t, uint64(0), h.DAHint()) + assert.Equal(t, headerHash, h.Hash()) + + h2, err := headerSvc.getter(ctx, headerHash) + require.NoError(t, err) + assert.Equal(t, uint64(0), h2.DAHint()) + }) + + t.Run("data sync service resilience", func(t *testing.T) { + dataSvc, err := NewDataSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) + require.NoError(t, err) + + d, err := dataSvc.getterByHeight(ctx, height) + require.NoError(t, err) + assert.Equal(t, uint64(0), d.DAHint()) + assert.Equal(t, dataHash, d.Hash()) + + d2, err := dataSvc.getter(ctx, dataHash) + require.NoError(t, err) + assert.Equal(t, uint64(0), d2.DAHint()) + }) +} + func TestHeaderSyncServiceRestart(t *testing.T) { mainKV := sync.MutexWrap(datastore.NewMapDatastore()) pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) @@ -60,8 +326,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - rktStore := store.New(mainKV) - svc, err := NewHeaderSyncService(mainKV, rktStore, conf, genesisDoc, p2pClient, logger) + svc, err := NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) err = svc.Start(ctx) require.NoError(t, err) @@ -101,7 +366,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = p2pClient.Close() }) - svc, err = NewHeaderSyncService(mainKV, rktStore, conf, genesisDoc, p2pClient, logger) + svc, err = NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) err = svc.Start(ctx) require.NoError(t, err) @@ -152,8 +417,7 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - rktStore := store.New(mainKV) - svc, err := NewHeaderSyncService(mainKV, rktStore, conf, genesisDoc, p2pClient, logger) + svc, err := NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, svc.Start(ctx)) t.Cleanup(func() { _ = svc.Stop(context.Background()) }) @@ -205,7 +469,7 @@ func TestDAHintStorageHeader(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - headerSvc, err := NewHeaderSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + headerSvc, err := NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, headerSvc.Start(ctx)) @@ -244,7 +508,7 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - headerSvc, err = NewHeaderSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + headerSvc, err = NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, headerSvc.Start(ctx)) t.Cleanup(func() { _ = headerSvc.Stop(context.Background()) }) @@ -289,7 +553,7 @@ func TestDAHintStorageData(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - dataSvc, err := NewDataSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + dataSvc, err := NewDataSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, dataSvc.Start(ctx)) @@ -335,7 +599,7 @@ func TestDAHintStorageData(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - dataSvc, err = NewDataSyncService(mainKV, conf, genesisDoc, p2pClient, logger) + dataSvc, err = NewDataSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, dataSvc.Start(ctx)) t.Cleanup(func() { _ = dataSvc.Stop(context.Background()) }) From 9c9510121fd5f9ba3c1bd95d59723de88d817463 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 15:01:57 +0100 Subject: [PATCH 18/41] updates --- pkg/store/store_adapter.go | 53 +++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 79571c20f7..fbfcf11ede 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -527,14 +527,29 @@ func NewHeaderStoreGetter(store Store) *HeaderStoreGetter { } // GetByHeight implements StoreGetter. -func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*types.SignedHeader, error) { - return g.store.GetHeader(ctx, height) +func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*types.P2PSignedHeader, error) { + header, err := g.store.GetHeader(ctx, height) + if err != nil { + return nil, err + } + + return &types.P2PSignedHeader{ + Message: header, + DAHeightHint: 0, // TODO: fetch DA hint. + }, nil } // GetByHash implements StoreGetter. -func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.SignedHeader, error) { +func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2PSignedHeader, error) { hdr, _, err := g.store.GetBlockByHash(ctx, hash) - return hdr, err + if err != nil { + return nil, err + } + + return &types.P2PSignedHeader{ + Message: hdr, + DAHeightHint: 0, // TODO: fetch DA hint. + }, nil } // Height implements StoreGetter. @@ -559,15 +574,29 @@ func NewDataStoreGetter(store Store) *DataStoreGetter { } // GetByHeight implements StoreGetter. -func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*types.Data, error) { +func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*types.P2PData, error) { _, data, err := g.store.GetBlockData(ctx, height) - return data, err + if err != nil { + return nil, err + } + + return &types.P2PData{ + Message: data, + DAHeightHint: 0, // TODO: fetch DA hint. + }, nil } // GetByHash implements StoreGetter. -func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.Data, error) { +func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2PData, error) { _, data, err := g.store.GetBlockByHash(ctx, hash) - return data, err + if err != nil { + return nil, err + } + + return &types.P2PData{ + Message: data, + DAHeightHint: 0, // TODO: fetch DA hint. + }, nil } // Height implements StoreGetter. @@ -582,17 +611,17 @@ func (g *DataStoreGetter) HasAt(ctx context.Context, height uint64) bool { } // Type aliases for convenience -type HeaderStoreAdapter = StoreAdapter[*types.SignedHeader] -type DataStoreAdapter = StoreAdapter[*types.Data] +type HeaderStoreAdapter = StoreAdapter[*types.P2PSignedHeader] +type DataStoreAdapter = StoreAdapter[*types.P2PData] // NewHeaderStoreAdapter creates a new StoreAdapter for headers. // The genesis is used to determine the initial height for efficient Tail lookups. func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter { - return NewStoreAdapter[*types.SignedHeader](NewHeaderStoreGetter(store), gen) + return NewStoreAdapter(NewHeaderStoreGetter(store), gen) } // NewDataStoreAdapter creates a new StoreAdapter for data. // The genesis is used to determine the initial height for efficient Tail lookups. func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { - return NewStoreAdapter[*types.Data](NewDataStoreGetter(store), gen) + return NewStoreAdapter(NewDataStoreGetter(store), gen) } From 19835e8540c268dbb14291a02d16c6a6525c7fdb Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 15:16:41 +0100 Subject: [PATCH 19/41] build fixes --- pkg/store/data_store_adapter_test.go | 68 +++--- pkg/store/header_store_adapter_test.go | 130 ++++++++--- pkg/sync/sync_service_test.go | 287 +------------------------ 3 files changed, 157 insertions(+), 328 deletions(-) diff --git a/pkg/store/data_store_adapter_test.go b/pkg/store/data_store_adapter_test.go index a43e7838e9..ec260d2fbd 100644 --- a/pkg/store/data_store_adapter_test.go +++ b/pkg/store/data_store_adapter_test.go @@ -33,6 +33,17 @@ func computeDataIndexHash(h *types.SignedHeader) []byte { return hash[:] } +// wrapData wraps a *types.Data in a *types.P2PData for use with the DataStoreAdapter. +func wrapData(d *types.Data) *types.P2PData { + if d == nil { + return nil + } + return &types.P2PData{ + Message: d, + DAHeightHint: 0, + } +} + func TestDataStoreAdapter_NewDataStoreAdapter(t *testing.T) { t.Parallel() ctx := context.Background() @@ -66,7 +77,7 @@ func TestDataStoreAdapter_AppendAndRetrieve(t *testing.T) { _, d2 := types.GetRandomBlock(2, 2, "test-chain") // Append data - these go to pending cache - err = adapter.Append(ctx, d1, d2) + err = adapter.Append(ctx, wrapData(d1), wrapData(d2)) require.NoError(t, err) // Check height is updated (from pending) @@ -103,12 +114,10 @@ func TestDataStoreAdapter_GetFromStore(t *testing.T) { require.NoError(t, batch.SetHeight(1)) require.NoError(t, batch.Commit()) - // Create adapter after data is in store + // Now create adapter and verify we can get from store adapter := NewDataStoreAdapter(store, testGenesisData()) - // Get by hash - need to use the index hash (sha256 of marshaled SignedHeader) - hash := computeDataIndexHash(h1) - retrieved, err := adapter.Get(ctx, hash) + retrieved, err := adapter.GetByHeight(ctx, 1) require.NoError(t, err) assert.Equal(t, d1.Height(), retrieved.Height()) @@ -135,12 +144,14 @@ func TestDataStoreAdapter_Has(t *testing.T) { // Create adapter after data is in store adapter := NewDataStoreAdapter(store, testGenesisData()) - // Has should return true for existing data - use index hash - has, err := adapter.Has(ctx, computeDataIndexHash(h1)) + require.NoError(t, adapter.Append(ctx, wrapData(d1))) + + // Has should return true for existing hash + has, err := adapter.Has(ctx, d1.Hash()) require.NoError(t, err) assert.True(t, has) - // Has should return false for non-existent + // Has should return false for non-existent hash has, err = adapter.Has(ctx, []byte("nonexistent")) require.NoError(t, err) assert.False(t, has) @@ -156,7 +167,7 @@ func TestDataStoreAdapter_HasAt(t *testing.T) { adapter := NewDataStoreAdapter(store, testGenesisData()) _, d1 := types.GetRandomBlock(1, 2, "test-chain") - require.NoError(t, adapter.Append(ctx, d1)) + require.NoError(t, adapter.Append(ctx, wrapData(d1))) // HasAt should return true for pending height assert.True(t, adapter.HasAt(ctx, 1)) @@ -203,7 +214,7 @@ func TestDataStoreAdapter_GetRange(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") _, d3 := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2, d3)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2), wrapData(d3))) // GetRange [1, 3) should return data 1 and 2 dataList, err := adapter.GetRange(ctx, 1, 3) @@ -230,10 +241,10 @@ func TestDataStoreAdapter_GetRangeByHeight(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") _, d3 := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2, d3)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2), wrapData(d3))) // GetRangeByHeight from d1 to 4 should return data 2 and 3 - dataList, err := adapter.GetRangeByHeight(ctx, d1, 4) + dataList, err := adapter.GetRangeByHeight(ctx, wrapData(d1), 4) require.NoError(t, err) require.Len(t, dataList, 2) assert.Equal(t, uint64(2), dataList[0].Height()) @@ -252,7 +263,7 @@ func TestDataStoreAdapter_Init(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") // Init should add data to pending - err = adapter.Init(ctx, d1) + err = adapter.Init(ctx, wrapData(d1)) require.NoError(t, err) // Verify it's retrievable from pending @@ -262,7 +273,7 @@ func TestDataStoreAdapter_Init(t *testing.T) { // Init again should be a no-op (already initialized) _, d2 := types.GetRandomBlock(2, 1, "test-chain") - err = adapter.Init(ctx, d2) + err = adapter.Init(ctx, wrapData(d2)) require.NoError(t, err) // Height 2 should not be in pending since Init was already done @@ -284,7 +295,7 @@ func TestDataStoreAdapter_Tail(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2))) // Tail should return the first data from pending tail, err := adapter.Tail(ctx) @@ -346,7 +357,7 @@ func TestDataStoreAdapter_DeleteRange(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") _, d3 := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2, d3)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2), wrapData(d3))) assert.Equal(t, uint64(3), adapter.Height()) @@ -376,7 +387,7 @@ func TestDataStoreAdapter_OnDelete(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2))) // Track deleted heights var deletedHeights []uint64 @@ -410,7 +421,7 @@ func TestDataStoreAdapter_AppendSkipsExisting(t *testing.T) { adapter := NewDataStoreAdapter(store, testGenesisData()) // Append the same data again should not error (skips existing in store) - err = adapter.Append(ctx, d1) + err = adapter.Append(ctx, wrapData(d1)) require.NoError(t, err) // Height should still be 1 @@ -430,7 +441,7 @@ func TestDataStoreAdapter_AppendNilData(t *testing.T) { err = adapter.Append(ctx) require.NoError(t, err) - var nilData *types.Data + var nilData *types.P2PData err = adapter.Append(ctx, nilData) require.NoError(t, err) @@ -524,7 +535,7 @@ func TestDataStoreAdapter_ContextTimeout(t *testing.T) { _, d1 := types.GetRandomBlock(1, 1, "test-chain") // Note: In-memory store doesn't actually check context, but this verifies // the adapter passes the context through - _ = adapter.Append(ctx, d1) + _ = adapter.Append(ctx, wrapData(d1)) } func TestDataStoreAdapter_GetRangePartial(t *testing.T) { @@ -541,7 +552,7 @@ func TestDataStoreAdapter_GetRangePartial(t *testing.T) { // Only append data for heights 1 and 2, not 3 _, d1 := types.GetRandomBlock(1, 1, "test-chain") _, d2 := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1, d2)) + require.NoError(t, adapter.Append(ctx, wrapData(d1), wrapData(d2))) // GetRange [1, 5) should return data 1 and 2 (partial result) dataList, err := adapter.GetRange(ctx, 1, 5) @@ -580,15 +591,15 @@ func TestDataStoreAdapter_MultipleAppends(t *testing.T) { // Append data in multiple batches _, d1 := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1)) + require.NoError(t, adapter.Append(ctx, wrapData(d1))) assert.Equal(t, uint64(1), adapter.Height()) _, d2 := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d2)) + require.NoError(t, adapter.Append(ctx, wrapData(d2))) assert.Equal(t, uint64(2), adapter.Height()) _, d3 := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d3)) + require.NoError(t, adapter.Append(ctx, wrapData(d3))) assert.Equal(t, uint64(3), adapter.Height()) // Verify all data is retrievable @@ -608,7 +619,7 @@ func TestDataStoreAdapter_PendingAndStoreInteraction(t *testing.T) { // Add data to pending _, d1 := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1)) + require.NoError(t, adapter.Append(ctx, wrapData(d1))) // Verify it's in pending retrieved, err := adapter.GetByHeight(ctx, 1) @@ -650,7 +661,7 @@ func TestDataStoreAdapter_HeadPrefersPending(t *testing.T) { // Add height 2 to pending _, d2 := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d2)) + require.NoError(t, adapter.Append(ctx, wrapData(d2))) // Head should return the pending data (higher height) head, err := adapter.Head(ctx) @@ -669,10 +680,11 @@ func TestDataStoreAdapter_GetFromPendingByHash(t *testing.T) { // Add data to pending _, d1 := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, d1)) + p2pD1 := wrapData(d1) + require.NoError(t, adapter.Append(ctx, p2pD1)) // Get by hash from pending (uses data's Hash() method) - retrieved, err := adapter.Get(ctx, d1.Hash()) + retrieved, err := adapter.Get(ctx, p2pD1.Hash()) require.NoError(t, err) assert.Equal(t, d1.Height(), retrieved.Height()) } diff --git a/pkg/store/header_store_adapter_test.go b/pkg/store/header_store_adapter_test.go index bb1a281936..c39805c0b2 100644 --- a/pkg/store/header_store_adapter_test.go +++ b/pkg/store/header_store_adapter_test.go @@ -3,6 +3,7 @@ package store import ( "context" "crypto/sha256" + "errors" "testing" "time" @@ -31,6 +32,17 @@ func computeHeaderIndexHash(h *types.SignedHeader) []byte { return hash[:] } +// wrapHeader wraps a *types.SignedHeader in a *types.P2PSignedHeader for use with the HeaderStoreAdapter. +func wrapHeader(h *types.SignedHeader) *types.P2PSignedHeader { + if h == nil { + return nil + } + return &types.P2PSignedHeader{ + Message: h, + DAHeightHint: 0, + } +} + func TestHeaderStoreAdapter_NewHeaderStoreAdapter(t *testing.T) { t.Parallel() ctx := context.Background() @@ -64,7 +76,7 @@ func TestHeaderStoreAdapter_AppendAndRetrieve(t *testing.T) { h2, _ := types.GetRandomBlock(2, 2, "test-chain") // Append headers - these go to pending cache - err = adapter.Append(ctx, h1, h2) + err = adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2)) require.NoError(t, err) // Check height is updated (from pending) @@ -101,12 +113,10 @@ func TestHeaderStoreAdapter_GetFromStore(t *testing.T) { require.NoError(t, batch.SetHeight(1)) require.NoError(t, batch.Commit()) - // Create adapter after data is in store + // Now create adapter and verify we can get from store adapter := NewHeaderStoreAdapter(store, testGenesis()) - // Get by hash - need to use the index hash (sha256 of marshaled SignedHeader) - hash := computeHeaderIndexHash(h1) - retrieved, err := adapter.Get(ctx, hash) + retrieved, err := adapter.GetByHeight(ctx, 1) require.NoError(t, err) assert.Equal(t, h1.Height(), retrieved.Height()) @@ -133,12 +143,13 @@ func TestHeaderStoreAdapter_Has(t *testing.T) { adapter := NewHeaderStoreAdapter(store, testGenesis()) - // Has should return true for existing header - use index hash - has, err := adapter.Has(ctx, computeHeaderIndexHash(h1)) + p2pH1 := wrapHeader(h1) + // Has should return true for existing hash + has, err := adapter.Has(ctx, p2pH1.Hash()) require.NoError(t, err) assert.True(t, has) - // Has should return false for non-existent + // Has should return false for non-existent hash has, err = adapter.Has(ctx, []byte("nonexistent")) require.NoError(t, err) assert.False(t, has) @@ -154,7 +165,7 @@ func TestHeaderStoreAdapter_HasAt(t *testing.T) { adapter := NewHeaderStoreAdapter(store, testGenesis()) h1, _ := types.GetRandomBlock(1, 2, "test-chain") - require.NoError(t, adapter.Append(ctx, h1)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1))) // HasAt should return true for pending height assert.True(t, adapter.HasAt(ctx, 1)) @@ -201,7 +212,7 @@ func TestHeaderStoreAdapter_GetRange(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") h2, _ := types.GetRandomBlock(2, 1, "test-chain") h3, _ := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1, h2, h3)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2), wrapHeader(h3))) // GetRange [1, 3) should return headers 1 and 2 headers, err := adapter.GetRange(ctx, 1, 3) @@ -228,10 +239,10 @@ func TestHeaderStoreAdapter_GetRangeByHeight(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") h2, _ := types.GetRandomBlock(2, 1, "test-chain") h3, _ := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1, h2, h3)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2), wrapHeader(h3))) // GetRangeByHeight from h1 to 4 should return headers 2 and 3 - headers, err := adapter.GetRangeByHeight(ctx, h1, 4) + headers, err := adapter.GetRangeByHeight(ctx, wrapHeader(h1), 4) require.NoError(t, err) require.Len(t, headers, 2) assert.Equal(t, uint64(2), headers[0].Height()) @@ -250,7 +261,7 @@ func TestHeaderStoreAdapter_Init(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") // Init should add header to pending - err = adapter.Init(ctx, h1) + err = adapter.Init(ctx, wrapHeader(h1)) require.NoError(t, err) // Verify it's retrievable from pending @@ -260,7 +271,7 @@ func TestHeaderStoreAdapter_Init(t *testing.T) { // Init again should be a no-op (already initialized) h2, _ := types.GetRandomBlock(2, 1, "test-chain") - err = adapter.Init(ctx, h2) + err = adapter.Init(ctx, wrapHeader(h2)) require.NoError(t, err) // Height 2 should not be in pending since Init was already done @@ -282,7 +293,7 @@ func TestHeaderStoreAdapter_Tail(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") h2, _ := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1, h2)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2))) // Tail should return the first header tail, err := adapter.Tail(ctx) @@ -344,7 +355,7 @@ func TestHeaderStoreAdapter_DeleteRange(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") h2, _ := types.GetRandomBlock(2, 1, "test-chain") h3, _ := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1, h2, h3)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2), wrapHeader(h3))) assert.Equal(t, uint64(3), adapter.Height()) @@ -374,7 +385,7 @@ func TestHeaderStoreAdapter_OnDelete(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") h2, _ := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1, h2)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2))) // Track deleted heights var deletedHeights []uint64 @@ -408,7 +419,7 @@ func TestHeaderStoreAdapter_AppendSkipsExisting(t *testing.T) { adapter := NewHeaderStoreAdapter(store, testGenesis()) // Append the same header again should not error (skips existing in store) - err = adapter.Append(ctx, h1) + err = adapter.Append(ctx, wrapHeader(h1)) require.NoError(t, err) // Height should still be 1 @@ -428,7 +439,7 @@ func TestHeaderStoreAdapter_AppendNilHeaders(t *testing.T) { err = adapter.Append(ctx) require.NoError(t, err) - var nilHeader *types.SignedHeader + var nilHeader *types.P2PSignedHeader err = adapter.Append(ctx, nilHeader) require.NoError(t, err) @@ -522,7 +533,77 @@ func TestHeaderStoreAdapter_ContextTimeout(t *testing.T) { h1, _ := types.GetRandomBlock(1, 1, "test-chain") // Note: In-memory store doesn't actually check context, but this verifies // the adapter passes the context through - _ = adapter.Append(ctx, h1) + _ = adapter.Append(ctx, wrapHeader(h1)) +} + +func TestHeaderStoreAdapter_GetRangePartial(t *testing.T) { + t.Parallel() + // Use a short timeout since GetByHeight now blocks waiting for the height + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + ds, err := NewTestInMemoryKVStore() + require.NoError(t, err) + store := New(ds) + adapter := NewHeaderStoreAdapter(store, testGenesis()) + + // Only append headers for heights 1 and 2, not 3 + h1, _ := types.GetRandomBlock(1, 1, "test-chain") + h2, _ := types.GetRandomBlock(2, 1, "test-chain") + require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2))) + + // GetRange [1, 5) should return headers 1 and 2 (partial result) + headers, err := adapter.GetRange(ctx, 1, 5) + require.NoError(t, err) + require.Len(t, headers, 2) + assert.Equal(t, uint64(1), headers[0].Height()) + assert.Equal(t, uint64(2), headers[1].Height()) +} + +func TestHeaderStoreAdapter_GetRangeEmpty(t *testing.T) { + t.Parallel() + // Use a short timeout since GetByHeight now blocks waiting for the height + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + ds, err := NewTestInMemoryKVStore() + require.NoError(t, err) + store := New(ds) + adapter := NewHeaderStoreAdapter(store, testGenesis()) + + // GetRange on empty store will block until context timeout + _, err = adapter.GetRange(ctx, 1, 5) + // GetByHeight now blocks - we may get context.DeadlineExceeded or ErrNotFound depending on timing + assert.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, header.ErrNotFound), + "expected DeadlineExceeded or ErrNotFound, got: %v", err) +} + +func TestHeaderStoreAdapter_MultipleAppends(t *testing.T) { + t.Parallel() + ctx := context.Background() + + ds, err := NewTestInMemoryKVStore() + require.NoError(t, err) + store := New(ds) + adapter := NewHeaderStoreAdapter(store, testGenesis()) + + // Append headers in multiple batches + h1, _ := types.GetRandomBlock(1, 1, "test-chain") + require.NoError(t, adapter.Append(ctx, wrapHeader(h1))) + assert.Equal(t, uint64(1), adapter.Height()) + + h2, _ := types.GetRandomBlock(2, 1, "test-chain") + require.NoError(t, adapter.Append(ctx, wrapHeader(h2))) + assert.Equal(t, uint64(2), adapter.Height()) + + h3, _ := types.GetRandomBlock(3, 1, "test-chain") + require.NoError(t, adapter.Append(ctx, wrapHeader(h3))) + assert.Equal(t, uint64(3), adapter.Height()) + + // Verify all headers are retrievable + for h := uint64(1); h <= 3; h++ { + assert.True(t, adapter.HasAt(ctx, h)) + } } func TestHeaderStoreAdapter_PendingAndStoreInteraction(t *testing.T) { @@ -536,7 +617,7 @@ func TestHeaderStoreAdapter_PendingAndStoreInteraction(t *testing.T) { // Add header to pending h1, _ := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1))) // Verify it's in pending retrieved, err := adapter.GetByHeight(ctx, 1) @@ -578,7 +659,7 @@ func TestHeaderStoreAdapter_HeadPrefersPending(t *testing.T) { // Add height 2 to pending h2, _ := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h2)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h2))) // Head should return the pending header (higher height) head, err := adapter.Head(ctx) @@ -597,10 +678,11 @@ func TestHeaderStoreAdapter_GetFromPendingByHash(t *testing.T) { // Add header to pending h1, _ := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, h1)) + p2pH1 := wrapHeader(h1) + require.NoError(t, adapter.Append(ctx, p2pH1)) // Get by hash from pending (uses header's Hash() method) - retrieved, err := adapter.Get(ctx, h1.Hash()) + retrieved, err := adapter.Get(ctx, p2pH1.Hash()) require.NoError(t, err) assert.Equal(t, h1.Height(), retrieved.Height()) } diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 173cddc419..640e291c53 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -21,276 +21,9 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -type mockStore struct { - mock.Mock -} - -func (m *mockStore) Height(ctx context.Context) (uint64, error) { - args := m.Called(ctx) - return args.Get(0).(uint64), args.Error(1) -} - -func (m *mockStore) GetBlockData(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) { - args := m.Called(ctx, height) - return args.Get(0).(*types.SignedHeader), args.Get(1).(*types.Data), args.Error(2) -} - -func (m *mockStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { - args := m.Called(ctx, hash) - return args.Get(0).(*types.SignedHeader), args.Get(1).(*types.Data), args.Error(2) -} - -func (m *mockStore) GetHeader(ctx context.Context, height uint64) (*types.SignedHeader, error) { - args := m.Called(ctx, height) - return args.Get(0).(*types.SignedHeader), args.Error(1) -} - -func (m *mockStore) GetStateAtHeight(ctx context.Context, height uint64) (types.State, error) { - args := m.Called(ctx, height) - return args.Get(0).(types.State), args.Error(1) -} - -func (m *mockStore) GetSignature(ctx context.Context, height uint64) (*types.Signature, error) { - args := m.Called(ctx, height) - return args.Get(0).(*types.Signature), args.Error(1) -} - -func (m *mockStore) GetSignatureByHash(ctx context.Context, hash []byte) (*types.Signature, error) { - args := m.Called(ctx, hash) - return args.Get(0).(*types.Signature), args.Error(1) -} - -func (m *mockStore) GetState(ctx context.Context) (types.State, error) { - args := m.Called(ctx) - return args.Get(0).(types.State), args.Error(1) -} - -func (m *mockStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { - args := m.Called(ctx, key) - return args.Get(0).([]byte), args.Error(1) -} - -func (m *mockStore) SetMetadata(ctx context.Context, key string, value []byte) error { - args := m.Called(ctx, key, value) - return args.Error(0) -} - -func (m *mockStore) Close() error { - args := m.Called() - return args.Error(0) -} - -func (m *mockStore) NewBatch(ctx context.Context) (store.Batch, error) { - args := m.Called(ctx) - return args.Get(0).(store.Batch), args.Error(1) -} - -func (m *mockStore) Rollback(ctx context.Context, height uint64, aggregator bool) error { - args := m.Called(ctx, height, aggregator) - return args.Error(0) -} - -func TestDAHintFromDAStore(t *testing.T) { - mainKV := sync.MutexWrap(datastore.NewMapDatastore()) - daStore := new(mockStore) - - pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) - require.NoError(t, err) - noopSigner, err := noop.NewNoopSigner(pk) - require.NoError(t, err) - rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only - mn := mocknet.New() - - chainId := "test-chain-id" - genesisDoc := genesispkg.Genesis{ - ChainID: chainId, - StartTime: time.Now(), - InitialHeight: 1, - ProposerAddress: []byte("test"), - } - conf := config.DefaultConfig() - conf.RootDir = t.TempDir() - nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) - require.NoError(t, err) - logger := zerolog.Nop() - - p2pHost, err := mn.AddPeer(nodeKey.PrivKey, nil) - require.NoError(t, err) - p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - require.NoError(t, p2pClient.Start(ctx)) - - // Prepare data - height := uint64(1) - daHeight := uint64(100) - headerConfig := types.HeaderConfig{ - Height: height, - DataHash: bytesN(rnd, 32), - AppHash: bytesN(rnd, 32), - Signer: noopSigner, - } - signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, chainId) - require.NoError(t, err) - data := &types.Data{ - Metadata: &types.Metadata{ - ChainID: chainId, - Height: height, - }, - Txs: types.Txs{[]byte("tx1")}, - } - headerHash := signedHeader.Hash() - dataHash := data.Hash() - - state := types.State{ - ChainID: chainId, - LastBlockHeight: height, - DAHeight: daHeight, - } - - daStore.On("GetHeader", mock.Anything, height).Return(signedHeader, nil) - daStore.On("GetBlockByHash", mock.Anything, []byte(headerHash)).Return(signedHeader, data, nil) - daStore.On("GetBlockByHash", mock.Anything, []byte(dataHash)).Return(signedHeader, data, nil) - daStore.On("GetBlockData", mock.Anything, height).Return(signedHeader, data, nil) - daStore.On("GetStateAtHeight", mock.Anything, height).Return(state, nil) - - t.Run("header sync service", func(t *testing.T) { - headerSvc, err := NewHeaderSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) - require.NoError(t, err) - - h, err := headerSvc.getterByHeight(ctx, height) - require.NoError(t, err) - assert.Equal(t, daHeight, h.DAHint()) - assert.Equal(t, headerHash, h.Hash()) - - h2, err := headerSvc.getter(ctx, headerHash) - require.NoError(t, err) - assert.Equal(t, daHeight, h2.DAHint()) - - hRange, next, err := headerSvc.rangeGetter(ctx, height, height+1) - require.NoError(t, err) - assert.Equal(t, height+1, next) - require.Len(t, hRange, 1) - assert.Equal(t, daHeight, hRange[0].DAHint()) - }) - - t.Run("data sync service", func(t *testing.T) { - dataSvc, err := NewDataSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) - require.NoError(t, err) - - d, err := dataSvc.getterByHeight(ctx, height) - require.NoError(t, err) - assert.Equal(t, daHeight, d.DAHint()) - assert.Equal(t, dataHash, d.Hash()) - - d2, err := dataSvc.getter(ctx, dataHash) - require.NoError(t, err) - assert.Equal(t, daHeight, d2.DAHint()) - - dRange, next, err := dataSvc.rangeGetter(ctx, height, height+1) - require.NoError(t, err) - assert.Equal(t, height+1, next) - require.Len(t, dRange, 1) - assert.Equal(t, daHeight, dRange[0].DAHint()) - }) -} - -func TestDAHintFromDAStoreResilience(t *testing.T) { - mainKV := sync.MutexWrap(datastore.NewMapDatastore()) - daStore := new(mockStore) - - pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) - require.NoError(t, err) - noopSigner, err := noop.NewNoopSigner(pk) - require.NoError(t, err) - rnd := rand.New(rand.NewSource(1)) // nolint:gosec // test code only - mn := mocknet.New() - - chainId := "test-chain-id" - genesisDoc := genesispkg.Genesis{ - ChainID: chainId, - StartTime: time.Now(), - InitialHeight: 1, - ProposerAddress: []byte("test"), - } - conf := config.DefaultConfig() - conf.RootDir = t.TempDir() - nodeKey, err := key.LoadOrGenNodeKey(filepath.Dir(conf.ConfigPath())) - require.NoError(t, err) - logger := zerolog.Nop() - - p2pHost, err := mn.AddPeer(nodeKey.PrivKey, nil) - require.NoError(t, err) - p2pClient, err := p2p.NewClientWithHost(conf.P2P, nodeKey.PrivKey, mainKV, chainId, logger, p2p.NopMetrics(), p2pHost) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - require.NoError(t, p2pClient.Start(ctx)) - - // Prepare data - height := uint64(1) - headerConfig := types.HeaderConfig{ - Height: height, - DataHash: bytesN(rnd, 32), - AppHash: bytesN(rnd, 32), - Signer: noopSigner, - } - signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, chainId) - require.NoError(t, err) - data := &types.Data{ - Metadata: &types.Metadata{ - ChainID: chainId, - Height: height, - }, - Txs: types.Txs{[]byte("tx1")}, - } - headerHash := signedHeader.Hash() - dataHash := data.Hash() - - daStore.On("GetHeader", mock.Anything, height).Return(signedHeader, nil) - daStore.On("GetBlockByHash", mock.Anything, []byte(headerHash)).Return(signedHeader, data, nil) - daStore.On("GetBlockByHash", mock.Anything, []byte(dataHash)).Return(signedHeader, data, nil) - daStore.On("GetBlockData", mock.Anything, height).Return(signedHeader, data, nil) - // Return "not found" error for state - daStore.On("GetStateAtHeight", mock.Anything, height).Return(types.State{}, store.ErrNotFound) - - t.Run("header sync service resilience", func(t *testing.T) { - headerSvc, err := NewHeaderSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) - require.NoError(t, err) - - h, err := headerSvc.getterByHeight(ctx, height) - require.NoError(t, err) - assert.Equal(t, uint64(0), h.DAHint()) - assert.Equal(t, headerHash, h.Hash()) - - h2, err := headerSvc.getter(ctx, headerHash) - require.NoError(t, err) - assert.Equal(t, uint64(0), h2.DAHint()) - }) - - t.Run("data sync service resilience", func(t *testing.T) { - dataSvc, err := NewDataSyncService(mainKV, daStore, conf, genesisDoc, p2pClient, logger) - require.NoError(t, err) - - d, err := dataSvc.getterByHeight(ctx, height) - require.NoError(t, err) - assert.Equal(t, uint64(0), d.DAHint()) - assert.Equal(t, dataHash, d.Hash()) - - d2, err := dataSvc.getter(ctx, dataHash) - require.NoError(t, err) - assert.Equal(t, uint64(0), d2.DAHint()) - }) -} - func TestHeaderSyncServiceRestart(t *testing.T) { mainKV := sync.MutexWrap(datastore.NewMapDatastore()) pk, _, err := crypto.GenerateEd25519Key(cryptoRand.Reader) @@ -326,8 +59,8 @@ func TestHeaderSyncServiceRestart(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - rktStore := store.New(mainKV) - svc, err := NewHeaderSyncService(rktStore, conf, genesisDoc, p2pClient, logger) + evStore := store.New(mainKV) + svc, err := NewHeaderSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) err = svc.Start(ctx) require.NoError(t, err) @@ -367,7 +100,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = p2pClient.Close() }) - svc, err = NewHeaderSyncService(rktStore, conf, genesisDoc, p2pClient, logger) + svc, err = NewHeaderSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) err = svc.Start(ctx) require.NoError(t, err) @@ -418,8 +151,8 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - rktStore := store.New(mainKV) - svc, err := NewHeaderSyncService(rktStore, conf, genesisDoc, p2pClient, logger) + evStore := store.New(mainKV) + svc, err := NewHeaderSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, svc.Start(ctx)) t.Cleanup(func() { _ = svc.Stop(context.Background()) }) @@ -471,7 +204,8 @@ func TestDAHintStorageHeader(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - headerSvc, err := NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) + evStore := store.New(mainKV) + headerSvc, err := NewHeaderSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, headerSvc.Start(ctx)) @@ -510,7 +244,7 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - headerSvc, err = NewHeaderSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) + headerSvc, err = NewHeaderSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, headerSvc.Start(ctx)) t.Cleanup(func() { _ = headerSvc.Stop(context.Background()) }) @@ -555,7 +289,8 @@ func TestDAHintStorageData(t *testing.T) { defer cancel() require.NoError(t, p2pClient.Start(ctx)) - dataSvc, err := NewDataSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) + evStore := store.New(mainKV) + dataSvc, err := NewDataSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, dataSvc.Start(ctx)) @@ -601,7 +336,7 @@ func TestDAHintStorageData(t *testing.T) { require.NoError(t, p2pClient.Start(ctx)) t.Cleanup(func() { _ = p2pClient.Close() }) - dataSvc, err = NewDataSyncService(mainKV, nil, conf, genesisDoc, p2pClient, logger) + dataSvc, err = NewDataSyncService(evStore, conf, genesisDoc, p2pClient, logger) require.NoError(t, err) require.NoError(t, dataSvc.Start(ctx)) t.Cleanup(func() { _ = dataSvc.Stop(context.Background()) }) From c6e569670333d1f7ad4788081bd6f81e4c1e123c Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 15:26:53 +0100 Subject: [PATCH 20/41] fixes --- pkg/store/data_store_adapter_test.go | 19 +++--- pkg/store/header_store_adapter_test.go | 89 +++----------------------- 2 files changed, 18 insertions(+), 90 deletions(-) diff --git a/pkg/store/data_store_adapter_test.go b/pkg/store/data_store_adapter_test.go index ec260d2fbd..bde29ce1ed 100644 --- a/pkg/store/data_store_adapter_test.go +++ b/pkg/store/data_store_adapter_test.go @@ -114,10 +114,12 @@ func TestDataStoreAdapter_GetFromStore(t *testing.T) { require.NoError(t, batch.SetHeight(1)) require.NoError(t, batch.Commit()) - // Now create adapter and verify we can get from store + // Create adapter after data is in store adapter := NewDataStoreAdapter(store, testGenesisData()) - retrieved, err := adapter.GetByHeight(ctx, 1) + // Get by hash - need to use the index hash (sha256 of marshaled SignedHeader) + hash := computeDataIndexHash(h1) + retrieved, err := adapter.Get(ctx, hash) require.NoError(t, err) assert.Equal(t, d1.Height(), retrieved.Height()) @@ -144,14 +146,12 @@ func TestDataStoreAdapter_Has(t *testing.T) { // Create adapter after data is in store adapter := NewDataStoreAdapter(store, testGenesisData()) - require.NoError(t, adapter.Append(ctx, wrapData(d1))) - - // Has should return true for existing hash - has, err := adapter.Has(ctx, d1.Hash()) + // Has should return true for existing data - use index hash + has, err := adapter.Has(ctx, computeDataIndexHash(h1)) require.NoError(t, err) assert.True(t, has) - // Has should return false for non-existent hash + // Has should return false for non-existent has, err = adapter.Has(ctx, []byte("nonexistent")) require.NoError(t, err) assert.False(t, has) @@ -680,11 +680,10 @@ func TestDataStoreAdapter_GetFromPendingByHash(t *testing.T) { // Add data to pending _, d1 := types.GetRandomBlock(1, 1, "test-chain") - p2pD1 := wrapData(d1) - require.NoError(t, adapter.Append(ctx, p2pD1)) + require.NoError(t, adapter.Append(ctx, wrapData(d1))) // Get by hash from pending (uses data's Hash() method) - retrieved, err := adapter.Get(ctx, p2pD1.Hash()) + retrieved, err := adapter.Get(ctx, d1.Hash()) require.NoError(t, err) assert.Equal(t, d1.Height(), retrieved.Height()) } diff --git a/pkg/store/header_store_adapter_test.go b/pkg/store/header_store_adapter_test.go index c39805c0b2..04e238c9c5 100644 --- a/pkg/store/header_store_adapter_test.go +++ b/pkg/store/header_store_adapter_test.go @@ -3,7 +3,6 @@ package store import ( "context" "crypto/sha256" - "errors" "testing" "time" @@ -113,10 +112,12 @@ func TestHeaderStoreAdapter_GetFromStore(t *testing.T) { require.NoError(t, batch.SetHeight(1)) require.NoError(t, batch.Commit()) - // Now create adapter and verify we can get from store + // Create adapter after data is in store adapter := NewHeaderStoreAdapter(store, testGenesis()) - retrieved, err := adapter.GetByHeight(ctx, 1) + // Get by hash - need to use the index hash (sha256 of marshaled SignedHeader) + hash := computeHeaderIndexHash(h1) + retrieved, err := adapter.Get(ctx, hash) require.NoError(t, err) assert.Equal(t, h1.Height(), retrieved.Height()) @@ -143,13 +144,12 @@ func TestHeaderStoreAdapter_Has(t *testing.T) { adapter := NewHeaderStoreAdapter(store, testGenesis()) - p2pH1 := wrapHeader(h1) - // Has should return true for existing hash - has, err := adapter.Has(ctx, p2pH1.Hash()) + // Has should return true for existing header - use index hash + has, err := adapter.Has(ctx, computeHeaderIndexHash(h1)) require.NoError(t, err) assert.True(t, has) - // Has should return false for non-existent hash + // Has should return false for non-existent has, err = adapter.Has(ctx, []byte("nonexistent")) require.NoError(t, err) assert.False(t, has) @@ -536,76 +536,6 @@ func TestHeaderStoreAdapter_ContextTimeout(t *testing.T) { _ = adapter.Append(ctx, wrapHeader(h1)) } -func TestHeaderStoreAdapter_GetRangePartial(t *testing.T) { - t.Parallel() - // Use a short timeout since GetByHeight now blocks waiting for the height - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - - ds, err := NewTestInMemoryKVStore() - require.NoError(t, err) - store := New(ds) - adapter := NewHeaderStoreAdapter(store, testGenesis()) - - // Only append headers for heights 1 and 2, not 3 - h1, _ := types.GetRandomBlock(1, 1, "test-chain") - h2, _ := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, wrapHeader(h1), wrapHeader(h2))) - - // GetRange [1, 5) should return headers 1 and 2 (partial result) - headers, err := adapter.GetRange(ctx, 1, 5) - require.NoError(t, err) - require.Len(t, headers, 2) - assert.Equal(t, uint64(1), headers[0].Height()) - assert.Equal(t, uint64(2), headers[1].Height()) -} - -func TestHeaderStoreAdapter_GetRangeEmpty(t *testing.T) { - t.Parallel() - // Use a short timeout since GetByHeight now blocks waiting for the height - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - ds, err := NewTestInMemoryKVStore() - require.NoError(t, err) - store := New(ds) - adapter := NewHeaderStoreAdapter(store, testGenesis()) - - // GetRange on empty store will block until context timeout - _, err = adapter.GetRange(ctx, 1, 5) - // GetByHeight now blocks - we may get context.DeadlineExceeded or ErrNotFound depending on timing - assert.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, header.ErrNotFound), - "expected DeadlineExceeded or ErrNotFound, got: %v", err) -} - -func TestHeaderStoreAdapter_MultipleAppends(t *testing.T) { - t.Parallel() - ctx := context.Background() - - ds, err := NewTestInMemoryKVStore() - require.NoError(t, err) - store := New(ds) - adapter := NewHeaderStoreAdapter(store, testGenesis()) - - // Append headers in multiple batches - h1, _ := types.GetRandomBlock(1, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, wrapHeader(h1))) - assert.Equal(t, uint64(1), adapter.Height()) - - h2, _ := types.GetRandomBlock(2, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, wrapHeader(h2))) - assert.Equal(t, uint64(2), adapter.Height()) - - h3, _ := types.GetRandomBlock(3, 1, "test-chain") - require.NoError(t, adapter.Append(ctx, wrapHeader(h3))) - assert.Equal(t, uint64(3), adapter.Height()) - - // Verify all headers are retrievable - for h := uint64(1); h <= 3; h++ { - assert.True(t, adapter.HasAt(ctx, h)) - } -} - func TestHeaderStoreAdapter_PendingAndStoreInteraction(t *testing.T) { t.Parallel() ctx := context.Background() @@ -678,11 +608,10 @@ func TestHeaderStoreAdapter_GetFromPendingByHash(t *testing.T) { // Add header to pending h1, _ := types.GetRandomBlock(1, 1, "test-chain") - p2pH1 := wrapHeader(h1) - require.NoError(t, adapter.Append(ctx, p2pH1)) + require.NoError(t, adapter.Append(ctx, wrapHeader(h1))) // Get by hash from pending (uses header's Hash() method) - retrieved, err := adapter.Get(ctx, p2pH1.Hash()) + retrieved, err := adapter.Get(ctx, h1.Hash()) require.NoError(t, err) assert.Equal(t, h1.Height(), retrieved.Height()) } From b7b21001c89c1b47ec978dcd980ca46752904c03 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 15:46:27 +0100 Subject: [PATCH 21/41] cleanup --- apps/testapp/go.mod | 2 -- apps/testapp/go.sum | 4 ++++ pkg/store/store.go | 2 +- pkg/store/types.go | 6 ------ 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/apps/testapp/go.mod b/apps/testapp/go.mod index 58ed351f47..07fc9f3a74 100644 --- a/apps/testapp/go.mod +++ b/apps/testapp/go.mod @@ -183,8 +183,6 @@ require ( // pin google genproto to a single version to avoid ambiguous imports pulled by transitive deps replace ( - github.com/evstack/ev-node => ../../ - github.com/evstack/ev-node/core => ../../core google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 diff --git a/apps/testapp/go.sum b/apps/testapp/go.sum index de36566f76..ff0e8ec88b 100644 --- a/apps/testapp/go.sum +++ b/apps/testapp/go.sum @@ -365,6 +365,10 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/evstack/ev-node v1.0.0-rc.1 h1:MO7DT3y1X4WK7pTgl/867NroqhXJ/oe2NbmvMr3jqq8= +github.com/evstack/ev-node v1.0.0-rc.1/go.mod h1:JtbvY2r6k6ZhGYMeDNZk7cx6ALj3d0f6dVyyJmJHBd4= +github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= +github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= diff --git a/pkg/store/store.go b/pkg/store/store.go index 1c5abda764..40d00547b5 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -152,7 +152,7 @@ func (s *DefaultStore) GetStateAtHeight(ctx context.Context, height uint64) (typ blob, err := s.db.Get(ctx, ds.NewKey(getStateAtHeightKey(height))) if err != nil { if errors.Is(err, ds.ErrNotFound) { - return types.State{}, fmt.Errorf("get state at height %d: %w", height, ErrNotFound) + return types.State{}, fmt.Errorf("get state at height %d: %w", height, ds.ErrNotFound) } return types.State{}, fmt.Errorf("failed to retrieve state at height %d: %w", height, err) } diff --git a/pkg/store/types.go b/pkg/store/types.go index 8a01f7c467..bf1cb6ced8 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -2,18 +2,12 @@ package store import ( "context" - "errors" ds "github.com/ipfs/go-datastore" "github.com/evstack/ev-node/types" ) -var ( - // ErrNotFound is returned when the entry is not found in the store. - ErrNotFound = errors.New("not found") -) - // Batch provides atomic operations for the store type Batch interface { // SaveBlockData atomically saves the block header, data, and signature From 7b1c80250e4cb71f6df82a111238928ef8f7a45b Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 15:47:59 +0100 Subject: [PATCH 22/41] updates --- block/internal/syncing/p2p_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index 86fc95dee1..3326583bc8 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -20,7 +20,7 @@ type p2pHandler interface { SetProcessedHeight(height uint64) } -// HeightStore is a subset of goheader.Store +// HeightStore is a subset of the sync service. type HeightStore[H header.Header[H]] interface { GetByHeight(ctx context.Context, height uint64) (H, uint64, error) } From 14b83ff2addca726a5b442b1985d474143b5d781 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 16:05:52 +0100 Subject: [PATCH 23/41] implement setting --- pkg/store/store_adapter.go | 122 ++++++++++++++++++++++++++++++++++--- pkg/sync/sync_service.go | 10 +-- 2 files changed, 116 insertions(+), 16 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index fbfcf11ede..45bf175a79 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -3,7 +3,9 @@ package store import ( "bytes" "context" + "encoding/binary" "errors" + "fmt" "sync" "sync/atomic" @@ -30,6 +32,19 @@ type StoreGetter[H header.Header[H]] interface { Height(ctx context.Context) (uint64, error) // HasAt checks if an item exists at the given height. HasAt(ctx context.Context, height uint64) bool + // GetDAHint retrieves the DA hint for a given height. + GetDAHint(ctx context.Context, height uint64) (uint64, error) + // SetDAHint stores the DA hint for a given height. + SetDAHint(ctx context.Context, height uint64, daHint uint64) error +} + +// EntityWithDAHint extends header.Header with DA hint methods. +// This interface is used by sync services and store adapters to track +// which DA height contains the data for a given block. +type EntityWithDAHint[H any] interface { + header.Header[H] + SetDAHint(daHeight uint64) + DAHint() uint64 } // heightSub provides a mechanism for waiting on a specific height to be stored. @@ -111,7 +126,7 @@ func (hs *heightSub) notifyUpTo(h uint64) { // This cache allows the go-header syncer and P2P handler to access items before they // are validated and persisted by the ev-node syncer. Once the ev-node syncer processes // a block, it writes to the underlying store, and subsequent reads will come from the store. -type StoreAdapter[H header.Header[H]] struct { +type StoreAdapter[H EntityWithDAHint[H]] struct { getter StoreGetter[H] genesisInitialHeight uint64 @@ -127,15 +142,20 @@ type StoreAdapter[H header.Header[H]] struct { // written to the store yet. Keyed by height. Using LRU prevents unbounded growth. pending *lru.Cache[uint64, H] + // daHints caches DA height hints by block height for fast access. + // Hints are also persisted to disk via the getter. + daHints *lru.Cache[uint64, uint64] + // onDeleteFn is called when items are deleted (for rollback scenarios) onDeleteFn func(context.Context, uint64) error } // NewStoreAdapter creates a new StoreAdapter wrapping the given store getter. // The genesis is used to determine the initial height for efficient Tail lookups. -func NewStoreAdapter[H header.Header[H]](getter StoreGetter[H], gen genesis.Genesis) *StoreAdapter[H] { +func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], gen genesis.Genesis) *StoreAdapter[H] { // Create LRU cache for pending items - ignore error as size is constant and valid pendingCache, _ := lru.New[uint64, H](defaultPendingCacheSize) + daHintsCache, _ := lru.New[uint64, uint64](defaultPendingCacheSize) // Get actual current height from store (0 if empty) var storeHeight uint64 @@ -147,6 +167,7 @@ func NewStoreAdapter[H header.Header[H]](getter StoreGetter[H], gen genesis.Gene getter: getter, genesisInitialHeight: max(gen.InitialHeight, 1), pending: pendingCache, + daHints: daHintsCache, heightSub: newHeightSub(storeHeight), } @@ -276,6 +297,7 @@ func (a *StoreAdapter[H]) Get(ctx context.Context, hash header.Hash) (H, error) // First try the store item, err := a.getter.GetByHash(ctx, hash) if err == nil { + a.applyDAHint(item) return item, nil } @@ -316,6 +338,7 @@ func (a *StoreAdapter[H]) getByHeightNoWait(ctx context.Context, height uint64) // First try the store item, err := a.getter.GetByHeight(ctx, height) if err == nil { + a.applyDAHint(item) return item, nil } @@ -327,6 +350,27 @@ func (a *StoreAdapter[H]) getByHeightNoWait(ctx context.Context, height uint64) return zero, header.ErrNotFound } +// applyDAHint sets the DA hint on the item from cache or disk. +func (a *StoreAdapter[H]) applyDAHint(item H) { + if item.IsZero() { + return + } + + height := item.Height() + + // Check cache first + if hint, found := a.daHints.Get(height); found { + item.SetDAHint(hint) + return + } + + // Try to load from disk + if hint, err := a.getter.GetDAHint(context.Background(), height); err == nil && hint > 0 { + a.daHints.Add(height, hint) + item.SetDAHint(hint) + } +} + // GetRangeByHeight returns items in the range [from.Height()+1, to). // This follows go-header's convention where 'from' is the trusted item // and we return items starting from the next height. @@ -431,6 +475,7 @@ func (a *StoreAdapter[H]) Height() uint64 { // Append stores items in the pending cache. // These items are received via P2P and will be available for retrieval // until the ev-node syncer processes and persists them to the store. +// If items have a DA hint set, it will be cached for later retrieval. func (a *StoreAdapter[H]) Append(ctx context.Context, items ...H) error { if len(items) == 0 { return nil @@ -443,9 +488,16 @@ func (a *StoreAdapter[H]) Append(ctx context.Context, items ...H) error { height := item.Height() + // Cache and persist DA hint if present + if hint := item.DAHint(); hint > 0 { + a.daHints.Add(height, hint) + // Persist to disk + _ = a.getter.SetDAHint(ctx, height, hint) + } + // Check if already in store if a.getter.HasAt(ctx, height) { - // Already persisted, skip + // Already persisted, skip adding to pending continue } @@ -492,9 +544,10 @@ func (a *StoreAdapter[H]) Sync(ctx context.Context) error { // DeleteRange deletes items in the range [from, to). // This is used for rollback operations. func (a *StoreAdapter[H]) DeleteRange(ctx context.Context, from, to uint64) error { - // Remove from pending cache + // Remove from pending cache and DA hints cache for height := from; height < to; height++ { a.pending.Remove(height) + a.daHints.Remove(height) if a.onDeleteFn != nil { if err := a.onDeleteFn(ctx, height); err != nil { @@ -533,12 +586,24 @@ func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*ty return nil, err } + daHint, _ := getDAHintFromStore(ctx, g.store, height) + return &types.P2PSignedHeader{ Message: header, - DAHeightHint: 0, // TODO: fetch DA hint. + DAHeightHint: daHint, }, nil } +// GetDAHint implements StoreGetter. +func (g *HeaderStoreGetter) GetDAHint(ctx context.Context, height uint64) (uint64, error) { + return getDAHintFromStore(ctx, g.store, height) +} + +// SetDAHint implements StoreGetter. +func (g *HeaderStoreGetter) SetDAHint(ctx context.Context, height uint64, daHint uint64) error { + return setDAHintToStore(ctx, g.store, height, daHint) +} + // GetByHash implements StoreGetter. func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2PSignedHeader, error) { hdr, _, err := g.store.GetBlockByHash(ctx, hash) @@ -546,9 +611,11 @@ func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types. return nil, err } + daHint, _ := getDAHintFromStore(ctx, g.store, hdr.Height()) + return &types.P2PSignedHeader{ Message: hdr, - DAHeightHint: 0, // TODO: fetch DA hint. + DAHeightHint: daHint, }, nil } @@ -580,12 +647,24 @@ func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*type return nil, err } + daHint, _ := getDAHintFromStore(ctx, g.store, height) + return &types.P2PData{ Message: data, - DAHeightHint: 0, // TODO: fetch DA hint. + DAHeightHint: daHint, }, nil } +// GetDAHint implements StoreGetter. +func (g *DataStoreGetter) GetDAHint(ctx context.Context, height uint64) (uint64, error) { + return getDAHintFromStore(ctx, g.store, height) +} + +// SetDAHint implements StoreGetter. +func (g *DataStoreGetter) SetDAHint(ctx context.Context, height uint64, daHint uint64) error { + return setDAHintToStore(ctx, g.store, height, daHint) +} + // GetByHash implements StoreGetter. func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2PData, error) { _, data, err := g.store.GetBlockByHash(ctx, hash) @@ -593,9 +672,11 @@ func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2 return nil, err } + daHint, _ := getDAHintFromStore(ctx, g.store, data.Height()) + return &types.P2PData{ Message: data, - DAHeightHint: 0, // TODO: fetch DA hint. + DAHeightHint: daHint, }, nil } @@ -625,3 +706,28 @@ func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { return NewStoreAdapter(NewDataStoreGetter(store), gen) } + +// DA hint persistence helpers + +const daHintKeyPrefix = "da_hint:" + +func daHintKey(height uint64) string { + return fmt.Sprintf("%s%d", daHintKeyPrefix, height) +} + +func getDAHintFromStore(ctx context.Context, store Store, height uint64) (uint64, error) { + data, err := store.GetMetadata(ctx, daHintKey(height)) + if err != nil { + return 0, err + } + if len(data) != 8 { + return 0, fmt.Errorf("invalid da hint data length: %d", len(data)) + } + return binary.BigEndian.Uint64(data), nil +} + +func setDAHintToStore(ctx context.Context, store Store, height uint64, daHint uint64) error { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, daHint) + return store.SetMetadata(ctx, daHintKey(height), data) +} diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index de22b92ad5..be9bfcfda3 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -32,12 +32,6 @@ const ( dataSync syncType = "dataSync" ) -type EntityWithDAHint[H any] interface { - header.Header[H] - SetDAHint(daHeight uint64) - DAHint() uint64 -} - // HeaderSyncService is the P2P Sync Service for headers. type HeaderSyncService = SyncService[*types.P2PSignedHeader] @@ -47,7 +41,7 @@ type DataSyncService = SyncService[*types.P2PData] // SyncService is the P2P Sync Service for blocks and headers. // // Uses the go-header library for handling all P2P logic. -type SyncService[H EntityWithDAHint[H]] struct { +type SyncService[H store.EntityWithDAHint[H]] struct { conf config.Config logger zerolog.Logger syncType syncType @@ -91,7 +85,7 @@ func NewHeaderSyncService( return newSyncService(storeAdapter, headerSync, conf, genesis, p2p, logger) } -func newSyncService[H EntityWithDAHint[H]]( +func newSyncService[H store.EntityWithDAHint[H]]( storeAdapter header.Store[H], syncType syncType, conf config.Config, From 3a2ba3fe7a842bd097812785343666573255609e Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 16:31:30 +0100 Subject: [PATCH 24/41] limit abstractions --- .mockery.yaml | 6 +- block/components.go | 20 +- block/internal/common/broadcaster_mock.go | 151 ----------- block/internal/common/expected_interfaces.go | 2 - block/internal/executing/executor.go | 4 +- block/internal/syncing/height_store_mock.go | 113 --------- block/internal/syncing/p2p_handler.go | 35 ++- block/internal/syncing/p2p_handler_test.go | 65 ++--- block/internal/syncing/syncer.go | 13 +- block/internal/syncing/syncer_backoff_test.go | 4 +- .../internal/syncing/syncer_benchmark_test.go | 5 +- .../syncing/syncer_forced_inclusion_test.go | 33 +-- block/internal/syncing/syncer_test.go | 20 +- pkg/rpc/client/client_test.go | 4 +- pkg/rpc/server/server_test.go | 4 +- pkg/store/data_store_adapter_test.go | 2 +- pkg/store/header_store_adapter_test.go | 2 +- pkg/store/store_adapter.go | 8 +- pkg/sync/sync_service.go | 9 - pkg/sync/sync_service_test.go | 28 +-- types/p2p_envelope.go | 237 ++++++++++-------- types/p2p_envelope_test.go | 38 +-- 22 files changed, 273 insertions(+), 530 deletions(-) delete mode 100644 block/internal/syncing/height_store_mock.go diff --git a/.mockery.yaml b/.mockery.yaml index e38a8f9dc4..106be368cb 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -61,11 +61,7 @@ packages: dir: ./block/internal/syncing pkgname: syncing filename: syncer_mock.go - HeightStore: - config: - dir: ./block/internal/syncing - pkgname: syncing - filename: height_store_mock.go + github.com/evstack/ev-node/block/internal/common: interfaces: Broadcaster: diff --git a/block/components.go b/block/components.go index 2296cd75d4..d16a16863f 100644 --- a/block/components.go +++ b/block/components.go @@ -127,8 +127,8 @@ func NewSyncComponents( store store.Store, exec coreexecutor.Executor, daClient da.Client, - headerStore *sync.HeaderSyncService, - dataStore *sync.DataSyncService, + headerSyncService *sync.HeaderSyncService, + dataSyncService *sync.DataSyncService, logger zerolog.Logger, metrics *Metrics, blockOpts BlockOptions, @@ -150,8 +150,8 @@ func NewSyncComponents( metrics, config, genesis, - headerStore, - dataStore, + headerSyncService.Store(), + dataSyncService.Store(), logger, blockOpts, errorCh, @@ -163,7 +163,7 @@ func NewSyncComponents( } // Create submitter for sync nodes (no signer, only DA inclusion processing) - var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerStore, dataStore) + var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerSyncService, dataSyncService) if config.Instrumentation.IsTracingEnabled() { daSubmitter = submitting.WithTracingDASubmitter(daSubmitter) } @@ -200,8 +200,8 @@ func NewAggregatorComponents( sequencer coresequencer.Sequencer, daClient da.Client, signer signer.Signer, - headerBroadcaster *sync.HeaderSyncService, - dataBroadcaster *sync.DataSyncService, + headerSyncService *sync.HeaderSyncService, + dataSyncService *sync.DataSyncService, logger zerolog.Logger, metrics *Metrics, blockOpts BlockOptions, @@ -229,8 +229,8 @@ func NewAggregatorComponents( metrics, config, genesis, - headerBroadcaster, - dataBroadcaster, + headerSyncService, + dataSyncService, logger, blockOpts, errorCh, @@ -266,7 +266,7 @@ func NewAggregatorComponents( }, nil } - var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerBroadcaster, dataBroadcaster) + var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerSyncService, dataSyncService) if config.Instrumentation.IsTracingEnabled() { daSubmitter = submitting.WithTracingDASubmitter(daSubmitter) } diff --git a/block/internal/common/broadcaster_mock.go b/block/internal/common/broadcaster_mock.go index 04fde0982e..b638e6b491 100644 --- a/block/internal/common/broadcaster_mock.go +++ b/block/internal/common/broadcaster_mock.go @@ -8,7 +8,6 @@ import ( "context" "github.com/celestiaorg/go-header" - "github.com/evstack/ev-node/types" "github.com/libp2p/go-libp2p-pubsub" mock "github.com/stretchr/testify/mock" ) @@ -40,156 +39,6 @@ func (_m *MockBroadcaster[H]) EXPECT() *MockBroadcaster_Expecter[H] { return &MockBroadcaster_Expecter[H]{mock: &_m.Mock} } -// AppendDAHint provides a mock function for the type MockBroadcaster -func (_mock *MockBroadcaster[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { - // types.Hash - _va := make([]interface{}, len(hashes)) - for _i := range hashes { - _va[_i] = hashes[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, daHeight) - _ca = append(_ca, _va...) - ret := _mock.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for AppendDAHint") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64, ...types.Hash) error); ok { - r0 = returnFunc(ctx, daHeight, hashes...) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockBroadcaster_AppendDAHint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AppendDAHint' -type MockBroadcaster_AppendDAHint_Call[H header.Header[H]] struct { - *mock.Call -} - -// AppendDAHint is a helper method to define mock.On call -// - ctx context.Context -// - daHeight uint64 -// - hashes ...types.Hash -func (_e *MockBroadcaster_Expecter[H]) AppendDAHint(ctx interface{}, daHeight interface{}, hashes ...interface{}) *MockBroadcaster_AppendDAHint_Call[H] { - return &MockBroadcaster_AppendDAHint_Call[H]{Call: _e.mock.On("AppendDAHint", - append([]interface{}{ctx, daHeight}, hashes...)...)} -} - -func (_c *MockBroadcaster_AppendDAHint_Call[H]) Run(run func(ctx context.Context, daHeight uint64, hashes ...types.Hash)) *MockBroadcaster_AppendDAHint_Call[H] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 uint64 - if args[1] != nil { - arg1 = args[1].(uint64) - } - var arg2 []types.Hash - variadicArgs := make([]types.Hash, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(types.Hash) - } - } - arg2 = variadicArgs - run( - arg0, - arg1, - arg2..., - ) - }) - return _c -} - -func (_c *MockBroadcaster_AppendDAHint_Call[H]) Return(err error) *MockBroadcaster_AppendDAHint_Call[H] { - _c.Call.Return(err) - return _c -} - -func (_c *MockBroadcaster_AppendDAHint_Call[H]) RunAndReturn(run func(ctx context.Context, daHeight uint64, hashes ...types.Hash) error) *MockBroadcaster_AppendDAHint_Call[H] { - _c.Call.Return(run) - return _c -} - -// GetByHeight provides a mock function for the type MockBroadcaster -func (_mock *MockBroadcaster[H]) GetByHeight(ctx context.Context, height uint64) (H, uint64, error) { - ret := _mock.Called(ctx, height) - - if len(ret) == 0 { - panic("no return value specified for GetByHeight") - } - - var r0 H - var r1 uint64 - var r2 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) (H, uint64, error)); ok { - return returnFunc(ctx, height) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) H); ok { - r0 = returnFunc(ctx, height) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(H) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) uint64); ok { - r1 = returnFunc(ctx, height) - } else { - r1 = ret.Get(1).(uint64) - } - if returnFunc, ok := ret.Get(2).(func(context.Context, uint64) error); ok { - r2 = returnFunc(ctx, height) - } else { - r2 = ret.Error(2) - } - return r0, r1, r2 -} - -// MockBroadcaster_GetByHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByHeight' -type MockBroadcaster_GetByHeight_Call[H header.Header[H]] struct { - *mock.Call -} - -// GetByHeight is a helper method to define mock.On call -// - ctx context.Context -// - height uint64 -func (_e *MockBroadcaster_Expecter[H]) GetByHeight(ctx interface{}, height interface{}) *MockBroadcaster_GetByHeight_Call[H] { - return &MockBroadcaster_GetByHeight_Call[H]{Call: _e.mock.On("GetByHeight", ctx, height)} -} - -func (_c *MockBroadcaster_GetByHeight_Call[H]) Run(run func(ctx context.Context, height uint64)) *MockBroadcaster_GetByHeight_Call[H] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 uint64 - if args[1] != nil { - arg1 = args[1].(uint64) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *MockBroadcaster_GetByHeight_Call[H]) Return(v H, v1 uint64, err error) *MockBroadcaster_GetByHeight_Call[H] { - _c.Call.Return(v, v1, err) - return _c -} - -func (_c *MockBroadcaster_GetByHeight_Call[H]) RunAndReturn(run func(ctx context.Context, height uint64) (H, uint64, error)) *MockBroadcaster_GetByHeight_Call[H] { - _c.Call.Return(run) - return _c -} - // Height provides a mock function for the type MockBroadcaster func (_mock *MockBroadcaster[H]) Height() uint64 { ret := _mock.Called() diff --git a/block/internal/common/expected_interfaces.go b/block/internal/common/expected_interfaces.go index 03e49a0866..9f3a7fa1d1 100644 --- a/block/internal/common/expected_interfaces.go +++ b/block/internal/common/expected_interfaces.go @@ -17,8 +17,6 @@ type ( // Broadcaster interface for P2P broadcasting type Broadcaster[H header.Header[H]] interface { WriteToStoreAndBroadcast(ctx context.Context, payload H, opts ...pubsub.PubOpt) error - AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error - GetByHeight(ctx context.Context, height uint64) (H, uint64, error) Store() header.Store[H] Height() uint64 } diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 6e6242df3f..bf1b44b6cb 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -549,10 +549,10 @@ func (e *Executor) ProduceBlock(ctx context.Context) error { // broadcast header and data to P2P network g, broadcastCtx := errgroup.WithContext(e.ctx) g.Go(func() error { - return e.headerBroadcaster.WriteToStoreAndBroadcast(broadcastCtx, &types.P2PSignedHeader{Message: header}) + return e.headerBroadcaster.WriteToStoreAndBroadcast(broadcastCtx, &types.P2PSignedHeader{SignedHeader: header}) }) g.Go(func() error { - return e.dataBroadcaster.WriteToStoreAndBroadcast(broadcastCtx, &types.P2PData{Message: data}) + return e.dataBroadcaster.WriteToStoreAndBroadcast(broadcastCtx, &types.P2PData{Data: data}) }) if err := g.Wait(); err != nil { e.logger.Error().Err(err).Msg("failed to broadcast header and/data") diff --git a/block/internal/syncing/height_store_mock.go b/block/internal/syncing/height_store_mock.go deleted file mode 100644 index b4857accfa..0000000000 --- a/block/internal/syncing/height_store_mock.go +++ /dev/null @@ -1,113 +0,0 @@ -// Code generated by mockery; DO NOT EDIT. -// github.com/vektra/mockery -// template: testify - -package syncing - -import ( - "context" - - "github.com/celestiaorg/go-header" - mock "github.com/stretchr/testify/mock" -) - -// NewMockHeightStore creates a new instance of MockHeightStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewMockHeightStore[H header.Header[H]](t interface { - mock.TestingT - Cleanup(func()) -}) *MockHeightStore[H] { - mock := &MockHeightStore[H]{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// MockHeightStore is an autogenerated mock type for the HeightStore type -type MockHeightStore[H header.Header[H]] struct { - mock.Mock -} - -type MockHeightStore_Expecter[H header.Header[H]] struct { - mock *mock.Mock -} - -func (_m *MockHeightStore[H]) EXPECT() *MockHeightStore_Expecter[H] { - return &MockHeightStore_Expecter[H]{mock: &_m.Mock} -} - -// GetByHeight provides a mock function for the type MockHeightStore -func (_mock *MockHeightStore[H]) GetByHeight(ctx context.Context, height uint64) (H, uint64, error) { - ret := _mock.Called(ctx, height) - - if len(ret) == 0 { - panic("no return value specified for GetByHeight") - } - - var r0 H - var r1 uint64 - var r2 error - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) (H, uint64, error)); ok { - return returnFunc(ctx, height) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) H); ok { - r0 = returnFunc(ctx, height) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(H) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, uint64) uint64); ok { - r1 = returnFunc(ctx, height) - } else { - r1 = ret.Get(1).(uint64) - } - if returnFunc, ok := ret.Get(2).(func(context.Context, uint64) error); ok { - r2 = returnFunc(ctx, height) - } else { - r2 = ret.Error(2) - } - return r0, r1, r2 -} - -// MockHeightStore_GetByHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByHeight' -type MockHeightStore_GetByHeight_Call[H header.Header[H]] struct { - *mock.Call -} - -// GetByHeight is a helper method to define mock.On call -// - ctx context.Context -// - height uint64 -func (_e *MockHeightStore_Expecter[H]) GetByHeight(ctx interface{}, height interface{}) *MockHeightStore_GetByHeight_Call[H] { - return &MockHeightStore_GetByHeight_Call[H]{Call: _e.mock.On("GetByHeight", ctx, height)} -} - -func (_c *MockHeightStore_GetByHeight_Call[H]) Run(run func(ctx context.Context, height uint64)) *MockHeightStore_GetByHeight_Call[H] { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 uint64 - if args[1] != nil { - arg1 = args[1].(uint64) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *MockHeightStore_GetByHeight_Call[H]) Return(v H, v1 uint64, err error) *MockHeightStore_GetByHeight_Call[H] { - _c.Call.Return(v, v1, err) - return _c -} - -func (_c *MockHeightStore_GetByHeight_Call[H]) RunAndReturn(run func(ctx context.Context, height uint64) (H, uint64, error)) *MockHeightStore_GetByHeight_Call[H] { - _c.Call.Return(run) - return _c -} diff --git a/block/internal/syncing/p2p_handler.go b/block/internal/syncing/p2p_handler.go index 3326583bc8..49bbaa3f12 100644 --- a/block/internal/syncing/p2p_handler.go +++ b/block/internal/syncing/p2p_handler.go @@ -7,12 +7,12 @@ import ( "sync/atomic" "github.com/celestiaorg/go-header" - "github.com/evstack/ev-node/types" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" "github.com/evstack/ev-node/pkg/genesis" + "github.com/evstack/ev-node/types" ) type p2pHandler interface { @@ -20,11 +20,6 @@ type p2pHandler interface { SetProcessedHeight(height uint64) } -// HeightStore is a subset of the sync service. -type HeightStore[H header.Header[H]] interface { - GetByHeight(ctx context.Context, height uint64) (H, uint64, error) -} - // P2PHandler coordinates block retrieval from P2P stores for the syncer. // It waits for both header and data to be available at a given height, // validates their consistency, and emits events to the syncer for processing. @@ -32,8 +27,8 @@ type HeightStore[H header.Header[H]] interface { // The handler maintains a processedHeight to track the highest block that has been // successfully validated and sent to the syncer, preventing duplicate processing. type P2PHandler struct { - headerStore HeightStore[*types.P2PSignedHeader] - dataStore HeightStore[*types.P2PData] + headerStore header.Store[*types.P2PSignedHeader] + dataStore header.Store[*types.P2PData] cache cache.CacheManager genesis genesis.Genesis logger zerolog.Logger @@ -43,8 +38,8 @@ type P2PHandler struct { // NewP2PHandler creates a new P2P handler. func NewP2PHandler( - headerStore HeightStore[*types.P2PSignedHeader], - dataStore HeightStore[*types.P2PData], + headerStore header.Store[*types.P2PSignedHeader], + dataStore header.Store[*types.P2PData], cache cache.CacheManager, genesis genesis.Genesis, logger zerolog.Logger, @@ -79,30 +74,28 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC return nil } - p2pHeader, headerDAHint, err := h.headerStore.GetByHeight(ctx, height) + p2pHeader, err := h.headerStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("header unavailable in store") } return err } - header := p2pHeader.Message - if err := h.assertExpectedProposer(header.ProposerAddress); err != nil { + if err := h.assertExpectedProposer(p2pHeader.ProposerAddress); err != nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("invalid header from P2P") return err } - p2pData, dataDAHint, err := h.dataStore.GetByHeight(ctx, height) + p2pData, err := h.dataStore.GetByHeight(ctx, height) if err != nil { if ctx.Err() == nil { h.logger.Debug().Uint64("height", height).Err(err).Msg("data unavailable in store") } return err } - data := p2pData.Message - dataCommitment := data.DACommitment() - if !bytes.Equal(header.DataHash[:], dataCommitment[:]) { - err := fmt.Errorf("data hash mismatch: header %x, data %x", header.DataHash, dataCommitment) + dataCommitment := p2pData.DACommitment() + if !bytes.Equal(p2pHeader.DataHash[:], dataCommitment[:]) { + err := fmt.Errorf("data hash mismatch: header %x, data %x", p2pHeader.DataHash, dataCommitment) h.logger.Warn().Uint64("height", height).Err(err).Msg("discarding inconsistent block from P2P") return err } @@ -110,10 +103,10 @@ func (h *P2PHandler) ProcessHeight(ctx context.Context, height uint64, heightInC // further header validation (signature) is done in validateBlock. // we need to be sure that the previous block n-1 was executed before validating block n event := common.DAHeightEvent{ - Header: header, - Data: data, + Header: p2pHeader.SignedHeader, + Data: p2pData.Data, Source: common.SourceP2P, - DaHeightHints: [2]uint64{headerDAHint, dataDAHint}, + DaHeightHints: [2]uint64{p2pHeader.DAHint(), p2pData.DAHint()}, } select { diff --git a/block/internal/syncing/p2p_handler_test.go b/block/internal/syncing/p2p_handler_test.go index 0e0604944b..40c6876d84 100644 --- a/block/internal/syncing/p2p_handler_test.go +++ b/block/internal/syncing/p2p_handler_test.go @@ -18,6 +18,7 @@ import ( "github.com/evstack/ev-node/pkg/genesis" signerpkg "github.com/evstack/ev-node/pkg/signer" "github.com/evstack/ev-node/pkg/signer/noop" + extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" ) @@ -50,14 +51,14 @@ func p2pMakeSignedHeader(t *testing.T, chainID string, height uint64, proposer [ sig, err := signer.Sign(bz) require.NoError(t, err, "failed to sign header bytes") hdr.Signature = sig - return &types.P2PSignedHeader{Message: hdr} + return &types.P2PSignedHeader{SignedHeader: hdr} } // P2PTestData aggregates dependencies used by P2P handler tests. type P2PTestData struct { Handler *P2PHandler - HeaderStore *MockHeightStore[*types.P2PSignedHeader] - DataStore *MockHeightStore[*types.P2PData] + HeaderStore *extmocks.MockStore[*types.P2PSignedHeader] + DataStore *extmocks.MockStore[*types.P2PData] Cache cache.CacheManager Genesis genesis.Genesis ProposerAddr []byte @@ -72,8 +73,8 @@ func setupP2P(t *testing.T) *P2PTestData { gen := genesis.Genesis{ChainID: "p2p-test", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: proposerAddr} - headerStoreMock := NewMockHeightStore[*types.P2PSignedHeader](t) - dataStoreMock := NewMockHeightStore[*types.P2PData](t) + headerStoreMock := extmocks.NewMockStore[*types.P2PSignedHeader](t) + dataStoreMock := extmocks.NewMockStore[*types.P2PData](t) cfg := config.Config{ RootDir: t.TempDir(), @@ -128,16 +129,16 @@ func TestP2PHandler_ProcessHeight_EmitsEventWhenHeaderAndDataPresent(t *testing. require.Equal(t, string(p.Genesis.ProposerAddress), string(p.ProposerAddr)) header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 5, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 5, 1)} - header.Message.DataHash = data.Message.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) + data := &types.P2PData{Data: makeData(p.Genesis.ChainID, 5, 1)} + header.DataHash = data.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Message.Signature = sig + header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(header, 0, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, 0, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(header, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(5)).Return(data, nil).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 5, ch) @@ -154,16 +155,16 @@ func TestP2PHandler_ProcessHeight_SkipsWhenDataMissing(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 7, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 7, 1)} - header.Message.DataHash = data.Message.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) + data := &types.P2PData{Data: makeData(p.Genesis.ChainID, 7, 1)} + header.DataHash = data.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Message.Signature = sig + header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(header, 0, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, 0, errors.New("missing")).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(header, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(7)).Return(nil, errors.New("missing")).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 7, ch) @@ -176,7 +177,7 @@ func TestP2PHandler_ProcessHeight_SkipsWhenHeaderMissing(t *testing.T) { p := setupP2P(t) ctx := context.Background() - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(9)).Return(nil, 0, errors.New("missing")).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(9)).Return(nil, errors.New("missing")).Once() ch := make(chan common.DAHeightEvent, 1) err := p.Handler.ProcessHeight(ctx, 9, ch) @@ -195,9 +196,9 @@ func TestP2PHandler_ProcessHeight_SkipsOnProposerMismatch(t *testing.T) { require.NotEqual(t, string(p.Genesis.ProposerAddress), string(badAddr)) header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 11, badAddr, pub, signer) - header.Message.DataHash = common.DataHashForEmptyTxs + header.DataHash = common.DataHashForEmptyTxs - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(header, 0, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(11)).Return(header, nil).Once() ch := make(chan common.DAHeightEvent, 1) err = p.Handler.ProcessHeight(ctx, 11, ch) @@ -224,16 +225,16 @@ func TestP2PHandler_ProcessedHeightSkipsPreviouslyHandledBlocks(t *testing.T) { // Height 6 should be fetched normally. header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 6, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 6, 1)} - header.Message.DataHash = data.Message.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) + data := &types.P2PData{Data: makeData(p.Genesis.ChainID, 6, 1)} + header.DataHash = data.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Message.Signature = sig + header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(header, 0, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, 0, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(header, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(6)).Return(data, nil).Once() require.NoError(t, p.Handler.ProcessHeight(ctx, 6, ch)) @@ -247,16 +248,16 @@ func TestP2PHandler_SetProcessedHeightPreventsDuplicates(t *testing.T) { ctx := context.Background() header := p2pMakeSignedHeader(t, p.Genesis.ChainID, 8, p.ProposerAddr, p.ProposerPub, p.Signer) - data := &types.P2PData{Message: makeData(p.Genesis.ChainID, 8, 0)} - header.Message.DataHash = data.Message.DACommitment() - bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Message.Header) + data := &types.P2PData{Data: makeData(p.Genesis.ChainID, 8, 0)} + header.DataHash = data.DACommitment() + bz, err := types.DefaultAggregatorNodeSignatureBytesProvider(&header.Header) require.NoError(t, err) sig, err := p.Signer.Sign(bz) require.NoError(t, err) - header.Message.Signature = sig + header.Signature = sig - p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(header, 0, nil).Once() - p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, 0, nil).Once() + p.HeaderStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(header, nil).Once() + p.DataStore.EXPECT().GetByHeight(mock.Anything, uint64(8)).Return(data, nil).Once() ch := make(chan common.DAHeightEvent, 1) require.NoError(t, p.Handler.ProcessHeight(ctx, 8, ch)) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index f28e8339eb..105f589600 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -14,16 +14,17 @@ import ( "sync/atomic" "time" + "github.com/celestiaorg/go-header" coreexecutor "github.com/evstack/ev-node/core/execution" - datypes "github.com/evstack/ev-node/pkg/da/types" - "github.com/evstack/ev-node/pkg/raft" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" "github.com/evstack/ev-node/block/internal/common" "github.com/evstack/ev-node/block/internal/da" "github.com/evstack/ev-node/pkg/config" + datypes "github.com/evstack/ev-node/pkg/da/types" "github.com/evstack/ev-node/pkg/genesis" + "github.com/evstack/ev-node/pkg/raft" "github.com/evstack/ev-node/pkg/store" "github.com/evstack/ev-node/types" ) @@ -95,8 +96,8 @@ type Syncer struct { daRetrieverHeight *atomic.Uint64 // P2P stores - headerStore common.HeaderP2PBroadcaster - dataStore common.DataP2PBroadcaster + headerStore header.Store[*types.P2PSignedHeader] + dataStore header.Store[*types.P2PData] // Channels for coordination heightInCh chan common.DAHeightEvent @@ -147,8 +148,8 @@ func NewSyncer( metrics *common.Metrics, config config.Config, genesis genesis.Genesis, - headerStore common.HeaderP2PBroadcaster, - dataStore common.DataP2PBroadcaster, + headerStore header.Store[*types.P2PSignedHeader], + dataStore header.Store[*types.P2PData], logger zerolog.Logger, options common.BlockOptions, errorCh chan<- error, diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 3c9be5f137..8651b8c784 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -326,8 +326,8 @@ func setupTestSyncer(t *testing.T, daBlockTime time.Duration) *Syncer { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 5217bb3043..5accde5a25 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -12,6 +12,7 @@ import ( "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/store" testmocks "github.com/evstack/ev-node/test/mocks" + extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -154,9 +155,9 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay mockP2P := newMockp2pHandler(b) // not used directly in this benchmark path mockP2P.On("SetProcessedHeight", mock.Anything).Return().Maybe() s.p2pHandler = mockP2P - headerP2PStore := common.NewMockBroadcaster[*types.P2PSignedHeader](b) + headerP2PStore := extmocks.NewMockStore[*types.P2PSignedHeader](b) s.headerStore = headerP2PStore - dataP2PStore := common.NewMockBroadcaster[*types.P2PData](b) + dataP2PStore := extmocks.NewMockStore[*types.P2PData](b) s.dataStore = dataP2PStore return &benchFixture{s: s, st: st, cm: cm, cancel: cancel} } diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index a76b2b9ced..075b6f1694 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -21,6 +21,7 @@ import ( "github.com/evstack/ev-node/pkg/genesis" "github.com/evstack/ev-node/pkg/store" testmocks "github.com/evstack/ev-node/test/mocks" + extmocks "github.com/evstack/ev-node/test/mocks/external" "github.com/evstack/ev-node/types" ) @@ -399,8 +400,8 @@ func TestVerifyForcedInclusionTxs_AllTransactionsIncluded(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -475,8 +476,8 @@ func TestVerifyForcedInclusionTxs_MissingTransactions(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -581,8 +582,8 @@ func TestVerifyForcedInclusionTxs_PartiallyIncluded(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -691,8 +692,8 @@ func TestVerifyForcedInclusionTxs_NoForcedTransactions(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -761,8 +762,8 @@ func TestVerifyForcedInclusionTxs_NamespaceNotConfigured(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -829,8 +830,8 @@ func TestVerifyForcedInclusionTxs_DeferralWithinEpoch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -955,8 +956,8 @@ func TestVerifyForcedInclusionTxs_MaliciousAfterEpochEnd(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -1047,8 +1048,8 @@ func TestVerifyForcedInclusionTxs_SmoothingExceedsEpoch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 9f6957d80b..8543e4e794 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -123,8 +123,8 @@ func TestSyncer_validateBlock_DataHashMismatch(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), @@ -175,8 +175,8 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), errChan, @@ -230,8 +230,8 @@ func TestSequentialBlockSync(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), errChan, @@ -349,8 +349,8 @@ func TestSyncLoopPersistState(t *testing.T) { mockDataStore := extmocks.NewMockStore[*types.Data](t) mockDataStore.EXPECT().Height().Return(uint64(0)).Maybe() - mockP2PHeaderStore := common.NewMockBroadcaster[*types.P2PSignedHeader](t) - mockP2PDataStore := common.NewMockBroadcaster[*types.P2PData](t) + mockP2PHeaderStore := extmocks.NewMockStore[*types.P2PSignedHeader](t) + mockP2PDataStore := extmocks.NewMockStore[*types.P2PData](t) errorCh := make(chan error, 1) syncerInst1 := NewSyncer( @@ -731,8 +731,8 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { common.NopMetrics(), cfg, gen, - common.NewMockBroadcaster[*types.P2PSignedHeader](t), - common.NewMockBroadcaster[*types.P2PData](t), + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), zerolog.Nop(), common.DefaultBlockOptions(), make(chan error, 1), diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index 3841156635..31aa938b04 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -253,7 +253,7 @@ func TestClientGetNamespace(t *testing.T) { func testSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { return &types.P2PSignedHeader{ - Message: &types.SignedHeader{ + SignedHeader: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -270,7 +270,7 @@ func testSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { func testData(height uint64, ts time.Time) *types.P2PData { return &types.P2PData{ - Message: &types.Data{ + Data: &types.Data{ Metadata: &types.Metadata{ ChainID: "test-chain", Height: height, diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index 72830e85bd..ddd50b550e 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -629,7 +629,7 @@ func TestHealthReadyEndpoint(t *testing.T) { func makeTestSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { return &types.P2PSignedHeader{ - Message: &types.SignedHeader{ + SignedHeader: &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ Height: height, @@ -646,7 +646,7 @@ func makeTestSignedHeader(height uint64, ts time.Time) *types.P2PSignedHeader { func makeTestData(height uint64, ts time.Time) *types.P2PData { return &types.P2PData{ - Message: &types.Data{ + Data: &types.Data{ Metadata: &types.Metadata{ ChainID: "test-chain", Height: height, diff --git a/pkg/store/data_store_adapter_test.go b/pkg/store/data_store_adapter_test.go index bde29ce1ed..c05db183c4 100644 --- a/pkg/store/data_store_adapter_test.go +++ b/pkg/store/data_store_adapter_test.go @@ -39,7 +39,7 @@ func wrapData(d *types.Data) *types.P2PData { return nil } return &types.P2PData{ - Message: d, + Data: d, DAHeightHint: 0, } } diff --git a/pkg/store/header_store_adapter_test.go b/pkg/store/header_store_adapter_test.go index 04e238c9c5..12635f097d 100644 --- a/pkg/store/header_store_adapter_test.go +++ b/pkg/store/header_store_adapter_test.go @@ -37,7 +37,7 @@ func wrapHeader(h *types.SignedHeader) *types.P2PSignedHeader { return nil } return &types.P2PSignedHeader{ - Message: h, + SignedHeader: h, DAHeightHint: 0, } } diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 45bf175a79..3d3395d377 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -589,7 +589,7 @@ func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*ty daHint, _ := getDAHintFromStore(ctx, g.store, height) return &types.P2PSignedHeader{ - Message: header, + SignedHeader: header, DAHeightHint: daHint, }, nil } @@ -614,7 +614,7 @@ func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types. daHint, _ := getDAHintFromStore(ctx, g.store, hdr.Height()) return &types.P2PSignedHeader{ - Message: hdr, + SignedHeader: hdr, DAHeightHint: daHint, }, nil } @@ -650,7 +650,7 @@ func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*type daHint, _ := getDAHintFromStore(ctx, g.store, height) return &types.P2PData{ - Message: data, + Data: data, DAHeightHint: daHint, }, nil } @@ -675,7 +675,7 @@ func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2 daHint, _ := getDAHintFromStore(ctx, g.store, data.Height()) return &types.P2PData{ - Message: data, + Data: data, DAHeightHint: daHint, }, nil } diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index be9bfcfda3..27344fe50c 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -179,15 +179,6 @@ func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, hash return s.store.Append(ctx, entries...) } -func (s *SyncService[H]) GetByHeight(ctx context.Context, height uint64) (H, uint64, error) { - c, err := s.store.GetByHeight(ctx, height) - if err != nil { - var zero H - return zero, 0, err - } - return c, c.DAHint(), nil -} - // Start is a part of Service interface. func (syncService *SyncService[H]) Start(ctx context.Context) error { // setup P2P infrastructure, but don't start Subscriber yet. diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 640e291c53..9d89861bfd 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -75,12 +75,12 @@ func TestHeaderSyncServiceRestart(t *testing.T) { signedHeader, err := types.GetRandomSignedHeaderCustom(&headerConfig, genesisDoc.ChainID) require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) for i := genesisDoc.InitialHeight + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) } // then stop and restart service @@ -111,7 +111,7 @@ func TestHeaderSyncServiceRestart(t *testing.T) { for i := signedHeader.Height() + 1; i < 2; i++ { signedHeader = nextHeader(t, signedHeader, genesisDoc.ChainID, noopSigner) t.Logf("signed header: %d", i) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) } cancel() } @@ -167,7 +167,7 @@ func TestHeaderSyncServiceInitFromHigherHeight(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) + require.NoError(t, svc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) } func TestDAHintStorageHeader(t *testing.T) { @@ -219,15 +219,15 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, err) require.NoError(t, signedHeader.Validate()) - require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{Message: signedHeader})) + require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) daHeight := uint64(100) require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Hash())) - h, hint, err := headerSvc.GetByHeight(ctx, signedHeader.Height()) + h, err := headerSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) require.Equal(t, signedHeader.Hash(), h.Hash()) - require.Equal(t, daHeight, hint) + require.Equal(t, daHeight, h.DAHint()) _ = p2pClient.Close() _ = headerSvc.Stop(ctx) @@ -249,10 +249,10 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, headerSvc.Start(ctx)) t.Cleanup(func() { _ = headerSvc.Stop(context.Background()) }) - h, hint, err = headerSvc.GetByHeight(ctx, signedHeader.Height()) + h, err = headerSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) require.Equal(t, signedHeader.Hash(), h.Hash()) - require.Equal(t, daHeight, hint) + require.Equal(t, daHeight, h.DAHint()) } func TestDAHintStorageData(t *testing.T) { @@ -311,15 +311,15 @@ func TestDAHintStorageData(t *testing.T) { }, } - require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Message: &data})) + require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: &data})) daHeight := uint64(100) require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Hash())) - d, hint, err := dataSvc.GetByHeight(ctx, signedHeader.Height()) + d, err := dataSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) require.Equal(t, data.Hash(), d.Hash()) - require.Equal(t, daHeight, hint) + require.Equal(t, daHeight, d.DAHint()) _ = p2pClient.Close() _ = dataSvc.Stop(ctx) @@ -341,10 +341,10 @@ func TestDAHintStorageData(t *testing.T) { require.NoError(t, dataSvc.Start(ctx)) t.Cleanup(func() { _ = dataSvc.Stop(context.Background()) }) - d, hint, err = dataSvc.GetByHeight(ctx, signedHeader.Height()) + d, err = dataSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) require.Equal(t, data.Hash(), d.Hash()) - require.Equal(t, daHeight, hint) + require.Equal(t, daHeight, d.DAHint()) } func nextHeader(t *testing.T, previousHeader *types.SignedHeader, chainID string, noopSigner signer.Signer) *types.SignedHeader { diff --git a/types/p2p_envelope.go b/types/p2p_envelope.go index eb76e9f896..4386d40eff 100644 --- a/types/p2p_envelope.go +++ b/types/p2p_envelope.go @@ -1,7 +1,6 @@ package types import ( - "fmt" "time" "github.com/celestiaorg/go-header" @@ -10,144 +9,170 @@ import ( pb "github.com/evstack/ev-node/types/pb/evnode/v1" ) -type ( - P2PSignedHeader = P2PEnvelope[*SignedHeader] - P2PData = P2PEnvelope[*Data] -) - var ( _ header.Header[*P2PData] = &P2PData{} _ header.Header[*P2PSignedHeader] = &P2PSignedHeader{} ) -// P2PEnvelope is a generic envelope for P2P messages that includes a DA height hint. -type P2PEnvelope[H header.Header[H]] struct { - Message H +// P2PSignedHeader wraps SignedHeader with an optional DA height hint for P2P sync optimization. +type P2PSignedHeader struct { + *SignedHeader DAHeightHint uint64 } -// New creates a new P2PEnvelope. -func (e *P2PEnvelope[H]) New() *P2PEnvelope[H] { - var empty H - return &P2PEnvelope[H]{Message: empty.New()} +// New creates a new P2PSignedHeader. +func (p *P2PSignedHeader) New() *P2PSignedHeader { + return &P2PSignedHeader{SignedHeader: new(SignedHeader)} } -// IsZero checks if the envelope or its message is zero. -func (e *P2PEnvelope[H]) IsZero() bool { - return e == nil || e.Message.IsZero() +// IsZero checks if the header is nil or zero. +func (p *P2PSignedHeader) IsZero() bool { + return p == nil || p.SignedHeader == nil || p.SignedHeader.IsZero() } // SetDAHint sets the DA height hint. -func (e *P2PEnvelope[H]) SetDAHint(daHeight uint64) { - e.DAHeightHint = daHeight +func (p *P2PSignedHeader) SetDAHint(daHeight uint64) { + p.DAHeightHint = daHeight } // DAHint returns the DA height hint. -func (e *P2PEnvelope[H]) DAHint() uint64 { - return e.DAHeightHint +func (p *P2PSignedHeader) DAHint() uint64 { + return p.DAHeightHint } -// Verify verifies the envelope message against an untrusted envelope. -func (e *P2PEnvelope[H]) Verify(untrst *P2PEnvelope[H]) error { - return e.Message.Verify(untrst.Message) +// Verify verifies against an untrusted header. +func (p *P2PSignedHeader) Verify(untrusted *P2PSignedHeader) error { + return p.SignedHeader.Verify(untrusted.SignedHeader) } -// ChainID returns the ChainID of the message. -func (e *P2PEnvelope[H]) ChainID() string { - return e.Message.ChainID() +// MarshalBinary marshals the header to binary using P2P protobuf format. +func (p *P2PSignedHeader) MarshalBinary() ([]byte, error) { + psh, err := p.SignedHeader.ToProto() + if err != nil { + return nil, err + } + msg := &pb.P2PSignedHeader{ + Header: psh.Header, + Signature: psh.Signature, + Signer: psh.Signer, + DaHeightHint: &p.DAHeightHint, + } + return proto.Marshal(msg) } -// Height returns the Height of the message. -func (e *P2PEnvelope[H]) Height() uint64 { - return e.Message.Height() +// UnmarshalBinary unmarshals the header from binary using P2P protobuf format. +func (p *P2PSignedHeader) UnmarshalBinary(data []byte) error { + var msg pb.P2PSignedHeader + if err := proto.Unmarshal(data, &msg); err != nil { + return err + } + psh := &pb.SignedHeader{ + Header: msg.Header, + Signature: msg.Signature, + Signer: msg.Signer, + } + if p.SignedHeader == nil { + p.SignedHeader = new(SignedHeader) + } + if err := p.SignedHeader.FromProto(psh); err != nil { + return err + } + if msg.DaHeightHint != nil { + p.DAHeightHint = *msg.DaHeightHint + } + return nil } -// LastHeader returns the LastHeader hash of the message. -func (e *P2PEnvelope[H]) LastHeader() Hash { - return e.Message.LastHeader() +// P2PData wraps Data with an optional DA height hint for P2P sync optimization. +type P2PData struct { + *Data + DAHeightHint uint64 +} + +// New creates a new P2PData. +func (p *P2PData) New() *P2PData { + return &P2PData{Data: new(Data)} +} + +// IsZero checks if the data is nil or zero. +func (p *P2PData) IsZero() bool { + return p == nil || p.Data == nil || p.Data.IsZero() } -// Time returns the Time of the message. -func (e *P2PEnvelope[H]) Time() time.Time { - return e.Message.Time() +// SetDAHint sets the DA height hint. +func (p *P2PData) SetDAHint(daHeight uint64) { + p.DAHeightHint = daHeight } -// Hash returns the hash of the message. -func (e *P2PEnvelope[H]) Hash() Hash { - return e.Message.Hash() +// DAHint returns the DA height hint. +func (p *P2PData) DAHint() uint64 { + return p.DAHeightHint } -// Validate performs basic validation on the message. -func (e *P2PEnvelope[H]) Validate() error { - return e.Message.Validate() +// Verify verifies against untrusted data. +func (p *P2PData) Verify(untrusted *P2PData) error { + return p.Data.Verify(untrusted.Data) } -// MarshalBinary marshals the envelope to binary. -func (e *P2PEnvelope[H]) MarshalBinary() ([]byte, error) { - var mirrorPb proto.Message +// ChainID returns chain ID of the data. +func (p *P2PData) ChainID() string { + return p.Data.ChainID() +} - switch msg := any(e.Message).(type) { - case *Data: - pData := msg.ToProto() - mirrorPb = &pb.P2PData{ - Metadata: pData.Metadata, - Txs: pData.Txs, - DaHeightHint: &e.DAHeightHint, - } - case *SignedHeader: - psh, err := msg.ToProto() - if err != nil { - return nil, err - } - mirrorPb = &pb.P2PSignedHeader{ - Header: psh.Header, - Signature: psh.Signature, - Signer: psh.Signer, - DaHeightHint: &e.DAHeightHint, - } - default: - return nil, fmt.Errorf("unsupported type for toProto: %T", msg) +// Height returns height of the data. +func (p *P2PData) Height() uint64 { + return p.Data.Height() +} + +// LastHeader returns last header hash of the data. +func (p *P2PData) LastHeader() Hash { + return p.Data.LastHeader() +} + +// Time returns time of the data. +func (p *P2PData) Time() time.Time { + return p.Data.Time() +} + +// Hash returns the hash of the data. +func (p *P2PData) Hash() Hash { + return p.Data.Hash() +} + +// Validate performs basic validation on the data. +func (p *P2PData) Validate() error { + return p.Data.Validate() +} + +// MarshalBinary marshals the data to binary using P2P protobuf format. +func (p *P2PData) MarshalBinary() ([]byte, error) { + pData := p.Data.ToProto() + msg := &pb.P2PData{ + Metadata: pData.Metadata, + Txs: pData.Txs, + DaHeightHint: &p.DAHeightHint, + } + return proto.Marshal(msg) +} + +// UnmarshalBinary unmarshals the data from binary using P2P protobuf format. +func (p *P2PData) UnmarshalBinary(data []byte) error { + var msg pb.P2PData + if err := proto.Unmarshal(data, &msg); err != nil { + return err + } + pData := &pb.Data{ + Metadata: msg.Metadata, + Txs: msg.Txs, + } + if p.Data == nil { + p.Data = new(Data) + } + if err := p.Data.FromProto(pData); err != nil { + return err } - return proto.Marshal(mirrorPb) -} - -// UnmarshalBinary unmarshals the envelope from binary. -func (e *P2PEnvelope[H]) UnmarshalBinary(data []byte) error { - switch target := any(e.Message).(type) { - case *Data: - var pData pb.P2PData - if err := proto.Unmarshal(data, &pData); err != nil { - return err - } - mirrorData := &pb.Data{ - Metadata: pData.Metadata, - Txs: pData.Txs, - } - if err := target.FromProto(mirrorData); err != nil { - return err - } - if pData.DaHeightHint != nil { - e.DAHeightHint = *pData.DaHeightHint - } - return nil - case *SignedHeader: - var pHeader pb.P2PSignedHeader - if err := proto.Unmarshal(data, &pHeader); err != nil { - return err - } - psh := &pb.SignedHeader{ - Header: pHeader.Header, - Signature: pHeader.Signature, - Signer: pHeader.Signer, - } - if err := target.FromProto(psh); err != nil { - return err - } - if pHeader.DaHeightHint != nil { - e.DAHeightHint = *pHeader.DaHeightHint - } - return nil - default: - return fmt.Errorf("unsupported type for UnmarshalBinary: %T", target) + if msg.DaHeightHint != nil { + p.DAHeightHint = *msg.DaHeightHint } + return nil } diff --git a/types/p2p_envelope_test.go b/types/p2p_envelope_test.go index 3dc2127fed..50485cb146 100644 --- a/types/p2p_envelope_test.go +++ b/types/p2p_envelope_test.go @@ -22,24 +22,24 @@ func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { Txs: Txs{[]byte{0x1}, []byte{0x2}}, } envelope := &P2PData{ - Message: data, + Data: data, DAHeightHint: 100, } // Marshaling - bytes, err := envelope.MarshalBinary() + bz, err := envelope.MarshalBinary() require.NoError(t, err) - assert.NotEmpty(t, bytes) + assert.NotEmpty(t, bz) // Unmarshaling newEnvelope := (&P2PData{}).New() - err = newEnvelope.UnmarshalBinary(bytes) + err = newEnvelope.UnmarshalBinary(bz) require.NoError(t, err) assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) - assert.Equal(t, envelope.Message.Height(), newEnvelope.Message.Height()) - assert.Equal(t, envelope.Message.ChainID(), newEnvelope.Message.ChainID()) - assert.Equal(t, envelope.Message.LastDataHash, newEnvelope.Message.LastDataHash) - assert.Equal(t, envelope.Message.Txs, newEnvelope.Message.Txs) + assert.Equal(t, envelope.Data.Height(), newEnvelope.Data.Height()) + assert.Equal(t, envelope.Data.ChainID(), newEnvelope.Data.ChainID()) + assert.Equal(t, envelope.Data.LastDataHash, newEnvelope.Data.LastDataHash) + assert.Equal(t, envelope.Data.Txs, newEnvelope.Data.Txs) } func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { @@ -71,7 +71,7 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { } envelope := &P2PSignedHeader{ - Message: header, + SignedHeader: header, DAHeightHint: 200, } @@ -85,23 +85,23 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { err = newEnvelope.UnmarshalBinary(bz) require.NoError(t, err) assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) - assert.Equal(t, envelope.Message.Signer, newEnvelope.Message.Signer) + assert.Equal(t, envelope.SignedHeader.Signer, newEnvelope.SignedHeader.Signer) assert.Equal(t, envelope, newEnvelope) } func TestSignedHeaderBinaryCompatibility(t *testing.T) { signedHeader, _, err := GetRandomSignedHeader("chain-id") require.NoError(t, err) - bytes, err := signedHeader.MarshalBinary() + bz, err := signedHeader.MarshalBinary() require.NoError(t, err) p2pHeader := (&P2PSignedHeader{}).New() - err = p2pHeader.UnmarshalBinary(bytes) + err = p2pHeader.UnmarshalBinary(bz) require.NoError(t, err) - assert.Equal(t, signedHeader.Header, p2pHeader.Message.Header) - assert.Equal(t, signedHeader.Signature, p2pHeader.Message.Signature) - assert.Equal(t, signedHeader.Signer, p2pHeader.Message.Signer) + assert.Equal(t, signedHeader.Header, p2pHeader.SignedHeader.Header) + assert.Equal(t, signedHeader.Signature, p2pHeader.SignedHeader.Signature) + assert.Equal(t, signedHeader.Signer, p2pHeader.SignedHeader.Signer) assert.Zero(t, p2pHeader.DAHeightHint) p2pHeader.DAHeightHint = 100 @@ -129,15 +129,15 @@ func TestDataBinaryCompatibility(t *testing.T) { []byte("tx2"), }, } - bytes, err := data.MarshalBinary() + bz, err := data.MarshalBinary() require.NoError(t, err) p2pData := (&P2PData{}).New() - err = p2pData.UnmarshalBinary(bytes) + err = p2pData.UnmarshalBinary(bz) require.NoError(t, err) - assert.Equal(t, data.Metadata, p2pData.Message.Metadata) - assert.Equal(t, data.Txs, p2pData.Message.Txs) + assert.Equal(t, data.Metadata, p2pData.Data.Metadata) + assert.Equal(t, data.Txs, p2pData.Data.Txs) assert.Zero(t, p2pData.DAHeightHint) p2pData.DAHeightHint = 200 From c9d3251550de4a842c40b7e09f416e2549de76b6 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 16:42:08 +0100 Subject: [PATCH 25/41] linting --- types/p2p_envelope.go | 8 ++++---- types/p2p_envelope_test.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/types/p2p_envelope.go b/types/p2p_envelope.go index 4386d40eff..d5c5957eef 100644 --- a/types/p2p_envelope.go +++ b/types/p2p_envelope.go @@ -47,7 +47,7 @@ func (p *P2PSignedHeader) Verify(untrusted *P2PSignedHeader) error { // MarshalBinary marshals the header to binary using P2P protobuf format. func (p *P2PSignedHeader) MarshalBinary() ([]byte, error) { - psh, err := p.SignedHeader.ToProto() + psh, err := p.ToProto() if err != nil { return nil, err } @@ -74,7 +74,7 @@ func (p *P2PSignedHeader) UnmarshalBinary(data []byte) error { if p.SignedHeader == nil { p.SignedHeader = new(SignedHeader) } - if err := p.SignedHeader.FromProto(psh); err != nil { + if err := p.FromProto(psh); err != nil { return err } if msg.DaHeightHint != nil { @@ -146,7 +146,7 @@ func (p *P2PData) Validate() error { // MarshalBinary marshals the data to binary using P2P protobuf format. func (p *P2PData) MarshalBinary() ([]byte, error) { - pData := p.Data.ToProto() + pData := p.ToProto() msg := &pb.P2PData{ Metadata: pData.Metadata, Txs: pData.Txs, @@ -168,7 +168,7 @@ func (p *P2PData) UnmarshalBinary(data []byte) error { if p.Data == nil { p.Data = new(Data) } - if err := p.Data.FromProto(pData); err != nil { + if err := p.FromProto(pData); err != nil { return err } if msg.DaHeightHint != nil { diff --git a/types/p2p_envelope_test.go b/types/p2p_envelope_test.go index 50485cb146..01ffd61b74 100644 --- a/types/p2p_envelope_test.go +++ b/types/p2p_envelope_test.go @@ -38,8 +38,8 @@ func TestP2PEnvelope_MarshalUnmarshal(t *testing.T) { assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) assert.Equal(t, envelope.Data.Height(), newEnvelope.Data.Height()) assert.Equal(t, envelope.Data.ChainID(), newEnvelope.Data.ChainID()) - assert.Equal(t, envelope.Data.LastDataHash, newEnvelope.Data.LastDataHash) - assert.Equal(t, envelope.Data.Txs, newEnvelope.Data.Txs) + assert.Equal(t, envelope.LastDataHash, newEnvelope.LastDataHash) + assert.Equal(t, envelope.Txs, newEnvelope.Txs) } func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { @@ -85,7 +85,7 @@ func TestP2PSignedHeader_MarshalUnmarshal(t *testing.T) { err = newEnvelope.UnmarshalBinary(bz) require.NoError(t, err) assert.Equal(t, envelope.DAHeightHint, newEnvelope.DAHeightHint) - assert.Equal(t, envelope.SignedHeader.Signer, newEnvelope.SignedHeader.Signer) + assert.Equal(t, envelope.Signer, newEnvelope.Signer) assert.Equal(t, envelope, newEnvelope) } @@ -99,9 +99,9 @@ func TestSignedHeaderBinaryCompatibility(t *testing.T) { err = p2pHeader.UnmarshalBinary(bz) require.NoError(t, err) - assert.Equal(t, signedHeader.Header, p2pHeader.SignedHeader.Header) - assert.Equal(t, signedHeader.Signature, p2pHeader.SignedHeader.Signature) - assert.Equal(t, signedHeader.Signer, p2pHeader.SignedHeader.Signer) + assert.Equal(t, signedHeader.Header, p2pHeader.Header) + assert.Equal(t, signedHeader.Signature, p2pHeader.Signature) + assert.Equal(t, signedHeader.Signer, p2pHeader.Signer) assert.Zero(t, p2pHeader.DAHeightHint) p2pHeader.DAHeightHint = 100 @@ -136,8 +136,8 @@ func TestDataBinaryCompatibility(t *testing.T) { err = p2pData.UnmarshalBinary(bz) require.NoError(t, err) - assert.Equal(t, data.Metadata, p2pData.Data.Metadata) - assert.Equal(t, data.Txs, p2pData.Data.Txs) + assert.Equal(t, data.Metadata, p2pData.Metadata) + assert.Equal(t, data.Txs, p2pData.Txs) assert.Zero(t, p2pData.DAHeightHint) p2pData.DAHeightHint = 200 From fcb3e7548e0b6ec401b69e4bf81bb183c54df131 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 16:56:30 +0100 Subject: [PATCH 26/41] use same key as sequencer --- pkg/store/store_adapter.go | 59 +++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 3d3395d377..166a54ea66 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -586,7 +586,7 @@ func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*ty return nil, err } - daHint, _ := getDAHintFromStore(ctx, g.store, height) + daHint, _ := g.GetDAHint(ctx, height) return &types.P2PSignedHeader{ SignedHeader: header, @@ -596,12 +596,21 @@ func (g *HeaderStoreGetter) GetByHeight(ctx context.Context, height uint64) (*ty // GetDAHint implements StoreGetter. func (g *HeaderStoreGetter) GetDAHint(ctx context.Context, height uint64) (uint64, error) { - return getDAHintFromStore(ctx, g.store, height) + data, err := g.store.GetMetadata(ctx, GetHeightToDAHeightHeaderKey(height)) + if err != nil { + return 0, err + } + if len(data) != 8 { + return 0, fmt.Errorf("invalid da hint data length: %d", len(data)) + } + return binary.LittleEndian.Uint64(data), nil } // SetDAHint implements StoreGetter. func (g *HeaderStoreGetter) SetDAHint(ctx context.Context, height uint64, daHint uint64) error { - return setDAHintToStore(ctx, g.store, height, daHint) + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, daHint) + return g.store.SetMetadata(ctx, GetHeightToDAHeightHeaderKey(height), data) } // GetByHash implements StoreGetter. @@ -611,7 +620,7 @@ func (g *HeaderStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types. return nil, err } - daHint, _ := getDAHintFromStore(ctx, g.store, hdr.Height()) + daHint, _ := g.GetDAHint(ctx, hdr.Height()) return &types.P2PSignedHeader{ SignedHeader: hdr, @@ -647,7 +656,7 @@ func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*type return nil, err } - daHint, _ := getDAHintFromStore(ctx, g.store, height) + daHint, _ := g.GetDAHint(ctx, height) return &types.P2PData{ Data: data, @@ -657,12 +666,21 @@ func (g *DataStoreGetter) GetByHeight(ctx context.Context, height uint64) (*type // GetDAHint implements StoreGetter. func (g *DataStoreGetter) GetDAHint(ctx context.Context, height uint64) (uint64, error) { - return getDAHintFromStore(ctx, g.store, height) + data, err := g.store.GetMetadata(ctx, GetHeightToDAHeightDataKey(height)) + if err != nil { + return 0, err + } + if len(data) != 8 { + return 0, fmt.Errorf("invalid da hint data length: %d", len(data)) + } + return binary.LittleEndian.Uint64(data), nil } // SetDAHint implements StoreGetter. func (g *DataStoreGetter) SetDAHint(ctx context.Context, height uint64, daHint uint64) error { - return setDAHintToStore(ctx, g.store, height, daHint) + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, daHint) + return g.store.SetMetadata(ctx, GetHeightToDAHeightDataKey(height), data) } // GetByHash implements StoreGetter. @@ -672,7 +690,7 @@ func (g *DataStoreGetter) GetByHash(ctx context.Context, hash []byte) (*types.P2 return nil, err } - daHint, _ := getDAHintFromStore(ctx, g.store, data.Height()) + daHint, _ := g.GetDAHint(ctx, data.Height()) return &types.P2PData{ Data: data, @@ -706,28 +724,3 @@ func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { return NewStoreAdapter(NewDataStoreGetter(store), gen) } - -// DA hint persistence helpers - -const daHintKeyPrefix = "da_hint:" - -func daHintKey(height uint64) string { - return fmt.Sprintf("%s%d", daHintKeyPrefix, height) -} - -func getDAHintFromStore(ctx context.Context, store Store, height uint64) (uint64, error) { - data, err := store.GetMetadata(ctx, daHintKey(height)) - if err != nil { - return 0, err - } - if len(data) != 8 { - return 0, fmt.Errorf("invalid da hint data length: %d", len(data)) - } - return binary.BigEndian.Uint64(data), nil -} - -func setDAHintToStore(ctx context.Context, store Store, height uint64, daHint uint64) error { - data := make([]byte, 8) - binary.BigEndian.PutUint64(data, daHint) - return store.SetMetadata(ctx, daHintKey(height), data) -} From 1b3802eaabd14cf202d6787f789d1ad0996a6dc2 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 17:02:35 +0100 Subject: [PATCH 27/41] fix unit test --- pkg/sync/sync_service_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 9d89861bfd..3f7b0e529c 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -229,6 +229,15 @@ func TestDAHintStorageHeader(t *testing.T) { require.Equal(t, signedHeader.Hash(), h.Hash()) require.Equal(t, daHeight, h.DAHint()) + // Persist header to underlying store so it survives restart + // (WriteToStoreAndBroadcast only writes to P2P pending cache) + data := &types.Data{Metadata: &types.Metadata{Height: signedHeader.Height()}} + batch, err := evStore.NewBatch(ctx) + require.NoError(t, err) + require.NoError(t, batch.SaveBlockData(signedHeader, data, &signedHeader.Signature)) + require.NoError(t, batch.SetHeight(signedHeader.Height())) + require.NoError(t, batch.Commit()) + _ = p2pClient.Close() _ = headerSvc.Stop(ctx) cancel() @@ -321,6 +330,14 @@ func TestDAHintStorageData(t *testing.T) { require.Equal(t, data.Hash(), d.Hash()) require.Equal(t, daHeight, d.DAHint()) + // Persist data to underlying store so it survives restart + // (WriteToStoreAndBroadcast only writes to P2P pending cache) + batch, err := evStore.NewBatch(ctx) + require.NoError(t, err) + require.NoError(t, batch.SaveBlockData(signedHeader, &data, &signedHeader.Signature)) + require.NoError(t, batch.SetHeight(signedHeader.Height())) + require.NoError(t, batch.Commit()) + _ = p2pClient.Close() _ = dataSvc.Stop(ctx) cancel() From aac3f47928eb98fc7d6f22acdded33b26b739df9 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 17:07:50 +0100 Subject: [PATCH 28/41] rename working pool to avoid naming confusion --- ...triever.go => da_retrieval_worker_pool.go} | 32 ++++++++++--------- ...st.go => da_retrieval_worker_pool_test.go} | 28 ++++++++-------- block/internal/syncing/syncer.go | 10 +++--- block/internal/syncing/syncer_test.go | 12 +++---- 4 files changed, 42 insertions(+), 40 deletions(-) rename block/internal/syncing/{async_da_retriever.go => da_retrieval_worker_pool.go} (63%) rename block/internal/syncing/{async_da_retriever_test.go => da_retrieval_worker_pool_test.go} (84%) diff --git a/block/internal/syncing/async_da_retriever.go b/block/internal/syncing/da_retrieval_worker_pool.go similarity index 63% rename from block/internal/syncing/async_da_retriever.go rename to block/internal/syncing/da_retrieval_worker_pool.go index 7c79a37512..e99cb67e0d 100644 --- a/block/internal/syncing/async_da_retriever.go +++ b/block/internal/syncing/da_retrieval_worker_pool.go @@ -8,8 +8,10 @@ import ( "github.com/rs/zerolog" ) -// AsyncDARetriever handles concurrent DA retrieval operations. -type AsyncDARetriever struct { +// DARetrievalWorkerPool handles concurrent on-demand DA retrieval operations +// triggered by P2P hints. Unlike the background prefetcher (AsyncBlockRetriever), +// this worker pool processes explicit retrieval requests for specific DA heights. +type DARetrievalWorkerPool struct { retriever DARetriever resultCh chan<- common.DAHeightEvent workCh chan uint64 @@ -21,44 +23,44 @@ type AsyncDARetriever struct { cancel context.CancelFunc } -// NewAsyncDARetriever creates a new AsyncDARetriever. -func NewAsyncDARetriever( +// NewDARetrievalWorkerPool creates a new DARetrievalWorkerPool. +func NewDARetrievalWorkerPool( retriever DARetriever, resultCh chan<- common.DAHeightEvent, logger zerolog.Logger, -) *AsyncDARetriever { - return &AsyncDARetriever{ +) *DARetrievalWorkerPool { + return &DARetrievalWorkerPool{ retriever: retriever, resultCh: resultCh, workCh: make(chan uint64, 100), // Buffer size 100 inFlight: make(map[uint64]struct{}), - logger: logger.With().Str("component", "async_da_retriever").Logger(), + logger: logger.With().Str("component", "da_retrieval_worker_pool").Logger(), } } // Start starts the worker pool. -func (r *AsyncDARetriever) Start(ctx context.Context) { +func (r *DARetrievalWorkerPool) Start(ctx context.Context) { r.ctx, r.cancel = context.WithCancel(ctx) // Start 5 workers for i := 0; i < 5; i++ { r.wg.Add(1) go r.worker() } - r.logger.Info().Msg("AsyncDARetriever started") + r.logger.Info().Msg("DARetrievalWorkerPool started") } // Stop stops the worker pool. -func (r *AsyncDARetriever) Stop() { +func (r *DARetrievalWorkerPool) Stop() { if r.cancel != nil { r.cancel() } r.wg.Wait() - r.logger.Info().Msg("AsyncDARetriever stopped") + r.logger.Info().Msg("DARetrievalWorkerPool stopped") } // RequestRetrieval requests a DA retrieval for the given height. // It is non-blocking and idempotent. -func (r *AsyncDARetriever) RequestRetrieval(height uint64) { +func (r *DARetrievalWorkerPool) RequestRetrieval(height uint64) { r.mu.Lock() defer r.mu.Unlock() @@ -75,7 +77,7 @@ func (r *AsyncDARetriever) RequestRetrieval(height uint64) { } } -func (r *AsyncDARetriever) worker() { +func (r *DARetrievalWorkerPool) worker() { defer r.wg.Done() for { @@ -88,7 +90,7 @@ func (r *AsyncDARetriever) worker() { } } -func (r *AsyncDARetriever) processRetrieval(height uint64) { +func (r *DARetrievalWorkerPool) processRetrieval(height uint64) { defer func() { r.mu.Lock() delete(r.inFlight, height) @@ -97,7 +99,7 @@ func (r *AsyncDARetriever) processRetrieval(height uint64) { events, err := r.retriever.RetrieveFromDA(r.ctx, height) if err != nil { - r.logger.Debug().Err(err).Uint64("height", height).Msg("async DA retrieval failed") + r.logger.Debug().Err(err).Uint64("height", height).Msg("DA retrieval failed") return } diff --git a/block/internal/syncing/async_da_retriever_test.go b/block/internal/syncing/da_retrieval_worker_pool_test.go similarity index 84% rename from block/internal/syncing/async_da_retriever_test.go rename to block/internal/syncing/da_retrieval_worker_pool_test.go index dfaecc922e..eb628e6981 100644 --- a/block/internal/syncing/async_da_retriever_test.go +++ b/block/internal/syncing/da_retrieval_worker_pool_test.go @@ -11,23 +11,23 @@ import ( "github.com/stretchr/testify/mock" ) -func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { +func TestDARetrievalWorkerPool_RequestRetrieval(t *testing.T) { logger := zerolog.Nop() mockRetriever := NewMockDARetriever(t) resultCh := make(chan common.DAHeightEvent, 10) - asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger) + workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - asyncRetriever.Start(ctx) - defer asyncRetriever.Stop() + workerPool.Start(ctx) + defer workerPool.Stop() // 1. Test successful retrieval height1 := uint64(100) mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height1).Return([]common.DAHeightEvent{{DaHeight: height1}}, nil).Once() - asyncRetriever.RequestRetrieval(height1) + workerPool.RequestRetrieval(height1) select { case event := <-resultCh: @@ -52,7 +52,7 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { }).Once() // Should be called only once despite multiple requests // Send first request - asyncRetriever.RequestRetrieval(height2) + workerPool.RequestRetrieval(height2) // Wait for the worker to pick it up select { @@ -62,8 +62,8 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { } // Send duplicate requests while the first one is still in flight - asyncRetriever.RequestRetrieval(height2) - asyncRetriever.RequestRetrieval(height2) + workerPool.RequestRetrieval(height2) + workerPool.RequestRetrieval(height2) // Unblock the worker close(unblockCh) @@ -84,17 +84,17 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) { } } -func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) { +func TestDARetrievalWorkerPool_WorkerPoolLimit(t *testing.T) { logger := zerolog.Nop() mockRetriever := NewMockDARetriever(t) resultCh := make(chan common.DAHeightEvent, 100) - asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger) + workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - asyncRetriever.Start(ctx) - defer asyncRetriever.Stop() + workerPool.Start(ctx) + defer workerPool.Stop() // We have 5 workers. We'll block them all. unblockCh := make(chan struct{}) @@ -106,7 +106,7 @@ func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) { <-unblockCh return []common.DAHeightEvent{{DaHeight: h}}, nil }).Once() - asyncRetriever.RequestRetrieval(h) + workerPool.RequestRetrieval(h) } // Give workers time to pick up tasks @@ -120,7 +120,7 @@ func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) { return []common.DAHeightEvent{{DaHeight: h}}, nil }).Once() - asyncRetriever.RequestRetrieval(height6) + workerPool.RequestRetrieval(height6) // Ensure 6th request is NOT processed yet select { diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 105f589600..668d99a024 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -127,8 +127,8 @@ type Syncer struct { // defaults to self, but can be wrapped with tracing. blockSyncer BlockSyncer - // Async DA retriever - asyncDARetriever *AsyncDARetriever + // Worker pool for on-demand DA retrieval triggered by P2P hints + daRetrievalWorkerPool *DARetrievalWorkerPool } // pendingForcedInclusionTx represents a forced inclusion transaction that hasn't been included yet @@ -216,8 +216,8 @@ func (s *Syncer) Start(ctx context.Context) error { // Initialize handlers s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger) - s.asyncDARetriever = NewAsyncDARetriever(s.daRetriever, s.heightInCh, s.logger) - s.asyncDARetriever.Start(s.ctx) + s.daRetrievalWorkerPool = NewDARetrievalWorkerPool(s.daRetriever, s.heightInCh, s.logger) + s.daRetrievalWorkerPool.Start(s.ctx) if s.config.Instrumentation.IsTracingEnabled() { s.daRetriever = WithTracingDARetriever(s.daRetriever) } @@ -634,7 +634,7 @@ func (s *Syncer) processHeightEvent(ctx context.Context, event *common.DAHeightE Msg("P2P event with DA height hint, triggering targeted DA retrieval") // Trigger targeted DA retrieval in background via worker pool - s.asyncDARetriever.RequestRetrieval(daHeightHint) + s.daRetrievalWorkerPool.RequestRetrieval(daHeightHint) } } } diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 8543e4e794..ef42a633df 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -741,14 +741,14 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() - // Mock AsyncDARetriever + // Mock DARetrievalWorkerPool mockRetriever := NewMockDARetriever(t) - asyncRetriever := NewAsyncDARetriever(mockRetriever, s.heightInCh, zerolog.Nop()) - // We don't start the async retriever to avoid race conditions in test, + workerPool := NewDARetrievalWorkerPool(mockRetriever, s.heightInCh, zerolog.Nop()) + // We don't start the worker pool to avoid race conditions in test, // we just want to verify RequestRetrieval queues the request. // However, RequestRetrieval writes to a channel, so we need a consumer or a buffered channel. // The workCh is buffered (100), so we are good. - s.asyncDARetriever = asyncRetriever + s.daRetrievalWorkerPool = workerPool // Create event with DA height hint evt := common.DAHeightEvent{ @@ -778,9 +778,9 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { s.processHeightEvent(t.Context(), &evt) - // Verify that the request was queued in the async retriever + // Verify that the request was queued in the worker pool select { - case h := <-asyncRetriever.workCh: + case h := <-workerPool.workCh: assert.Equal(t, uint64(100), h) default: t.Fatal("expected DA retrieval request to be queued") From 68b3bcb6b71460d7a002ea21a09b5f7a9d7fbe98 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 17:17:58 +0100 Subject: [PATCH 29/41] persist da/height for da synced nodes as well for consistency --- block/internal/syncing/syncer.go | 15 ++++++ block/internal/syncing/syncer_test.go | 71 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 668d99a024..0cb1edf3b6 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -757,6 +757,21 @@ func (s *Syncer) TrySyncNextBlock(ctx context.Context, event *common.DAHeightEve return fmt.Errorf("failed to commit batch: %w", err) } + // Persist DA height mapping for blocks synced from DA + // This ensures consistency with the sequencer's submitter which also persists this mapping + // Note: P2P hints are already persisted via store_adapter.Append when items have DAHint set + if event.Source == common.SourceDA && event.DaHeight > 0 { + daHeightBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(daHeightBytes, event.DaHeight) + + if err := s.store.SetMetadata(ctx, store.GetHeightToDAHeightHeaderKey(nextHeight), daHeightBytes); err != nil { + s.logger.Warn().Err(err).Uint64("height", nextHeight).Msg("failed to persist header DA height mapping") + } + if err := s.store.SetMetadata(ctx, store.GetHeightToDAHeightDataKey(nextHeight), daHeightBytes); err != nil { + s.logger.Warn().Err(err).Uint64("height", nextHeight).Msg("failed to persist data DA height mapping") + } + } + // Update in-memory state after successful commit s.SetLastState(newState) s.metrics.Height.Set(float64(newState.LastBlockHeight)) diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index ef42a633df..75e1c81ab6 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -207,6 +207,77 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { assert.Equal(t, uint64(1), st1.LastBlockHeight) } +func TestSyncer_PersistsDAHeightMapping_WhenSyncingFromDA(t *testing.T) { + ds := dssync.MutexWrap(datastore.NewMapDatastore()) + st := store.New(ds) + + cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) + require.NoError(t, err) + + addr, pub, signer := buildSyncTestSigner(t) + + cfg := config.DefaultConfig() + gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} + + mockExec := testmocks.NewMockExecutor(t) + mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), nil).Once() + + errChan := make(chan error, 1) + s := NewSyncer( + st, + mockExec, + nil, + cm, + common.NopMetrics(), + cfg, + gen, + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), + zerolog.Nop(), + common.DefaultBlockOptions(), + errChan, + nil, + ) + + require.NoError(t, s.initializeState()) + s.ctx = t.Context() + + // Create signed header & data for height 1 + lastState := s.getLastState() + data := makeData(gen.ChainID, 1, 0) + _, hdr := makeSignedHeaderBytes(t, gen.ChainID, 1, addr, pub, signer, lastState.AppHash, data, nil) + + // Expect ExecuteTxs call for height 1 + mockExec.EXPECT().ExecuteTxs(mock.Anything, mock.Anything, uint64(1), mock.Anything, lastState.AppHash). + Return([]byte("app1"), nil).Once() + + // Process event with DA source and a specific DA height + daHeight := uint64(42) + evt := common.DAHeightEvent{Header: hdr, Data: data, DaHeight: daHeight, Source: common.SourceDA} + s.processHeightEvent(t.Context(), &evt) + + requireEmptyChan(t, errChan) + + // Verify block was synced + h, err := st.Height(t.Context()) + require.NoError(t, err) + assert.Equal(t, uint64(1), h) + + // Verify DA height mapping was persisted for header + headerDABytes, err := st.GetMetadata(t.Context(), store.GetHeightToDAHeightHeaderKey(1)) + require.NoError(t, err) + require.Len(t, headerDABytes, 8) + headerDAHeight := binary.LittleEndian.Uint64(headerDABytes) + assert.Equal(t, daHeight, headerDAHeight) + + // Verify DA height mapping was persisted for data + dataDABytes, err := st.GetMetadata(t.Context(), store.GetHeightToDAHeightDataKey(1)) + require.NoError(t, err) + require.Len(t, dataDABytes, 8) + dataDAHeight := binary.LittleEndian.Uint64(dataDABytes) + assert.Equal(t, daHeight, dataDAHeight) +} + func TestSequentialBlockSync(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) st := store.New(ds) From 025ee3fddb6e2f13cc12a787b5802530d14ba6ba Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 2 Feb 2026 17:50:05 +0100 Subject: [PATCH 30/41] fixes --- block/components.go | 14 +++++++++----- block/components_test.go | 23 +++++++++++++++++++++-- node/failover.go | 2 ++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/block/components.go b/block/components.go index d16a16863f..903402cb12 100644 --- a/block/components.go +++ b/block/components.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/celestiaorg/go-header" "github.com/rs/zerolog" "github.com/evstack/ev-node/block/internal/cache" @@ -22,6 +23,7 @@ import ( "github.com/evstack/ev-node/pkg/store" "github.com/evstack/ev-node/pkg/sync" "github.com/evstack/ev-node/pkg/telemetry" + "github.com/evstack/ev-node/types" ) // Components represents the block-related components @@ -127,8 +129,10 @@ func NewSyncComponents( store store.Store, exec coreexecutor.Executor, daClient da.Client, - headerSyncService *sync.HeaderSyncService, - dataSyncService *sync.DataSyncService, + headerStore header.Store[*types.P2PSignedHeader], + dataStore header.Store[*types.P2PData], + headerDAHintAppender submitting.DAHintAppender, + dataDAHintAppender submitting.DAHintAppender, logger zerolog.Logger, metrics *Metrics, blockOpts BlockOptions, @@ -150,8 +154,8 @@ func NewSyncComponents( metrics, config, genesis, - headerSyncService.Store(), - dataSyncService.Store(), + headerStore, + dataStore, logger, blockOpts, errorCh, @@ -163,7 +167,7 @@ func NewSyncComponents( } // Create submitter for sync nodes (no signer, only DA inclusion processing) - var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerSyncService, dataSyncService) + var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerDAHintAppender, dataDAHintAppender) if config.Instrumentation.IsTracingEnabled() { daSubmitter = submitting.WithTracingDASubmitter(daSubmitter) } diff --git a/block/components_test.go b/block/components_test.go index 87ec922563..c4d3871723 100644 --- a/block/components_test.go +++ b/block/components_test.go @@ -22,8 +22,17 @@ import ( "github.com/evstack/ev-node/pkg/signer/noop" "github.com/evstack/ev-node/pkg/store" testmocks "github.com/evstack/ev-node/test/mocks" + extmocks "github.com/evstack/ev-node/test/mocks/external" + "github.com/evstack/ev-node/types" ) +// noopDAHintAppender is a no-op implementation of DAHintAppender for testing +type noopDAHintAppender struct{} + +func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error { + return nil +} + func TestBlockComponents_ExecutionClientFailure_StopsNode(t *testing.T) { // Test the error channel mechanism works as intended @@ -86,6 +95,14 @@ func TestNewSyncComponents_Creation(t *testing.T) { daClient.On("GetForcedInclusionNamespace").Return([]byte(nil)).Maybe() daClient.On("HasForcedInclusionNamespace").Return(false).Maybe() + // Create mock P2P stores + mockHeaderStore := extmocks.NewMockStore[*types.P2PSignedHeader](t) + mockDataStore := extmocks.NewMockStore[*types.P2PData](t) + + // Create noop DAHintAppenders for testing + headerHintAppender := noopDAHintAppender{} + dataHintAppender := noopDAHintAppender{} + // Just test that the constructor doesn't panic - don't start the components // to avoid P2P store dependencies components, err := NewSyncComponents( @@ -94,8 +111,10 @@ func TestNewSyncComponents_Creation(t *testing.T) { memStore, mockExec, daClient, - nil, - nil, + mockHeaderStore, + mockDataStore, + headerHintAppender, + dataHintAppender, zerolog.Nop(), NopMetrics(), DefaultBlockOptions(), diff --git a/node/failover.go b/node/failover.go index 27f4ddf685..787f627ce6 100644 --- a/node/failover.go +++ b/node/failover.go @@ -57,6 +57,8 @@ func newSyncMode( rktStore, exec, da, + headerSyncService.Store(), + dataSyncService.Store(), headerSyncService, dataSyncService, logger, From 0298cead4e26d6f87821c36aaafa8084dc866fd8 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 09:38:13 +0100 Subject: [PATCH 31/41] use height instead of hashes --- block/components_test.go | 2 +- block/internal/submitting/da_submitter.go | 14 +++++++------- .../submitting/da_submitter_integration_test.go | 2 +- pkg/sync/sync_service.go | 9 +++++---- pkg/sync/sync_service_test.go | 4 ++-- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/block/components_test.go b/block/components_test.go index c4d3871723..f1fbac743b 100644 --- a/block/components_test.go +++ b/block/components_test.go @@ -29,7 +29,7 @@ import ( // noopDAHintAppender is a no-op implementation of DAHintAppender for testing type noopDAHintAppender struct{} -func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error { +func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, heights ...uint64) error { return nil } diff --git a/block/internal/submitting/da_submitter.go b/block/internal/submitting/da_submitter.go index c71b2aeae4..a2e0adcf75 100644 --- a/block/internal/submitting/da_submitter.go +++ b/block/internal/submitting/da_submitter.go @@ -104,7 +104,7 @@ func clamp(v, min, max time.Duration) time.Duration { } type DAHintAppender interface { - AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error + AppendDAHint(ctx context.Context, daHeight uint64, heights ...uint64) error } // DASubmitter handles DA submission operations @@ -232,13 +232,13 @@ func (s *DASubmitter) SubmitHeaders(ctx context.Context, headers []*types.Signed return submitToDA(s, ctx, headers, envelopes, func(submitted []*types.SignedHeader, res *datypes.ResultSubmit) { - hashes := make([]types.Hash, len(submitted)) + heights := make([]uint64, len(submitted)) for i, header := range submitted { headerHash := header.Hash() cache.SetHeaderDAIncluded(headerHash.String(), res.Height, header.Height()) - hashes[i] = headerHash + heights[i] = header.Height() } - if err := s.headerDAHintAppender.AppendDAHint(ctx, res.Height, hashes...); err != nil { + if err := s.headerDAHintAppender.AppendDAHint(ctx, res.Height, heights...); err != nil { s.logger.Error().Err(err).Msg("failed to append da height hint in header p2p store") // ignoring error here, since we don't want to block the block submission' } @@ -440,12 +440,12 @@ func (s *DASubmitter) SubmitData(ctx context.Context, unsignedDataList []*types. return submitToDA(s, ctx, signedDataList, signedDataListBz, func(submitted []*types.SignedData, res *datypes.ResultSubmit) { - hashes := make([]types.Hash, len(submitted)) + heights := make([]uint64, len(submitted)) for i, sd := range submitted { cache.SetDataDAIncluded(sd.Data.DACommitment().String(), res.Height, sd.Height()) - hashes[i] = sd.Hash() + heights[i] = sd.Height() } - if err := s.dataDAHintAppender.AppendDAHint(ctx, res.Height, hashes...); err != nil { + if err := s.dataDAHintAppender.AppendDAHint(ctx, res.Height, heights...); err != nil { s.logger.Error().Err(err).Msg("failed to append da height hint in data p2p store") // ignoring error here, since we don't want to block the block submission' } diff --git a/block/internal/submitting/da_submitter_integration_test.go b/block/internal/submitting/da_submitter_integration_test.go index 4818afa7e1..8672cd1ae2 100644 --- a/block/internal/submitting/da_submitter_integration_test.go +++ b/block/internal/submitting/da_submitter_integration_test.go @@ -121,6 +121,6 @@ func TestDASubmitter_SubmitHeadersAndData_MarksInclusionAndUpdatesLastSubmitted( type noopDAHintAppender struct{} -func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, hash ...types.Hash) error { +func (n noopDAHintAppender) AppendDAHint(ctx context.Context, daHeight uint64, heights ...uint64) error { return nil } diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 27344fe50c..8567e79764 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -163,12 +163,13 @@ func (syncService *SyncService[H]) WriteToStoreAndBroadcast(ctx context.Context, return nil } -func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, hashes ...types.Hash) error { - entries := make([]H, 0, len(hashes)) - for _, h := range hashes { - v, err := s.store.Get(ctx, h) +func (s *SyncService[H]) AppendDAHint(ctx context.Context, daHeight uint64, heights ...uint64) error { + entries := make([]H, 0, len(heights)) + for _, height := range heights { + v, err := s.store.GetByHeight(ctx, height) if err != nil { if errors.Is(err, header.ErrNotFound) { + s.logger.Debug().Uint64("height", height).Msg("cannot append DA height hint; header/data not found in store") continue } return err diff --git a/pkg/sync/sync_service_test.go b/pkg/sync/sync_service_test.go index 3f7b0e529c..ea600f3bdd 100644 --- a/pkg/sync/sync_service_test.go +++ b/pkg/sync/sync_service_test.go @@ -222,7 +222,7 @@ func TestDAHintStorageHeader(t *testing.T) { require.NoError(t, headerSvc.WriteToStoreAndBroadcast(ctx, &types.P2PSignedHeader{SignedHeader: signedHeader})) daHeight := uint64(100) - require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Hash())) + require.NoError(t, headerSvc.AppendDAHint(ctx, daHeight, signedHeader.Height())) h, err := headerSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) @@ -323,7 +323,7 @@ func TestDAHintStorageData(t *testing.T) { require.NoError(t, dataSvc.WriteToStoreAndBroadcast(ctx, &types.P2PData{Data: &data})) daHeight := uint64(100) - require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Hash())) + require.NoError(t, dataSvc.AppendDAHint(ctx, daHeight, data.Height())) d, err := dataSvc.Store().GetByHeight(ctx, signedHeader.Height()) require.NoError(t, err) From d6ba01d32e73c621f374c1ba4671eda01a274ca7 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 10:29:44 +0100 Subject: [PATCH 32/41] fixes --- block/internal/syncing/syncer.go | 8 +++----- pkg/store/store_adapter.go | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index 0cb1edf3b6..b357c1e266 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -240,11 +240,7 @@ func (s *Syncer) Start(ctx context.Context) error { } // Start main processing loop - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.processLoop() - }() + s.wg.Go(s.processLoop) // Start dedicated workers for DA, and pending processing s.startSyncWorkers() @@ -259,6 +255,8 @@ func (s *Syncer) Stop() error { return nil } + s.daRetrievalWorkerPool.Stop() + s.cancel() s.cancelP2PWait(0) s.wg.Wait() diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 166a54ea66..ecd4da7abc 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -304,6 +304,7 @@ func (a *StoreAdapter[H]) Get(ctx context.Context, hash header.Hash) (H, error) // Check pending items for _, h := range a.pending.Keys() { if pendingItem, ok := a.pending.Peek(h); ok && !pendingItem.IsZero() && bytes.Equal(pendingItem.Hash(), hash) { + a.applyDAHint(pendingItem) return pendingItem, nil } } @@ -344,6 +345,7 @@ func (a *StoreAdapter[H]) getByHeightNoWait(ctx context.Context, height uint64) // Check pending items if pendingItem, ok := a.pending.Peek(height); ok { + a.applyDAHint(pendingItem) return pendingItem, nil } From f662a50d444cacb5fdba67f878543348cb895425 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 11:04:22 +0100 Subject: [PATCH 33/41] add docs --- block/internal/syncing/da_retriever.go | 2 +- block/internal/syncing/syncer.go | 3 +- docs/learn/specs/header-sync.md | 64 ++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 506840f5c3..7ba20b442f 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -360,7 +360,7 @@ func isEmptyDataExpected(header *types.SignedHeader) bool { } // createEmptyDataForHeader creates empty data for a header -func createEmptyDataForHeader(ctx context.Context, header *types.SignedHeader) *types.Data { +func createEmptyDataForHeader(_ context.Context, header *types.SignedHeader) *types.Data { return &types.Data{ Txs: make(types.Txs, 0), Metadata: &types.Metadata{ diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index b357c1e266..b4fd938031 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -758,7 +758,8 @@ func (s *Syncer) TrySyncNextBlock(ctx context.Context, event *common.DAHeightEve // Persist DA height mapping for blocks synced from DA // This ensures consistency with the sequencer's submitter which also persists this mapping // Note: P2P hints are already persisted via store_adapter.Append when items have DAHint set - if event.Source == common.SourceDA && event.DaHeight > 0 { + // But DaHeight from events always take precedence as they are authoritative (comes from DA) + if event.DaHeight > 0 { daHeightBytes := make([]byte, 8) binary.LittleEndian.PutUint64(daHeightBytes, event.DaHeight) diff --git a/docs/learn/specs/header-sync.md b/docs/learn/specs/header-sync.md index 750f325933..6291356975 100644 --- a/docs/learn/specs/header-sync.md +++ b/docs/learn/specs/header-sync.md @@ -4,13 +4,13 @@ The nodes in the P2P network sync headers and data using separate sync services that implement the [go-header][go-header] interface. Evolve uses a header/data separation architecture where headers and transaction data are synchronized independently through parallel services. Each sync service consists of several components as listed below. -|Component|Description| -|---|---| -|store| a prefixed [datastore][datastore] where synced items are stored (`headerSync` prefix for headers, `dataSync` prefix for data)| -|subscriber| a [libp2p][libp2p] node pubsub subscriber for the specific data type| -|P2P server| a server for handling requests between peers in the P2P network| -|exchange| a client that enables sending in/out-bound requests from/to the P2P network| -|syncer| a service for efficient synchronization. When a P2P node falls behind and wants to catch up to the latest network head via P2P network, it can use the syncer.| +| Component | Description | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| store | a prefixed [datastore][datastore] where synced items are stored (`headerSync` prefix for headers, `dataSync` prefix for data) | +| subscriber | a [libp2p][libp2p] node pubsub subscriber for the specific data type | +| P2P server | a server for handling requests between peers in the P2P network | +| exchange | a client that enables sending in/out-bound requests from/to the P2P network | +| syncer | a service for efficient synchronization. When a P2P node falls behind and wants to catch up to the latest network head via P2P network, it can use the syncer. | ## Details @@ -22,7 +22,7 @@ Evolve implements two separate sync services: - Used by all node types (sequencer, full, and light) - Essential for maintaining the canonical view of the chain -### Data Sync Service +### Data Sync Service - Synchronizes `Data` structures containing transaction data - Used only by full nodes and sequencers @@ -90,6 +90,54 @@ The block components integrate with both services through: - The Executor component publishes headers and data through broadcast channels - Separate stores and channels manage header and data synchronization +## DA Height Hints + +DA Height Hints (DAHint) provide an optimization for P2P synchronization by indicating which DA layer height contains a block's header or data. This allows syncing nodes to fetch missing DA data directly instead of performing sequential DA scanning. + +### Naming Considerations + +The naming convention follows this pattern: + +| Name | Usage | +| ----------------- | ---------------------------------------------------------- | +| `DAHeightHint` | Internal struct field storing the hint value | +| `DAHint()` | Getter method returning the DA height hint | +| `SetDAHint()` | Setter method for the DA height hint | +| `P2PSignedHeader` | Wrapper around `SignedHeader` that includes `DAHeightHint` | +| `P2PData` | Wrapper around `Data` that includes `DAHeightHint` | + +The term "hint" is used deliberately because: + +1. **It's advisory, not authoritative**: The hint suggests where to find data on the DA layer, but the authoritative source is always the DA layer itself +2. **It may be absent**: Hints are only populated during certain sync scenarios (see below) +3. **It optimizes but doesn't replace**: Nodes can still function without hints by scanning the DA layer sequentially + +### When DAHints Are Populated + +DAHints are **only populated when a node catches up from P2P** and is not yet synced to the head. When a node is already synced to the head: + +- The executor broadcasts headers/data immediately after block creation +- At this point, DA submission has not occurred yet (it happens later in the flow) +- Therefore, the broadcasted P2P messages do not contain DA hints + +This means: + +- **Syncing nodes** (catching up): Receive headers/data with DA hints populated +- **Synced nodes** (at head): Receive headers/data without DA hints + +The DA hints are set by the DA submitter after successful inclusion on the DA layer and stored for later P2P propagation to syncing peers. + +### Implementation Details + +The P2P wrapper types (`P2PSignedHeader` and `P2PData`) extend the base types with an optional `DAHeightHint`. + +The hint is: + +1. **Set by the DA Submitter** when headers/data are successfully included on the DA layer +2. **Stored in the P2P store** alongside the header/data +3. **Propagated via P2P** when syncing nodes request blocks +4. **Used by the Syncer** to trigger targeted DA retrieval instead of sequential scanning + ## References [1] [Header Sync][sync-service] From 0a8e542b01a251b2510049a621302f0c377bcf47 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 11:35:53 +0100 Subject: [PATCH 34/41] remove worker pool --- .../syncing/da_retrieval_worker_pool.go | 113 -------------- .../syncing/da_retrieval_worker_pool_test.go | 141 ------------------ block/internal/syncing/da_retriever.go | 60 +++++++- block/internal/syncing/da_retriever_mock.go | 84 +++++++++++ .../internal/syncing/da_retriever_tracing.go | 8 + .../syncing/da_retriever_tracing_test.go | 4 + block/internal/syncing/syncer.go | 48 ++++-- block/internal/syncing/syncer_test.go | 96 ++++++++++-- docs/learn/specs/header-sync.md | 22 ++- 9 files changed, 285 insertions(+), 291 deletions(-) delete mode 100644 block/internal/syncing/da_retrieval_worker_pool.go delete mode 100644 block/internal/syncing/da_retrieval_worker_pool_test.go diff --git a/block/internal/syncing/da_retrieval_worker_pool.go b/block/internal/syncing/da_retrieval_worker_pool.go deleted file mode 100644 index e99cb67e0d..0000000000 --- a/block/internal/syncing/da_retrieval_worker_pool.go +++ /dev/null @@ -1,113 +0,0 @@ -package syncing - -import ( - "context" - "sync" - - "github.com/evstack/ev-node/block/internal/common" - "github.com/rs/zerolog" -) - -// DARetrievalWorkerPool handles concurrent on-demand DA retrieval operations -// triggered by P2P hints. Unlike the background prefetcher (AsyncBlockRetriever), -// this worker pool processes explicit retrieval requests for specific DA heights. -type DARetrievalWorkerPool struct { - retriever DARetriever - resultCh chan<- common.DAHeightEvent - workCh chan uint64 - inFlight map[uint64]struct{} - mu sync.Mutex - logger zerolog.Logger - wg sync.WaitGroup - ctx context.Context - cancel context.CancelFunc -} - -// NewDARetrievalWorkerPool creates a new DARetrievalWorkerPool. -func NewDARetrievalWorkerPool( - retriever DARetriever, - resultCh chan<- common.DAHeightEvent, - logger zerolog.Logger, -) *DARetrievalWorkerPool { - return &DARetrievalWorkerPool{ - retriever: retriever, - resultCh: resultCh, - workCh: make(chan uint64, 100), // Buffer size 100 - inFlight: make(map[uint64]struct{}), - logger: logger.With().Str("component", "da_retrieval_worker_pool").Logger(), - } -} - -// Start starts the worker pool. -func (r *DARetrievalWorkerPool) Start(ctx context.Context) { - r.ctx, r.cancel = context.WithCancel(ctx) - // Start 5 workers - for i := 0; i < 5; i++ { - r.wg.Add(1) - go r.worker() - } - r.logger.Info().Msg("DARetrievalWorkerPool started") -} - -// Stop stops the worker pool. -func (r *DARetrievalWorkerPool) Stop() { - if r.cancel != nil { - r.cancel() - } - r.wg.Wait() - r.logger.Info().Msg("DARetrievalWorkerPool stopped") -} - -// RequestRetrieval requests a DA retrieval for the given height. -// It is non-blocking and idempotent. -func (r *DARetrievalWorkerPool) RequestRetrieval(height uint64) { - r.mu.Lock() - defer r.mu.Unlock() - - if _, exists := r.inFlight[height]; exists { - return - } - - select { - case r.workCh <- height: - r.inFlight[height] = struct{}{} - r.logger.Debug().Uint64("height", height).Msg("queued DA retrieval request") - default: - r.logger.Debug().Uint64("height", height).Msg("DA retrieval worker pool full, dropping request") - } -} - -func (r *DARetrievalWorkerPool) worker() { - defer r.wg.Done() - - for { - select { - case <-r.ctx.Done(): - return - case height := <-r.workCh: - r.processRetrieval(height) - } - } -} - -func (r *DARetrievalWorkerPool) processRetrieval(height uint64) { - defer func() { - r.mu.Lock() - delete(r.inFlight, height) - r.mu.Unlock() - }() - - events, err := r.retriever.RetrieveFromDA(r.ctx, height) - if err != nil { - r.logger.Debug().Err(err).Uint64("height", height).Msg("DA retrieval failed") - return - } - - for _, event := range events { - select { - case r.resultCh <- event: - case <-r.ctx.Done(): - return - } - } -} diff --git a/block/internal/syncing/da_retrieval_worker_pool_test.go b/block/internal/syncing/da_retrieval_worker_pool_test.go deleted file mode 100644 index eb628e6981..0000000000 --- a/block/internal/syncing/da_retrieval_worker_pool_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package syncing - -import ( - "context" - "testing" - "time" - - "github.com/evstack/ev-node/block/internal/common" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestDARetrievalWorkerPool_RequestRetrieval(t *testing.T) { - logger := zerolog.Nop() - mockRetriever := NewMockDARetriever(t) - resultCh := make(chan common.DAHeightEvent, 10) - - workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - workerPool.Start(ctx) - defer workerPool.Stop() - - // 1. Test successful retrieval - height1 := uint64(100) - mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height1).Return([]common.DAHeightEvent{{DaHeight: height1}}, nil).Once() - - workerPool.RequestRetrieval(height1) - - select { - case event := <-resultCh: - assert.Equal(t, height1, event.DaHeight) - case <-time.After(1 * time.Second): - t.Fatal("timeout waiting for result") - } - - // 2. Test deduplication (idempotency) - // We'll block the retriever to simulate a slow request, then send multiple requests for the same height - height2 := uint64(200) - - // Create a channel to signal when the mock is called - calledCh := make(chan struct{}) - // Create a channel to unblock the mock - unblockCh := make(chan struct{}) - - mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height2).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { - close(calledCh) - <-unblockCh - return []common.DAHeightEvent{{DaHeight: h}}, nil - }).Once() // Should be called only once despite multiple requests - - // Send first request - workerPool.RequestRetrieval(height2) - - // Wait for the worker to pick it up - select { - case <-calledCh: - case <-time.After(1 * time.Second): - t.Fatal("timeout waiting for retriever call") - } - - // Send duplicate requests while the first one is still in flight - workerPool.RequestRetrieval(height2) - workerPool.RequestRetrieval(height2) - - // Unblock the worker - close(unblockCh) - - // We should receive exactly one result - select { - case event := <-resultCh: - assert.Equal(t, height2, event.DaHeight) - case <-time.After(1 * time.Second): - t.Fatal("timeout waiting for result") - } - - // Ensure no more results come through - select { - case <-resultCh: - t.Fatal("received duplicate result") - default: - } -} - -func TestDARetrievalWorkerPool_WorkerPoolLimit(t *testing.T) { - logger := zerolog.Nop() - mockRetriever := NewMockDARetriever(t) - resultCh := make(chan common.DAHeightEvent, 100) - - workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - workerPool.Start(ctx) - defer workerPool.Stop() - - // We have 5 workers. We'll block them all. - unblockCh := make(chan struct{}) - - // Expect 5 calls that block - for i := 0; i < 5; i++ { - h := uint64(1000 + i) - mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, h).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { - <-unblockCh - return []common.DAHeightEvent{{DaHeight: h}}, nil - }).Once() - workerPool.RequestRetrieval(h) - } - - // Give workers time to pick up tasks - time.Sleep(100 * time.Millisecond) - - // Now send a 6th request. It should be queued but not processed yet. - height6 := uint64(1005) - processed6 := make(chan struct{}) - mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height6).RunAndReturn(func(ctx context.Context, h uint64) ([]common.DAHeightEvent, error) { - close(processed6) - return []common.DAHeightEvent{{DaHeight: h}}, nil - }).Once() - - workerPool.RequestRetrieval(height6) - - // Ensure 6th request is NOT processed yet - select { - case <-processed6: - t.Fatal("6th request processed too early") - default: - } - - // Unblock workers - close(unblockCh) - - // Now 6th request should be processed - select { - case <-processed6: - case <-time.After(1 * time.Second): - t.Fatal("timeout waiting for 6th request") - } -} diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 7ba20b442f..270d9acd32 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -5,6 +5,8 @@ import ( "context" "errors" "fmt" + "slices" + "sync" "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -20,7 +22,13 @@ import ( // DARetriever defines the interface for retrieving events from the DA layer type DARetriever interface { + // RetrieveFromDA retrieves blocks from the specified DA height and returns height events RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) + // QueuePriorityHeight queues a DA height for priority retrieval (from P2P hints). + // These heights take precedence over sequential fetching. + QueuePriorityHeight(daHeight uint64) + // PopPriorityHeight returns the next priority height to fetch, or 0 if none. + PopPriorityHeight() uint64 } // daRetriever handles DA retrieval operations for syncing @@ -38,6 +46,12 @@ type daRetriever struct { // strictMode indicates if the node has seen a valid DAHeaderEnvelope // and should now reject all legacy/unsigned headers. strictMode bool + + // priorityMu protects priorityHeights from concurrent access + priorityMu sync.Mutex + // priorityHeights holds DA heights from P2P hints that should be fetched + // before continuing sequential retrieval. Sorted in ascending order. + priorityHeights []uint64 } // NewDARetriever creates a new DA retriever @@ -48,14 +62,46 @@ func NewDARetriever( logger zerolog.Logger, ) *daRetriever { return &daRetriever{ - client: client, - cache: cache, - genesis: genesis, - logger: logger.With().Str("component", "da_retriever").Logger(), - pendingHeaders: make(map[uint64]*types.SignedHeader), - pendingData: make(map[uint64]*types.Data), - strictMode: false, + client: client, + cache: cache, + genesis: genesis, + logger: logger.With().Str("component", "da_retriever").Logger(), + pendingHeaders: make(map[uint64]*types.SignedHeader), + pendingData: make(map[uint64]*types.Data), + strictMode: false, + priorityHeights: make([]uint64, 0), + } +} + +// QueuePriorityHeight queues a DA height for priority retrieval. +// Heights from P2P hints take precedence over sequential fetching. +func (r *daRetriever) QueuePriorityHeight(daHeight uint64) { + r.priorityMu.Lock() + defer r.priorityMu.Unlock() + + // Skip if already queued + if slices.Contains(r.priorityHeights, daHeight) { + return + } + + r.priorityHeights = append(r.priorityHeights, daHeight) + // Keep sorted in ascending order so we process lower heights first + slices.Sort(r.priorityHeights) +} + +// PopPriorityHeight returns the next priority height to fetch, or 0 if none. +func (r *daRetriever) PopPriorityHeight() uint64 { + r.priorityMu.Lock() + defer r.priorityMu.Unlock() + + if len(r.priorityHeights) == 0 { + return 0 } + + height := r.priorityHeights[0] + r.priorityHeights = r.priorityHeights[1:] + + return height } // RetrieveFromDA retrieves blocks from the specified DA height and returns height events diff --git a/block/internal/syncing/da_retriever_mock.go b/block/internal/syncing/da_retriever_mock.go index d94dff4d62..2e191c8851 100644 --- a/block/internal/syncing/da_retriever_mock.go +++ b/block/internal/syncing/da_retriever_mock.go @@ -38,6 +38,90 @@ func (_m *MockDARetriever) EXPECT() *MockDARetriever_Expecter { return &MockDARetriever_Expecter{mock: &_m.Mock} } +// PopPriorityHeight provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) PopPriorityHeight() uint64 { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for PopPriorityHeight") + } + + var r0 uint64 + if returnFunc, ok := ret.Get(0).(func() uint64); ok { + r0 = returnFunc() + } else { + r0 = ret.Get(0).(uint64) + } + return r0 +} + +// MockDARetriever_PopPriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PopPriorityHeight' +type MockDARetriever_PopPriorityHeight_Call struct { + *mock.Call +} + +// PopPriorityHeight is a helper method to define mock.On call +func (_e *MockDARetriever_Expecter) PopPriorityHeight() *MockDARetriever_PopPriorityHeight_Call { + return &MockDARetriever_PopPriorityHeight_Call{Call: _e.mock.On("PopPriorityHeight")} +} + +func (_c *MockDARetriever_PopPriorityHeight_Call) Run(run func()) *MockDARetriever_PopPriorityHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockDARetriever_PopPriorityHeight_Call) Return(v uint64) *MockDARetriever_PopPriorityHeight_Call { + _c.Call.Return(v) + return _c +} + +func (_c *MockDARetriever_PopPriorityHeight_Call) RunAndReturn(run func() uint64) *MockDARetriever_PopPriorityHeight_Call { + _c.Call.Return(run) + return _c +} + +// QueuePriorityHeight provides a mock function for the type MockDARetriever +func (_mock *MockDARetriever) QueuePriorityHeight(daHeight uint64) { + _mock.Called(daHeight) + return +} + +// MockDARetriever_QueuePriorityHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueuePriorityHeight' +type MockDARetriever_QueuePriorityHeight_Call struct { + *mock.Call +} + +// QueuePriorityHeight is a helper method to define mock.On call +// - daHeight uint64 +func (_e *MockDARetriever_Expecter) QueuePriorityHeight(daHeight interface{}) *MockDARetriever_QueuePriorityHeight_Call { + return &MockDARetriever_QueuePriorityHeight_Call{Call: _e.mock.On("QueuePriorityHeight", daHeight)} +} + +func (_c *MockDARetriever_QueuePriorityHeight_Call) Run(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 uint64 + if args[0] != nil { + arg0 = args[0].(uint64) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockDARetriever_QueuePriorityHeight_Call) Return() *MockDARetriever_QueuePriorityHeight_Call { + _c.Call.Return() + return _c +} + +func (_c *MockDARetriever_QueuePriorityHeight_Call) RunAndReturn(run func(daHeight uint64)) *MockDARetriever_QueuePriorityHeight_Call { + _c.Run(run) + return _c +} + // RetrieveFromDA provides a mock function for the type MockDARetriever func (_mock *MockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ([]common.DAHeightEvent, error) { ret := _mock.Called(ctx, daHeight) diff --git a/block/internal/syncing/da_retriever_tracing.go b/block/internal/syncing/da_retriever_tracing.go index 894fc67ba1..2bc7a4094d 100644 --- a/block/internal/syncing/da_retriever_tracing.go +++ b/block/internal/syncing/da_retriever_tracing.go @@ -55,3 +55,11 @@ func (t *tracedDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) return events, nil } + +func (t *tracedDARetriever) QueuePriorityHeight(daHeight uint64) { + t.inner.QueuePriorityHeight(daHeight) +} + +func (t *tracedDARetriever) PopPriorityHeight() uint64 { + return t.inner.PopPriorityHeight() +} diff --git a/block/internal/syncing/da_retriever_tracing_test.go b/block/internal/syncing/da_retriever_tracing_test.go index d83ed99d23..99ce1eb639 100644 --- a/block/internal/syncing/da_retriever_tracing_test.go +++ b/block/internal/syncing/da_retriever_tracing_test.go @@ -27,6 +27,10 @@ func (m *mockDARetriever) RetrieveFromDA(ctx context.Context, daHeight uint64) ( return nil, nil } +func (m *mockDARetriever) QueuePriorityHeight(daHeight uint64) {} + +func (m *mockDARetriever) PopPriorityHeight() uint64 { return 0 } + func setupDARetrieverTrace(t *testing.T, inner DARetriever) (DARetriever, *tracetest.SpanRecorder) { t.Helper() sr := tracetest.NewSpanRecorder() diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index b4fd938031..ab9edb53bc 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -126,9 +126,6 @@ type Syncer struct { // blockSyncer is the interface used for block sync operations. // defaults to self, but can be wrapped with tracing. blockSyncer BlockSyncer - - // Worker pool for on-demand DA retrieval triggered by P2P hints - daRetrievalWorkerPool *DARetrievalWorkerPool } // pendingForcedInclusionTx represents a forced inclusion transaction that hasn't been included yet @@ -216,11 +213,10 @@ func (s *Syncer) Start(ctx context.Context) error { // Initialize handlers s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger) - s.daRetrievalWorkerPool = NewDARetrievalWorkerPool(s.daRetriever, s.heightInCh, s.logger) - s.daRetrievalWorkerPool.Start(s.ctx) if s.config.Instrumentation.IsTracingEnabled() { s.daRetriever = WithTracingDARetriever(s.daRetriever) } + s.fiRetriever = da.NewForcedInclusionRetriever(s.daClient, s.logger, s.config, s.genesis.DAStartHeight, s.genesis.DAEpochForcedInclusion) s.p2pHandler = NewP2PHandler(s.headerStore, s.dataStore, s.cache, s.genesis, s.logger) if currentHeight, err := s.store.Height(s.ctx); err != nil { @@ -255,8 +251,6 @@ func (s *Syncer) Stop() error { return nil } - s.daRetrievalWorkerPool.Stop() - s.cancel() s.cancelP2PWait(0) s.wg.Wait() @@ -432,7 +426,19 @@ func (s *Syncer) fetchDAUntilCaughtUp() error { default: } - daHeight := max(s.daRetrieverHeight.Load(), s.cache.DaHeight()) + // Check for priority heights from P2P hints first + var daHeight uint64 + if priorityHeight := s.daRetriever.PopPriorityHeight(); priorityHeight > 0 { + // Skip if we've already fetched past this height + currentHeight := s.daRetrieverHeight.Load() + if priorityHeight < currentHeight { + continue + } + daHeight = priorityHeight + s.logger.Debug().Uint64("da_height", daHeight).Msg("fetching priority DA height from P2P hint") + } else { + daHeight = max(s.daRetrieverHeight.Load(), s.cache.DaHeight()) + } events, err := s.daRetriever.RetrieveFromDA(s.ctx, daHeight) if err != nil { @@ -461,8 +467,19 @@ func (s *Syncer) fetchDAUntilCaughtUp() error { } } - // increment DA retrieval height on successful retrieval - s.daRetrieverHeight.Store(daHeight + 1) + // Update DA retrieval height on successful retrieval + // For priority fetches, only update if the priority height is ahead of current + // For sequential fetches, always increment + newHeight := daHeight + 1 + for { + current := s.daRetrieverHeight.Load() + if newHeight <= current { + break // Already at or past this height + } + if s.daRetrieverHeight.CompareAndSwap(current, newHeight) { + break + } + } } } @@ -626,13 +643,18 @@ func (s *Syncer) processHeightEvent(ctx context.Context, event *common.DAHeightE } if len(daHeightHints) > 0 { for _, daHeightHint := range daHeightHints { + // Skip if we've already fetched past this height + if daHeightHint < s.daRetrieverHeight.Load() { + continue + } + s.logger.Debug(). Uint64("height", height). Uint64("da_height_hint", daHeightHint). - Msg("P2P event with DA height hint, triggering targeted DA retrieval") + Msg("P2P event with DA height hint, queuing priority DA retrieval") - // Trigger targeted DA retrieval in background via worker pool - s.daRetrievalWorkerPool.RequestRetrieval(daHeightHint) + // Queue priority DA retrieval - will be processed in fetchDAUntilCaughtUp + s.daRetriever.QueuePriorityHeight(daHeightHint) } } } diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 75e1c81ab6..b7bc6beb8d 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -812,14 +812,8 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { require.NoError(t, s.initializeState()) s.ctx = context.Background() - // Mock DARetrievalWorkerPool - mockRetriever := NewMockDARetriever(t) - workerPool := NewDARetrievalWorkerPool(mockRetriever, s.heightInCh, zerolog.Nop()) - // We don't start the worker pool to avoid race conditions in test, - // we just want to verify RequestRetrieval queues the request. - // However, RequestRetrieval writes to a channel, so we need a consumer or a buffered channel. - // The workCh is buffered (100), so we are good. - s.daRetrievalWorkerPool = workerPool + // Create a real daRetriever to test priority queue + s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) // Create event with DA height hint evt := common.DAHeightEvent{ @@ -849,11 +843,85 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) { s.processHeightEvent(t.Context(), &evt) - // Verify that the request was queued in the worker pool - select { - case h := <-workerPool.workCh: - assert.Equal(t, uint64(100), h) - default: - t.Fatal("expected DA retrieval request to be queued") + // Verify that the priority height was queued in the daRetriever + priorityHeight := s.daRetriever.PopPriorityHeight() + assert.Equal(t, uint64(100), priorityHeight) +} + +func TestProcessHeightEvent_SkipsDAHintWhenAlreadyFetched(t *testing.T) { + ds := dssync.MutexWrap(datastore.NewMapDatastore()) + st := store.New(ds) + cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) + require.NoError(t, err) + + addr, _, _ := buildSyncTestSigner(t) + cfg := config.DefaultConfig() + gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} + + mockExec := testmocks.NewMockExecutor(t) + mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), nil).Once() + + s := NewSyncer( + st, + mockExec, + nil, + cm, + common.NopMetrics(), + cfg, + gen, + extmocks.NewMockStore[*types.P2PSignedHeader](t), + extmocks.NewMockStore[*types.P2PData](t), + zerolog.Nop(), + common.DefaultBlockOptions(), + make(chan error, 1), + nil, + ) + require.NoError(t, s.initializeState()) + s.ctx = context.Background() + + // Create a real daRetriever to test priority queue + s.daRetriever = NewDARetriever(nil, cm, gen, zerolog.Nop()) + + // Set DA retriever height to 150 - simulating we've already fetched past height 100 + s.daRetrieverHeight.Store(150) + + // Set the store height to 1 so the event can be processed + batch, err := st.NewBatch(context.Background()) + require.NoError(t, err) + require.NoError(t, batch.SetHeight(1)) + require.NoError(t, batch.Commit()) + + // Create event with DA height hint that is BELOW the current daRetrieverHeight + evt := common.DAHeightEvent{ + Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: "c", Height: 2}}}, + Data: &types.Data{Metadata: &types.Metadata{ChainID: "c", Height: 2}}, + Source: common.SourceP2P, + DaHeightHints: [2]uint64{100, 100}, // Both hints are below 150 + } + + s.processHeightEvent(t.Context(), &evt) + + // Verify that no priority height was queued since we've already fetched past it + priorityHeight := s.daRetriever.PopPriorityHeight() + assert.Equal(t, uint64(0), priorityHeight, "should not queue DA hint that is below current daRetrieverHeight") + + // Now test with a hint that is ABOVE the current daRetrieverHeight + evt2 := common.DAHeightEvent{ + Header: &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{ChainID: "c", Height: 3}}}, + Data: &types.Data{Metadata: &types.Metadata{ChainID: "c", Height: 3}}, + Source: common.SourceP2P, + DaHeightHints: [2]uint64{200, 200}, // Both hints are above 150 } + + // Set the store height to 2 so the event can be processed + batch, err = st.NewBatch(context.Background()) + require.NoError(t, err) + require.NoError(t, batch.SetHeight(2)) + require.NoError(t, batch.Commit()) + + s.processHeightEvent(t.Context(), &evt2) + + // Verify that the priority height WAS queued since it's above daRetrieverHeight + priorityHeight = s.daRetriever.PopPriorityHeight() + assert.Equal(t, uint64(200), priorityHeight, "should queue DA hint that is above current daRetrieverHeight") } diff --git a/docs/learn/specs/header-sync.md b/docs/learn/specs/header-sync.md index 6291356975..ae237f9233 100644 --- a/docs/learn/specs/header-sync.md +++ b/docs/learn/specs/header-sync.md @@ -129,14 +129,30 @@ The DA hints are set by the DA submitter after successful inclusion on the DA la ### Implementation Details -The P2P wrapper types (`P2PSignedHeader` and `P2PData`) extend the base types with an optional `DAHeightHint`. +The P2P wrapper types (`P2PSignedHeader` and `P2PData`) extend the base types with an optional `DAHeightHint` field: -The hint is: +- Uses protobuf optional fields (`optional uint64 da_height_hint`) for backward compatibility +- Old nodes can still unmarshal new messages (the hint field is simply ignored) +- New nodes can unmarshal old messages (the hint field defaults to zero/absent) + +The hint flow: 1. **Set by the DA Submitter** when headers/data are successfully included on the DA layer 2. **Stored in the P2P store** alongside the header/data 3. **Propagated via P2P** when syncing nodes request blocks -4. **Used by the Syncer** to trigger targeted DA retrieval instead of sequential scanning +4. **Queued as priority** by the Syncer's DA retriever when received via P2P +5. **Fetched before sequential heights** - priority heights take precedence over normal DA scanning + +### Priority Queue Mechanism + +When a P2P event arrives with a DA height hint, the hint is queued as a priority height in the DA retriever. The `fetchDAUntilCaughtUp` loop checks for priority heights first: + +1. If priority heights are queued, pop and fetch the lowest one first +2. If no priority heights, continue sequential DA fetching (form last known da height) +3. Priority heights are sorted ascending to process lower heights first +4. Already-processed priority heights are tracked to avoid duplicate fetches + +This ensures that when syncing from P2P, the node can immediately fetch the DA data for blocks it receives, rather than waiting for sequential scanning to reach that height. ## References From 2485ab507da6277ed6c2512fc9260177fcb0d9c1 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 11:51:54 +0100 Subject: [PATCH 35/41] fix mocks --- block/internal/syncing/syncer_backoff_test.go | 9 +++++++++ block/internal/syncing/syncer_benchmark_test.go | 1 + block/internal/syncing/syncer_test.go | 2 ++ 3 files changed, 12 insertions(+) diff --git a/block/internal/syncing/syncer_backoff_test.go b/block/internal/syncing/syncer_backoff_test.go index 8651b8c784..7bc6b4f5c6 100644 --- a/block/internal/syncing/syncer_backoff_test.go +++ b/block/internal/syncing/syncer_backoff_test.go @@ -76,6 +76,9 @@ func TestSyncer_BackoffOnDAError(t *testing.T) { syncer.p2pHandler = p2pHandler p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() + // Mock PopPriorityHeight to always return 0 (no priority heights) + daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + // Create mock stores for P2P mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() @@ -164,6 +167,9 @@ func TestSyncer_BackoffResetOnSuccess(t *testing.T) { syncer.p2pHandler = p2pHandler p2pHandler.On("SetProcessedHeight", mock.Anything).Return().Maybe() + // Mock PopPriorityHeight to always return 0 (no priority heights) + daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + // Create mock stores for P2P mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() @@ -246,6 +252,9 @@ func TestSyncer_BackoffBehaviorIntegration(t *testing.T) { syncer.daRetriever = daRetriever syncer.p2pHandler = p2pHandler + // Mock PopPriorityHeight to always return 0 (no priority heights) + daRetriever.On("PopPriorityHeight").Return(uint64(0)).Maybe() + // Create mock stores for P2P mockHeaderStore := extmocks.NewMockStore[*types.SignedHeader](t) mockHeaderStore.EXPECT().Height().Return(uint64(0)).Maybe() diff --git a/block/internal/syncing/syncer_benchmark_test.go b/block/internal/syncing/syncer_benchmark_test.go index 5accde5a25..a6529d5562 100644 --- a/block/internal/syncing/syncer_benchmark_test.go +++ b/block/internal/syncing/syncer_benchmark_test.go @@ -136,6 +136,7 @@ func newBenchFixture(b *testing.B, totalHeights uint64, shuffledTx bool, daDelay // Mock DA retriever to emit exactly totalHeights events, then HFF and cancel daR := NewMockDARetriever(b) + daR.On("PopPriorityHeight").Return(uint64(0)).Maybe() for i := uint64(0); i < totalHeights; i++ { daHeight := i + daHeightOffset daR.On("RetrieveFromDA", mock.Anything, daHeight). diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index b7bc6beb8d..41c89e7a84 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -446,6 +446,7 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock := NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() + daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() syncerInst1.daRetriever, syncerInst1.p2pHandler = daRtrMock, p2pHndlMock // with n da blobs fetched @@ -537,6 +538,7 @@ func TestSyncLoopPersistState(t *testing.T) { daRtrMock, p2pHndlMock = NewMockDARetriever(t), newMockp2pHandler(t) p2pHndlMock.On("ProcessHeight", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() p2pHndlMock.On("SetProcessedHeight", mock.Anything).Return().Maybe() + daRtrMock.On("PopPriorityHeight").Return(uint64(0)).Maybe() syncerInst2.daRetriever, syncerInst2.p2pHandler = daRtrMock, p2pHndlMock daRtrMock.On("RetrieveFromDA", mock.Anything, mock.Anything). From ffc9ff4744683cd19db47156458db1cb2737bd43 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 12:10:12 +0100 Subject: [PATCH 36/41] cleanup (syncer should not care about da included height) --- block/internal/submitting/submitter.go | 12 ++---------- block/internal/syncing/syncer.go | 16 ---------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 68fadbef00..097acf74c0 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -129,19 +129,11 @@ func (s *Submitter) Start(ctx context.Context) error { // Start DA submission loop if signer is available (aggregator nodes only) if s.signer != nil { s.logger.Info().Msg("starting DA submission loop") - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.daSubmissionLoop() - }() + s.wg.Go(s.daSubmissionLoop) } // Start DA inclusion processing loop (both sync and aggregator nodes) - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.processDAInclusionLoop() - }() + s.wg.Go(s.processDAInclusionLoop) return nil } diff --git a/block/internal/syncing/syncer.go b/block/internal/syncing/syncer.go index ab9edb53bc..88f46aafeb 100644 --- a/block/internal/syncing/syncer.go +++ b/block/internal/syncing/syncer.go @@ -777,22 +777,6 @@ func (s *Syncer) TrySyncNextBlock(ctx context.Context, event *common.DAHeightEve return fmt.Errorf("failed to commit batch: %w", err) } - // Persist DA height mapping for blocks synced from DA - // This ensures consistency with the sequencer's submitter which also persists this mapping - // Note: P2P hints are already persisted via store_adapter.Append when items have DAHint set - // But DaHeight from events always take precedence as they are authoritative (comes from DA) - if event.DaHeight > 0 { - daHeightBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(daHeightBytes, event.DaHeight) - - if err := s.store.SetMetadata(ctx, store.GetHeightToDAHeightHeaderKey(nextHeight), daHeightBytes); err != nil { - s.logger.Warn().Err(err).Uint64("height", nextHeight).Msg("failed to persist header DA height mapping") - } - if err := s.store.SetMetadata(ctx, store.GetHeightToDAHeightDataKey(nextHeight), daHeightBytes); err != nil { - s.logger.Warn().Err(err).Uint64("height", nextHeight).Msg("failed to persist data DA height mapping") - } - } - // Update in-memory state after successful commit s.SetLastState(newState) s.metrics.Height.Set(float64(newState.LastBlockHeight)) From 7f39a410cac6342928bcdbf321f3c5a8436be820 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 12:15:07 +0100 Subject: [PATCH 37/41] rename for clarity --- block/internal/submitting/submitter.go | 10 +++++----- block/internal/submitting/submitter_test.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 097acf74c0..cac8ebd1cc 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -337,9 +337,9 @@ func (s *Submitter) processDAInclusionLoop() { s.logger.Debug().Uint64("height", nextHeight).Msg("advancing DA included height") - // Set sequencer height to DA height mapping using already retrieved data - if err := s.setSequencerHeightToDAHeight(s.ctx, nextHeight, header, data, currentDAIncluded == 0); err != nil { - s.logger.Error().Err(err).Uint64("height", nextHeight).Msg("failed to set sequencer height to DA height mapping") + // Set node height to DA height mapping using already retrieved data + if err := s.setNodeHeightToDAHeight(s.ctx, nextHeight, header, data, currentDAIncluded == 0); err != nil { + s.logger.Error().Err(err).Uint64("height", nextHeight).Msg("failed to set node height to DA height mapping") break } @@ -427,14 +427,14 @@ func (s *Submitter) sendCriticalError(err error) { } } -// setSequencerHeightToDAHeight stores the mapping from a ev-node block height to the corresponding +// setNodeHeightToDAHeight stores the mapping from a ev-node block height to the corresponding // DA (Data Availability) layer heights where the block's header and data were included. // This mapping is persisted in the store metadata and is used to track which DA heights // contain the block components for a given ev-node height. // // For blocks with empty transactions, both header and data use the same DA height since // empty transaction data is not actually published to the DA layer. -func (s *Submitter) setSequencerHeightToDAHeight(ctx context.Context, height uint64, header *types.SignedHeader, data *types.Data, genesisInclusion bool) error { +func (s *Submitter) setNodeHeightToDAHeight(ctx context.Context, height uint64, header *types.SignedHeader, data *types.Data, genesisInclusion bool) error { headerHash, dataHash := header.Hash(), data.DACommitment() headerDaHeightBytes := make([]byte, 8) diff --git a/block/internal/submitting/submitter_test.go b/block/internal/submitting/submitter_test.go index 0b243a1f97..3e0e0b343d 100644 --- a/block/internal/submitting/submitter_test.go +++ b/block/internal/submitting/submitter_test.go @@ -191,10 +191,10 @@ func TestSubmitter_setSequencerHeightToDAHeight(t *testing.T) { mockStore.On("SetMetadata", mock.Anything, dataKey, dBz).Return(nil).Once() mockStore.On("SetMetadata", mock.Anything, store.GenesisDAHeightKey, gBz).Return(nil).Once() - require.NoError(t, s.setSequencerHeightToDAHeight(ctx, 1, h, d, true)) + require.NoError(t, s.setNodeHeightToDAHeight(ctx, 1, h, d, true)) } -func TestSubmitter_setSequencerHeightToDAHeight_Errors(t *testing.T) { +func TestSubmitter_setNodeHeightToDAHeight_Errors(t *testing.T) { ctx := t.Context() cm, st := newTestCacheAndStore(t) @@ -205,11 +205,11 @@ func TestSubmitter_setSequencerHeightToDAHeight_Errors(t *testing.T) { // No cache entries -> expect error on missing header _, ok := cm.GetHeaderDAIncluded(h.Hash().String()) assert.False(t, ok) - assert.Error(t, s.setSequencerHeightToDAHeight(ctx, 1, h, d, false)) + assert.Error(t, s.setNodeHeightToDAHeight(ctx, 1, h, d, false)) // Add header, missing data cm.SetHeaderDAIncluded(h.Hash().String(), 10, 1) - assert.Error(t, s.setSequencerHeightToDAHeight(ctx, 1, h, d, false)) + assert.Error(t, s.setNodeHeightToDAHeight(ctx, 1, h, d, false)) } func TestSubmitter_initializeDAIncludedHeight(t *testing.T) { From c5e0ac7ea88792feba2c71073efc27dbbad62f9b Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 12:20:31 +0100 Subject: [PATCH 38/41] Update syncer_test.go --- block/internal/syncing/syncer_test.go | 71 --------------------------- 1 file changed, 71 deletions(-) diff --git a/block/internal/syncing/syncer_test.go b/block/internal/syncing/syncer_test.go index 41c89e7a84..5edec1cce5 100644 --- a/block/internal/syncing/syncer_test.go +++ b/block/internal/syncing/syncer_test.go @@ -207,77 +207,6 @@ func TestProcessHeightEvent_SyncsAndUpdatesState(t *testing.T) { assert.Equal(t, uint64(1), st1.LastBlockHeight) } -func TestSyncer_PersistsDAHeightMapping_WhenSyncingFromDA(t *testing.T) { - ds := dssync.MutexWrap(datastore.NewMapDatastore()) - st := store.New(ds) - - cm, err := cache.NewManager(config.DefaultConfig(), st, zerolog.Nop()) - require.NoError(t, err) - - addr, pub, signer := buildSyncTestSigner(t) - - cfg := config.DefaultConfig() - gen := genesis.Genesis{ChainID: "tchain", InitialHeight: 1, StartTime: time.Now().Add(-time.Second), ProposerAddress: addr} - - mockExec := testmocks.NewMockExecutor(t) - mockExec.EXPECT().InitChain(mock.Anything, mock.Anything, uint64(1), "tchain").Return([]byte("app0"), nil).Once() - - errChan := make(chan error, 1) - s := NewSyncer( - st, - mockExec, - nil, - cm, - common.NopMetrics(), - cfg, - gen, - extmocks.NewMockStore[*types.P2PSignedHeader](t), - extmocks.NewMockStore[*types.P2PData](t), - zerolog.Nop(), - common.DefaultBlockOptions(), - errChan, - nil, - ) - - require.NoError(t, s.initializeState()) - s.ctx = t.Context() - - // Create signed header & data for height 1 - lastState := s.getLastState() - data := makeData(gen.ChainID, 1, 0) - _, hdr := makeSignedHeaderBytes(t, gen.ChainID, 1, addr, pub, signer, lastState.AppHash, data, nil) - - // Expect ExecuteTxs call for height 1 - mockExec.EXPECT().ExecuteTxs(mock.Anything, mock.Anything, uint64(1), mock.Anything, lastState.AppHash). - Return([]byte("app1"), nil).Once() - - // Process event with DA source and a specific DA height - daHeight := uint64(42) - evt := common.DAHeightEvent{Header: hdr, Data: data, DaHeight: daHeight, Source: common.SourceDA} - s.processHeightEvent(t.Context(), &evt) - - requireEmptyChan(t, errChan) - - // Verify block was synced - h, err := st.Height(t.Context()) - require.NoError(t, err) - assert.Equal(t, uint64(1), h) - - // Verify DA height mapping was persisted for header - headerDABytes, err := st.GetMetadata(t.Context(), store.GetHeightToDAHeightHeaderKey(1)) - require.NoError(t, err) - require.Len(t, headerDABytes, 8) - headerDAHeight := binary.LittleEndian.Uint64(headerDABytes) - assert.Equal(t, daHeight, headerDAHeight) - - // Verify DA height mapping was persisted for data - dataDABytes, err := st.GetMetadata(t.Context(), store.GetHeightToDAHeightDataKey(1)) - require.NoError(t, err) - require.Len(t, dataDABytes, 8) - dataDAHeight := binary.LittleEndian.Uint64(dataDABytes) - assert.Equal(t, daHeight, dataDAHeight) -} - func TestSequentialBlockSync(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) st := store.New(ds) From 627b7986e2a2b4e2844d844aa017748549b207a0 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 12:55:47 +0100 Subject: [PATCH 39/41] speed-up queue --- block/internal/syncing/da_retriever.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/block/internal/syncing/da_retriever.go b/block/internal/syncing/da_retriever.go index 270d9acd32..a6c9d43c7c 100644 --- a/block/internal/syncing/da_retriever.go +++ b/block/internal/syncing/da_retriever.go @@ -79,14 +79,11 @@ func (r *daRetriever) QueuePriorityHeight(daHeight uint64) { r.priorityMu.Lock() defer r.priorityMu.Unlock() - // Skip if already queued - if slices.Contains(r.priorityHeights, daHeight) { - return + idx, found := slices.BinarySearch(r.priorityHeights, daHeight) + if found { + return // Already queued } - - r.priorityHeights = append(r.priorityHeights, daHeight) - // Keep sorted in ascending order so we process lower heights first - slices.Sort(r.priorityHeights) + r.priorityHeights = slices.Insert(r.priorityHeights, idx, daHeight) } // PopPriorityHeight returns the next priority height to fetch, or 0 if none. From 86e91a56996945c3eb1958c0331a724598fea9b6 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 13:13:03 +0100 Subject: [PATCH 40/41] uncomment replaces --- apps/evm/go.mod | 8 ++++---- apps/grpc/go.mod | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/evm/go.mod b/apps/evm/go.mod index bd1b1a930b..151585228f 100644 --- a/apps/evm/go.mod +++ b/apps/evm/go.mod @@ -2,10 +2,10 @@ module github.com/evstack/ev-node/apps/evm go 1.25.0 -//replace ( -// github.com/evstack/ev-node => ../../ -// github.com/evstack/ev-node/execution/evm => ../../execution/evm -//) +replace ( + github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/execution/evm => ../../execution/evm +) require ( github.com/ethereum/go-ethereum v1.16.8 diff --git a/apps/grpc/go.mod b/apps/grpc/go.mod index 912f94c40c..c1b30d70e3 100644 --- a/apps/grpc/go.mod +++ b/apps/grpc/go.mod @@ -2,10 +2,10 @@ module github.com/evstack/ev-node/apps/grpc go 1.25.0 -//replace ( -// github.com/evstack/ev-node => ../../ -// github.com/evstack/ev-node/execution/grpc => ../../execution/grpc -//) +replace ( + github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/execution/grpc => ../../execution/grpc +) require ( github.com/evstack/ev-node v1.0.0-rc.2 From de03ad968e1b1f9365d1751012e40161d3350e9d Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 3 Feb 2026 13:18:35 +0100 Subject: [PATCH 41/41] go mod tidy --- apps/evm/go.sum | 4 ---- apps/grpc/go.sum | 4 ---- 2 files changed, 8 deletions(-) diff --git a/apps/evm/go.sum b/apps/evm/go.sum index 109afc76fa..1da058e8e2 100644 --- a/apps/evm/go.sum +++ b/apps/evm/go.sum @@ -409,12 +409,8 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/evstack/ev-node v1.0.0-rc.2 h1:gUQzLTkCj6D751exm/FIR/yw2aXWiW2aEREEwtxMvw0= -github.com/evstack/ev-node v1.0.0-rc.2/go.mod h1:Qa2nN1D6PJQRU2tiarv6X5Der5OZg/+2QGY/K2mA760= github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= -github.com/evstack/ev-node/execution/evm v1.0.0-rc.2 h1:t7os7ksmPhf2rWY2psVBowyc+iuneMDPwBGQaxSckus= -github.com/evstack/ev-node/execution/evm v1.0.0-rc.2/go.mod h1:ahxKQfPlJ5C7g15Eq9Mjn2tQnn59T0kIm9B10zDhcTI= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/apps/grpc/go.sum b/apps/grpc/go.sum index b99a59e18a..94c6bc63fa 100644 --- a/apps/grpc/go.sum +++ b/apps/grpc/go.sum @@ -365,12 +365,8 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node v1.0.0-rc.2 h1:gUQzLTkCj6D751exm/FIR/yw2aXWiW2aEREEwtxMvw0= -github.com/evstack/ev-node v1.0.0-rc.2/go.mod h1:Qa2nN1D6PJQRU2tiarv6X5Der5OZg/+2QGY/K2mA760= github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= -github.com/evstack/ev-node/execution/grpc v1.0.0-rc.1 h1:OzrWLDDY6/9+LWx0XmUqPzxs/CHZRJICOwQ0Me/i6dY= -github.com/evstack/ev-node/execution/grpc v1.0.0-rc.1/go.mod h1:Pr/sF6Zx8am9ZeWFcoz1jYPs0kXmf+OmL8Tz2Gyq7E4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=