fix(block): halt on invalid blocks and wait on unavailable execution client (#2719)

<!--
Please read and fill out this form before submitting your PR.

Please make sure you have reviewed our contributors guide before
submitting your
first PR.

NOTE: PR titles should follow semantic commits:
https://www.conventionalcommits.org/en/v1.0.0/
-->

## Overview

aggregator:
we halt on invalid blocks only and unavailable execution client in the
aggregator flow.
all other errors should be intermittent so retrying is fine.

syncer:
in the syncer, the availability of the execution client is less
important (block time isn't relevant), so we can retry.

<!-- 
Please provide an explanation of the PR, including the appropriate
context,
background, goal, and rationale. If there is an issue with this
information,
please provide a tl;dr and link the issue. 

Ex: Closes #<issue number>
-->

Author: julienrbrt

block/internal/executing/executor.go

@@ -362,6 +362,7 @@ func (e *Executor) produceBlock() error {
 	header.Signature = signature
 
 	if err := e.validateBlock(currentState, header, data); err != nil {
+		e.sendCriticalError(fmt.Errorf("failed to validate block: %w", err))
 		return fmt.Errorf("failed to validate block: %w", err)
 	}
 

block/internal/syncing/syncer.go

@@ -30,8 +30,12 @@ type p2pHandler interface {
 	ProcessDataRange(ctx context.Context, fromHeight, toHeight uint64) []common.DAHeightEvent
 }
 
-// maxRetriesBeforeHalt is the maximum number of retries against the execution client before halting the syncer.
-const maxRetriesBeforeHalt = 3
+const (
+	// maxRetriesBeforeHalt is the maximum number of retries against the execution client before halting the syncer.
+	maxRetriesBeforeHalt = 3
+	// maxRetriesTimeout is the maximum time to wait for a retry before halting the syncer.
+	maxRetriesTimeout = time.Second * 10
+)
 
 // Syncer handles block synchronization from DA and P2P sources.
 type Syncer struct {
@@ -494,6 +498,7 @@ func (s *Syncer) applyBlock(header types.Header, data *types.Data, currentState
 			return types.State{}, fmt.Errorf("failed to execute transactions: %w", err)
 		}
 
+		time.Sleep(maxRetriesTimeout) // sleep before retrying
 		return types.State{}, fmt.Errorf("failed to execute transactions (retry %d / %d): %w", s.retriesBeforeHalt[header.Height()], maxRetriesBeforeHalt, err)
 	}
 	delete(s.retriesBeforeHalt, header.Height())