Probe scripts

README.md

@@ -142,7 +142,7 @@ $ builder-playground logs validator
 Builder-playground supports inspecting the connection of a service to a specific port.
 
 ```bash
-$ builder-playground inspect <service> <port>
+$ builder-playground debug inspect <service> <port>
 ```
 
 Example:

main.go

@@ -105,6 +105,27 @@ var inspectCmd = &cobra.Command{
 	},
 }
 
+var debugCmd = &cobra.Command{
+	Use: "debug",
+}
+
+var probeCmd = &cobra.Command{
+	Use: "probe",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		serviceName := args[0]
+
+		resp, err := playground.ExecuteHealthCheckManually(serviceName)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Exit code: %d\n", resp.ExitCode)
+		fmt.Printf("Output: %s\n", resp.Output)
+
+		return nil
+	},
+}
+
 var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Show logs for a service",
@@ -193,6 +214,10 @@ func main() {
 	rootCmd.AddCommand(stopCmd)
 	stopCmd.Flags().StringVar(&outputFlag, "output", "", "Output folder for the artifacts")
 
+	debugCmd.AddCommand(probeCmd)
+	debugCmd.AddCommand(inspectCmd)
+	rootCmd.AddCommand(debugCmd)
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -280,14 +305,14 @@ func runIt(recipe playground.Recipe) error {
 
 	if interactive {
 		i := playground.NewInteractiveDisplay(svcManager)
-		cfg.Callback = i.HandleUpdate
+		cfg.AddCallback(i.HandleUpdate)
 	}
 
 	// Add callback to log service updates in debug mode
 	if logLevel == playground.LevelDebug {
-		cfg.Callback = func(serviceName string, update playground.TaskStatus) {
+		cfg.AddCallback(func(serviceName string, update playground.TaskStatus) {
 			log.Printf("[DEBUG] [%s] %s\n", serviceName, update)
-		}
+		})
 	}
 
 	dockerRunner, err := playground.NewLocalRunner(cfg)
@@ -328,14 +353,7 @@ func runIt(recipe playground.Recipe) error {
 		return fmt.Errorf("failed to wait for service readiness: %w", err)
 	}
 
-	fmt.Printf("\nWaiting for network to be ready for transactions...\n")
-	networkReadyStart := time.Now()
-	if err := playground.CompleteReady(ctx, svcManager.Services); err != nil {
-		dockerRunner.Stop()
-		return fmt.Errorf("network not ready: %w", err)
-	}
-	fmt.Printf("Network is ready for transactions (took %.1fs)\n", time.Since(networkReadyStart).Seconds())
-	fmt.Println("Session ID:", svcManager.ID)
+	fmt.Println("\nServices healthy... Ready to accept transactions")
 
 	// get the output from the recipe
 	output := recipe.Output(svcManager)
@@ -353,9 +371,11 @@ func runIt(recipe playground.Recipe) error {
 	watchdogErr := make(chan error, 1)
 	if watchdog {
 		go func() {
-			if err := playground.RunWatchdog(artifacts.Out, svcManager.Services); err != nil {
-				watchdogErr <- fmt.Errorf("watchdog failed: %w", err)
-			}
+			cfg.AddCallback(func(name string, status playground.TaskStatus) {
+				if status == playground.TaskStatusUnhealty {
+					watchdogErr <- fmt.Errorf("watchdog failed: %w", fmt.Errorf("task '%s' is not healthy anymore", name))
+				}
+			})
 		}()
 	}
 

playground/components.go

@@ -1,9 +1,7 @@
 package playground
 
 import (
-	"context"
 	"fmt"
-	"io"
 	"strconv"
 	"strings"
 	"time"
@@ -28,7 +26,6 @@ func (r *RollupBoost) Apply(manifest *Manifest) {
 	service := manifest.NewService("rollup-boost").
 		WithImage("docker.io/flashbots/rollup-boost").
 		WithTag("v0.7.5").
-        DependsOnHealthy(r.ELNode).
 		WithArgs(
 			"--rpc-host", "0.0.0.0",
 			"--rpc-port", `{{Port "authrpc" 8551}}`,
@@ -229,11 +226,9 @@ type ChainMonitor struct {
 
 func (c *ChainMonitor) Apply(manifest *Manifest) {
 	manifest.NewService("chain-monitor").
-        WithPort("metrics", 8080).
-        WithImage("ghcr.io/flashbots/chain-monitor").
-        WithTag("v0.0.54").
-		DependsOnHealthy(c.L1RPC).
-		DependsOnHealthy(c.L2RPC).
+		WithPort("metrics", 8080).
+		WithImage("ghcr.io/flashbots/chain-monitor").
+		WithTag("v0.0.54").
 		WithArgs(
 			"serve",
 			"--l1-rpc", Connect(c.L1RPC, "http"),
@@ -343,7 +338,7 @@ func (o *OpGeth) Apply(manifest *Manifest) {
 		trustedPeers = fmt.Sprintf("--bootnodes %s ", manifest.ctx.Bootnode.Connect())
 	}
 
-	manifest.NewService("op-geth").
+	svc := manifest.NewService("op-geth").
 		WithImage("us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth").
 		WithTag("v1.101503.2-rc.5").
 		WithEntrypoint("/bin/sh").
@@ -383,28 +378,11 @@ func (o *OpGeth) Apply(manifest *Manifest) {
 				"--metrics.port "+`{{Port "metrics" 6061}}`,
 		).
 		WithVolume("data", "/data_opgeth").
-		WithWatchdog(opGethWatchdogFn).
-		WithReadyFn(opGethReadyFn).
 		WithArtifact("/data/l2-genesis.json", "l2-genesis.json").
 		WithArtifact("/data/jwtsecret", "jwtsecret").
-		WithArtifact("/data/p2p_key.txt", o.Enode.Artifact).
-		WithReady(ReadyCheck{
-			QueryURL:    "http://localhost:8545",
-			Interval:    1 * time.Second,
-			Timeout:     10 * time.Second,
-			Retries:     20,
-			StartPeriod: 1 * time.Second,
-		})
-}
-
-func opGethReadyFn(ctx context.Context, service *Service) error {
-	opGethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-	return waitForFirstBlock(ctx, opGethURL, 60*time.Second)
-}
+		WithArtifact("/data/p2p_key.txt", o.Enode.Artifact)
 
-func opGethWatchdogFn(out io.Writer, service *Service, ctx context.Context) error {
-	gethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-	return watchChainHead(out, gethURL, 2*time.Second)
+	UseHealthmon(manifest, svc)
 }
 
 type RethEL struct {
@@ -480,24 +458,11 @@ func (r *RethEL) Apply(manifest *Manifest) {
 			logLevelToRethVerbosity(manifest.ctx.LogLevel),
 		).
 		WithRelease(rethELRelease).
-		WithWatchdog(func(out io.Writer, service *Service, ctx context.Context) error {
-			rethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-			return watchChainHead(out, rethURL, 12*time.Second)
-		}).
-		WithReadyFn(func(ctx context.Context, service *Service) error {
-			elURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-			return waitForFirstBlock(ctx, elURL, 60*time.Second)
-		}).
 		WithArtifact("/data/genesis.json", "genesis.json").
 		WithArtifact("/data/jwtsecret", "jwtsecret").
-		WithVolume("data", "/data_reth").
-		WithReady(ReadyCheck{
-			QueryURL:    "http://localhost:8545",
-			Interval:    1 * time.Second,
-			Timeout:     10 * time.Second,
-			Retries:     20,
-			StartPeriod: 1 * time.Second,
-		})
+		WithVolume("data", "/data_reth")
+
+	UseHealthmon(manifest, svc)
 
 	if r.UseNativeReth {
 		// we need to use this otherwise the db cannot be binded
@@ -615,7 +580,6 @@ func (m *MevBoostRelay) Apply(manifest *Manifest) {
 		WithEnv("ALLOW_SYNCING_BEACON_NODE", "1").
 		WithEntrypoint("mev-boost-relay").
 		DependsOnHealthy(m.BeaconClient).
-		WithWatchdog(mevboostRelayWatchdogFn).
 		WithArgs(
 			"--api-listen-addr", "0.0.0.0",
 			"--api-listen-port", `{{Port "http" 5555}}`,
@@ -627,20 +591,6 @@ func (m *MevBoostRelay) Apply(manifest *Manifest) {
 	}
 }
 
-func mevboostRelayWatchdogFn(out io.Writer, service *Service, ctx context.Context) error {
-	beaconNodeURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-
-	watchGroup := newWatchGroup()
-	watchGroup.watch(func() error {
-		return watchProposerPayloads(beaconNodeURL)
-	})
-	watchGroup.watch(func() error {
-		return validateProposerPayloads(out, beaconNodeURL)
-	})
-
-	return watchGroup.wait()
-}
-
 type OpReth struct{}
 
 var opRethRelease = &release{
@@ -661,7 +611,7 @@ var opRethRelease = &release{
 }
 
 func (o *OpReth) Apply(manifest *Manifest) {
-	manifest.NewService("op-reth").
+	svc := manifest.NewService("op-reth").
 		WithImage("ghcr.io/paradigmxyz/op-reth").
 		WithTag("nightly").
 		WithEntrypoint("op-reth").
@@ -681,20 +631,11 @@ func (o *OpReth) Apply(manifest *Manifest) {
 			"--addr", "0.0.0.0",
 			"--port", `{{Port "rpc" 30303}}`).
 		WithRelease(opRethRelease).
-		WithWatchdog(func(out io.Writer, service *Service, ctx context.Context) error {
-			rethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
-			return watchChainHead(out, rethURL, 2*time.Second)
-		}).
 		WithArtifact("/data/jwtsecret", "jwtsecret").
 		WithArtifact("/data/l2-genesis.json", "l2-genesis.json").
-		WithVolume("data", "/data_op_reth").
-		WithReady(ReadyCheck{
-			QueryURL:    "http://localhost:8545",
-			Interval:    1 * time.Second,
-			Timeout:     10 * time.Second,
-			Retries:     20,
-			StartPeriod: 1 * time.Second,
-		})
+		WithVolume("data", "/data_op_reth")
+
+	UseHealthmon(manifest, svc)
 }
 
 type MevBoost struct {
@@ -933,3 +874,18 @@ func (b *BuilderHub) Apply(manifest *Manifest) {
 		WithEnv("TARGET", Connect("web", "http")).
 		DependsOnHealthy("web")
 }
+
+func UseHealthmon(m *Manifest, s *Service) {
+	m.NewService(s.Name+"_healthmon").
+		WithImage("ghcr.io/flashbots/ethereum-healthmon").
+		WithTag("v0.0.1").
+		// TODO: Use this also for beacon node
+		WithArgs("--chain", "execution", "--url", Connect(s.Name, "http")).
+		WithReady(ReadyCheck{
+			Test:        []string{"CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:21171/ready"},
+			Interval:    1 * time.Second,
+			Timeout:     10 * time.Second,
+			Retries:     20,
+			StartPeriod: 1 * time.Second,
+		})
+}

playground/components_test.go

@@ -12,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/stretchr/testify/require"
 )
 
@@ -143,8 +145,6 @@ func (tt *testFramework) test(s ServiceGen, args []string) *Manifest {
 	require.NoError(t, err)
 
 	require.NoError(t, dockerRunner.WaitForReady(context.Background(), 20*time.Second))
-	require.NoError(t, CompleteReady(context.Background(), svcManager.Services))
-
 	return svcManager
 }
 
@@ -164,3 +164,29 @@ func toSnakeCase(s string) string {
 	// Convert to lowercase
 	return strings.ToLower(snake)
 }
+
+func waitForBlock(elURL string, targetBlock uint64, timeout time.Duration) error {
+	rpcClient, err := rpc.Dial(elURL)
+	if err != nil {
+		return fmt.Errorf("failed to connect to %s: %w", elURL, err)
+	}
+	defer rpcClient.Close()
+
+	clt := ethclient.NewClient(rpcClient)
+	timeoutCh := time.After(timeout)
+
+	for {
+		select {
+		case <-timeoutCh:
+			return fmt.Errorf("timeout waiting for block %d on %s", targetBlock, elURL)
+		case <-time.After(500 * time.Millisecond):
+			num, err := clt.BlockNumber(context.Background())
+			if err != nil {
+				continue
+			}
+			if num >= targetBlock {
+				return nil
+			}
+		}
+	}
+}