diff --git a/pkg/cmd/run_node.go b/pkg/cmd/run_node.go index 30f8489822..fe42707f85 100644 --- a/pkg/cmd/run_node.go +++ b/pkg/cmd/run_node.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "strings" "syscall" "time" @@ -155,8 +156,10 @@ func StartNode( go func() { defer func() { if r := recover(); r != nil { - err := fmt.Errorf("node panicked: %v", r) - logger.Error().Interface("panic", r).Msg("Recovered from panic in node") + buf := make([]byte, 1024) + n := runtime.Stack(buf, false) + err := fmt.Errorf("node panicked: %v\nstack trace:\n%s", r, buf[:n]) + logger.Error().Interface("panic", r).Str("stacktrace", string(buf[:n])).Msg("Recovered from panic in node") select { case errCh <- err: default: diff --git a/test/e2e/sut_helper.go b/test/e2e/sut_helper.go index fca77a122c..3aaa032a0a 100644 --- a/test/e2e/sut_helper.go +++ b/test/e2e/sut_helper.go @@ -76,25 +76,33 @@ func (s *SystemUnderTest) RunCmd(cmd string, args ...string) (string, error) { } // ExecCmd starts a process for the given command and manages it cleanup on test end. -func (s *SystemUnderTest) ExecCmd(cmd string, args ...string) { +func (s *SystemUnderTest) ExecCmd(cmd string, args ...string) *os.Process { + return s.ExecCmdWithLogPrefix("", cmd, args...) +} + +// ExecCmdWithLogPrefix same as ExecCmd but prepends the given prefix to the log output +func (s *SystemUnderTest) ExecCmdWithLogPrefix(prefix, cmd string, args ...string) *os.Process { + executable := locateExecutable(cmd) c := exec.Command( //nolint:gosec // used by tests only executable, args..., ) c.Dir = WorkDir - s.watchLogs(c) + s.watchLogs(prefix, c) err := c.Start() require.NoError(s.t, err) if s.debug { - s.logf("Exec cmd (pid: %d): %s %s", c.Process.Pid, executable, strings.Join(c.Args, " ")) + s.logf("Exec cmd (pid: %d): %s %s", c.Process.Pid, executable, strings.Join(args, " ")) } // cleanup when stopped s.awaitProcessCleanup(c) + return c.Process } // AwaitNodeUp waits until a node is operational by checking both liveness and readiness. +// Fails tests when node is not up within the specified timeout. func (s *SystemUnderTest) AwaitNodeUp(t *testing.T, rpcAddr string, timeout time.Duration) { t.Helper() t.Logf("Await node is up: %s", rpcAddr) @@ -168,7 +176,7 @@ func (s *SystemUnderTest) awaitProcessCleanup(cmd *exec.Cmd) { }() } -func (s *SystemUnderTest) watchLogs(cmd *exec.Cmd) { +func (s *SystemUnderTest) watchLogs(prefix string, cmd *exec.Cmd) { errReader, err := cmd.StderrPipe() require.NoError(s.t, err) outReader, err := cmd.StdoutPipe() @@ -178,7 +186,7 @@ func (s *SystemUnderTest) watchLogs(cmd *exec.Cmd) { logDir := filepath.Join(WorkDir, "testnet") require.NoError(s.t, os.MkdirAll(logDir, 0o750)) testName := strings.ReplaceAll(s.t.Name(), "/", "-") - logfileName := filepath.Join(logDir, fmt.Sprintf("exec-%s-%s-%d.out", filepath.Base(cmd.Args[0]), testName, time.Now().UnixNano())) + logfileName := filepath.Join(logDir, prefix+fmt.Sprintf("exec-%s-%s-%d.out", filepath.Base(cmd.Args[0]), testName, time.Now().UnixNano())) logfile, err := os.Create(logfileName) require.NoError(s.t, err) errReader = io.NopCloser(io.TeeReader(errReader, logfile))