agreement: dump demux queues on fuzzer test failure (#5882)

5 files changed, 33 insertions, 2 deletions

diff --git a/agreement/demux.go b/agreement/demux.go
index 7f888dc97..2653b8d8c 100644
--- a/agreement/demux.go
+++ b/agreement/demux.go
@@ -19,6 +19,7 @@ package agreement
 import (
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/logging"
@@ -361,6 +362,17 @@ func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, curre
 	return
 }
 
+// dumpQueues dumps the current state of the demux queues to the given writer.
+func (d *demux) dumpQueues(w io.Writer) {
+	fmt.Fprintf(w, "rawVotes: %d\n", len(d.rawVotes))
+	fmt.Fprintf(w, "rawProposals: %d\n", len(d.rawProposals))
+	fmt.Fprintf(w, "rawBundles: %d\n", len(d.rawBundles))
+
+	fmt.Fprintf(w, "cryptoVerifiedVotes: %d\n", len(d.crypto.VerifiedVotes()))
+	fmt.Fprintf(w, "cryptoVerified ProposalPayloadTag: %d\n", len(d.crypto.Verified(protocol.ProposalPayloadTag)))
+	fmt.Fprintf(w, "cryptoVerified VoteBundleTag: %d\n", len(d.crypto.Verified(protocol.VoteBundleTag)))
+}
+
 // setupCompoundMessage processes compound messages: distinct messages which are delivered together
 func setupCompoundMessage(l LedgerReader, m message) (res externalEvent) {
 	compound := m.CompoundMessage
diff --git a/agreement/fuzzer/fuzzer_test.go b/agreement/fuzzer/fuzzer_test.go
index e505faa19..77a52401e 100644
--- a/agreement/fuzzer/fuzzer_test.go
+++ b/agreement/fuzzer/fuzzer_test.go
@@ -261,6 +261,13 @@ func (n *Fuzzer) Start() {
 	}
 }
 
+// DumpQueues dumps the queues of all the nodes.
+func (n *Fuzzer) DumpQueues() {
+	for _, f := range n.agreements {
+		f.DumpDemuxQueues(os.Stderr)
+	}
+}
+
 func (n *Fuzzer) InvokeFiltersShutdown(preshutdown bool) {
 	for _, facade := range n.facades {
 		dsFilter := facade.GetDownstreamFilter()
diff --git a/agreement/fuzzer/networkFacade_test.go b/agreement/fuzzer/networkFacade_test.go
index d18f8251d..86c7ff712 100644
--- a/agreement/fuzzer/networkFacade_test.go
+++ b/agreement/fuzzer/networkFacade_test.go
@@ -122,7 +122,7 @@ func (n *NetworkFacade) DumpQueues() {
 	}
 	n.eventsQueuesMu.Unlock()
 	queues += "----------------------\n"
-	fmt.Printf(queues)
+	fmt.Print(queues)
 }
 
 func (n *NetworkFacade) WaitForEventsQueue(cleared bool) {
@@ -151,7 +151,7 @@ func (n *NetworkFacade) WaitForEventsQueue(cleared bool) {
 				n.DumpQueues()
 				//panic("Waiting for event processing for 0 took too long")
 				pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
-				os.Exit(1)
+				panic(fmt.Sprintf("maxWait %d sec exceeded", maxEventQueueWait/time.Second))
 			}
 
 		}
diff --git a/agreement/fuzzer/validator_test.go b/agreement/fuzzer/validator_test.go
index 87425a099..8c7352ccc 100644
--- a/agreement/fuzzer/validator_test.go
+++ b/agreement/fuzzer/validator_test.go
@@ -47,6 +47,12 @@ func (v *Validator) Go(netConfig *FuzzerConfig) {
 	network := MakeFuzzer(*netConfig)
 	require.NotNil(v.tb, network)
 
+	defer func() {
+		if r := recover(); r != nil {
+			network.DumpQueues()
+		}
+	}()
+
 	network.Start()
 	//_, runRes := network.Run(v.config.NetworkRunDuration /*time.Millisecond*5000*/, time.Millisecond*3000, time.Second)
 	_, v.runResult = network.Run(v.config.NetworkRunTicks, v.config.NetworkRecoverTicks, 100)
diff --git a/agreement/service.go b/agreement/service.go
index 433a17d5c..852f17145 100644
--- a/agreement/service.go
+++ b/agreement/service.go
@@ -19,6 +19,7 @@ package agreement
 //go:generate dbgen -i agree.sql -p agreement -n agree -o agreeInstall.go -h ../scripts/LICENSE_HEADER
 import (
 	"context"
+	"io"
 	"time"
 
 	"github.com/algorand/go-algorand/config"
@@ -178,6 +179,11 @@ func (s *Service) Shutdown() {
 	s.persistenceLoop.Quit()
 }
 
+// DumpDemuxQueues dumps the demux queues to the given writer.
+func (s *Service) DumpDemuxQueues(w io.Writer) {
+	s.demux.dumpQueues(w)
+}
+
 // demuxLoop repeatedly executes pending actions and then requests the next event from the Service.demux.
 func (s *Service) demuxLoop(ctx context.Context, input chan<- externalEvent, output <-chan []action, ready <-chan externalDemuxSignals) {
 	for a := range output {