catchpointdump: allow print filters (#3566)

## Summary

Added a new option `--exclude-fields` to `file` subcommand.
This simplifies large dumps comparison by excluding all exclude some header fields.
Also moved some shared cmd code into a new cmdutil package.

## Test Plan

Tested manually

6 files changed, 193 insertions, 89 deletions

diff --git a/cmd/catchpointdump/database.go b/cmd/catchpointdump/database.go
index 4d8282d40..c1ec554e2 100644
--- a/cmd/catchpointdump/database.go
+++ b/cmd/catchpointdump/database.go
@@ -58,7 +58,7 @@ var databaseCmd = &cobra.Command{
 			}
 			defer outFile.Close()
 		}
-		err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile)
+		err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
 		if err != nil {
 			reportErrorf("Unable to print account database : %v", err)
 		}
diff --git a/cmd/catchpointdump/file.go b/cmd/catchpointdump/file.go
index 3335575b0..b6da6029a 100644
--- a/cmd/catchpointdump/file.go
+++ b/cmd/catchpointdump/file.go
@@ -30,6 +30,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	cmdutil "github.com/algorand/go-algorand/cmd/util"
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/ledger"
@@ -41,10 +42,12 @@ import (
 
 var tarFile string
 var outFileName string
+var excludedFields *cmdutil.CobraStringSliceValue = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
 
 func init() {
 	fileCmd.Flags().StringVarP(&tarFile, "tar", "t", "", "Specify the tar file to process")
 	fileCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )")
+	fileCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]")
 }
 
 var fileCmd = &cobra.Command{
@@ -108,7 +111,7 @@ var fileCmd = &cobra.Command{
 			defer outFile.Close()
 		}
 
-		err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile)
+		err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
 		if err != nil {
 			reportErrorf("Unable to print account database : %v", err)
 		}
@@ -187,7 +190,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
 	fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
 }
 
-func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File) error {
+func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
 	lastProgressUpdate := time.Now()
 	progress := uint64(0)
 	defer printDumpingCatchpointProgressLine(0, 0, 0)
@@ -200,14 +203,54 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 		return err
 	}
 	if fileHeader.Version != 0 {
-		fmt.Fprintf(fileWriter, "Version: %d\nBalances Round: %d\nBlock Round: %d\nBlock Header Digest: %s\nCatchpoint: %s\nTotal Accounts: %d\nTotal Chunks: %d\n",
+		var headerFields = []string{
+			"Version: %d",
+			"Balances Round: %d",
+			"Block Round: %d",
+			"Block Header Digest: %s",
+			"Catchpoint: %s",
+			"Total Accounts: %d",
+			"Total Chunks: %d",
+		}
+		var headerValues = []interface{}{
 			fileHeader.Version,
 			fileHeader.BalancesRound,
 			fileHeader.BlocksRound,
 			fileHeader.BlockHeaderDigest.String(),
 			fileHeader.Catchpoint,
 			fileHeader.TotalAccounts,
-			fileHeader.TotalChunks)
+			fileHeader.TotalChunks,
+		}
+		// safety check
+		if len(headerFields) != len(headerValues) {
+			return fmt.Errorf("printing failed: header formatting mismatch")
+		}
+
+		var actualFields []string
+		var actualValues []interface{}
+		if len(excludeFields) == 0 {
+			actualFields = headerFields
+			actualValues = headerValues
+		} else {
+			actualFields = make([]string, 0, len(headerFields)-len(excludeFields))
+			actualValues = make([]interface{}, 0, len(headerFields)-len(excludeFields))
+			for i, field := range headerFields {
+				lower := strings.ToLower(field)
+				excluded := false
+				for _, filter := range excludeFields {
+					if strings.HasPrefix(lower, filter) {
+						excluded = true
+						break
+					}
+				}
+				if !excluded {
+					actualFields = append(actualFields, field)
+					actualValues = append(actualValues, headerValues[i])
+				}
+			}
+		}
+
+		fmt.Fprintf(fileWriter, strings.Join(actualFields, "\n")+"\n", actualValues...)
 
 		totals := fileHeader.Totals
 		fmt.Fprintf(fileWriter, "AccountTotals - Online Money: %d\nAccountTotals - Online RewardUnits : %d\nAccountTotals - Offline Money: %d\nAccountTotals - Offline RewardUnits : %d\nAccountTotals - Not Participating Money: %d\nAccountTotals - Not Participating Money RewardUnits: %d\nAccountTotals - Rewards Level: %d\n",
diff --git a/cmd/catchpointdump/net.go b/cmd/catchpointdump/net.go
index f4a31bff9..9e46fea14 100644
--- a/cmd/catchpointdump/net.go
+++ b/cmd/catchpointdump/net.go
@@ -277,7 +277,7 @@ func makeFileDump(addr string, tarFile string) error {
 	if err != nil {
 		return err
 	}
-	err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile)
+	err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, nil)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/goal/common.go b/cmd/goal/common.go
index a392410e7..56dde3af7 100644
--- a/cmd/goal/common.go
+++ b/cmd/goal/common.go
@@ -17,10 +17,9 @@
 package main
 
 import (
-	"fmt"
-	"strings"
-
 	"github.com/spf13/cobra"
+
+	cmdutil "github.com/algorand/go-algorand/cmd/util"
 )
 
 const (
@@ -52,7 +51,7 @@ var (
 	dumpForDryrunAccts []string
 )
 
-var dumpForDryrunFormat cobraStringValue = *makeCobraStringValue("json", []string{"msgp"})
+var dumpForDryrunFormat cmdutil.CobraStringValue = *cmdutil.MakeCobraStringValue("json", []string{"msgp"})
 
 func addTxnFlags(cmd *cobra.Command) {
 	cmd.Flags().Uint64Var(&fee, "fee", 0, "The transaction fee (automatically determined by default), in microAlgos")
@@ -69,39 +68,3 @@ func addTxnFlags(cmd *cobra.Command) {
 	cmd.Flags().Var(&dumpForDryrunFormat, "dryrun-dump-format", "Dryrun dump format: "+dumpForDryrunFormat.AllowedString())
 	cmd.Flags().StringSliceVar(&dumpForDryrunAccts, "dryrun-accounts", nil, "additional accounts to include into dryrun request obj")
 }
-
-type cobraStringValue struct {
-	value   string
-	allowed []string
-	isSet   bool
-}
-
-func makeCobraStringValue(value string, others []string) *cobraStringValue {
-	c := new(cobraStringValue)
-	c.value = value
-	c.allowed = make([]string, 0, len(others)+1)
-	c.allowed = append(c.allowed, value)
-	for _, s := range others {
-		c.allowed = append(c.allowed, s)
-	}
-	return c
-}
-
-func (c *cobraStringValue) String() string { return c.value }
-func (c *cobraStringValue) Type() string   { return "string" }
-func (c *cobraStringValue) IsSet() bool    { return c.isSet }
-
-func (c *cobraStringValue) Set(other string) error {
-	for _, s := range c.allowed {
-		if other == s {
-			c.value = other
-			c.isSet = true
-			return nil
-		}
-	}
-	return fmt.Errorf("value %s not allowed", other)
-}
-
-func (c *cobraStringValue) AllowedString() string {
-	return strings.Join(c.allowed, ", ")
-}
diff --git a/cmd/tealdbg/main.go b/cmd/tealdbg/main.go
index 288db8709..97701b979 100644
--- a/cmd/tealdbg/main.go
+++ b/cmd/tealdbg/main.go
@@ -17,14 +17,14 @@
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
-	"strings"
 
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
+
+	cmdutil "github.com/algorand/go-algorand/cmd/util"
 )
 
 func main() {
@@ -65,49 +65,16 @@ var remoteCmd = &cobra.Command{
 	},
 }
 
-// cobraStringValue is a cobra's string flag with restricted values
-type cobraStringValue struct {
-	value   string
-	allowed []string
-	isSet   bool
-}
-
-func makeCobraStringValue(value string, others []string) *cobraStringValue {
-	c := new(cobraStringValue)
-	c.value = value
-	c.allowed = make([]string, 0, len(others)+1)
-	c.allowed = append(c.allowed, value)
-	for _, s := range others {
-		c.allowed = append(c.allowed, s)
-	}
-	return c
-}
-
-func (c *cobraStringValue) String() string { return c.value }
-func (c *cobraStringValue) Type() string   { return "string" }
-func (c *cobraStringValue) IsSet() bool    { return c.isSet }
-
-func (c *cobraStringValue) Set(other string) error {
-	for _, s := range c.allowed {
-		if other == s {
-			c.value = other
-			c.isSet = true
-			return nil
-		}
-	}
-	return fmt.Errorf("value %s not allowed", other)
-}
-
-func (c *cobraStringValue) AllowedString() string {
-	return strings.Join(c.allowed, ", ")
+type frontendValue struct {
+	*cmdutil.CobraStringValue
 }
 
-type frontendValue struct {
-	*cobraStringValue
+func (f *frontendValue) value() string {
+	return f.CobraStringValue.String()
 }
 
 func (f *frontendValue) Make(router *mux.Router, appAddress string) (da DebugAdapter) {
-	switch f.value {
+	switch f.value() {
 	case "web":
 		wa := MakeWebPageFrontend(&WebPageFrontendParams{router, appAddress})
 		return wa
@@ -120,10 +87,10 @@ func (f *frontendValue) Make(router *mux.Router, appAddress string) (da DebugAda
 }
 
 type runModeValue struct {
-	*cobraStringValue
+	*cmdutil.CobraStringValue
 }
 
-var frontend frontendValue = frontendValue{makeCobraStringValue("cdt", []string{"web"})}
+var frontend frontendValue = frontendValue{cmdutil.MakeCobraStringValue("cdt", []string{"web"})}
 var proto string
 var txnFile string
 var groupIndex int
@@ -133,7 +100,7 @@ var indexerURL string
 var indexerToken string
 var roundNumber uint64
 var timestamp int64
-var runMode runModeValue = runModeValue{makeCobraStringValue("auto", []string{"signature", "application"})}
+var runMode runModeValue = runModeValue{cmdutil.MakeCobraStringValue("auto", []string{"signature", "application"})}
 var port int
 var iface string
 var noFirstRun bool
diff --git a/cmd/util/cmd.go b/cmd/util/cmd.go
new file mode 100644
index 000000000..e38972e9c
--- /dev/null
+++ b/cmd/util/cmd.go
@@ -0,0 +1,131 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmdutil
+
+import (
+	"fmt"
+	"strings"
+)
+
+// CobraStringValue is similar to spf13.pflag.stringValue but enforces allowed values
+type CobraStringValue struct {
+	value   string
+	allowed []string
+	isSet   bool
+}
+
+// MakeCobraStringValue creates a string value (satisfying spf13/pflag.Value interface)
+// for a limited number of valid options
+func MakeCobraStringValue(value string, others []string) *CobraStringValue {
+	c := new(CobraStringValue)
+	c.value = value
+	c.allowed = make([]string, 0, len(others)+1)
+	c.allowed = append(c.allowed, value)
+	c.allowed = append(c.allowed, others...)
+	return c
+}
+
+func (c *CobraStringValue) String() string { return c.value }
+
+// Type returns the value type as a string
+func (c *CobraStringValue) Type() string { return "string" }
+
+// IsSet returns a boolean flag indicating of the value was changed from defaults
+func (c *CobraStringValue) IsSet() bool { return c.isSet }
+
+// Set sets a value and fails if it is not allowed
+func (c *CobraStringValue) Set(other string) error {
+	for _, s := range c.allowed {
+		if other == s {
+			c.value = other
+			c.isSet = true
+			return nil
+		}
+	}
+	return fmt.Errorf("value %s not allowed", other)
+}
+
+// AllowedString returns a comma-separated string of allowed values
+func (c *CobraStringValue) AllowedString() string {
+	return strings.Join(c.allowed, ", ")
+}
+
+// CobraStringSliceValue is similar to spf13.pflag.stringSliceValue but enforces allowed values
+type CobraStringSliceValue struct {
+	value      []string
+	allowed    []string
+	allowedMap map[string]int
+	isSet      bool
+}
+
+// MakeCobraStringSliceValue creates a string slice value (satisfying spf13/pflag.Value interface)
+// for a limited number of valid options
+func MakeCobraStringSliceValue(value *[]string, others []string) *CobraStringSliceValue {
+	c := new(CobraStringSliceValue)
+	if value != nil {
+		c.value = *value
+	}
+
+	// make allowed values by filtering out duplicates and preseve the order
+	c.allowedMap = make(map[string]int, len(others)+len(c.value))
+	var dups int
+	for i, v := range append(c.value, others...) {
+		if _, ok := c.allowedMap[v]; !ok {
+			c.allowedMap[v] = i - dups
+		} else {
+			dups++
+		}
+	}
+	c.allowed = make([]string, len(c.allowedMap))
+	for v, i := range c.allowedMap {
+		c.allowed[i] = v
+	}
+	return c
+}
+
+func (c *CobraStringSliceValue) String() string { return "[" + strings.Join(c.value, ", ") + "]" }
+
+// Type returns the value type as a string
+func (c *CobraStringSliceValue) Type() string { return "stringSlice" }
+
+// IsSet returns a boolean flag indicating of the value was changed from defaults
+func (c *CobraStringSliceValue) IsSet() bool { return c.isSet }
+
+// Set sets a value and fails if it is not allowed
+func (c *CobraStringSliceValue) Set(values string) error {
+	others := strings.Split(values, ",")
+	for _, other := range others {
+		other = strings.TrimSpace(other)
+		if _, ok := c.allowedMap[other]; ok {
+			c.value = append(c.value, other)
+			c.isSet = true
+		} else {
+			return fmt.Errorf("value %s not allowed", other)
+		}
+	}
+	return nil
+}
+
+// AllowedString returns a comma-separated string of allowed values
+func (c *CobraStringSliceValue) AllowedString() string {
+	return strings.Join(c.allowed, ", ")
+}
+
+// GetSlice returns a current value as a string slice
+func (c *CobraStringSliceValue) GetSlice() []string {
+	return c.value
+}