Merge pull request #3625 from Algo-devops-service/relbeta3.4.0v3.4.0-beta

go-algorand 3.4.0-beta 

8 files changed, 99 insertions, 21 deletions

diff --git a/crypto/merklesignature/persistentMerkleSignatureScheme.go b/crypto/merklesignature/persistentMerkleSignatureScheme.go
index cbcc44b70..1ce67c8d5 100644
--- a/crypto/merklesignature/persistentMerkleSignatureScheme.go
+++ b/crypto/merklesignature/persistentMerkleSignatureScheme.go
@@ -41,7 +41,8 @@ var (
 	errKeyDecodeError  = errors.New("failed to decode stateproof key")
 )
 
-func merkleSignatureInstallDatabase(tx *sql.Tx) error {
+// InstallStateProofTable creates (or migrates if exists already) the StateProofKeys database table
+func InstallStateProofTable(tx *sql.Tx) error {
 	var schemaVersion sql.NullInt32
 	err := tx.QueryRow("SELECT version FROM schema where tablename = ?", merkleSignatureTableSchemaName).Scan(&schemaVersion)
 	switch err {
@@ -99,7 +100,7 @@ func (s *Secrets) Persist(store db.Accessor) error {
 	round := indexToRound(s.FirstValid, s.Interval, 0)
 	encodedKey := protocol.GetEncodingBuf()
 	err := store.Atomic(func(ctx context.Context, tx *sql.Tx) error {
-		err := merkleSignatureInstallDatabase(tx) // assumes schema table already exists (created by partInstallDatabase)
+		err := InstallStateProofTable(tx) // assumes schema table already exists (created by partInstallDatabase)
 		if err != nil {
 			return err
 		}
diff --git a/crypto/merklesignature/persistentMerkleSignatureScheme_test.go b/crypto/merklesignature/persistentMerkleSignatureScheme_test.go
index a736a9e1b..12aa8a02d 100644
--- a/crypto/merklesignature/persistentMerkleSignatureScheme_test.go
+++ b/crypto/merklesignature/persistentMerkleSignatureScheme_test.go
@@ -48,7 +48,7 @@ func TestSecretsDatabaseUpgrade(t *testing.T) {
 	a.NoError(err)
 
 	err = store.Atomic(func(ctx context.Context, tx *sql.Tx) error {
-		err := merkleSignatureInstallDatabase(tx) // assumes schema table already exists (created by partInstallDatabase)
+		err := InstallStateProofTable(tx) // assumes schema table already exists (created by partInstallDatabase)
 		if err != nil {
 			return err
 		}
@@ -56,6 +56,9 @@ func TestSecretsDatabaseUpgrade(t *testing.T) {
 	})
 
 	a.NoError(err)
+	version, err := getStateProofTableSchemaVersions(*store)
+	a.NoError(err)
+	a.Equal(merkleSignatureSchemaVersion, version)
 }
 
 func TestFetchRestoreAllSecrets(t *testing.T) {
@@ -114,3 +117,18 @@ func createTestDB(a *require.Assertions) *db.Accessor {
 
 	return &store
 }
+
+func getStateProofTableSchemaVersions(db db.Accessor) (int, error) {
+	var version int
+	err := db.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
+		row := tx.QueryRow("SELECT version FROM schema where tablename = ?", merkleSignatureTableSchemaName)
+		return row.Scan(&version)
+	})
+	if err == sql.ErrNoRows {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	return version, nil
+}
diff --git a/data/account/account.go b/data/account/account.go
index 47effa120..8e7bb5337 100644
--- a/data/account/account.go
+++ b/data/account/account.go
@@ -196,12 +196,16 @@ func RestoreParticipation(store db.Accessor) (acc PersistedParticipation, err er
 
 // RestoreParticipationWithSecrets restores a Participation from a database
 // handle. In addition, this function also restores all stateproof secrets
-func RestoreParticipationWithSecrets(store db.Accessor) (acc PersistedParticipation, err error) {
+func RestoreParticipationWithSecrets(store db.Accessor) (PersistedParticipation, error) {
 	persistedParticipation, err := RestoreParticipation(store)
 	if err != nil {
 		return PersistedParticipation{}, err
 	}
 
+	if persistedParticipation.StateProofSecrets == nil { // no state proof keys to restore
+		return persistedParticipation, nil
+	}
+
 	err = persistedParticipation.StateProofSecrets.RestoreAllSecrets(store)
 	if err != nil {
 		return PersistedParticipation{}, err
diff --git a/data/account/participation.go b/data/account/participation.go
index 43441cf5a..6b05ff4ba 100644
--- a/data/account/participation.go
+++ b/data/account/participation.go
@@ -48,7 +48,7 @@ type Participation struct {
 
 	VRF    *crypto.VRFSecrets
 	Voting *crypto.OneTimeSignatureSecrets
-	// StateProofSecrets is used to sign compact certificates. might be nil
+	// StateProofSecrets is used to sign compact certificates.
 	StateProofSecrets *merklesignature.Secrets
 
 	// The first and last rounds for which this account is valid, respectively.
@@ -303,7 +303,12 @@ func (part PersistedParticipation) Persist() error {
 // Calls through to the migration helper and returns the result.
 func Migrate(partDB db.Accessor) error {
 	return partDB.Atomic(func(ctx context.Context, tx *sql.Tx) error {
-		return partMigrate(tx)
+		err := partMigrate(tx)
+		if err != nil {
+			return err
+		}
+
+		return merklesignature.InstallStateProofTable(tx)
 	})
 }
 
diff --git a/data/account/participation_test.go b/data/account/participation_test.go
index 2ba7d3025..843509f96 100644
--- a/data/account/participation_test.go
+++ b/data/account/participation_test.go
@@ -205,9 +205,15 @@ func TestRetrieveFromDBAtVersion1(t *testing.T) {
 	retrivedPart, err := RestoreParticipation(partDB)
 	a.NoError(err)
 	assertionForRestoringFromDBAtLowVersion(a, retrivedPart)
+	assertStateProofTablesExists(a, partDB)
+
+	retrivedPart, err = RestoreParticipationWithSecrets(partDB)
+	a.NoError(err)
+	assertionForRestoringFromDBAtLowVersion(a, retrivedPart)
+	assertStateProofTablesExists(a, partDB)
 }
 
-func TestRetriveFromDBAtVersion2(t *testing.T) {
+func TestRetrieveFromDBAtVersion2(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
 	a := require.New(t)
@@ -222,6 +228,18 @@ func TestRetriveFromDBAtVersion2(t *testing.T) {
 	retrivedPart, err := RestoreParticipation(partDB)
 	a.NoError(err)
 	assertionForRestoringFromDBAtLowVersion(a, retrivedPart)
+	assertStateProofTablesExists(a, partDB)
+	versions, err := getSchemaVersions(partDB)
+	a.NoError(err)
+	a.Equal(versions[PartTableSchemaName], PartTableSchemaVersion)
+
+	retrivedPart, err = RestoreParticipationWithSecrets(partDB)
+	a.NoError(err)
+	assertionForRestoringFromDBAtLowVersion(a, retrivedPart)
+	assertStateProofTablesExists(a, partDB)
+	versions, err = getSchemaVersions(partDB)
+	a.NoError(err)
+	a.Equal(versions[PartTableSchemaName], PartTableSchemaVersion)
 }
 
 func TestKeyRegCreation(t *testing.T) {
@@ -244,6 +262,14 @@ func closeDBS(dbAccessor ...db.Accessor) {
 	}
 }
 
+func assertStateProofTablesExists(a *require.Assertions, store db.Accessor) {
+	err := store.Atomic(func(ctx context.Context, tx *sql.Tx) error {
+		_, err := tx.Exec("select count(*) From StateProofKeys;")
+		return err
+	})
+	a.NoError(err)
+
+}
 func assertionForRestoringFromDBAtLowVersion(a *require.Assertions, retrivedPart PersistedParticipation) {
 	a.NotNil(retrivedPart)
 	a.Nil(retrivedPart.StateProofSecrets)
diff --git a/node/node.go b/node/node.go
index c9aaffe46..40fa4cdb6 100644
--- a/node/node.go
+++ b/node/node.go
@@ -1006,6 +1006,10 @@ func (node *AlgorandFullNode) loadParticipationKeys() error {
 
 func insertStateProofToRegistry(part account.PersistedParticipation, node *AlgorandFullNode) error {
 	partID := part.ID()
+	// in case there are no state proof keys for that participant
+	if part.StateProofSecrets == nil {
+		return nil
+	}
 	keys := part.StateProofSecrets.GetAllKeys()
 	keysSinger := make(account.StateProofKeys, len(keys))
 	for i := uint64(0); i < uint64(len(keys)); i++ {
diff --git a/test/e2e-go/features/participation/participationExpiration_test.go b/test/e2e-go/features/participation/participationExpiration_test.go
index 795dd9c98..126b5acf0 100644
--- a/test/e2e-go/features/participation/participationExpiration_test.go
+++ b/test/e2e-go/features/participation/participationExpiration_test.go
@@ -46,9 +46,9 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 	accountList, err := fixture.GetWalletsSortedByBalance()
 	a.NoError(err)
 	richAccount := accountList[0].Address
-	_, initialRound := fixture.GetBalanceAndRound(richAccount)
+	latestRound := fetchLatestRound(fixture, a)
 
-	minTxnFee, minAcctBalance, err := fixture.MinFeeAndBalance(initialRound)
+	minTxnFee, minAcctBalance, err := fixture.MinFeeAndBalance(latestRound)
 	a.NoError(err)
 
 	transactionFee := minTxnFee
@@ -57,7 +57,7 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 	initialAmt, err := sClient.GetBalance(sAccount)
 	a.NoError(err)
 
-	fixture.SendMoneyAndWait(initialRound, amountToSendInitial, transactionFee, richAccount, sAccount, "")
+	fixture.SendMoneyAndWait(latestRound, amountToSendInitial, transactionFee, richAccount, sAccount, "")
 
 	newAmt, err := sClient.GetBalance(sAccount)
 	a.NoError(err)
@@ -73,7 +73,8 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 
 	startTime := time.Now()
 	for time.Since(startTime) < 2*time.Minute {
-		_, currentRound := fixture.GetBalanceAndRound(richAccount)
+		currentRound := fetchLatestRound(fixture, a)
+
 		// account adds part key
 		partKeyFirstValid := uint64(0)
 		partKeyValidityPeriod := uint64(10)
@@ -110,14 +111,24 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 	a.NoError(err)
 	seededRound := sNodeStatus.LastRound
 
-	fixture.WaitForTxnConfirmation(seededRound+maxRoundsToWaitForTxnConfirm, sAccount, onlineTxID)
-	sNodeStatus, _ = sClient.Status()
+	txnConfirmed := fixture.WaitForTxnConfirmation(seededRound+maxRoundsToWaitForTxnConfirm, sAccount, onlineTxID)
+	a.True(txnConfirmed)
+
 	newAccountStatus, err = pClient.AccountInformation(sAccount)
 	a.NoError(err)
 	a.Equal(basics.Online.String(), newAccountStatus.Status)
-	sAccountData, err := sClient.AccountData(sAccount)
+
+	// get the round number of the primary node
+	pNodeStatus, err := pClient.Status()
 	a.NoError(err)
 
+	// ensure the secondary node reaches that number
+	_, err = sClient.WaitForRound(pNodeStatus.LastRound)
+	a.NoError(err)
+
+	// get the account data ( which now is syncronized across the network )
+	sAccountData, err := sClient.AccountData(sAccount)
+	a.NoError(err)
 	lastValidRound := sAccountData.VoteLastValid
 
 	a.Equal(basics.Round(partKeyLastValid), lastValidRound)
@@ -133,15 +144,20 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 	// Now we want to send a transaction to the account and test that
 	// it was taken offline after we sent it something
 
-	_, initialRound = fixture.GetBalanceAndRound(richAccount)
+	latestRound = fetchLatestRound(fixture, a)
+
+	// making certain sClient has the same blocks as pClient.
+	_, err = sClient.WaitForRound(uint64(lastValidRound + 1))
+	a.NoError(err)
 
-	blk, err := sClient.Block(initialRound)
+	blk, err := sClient.Block(latestRound)
 	a.NoError(err)
 	a.Equal(blk.CurrentProtocol, protocolCheck)
 
-	fixture.SendMoneyAndWait(initialRound, amountToSendInitial, transactionFee, richAccount, sAccount, "")
+	sendMoneyTxn := fixture.SendMoneyAndWait(latestRound, amountToSendInitial, transactionFee, richAccount, sAccount, "")
 
-	err = fixture.WaitForRoundWithTimeout(uint64(initialRound) + 3)
+	txnConfirmed = fixture.WaitForTxnConfirmation(latestRound+maxRoundsToWaitForTxnConfirm, sAccount, sendMoneyTxn.TxID)
+	a.True(txnConfirmed)
 
 	newAccountStatus, err = pClient.AccountInformation(sAccount)
 	a.NoError(err)
@@ -150,6 +166,12 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 	a.Equal(finalStatus.String(), newAccountStatus.Status)
 }
 
+func fetchLatestRound(fixture *fixtures.RestClientFixture, a *require.Assertions) uint64 {
+	status, err := fixture.LibGoalClient.Status()
+	a.NoError(err)
+	return status.LastRound
+}
+
 // TestParticipationAccountsExpirationFuture tests that sending a transaction to an account with
 // its last valid round being less than the current round will turn it offline.  This test will only
 // work when the consensus protocol enables it (in this case the future protocol)
diff --git a/test/e2e-go/upgrades/stateproof_test.go b/test/e2e-go/upgrades/stateproof_test.go
index ba9c72505..6f9cf7f13 100644
--- a/test/e2e-go/upgrades/stateproof_test.go
+++ b/test/e2e-go/upgrades/stateproof_test.go
@@ -85,11 +85,9 @@ func TestKeysWithoutStateProofKeyCanRegister(t *testing.T) {
 	defer fixtures.ShutdownSynchronizedTest(t)
 
 	a := require.New(fixtures.SynchronizedTest(t))
-	consensus := getStateProofConsensus()
 
 	var fixture fixtures.RestClientFixture
-	fixture.SetConsensus(consensus)
-	fixture.Setup(t, filepath.Join("nettemplates", "TwoNodesWithoutStateProofPartkeys.json"))
+	fixture.Setup(t, filepath.Join("nettemplates", "TwoNodes50EachV30.json"))
 	defer fixture.Shutdown()
 	lastValid := uint64(1000 * 5)