diff options
author | John Lee <64482439+algojohnlee@users.noreply.github.com> | 2020-08-27 10:03:14 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-08-27 10:03:14 -0400 |
commit | 65e359d67e2a6ada062bf7d5341227a7219f5209 (patch) | |
tree | 883a0b0bfb3970c3ed3c271c558cfd94e7ebabbc | |
parent | 30c8dd683bb9f0d5016527ddd9fa50cac1261385 (diff) | |
parent | 816bc8a1fcfba370c2f3e401f911f97018bea851 (diff) |
Merge pull request #1455 from onetechnical/onetechnical/relstable2.1.4v2.1.4-stable
Onetechnical/relstable2.1.4
34 files changed, 1650 insertions, 148 deletions
diff --git a/buildnumber.dat b/buildnumber.dat index 00750edc0..b8626c4cf 100644 --- a/buildnumber.dat +++ b/buildnumber.dat @@ -1 +1 @@ -3 +4 diff --git a/cmd/pingpong/runCmd.go b/cmd/pingpong/runCmd.go index d8fcc5ec5..8755063c2 100644 --- a/cmd/pingpong/runCmd.go +++ b/cmd/pingpong/runCmd.go @@ -20,7 +20,6 @@ import ( "context" "encoding/base64" "fmt" - "github.com/algorand/go-algorand/data/transactions/logic" "io/ioutil" "os" "path/filepath" @@ -29,6 +28,7 @@ import ( "github.com/spf13/cobra" + "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/libgoal" "github.com/algorand/go-algorand/shared/pingpong" ) @@ -59,6 +59,7 @@ var groupSize uint32 var numAsset uint32 var numApp uint32 var appProgOps uint32 +var rekey bool func init() { rootCmd.AddCommand(runCmd) @@ -91,6 +92,7 @@ func init() { runCmd.Flags().Uint32Var(&numApp, "numapp", 0, "The number of apps each account opts in to") runCmd.Flags().Uint32Var(&appProgOps, "appprogops", 0, "The approximate number of TEAL operations to perform in each ApplicationCall transaction") runCmd.Flags().BoolVar(&randomLease, "randomlease", false, "set the lease to contain a random value") + runCmd.Flags().BoolVar(&rekey, "rekey", false, "Create RekeyTo transactions. Requires groupsize=2 and any of random flags exc random dst") } var runCmd = &cobra.Command{ @@ -254,7 +256,17 @@ var runCmd = &cobra.Command{ } if numAsset != 0 && numApp != 0 { - reportErrorf("only one of numapp and numasset may be specified") + reportErrorf("only one of numapp and numasset may be specified\n") + } + + if rekey { + cfg.Rekey = rekey + if !cfg.RandomLease && !cfg.RandomNote && !cfg.RandomizeFee && !cfg.RandomizeAmt { + reportErrorf("RandomNote, RandomLease, RandomizeFee or RandomizeAmt must be used with rekeying\n") + } + if cfg.GroupSize != 2 { + reportErrorf("Rekeying requires txn groups of size 2\n") + } } reportInfof("Preparing to initialize PingPong with config:\n") diff --git a/config/config.go b/config/config.go index 86bbc6018..4ade54361 100644 --- a/config/config.go +++ b/config/config.go @@ -290,7 +290,8 @@ type Local struct { // NetworkProtocolVersion overrides network protocol version ( if present ) NetworkProtocolVersion string `version[6]:""` - // CatchpointInterval set the interval at which catchpoint are being generated. + // CatchpointInterval sets the interval at which catchpoint are being generated. Setting this to 0 disables the catchpoint from being generated. + // See CatchpointTracking for more details. CatchpointInterval uint64 `version[7]:"10000"` // CatchpointFileHistoryLength defines how many catchpoint files we want to store back. @@ -330,7 +331,7 @@ type Local struct { OptimizeAccountsDatabaseOnStartup bool `version[10]:"false"` // CatchpointTracking determines if catchpoints are going to be tracked. The value is interpreted as follows: - // A of -1 means "don't track catchpoints". + // A value of -1 means "don't track catchpoints". // A value of 1 means "track catchpoints as long as CatchpointInterval is also set to a positive non-zero value". If CatchpointInterval <= 0, no catchpoint tracking would be performed. // A value of 0 means automatic, which is the default value. In this mode, a non archival node would not track the catchpoints, and an archival node would track the catchpoints as long as CatchpointInterval > 0. // Other values of CatchpointTracking would give a warning in the log file, and would behave as if the default value was provided. diff --git a/crypto/merklearray/array.go b/crypto/merklearray/array.go new file mode 100644 index 000000000..d971b2944 --- /dev/null +++ b/crypto/merklearray/array.go @@ -0,0 +1,28 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see <https://www.gnu.org/licenses/>. + +package merklearray + +import ( + "github.com/algorand/go-algorand/crypto" +) + +// An Array represents a dense array of leaf elements that are +// combined into a Merkle tree. Each element must be hashable. +type Array interface { + Length() uint64 + Get(pos uint64) (crypto.Hashable, error) +} diff --git a/crypto/merklearray/layer.go b/crypto/merklearray/layer.go new file mode 100644 index 000000000..f5ccf5835 --- /dev/null +++ b/crypto/merklearray/layer.go @@ -0,0 +1,56 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see <https://www.gnu.org/licenses/>. + +package merklearray + +import ( + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/protocol" +) + +// A layer of the Merkle tree consists of a dense array of hashes at that +// level of the tree. Hashes beyond the end of the array (e.g., if the +// number of leaves is not an exact power of 2) are implicitly zero. +type layer []crypto.Digest + +// A pair represents an internal node in the Merkle tree. +type pair struct { + l crypto.Digest + r crypto.Digest +} + +func (p *pair) ToBeHashed() (protocol.HashID, []byte) { + var buf [2 * crypto.DigestSize]byte + copy(buf[:crypto.DigestSize], p.l[:]) + copy(buf[crypto.DigestSize:], p.r[:]) + return protocol.MerkleArrayNode, buf[:] +} + +// up takes a layer representing some level in the tree, +// and returns the next-higher level in the tree, +// represented as a layer. +func (l layer) up() layer { + res := make(layer, (uint64(len(l))+1)/2) + for i := 0; i < len(l); i += 2 { + var p pair + p.l = l[i] + if i+1 < len(l) { + p.r = l[i+1] + } + res[i/2] = crypto.HashObj(&p) + } + return res +} diff --git a/crypto/merklearray/merkle.go b/crypto/merklearray/merkle.go new file mode 100644 index 000000000..cbccf607b --- /dev/null +++ b/crypto/merklearray/merkle.go @@ -0,0 +1,178 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see <https://www.gnu.org/licenses/>. + +package merklearray + +import ( + "fmt" + "sort" + + "github.com/algorand/go-algorand/crypto" +) + +// Tree is a Merkle tree, represented by layers of nodes (hashes) in the tree +// at each height. +type Tree struct { + // Level 0 is the leaves. + levels []layer +} + +func (tree *Tree) topLayer() layer { + return tree.levels[len(tree.levels)-1] +} + +// Build constructs a Merkle tree given an array. +func Build(array Array) (*Tree, error) { + tree := &Tree{} + + var leaves layer + arraylen := array.Length() + for i := uint64(0); i < arraylen; i++ { + data, err := array.Get(i) + if err != nil { + return nil, err + } + + leaves = append(leaves, crypto.HashObj(data)) + } + + if arraylen > 0 { + tree.levels = []layer{leaves} + + for len(tree.topLayer()) > 1 { + tree.levels = append(tree.levels, tree.topLayer().up()) + } + } + + return tree, nil +} + +// Root returns the root hash of the tree. +func (tree *Tree) Root() crypto.Digest { + // Special case: commitment to zero-length array + if len(tree.levels) == 0 { + var zero crypto.Digest + return zero + } + + return tree.topLayer()[0] +} + +const validateProof = false + +// Prove constructs a proof for some set of positions in the array that was +// used to construct the tree. +func (tree *Tree) Prove(idxs []uint64) ([]crypto.Digest, error) { + if len(idxs) == 0 { + return nil, nil + } + + // Special case: commitment to zero-length array + if len(tree.levels) == 0 { + return nil, fmt.Errorf("proving in zero-length commitment") + } + + sort.Slice(idxs, func(i, j int) bool { return idxs[i] < idxs[j] }) + + pl := make(partialLayer, 0, len(idxs)) + for _, pos := range idxs { + if pos >= uint64(len(tree.levels[0])) { + return nil, fmt.Errorf("pos %d larger than leaf count %d", pos, len(tree.levels[0])) + } + + // Discard duplicates + if len(pl) > 0 && pl[len(pl)-1].pos == pos { + continue + } + + pl = append(pl, layerItem{ + pos: pos, + hash: tree.levels[0][pos], + }) + } + + s := &siblings{ + tree: tree, + } + + for l := uint64(0); l < uint64(len(tree.levels)-1); l++ { + var err error + pl, err = pl.up(s, l, validateProof) + if err != nil { + return nil, err + } + } + + // Confirm that we got the same root hash + if len(pl) != 1 { + return nil, fmt.Errorf("internal error: partial layer produced %d hashes", len(pl)) + } + + if validateProof { + computedroot := pl[0] + if computedroot.pos != 0 || computedroot.hash != tree.topLayer()[0] { + return nil, fmt.Errorf("internal error: root mismatch during proof") + } + } + + return s.hints, nil +} + +// Verify ensures that the positions in elems correspond to the hashes of their respective +// crypto.Hashable objects in a tree with the given root hash. The proof is expected to +// be the proof returned by Prove(). +func Verify(root crypto.Digest, elems map[uint64]crypto.Hashable, proof []crypto.Digest) error { + if len(elems) == 0 { + if len(proof) != 0 { + return fmt.Errorf("non-empty proof for empty set of elements") + } + + return nil + } + + pl := make(partialLayer, 0, len(elems)) + for pos, elem := range elems { + pl = append(pl, layerItem{ + pos: pos, + hash: crypto.HashObj(elem), + }) + } + + sort.Slice(pl, func(i, j int) bool { return pl[i].pos < pl[j].pos }) + + s := &siblings{ + hints: proof, + } + + for l := uint64(0); len(s.hints) > 0 || len(pl) > 1; l++ { + var err error + pl, err = pl.up(s, l, true) + if err != nil { + return err + } + + if l > 64 { + return fmt.Errorf("Verify exceeded 64 levels, more than 2^64 leaves not supported") + } + } + + computedroot := pl[0] + if computedroot.pos != 0 || computedroot.hash != root { + return fmt.Errorf("root mismatch") + } + + return nil +} diff --git a/crypto/merklearray/merkle_test.go b/crypto/merklearray/merkle_test.go new file mode 100644 index 000000000..46710f689 --- /dev/null +++ b/crypto/merklearray/merkle_test.go @@ -0,0 +1,227 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see <https://www.gnu.org/licenses/>. + +package merklearray + +import ( + "fmt" + "testing" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/protocol" +) + +type TestMessage string + +func (m TestMessage) ToBeHashed() (protocol.HashID, []byte) { + return protocol.Message, []byte(m) +} + +type TestData crypto.Digest + +func (d TestData) ToBeHashed() (protocol.HashID, []byte) { + return protocol.Message, d[:] +} + +type TestArray []TestData + +func (a TestArray) Length() uint64 { + return uint64(len(a)) +} + +func (a TestArray) Get(pos uint64) (crypto.Hashable, error) { + if pos >= uint64(len(a)) { + return nil, fmt.Errorf("pos %d larger than length %d", pos, len(a)) + } + + return a[pos], nil +} + +type TestRepeatingArray struct { + item crypto.Hashable + count uint64 +} + +func (a TestRepeatingArray) Length() uint64 { + return a.count +} + +func (a TestRepeatingArray) Get(pos uint64) (crypto.Hashable, error) { + if pos >= a.count { + return nil, fmt.Errorf("pos %d larger than length %d", pos, a.count) + } + + return a.item, nil +} + +func TestMerkle(t *testing.T) { + var junk TestData + crypto.RandBytes(junk[:]) + + for sz := uint64(0); sz < 1024; sz++ { + a := make(TestArray, sz) + for i := uint64(0); i < sz; i++ { + crypto.RandBytes(a[i][:]) + } + + tree, err := Build(a) + if err != nil { + t.Error(err) + } + + root := tree.Root() + + var allpos []uint64 + allmap := make(map[uint64]crypto.Hashable) + + for i := uint64(0); i < sz; i++ { + proof, err := tree.Prove([]uint64{i}) + if err != nil { + t.Error(err) + } + + err = Verify(root, map[uint64]crypto.Hashable{i: a[i]}, proof) + if err != nil { + t.Error(err) + } + + err = Verify(root, map[uint64]crypto.Hashable{i: junk}, proof) + if err == nil { + t.Errorf("no error when verifying junk") + } + + allpos = append(allpos, i) + allmap[i] = a[i] + } + + proof, err := tree.Prove(allpos) + if err != nil { + t.Error(err) + } + + err = Verify(root, allmap, proof) + if err != nil { + t.Error(err) + } + + err = Verify(root, map[uint64]crypto.Hashable{0: junk}, proof) + if err == nil { + t.Errorf("no error when verifying junk batch") + } + + err = Verify(root, map[uint64]crypto.Hashable{0: junk}, nil) + if err == nil { + t.Errorf("no error when verifying junk batch") + } + + _, err = tree.Prove([]uint64{sz}) + if err == nil { + t.Errorf("no error when proving past the end") + } + + err = Verify(root, map[uint64]crypto.Hashable{sz: junk}, nil) + if err == nil { + t.Errorf("no error when verifying past the end") + } + + if sz > 0 { + var somepos []uint64 + somemap := make(map[uint64]crypto.Hashable) + for i := 0; i < 10; i++ { + pos := crypto.RandUint64() % sz + somepos = append(somepos, pos) + somemap[pos] = a[pos] + } + + proof, err = tree.Prove(somepos) + if err != nil { + t.Error(err) + } + + err = Verify(root, somemap, proof) + if err != nil { + t.Error(err) + } + } + } +} + +func BenchmarkMerkleCommit(b *testing.B) { + msg := TestMessage("Hello world") + + var a TestRepeatingArray + a.item = msg + a.count = uint64(b.N) + + tree, err := Build(a) + if err != nil { + b.Error(err) + } + tree.Root() +} + +func BenchmarkMerkleProve1M(b *testing.B) { + msg := TestMessage("Hello world") + + var a TestRepeatingArray + a.item = msg + a.count = 1024 * 1024 + + tree, err := Build(a) + if err != nil { + b.Error(err) + } + + b.ResetTimer() + + for i := uint64(0); i < uint64(b.N); i++ { + _, err := tree.Prove([]uint64{i % a.count}) + if err != nil { + b.Error(err) + } + } +} + +func BenchmarkMerkleVerify1M(b *testing.B) { + msg := TestMessage("Hello world") + + var a TestRepeatingArray + a.item = msg + a.count = 1024 * 1024 + + tree, err := Build(a) + if err != nil { + b.Error(err) + } + root := tree.Root() + + proofs := make([][]crypto.Digest, a.count) + for i := uint64(0); i < a.count; i++ { + proofs[i], err = tree.Prove([]uint64{i}) + if err != nil { + b.Error(err) + } + } + + b.ResetTimer() + + for i := uint64(0); i < uint64(b.N); i++ { + err := Verify(root, map[uint64]crypto.Hashable{i % a.count: msg}, proofs[i]) + if err != nil { + b.Error(err) + } + } +} diff --git a/crypto/merklearray/partial.go b/crypto/merklearray/partial.go new file mode 100644 index 000000000..c349bb317 --- /dev/null +++ b/crypto/merklearray/partial.go @@ -0,0 +1,127 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see <https://www.gnu.org/licenses/>. + +package merklearray + +import ( + "fmt" + + "github.com/algorand/go-algorand/crypto" +) + +// siblings represents the siblings needed to compute the root hash +// given a set of leaf nodes. This data structure can operate in two +// modes: either build up the set of sibling hints, if tree is not nil, +// or use the set of sibling hints, if tree is nil. +type siblings struct { + tree *Tree + hints []crypto.Digest +} + +// get returns the sibling from tree level l (0 being the leaves) +// position i. +func (s *siblings) get(l uint64, i uint64) (res crypto.Digest, err error) { + if s.tree == nil { + if len(s.hints) > 0 { + res = s.hints[0] + s.hints = s.hints[1:] + return + } + + err = fmt.Errorf("no more sibling hints") + return + } + + if l >= uint64(len(s.tree.levels)) { + err = fmt.Errorf("level %d beyond tree height %d", l, len(s.tree.levels)) + return + } + + if i < uint64(len(s.tree.levels[l])) { + res = s.tree.levels[l][i] + } + + s.hints = append(s.hints, res) + return +} + +// partialLayer represents a subset of a layer (i.e., nodes at some +// level in the Merkle tree). layerItem represents one element in the +// partial layer. +type partialLayer []layerItem + +type layerItem struct { + pos uint64 + hash crypto.Digest +} + +// up takes a partial layer at level l, and returns the next-higher (partial) +// level in the tree. Since the layer is partial, up() requires siblings. +// +// The implementation is deterministic to ensure that up() asks for siblings +// in the same order both when generating a proof, as well as when checking +// the proof. +// +// If doHash is false, fill in zero hashes, which suffices for constructing +// a proof. +func (pl partialLayer) up(s *siblings, l uint64, doHash bool) (partialLayer, error) { + var res partialLayer + for i := 0; i < len(pl); i++ { + item := pl[i] + pos := item.pos + posHash := item.hash + + siblingPos := pos ^ 1 + var siblingHash crypto.Digest + if i+1 < len(pl) && pl[i+1].pos == siblingPos { + // If our sibling is also in the partial layer, use its + // hash (and skip over its position). + siblingHash = pl[i+1].hash + i++ + } else { + // Ask for the sibling hash from the tree / proof. + var err error + siblingHash, err = s.get(l, siblingPos) + if err != nil { + return nil, err + } + } + + nextLayerPos := pos / 2 + var nextLayerHash crypto.Digest + + if doHash { + var p pair + if pos&1 == 0 { + // We are left + p.l = posHash + p.r = siblingHash + } else { + // We are right + p.l = siblingHash + p.r = posHash + } + nextLayerHash = crypto.HashObj(&p) + } + + res = append(res, layerItem{ + pos: nextLayerPos, + hash: nextLayerHash, + }) + } + + return res, nil +} diff --git a/crypto/merkletrie/cache.go b/crypto/merkletrie/cache.go index 1f53edfef..2950a4cad 100644 --- a/crypto/merkletrie/cache.go +++ b/crypto/merkletrie/cache.go @@ -177,12 +177,15 @@ func (mtc *merkleTrieCache) loadPage(page uint64) (err error) { return } if mtc.pageToNIDsPtr[page] == nil { - mtc.pageToNIDsPtr[page] = make(map[storedNodeIdentifier]*node, mtc.nodesPerPage) - } - for nodeID, pnode := range decodedNodes { - mtc.pageToNIDsPtr[page][nodeID] = pnode + mtc.pageToNIDsPtr[page] = decodedNodes + mtc.cachedNodeCount += len(mtc.pageToNIDsPtr[page]) + } else { + mtc.cachedNodeCount -= len(mtc.pageToNIDsPtr[page]) + for nodeID, pnode := range decodedNodes { + mtc.pageToNIDsPtr[page][nodeID] = pnode + } + mtc.cachedNodeCount += len(mtc.pageToNIDsPtr[page]) } - mtc.cachedNodeCount += len(mtc.pageToNIDsPtr[page]) // if we've just loaded a deferred page, no need to reload it during the commit. if mtc.deferedPageLoad != page { @@ -438,10 +441,18 @@ func (mtc *merkleTrieCache) encodePage(nodeIDs map[storedNodeIdentifier]*node) [ return serializedBuffer[:walk] } -// evict releases the least used pages from cache until the number of elements in cache are less than cachedNodeCountTarget +// evict releases the least used pages from cache until the number of elements in cache are less than cachedNodeCountTarget. +// the root element page is being moved to the front so that it would get flushed last. func (mtc *merkleTrieCache) evict() (removedNodes int) { removedNodes = mtc.cachedNodeCount - + // check the root element ( if there is such ), and give it a higher priority, since we want + // to release the page with the root element last. + if mtc.mt.root != storedNodeIdentifierNull { + rootPage := uint64(mtc.mt.root) / uint64(mtc.nodesPerPage) + if element, has := mtc.pagesPrioritizationMap[rootPage]; has && element != nil { + mtc.pagesPrioritizationList.MoveToFront(element) + } + } for mtc.cachedNodeCount > mtc.cachedNodeCountTarget { // get the least used page off the pagesPrioritizationList element := mtc.pagesPrioritizationList.Back() diff --git a/crypto/merkletrie/cache_test.go b/crypto/merkletrie/cache_test.go index 6f0fbae58..889f1759f 100644 --- a/crypto/merkletrie/cache_test.go +++ b/crypto/merkletrie/cache_test.go @@ -370,3 +370,76 @@ func TestCacheDeleteNodeMidTransaction(t *testing.T) { require.NotZerof(t, len(pageContent), "Memory page %d has zero nodes", page) } } + +// TestCachePageLoading ensures that during page loading, the number of cachedNodeCount is not +// increased if the page was already loaded previously into memory. +func TestCachePageReloading(t *testing.T) { + var memoryCommitter InMemoryCommitter + mt1, _ := MakeTrie(&memoryCommitter, defaultTestEvictSize) + // create 10 hashes. + leafsCount := 10 + hashes := make([]crypto.Digest, leafsCount) + for i := 0; i < len(hashes); i++ { + hashes[i] = crypto.Hash([]byte{byte(i % 256), byte((i / 256) % 256), byte(i / 65536)}) + } + + for i := 0; i < len(hashes); i++ { + mt1.Add(hashes[i][:]) + } + _, err := mt1.Evict(true) + require.NoError(t, err) + + earlyCachedNodeCount := mt1.cache.cachedNodeCount + // reloading existing cached page multiple time should not cause increase cached node count. + page := uint64(mt1.nextNodeID) / uint64(memoryCommitter.GetNodesCountPerPage()) + err = mt1.cache.loadPage(page) + require.NoError(t, err) + lateCachedNodeCount := mt1.cache.cachedNodeCount + require.Equal(t, earlyCachedNodeCount, lateCachedNodeCount) + + // manually remove one item off this page + for k := range mt1.cache.pageToNIDsPtr[page] { + delete(mt1.cache.pageToNIDsPtr[page], k) + break + } + mt1.cache.cachedNodeCount-- + + // reload to see if that would "fix" the missing entry. + err = mt1.cache.loadPage(page) + require.NoError(t, err) + lateCachedNodeCount = mt1.cache.cachedNodeCount + require.Equal(t, earlyCachedNodeCount, lateCachedNodeCount) +} + +// TestCachePagedOutTip verifies that the evict function would prioritize +// evicting other pages before evicting the top page. +func TestCachePagedOutTip(t *testing.T) { + var memoryCommitter InMemoryCommitter + mt1, _ := MakeTrie(&memoryCommitter, 600) + // create 2048 hashes. + leafsCount := 2048 + hashes := make([]crypto.Digest, leafsCount) + for i := 0; i < len(hashes); i++ { + hashes[i] = crypto.Hash([]byte{byte(i % 256), byte((i / 256) % 256), byte(i / 65536)}) + } + + for i := 0; i < len(hashes)/2; i++ { + mt1.Add(hashes[i][:]) + } + err := mt1.Commit() + require.NoError(t, err) + + for i := 0; i < len(hashes)/2; i++ { + mt1.Add(hashes[i+len(hashes)/2][:]) + } + + // check the tip page before evicting + page := uint64(mt1.nextNodeID) / uint64(memoryCommitter.GetNodesCountPerPage()) + require.NotNil(t, mt1.cache.pageToNIDsPtr[page]) + + _, err = mt1.Evict(true) + require.NoError(t, err) + + // ensures that the tip page was not flushed out. + require.NotNil(t, mt1.cache.pageToNIDsPtr[page]) +} diff --git a/daemon/algod/api/server/v2/handlers.go b/daemon/algod/api/server/v2/handlers.go index e13736e31..192fbbe52 100644 --- a/daemon/algod/api/server/v2/handlers.go +++ b/daemon/algod/api/server/v2/handlers.go @@ -625,7 +625,7 @@ func (v2 *Handlers) GetApplicationByID(ctx echo.Context, applicationID uint64) e } // GetAssetByID returns application information by app idx. -// (GET /v2/asset/{asset-id}) +// (GET /v2/assets/{asset-id}) func (v2 *Handlers) GetAssetByID(ctx echo.Context, assetID uint64) error { assetIdx := basics.AssetIndex(assetID) ledger := v2.Node.Ledger() diff --git a/daemon/kmd/wallet/driver/sqlite.go b/daemon/kmd/wallet/driver/sqlite.go index 3620adc67..b3a8cedae 100644 --- a/daemon/kmd/wallet/driver/sqlite.go +++ b/daemon/kmd/wallet/driver/sqlite.go @@ -87,6 +87,8 @@ type SQLiteWalletDriver struct { globalCfg config.KMDConfig sqliteCfg config.SQLiteWalletDriverConfig mux *deadlock.Mutex + + claimedWallets [][][]byte } // SQLiteWallet represents a particular SQLiteWallet under the @@ -360,50 +362,70 @@ func checkDBError(err error) error { return nil } -// CreateWallet ensures that a wallet of the given name/id combo doesn't exist, -// and initializes a database with the appropriate name. -func (swd *SQLiteWalletDriver) CreateWallet(name []byte, id []byte, pw []byte, mdk crypto.MasterDerivationKey) error { +func (swd *SQLiteWalletDriver) claimWalletNameID(name []byte, id []byte) (dbPath string, err error) { // Grab our lock to avoid races with duplicate wallet names/ids swd.mux.Lock() defer swd.mux.Unlock() - if len(name) > sqliteMaxWalletNameLen { - return errNameTooLong - } - - if len(id) > sqliteMaxWalletIDLen { - return errIDTooLong + for _, nameID := range swd.claimedWallets { + if bytes.Equal(nameID[0], name) { + return "", errSameName + } + if bytes.Equal(nameID[1], id) { + return "", errSameID + } } // name, id -> "/data/dir/name-id.db" - dbPath := swd.nameIDToPath(name, id) + dbPath = swd.nameIDToPath(name, id) // Ensure the wallet with this filename doesn't already exist, and that we // have permissions to access the wallet directory - _, err := os.Stat(dbPath) + _, err = os.Stat(dbPath) if !os.IsNotExist(err) { - return err + return } // Ensure a wallet with this name doesn't already exist. swd.mux is // locked above to avoid races here sameNameDBPaths, err := swd.findDBPathsByName(name) if err != nil { - return err + return } if len(sameNameDBPaths) != 0 { - return errSameName + return "", errSameName } // Ensure a wallet with this id doesn't already exist. As above, we use // swd.mux to avoid races sameIDDBPaths, err := swd.findDBPathsByID(id) if err != nil { - return err + return } if len(sameIDDBPaths) != 0 { - return errSameID + return "", errSameID + } + + swd.claimedWallets = append(swd.claimedWallets, [][]byte{name, id}) + return +} + +// CreateWallet ensures that a wallet of the given name/id combo doesn't exist, +// and initializes a database with the appropriate name. +func (swd *SQLiteWalletDriver) CreateWallet(name []byte, id []byte, pw []byte, mdk crypto.MasterDerivationKey) error { + if len(name) > sqliteMaxWalletNameLen { + return errNameTooLong + } + + if len(id) > sqliteMaxWalletIDLen { + return errIDTooLong + } + + dbPath, err := swd.claimWalletNameID(name, id) + if err != nil { + return err } + // TODO? drop the entry in swd.claimedWallets on exit? // Create the database db, err := sqlx.Connect("sqlite3", dbConnectionURL(dbPath)) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 23e35be48..0b972d9b1 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -48,6 +48,10 @@ import ( // TransactionPool.AssembleBlock constructs a valid block for // proposal given a deadline. type TransactionPool struct { + // feePerByte is stored at the begining of this struct to ensure it has a 64 bit aligned address. This is needed as it's being used + // with atomic operations which require 64 bit alignment on arm. + feePerByte uint64 + // const logProcessBlockStats bool logAssembleStats bool @@ -61,13 +65,14 @@ type TransactionPool struct { pendingBlockEvaluator *ledger.BlockEvaluator numPendingWholeBlocks basics.Round feeThresholdMultiplier uint64 - feePerByte uint64 statusCache *statusCache assemblyMu deadlock.Mutex assemblyCond sync.Cond assemblyDeadline time.Time - assemblyResults poolAsmResults + // assemblyRound indicates which round number we're currently waiting for or waited for last. + assemblyRound basics.Round + assemblyResults poolAsmResults // pendingMu protects pendingTxGroups and pendingTxids pendingMu deadlock.RWMutex @@ -117,6 +122,9 @@ type poolAsmResults struct { blk *ledger.ValidatedBlock stats telemetryspec.AssembleBlockMetrics err error + // roundStartedEvaluating is the round which we were attempted to evaluate last. It's a good measure for + // which round we started evaluating, but not a measure to whether the evaluation is complete. + roundStartedEvaluating basics.Round } // TODO I moved this number to be a constant in the module, we should consider putting it in the local config @@ -128,11 +136,11 @@ const timeoutOnNewBlock = time.Second // assemblyWaitEps is the extra time AssembleBlock() waits past the // deadline before giving up. -const assemblyWaitEps = 10 * time.Millisecond +const assemblyWaitEps = 150 * time.Millisecond -// ErrTxPoolStaleBlockAssembly returned by AssembleBlock when requested block number is older than the current transaction pool round +// ErrStaleBlockAssemblyRequest returned by AssembleBlock when requested block number is older than the current transaction pool round // i.e. typically it means that we're trying to make a proposal for an older round than what the ledger is currently pointing at. -var ErrTxPoolStaleBlockAssembly = fmt.Errorf("AssembleBlock: requested block assembly specified a round that is older than current transaction pool round") +var ErrStaleBlockAssemblyRequest = fmt.Errorf("AssembleBlock: requested block assembly specified a round that is older than current transaction pool round") // Reset resets the content of the transaction pool func (pool *TransactionPool) Reset() { @@ -545,19 +553,18 @@ func (pool *TransactionPool) addToPendingBlockEvaluatorOnce(txgroup []transactio pool.assemblyMu.Lock() defer pool.assemblyMu.Unlock() if !pool.assemblyResults.ok { - if err == ledger.ErrNoSpace || (pool.assemblyDeadline != time.Time{} && time.Now().After(pool.assemblyDeadline)) { + if (err == ledger.ErrNoSpace || (pool.assemblyDeadline != time.Time{} && time.Now().After(pool.assemblyDeadline))) && (pool.assemblyRound <= pool.pendingBlockEvaluator.Round()) { pool.assemblyResults.ok = true - - stats.StopReason = telemetryspec.AssembleBlockTimeout if err == ledger.ErrNoSpace { stats.StopReason = telemetryspec.AssembleBlockFull + } else { + stats.StopReason = telemetryspec.AssembleBlockTimeout } pool.assemblyResults.stats = *stats lvb, gerr := pool.pendingBlockEvaluator.GenerateBlock() if gerr != nil { - rnd := pool.pendingBlockEvaluator.Round() - pool.assemblyResults.err = fmt.Errorf("could not generate block for %d: %v", rnd, gerr) + pool.assemblyResults.err = fmt.Errorf("could not generate block for %d: %v", pool.assemblyResults.roundStartedEvaluating, gerr) } else { pool.assemblyResults.blk = lvb } @@ -580,7 +587,7 @@ func (pool *TransactionPool) addToPendingBlockEvaluator(txgroup []transactions.S // recomputeBlockEvaluator constructs a new BlockEvaluator and feeds all // in-pool transactions to it (removing any transactions that are rejected -// by the BlockEvaluator). +// by the BlockEvaluator). Expects that the pool.mu mutex would be already taken. func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]basics.Round) (stats telemetryspec.ProcessBlockMetrics) { pool.pendingBlockEvaluator = nil @@ -615,7 +622,9 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact pool.pendingMu.RUnlock() pool.assemblyMu.Lock() - pool.assemblyResults = poolAsmResults{} + pool.assemblyResults = poolAsmResults{ + roundStartedEvaluating: prev.Round + basics.Round(1), + } pool.assemblyMu.Unlock() next := bookkeeping.MakeBlock(prev) @@ -666,13 +675,12 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact } pool.assemblyMu.Lock() - if !pool.assemblyResults.ok { + if !pool.assemblyResults.ok && pool.assemblyRound <= pool.pendingBlockEvaluator.Round() { pool.assemblyResults.ok = true pool.assemblyResults.stats = asmStats lvb, err := pool.pendingBlockEvaluator.GenerateBlock() if err != nil { - rnd := pool.pendingBlockEvaluator.Round() - pool.assemblyResults.err = fmt.Errorf("could not generate block for %d (end): %v", rnd, err) + pool.assemblyResults.err = fmt.Errorf("could not generate block for %d (end): %v", pool.assemblyResults.roundStartedEvaluating, err) } else { pool.assemblyResults.blk = lvb } @@ -743,29 +751,98 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim } pool.assemblyMu.Lock() + + // if the transaction pool is more than two rounds behind, we don't want to wait. + if pool.assemblyResults.roundStartedEvaluating <= round.SubSaturate(2) { + logging.Base().Infof("AssembleBlock: requested round is more than a single round ahead of the transaction pool %d <= %d-2", pool.assemblyResults.roundStartedEvaluating, round) + stats.StopReason = telemetryspec.AssembleBlockEmpty + pool.assemblyMu.Unlock() + return pool.assembleEmptyBlock(round) + } + defer pool.assemblyMu.Unlock() + if pool.assemblyResults.roundStartedEvaluating > round { + // we've already assembled a round in the future. Since we're clearly won't go backward, it means + // that the agreement is far behind us, so we're going to return here with error code to let + // the agreement know about it. + // since the network is already ahead of us, there is no issue here in not generating a block ( since the block would get discarded anyway ) + logging.Base().Infof("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.roundStartedEvaluating) + return nil, ErrStaleBlockAssemblyRequest + } + pool.assemblyDeadline = deadline - deadline = deadline.Add(assemblyWaitEps) - for time.Now().Before(deadline) && (!pool.assemblyResults.ok || pool.assemblyResults.blk.Block().Round() < round) { + pool.assemblyRound = round + for time.Now().Before(deadline) && (!pool.assemblyResults.ok || pool.assemblyResults.roundStartedEvaluating != round) { condvar.TimedWait(&pool.assemblyCond, deadline.Sub(time.Now())) } - pool.assemblyDeadline = time.Time{} if !pool.assemblyResults.ok { - return nil, fmt.Errorf("AssembleBlock: ran out of time for round %d", round) + // we've passed the deadline, so we're either going to have a partial block, or that we won't make it on time. + // start preparing an empty block in case we'll miss the extra time (assemblyWaitEps). + // the assembleEmptyBlock is using the database, so we want to unlock here and take the lock again later on. + pool.assemblyMu.Unlock() + emptyBlock, emptyBlockErr := pool.assembleEmptyBlock(round) + pool.assemblyMu.Lock() + + if pool.assemblyResults.roundStartedEvaluating > round { + // this case is expected to happen only if the transaction pool was able to construct *two* rounds during the time we were trying to assemble the empty block. + // while this is extreamly unlikely, we need to handle this. the handling it quite straight-forward : + // since the network is already ahead of us, there is no issue here in not generating a block ( since the block would get discarded anyway ) + logging.Base().Infof("AssembleBlock: requested round is behind transaction pool round after timing out %d < %d", round, pool.assemblyResults.roundStartedEvaluating) + return nil, ErrStaleBlockAssemblyRequest + } + + deadline = deadline.Add(assemblyWaitEps) + for time.Now().Before(deadline) && (!pool.assemblyResults.ok || pool.assemblyResults.roundStartedEvaluating != round) { + condvar.TimedWait(&pool.assemblyCond, deadline.Sub(time.Now())) + } + + // check to see if the extra time helped us to get a block. + if !pool.assemblyResults.ok { + // it didn't. Lucky us - we already prepared an empty block, so we can return this right now. + logging.Base().Warnf("AssembleBlock: ran out of time for round %d", round) + stats.StopReason = telemetryspec.AssembleBlockTimeout + if emptyBlockErr != nil { + emptyBlockErr = fmt.Errorf("AssembleBlock: failed to construct empty block : %v", emptyBlockErr) + } + return emptyBlock, emptyBlockErr + } } + pool.assemblyDeadline = time.Time{} + if pool.assemblyResults.err != nil { return nil, fmt.Errorf("AssemblyBlock: encountered error for round %d: %v", round, pool.assemblyResults.err) } - if pool.assemblyResults.blk.Block().Round() > round { - logging.Base().Infof("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.blk.Block().Round()) - return nil, ErrTxPoolStaleBlockAssembly - } else if pool.assemblyResults.blk.Block().Round() != round { + if pool.assemblyResults.roundStartedEvaluating > round { + // this scenario should not happen unless the txpool is receiving the new blocks via OnNewBlocks + // with "jumps" between consecutive blocks ( which is why it's a warning ) + // The "normal" usecase is evaluated on the top of the function. + logging.Base().Warnf("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.roundStartedEvaluating) + return nil, ErrStaleBlockAssemblyRequest + } else if pool.assemblyResults.roundStartedEvaluating != round { return nil, fmt.Errorf("AssembleBlock: assembled block round does not match: %d != %d", - pool.assemblyResults.blk.Block().Round(), round) + pool.assemblyResults.roundStartedEvaluating, round) } stats = pool.assemblyResults.stats return pool.assemblyResults.blk, nil } + +// assembleEmptyBlock construct a new block for the given round. Internally it's using the ledger database calls, so callers +// need to be aware that it might take a while before it would return. +func (pool *TransactionPool) assembleEmptyBlock(round basics.Round) (assembled *ledger.ValidatedBlock, err error) { + prevRound := round - 1 + prev, err := pool.ledger.BlockHdr(prevRound) + if err != nil { + err = fmt.Errorf("TransactionPool.assembleEmptyBlock: cannot get prev header for %d: %v", prevRound, err) + return nil, err + } + next := bookkeeping.MakeBlock(prev) + blockEval, err := pool.ledger.StartEvaluator(next.BlockHeader, 0) + if err != nil { + err = fmt.Errorf("TransactionPool.assembleEmptyBlock: cannot start evaluator for %d: %v", round, err) + return nil, err + } + return blockEval.GenerateBlock() +} diff --git a/data/transactions/logic/assembler_test.go b/data/transactions/logic/assembler_test.go index 90014cf80..e44cb372d 100644 --- a/data/transactions/logic/assembler_test.go +++ b/data/transactions/logic/assembler_test.go @@ -1330,6 +1330,18 @@ func TestStringLiteralParsing(t *testing.T) { require.NoError(t, err) require.Equal(t, e, result) + s = `"test\ra"` + e = []byte("test\x0da") + result, err = parseStringLiteral(s) + require.NoError(t, err) + require.Equal(t, e, result) + + s = `"test\\"` + e = []byte(`test\`) + result, err = parseStringLiteral(s) + require.NoError(t, err) + require.Equal(t, e, result) + s = `"test 123"` e = []byte(`test 123`) result, err = parseStringLiteral(s) @@ -1,6 +1,6 @@ module github.com/algorand/go-algorand -go 1.12 +go 1.14 require ( github.com/algorand/go-codec/codec v0.0.0-20190507210007-269d70b6135d diff --git a/installer/debian/preinst b/installer/debian/preinst index e49fa261a..6817072ee 100755 --- a/installer/debian/preinst +++ b/installer/debian/preinst @@ -1,21 +1,26 @@ -#!/bin/bash +#!/usr/bin/env bash + +set -e PKG_NAME=@PKG_NAME@ -VERSION="$2" -if test "$1" = install && - dpkg-query --list 'algorand*' > /dev/null +if [ "$1" != install ] +then + exit 0 +fi + +if dpkg-query --list 'algorand*' &> /dev/null then if PKG_INFO=$(dpkg-query --show --showformat='${Package} ${Status}\n' 'algorand*' | grep "install ok installed") then - INSTALLED_PKG=$(echo "$PKG_INFO" | awk '{print $1}') + INSTALLED_PKG=$(grep -v algorand-indexer <<< "$PKG_INFO" | awk '{print $1}') - if [ "$INSTALLED_PKG" != "$PKG_NAME" ] + if [ -n "$INSTALLED_PKG" ] then echo -e "\nAlgorand does not currently support multi-distribution installations!\n\ -To install this package, it is necessary to first remove the \`$INSTALLED_PKG\` package:\n\n\ -apt remove $INSTALLED_PKG\n\ -apt install $PKG_NAME=$VERSION\n" + To install this package, it is necessary to first remove the \`$INSTALLED_PKG\` package:\n\n\ + sudo apt-get remove $INSTALLED_PKG\n\ + sudo apt-get install $PKG_NAME\n" exit 1 fi fi diff --git a/network/msgOfInterest.go b/network/msgOfInterest.go index 2c4af3362..5f876feec 100644 --- a/network/msgOfInterest.go +++ b/network/msgOfInterest.go @@ -28,6 +28,7 @@ var errInvalidMessageOfInterest = errors.New("unmarshalMessageOfInterest: messag var errInvalidMessageOfInterestLength = errors.New("unmarshalMessageOfInterest: message length is too long") const maxMessageOfInterestTags = 1024 +const topicsEncodingSeparator = "," func unmarshallMessageOfInterest(data []byte) (map[protocol.Tag]bool, error) { // decode the message, and ensure it's a valid message. @@ -44,7 +45,7 @@ func unmarshallMessageOfInterest(data []byte) (map[protocol.Tag]bool, error) { } // convert the tags into a tags map. msgTagsMap := make(map[protocol.Tag]bool, len(tags)) - for _, tag := range strings.Split(string(tags), ",") { + for _, tag := range strings.Split(string(tags), topicsEncodingSeparator) { msgTagsMap[protocol.Tag(tag)] = true } return msgTagsMap, nil @@ -55,10 +56,26 @@ func MarshallMessageOfInterest(messageTags []protocol.Tag) []byte { // create a long string with all these messages. tags := "" for _, tag := range messageTags { - tags += "," + string(tag) + tags += topicsEncodingSeparator + string(tag) } if len(tags) > 0 { - tags = tags[1:] + tags = tags[len(topicsEncodingSeparator):] + } + topics := Topics{Topic{key: "tags", data: []byte(tags)}} + return topics.MarshallTopics() +} + +// MarshallMessageOfInterestMap generates a message of interest message body +// for the message tags that map to "true" in the map argument. +func MarshallMessageOfInterestMap(tagmap map[protocol.Tag]bool) []byte { + tags := "" + for tag, flag := range tagmap { + if flag { + tags += topicsEncodingSeparator + string(tag) + } + } + if len(tags) > 0 { + tags = tags[len(topicsEncodingSeparator):] } topics := Topics{Topic{key: "tags", data: []byte(tags)}} return topics.MarshallTopics() diff --git a/network/wsNetwork.go b/network/wsNetwork.go index af836e282..5a154156e 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -189,6 +189,13 @@ type GossipNode interface { // GetHTTPRequestConnection returns the underlying connection for the given request. Note that the request must be the same // request that was provided to the http handler ( or provide a fallback Context() to that ) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) + + // RegisterMessageInterest notifies the network library that this node + // wants to receive messages with the specified tag. This will cause + // this node to send corresponding MsgOfInterest notifications to any + // newly connecting peers. This should be called before the network + // is started. + RegisterMessageInterest(protocol.Tag) error } // IncomingMessage represents a message arriving from some peer in our p2p network @@ -358,6 +365,23 @@ type WebsocketNetwork struct { // connection in compliance with connectionsRateLimitingCount. transport rateLimitingTransport dialer Dialer + + // messagesOfInterest specifies the message types that this node + // wants to receive. nil means default. non-nil causes this + // map to be sent to new peers as a MsgOfInterest message type. + messagesOfInterest map[protocol.Tag]bool + + // messagesOfInterestEnc is the encoding of messagesOfInterest, + // to be sent to new peers. This is filled in at network start, + // at which point messagesOfInterestEncoded is set to prevent + // further changes. + messagesOfInterestEnc []byte + messagesOfInterestEncoded bool + + // messagesOfInterestMu protects messagesOfInterest and ensures + // that messagesOfInterestEnc does not change once it is set during + // network start. + messagesOfInterestMu deadlock.Mutex } type broadcastRequest struct { @@ -675,6 +699,13 @@ func (wn *WebsocketNetwork) Start() { wn.config.IncomingConnectionsLimit = MaxInt } + wn.messagesOfInterestMu.Lock() + defer wn.messagesOfInterestMu.Unlock() + wn.messagesOfInterestEncoded = true + if wn.messagesOfInterest != nil { + wn.messagesOfInterestEnc = MarshallMessageOfInterestMap(wn.messagesOfInterest) + } + // Make sure we do not accept more incoming connections than our // open file rlimit, with some headroom for other FDs (DNS, log // files, SQLite files, telemetry, ...) @@ -783,6 +814,12 @@ func (wn *WebsocketNetwork) Stop() { if wn.listener != nil { wn.log.Debugf("closed %s", listenAddr) } + + wn.messagesOfInterestMu.Lock() + defer wn.messagesOfInterestMu.Unlock() + wn.messagesOfInterestEncoded = false + wn.messagesOfInterestEnc = nil + wn.messagesOfInterest = nil } // RegisterHandlers registers the set of given message handlers. @@ -1036,6 +1073,13 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt InstanceName: trackedRequest.otherInstanceName, }) + if wn.messagesOfInterestEnc != nil { + err = peer.Unicast(wn.ctx, wn.messagesOfInterestEnc, protocol.MsgOfInterestTag) + if err != nil { + wn.log.Infof("ws send msgOfInterest: %v", err) + } + } + peers.Set(float64(wn.NumPeers()), nil) incomingPeers.Set(float64(wn.numIncomingPeers()), nil) } @@ -2031,3 +2075,27 @@ func SetUserAgentHeader(header http.Header) { ua := fmt.Sprintf("algod/%d.%d (%s; commit=%s; %d) %s(%s)", version.Major, version.Minor, version.Channel, version.CommitHash, version.BuildNumber, runtime.GOOS, runtime.GOARCH) header.Set(UserAgentHeader, ua) } + +// RegisterMessageInterest notifies the network library that this node +// wants to receive messages with the specified tag. This will cause +// this node to send corresponding MsgOfInterest notifications to any +// newly connecting peers. This should be called before the network +// is started. +func (wn *WebsocketNetwork) RegisterMessageInterest(t protocol.Tag) error { + wn.messagesOfInterestMu.Lock() + defer wn.messagesOfInterestMu.Unlock() + + if wn.messagesOfInterestEncoded { + return fmt.Errorf("network already started") + } + + if wn.messagesOfInterest == nil { + wn.messagesOfInterest = make(map[protocol.Tag]bool) + for tag, flag := range defaultSendMessageTags { + wn.messagesOfInterest[tag] = flag + } + } + + wn.messagesOfInterest[t] = true + return nil +} diff --git a/node/node.go b/node/node.go index 6cc6c1f97..3fcb008ae 100644 --- a/node/node.go +++ b/node/node.go @@ -970,7 +970,7 @@ func (vb validatedBlock) Block() bookkeeping.Block { func (node *AlgorandFullNode) AssembleBlock(round basics.Round, deadline time.Time) (agreement.ValidatedBlock, error) { lvb, err := node.transactionPool.AssembleBlock(round, deadline) if err != nil { - if err == pools.ErrTxPoolStaleBlockAssembly { + if err == pools.ErrStaleBlockAssemblyRequest { // convert specific error to one that would have special handling in the agreement code. err = agreement.ErrAssembleBlockRoundStale diff --git a/protocol/hash.go b/protocol/hash.go index b42bfc1e8..d3f821690 100644 --- a/protocol/hash.go +++ b/protocol/hash.go @@ -33,6 +33,7 @@ const ( BalanceRecord HashID = "BR" Credential HashID = "CR" Genesis HashID = "GE" + MerkleArrayNode HashID = "MA" Message HashID = "MX" NetPrioResponse HashID = "NPR" OneTimeSigKey1 HashID = "OT1" diff --git a/protocol/tags.go b/protocol/tags.go index 9fff2316a..229be0323 100644 --- a/protocol/tags.go +++ b/protocol/tags.go @@ -21,16 +21,18 @@ package protocol type Tag string // Tags, in lexicographic sort order of tag values to avoid duplicates. +// These tags must not contain a comma character because lists of tags +// are encoded using a comma separator (see network/msgOfInterest.go). const ( UnknownMsgTag Tag = "??" AgreementVoteTag Tag = "AV" + MsgOfInterestTag Tag = "MI" MsgDigestSkipTag Tag = "MS" NetPrioResponseTag Tag = "NP" PingTag Tag = "pi" PingReplyTag Tag = "pj" ProposalPayloadTag Tag = "PP" TopicMsgRespTag Tag = "TS" - MsgOfInterestTag Tag = "MI" TxnTag Tag = "TX" UniCatchupReqTag Tag = "UC" UniEnsBlockReqTag Tag = "UE" diff --git a/scripts/get_golang_version.sh b/scripts/get_golang_version.sh index d98414482..48cf7d823 100755 --- a/scripts/get_golang_version.sh +++ b/scripts/get_golang_version.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Required GOLANG Version for project specified here -GOLANG_VERSION="1.12.17" +GOLANG_VERSION="1.14.7" # Check to make sure that it matches what is specified in go.mod diff --git a/scripts/release/build/stage/upload/run.sh b/scripts/release/build/stage/upload/run.sh index 54db9f4e8..e3c989f28 100755 --- a/scripts/release/build/stage/upload/run.sh +++ b/scripts/release/build/stage/upload/run.sh @@ -15,5 +15,5 @@ rm -rf pkg && mkdir -p pkg/"$FULLVERSION" ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash go/src/github.com/algorand/go-algorand/scripts/release/build/stage/upload/task.sh scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@"$INSTANCE":~/node_pkg/* pkg/"$FULLVERSION"/ -aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* pkg/"$FULLVERSION" "s3://algorand-staging/releases/$CHANNEL/$FULLVERSION/" +aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --exclude beta* pkg/"$FULLVERSION" "s3://algorand-staging/releases/$CHANNEL/$FULLVERSION/" diff --git a/scripts/release/test/deb/test_algorand.sh b/scripts/release/test/deb/test_algorand.sh index ad117da06..3c3d7bb87 100755 --- a/scripts/release/test/deb/test_algorand.sh +++ b/scripts/release/test/deb/test_algorand.sh @@ -5,6 +5,11 @@ set -ex export GOPATH=${HOME}/go export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} +PKG_NAME=algorand +if [ "${CHANNEL}" = beta ]; then + PKG_NAME=algorand-beta +fi + apt-get update apt-get install -y gnupg2 curl software-properties-common python3 @@ -22,7 +27,7 @@ apt-key add /root/keys/dev.pub apt-key add /root/keys/rpm.pub add-apt-repository "deb http://${DC_IP}:8111/ stable main" apt-get update -apt-get install -y algorand +apt-get install -y "${PKG_NAME}" algod -v # check that the installed version is now the current version algod -v | grep -q "${FULLVERSION}.${CHANNEL}" diff --git a/scripts/release/test/deb/test_apt-get.sh b/scripts/release/test/deb/test_apt-get.sh index e177ccb3c..da3a42971 100755 --- a/scripts/release/test/deb/test_apt-get.sh +++ b/scripts/release/test/deb/test_apt-get.sh @@ -1,18 +1,23 @@ #!/usr/bin/env bash -#set -xv +set -ex echo "test_apt-get starting within docker container" export GOPATH=${HOME}/go export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} +PKG_NAME=algorand +if [ "${CHANNEL}" = beta ]; then + PKG_NAME=algorand-beta +fi + apt-get update apt-get install -y gnupg2 curl software-properties-common python3 apt-key add /root/keys/dev.pub add-apt-repository -y "deb [trusted=yes] http://${DC_IP}:8111/ stable main" apt-get update -apt-get install -y algorand +apt-get install -y "${PKG_NAME}" apt-get install -y expect algod -v diff --git a/scripts/release/test/rpm/run_centos.sh b/scripts/release/test/rpm/run_centos.sh index 0277bc5e3..f15fb5cca 100755 --- a/scripts/release/test/rpm/run_centos.sh +++ b/scripts/release/test/rpm/run_centos.sh @@ -9,6 +9,11 @@ echo date "+build_release begin TEST stage %Y%m%d_%H%M%S" echo +if [ "$CHANNEL" = beta ]; then + echo "There is currently no support for RPM beta packages. Exiting RPM test stage..." + exit 0 +fi + # Run RPM build in Centos7 Docker container sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" diff --git a/shared/pingpong/config.go b/shared/pingpong/config.go index 46f5d83b4..7941f7d27 100644 --- a/shared/pingpong/config.go +++ b/shared/pingpong/config.go @@ -54,6 +54,7 @@ type PpConfig struct { MinAccountAsset uint64 NumApp uint32 AppProgOps uint32 + Rekey bool } // DefaultConfig object for Ping Pong @@ -75,6 +76,9 @@ var DefaultConfig = PpConfig{ GroupSize: 1, NumAsset: 0, MinAccountAsset: 10000000, + NumApp: 0, + AppProgOps: 0, + Rekey: false, } // LoadConfigFromFile reads and loads Ping Pong configuration diff --git a/shared/pingpong/pingpong.go b/shared/pingpong/pingpong.go index 8818800fe..b266e6a5f 100644 --- a/shared/pingpong/pingpong.go +++ b/shared/pingpong/pingpong.go @@ -25,7 +25,7 @@ import ( "time" "github.com/algorand/go-algorand/crypto" - "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" + v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/libgoal" @@ -234,13 +234,17 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin var totalSent, totalSucceeded uint64 for !time.Now().After(stopTime) { + minimumAmount := cfg.MinAccountFunds + (cfg.MaxAmt+cfg.MaxFee)*2 + fromList := listSufficientAccounts(accounts, minimumAmount, cfg.SrcAccount) + // in group tests txns are sent back and forth, so both parties need funds + if cfg.GroupSize == 1 { + minimumAmount = 0 + } + toList := listSufficientAccounts(accounts, minimumAmount, cfg.SrcAccount) - fromList := listSufficientAccounts(accounts, cfg.MinAccountFunds+(cfg.MaxAmt+cfg.MaxFee)*2, cfg.SrcAccount) - toList := listSufficientAccounts(accounts, 0, cfg.SrcAccount) - - sent, succeded, err := sendFromTo(fromList, toList, accounts, assetParam, appParam, ac, cfg) + sent, succeeded, err := sendFromTo(fromList, toList, accounts, assetParam, appParam, ac, cfg) totalSent += sent - totalSucceeded += succeded + totalSucceeded += succeeded if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error sending transactions: %v\n", err) } @@ -266,9 +270,40 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin } } -func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams, client libgoal.Client, cfg PpConfig) (sentCount, successCount uint64, err error) { +func getCreatableID(cfg PpConfig, assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams) (aidx uint64) { + if cfg.NumAsset > 0 { + rindex := rand.Intn(len(assetParams)) + i := 0 + for k := range assetParams { + if i == rindex { + aidx = k + break + } + i++ + } + } else if cfg.NumApp > 0 { + rindex := rand.Intn(len(appParams)) + i := 0 + for k := range appParams { + if i == rindex { + aidx = k + break + } + i++ + } + } + return +} + +func sendFromTo( + fromList, toList []string, accounts map[string]uint64, + assetParams map[uint64]v1.AssetParams, appParams map[uint64]v1.AppParams, + client libgoal.Client, cfg PpConfig, +) (sentCount, successCount uint64, err error) { + amt := cfg.MaxAmt fee := cfg.MaxFee + for i, from := range fromList { if cfg.RandomizeAmt { amt = rand.Uint64()%cfg.MaxAmt + 1 @@ -293,33 +328,9 @@ func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetPara amt = 1 } - // app or asset ID - var aidx uint64 if cfg.GroupSize == 1 { - if cfg.NumAsset > 0 { // generate random assetID if we send asset txns - rindex := rand.Intn(len(assetParams)) - i := 0 - for k := range assetParams { - if i == rindex { - aidx = k - break - } - i++ - } - } else if cfg.NumApp > 0 { - rindex := rand.Intn(len(appParams)) - i := 0 - for k := range appParams { - if i == rindex { - aidx = k - break - } - i++ - } - } else { - aidx = 0 - } - + // generate random assetID or appId if we send asset/app txns + aidx := getCreatableID(cfg, assetParams, appParams) // Construct single txn txn, consErr := constructTxn(from, to, fee, amt, aidx, client, cfg) if consErr != nil { @@ -333,6 +344,7 @@ func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetPara _, _ = fmt.Fprintf(os.Stdout, "Skipping sending %d : %s -> %s; Current cost too high.\n", amt, from, to) continue } + fromBalanceChange = -int64(txn.Fee.Raw + amt) toBalanceChange = int64(amt) @@ -348,23 +360,55 @@ func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetPara _, sendErr = client.BroadcastTransaction(stxn) } else { // Generate txn group + + // In rekeying test there are two txns sent in a group + // the first is from -> to with RekeyTo=to + // the second is from -> to with RekeyTo=from and AuthAddr=to + // So that rekeying test only supports groups of two + var txGroup []transactions.Transaction + var txSigners []string for j := 0; j < int(cfg.GroupSize); j++ { var txn transactions.Transaction + var signer string if j%2 == 0 { txn, err = constructTxn(from, to, fee, amt, 0, client, cfg) fromBalanceChange -= int64(txn.Fee.Raw + amt) toBalanceChange += int64(amt) + signer = from + } else if cfg.GroupSize == 2 && cfg.Rekey { + txn, err = constructTxn(from, to, fee, amt, 0, client, cfg) + fromBalanceChange -= int64(txn.Fee.Raw + amt) + toBalanceChange += int64(amt) + signer = to } else { txn, err = constructTxn(to, from, fee, amt, 0, client, cfg) toBalanceChange -= int64(txn.Fee.Raw + amt) fromBalanceChange += int64(amt) + signer = to } if err != nil { _, _ = fmt.Fprintf(os.Stderr, "group tx failed: %v\n", err) return } + if cfg.RandomizeAmt { + amt = rand.Uint64()%cfg.MaxAmt + 1 + } + if cfg.Rekey { + if from == signer { + // rekey to the receiver the first txn of the rekeying pair + txn.RekeyTo, err = basics.UnmarshalChecksumAddress(to) + } else { + // rekey to the sender the second txn of the rekeying pair + txn.RekeyTo, err = basics.UnmarshalChecksumAddress(from) + } + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Address unmarshalling failed: %v\n", err) + return + } + } txGroup = append(txGroup, txn) + txSigners = append(txSigners, signer) } // would we have enough money after taking into account the current updated fees ? @@ -392,13 +436,7 @@ func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetPara var stxGroup []transactions.SignedTxn for j, txn := range txGroup { txn.Group = gid - var lf string - if j%2 == 0 { - lf = from - } else { - lf = to - } - stxn, signErr := signTxn(lf, txn, client, cfg) + stxn, signErr := signTxn(txSigners[j], txn, client, cfg) if signErr != nil { err = signErr return @@ -410,19 +448,16 @@ func sendFromTo(fromList, toList []string, accounts map[string]uint64, assetPara sendErr = client.BroadcastTransactionGroup(stxGroup) } - if sendErr != nil && !cfg.Quiet { - _, _ = fmt.Fprintf(os.Stderr, "error sending transaction: %v\n", sendErr) - } else { - successCount++ - accounts[from] = uint64(fromBalanceChange + int64(accounts[from])) - accounts[to] = uint64(toBalanceChange + int64(accounts[to])) - } if sendErr != nil { _, _ = fmt.Fprintf(os.Stderr, "error sending Transaction, sleeping .5 seconds: %v\n", sendErr) err = sendErr time.Sleep(500 * time.Millisecond) return } + + successCount++ + accounts[from] = uint64(fromBalanceChange + int64(accounts[from])) + accounts[to] = uint64(toBalanceChange + int64(accounts[to])) if cfg.DelayBetweenTxn > 0 { time.Sleep(cfg.DelayBetweenTxn) } @@ -503,7 +538,7 @@ func constructTxn(from, to string, fee, amt, aidx uint64, client libgoal.Client, return } -func signTxn(from string, txn transactions.Transaction, client libgoal.Client, cfg PpConfig) (stxn transactions.SignedTxn, err error) { +func signTxn(signer string, txn transactions.Transaction, client libgoal.Client, cfg PpConfig) (stxn transactions.SignedTxn, err error) { // Get wallet handle token var h []byte h, err = client.GetUnencryptedWalletHandle() @@ -513,9 +548,11 @@ func signTxn(from string, txn transactions.Transaction, client libgoal.Client, c var psig crypto.Signature - if len(cfg.Program) > 0 { + if cfg.Rekey { + stxn, err = client.SignTransactionWithWalletAndSigner(h, nil, signer, txn) + } else if len(cfg.Program) > 0 { // If there's a program, sign it and use that in a lsig - psig, err = client.SignProgramWithWallet(h, nil, from, cfg.Program) + psig, err = client.SignProgramWithWallet(h, nil, signer, cfg.Program) if err != nil { return } @@ -527,9 +564,6 @@ func signTxn(from string, txn transactions.Transaction, client libgoal.Client, c } else { // Otherwise, just sign the transaction like normal stxn, err = client.SignTransactionWithWallet(h, nil, txn) - if err != nil { - return - } } return } diff --git a/test/e2e-go/cli/goal/expect/README.md b/test/e2e-go/cli/goal/expect/README.md index a4b47b9bd..3053632c2 100644 --- a/test/e2e-go/cli/goal/expect/README.md +++ b/test/e2e-go/cli/goal/expect/README.md @@ -56,6 +56,8 @@ There are three (optional) environment variables that can be used to control the set the filter to be `export TESTFILTER=[b,c]ar`. - Defaults to all tests (`.*`). +NOTE: the file name shoud have the suffix: "Test.exp" + To run the Goal Expect test, run the following command from the top level go-algorand directory: ``` diff --git a/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp new file mode 100644 index 000000000..25378849f --- /dev/null +++ b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp @@ -0,0 +1,178 @@ +#!/usr/bin/expect -f +#exp_internal 1 +set err 0 +log_user 1 + +### +# * The test creates four additional accounts, opt-in one to the app, then passes the four accounts +# as --app-account flag to goal app call. +# * The teal program writes the four passed account addresses into the local state of the caller and one +# * additional account. +# * The test verifies the correct addresses are written to the local states of both. +# * It also tests for the error when one flag is missing +### + +source goalExpectCommon.exp + + +set TEST_ALGO_DIR [lindex $argv 0] +set TEST_DATA_DIR [lindex $argv 1] + +proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { + + # Setup the test + set timeout 60 + set TIME_STAMP [clock seconds] + + # Setup the network + set TEST_ROOT_DIR $TEST_ALGO_DIR/root_$TIME_STAMP + set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ + set NETWORK_NAME test_net_expect_$TIME_STAMP + set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50EachFuture.json" + + set TEAL_PROGRAM "appAccountParams.teal" + + exec cp $TEST_DATA_DIR/../../gen/devnet/genesis.json $TEST_ALGO_DIR + + # Create network + ::AlgorandGoal::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + # Start network + ::AlgorandGoal::StartNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + set PRIMARY_NODE_ADDRESS [ ::AlgorandGoal::GetAlgodNetworkAddress $TEST_PRIMARY_NODE_DIR ] + puts "Primary Node Address: $PRIMARY_NODE_ADDRESS" + + set PRIMARY_WALLET_NAME unencrypted-default-wallet + + # Determine primary account + set PRIMARY_ACCOUNT_ADDRESS [::AlgorandGoal::GetHighestFundedAccountForWallet $PRIMARY_WALLET_NAME $TEST_PRIMARY_NODE_DIR] + + # Check the balance of the primary account + set PRIMARY_ACCOUNT_BALANCE [::AlgorandGoal::GetAccountBalance $PRIMARY_WALLET_NAME $PRIMARY_ACCOUNT_ADDRESS $TEST_PRIMARY_NODE_DIR] + puts "Primary Account Balance: $PRIMARY_ACCOUNT_BALANCE" + + ::AlgorandGoal::WaitForRound 1 $TEST_PRIMARY_NODE_DIR + + # Create wallet #1 + set WALLET_1_NAME Wallet_1_$TIME_STAMP + set WALLET_1_PASSWORD 1234 + set WALLET_1_PASSPHRASE [::AlgorandGoal::CreateWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + puts "WALLET_1_PASSPHRASE: $WALLET_1_PASSPHRASE" + ::AlgorandGoal::VerifyWallet $WALLET_1_NAME $TEST_PRIMARY_NODE_DIR + + # Associate 4 new accounts with the wallet + set ACCOUNT_1_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + set ACCOUNT_2_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + set ACCOUNT_3_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + set ACCOUNT_4_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + + # Transfer Algos from primary account to accounts 1-4 + set MIN_BALANCE 1000000 + set TRANSFER_AMOUNT [expr {1000 * $MIN_BALANCE}] + set FEE_AMOUNT 1000 + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_1_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_2_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_3_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_4_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + + set TEAL_PROGS_DIR "$TEST_DATA_DIR/../scripts/e2e_subs/tealprogs" + set GLOBAL_BYTE_SLICES 0 + set LOCAL_BYTE_SLICES 5 + + # Create the app + puts "calling goal app create" + set APP_ID [::AlgorandGoal::AppCreateOnCompletion \ + $PRIMARY_WALLET_NAME \ + "" \ + $PRIMARY_ACCOUNT_ADDRESS \ + ${TEAL_PROGS_DIR}/${TEAL_PROGRAM} \ + "" \ + $GLOBAL_BYTE_SLICES \ + $LOCAL_BYTE_SLICES \ + ${TEAL_PROGS_DIR}/clear_program_state.teal \ + $TEST_PRIMARY_NODE_DIR \ + "OptIn"] + + # Let account 2 opt in so it's local state is also written + ::AlgorandGoal::AppOptIn $APP_ID $WALLET_1_NAME $WALLET_1_PASSWORD $ACCOUNT_2_ADDRESS "" $TEST_PRIMARY_NODE_DIR + + # call the app + puts "Calling goal app call to get the local state params" + spawn goal app call --app-id $APP_ID --from $PRIMARY_ACCOUNT_ADDRESS -w $PRIMARY_WALLET_NAME -d $TEST_PRIMARY_NODE_DIR \ + --app-account $ACCOUNT_1_ADDRESS \ + --app-account $ACCOUNT_2_ADDRESS \ + --app-account $ACCOUNT_3_ADDRESS \ + --app-account $ACCOUNT_4_ADDRESS + expect { + timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } + "*committed in round*" {puts "app call successful"; close} + eof {close; ::AlgorandGoal::Abort "app call failed" } + } + + puts "Checking the results" + set EXPECTED_OUTPUT "Account0*$PRIMARY_ACCOUNT_ADDRESS" + set EXPECTED_OUTPUT "$EXPECTED_OUTPUT*Account1*$ACCOUNT_1_ADDRESS" + set EXPECTED_OUTPUT "$EXPECTED_OUTPUT*Account2*$ACCOUNT_2_ADDRESS" + set EXPECTED_OUTPUT "$EXPECTED_OUTPUT*Account3*$ACCOUNT_3_ADDRESS" + set EXPECTED_OUTPUT "$EXPECTED_OUTPUT*Account4*$ACCOUNT_4_ADDRESS" + set EXPECTED_OUTPUT "$EXPECTED_OUTPUT*Num Accounts*ui*4" + + # check the local state of the caller + spawn goal app read --app-id $APP_ID --local --guess-format \ + --from $PRIMARY_ACCOUNT_ADDRESS -w $PRIMARY_WALLET_NAME -d $TEST_PRIMARY_NODE_DIR + expect { + timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } + "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} + eof {close; ::AlgorandGoal::Abort "app read failed" } + } + + # check the local state of account 2 + spawn goal app read --app-id $APP_ID --local --guess-format \ + --from $ACCOUNT_2_ADDRESS -w $WALLET_1_NAME -d $TEST_PRIMARY_NODE_DIR + + expect { + timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } + "Please enter the password for wallet '$WALLET_1_NAME':" {send "$WALLET_1_PASSWORD\r" ; exp_continue} + "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} + eof {close; ::AlgorandGoal::Abort "app read failed" } + } + + # call the app with a missing app-account. It should fail + puts "Calling goal app call to get the local state params" + spawn goal app call --app-id $APP_ID --from $PRIMARY_ACCOUNT_ADDRESS -w $PRIMARY_WALLET_NAME -d $TEST_PRIMARY_NODE_DIR \ + --app-account $ACCOUNT_1_ADDRESS \ + --app-account $ACCOUNT_2_ADDRESS \ + --app-account $ACCOUNT_4_ADDRESS + expect { + timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } + "*Couldn't broadcast tx with algod: HTTP 400 Bad Request: TransactionPool.Remember: transaction*invalid Accounts index 4*" \ + {puts "Error received successfully "; close} + eof {close; ::AlgorandGoal::Abort "failed to get the expected error" } + } + + # Shutdown the network + ::AlgorandGoal::StopNetwork $NETWORK_NAME $TEST_ALGO_DIR $TEST_ROOT_DIR + + puts "Goal Stateful Teal test Successful" +} + + + +if { [catch { + source goalExpectCommon.exp + + puts "starting test" + + puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" + puts "TEST_DATA_DIR: $TEST_DATA_DIR" + + puts "calling goalAppAccountAddressTest" + + goalAppAccountAddress $TEST_ALGO_DIR $TEST_DATA_DIR + + exit 0 + +} EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in goalAppAccountAddressTest: $EXCEPTION" +} diff --git a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp index ea240d6af..91272ffba 100755 --- a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp +++ b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp @@ -431,7 +431,8 @@ proc ::AlgorandGoal::AssetCreate { CREATOR WALLET_NAME WALLET_PASSWORD TOTAL_SUP spawn goal asset create -d $TEST_PRIMARY_NODE_DIR -w $WALLET_NAME --creator $CREATOR --total $TOTAL_SUPPLY --unitname $UNIT_NAME expect { timeout { ::AlgorandGoal::Abort "Timed out create asset" } - "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r" } + "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r"; exp_continue } + eof } } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in AssetCreate: $EXCEPTION" @@ -444,7 +445,8 @@ proc ::AlgorandGoal::AssetTransfer { WALLET_NAME WALLET_PASSWORD FROM_ADDR TO_AD spawn goal asset send -d $TEST_PRIMARY_NODE_DIR -w $WALLET_NAME --from $FROM_ADDR --to $TO_ADDR --assetid $ASSET_ID --amount $ASSET_AMOUNT expect { timeout { ::AlgorandGoal::Abort "Timed out asset transfer" } - "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r" } + "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r"; exp_continue } + eof } } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in AssetTransfer: $EXCEPTION" @@ -484,15 +486,27 @@ proc ::AlgorandGoal::AssetLookup { CREATOR UNIT_NAME TEST_PRIMARY_NODE_DIR } { proc ::AlgorandGoal::AssembleGroup { INPUT_GROUP OUTPUT_GROUP } { if { [ catch { spawn goal clerk group -i $INPUT_GROUP -o $OUTPUT_GROUP - expect { + expect { timeout { ::AlgorandGoal::Abort "Timed out assembling group transaction" } - close + eof } } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in AssembleGroup: $EXCEPTION" } } +# Split group txn +proc ::AlgorandGoal::SplitGroup { INPUT_GROUP OUTPUT_GROUP } { + if { [ catch { + spawn goal clerk split -i $INPUT_GROUP -o $OUTPUT_GROUP + expect { + timeout { ::AlgorandGoal::Abort "Timed out splitting group transaction" } + eof + } + } EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in Split Group: $EXCEPTION" + } +} # Generate limit order code from teal template proc ::AlgorandGoal::LimitOrder {TEAL_DRIVER SWAP_N SWAP_D MIN_TRD OWNER FEE TIME_OUT ASSET_ID TEAL_OUTPUT} { @@ -551,14 +565,14 @@ proc ::AlgorandGoal::TealTxnCreate { TEAL_SOURCE TO_ADDR CLOSE_TO SEND_AMOUNT NO } } -# Transfer asset +# Sign Transaction proc ::AlgorandGoal::SignTransaction { WALLET_NAME WALLET_PASSWORD INPUT_TXN OUTPUT_TXN TEST_PRIMARY_NODE_DIR} { if { [ catch { spawn goal clerk sign -d $TEST_PRIMARY_NODE_DIR -w $WALLET_NAME -i $INPUT_TXN -o $OUTPUT_TXN expect { timeout { ::AlgorandGoal::Abort "Timed out signing transaction" } - "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r" } - close + "Please enter the password for wallet '$WALLET_NAME':" { send "$WALLET_PASSWORD\r" ; exp_continue} + eof } } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in SignTransaction: $EXCEPTION" @@ -574,7 +588,7 @@ proc ::AlgorandGoal::RawSend { TXN_FILE TEST_PRIMARY_NODE_DIR } { expect { timeout { close; ::AlgorandGoal::Abort "Timed out rawsend $TXN_FILE" } -re {Transaction ([A-Z0-9]{52}) committed} {set TRANSACTION_ID $expect_out(1,string); close } - -re {Rejected transactions written to (.+rej)} {::AlgorandGoal::Abort "RawSend rejected."} + -re {Rejected transactions written to (.+rej)} {::AlgorandGoal::Abort "RawSend rejected."} } } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in RawSend: $EXCEPTION" @@ -938,6 +952,46 @@ proc ::AlgorandGoal::AppCreate { WALLET_NAME WALLET_PASSWORD ACCOUNT_ADDRESS APP return $APP_ID } +# App Create with onCompletion argument +proc ::AlgorandGoal::AppCreateOnCompletion { \ + WALLET_NAME \ + WALLET_PASSWORD \ + ACCOUNT_ADDRESS \ + APPROVAL_PROGRAM \ + APP_ARG GLOBAL_BYTE_SLICES \ + LOCAL_BYTE_SLICES \ + CLEAR_PROGRAM \ + DATA_DIR \ + ON_COMPLETION} { + set timeout 60 + if { [ catch { + set APP_ID "NOT SET" + puts "calling goal app create" + spawn goal app create \ + --creator $ACCOUNT_ADDRESS \ + --on-completion $ON_COMPLETION \ + --approval-prog $APPROVAL_PROGRAM \ + --global-byteslices $GLOBAL_BYTE_SLICES \ + --global-ints 0 \ + --local-byteslices $LOCAL_BYTE_SLICES \ + --local-ints 1 \ + --app-arg $APP_ARG \ + --clear-prog $CLEAR_PROGRAM \ + -w $WALLET_NAME \ + -d $DATA_DIR + expect { + timeout { abort "\n Failed to see expected output" } + "Please enter the password for wallet '$WALLET_NAME':" {send "$WALLET_PASSWORD\r" ; exp_continue } + -re {Created app with app index ([0-9]+)} {set APP_ID $expect_out(1,string) ;close } + eof {close; ::AlgorandGoal::Abort "app not created" } + } + puts "App ID $APP_ID" + } EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in AppCreate: $EXCEPTION" + } + return $APP_ID +} + # App OptIn with 1 argument proc ::AlgorandGoal::AppOptIn { APP_ID WALLET_NAME WALLET_PASSWORD ACCOUNT_ADDRESS APP_ARG1 DATA_DIR } { set timeout 60 @@ -1044,3 +1098,12 @@ proc ::AlgorandGoal::CheckEOF { { ERROR_STRING "" } } { exit 1 } } + +# Print contents of transaction file +proc ::AlgorandGoal::InspectTransactionFile { TRX_FILE } { + puts "\n Inspect $TRX_FILE" + spawn goal clerk inspect $TRX_FILE + expect { + eof + } +} diff --git a/test/e2e-go/cli/goal/expect/tealAndStatefulTealTest.exp b/test/e2e-go/cli/goal/expect/tealAndStatefulTealTest.exp new file mode 100644 index 000000000..b9f1db858 --- /dev/null +++ b/test/e2e-go/cli/goal/expect/tealAndStatefulTealTest.exp @@ -0,0 +1,163 @@ +#!/usr/bin/expect -f +set err 0 +log_user 1 + +if { [catch { + source goalExpectCommon.exp + set TEST_ALGO_DIR [lindex $argv 0] + set TEST_DATA_DIR [lindex $argv 1] + + puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" + puts "TEST_DATA_DIR: $TEST_DATA_DIR" + + set TIME_STAMP [clock seconds] + + set TEST_ROOT_DIR $TEST_ALGO_DIR/root + set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ + set NETWORK_NAME test_net_expect_$TIME_STAMP + set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50EachFuture.json" + + # Create network + ::AlgorandGoal::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + # Start network + ::AlgorandGoal::StartNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + set PRIMARY_NODE_ADDRESS [ ::AlgorandGoal::GetAlgodNetworkAddress $TEST_PRIMARY_NODE_DIR ] + puts "Primary Node Address: $PRIMARY_NODE_ADDRESS" + + set PRIMARY_WALLET_NAME unencrypted-default-wallet + + # Determine primary account + set PRIMARY_ACCOUNT_ADDRESS [::AlgorandGoal::GetHighestFundedAccountForWallet $PRIMARY_WALLET_NAME $TEST_PRIMARY_NODE_DIR] + + # Check the balance of the primary account + set PRIMARY_ACCOUNT_BALANCE [::AlgorandGoal::GetAccountBalance $PRIMARY_WALLET_NAME $PRIMARY_ACCOUNT_ADDRESS $TEST_PRIMARY_NODE_DIR] + puts "Primary Account Balance: $PRIMARY_ACCOUNT_BALANCE" + + # Check the rewards of the primary account + set PRIMARY_ACCOUNT_EARNINGS [::AlgorandGoal::GetAccountRewards $PRIMARY_WALLET_NAME $PRIMARY_ACCOUNT_ADDRESS $TEST_PRIMARY_NODE_DIR] + puts "Primary Account Rewards: $PRIMARY_ACCOUNT_EARNINGS" + + # Create wallet #1 + set WALLET_1_NAME Wallet_1_$TIME_STAMP + set WALLET_1_PASSWORD 1234 + set WALLET_1_PASSPHRASE [::AlgorandGoal::CreateWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + puts "WALLET_1_PASSPHRASE: $WALLET_1_PASSPHRASE" + ::AlgorandGoal::VerifyWallet $WALLET_1_NAME $TEST_PRIMARY_NODE_DIR + + # Associate a new account with the wallet + set ACCOUNT_1_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_1_NAME $WALLET_1_PASSWORD $TEST_PRIMARY_NODE_DIR] + ::AlgorandGoal::VerifyAccount $WALLET_1_NAME $WALLET_1_PASSWORD $ACCOUNT_1_ADDRESS $TEST_PRIMARY_NODE_DIR + + # Create wallet #2 + set WALLET_2_NAME Wallet_2_$TIME_STAMP + set WALLET_2_PASSWORD 12345678 + set WALLET_2_PASSPHRASE [::AlgorandGoal::CreateWallet $WALLET_2_NAME $WALLET_2_PASSWORD $TEST_PRIMARY_NODE_DIR] + puts "WALLET_2_PASSPHRASE: $WALLET_2_PASSPHRASE" + ::AlgorandGoal::VerifyWallet $WALLET_2_NAME $TEST_PRIMARY_NODE_DIR + + # Associate a new account with the wallet + set ACCOUNT_2_ADDRESS [::AlgorandGoal::CreateAccountForWallet $WALLET_2_NAME $WALLET_2_PASSWORD $TEST_PRIMARY_NODE_DIR] + ::AlgorandGoal::VerifyAccount $WALLET_2_NAME $WALLET_2_PASSWORD $ACCOUNT_2_ADDRESS $TEST_PRIMARY_NODE_DIR + + # -------------------------- setup accounts ---------------------------------- + + # Transfer Algos from primary account to account 1 + set MIN_BALANCE 1000000 + set TRANSFER_AMOUNT [expr {1000 * $MIN_BALANCE}] + set FEE_AMOUNT 1000 + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_1_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + + # Print the transaction id + puts "TRANSACTION_ID 1: $TRANSACTION_ID" + + # Check to make sure that the transfer is reflected in the to account + ::AlgorandGoal::WaitForAccountBalance $WALLET_1_NAME $ACCOUNT_1_ADDRESS $TRANSFER_AMOUNT $TEST_PRIMARY_NODE_DIR + + # Transfer Algos from primary account to account 2 + set TRANSACTION_ID [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ACCOUNT_2_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + + # Print the transaction id + puts "TRANSACTION_ID 2: $TRANSACTION_ID" + + # Check to make sure that the transfer is reflected in the to account + ::AlgorandGoal::WaitForAccountBalance $WALLET_2_NAME $ACCOUNT_2_ADDRESS $TRANSFER_AMOUNT $TEST_PRIMARY_NODE_DIR + + # -------------------------- setup working dir ---------------------------------- + + set TEST_WORKING_DIR $TEST_ROOT_DIR/work + + puts "setting up working dir $TEST_WORKING_DIR" + + exec mkdir -p $TEST_WORKING_DIR + + catch { cd $TEST_WORKING_DIR } + + # -------------------------- setup application ---------------------------------- + + set TEAL_SOURCE "$TEST_WORKING_DIR/simple.teal" + + puts "\nwriting teal script to file '$TEAL_SOURCE'" + + set CHAN [open $TEAL_SOURCE w] + puts $CHAN "#pragma version 2\nint 1\n" + close $CHAN + + puts "reading from file $TEAL_SOURCE" + puts [exec cat $TEAL_SOURCE] + + # compile teal assembly to bytecode + set ESCROW_ADDRESS [::AlgorandGoal::TealCompile $TEAL_SOURCE] + + # -------------------------- create mixed teal and stateful teal transaction ---------------------------------- + + set TRANSACTION_ID_APP [::AlgorandGoal::AccountTransfer $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TRANSFER_AMOUNT $ESCROW_ADDRESS $FEE_AMOUNT $TEST_PRIMARY_NODE_DIR] + + # Print the transaction id + puts "TRANSACTION_ID_APP: $TRANSACTION_ID_APP, APP_ACCOUNT_ADDRESS: $ESCROW_ADDRESS" + + # Check to make sure that the transfer is reflected in the to account + ::AlgorandGoal::WaitForAccountBalance $WALLET_1_NAME $ESCROW_ADDRESS $TRANSFER_AMOUNT $TEST_PRIMARY_NODE_DIR + + puts "calling app create" + set GLOBAL_BYTE_SLICES 1 + set LOCAL_BYTE_SLICES 0 + set APP_ID [::AlgorandGoal::AppCreate0 $PRIMARY_WALLET_NAME "" $PRIMARY_ACCOUNT_ADDRESS $TEAL_SOURCE $GLOBAL_BYTE_SLICES $LOCAL_BYTE_SLICES $TEAL_SOURCE $TEST_PRIMARY_NODE_DIR] + + exec goal app call --app-id $APP_ID --from $ACCOUNT_1_ADDRESS --out=unsginedtransaction1.tx -d $TEST_PRIMARY_NODE_DIR + + exec goal clerk send --to=$ACCOUNT_1_ADDRESS --close-to=$ACCOUNT_1_ADDRESS --from-program=$TEAL_SOURCE --amount=409000 --out=unsginedtransaction2.tx -d $TEST_PRIMARY_NODE_DIR + + ::AlgorandGoal::SignTransaction $WALLET_1_NAME $WALLET_1_PASSWORD unsginedtransaction1.tx sginedtransaction1.tx $TEST_PRIMARY_NODE_DIR + + puts "form combined transaction" + exec cat unsginedtransaction1.tx unsginedtransaction2.tx > combinedtransactions.tx + + puts "create group transaction" + ::AlgorandGoal::AssembleGroup combinedtransactions.tx groupedtransactions.tx + + puts "split transaction" + ::AlgorandGoal::SplitGroup groupedtransactions.tx split.tx + + puts "sign the split transaction" + set RAW_TX_1 split-0.tx + set RAW_STX_1 signout-0.tx + ::AlgorandGoal::SignTransaction $WALLET_1_NAME $WALLET_1_PASSWORD $RAW_TX_1 $RAW_STX_1 $TEST_PRIMARY_NODE_DIR + + puts "\ncombine into the sign out transaction" + exec cat signout-0.tx split-1.tx > signout.tx + + puts "submit the signout transaction" + ::AlgorandGoal::RawSend signout.tx $TEST_PRIMARY_NODE_DIR + + # Shutdown the network + ::AlgorandGoal::StopNetwork $NETWORK_NAME $TEST_ALGO_DIR $TEST_ROOT_DIR + + puts "Mixed Teal Test Successful" + + exit 0 + +} EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in Mixed Teal Test: $EXCEPTION" +} diff --git a/test/scripts/e2e_subs/rest.sh b/test/scripts/e2e_subs/rest.sh index aaaa5bd08..037579205 100755 --- a/test/scripts/e2e_subs/rest.sh +++ b/test/scripts/e2e_subs/rest.sh @@ -16,23 +16,48 @@ PUB_TOKEN=$(cat "$ALGORAND_DATA"/algod.token) ADMIN_TOKEN=$(cat "$ALGORAND_DATA"/algod.admin.token) NET=$(cat "$ALGORAND_DATA"/algod.net) + +function base_call { + curl -o "$3" -w "%{http_code}" -q -s -H "Authorization: Bearer $1" "$NET$2" +} + + function call_admin { - curl -q -s -H "Authorization: Bearer ${ADMIN_TOKEN}" "$NET$1" + base_call "$ADMIN_TOKEN" "$1" "$2" } + function call { - curl -q -s -H "Authorization: Bearer ${PUB_TOKEN}" "$NET$1" + base_call "$PUB_TOKEN" "$1" "$2" +} + + +function fail_and_exit { + printf "\n\nFailed test - $1 ($2): $3\n\n" + exit 1 } + # $1 - test description. # $2 - query -# $3 - substring that should be in the response +# $3 - expected status code +# $4 - substring that should be in the response function call_and_verify { - local RES - RES=$(call "$2") - if [[ "$RES" != *"$3"* ]]; then - echo "Failed test - $2: $1" - exit 1 + local CODE + + set +e + CODE=$(call "$2" "${TEMPDIR}/curl_out.txt") + if [[ $? != 0 ]]; then + fail_and_exit "$1" "$2" "curl had a non-zero exit code." + fi + set -e + + RES=$(cat "${TEMPDIR}/curl_out.txt") + if [[ "$CODE" != "$3" ]]; then + fail_and_exit "$1" "$2" "unexpected HTTP status code expected $3 (actual $CODE)" + fi + if [[ "$RES" != *"$4"* ]]; then + fail_and_exit "$1" "$2" "unexpected response. should contain '$4', actual: $RES" fi } @@ -43,11 +68,11 @@ function test_applications_endpoint { APPID=$(${gcmd} app create --creator "${ACCOUNT}" --approval-prog "${TEMPDIR}/simple.teal" --clear-prog "${TEMPDIR}/simple.teal" --global-byteslices 0 --global-ints 2 --local-byteslices 0 --local-ints 0 | grep Created | awk '{ print $6 }') # Good request, non-existant app id - call_and_verify "Should not find app." "/v2/applications/987654321" 'application does not exist' + call_and_verify "Should not find app." "/v2/applications/987654321" 404 'application does not exist' # Good request - call_and_verify "Should contain app data." "/v2/applications/$APPID" '"global-state-schema":{"num-byte-slice":0,"num-uint":2}' + call_and_verify "Should contain app data." "/v2/applications/$APPID" 200 '"global-state-schema":{"num-byte-slice":0,"num-uint":2}' # Good request, pretty response - call_and_verify "Should contain app data." "/v2/applications/$APPID?pretty" ' + call_and_verify "Should contain app data." "/v2/applications/$APPID?pretty" 200 ' "global-state-schema": { "num-byte-slice": 0, "num-uint": 2 @@ -58,11 +83,36 @@ function test_applications_endpoint { } ' # Some invalid path parameters - call_and_verify "Parameter parsing error." /v2/applications/-2 "Invalid format for parameter application-id" - call_and_verify "Parameter parsing error." /v2/applications/not-a-number "Invalid format for parameter application-id" + call_and_verify "App parameter parsing error 1." "/v2/applications/-2" 400 "Invalid format for parameter application-id" + call_and_verify "App parameter parsing error 2." "/v2/applications/not-a-number" 400 "Invalid format for parameter application-id" # Good request, but invalid query parameters - call_and_verify "Invalid parameter" "/v2/applications/$APPID?this-should-fail=200" 'Unknown parameter detected: this-should-fail' + call_and_verify "App invalid parameter" "/v2/applications/$APPID?this-should-fail=200" 400 'Unknown parameter detected: this-should-fail' } + +function test_assets_endpoint { + local ASSET_ID + ASSET_ID=$(${gcmd} asset create --creator "${ACCOUNT}" --total 10000 --decimals 19 --name "spanish coin" --unitname "doubloon" | grep "Created asset with asset index" | rev | cut -d ' ' -f 1 | rev) + + # Good request, non-existant asset id + call_and_verify "Should not find asset." "/v2/assets/987654321" 404 'asset does not exist' + # Good request + call_and_verify "Should contain asset data." "/v2/assets/$ASSET_ID" 200 '","decimals":19,"default-frozen":false,"freeze":"' + # Good request, pretty response + call_and_verify "Should contain asset data." "/v2/assets/$ASSET_ID?pretty" 200 ' + "decimals": 19, + "default-frozen": false, + "freeze": "' + # Some invalid path parameters + call_and_verify "Asset parameter parsing error 1." "/v2/assets/-2" 400 "Invalid format for parameter asset-id" + call_and_verify "Asset parameter parsing error 2." "/v2/assets/not-a-number" 400 "Invalid format for parameter asset-id" + + # Good request, but invalid query parameters + call_and_verify "Asset invalid parameter" "/v2/assets/$ASSET_ID?this-should-fail=200" 400 'parameter detected: this-should-fail' +} + + +# Run the tests. test_applications_endpoint +test_assets_endpoint diff --git a/test/scripts/e2e_subs/tealprogs/appAccountParams.teal b/test/scripts/e2e_subs/tealprogs/appAccountParams.teal new file mode 100644 index 000000000..ac0385b5c --- /dev/null +++ b/test/scripts/e2e_subs/tealprogs/appAccountParams.teal @@ -0,0 +1,76 @@ +#pragma version 2 + +// Writes the passed account addresses to the caller's and +// account_2's local spaces. + +int OptIn +txn OnCompletion +== +bnz skipped + +// Write the number of accounts passed as parameters +int 0 // write to app caller local space +byte "Num Accounts" // key +txn NumAccounts // get number of accounts passed (value) +app_local_put + +// Write the passed accounts to the local space of the caller +int 0 +byte "Account0" +txn Accounts 0 +app_local_put + +int 0 +byte "Account1" +txn Accounts 1 +app_local_put + +int 0 +byte "Account2" +txn Accounts 2 +app_local_put + +int 0 +byte "Account3" +txn Accounts 3 +app_local_put + +int 0 +byte "Account4" +txn Accounts 4 +app_local_put + +// Write the passed accounts to the local space of account 2 +int 2 +byte "Account0" +txn Accounts 0 +app_local_put + +int 2 +byte "Account1" +txn Accounts 1 +app_local_put + +int 2 +byte "Account2" +txn Accounts 2 +app_local_put + +int 2 +byte "Account3" +txn Accounts 3 +app_local_put + +int 2 +byte "Account4" +txn Accounts 4 +app_local_put + +// Write the number of accounts passed as parameters +int 2 +byte "Num Accounts" // key +txn NumAccounts // get number of accounts passed (value) +app_local_put + +skipped: +int 1 |