telemetry: add TCP RTT info collection (#4745)feature/node-type

10 files changed, 423 insertions, 112 deletions

diff --git a/logging/telemetryspec/event.go b/logging/telemetryspec/event.go
index 91c7bddf9..b98830bc0 100644
--- a/logging/telemetryspec/event.go
+++ b/logging/telemetryspec/event.go
@@ -18,6 +18,8 @@ package telemetryspec
 
 import (
 	"time"
+
+	"github.com/algorand/go-algorand/util"
 )
 
 // Telemetry Events
@@ -302,6 +304,8 @@ type PeerConnectionDetails struct {
 	MessageDelay int64 `json:",omitempty"`
 	// DuplicateFilterCount is the number of times this peer has sent us a message hash to filter that it had already sent before.
 	DuplicateFilterCount uint64
+	// TCPInfo provides connection measurements from TCP.
+	TCP util.TCPInfo `json:",omitempty"`
 }
 
 // CatchpointGenerationEvent event
diff --git a/network/limitlistener/rejectingLimitListener.go b/network/limitlistener/rejectingLimitListener.go
index 60d89199c..9d6a5914b 100644
--- a/network/limitlistener/rejectingLimitListener.go
+++ b/network/limitlistener/rejectingLimitListener.go
@@ -83,3 +83,7 @@ func (l *rejectingLimitListenerConn) Close() error {
 	l.releaseOnce.Do(l.release)
 	return err
 }
+
+func (l *rejectingLimitListenerConn) UnderlyingConn() net.Conn {
+	return l.Conn
+}
diff --git a/network/requestTracker.go b/network/requestTracker.go
index fd78dadca..2ae34c81f 100644
--- a/network/requestTracker.go
+++ b/network/requestTracker.go
@@ -254,6 +254,10 @@ type requestTrackedConnection struct {
 	tracker *RequestTracker
 }
 
+func (c *requestTrackedConnection) UnderlyingConn() net.Conn {
+	return c.Conn
+}
+
 // Close removes the connection from the tracker's connections map and call the underlaying Close function.
 func (c *requestTrackedConnection) Close() error {
 	c.tracker.hostRequestsMu.Lock()
diff --git a/network/wsNetwork.go b/network/wsNetwork.go
index d5f8b111e..2dbe9b590 100644
--- a/network/wsNetwork.go
+++ b/network/wsNetwork.go
@@ -1805,14 +1805,23 @@ func (wn *WebsocketNetwork) OnNetworkAdvance() {
 // to the telemetry server. Internally, it's using a timer to ensure that it would only
 // send the information once every hour ( configurable via PeerConnectionsUpdateInterval )
 func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
+	if !wn.log.GetTelemetryEnabled() {
+		return
+	}
 	now := time.Now()
 	if wn.lastPeerConnectionsSent.Add(time.Duration(wn.config.PeerConnectionsUpdateInterval)*time.Second).After(now) || wn.config.PeerConnectionsUpdateInterval <= 0 {
 		// it's not yet time to send the update.
 		return
 	}
 	wn.lastPeerConnectionsSent = now
+
 	var peers []*wsPeer
 	peers, _ = wn.peerSnapshot(peers)
+	connectionDetails := wn.getPeerConnectionTelemetryDetails(now, peers)
+	wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails)
+}
+
+func (wn *WebsocketNetwork) getPeerConnectionTelemetryDetails(now time.Time, peers []*wsPeer) telemetryspec.PeersConnectionDetails {
 	var connectionDetails telemetryspec.PeersConnectionDetails
 	for _, peer := range peers {
 		connDetail := telemetryspec.PeerConnectionDetails{
@@ -1821,6 +1830,18 @@ func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
 			InstanceName:         peer.InstanceName,
 			DuplicateFilterCount: peer.duplicateFilterCount,
 		}
+		// unwrap websocket.Conn, requestTrackedConnection, rejectingLimitListenerConn
+		var uconn net.Conn = peer.conn.UnderlyingConn()
+		for i := 0; i < 10; i++ {
+			wconn, ok := uconn.(wrappedConn)
+			if !ok {
+				break
+			}
+			uconn = wconn.UnderlyingConn()
+		}
+		if tcpInfo, err := util.GetConnTCPInfo(uconn); err == nil && tcpInfo != nil {
+			connDetail.TCP = *tcpInfo
+		}
 		if peer.outgoing {
 			connDetail.Address = justHost(peer.conn.RemoteAddr().String())
 			connDetail.Endpoint = peer.GetAddress()
@@ -1831,8 +1852,7 @@ func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
 			connectionDetails.IncomingPeers = append(connectionDetails.IncomingPeers, connDetail)
 		}
 	}
-
-	wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails)
+	return connectionDetails
 }
 
 // prioWeightRefreshTime controls how often we refresh the weights
diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go
index 37dd646aa..c2e467859 100644
--- a/network/wsNetwork_test.go
+++ b/network/wsNetwork_test.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
@@ -268,14 +269,17 @@ func netStop(t testing.TB, wn *WebsocketNetwork, name string) {
 	t.Logf("%s done", name)
 }
 
-// Set up two nodes, test that a.Broadcast is received by B
-func TestWebsocketNetworkBasic(t *testing.T) {
-	partitiontest.PartitionTest(t)
+func setupWebsocketNetworkAB(t *testing.T, countTarget int) (*WebsocketNetwork, *WebsocketNetwork, *messageCounterHandler, func()) {
+	success := false
 
 	netA := makeTestWebsocketNode(t)
 	netA.config.GossipFanout = 1
 	netA.Start()
-	defer netStop(t, netA, "A")
+	defer func() {
+		if !success {
+			netStop(t, netA, "A")
+		}
+	}()
 	netB := makeTestWebsocketNode(t)
 	netB.config.GossipFanout = 1
 	addrA, postListen := netA.Address()
@@ -283,9 +287,12 @@ func TestWebsocketNetworkBasic(t *testing.T) {
 	t.Log(addrA)
 	netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
 	netB.Start()
-	defer netStop(t, netB, "B")
-	counter := newMessageCounter(t, 2)
-	counterDone := counter.done
+	defer func() {
+		if !success {
+			netStop(t, netB, "B")
+		}
+	}()
+	counter := newMessageCounter(t, countTarget)
 	netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
 
 	readyTimeout := time.NewTimer(2 * time.Second)
@@ -294,6 +301,21 @@ func TestWebsocketNetworkBasic(t *testing.T) {
 	waitReady(t, netB, readyTimeout.C)
 	t.Log("b ready")
 
+	success = true
+	closeFunc := func() {
+		netStop(t, netB, "B")
+		netStop(t, netB, "A")
+	}
+	return netA, netB, counter, closeFunc
+}
+
+// Set up two nodes, test that a.Broadcast is received by B
+func TestWebsocketNetworkBasic(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+	defer closeFunc()
+	counterDone := counter.done
 	netA.Broadcast(context.Background(), protocol.TxnTag, []byte("foo"), false, nil)
 	netA.Broadcast(context.Background(), protocol.TxnTag, []byte("bar"), false, nil)
 
@@ -384,27 +406,9 @@ func TestWebsocketProposalPayloadCompression(t *testing.T) {
 func TestWebsocketNetworkUnicast(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	netA := makeTestWebsocketNode(t)
-	netA.config.GossipFanout = 1
-	netA.Start()
-	defer netStop(t, netA, "A")
-	netB := makeTestWebsocketNode(t)
-	netB.config.GossipFanout = 1
-	addrA, postListen := netA.Address()
-	require.True(t, postListen)
-	t.Log(addrA)
-	netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
-	netB.Start()
-	defer netStop(t, netB, "B")
-	counter := newMessageCounter(t, 2)
+	netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+	defer closeFunc()
 	counterDone := counter.done
-	netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
-	readyTimeout := time.NewTimer(2 * time.Second)
-	waitReady(t, netA, readyTimeout.C)
-	t.Log("a ready")
-	waitReady(t, netB, readyTimeout.C)
-	t.Log("b ready")
 
 	require.Equal(t, 1, len(netA.peers))
 	require.Equal(t, 1, len(netA.GetPeers(PeersConnectedIn)))
@@ -425,26 +429,8 @@ func TestWebsocketNetworkUnicast(t *testing.T) {
 func TestWebsocketPeerData(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	netA := makeTestWebsocketNode(t)
-	netA.config.GossipFanout = 1
-	netA.Start()
-	defer netStop(t, netA, "A")
-	netB := makeTestWebsocketNode(t)
-	netB.config.GossipFanout = 1
-	addrA, postListen := netA.Address()
-	require.True(t, postListen)
-	t.Log(addrA)
-	netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
-	netB.Start()
-	defer netStop(t, netB, "B")
-	counter := newMessageCounter(t, 2)
-	netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
-	readyTimeout := time.NewTimer(2 * time.Second)
-	waitReady(t, netA, readyTimeout.C)
-	t.Log("a ready")
-	waitReady(t, netB, readyTimeout.C)
-	t.Log("b ready")
+	netA, _, _, closeFunc := setupWebsocketNetworkAB(t, 2)
+	defer closeFunc()
 
 	require.Equal(t, 1, len(netA.peers))
 	require.Equal(t, 1, len(netA.GetPeers(PeersConnectedIn)))
@@ -463,27 +449,9 @@ func TestWebsocketPeerData(t *testing.T) {
 func TestWebsocketNetworkArray(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	netA := makeTestWebsocketNode(t)
-	netA.config.GossipFanout = 1
-	netA.Start()
-	defer netStop(t, netA, "A")
-	netB := makeTestWebsocketNode(t)
-	netB.config.GossipFanout = 1
-	addrA, postListen := netA.Address()
-	require.True(t, postListen)
-	t.Log(addrA)
-	netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
-	netB.Start()
-	defer netStop(t, netB, "B")
-	counter := newMessageCounter(t, 3)
+	netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 3)
+	defer closeFunc()
 	counterDone := counter.done
-	netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
-	readyTimeout := time.NewTimer(2 * time.Second)
-	waitReady(t, netA, readyTimeout.C)
-	t.Log("a ready")
-	waitReady(t, netB, readyTimeout.C)
-	t.Log("b ready")
 
 	tags := []protocol.Tag{protocol.TxnTag, protocol.TxnTag, protocol.TxnTag}
 	data := [][]byte{[]byte("foo"), []byte("bar"), []byte("algo")}
@@ -500,27 +468,9 @@ func TestWebsocketNetworkArray(t *testing.T) {
 func TestWebsocketNetworkCancel(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	netA := makeTestWebsocketNode(t)
-	netA.config.GossipFanout = 1
-	netA.Start()
-	defer netStop(t, netA, "A")
-	netB := makeTestWebsocketNode(t)
-	netB.config.GossipFanout = 1
-	addrA, postListen := netA.Address()
-	require.True(t, postListen)
-	t.Log(addrA)
-	netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
-	netB.Start()
-	defer netStop(t, netB, "B")
-	counter := newMessageCounter(t, 100)
+	netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 100)
+	defer closeFunc()
 	counterDone := counter.done
-	netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
-	readyTimeout := time.NewTimer(2 * time.Second)
-	waitReady(t, netA, readyTimeout.C)
-	t.Log("a ready")
-	waitReady(t, netB, readyTimeout.C)
-	t.Log("b ready")
 
 	tags := make([]protocol.Tag, 100)
 	data := make([][]byte, 100)
@@ -721,29 +671,15 @@ func TestAddrToGossipAddr(t *testing.T) {
 
 type nopConn struct{}
 
-func (nc *nopConn) RemoteAddr() net.Addr {
-	return nil
-}
-func (nc *nopConn) NextReader() (int, io.Reader, error) {
-	return 0, nil, nil
-}
-func (nc *nopConn) WriteMessage(int, []byte) error {
-	return nil
-}
-func (nc *nopConn) WriteControl(int, []byte, time.Time) error {
-	return nil
-}
-func (nc *nopConn) SetReadLimit(limit int64) {
-}
-func (nc *nopConn) CloseWithoutFlush() error {
-	return nil
-}
-func (nc *nopConn) SetPingHandler(h func(appData string) error) {
-
-}
-func (nc *nopConn) SetPongHandler(h func(appData string) error) {
-
-}
+func (nc *nopConn) RemoteAddr() net.Addr                        { return nil }
+func (nc *nopConn) NextReader() (int, io.Reader, error)         { return 0, nil, nil }
+func (nc *nopConn) WriteMessage(int, []byte) error              { return nil }
+func (nc *nopConn) WriteControl(int, []byte, time.Time) error   { return nil }
+func (nc *nopConn) SetReadLimit(limit int64)                    {}
+func (nc *nopConn) CloseWithoutFlush() error                    { return nil }
+func (nc *nopConn) SetPingHandler(h func(appData string) error) {}
+func (nc *nopConn) SetPongHandler(h func(appData string) error) {}
+func (nc *nopConn) UnderlyingConn() net.Conn                    { return nil }
 
 var nopConnSingleton = nopConn{}
 
@@ -2739,3 +2675,65 @@ func TestPreparePeerData(t *testing.T) {
 		}
 	}
 }
+
+func TestWebsocketNetworkTelemetryTCP(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// start two networks and send 2 messages from A to B
+	closed := false
+	netA, netB, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+	defer func() {
+		if !closed {
+			closeFunc()
+		}
+	}()
+	counterDone := counter.done
+	netA.Broadcast(context.Background(), protocol.TxnTag, []byte("foo"), false, nil)
+	netA.Broadcast(context.Background(), protocol.TxnTag, []byte("bar"), false, nil)
+
+	select {
+	case <-counterDone:
+	case <-time.After(2 * time.Second):
+		t.Errorf("timeout, count=%d, wanted 2", counter.count)
+	}
+
+	// get RTT from both ends and assert nonzero
+	var peersA, peersB []*wsPeer
+	peersA, _ = netA.peerSnapshot(peersA)
+	detailsA := netA.getPeerConnectionTelemetryDetails(time.Now(), peersA)
+	peersB, _ = netB.peerSnapshot(peersB)
+	detailsB := netB.getPeerConnectionTelemetryDetails(time.Now(), peersB)
+	require.Len(t, detailsA.IncomingPeers, 1)
+	assert.NotZero(t, detailsA.IncomingPeers[0].TCP.RTT)
+	require.Len(t, detailsB.OutgoingPeers, 1)
+	assert.NotZero(t, detailsB.OutgoingPeers[0].TCP.RTT)
+
+	pcdA, err := json.Marshal(detailsA)
+	assert.NoError(t, err)
+	pcdB, err := json.Marshal(detailsB)
+	assert.NoError(t, err)
+	t.Log("detailsA", string(pcdA))
+	t.Log("detailsB", string(pcdB))
+
+	// close connections
+	closeFunc()
+	closed = true
+	// open more FDs by starting 2 more networks
+	_, _, _, closeFunc2 := setupWebsocketNetworkAB(t, 2)
+	defer closeFunc2()
+	//  use stale peers snapshot from closed networks to get telemetry
+	// *net.OpError "use of closed network connection" err results in 0 rtt values
+	detailsA = netA.getPeerConnectionTelemetryDetails(time.Now(), peersA)
+	detailsB = netB.getPeerConnectionTelemetryDetails(time.Now(), peersB)
+	require.Len(t, detailsA.IncomingPeers, 1)
+	assert.Zero(t, detailsA.IncomingPeers[0].TCP.RTT)
+	require.Len(t, detailsB.OutgoingPeers, 1)
+	assert.Zero(t, detailsB.OutgoingPeers[0].TCP.RTT)
+
+	pcdA, err = json.Marshal(detailsA)
+	assert.NoError(t, err)
+	pcdB, err = json.Marshal(detailsB)
+	assert.NoError(t, err)
+	t.Log("closed detailsA", string(pcdA))
+	t.Log("closed detailsB", string(pcdB))
+}
diff --git a/network/wsPeer.go b/network/wsPeer.go
index e1ac6ffb3..d551cf1a6 100644
--- a/network/wsPeer.go
+++ b/network/wsPeer.go
@@ -124,6 +124,11 @@ type wsPeerWebsocketConn interface {
 	CloseWithoutFlush() error
 	SetPingHandler(h func(appData string) error)
 	SetPongHandler(h func(appData string) error)
+	wrappedConn
+}
+
+type wrappedConn interface {
+	UnderlyingConn() net.Conn
 }
 
 type sendMessage struct {
diff --git a/util/tcpinfo.go b/util/tcpinfo.go
new file mode 100644
index 000000000..2b4c69d29
--- /dev/null
+++ b/util/tcpinfo.go
@@ -0,0 +1,72 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"errors"
+	"net"
+	"syscall"
+)
+
+// TCPInfo provides socket-level TCP information.
+type TCPInfo struct {
+	RTT            uint32 `json:",omitempty"` // smoothed RTT
+	RTTVar         uint32 `json:",omitempty"` // RTT variance
+	RTTMin         uint32 `json:",omitempty"` // smallest observed RTT on the connection
+	SndMSS, RcvMSS uint32 `json:",omitempty"` // send and receive maximum segment size
+	SndCwnd        uint32 `json:",omitempty"` // sender congestion window
+	SndWnd         uint32 `json:",omitempty"` // send window advertised to receiver
+	RcvWnd         uint32 `json:",omitempty"` // receive window advertised to sender
+	//  tcpi_delivery_rate: The most recent goodput, as measured by
+	//    tcp_rate_gen(). If the socket is limited by the sending
+	//    application (e.g., no data to send), it reports the highest
+	//    measurement instead of the most recent. The unit is bytes per
+	//    second (like other rate fields in tcp_info).
+	Rate uint64 `json:",omitempty"`
+	//  tcpi_delivery_rate_app_limited: A boolean indicating if the goodput
+	//    was measured when the socket's throughput was limited by the
+	//    sending application.
+	AppLimited bool `json:",omitempty"`
+}
+
+var (
+	// ErrNotSyscallConn is reported when GetConnTCPInfo is passed a connection that doesn't satisfy the syscall.Conn interface.
+	ErrNotSyscallConn = errors.New("conn doesn't satisfy syscall.Conn")
+	// ErrTCPInfoUnsupported is reported if TCP information is not available for this platform.
+	ErrTCPInfoUnsupported = errors.New("GetConnRTT not supported on this platform")
+	// ErrNoTCPInfo is reported if getsockopt returned no TCP info for some reason.
+	ErrNoTCPInfo = errors.New("getsockopt returned no TCP info")
+)
+
+// GetConnTCPInfo returns statistics for a TCP connection collected by the
+// underlying network implementation, using a system call on Linux and Mac
+// and returning an error for unsupported platforms.
+func GetConnTCPInfo(conn net.Conn) (*TCPInfo, error) {
+	if conn == nil {
+		return nil, ErrNotSyscallConn
+	}
+	sysconn, ok := conn.(syscall.Conn)
+	if sysconn == nil || !ok {
+		return nil, ErrNotSyscallConn
+	}
+	raw, err := sysconn.SyscallConn()
+	if err != nil {
+		return nil, err
+	}
+
+	return getConnTCPInfo(raw)
+}
diff --git a/util/tcpinfo_darwin.go b/util/tcpinfo_darwin.go
new file mode 100644
index 000000000..cae19d06d
--- /dev/null
+++ b/util/tcpinfo_darwin.go
@@ -0,0 +1,49 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+func getConnTCPInfo(raw syscall.RawConn) (*TCPInfo, error) {
+	var info *unix.TCPConnectionInfo
+	var getSockoptErr error
+	err := raw.Control(func(fd uintptr) {
+		info, getSockoptErr = unix.GetsockoptTCPConnectionInfo(int(fd), unix.IPPROTO_TCP, unix.TCP_CONNECTION_INFO)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if getSockoptErr != nil {
+		return nil, getSockoptErr
+	}
+	if info == nil {
+		return nil, ErrNoTCPInfo
+	}
+	return &TCPInfo{
+		RTT:     info.Srtt,
+		RTTVar:  info.Rttvar,
+		SndMSS:  info.Maxseg, // MSS is the same for snd/rcv according bsd/netinet/tcp_usrreq.c
+		RcvMSS:  info.Maxseg,
+		SndCwnd: info.Snd_cwnd, // Send congestion window
+		SndWnd:  info.Snd_wnd,  // Advertised send window
+		RcvWnd:  info.Rcv_wnd,  // Advertised recv window
+	}, nil
+}
diff --git a/util/tcpinfo_linux.go b/util/tcpinfo_linux.go
new file mode 100644
index 000000000..3da707e1c
--- /dev/null
+++ b/util/tcpinfo_linux.go
@@ -0,0 +1,129 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+func getConnTCPInfo(raw syscall.RawConn) (*TCPInfo, error) {
+	var info linuxTCPInfo
+	size := unsafe.Sizeof(info)
+
+	var errno syscall.Errno
+	err := raw.Control(func(fd uintptr) {
+		_, _, errno = unix.Syscall6(unix.SYS_GETSOCKOPT, fd, unix.IPPROTO_TCP, unix.TCP_INFO,
+			uintptr(unsafe.Pointer(&info)), uintptr(unsafe.Pointer(&size)), 0)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if errno != 0 {
+		return nil, errno
+	}
+	if info == (linuxTCPInfo{}) {
+		return nil, ErrNoTCPInfo
+	}
+	return &TCPInfo{
+		RTT:        info.rtt,
+		RTTVar:     info.rttvar,
+		RTTMin:     info.min_rtt,
+		SndMSS:     info.snd_mss,
+		RcvMSS:     info.rcv_mss,
+		SndCwnd:    info.snd_cwnd, // Send congestion window
+		RcvWnd:     info.snd_wnd,  // "tp->snd_wnd, the receive window that the receiver has advertised to the sender."
+		Rate:       info.delivery_rate,
+		AppLimited: bool((info.app_limited >> 7) != 0), // get first bit
+	}, nil
+}
+
+// linuxTCPInfo is based on linux include/uapi/linux/tcp.h struct tcp_info
+//revive:disable:var-naming
+type linuxTCPInfo struct {
+	state       uint8
+	ca_state    uint8
+	retransmits uint8
+	probes      uint8
+	backoff     uint8
+	options     uint8
+	wscale      uint8 // __u8 tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4;
+	app_limited uint8 // __u8 tcpi_delivery_rate_app_limited:1, tcpi_fastopen_client_fail:2;
+
+	rto     uint32
+	ato     uint32
+	snd_mss uint32
+	rcv_mss uint32
+
+	unacked uint32
+	sacked  uint32
+	lost    uint32
+	retrans uint32
+	fackets uint32
+
+	last_data_sent uint32
+	last_ack_sent  uint32
+	last_data_recv uint32
+	last_ack_recv  uint32
+
+	pmtu         uint32
+	rcv_ssthresh uint32
+	rtt          uint32
+	rttvar       uint32
+	snd_ssthresh uint32
+	snd_cwnd     uint32
+	advmss       uint32
+	reordering   uint32
+
+	rcv_rtt   uint32
+	rcv_space uint32
+
+	total_retrans uint32
+
+	// extended info beyond what's in syscall.TCPInfo
+	pacing_rate     uint64
+	max_pacing_rate uint64
+	byte_acked      uint64
+	bytes_received  uint64
+	segs_out        uint32
+	segs_in         uint32
+
+	notsent_bytes uint32
+	min_rtt       uint32
+	data_segs_in  uint32
+	data_segs_out uint32
+
+	delivery_rate uint64
+
+	busy_time      uint64
+	rwnd_limited   uint64
+	sndbuf_limited uint64
+
+	delivered    uint32
+	delivered_ce uint32
+
+	bytes_sent    uint64
+	bytes_retrans uint64
+	dsack_dups    uint32
+	reord_seen    uint32
+
+	rcv_ooopack uint32
+
+	snd_wnd uint32
+}
diff --git a/util/tcpinfo_noop.go b/util/tcpinfo_noop.go
new file mode 100644
index 000000000..a155ed929
--- /dev/null
+++ b/util/tcpinfo_noop.go
@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+//go:build !linux && !darwin
+// +build !linux,!darwin
+
+package util
+
+import "syscall"
+
+func getConnTCPInfo(conn syscall.RawConn) (*TCPInfo, error) {
+	return nil, ErrTCPInfoUnsupported
+}