test/BIP324: functional tests for v2 P2P encryption #24748

597@@ -598,6 +598,11 @@ def wait_for_disconnect(self, timeout=60):
598         test_function = lambda: not self.is_connected
599         self.wait_until(test_function, timeout=timeout, check_connected=False)
600 
601+    def wait_for_reconnect(self, timeout=60):

1912@@ -1913,7 +1913,11 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ
1913     CSemaphoreGrant grant(*semOutbound, true);
1914     if (!grant) return false;
1915 
1916-    OpenNetworkConnection(CAddress(), false, std::move(grant), address.c_str(), conn_type, /*use_v2transport=*/false);
1917+    CAddress addr(CService(), NODE_NONE);
1918+    if (use_v2transport) {

 96@@ -18,3 +97,143 @@ def v2_ecdh(priv, ellswift_theirs, ellswift_ours, initiating):
 97         else:
 98             # Responding, place their public key encoding first.
 99             return TaggedHash("bip324_ellswift_xonly_ecdh", ellswift_theirs + ellswift_ours + ecdh_point_x32)
100+
101+    def initiate_v2_handshake(self, garbage_len=random.randrange(4096)):

133+        # Send garbage terminator
134+        msg_to_send = self.peer['send_garbage_terminator']
135+        # Optionally send decoy packets after garbage terminator.
136+        aad = self.sent_garbage
137+        for decoy_content_len in [random.randint(1, 100) for _ in range(10)]:
138+            msg_to_send += self.v2_enc_packet(decoy_content_len * b'\x00', aad=aad)

1208@@ -1209,13 +1209,14 @@ class CConnman
1209      * @param[in]   address     Address of node to try connecting to
1210      * @param[in]   conn_type   ConnectionType::OUTBOUND, ConnectionType::BLOCK_RELAY,
1211      *                          ConnectionType::ADDR_FETCH or ConnectionType::FEELER
1212+     * @param[in]   use_v2transport  Set to true if node supports BIP324 v2 transport protocol

356@@ -357,6 +357,7 @@ static RPCHelpMan addconnection()
357         {
358             {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address and port to attempt connecting to."},
359             {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open (\"outbound-full-relay\", \"block-relay-only\", \"addr-fetch\" or \"feeler\")."},
360+            {"v2transport", RPCArg::Type::BOOL, RPCArg::Default{false}, "Node supports BIP324 v2 transport protocol"},

388@@ -388,11 +389,12 @@ static RPCHelpMan addconnection()
389     } else {
390         throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString());
391     }
392+    bool use_v2transport = !request.params[2].isNull() && request.params[2].get_bool();

1879@@ -1880,7 +1880,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock,
1880     RandAddEvent((uint32_t)id);
1881 }
1882 
1883-bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type)
1884+bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type, bool use_v2transport = false)

178+                response = response[1:]
179+        # disconnect since garbage terminator was not seen after 4 KiB of garbage.
180+        return processed_length, False
181+
182+    def initialize_v2_transport(self, ecdh_secret):
183+        """Return a peer object with various BIP324 derived keys and ciphers."""

225+        """
226+        if len(response) < LENGTH_FIELD_LEN:
227+            return 0, b""
228+        enc_contents_len = response[:LENGTH_FIELD_LEN]
229+        response = response[LENGTH_FIELD_LEN:]
230+        contents_len = int.from_bytes(self.peer['recv_L'].crypt(enc_contents_len), 'little')

77+        - see section #overall-handshake-pseudocode in BIP 324
78+    - encrypt/decrypt v2 P2P messages.
79+        - see section #overall_packet_encryption_and_decryption_pseudocode in BIP 324
80+    """
81+    def __init__(self, **kwargs):
82+        self.initiating = kwargs['initiating']  # True if initiator

17 
18+logger = logging.getLogger("TestFramework.v2_p2p")
19+
20+MAGIC_BYTES = {
21+    "regtest": b"\xfa\xbf\xb5\xda"   # regtest
22+}

 99@@ -18,3 +100,147 @@ def v2_ecdh(priv, ellswift_theirs, ellswift_ours, initiating):
100         else:
101             # Responding, place their public key encoding first.
102             return TaggedHash("bip324_ellswift_xonly_ecdh", ellswift_theirs + ellswift_ours + ecdh_point_x32)
103+
104+    def initiate_v2_handshake(self):
105+        """Initiator begins the v2 handshake by sending its ellswift bytes and garbage"""
106+        self.privkey_ours, self.ellswift_ours = ellswift_create()
107+        garbage_len = random.randrange(4096)
108+        self.sent_garbage = os.urandom(garbage_len)

119+                return b""
120+            self.received_prefix += byte
121+            if self.received_prefix[-1] != V1_PREFIX[len(self.received_prefix) - 1]:
122+                self.privkey_ours, self.ellswift_ours = ellswift_create()
123+                garbage_len = random.randrange(4096)
124+                self.sent_garbage = os.urandom(garbage_len)

158+        processed_length = len(received_garbage)
159+        for i in range(4096):
160+            if received_garbage[-16:] == self.peer['recv_garbage_terminator']:
161+                # Receive, decode, and ignore version packet.
162+                # This includes skipping decoys and authenticating the received garbage.
163+                aad = received_garbage[:-16]

719+            self.addconnection('%s:%d' % (address, port), connection_type, advertise_v2_p2p)
720 
721         p2p_conn.p2p_connected_to_node = False
722-        p2p_conn.peer_accept_connection(connect_cb=addconnection_callback, connect_id=p2p_idx + 1, net=self.chain, timeout_factor=self.timeout_factor, **kwargs)()
723+        if advertise_v2_p2p:
724+            kwargs['services'] = kwargs['services']|NODE_P2P_V2 if 'services' in kwargs else P2P_SERVICES|NODE_P2P_V2

854@@ -853,9 +855,11 @@ def send_blocks_and_test(self, blocks, node, *, success=True, force_send=False,
855 
856         reject_reason = [reject_reason] if reject_reason else []
857         with node.assert_debug_log(expected_msgs=reject_reason):
858+            if is_decoy:  # since decoy messages don't get sent - no need to wait for response

18+    assert_equal,
19+    assert_greater_than,
20+    check_node_connections,
21+)
22+
23+REKEY_INTERVAL = 224

50+        self.v2_state = None
51+
52+    def connection_made(self, *args, **kwargs):
53+        """Custom implementation so that 64 bytes ellswift is sent in 2 parts during `initial_v2_handshake()`"""
54+        self.v2_state = TestEncryptedP2PState()
55+        super(PeerEarlyKey, self).connection_made(*args, **kwargs)

34+            1. when `send_network_magic` = True, send first 4 bytes of ellswift (matches network magic bytes)
35+            2. when `send_network_magic` = False, send remaining 60 bytes of ellswift
36+        """
37+        if self.send_net_magic:
38+            self.privkey_ours, self.ellswift_ours = ellswift_create()
39+            self.sent_garbage = os.urandom(garbage_len)

86+
87+            if i:
88+                # check if node1 connected to node0 (but not to node0's p2p connection directly)
89+                # gets blocks produced by node0's p2p connection
90+                self.log.info("Check if blocks produced by node0's p2p connection is received by node0")
91+                peer6.send_blocks_and_test(test_blocks, node0, success=True) # node0's tip advances

197@@ -197,6 +198,10 @@ def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, env=None
198         """Start the node."""
199         if extra_args is None:
200             extra_args = self.extra_args
201+        if "-v2transport=1" in extra_args:
202+            self.advertise_v2_p2p = True

257+
258+            # if we're the responder, read ellswift bytes till the first mismatch from 12 bytes V1_PREFIX in
259+            # `respond_v2_handshake()`.
260+            # `complete_handshake()` reads the remaining `64 - length` ellswift bytes afterwards from recvbuf.
261+            # if we're the initiator, length = 0 and 64 bytes ellswift is read from recvbuf in `complete_handshake()`
262+            if len(self.recvbuf) < 64 - length:

268+            self.recvbuf = self.recvbuf[64:]
269+
270+        # `self.v2_state.peer` is instantiated only after shared ECDH secret/BIP324 derived keys and ciphers
271+        # is derived in `complete_handshake()`.
272+        # so `authenticate_handshake()` which uses the BIP324 derived ciphers gets called after `complete_handshake()`.
273+        if self.v2_state.peer:

270+        # `self.v2_state.peer` is instantiated only after shared ECDH secret/BIP324 derived keys and ciphers
271+        # is derived in `complete_handshake()`.
272+        # so `authenticate_handshake()` which uses the BIP324 derived ciphers gets called after `complete_handshake()`.
273+        if self.v2_state.peer:
274+            # authenticate v2 handshake
275+            if len(self.recvbuf) < 16 + 20:

559@@ -553,6 +560,11 @@ def on_verack(self, message):
560 
561     def on_version(self, message):
562         assert message.nVersion >= MIN_P2P_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_P2P_VERSION_SUPPORTED)
563+        # reconnection using v1 P2P has happened since version message can be processed, previously queued messages are sent using v1 P2P here
564+        if self.reconnect:
565+            while self.queue_messages:
566+                message = self.queue_messages.pop(0)
567+                self.send_message(message)

56+}
57+
58+
59+def GetShortIDFromMessageType(msgtype):
60+    """Returns 1-byte short message type ID for the P2P message"""
61+    msgtype_to_shortid = dict(map(reversed, SHORTID.items()))

57+
58+
59+def GetShortIDFromMessageType(msgtype):
60+    """Returns 1-byte short message type ID for the P2P message"""
61+    msgtype_to_shortid = dict(map(reversed, SHORTID.items()))
62+    return msgtype_to_shortid[msgtype].to_bytes(1, 'big') if msgtype in msgtype_to_shortid else b"\x00"

102+    def generate_keypair_and_garbage(self):
103+        """Generates ellswift keypair and 4095 bytes garbage at max"""
104+        self.privkey_ours, self.ellswift_ours = ellswift_create()
105+        garbage_len = random.randrange(MAX_GARBAGE_LEN + 1)
106+        self.sent_garbage = random.randbytes(garbage_len)
107+        logger.debug("sending %d bytes of garbage data" % garbage_len)

337+                    else:
338+                        # a short id
339+                        if shortid in SHORTID:
340+                            msgtype = SHORTID[shortid]
341+                        else:
342+                            msgtype = "unknown-" + str(shortid)

399-        tmsg += h[:4]
400-        tmsg += data
401-        return tmsg
402+        if self.supports_v2_p2p:
403+            msgtype = message.msgtype
404+            data = message.serialize()

53+
54+    def run_test(self):
55+        node0, node1 = self.nodes[0], self.nodes[1]
56+        self.log.info("Check inbound connections to v2 TestNode from v2 P2PConnection is v2")
57+        peer1 = node0.add_p2p_connection(P2PInterface(), wait_for_verack=True, supports_v2_p2p=True)
58+        assert peer1.supports_v2_p2p

664             kwargs['dstaddr'] = '127.0.0.1'
665 
666         p2p_conn.p2p_connected_to_node = True
667-        p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor)()
668+        if self.use_v2transport:
669+            kwargs['services'] = kwargs['services']|NODE_P2P_V2 if 'services' in kwargs else P2P_SERVICES|NODE_P2P_V2

162@@ -163,6 +163,7 @@ def __init__(self):
163         # The underlying transport of the connection.
164         # Should only call methods on this from the NetworkThread, c.f. call_soon_threadsafe
165         self._transport = None
166+        self._send_lock = threading.Lock()

58+# Dictionary which contains short message type ID for the P2P message
59+MSGTYPE_TO_SHORTID = {msgtype: shortid for shortid, msgtype in SHORTID.items()}
60+
61 
62 class EncryptedP2PState:
63+    """A class for performing v2 P2P protocol functions on P2PConnection:

133+        self.initialize_v2_transport(ecdh_secret)
134+        # Send garbage terminator
135+        msg_to_send = self.peer['send_garbage_terminator']
136+        # Optionally send decoy packets after garbage terminator.
137+        aad = self.sent_garbage
138+        for decoy_content_len in [random.randint(1, 100) for _ in range(10)]:

220+        Returns:
221+            1. length - length of packet processed in order to update recvbuf.
222+                      - return 0 if only part of packet is received. (recvbuf not updated since decryption not done yet)
223+                      - return -1 if there's a MAC tag mismatch and disconnect.
224+            2. decrypted packet contents
225+                     - return b"" if only part of packet is received/MAC tag mismatch.

163+                    aad = b""
164+                    if length == -1:
165+                        return processed_length, False
166+                    elif length == 0:
167+                        return 0, True
168+                    # currently version packet has empty content while decoy packets don't

210@@ -199,8 +211,13 @@ def connection_made(self, transport):
211         assert not self._transport
212         logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
213         self._transport = transport
214+        # in an inbound connection to the TestNode, P2PConnection is the initiator. [TestNode <---- P2PConnection]
215+        # ellswift is generated and sent immediately to begin the initial v2 handshake.
216+        if self.supports_v2_p2p and self.v2_state.initiating and not self.v2_state.tried_v2_handshake:
217+            ellswift_and_garbage_data = self.v2_state.initiate_v2_handshake()

157+                # Receive, decode, and ignore version packet.
158+                # This includes skipping decoys and authenticating the received garbage.
159+                aad = received_garbage[:-16]
160+                while not self.tried_v2_handshake:
161+                    length, contents = self.v2_receive_packet(response, aad=aad)
162+                    processed_length += length

115+            byte = response.read(1)
116+            # return b"" if we need to receive more bytes
117+            if not byte:
118+                return b""
119+            self.received_prefix += byte
120+            v1_prefix = MAGIC_BYTES[self.net] + b'version\x00\x00\x00\x00\x00'

331+                    self.recvbuf = self.recvbuf[msglen:]
332+                    shortid = msg[0]
333+                    if shortid == 0:
334+                        # a string command
335+                        msgtype = msg[1:13].rstrip(b'\x00')
336+                        msg = msg[13:]  # msg is set to be payload

322+                if self.supports_v2_p2p:
323+                    if not self.v2_state.tried_v2_handshake:
324+                        return
325+                    # v2 P2P messages are read
326+                    if len(self.recvbuf) < 3 + 1 + 16:
327+                        return

257+        Returns:
258+        1. int - length of packet processed in order to update recvbuf.
259+               - return 0 if only part of packet is received. (recvbuf not updated since decryption not done yet)
260+               - return -1 if there's a MAC tag mismatch and disconnect.
261+        2. bytes - decrypted packet contents
262+                 - return b"" if only part of packet is received/MAC tag mismatch.

179@@ -174,16 +180,22 @@ def peer_connect_helper(self, dstaddr, dstport, net, timeout_factor):
180         self.recvbuf = b""
181         self.magic_bytes = MAGIC_BYTES[net]
182 
183-    def peer_connect(self, dstaddr, dstport, *, net, timeout_factor):
184+    def peer_connect(self, dstaddr, dstport, *, net, timeout_factor, supports_v2_p2p=False):

199+                    # decoy packets have contents = None. v2 handshake is complete only when version packet
200+                    # (can be empty with contents = b"") with contents != None is received.
201+                    if contents is not None:
202+                        self.tried_v2_handshake = True
203+                        return processed_length, True
204+                    response = response[length:]

320@@ -321,6 +321,8 @@ def _on_data(self):
321                         return
322                     self.recvbuf = self.recvbuf[msglen:]
323 
324+                    if msg is None:  # reject decoy messages

740@@ -741,7 +741,10 @@ def addconnection_callback(address, port):
741             p2p_conn.wait_until(lambda: p2p_conn.message_count["version"] == 1, check_connected=False)
742             p2p_conn.wait_until(lambda: not p2p_conn.is_connected, check_connected=False)
743         else:
744-            p2p_conn.wait_for_connect()
745+            if reconnect:
746+                p2p_conn.wait_for_reconnect()

192+                    length, contents = self.v2_receive_packet(response, aad=aad)
193+                    aad = b""
194+                    if length == -1:
195+                        return processed_length, False
196+                    elif length == 0:
197+                        return 0, True

252+        return enc_plaintext_len + aead_ciphertext
253+
254+    def v2_receive_packet(self, response, aad=b''):
255+        """Decrypt a BIP324 packet
256+
257+        Returns:

210@@ -199,8 +211,13 @@ def connection_made(self, transport):
211         assert not self._transport
212         logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
213         self._transport = transport
214+        # in an inbound connection to the TestNode, P2PConnection is the initiator. [TestNode <---- P2PConnection]
215+        # ellswift is generated and sent immediately to begin the initial v2 handshake.

161@@ -159,6 +162,9 @@ def __init__(self):
162         # The underlying transport of the connection.
163         # Should only call methods on this from the NetworkThread, c.f. call_soon_threadsafe
164         self._transport = None
165+        self.v2_state = None  # EncryptedP2PState object needed for v2 p2p connections
166+        self.supports_v2_p2p = False  # set if the connection supports v2 p2p
167+        self.queue_messages = []  # queue messages to send after initial v2 handshake

161@@ -159,6 +162,9 @@ def __init__(self):
162         # The underlying transport of the connection.
163         # Should only call methods on this from the NetworkThread, c.f. call_soon_threadsafe
164         self._transport = None
165+        self.v2_state = None  # EncryptedP2PState object needed for v2 p2p connections
166+        self.supports_v2_p2p = False  # set if the connection supports v2 p2p

216+        if self.supports_v2_p2p and self.v2_state.initiating and not self.v2_state.tried_v2_handshake:
217+            send_handshake_bytes = self.v2_state.initiate_v2_handshake()
218+            self.send_raw_message(send_handshake_bytes)
219         if self.on_connection_send_msg:
220-            self.send_message(self.on_connection_send_msg)
221+            self.queue_messages.append(self.on_connection_send_msg) if self.supports_v2_p2p else self.send_message(self.on_connection_send_msg)

230@@ -231,12 +231,66 @@ def connection_lost(self, exc):
231         self.recvbuf = b""
232         self.on_close()
233 
234+    # v2 handshake method
235+    def v2_handshake(self):
236+        """v2 handshake performed before P2P messages are exchanged (see BIP324). P2PConnection is an initiator
237+        (in inbound connections to TestNode) and responder(in outbound connections from TestNode).

230@@ -231,12 +231,66 @@ def connection_lost(self, exc):
231         self.recvbuf = b""
232         self.on_close()
233 
234+    # v2 handshake method
235+    def v2_handshake(self):
236+        """v2 handshake performed before P2P messages are exchanged (see BIP324). P2PConnection is an initiator

238+        Performed by:
239+            * initiator using `initiate_v2_handshake()`, `complete_handshake()` and `authenticate_handshake()`
240+            * responder using `respond_v2_handshake()`, `complete_handshake()` and `authenticate_handshake()`
241+
242+        `initiate_v2_handshake()` is immediately done by the initiator when the connection is established in
243+        `connection_made()`. the rest of the initial v2 handshake functions are handled here.

241+
242+        `initiate_v2_handshake()` is immediately done by the initiator when the connection is established in
243+        `connection_made()`. the rest of the initial v2 handshake functions are handled here.
244+        """
245+        if not self.v2_state.peer:
246+            length = 0

245+        if not self.v2_state.peer:
246+            length = 0
247+            if not self.v2_state.initiating and not self.v2_state.sent_garbage:
248+                # if the responder hasn't sent garbage yet, the responder is still reading ellswift bytes
249+                send_handshake_bytes = self.v2_state.respond_v2_handshake(BytesIO(self.recvbuf))
250+                length = len(self.v2_state.received_prefix)