Merge #18334: test: Add basic test for BIP 37

fa15699969 test: Add basic test for BIP 37 (MarcoFalke)

Pull request description:

  This does not add full coverage, but should be a good start and can be extended in the future. Currently, none of the BIP 37 p2p code has test coverage.

ACKs for top commit:
  practicalswift:
    Code review ACK fa15699969 -- more testing coverage is better than less testing coverage

Tree-SHA512: d52e8be79240dffb769105c087ae0ae9305d599282546e4ca7379c4c7add2dbcd668265b46670aa07c357638044cf0f61a6fab7dba8971dd0f80c8f99768686e

test/functional/p2p_filter.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+Test BIP 37
+"""
+
+from test_framework.messages import (
+    MSG_BLOCK,
+    MSG_FILTERED_BLOCK,
+    msg_getdata,
+    msg_filterload,
+)
+from test_framework.mininode import (
+    P2PInterface,
+    mininode_lock,
+)
+from test_framework.test_framework import BitcoinTestFramework
+
+
+class FilterNode(P2PInterface):
+    # This is a P2SH watch-only wallet
+    watch_script_pubkey = 'a914ffffffffffffffffffffffffffffffffffffffff87'
+    # The initial filter (n=10, fp=0.000001) with just the above scriptPubKey added
+    watch_filter_init = msg_filterload(
+        data=
+        b'@\x00\x08\x00\x80\x00\x00 \x00\xc0\x00 \x04\x00\x08$\x00\x04\x80\x00\x00 \x00\x00\x00\x00\x80\x00\x00@\x00\x02@ \x00',
+        nHashFuncs=19,
+        nTweak=0,
+        nFlags=1,
+    )
+
+    def on_inv(self, message):
+        want = msg_getdata()
+        for i in message.inv:
+            # inv messages can only contain TX or BLOCK, so translate BLOCK to FILTERED_BLOCK
+            if i.type == MSG_BLOCK:
+                i.type = MSG_FILTERED_BLOCK
+            want.inv.append(i)
+        if len(want.inv):
+            self.send_message(want)
+
+    def on_merkleblock(self, message):
+        self.merkleblock_received = True
+
+    def on_tx(self, message):
+        self.tx_received = True
+
+
+class FilterTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 1
+        self.extra_args = [[
+            '-peerbloomfilters',
+            '-whitelist=noban@127.0.0.1',  # immediate tx relay
+        ]]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        self.log.info('Add filtered P2P connection to the node')
+        filter_node = self.nodes[0].add_p2p_connection(FilterNode())
+        filter_node.send_message(filter_node.watch_filter_init)
+        filter_node.sync_with_ping()
+        filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0]
+
+        self.log.info('Check that we receive merkleblock and tx if the filter matches a tx in a block')
+        filter_node.merkleblock_received = False
+        block_hash = self.nodes[0].generatetoaddress(1, filter_address)[0]
+        txid = self.nodes[0].getblock(block_hash)['tx'][0]
+        filter_node.wait_for_tx(txid)
+        assert filter_node.merkleblock_received
+
+        self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block')
+        with mininode_lock:
+            filter_node.last_message.pop("merkleblock", None)
+        filter_node.tx_received = False
+        self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())
+        filter_node.wait_for_merkleblock()
+        assert not filter_node.tx_received
+
+        self.log.info('Check that we not receive a tx if the filter does not match a mempool tx')
+        filter_node.merkleblock_received = False
+        filter_node.tx_received = False
+        self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 90)
+        filter_node.sync_with_ping()
+        filter_node.sync_with_ping()
+        assert not filter_node.merkleblock_received
+        assert not filter_node.tx_received
+
+        self.log.info('Check that we receive a tx in reply to a mempool msg if the filter matches a mempool tx')
+        filter_node.merkleblock_received = False
+        txid = self.nodes[0].sendtoaddress(filter_address, 90)
+        filter_node.wait_for_tx(txid)
+        assert not filter_node.merkleblock_received
+
+
+if __name__ == '__main__':
+    FilterTest().main()

test/functional/test_framework/messages.py

@@ -51,6 +51,7 @@ NODE_NETWORK_LIMITED = (1 << 10)
 
 MSG_TX = 1
 MSG_BLOCK = 2
+MSG_FILTERED_BLOCK = 3
 MSG_WITNESS_FLAG = 1 << 30
 MSG_TYPE_MASK = 0xffffffff >> 2
 
@@ -225,10 +226,11 @@ class CInv:
 
     typemap = {
         0: "Error",
-        1: "TX",
-        2: "Block",
-        1|MSG_WITNESS_FLAG: "WitnessTx",
-        2|MSG_WITNESS_FLAG : "WitnessBlock",
+        MSG_TX: "TX",
+        MSG_BLOCK: "Block",
+        MSG_TX | MSG_WITNESS_FLAG: "WitnessTx",
+        MSG_BLOCK | MSG_WITNESS_FLAG: "WitnessBlock",
+        MSG_FILTERED_BLOCK: "filtered Block",
         4: "CompactBlock"
     }
 
@@ -1318,6 +1320,41 @@ class msg_headers:
         return "msg_headers(headers=%s)" % repr(self.headers)
 
 
+class msg_merkleblock:
+    command = b"merkleblock"
+
+    def deserialize(self, f):
+        pass  # Placeholder for now
+
+
+class msg_filterload:
+    __slots__ = ("data", "nHashFuncs", "nTweak", "nFlags")
+    command = b"filterload"
+
+    def __init__(self, data=b'00', nHashFuncs=0, nTweak=0, nFlags=0):
+        self.data = data
+        self.nHashFuncs = nHashFuncs
+        self.nTweak = nTweak
+        self.nFlags = nFlags
+
+    def deserialize(self, f):
+        self.data = deser_string(f)
+        self.nHashFuncs = struct.unpack("<I", f.read(4))[0]
+        self.nTweak = struct.unpack("<I", f.read(4))[0]
+        self.nFlags = struct.unpack("<B", f.read(1))[0]
+
+    def serialize(self):
+        r = b""
+        r += ser_string(self.data)
+        r += struct.pack("<I", self.nHashFuncs)
+        r += struct.pack("<I", self.nTweak)
+        r += struct.pack("<B", self.nFlags)
+        return r
+
+    def __repr__(self):
+        return "msg_filterload(data={}, nHashFuncs={}, nTweak={}, nFlags={})".format(
+            self.data, self.nHashFuncs, self.nTweak, self.nFlags)
+
 
 class msg_feefilter:
     __slots__ = ("feerate",)

test/functional/test_framework/mininode.py

@@ -30,6 +30,7 @@ from test_framework.messages import (
     msg_blocktxn,
     msg_cmpctblock,
     msg_feefilter,
+    msg_filterload,
     msg_getaddr,
     msg_getblocks,
     msg_getblocktxn,
@@ -38,6 +39,7 @@ from test_framework.messages import (
     msg_headers,
     msg_inv,
     msg_mempool,
+    msg_merkleblock,
     msg_notfound,
     msg_ping,
     msg_pong,
@@ -62,6 +64,7 @@ MESSAGEMAP = {
     b"blocktxn": msg_blocktxn,
     b"cmpctblock": msg_cmpctblock,
     b"feefilter": msg_feefilter,
+    b"filterload": msg_filterload,
     b"getaddr": msg_getaddr,
     b"getblocks": msg_getblocks,
     b"getblocktxn": msg_getblocktxn,
@@ -70,6 +73,7 @@ MESSAGEMAP = {
     b"headers": msg_headers,
     b"inv": msg_inv,
     b"mempool": msg_mempool,
+    b"merkleblock": msg_merkleblock,
     b"notfound": msg_notfound,
     b"ping": msg_ping,
     b"pong": msg_pong,
@@ -318,6 +322,7 @@ class P2PInterface(P2PConnection):
     def on_blocktxn(self, message): pass
     def on_cmpctblock(self, message): pass
     def on_feefilter(self, message): pass
+    def on_filterload(self, message): pass
     def on_getaddr(self, message): pass
     def on_getblocks(self, message): pass
     def on_getblocktxn(self, message): pass
@@ -325,6 +330,7 @@ class P2PInterface(P2PConnection):
     def on_getheaders(self, message): pass
     def on_headers(self, message): pass
     def on_mempool(self, message): pass
+    def on_merkleblock(self, message): pass
     def on_notfound(self, message): pass
     def on_pong(self, message): pass
     def on_reject(self, message): pass
@@ -385,6 +391,17 @@ class P2PInterface(P2PConnection):
 
         wait_until(test_function, timeout=timeout, lock=mininode_lock)
 
+    def wait_for_merkleblock(self, timeout=60):
+        def test_function():
+            assert self.is_connected
+            last_filtered_block = self.last_message.get('merkleblock')
+            if not last_filtered_block:
+                return False
+            # TODO change this method to take a hash value and only return true if the correct block has been received
+            return True
+
+        wait_until(test_function, timeout=timeout, lock=mininode_lock)
+
     def wait_for_getdata(self, timeout=60):
         """Waits for a getdata message.
 

test/functional/test_runner.py

@@ -147,6 +147,7 @@ BASE_SCRIPTS = [
     'rpc_net.py',
     'wallet_keypool.py',
     'p2p_mempool.py',
+    'p2p_filter.py',
     'rpc_setban.py',
     'p2p_blocksonly.py',
     'mining_prioritisetransaction.py',