Merge #14007: tests: Run functional test on Windows and enable it on Appveyor

661ac15a4a appveyor: Run functional tests on appveyor (Chun Kuan Lee)
2148c36b6e tests: Make it possible to run functional tests on Windows (Chun Kuan Lee)

Pull request description:

  This PR do the following things:
  - Make functional tests compatible with Windows
  - Print color output in functional tests for Windows 10
  - Run util and functional tests on appveyor
  - Do not run symlink tests on Windows

  Note:
  - The wallet_multiwallet.py fail is unrelated to the test framework, it's a bug related to c++ code or maybe dependencies. `bitcoind` would exit with 0xC0000005(Access violation) during shutdown occasionally. Disable this for now.
  - Not using `--failfast` because this is still in experimental. We should track if there is any other error.
  - Disable ZMQ tests because the python zmq library could cause access violation sometimes.
  - Disable `feature_notifications` because Bitcoin Core handles the command in different thread, whicha can cause a race condition.

Tree-SHA512: b76db137d264e62a5c130e1cbca7a2ca002a7a0f4153fa0b92c1ea6c9c09ef0533e11c49bdbd566c472d8ff59f245758feb5e5a6ec6cb6bb66a1c67bab5fa48a
This commit is contained in:
MarcoFalke 2018-09-24 16:10:13 -04:00
commit 990fc0de1a
No known key found for this signature in database
GPG key ID: D2EA4850E7528B25
6 changed files with 74 additions and 27 deletions

View file

@ -7,6 +7,7 @@ environment:
APPVEYOR_SAVE_CACHE_ON_ERROR: true APPVEYOR_SAVE_CACHE_ON_ERROR: true
CLCACHE_SERVER: 1 CLCACHE_SERVER: 1
PACKAGES: boost-filesystem boost-signals2 boost-interprocess boost-test libevent openssl zeromq berkeleydb secp256k1 leveldb PACKAGES: boost-filesystem boost-signals2 boost-interprocess boost-test libevent openssl zeromq berkeleydb secp256k1 leveldb
PYTHONIOENCODING: utf-8
cache: cache:
- C:\tools\vcpkg\installed - C:\tools\vcpkg\installed
- C:\Users\appveyor\clcache - C:\Users\appveyor\clcache
@ -15,6 +16,8 @@ init:
- cmd: set PATH=C:\Python36-x64;C:\Python36-x64\Scripts;%PATH% - cmd: set PATH=C:\Python36-x64;C:\Python36-x64\Scripts;%PATH%
install: install:
- cmd: pip install git+https://github.com/frerich/clcache.git - cmd: pip install git+https://github.com/frerich/clcache.git
# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
# - cmd: pip install zmq
- ps: $packages = $env:PACKAGES -Split ' ' - ps: $packages = $env:PACKAGES -Split ' '
- ps: for ($i=0; $i -lt $packages.length; $i++) { - ps: for ($i=0; $i -lt $packages.length; $i++) {
$env:ALL_PACKAGES += $packages[$i] + ":" + $env:PLATFORM + "-windows-static " $env:ALL_PACKAGES += $packages[$i] + ":" + $env:PLATFORM + "-windows-static "
@ -40,6 +43,17 @@ after_build:
- cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.iobj build_msvc\cache\ - cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.iobj build_msvc\cache\
- cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.ipdb build_msvc\cache\ - cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.ipdb build_msvc\cache\
- cmd: del C:\Users\appveyor\clcache\stats.txt - cmd: del C:\Users\appveyor\clcache\stats.txt
before_test:
- ps: ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in")))
- ps: ${conf_ini} = $conf_ini.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER}).Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER}).Replace("@EXEEXT@", ".exe")
- ps: ${conf_ini} = $conf_ini.Replace("@ENABLE_WALLET_TRUE@", "").Replace("@BUILD_BITCOIN_UTILS_TRUE@", "").Replace("@BUILD_BITCOIND_TRUE@", "").Replace("@ENABLE_ZMQ_TRUE@", "")
- ps: ${utf8} = New-Object System.Text.UTF8Encoding ${false}
- ps: '[IO.File]::WriteAllLines([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini"), $conf_ini, ${utf8})'
- ps: move "build_msvc\${env:PLATFORM}\${env:CONFIGURATION}\*.exe" src
test_script: test_script:
- cmd: build_msvc\%PLATFORM%\%CONFIGURATION%\test_bitcoin.exe - cmd: src\test_bitcoin.exe
- ps: src\bench_bitcoin.exe -evals=1 -scaling=0
- ps: python test\util\bitcoin-util-test.py
- cmd: python test\util\rpcauth-test.py
- cmd: python test\functional\test_runner.py --force --quiet --combinedlogslen=4000 --exclude "feature_notifications,wallet_multiwallet,wallet_multiwallet.py --usecli"
deploy: off deploy: off

View file

@ -25,10 +25,6 @@ def main():
parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2') parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
args, unknown_args = parser.parse_known_args() args, unknown_args = parser.parse_known_args()
if args.color and os.name != 'posix':
print("Color output requires posix terminal colors.")
sys.exit(1)
if args.html and args.color: if args.html and args.color:
print("Only one out of --color or --html should be specified") print("Only one out of --color or --html should be specified")
sys.exit(1) sys.exit(1)

View file

@ -824,7 +824,7 @@ class FullBlockTest(BitcoinTestFramework):
tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0)))
b64a = self.update_block("64a", [tx]) b64a = self.update_block("64a", [tx])
assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8)
self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize(): iostream error') self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize():')
# bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently
# resend the header message, it won't send us the getdata message again. Just # resend the header message, it won't send us the getdata message again. Just

View file

@ -38,6 +38,7 @@ import decimal
import http.client import http.client
import json import json
import logging import logging
import os
import socket import socket
import time import time
import urllib.parse import urllib.parse
@ -71,19 +72,12 @@ class AuthServiceProxy():
self._service_name = service_name self._service_name = service_name
self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests
self.__url = urllib.parse.urlparse(service_url) self.__url = urllib.parse.urlparse(service_url)
port = 80 if self.__url.port is None else self.__url.port
user = None if self.__url.username is None else self.__url.username.encode('utf8') user = None if self.__url.username is None else self.__url.username.encode('utf8')
passwd = None if self.__url.password is None else self.__url.password.encode('utf8') passwd = None if self.__url.password is None else self.__url.password.encode('utf8')
authpair = user + b':' + passwd authpair = user + b':' + passwd
self.__auth_header = b'Basic ' + base64.b64encode(authpair) self.__auth_header = b'Basic ' + base64.b64encode(authpair)
self.timeout = timeout
if connection: self._set_conn(connection)
# Callables re-use the connection of the original proxy
self.__conn = connection
elif self.__url.scheme == 'https':
self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=timeout)
else:
self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=timeout)
def __getattr__(self, name): def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'): if name.startswith('__') and name.endswith('__'):
@ -102,6 +96,10 @@ class AuthServiceProxy():
'User-Agent': USER_AGENT, 'User-Agent': USER_AGENT,
'Authorization': self.__auth_header, 'Authorization': self.__auth_header,
'Content-type': 'application/json'} 'Content-type': 'application/json'}
if os.name == 'nt':
# Windows somehow does not like to re-use connections
# TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows
self._set_conn()
try: try:
self.__conn.request(method, path, postdata, headers) self.__conn.request(method, path, postdata, headers)
return self._get_response() return self._get_response()
@ -178,3 +176,13 @@ class AuthServiceProxy():
def __truediv__(self, relative_uri): def __truediv__(self, relative_uri):
return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn)
def _set_conn(self, connection=None):
port = 80 if self.__url.port is None else self.__url.port
if connection:
self.__conn = connection
self.timeout = connection.timeout
elif self.__url.scheme == 'https':
self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=self.timeout)
else:
self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=self.timeout)

View file

@ -29,7 +29,7 @@ import re
import logging import logging
# Formatting. Default colors to empty strings. # Formatting. Default colors to empty strings.
BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
try: try:
# Make sure python thinks it can write unicode to its stdout # Make sure python thinks it can write unicode to its stdout
"\u2713".encode("utf_8").decode(sys.stdout.encoding) "\u2713".encode("utf_8").decode(sys.stdout.encoding)
@ -41,11 +41,27 @@ except UnicodeDecodeError:
CROSS = "x " CROSS = "x "
CIRCLE = "o " CIRCLE = "o "
if os.name == 'posix': if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393):
if os.name == 'nt':
import ctypes
kernel32 = ctypes.windll.kernel32
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12
# Enable ascii color control to stdout
stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
stdout_mode = ctypes.c_int32()
kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode))
kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
# Enable ascii color control to stderr
stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE)
stderr_mode = ctypes.c_int32()
kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode))
kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
# primitive formatting on supported # primitive formatting on supported
# terminal via ANSI escape sequences: # terminal via ANSI escape sequences:
BOLD = ('\033[0m', '\033[1m') BOLD = ('\033[0m', '\033[1m')
BLUE = ('\033[0m', '\033[0;34m') GREEN = ('\033[0m', '\033[0;32m')
RED = ('\033[0m', '\033[0;31m') RED = ('\033[0m', '\033[0;31m')
GREY = ('\033[0m', '\033[1;30m') GREY = ('\033[0m', '\033[1;30m')
@ -227,6 +243,11 @@ def main():
# Create base test directory # Create base test directory
tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
# If we fixed the command-line and filename encoding issue on Windows, these two lines could be removed
if config["environment"]["EXEEXT"] == ".exe":
tmpdir = "%s/test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
os.makedirs(tmpdir) os.makedirs(tmpdir)
logging.debug("Temporary test directory at %s" % tmpdir) logging.debug("Temporary test directory at %s" % tmpdir)
@ -264,7 +285,7 @@ def main():
# Remove the test cases that the user has explicitly asked to exclude. # Remove the test cases that the user has explicitly asked to exclude.
if args.exclude: if args.exclude:
exclude_tests = [re.sub("\.py$", "", test) + ".py" for test in args.exclude.split(',')] exclude_tests = [re.sub("\.py$", "", test) + (".py" if ".py" not in test else "") for test in args.exclude.split(',')]
for exclude_test in exclude_tests: for exclude_test in exclude_tests:
if exclude_test in test_list: if exclude_test in test_list:
test_list.remove(exclude_test) test_list.remove(exclude_test)
@ -359,7 +380,10 @@ def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=Fal
print('\n============') print('\n============')
print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
print('============\n') print('============\n')
combined_logs, _ = subprocess.Popen([sys.executable, os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate() combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir]
if BOLD[0]:
combined_logs_args += ['--color']
combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate()
print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
if failfast: if failfast:
@ -498,7 +522,7 @@ class TestResult():
def __repr__(self): def __repr__(self):
if self.status == "Passed": if self.status == "Passed":
color = BLUE color = GREEN
glyph = TICK glyph = TICK
elif self.status == "Failed": elif self.status == "Failed":
color = RED color = RED

View file

@ -44,6 +44,7 @@ class MultiWalletTest(BitcoinTestFramework):
# create symlink to verify wallet directory path can be referenced # create symlink to verify wallet directory path can be referenced
# through symlink # through symlink
if os.name != 'nt':
os.mkdir(wallet_dir('w7')) os.mkdir(wallet_dir('w7'))
os.symlink('w7', wallet_dir('w7_symlink')) os.symlink('w7', wallet_dir('w7_symlink'))
@ -66,6 +67,8 @@ class MultiWalletTest(BitcoinTestFramework):
# w8 - to verify existing wallet file is loaded correctly # w8 - to verify existing wallet file is loaded correctly
# '' - to verify default wallet file is created correctly # '' - to verify default wallet file is created correctly
wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', ''] wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', '']
if os.name == 'nt':
wallet_names.remove('w7_symlink')
extra_args = ['-wallet={}'.format(n) for n in wallet_names] extra_args = ['-wallet={}'.format(n) for n in wallet_names]
self.start_node(0, extra_args) self.start_node(0, extra_args)
assert_equal(set(node.listwallets()), set(wallet_names)) assert_equal(set(node.listwallets()), set(wallet_names))
@ -76,7 +79,7 @@ class MultiWalletTest(BitcoinTestFramework):
assert_equal(os.path.isfile(wallet_file(wallet_name)), True) assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
# should not initialize if wallet path can't be created # should not initialize if wallet path can't be created
exp_stderr = "boost::filesystem::create_directory: (The system cannot find the path specified|Not a directory):" exp_stderr = "boost::filesystem::create_directory:"
self.nodes[0].assert_start_raises_init_error(['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.nodes[0].assert_start_raises_init_error(['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist') self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist')
@ -92,6 +95,7 @@ class MultiWalletTest(BitcoinTestFramework):
self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
# should not initialize if wallet file is a symlink # should not initialize if wallet file is a symlink
if os.name != 'nt':
os.symlink('w8', wallet_dir('w8_symlink')) os.symlink('w8', wallet_dir('w8_symlink'))
self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX) self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
@ -220,6 +224,7 @@ class MultiWalletTest(BitcoinTestFramework):
assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
# Fail to load if wallet file is a symlink # Fail to load if wallet file is a symlink
if os.name != 'nt':
assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink')
# Fail to load if a directory is specified that doesn't contain a wallet # Fail to load if a directory is specified that doesn't contain a wallet