From 0d31ef4762f5a1428a57439d26551a99f15ddc2e Mon Sep 17 00:00:00 2001 From: John Bampton Date: Fri, 11 May 2018 01:28:27 +1000 Subject: [PATCH] Enable W191 and W291 flake8 checks. Remove trailing whitespace from Python files. Convert tabs to spaces. --- contrib/devtools/copyright_header.py | 4 +- contrib/devtools/lint-python.sh | 4 +- contrib/devtools/security-check.py | 2 +- contrib/devtools/test-security-check.py | 16 +- contrib/linearize/linearize-data.py | 474 ++++++++++----------- contrib/linearize/linearize-hashes.py | 214 +++++----- contrib/seeds/generate-seeds.py | 2 +- test/functional/feature_bip68_sequence.py | 2 +- test/functional/feature_maxuploadtarget.py | 2 +- test/functional/feature_proxy.py | 6 +- test/functional/p2p_feefilter.py | 2 +- test/functional/p2p_segwit.py | 2 +- test/functional/wallet_dump.py | 2 +- 13 files changed, 367 insertions(+), 365 deletions(-) diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index e7cccaab0..82d3c1968 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -506,7 +506,7 @@ def file_has_hashbang(file_lines): def insert_python_header(filename, file_lines, start_year, end_year): if file_has_hashbang(file_lines): - insert_idx = 1 + insert_idx = 1 else: insert_idx = 0 header_lines = get_python_header_lines_to_insert(start_year, end_year) @@ -571,7 +571,7 @@ def insert_cmd(argv): if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: sys.exit("*** cannot insert for file extension %s" % extension) - if extension == '.py': + if extension == '.py': style = 'python' else: style = 'cpp' diff --git a/contrib/devtools/lint-python.sh b/contrib/devtools/lint-python.sh index 239337000..d0cd0a374 100755 --- a/contrib/devtools/lint-python.sh +++ b/contrib/devtools/lint-python.sh @@ -61,6 +61,8 @@ # F823 local variable name … referenced before assignment # F831 duplicate argument name in function definition # F841 local variable 'foo' is assigned to but never used +# W191 indentation contains tabs +# W291 trailing whitespace # W292 no newline at end of file # W293 blank line contains whitespace # W504 line break after binary operator @@ -71,4 +73,4 @@ # W605 invalid escape sequence "x" # W606 'async' and 'await' are reserved keywords starting with Python 3.7 -flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W504,W601,W602,W603,W604,W605,W606 . +flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 . diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 0f2099953..c9516ef83 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -150,7 +150,7 @@ def check_PE_DYNAMIC_BASE(executable): def check_PE_HIGH_ENTROPY_VA(executable): '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': + if arch == 'i386:x86-64': reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA else: # Unnecessary on 32-bit assert(arch == 'i386') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 37a895872..307e05773 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -32,15 +32,15 @@ class TestSecurityChecks(unittest.TestCase): cc = 'gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), (0, '')) def test_32bit_PE(self): @@ -49,11 +49,11 @@ class TestSecurityChecks(unittest.TestCase): cc = 'i686-w64-mingw32-gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, []), + self.assertEqual(call_security_check(cc, source, executable, []), (1, executable+': failed DYNAMIC_BASE NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), (1, executable+': failed DYNAMIC_BASE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), (0, '')) def test_64bit_PE(self): source = 'test1.c' diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index c609e9b33..f8aea2734 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -22,300 +22,300 @@ from binascii import hexlify, unhexlify settings = {} def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() def uint32(x): - return x & 0xffffffff + return x & 0xffffffff def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) + return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | + (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + word = struct.unpack('@I', in_buf[i:i+4])[0] + out_words.append(struct.pack('@I', bytereverse(word))) + return b''.join(out_words) def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + out_words.append(in_buf[i:i+4]) + out_words.reverse() + return b''.join(out_words) def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() + hash1 = hashlib.sha256() + hash1.update(blk_hdr) + hash1_o = hash1.digest() - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() + hash2 = hashlib.sha256() + hash2.update(hash1_o) + hash2_o = hash2.digest() - return hash2_o + return hash2_o def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') - return hash_str + hash = calc_hdr_hash(blk_hdr) + hash = bufreverse(hash) + hash = wordreverse(hash) + hash_str = hexlify(hash).decode('utf-8') + return hash_str def get_blk_dt(blk_hdr): - members = struct.unpack(" self.maxOutSz): - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 + def writeBlock(self, inhdr, blk_hdr, rawblock): + blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) + if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 - (blkDate, blkTS) = get_blk_dt(blk_hdr) - if self.timestampSplit and (blkDate > self.lastDate): - print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) - self.lastDate = blkDate - if self.outF: - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if self.timestampSplit and (blkDate > self.lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + self.lastDate = blkDate + if self.outF: + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 - if not self.outF: - if self.fileOutput: - self.outFname = self.settings['output_file'] - else: - self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) - print("Output file " + self.outFname) - self.outF = open(self.outFname, "wb") + if not self.outF: + if self.fileOutput: + self.outFname = self.settings['output_file'] + else: + self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) + print("Output file " + self.outFname) + self.outF = open(self.outFname, "wb") - self.outF.write(inhdr) - self.outF.write(blk_hdr) - self.outF.write(rawblock) - self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) + self.outF.write(inhdr) + self.outF.write(blk_hdr) + self.outF.write(rawblock) + self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) - self.blkCountOut = self.blkCountOut + 1 - if blkTS > self.highTS: - self.highTS = blkTS + self.blkCountOut = self.blkCountOut + 1 + if blkTS > self.highTS: + self.highTS = blkTS - if (self.blkCountOut % 1000) == 0: - print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % - (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) + if (self.blkCountOut % 1000) == 0: + print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % + (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) - def inFileName(self, fn): - return os.path.join(self.settings['input'], "blk%05d.dat" % fn) + def inFileName(self, fn): + return os.path.join(self.settings['input'], "blk%05d.dat" % fn) - def fetchBlock(self, extent): - '''Fetch block contents from disk given extents''' - with open(self.inFileName(extent.fn), "rb") as f: - f.seek(extent.offset) - return f.read(extent.size) + def fetchBlock(self, extent): + '''Fetch block contents from disk given extents''' + with open(self.inFileName(extent.fn), "rb") as f: + f.seek(extent.offset) + return f.read(extent.size) - def copyOneBlock(self): - '''Find the next block to be written in the input, and copy it to the output.''' - extent = self.blockExtents.pop(self.blkCountOut) - if self.blkCountOut in self.outOfOrderData: - # If the data is cached, use it from memory and remove from the cache - rawblock = self.outOfOrderData.pop(self.blkCountOut) - self.outOfOrderSize -= len(rawblock) - else: # Otherwise look up data on disk - rawblock = self.fetchBlock(extent) + def copyOneBlock(self): + '''Find the next block to be written in the input, and copy it to the output.''' + extent = self.blockExtents.pop(self.blkCountOut) + if self.blkCountOut in self.outOfOrderData: + # If the data is cached, use it from memory and remove from the cache + rawblock = self.outOfOrderData.pop(self.blkCountOut) + self.outOfOrderSize -= len(rawblock) + else: # Otherwise look up data on disk + rawblock = self.fetchBlock(extent) - self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) + self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) - def run(self): - while self.blkCountOut < len(self.blkindex): - if not self.inF: - fname = self.inFileName(self.inFn) - print("Input file " + fname) - try: - self.inF = open(fname, "rb") - except IOError: - print("Premature end of block data") - return + def run(self): + while self.blkCountOut < len(self.blkindex): + if not self.inF: + fname = self.inFileName(self.inFn) + print("Input file " + fname) + try: + self.inF = open(fname, "rb") + except IOError: + print("Premature end of block data") + return - inhdr = self.inF.read(8) - if (not inhdr or (inhdr[0] == "\0")): - self.inF.close() - self.inF = None - self.inFn = self.inFn + 1 - continue + inhdr = self.inF.read(8) + if (not inhdr or (inhdr[0] == "\0")): + self.inF.close() + self.inF = None + self.inFn = self.inFn + 1 + continue - inMagic = inhdr[:4] - if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) - return - inLenLE = inhdr[4:] - su = struct.unpack(" : diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index eee38ce64..6d8d9843e 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -67,7 +67,7 @@ class BIP68Test(BitcoinTestFramework): # If sequence locks were used, this would require 1 block for the # input to mature. sequence_value = SEQUENCE_LOCKTIME_DISABLE_FLAG | 1 - tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] + tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] tx1.vout = [CTxOut(value, CScript([b'a']))] tx1_signed = self.nodes[0].signrawtransactionwithwallet(ToHex(tx1))["hex"] diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 072ba6c7c..0946f27b9 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -100,7 +100,7 @@ class MaxUploadTest(BitcoinTestFramework): assert_equal(p2p_conns[0].block_receive_map[big_old_block], i+1) assert_equal(len(self.nodes[0].getpeerinfo()), 3) - # At most a couple more tries should succeed (depending on how long + # At most a couple more tries should succeed (depending on how long # the test has been running so far). for i in range(3): p2p_conns[0].send_message(getdata_request) diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 60859de7a..2d10c547e 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -79,9 +79,9 @@ class ProxyTest(BitcoinTestFramework): # Note: proxies are not used to connect to local nodes # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost args = [ - ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], - ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], - ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], + ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], + ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], + ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] ] if self.have_ipv6: diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 7c954cdca..c304bbba8 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -69,7 +69,7 @@ class FeeFilterTest(BitcoinTestFramework): # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - sync_mempools(self.nodes) # must be sure node 0 has received all txs + sync_mempools(self.nodes) # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we # we can sync the test on receipt (if node1's txs were relayed, they'd diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 4fecd4ffe..e56d2acfc 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1659,7 +1659,7 @@ class SegWitTest(BitcoinTestFramework): tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program ] total_value += tx.vout[i].nValue - tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ] + tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ] tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE]))) tx2.rehash() diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 997f67ec7..01d27dabe 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -136,7 +136,7 @@ class WalletDumpTest(BitcoinTestFramework): assert_equal(found_addr, test_addr_count) assert_equal(found_script_addr, 2) assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now - assert_equal(found_addr_rsv, 90*2) + assert_equal(found_addr_rsv, 90*2) assert_equal(witness_addr_ret, witness_addr) # Overwriting should fail