2018-07-16 15:01:09 +02:00
#!/usr/bin/env python3
2019-12-07 17:14:40 +01:00
# Copyright (c) 2018-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2018-07-16 15:01:09 +02:00
import argparse
import os
import subprocess
import sys
def setup ( ) :
global args , workdir
2019-05-28 18:51:53 +02:00
programs = [ ' ruby ' , ' git ' , ' make ' , ' wget ' , ' curl ' ]
2018-07-16 15:01:09 +02:00
if args . kvm :
2019-05-18 15:40:39 +02:00
programs + = [ ' apt-cacher-ng ' , ' python-vm-builder ' , ' qemu-kvm ' , ' qemu-utils ' ]
elif args . docker and not os . path . isfile ( ' /lib/systemd/system/docker.service ' ) :
2018-07-19 11:30:47 +02:00
dockers = [ ' docker.io ' , ' docker-ce ' ]
for i in dockers :
return_code = subprocess . call ( [ ' sudo ' , ' apt-get ' , ' install ' , ' -qq ' , i ] )
if return_code == 0 :
break
if return_code != 0 :
2019-05-18 15:40:39 +02:00
print ( ' Cannot find any way to install Docker. ' , file = sys . stderr )
sys . exit ( 1 )
2018-07-16 15:01:09 +02:00
else :
2019-05-18 15:40:39 +02:00
programs + = [ ' apt-cacher-ng ' , ' lxc ' , ' debootstrap ' ]
2018-07-16 15:01:09 +02:00
subprocess . check_call ( [ ' sudo ' , ' apt-get ' , ' install ' , ' -qq ' ] + programs )
if not os . path . isdir ( ' gitian.sigs ' ) :
subprocess . check_call ( [ ' git ' , ' clone ' , ' https://github.com/bitcoin-core/gitian.sigs.git ' ] )
if not os . path . isdir ( ' bitcoin-detached-sigs ' ) :
subprocess . check_call ( [ ' git ' , ' clone ' , ' https://github.com/bitcoin-core/bitcoin-detached-sigs.git ' ] )
if not os . path . isdir ( ' gitian-builder ' ) :
subprocess . check_call ( [ ' git ' , ' clone ' , ' https://github.com/devrandom/gitian-builder.git ' ] )
2018-07-17 21:53:12 +02:00
if not os . path . isdir ( ' bitcoin ' ) :
subprocess . check_call ( [ ' git ' , ' clone ' , ' https://github.com/bitcoin/bitcoin.git ' ] )
2018-07-16 15:01:09 +02:00
os . chdir ( ' gitian-builder ' )
make_image_prog = [ ' bin/make-base-vm ' , ' --suite ' , ' bionic ' , ' --arch ' , ' amd64 ' ]
if args . docker :
make_image_prog + = [ ' --docker ' ]
elif not args . kvm :
make_image_prog + = [ ' --lxc ' ]
subprocess . check_call ( make_image_prog )
os . chdir ( workdir )
2018-07-19 11:30:47 +02:00
if args . is_bionic and not args . kvm and not args . docker :
subprocess . check_call ( [ ' sudo ' , ' sed ' , ' -i ' , ' s/lxcbr0/br0/ ' , ' /etc/default/lxc-net ' ] )
print ( ' Reboot is required ' )
2019-05-18 15:40:39 +02:00
sys . exit ( 0 )
2018-07-16 15:01:09 +02:00
def build ( ) :
global args , workdir
os . makedirs ( ' bitcoin-binaries/ ' + args . version , exist_ok = True )
print ( ' \n Building Dependencies \n ' )
os . chdir ( ' gitian-builder ' )
os . makedirs ( ' inputs ' , exist_ok = True )
2019-12-05 01:30:29 +01:00
subprocess . check_call ( [ ' wget ' , ' -O ' , ' inputs/osslsigncode-2.0.tar.gz ' , ' https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz ' ] )
2019-08-20 08:33:39 +02:00
subprocess . check_call ( [ " echo ' 5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz ' | sha256sum -c " ] , shell = True )
2018-07-16 15:01:09 +02:00
subprocess . check_call ( [ ' make ' , ' -C ' , ' ../bitcoin/depends ' , ' download ' , ' SOURCES_PATH= ' + os . getcwd ( ) + ' /cache/common ' ] )
if args . linux :
print ( ' \n Compiling ' + args . version + ' Linux ' )
subprocess . check_call ( [ ' bin/gbuild ' , ' -j ' , args . jobs , ' -m ' , args . memory , ' --commit ' , ' bitcoin= ' + args . commit , ' --url ' , ' bitcoin= ' + args . url , ' ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ' ] )
subprocess . check_call ( [ ' bin/gsign ' , ' -p ' , args . sign_prog , ' --signer ' , args . signer , ' --release ' , args . version + ' -linux ' , ' --destination ' , ' ../gitian.sigs/ ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ' ] )
subprocess . check_call ( ' mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/ ' + args . version , shell = True )
if args . windows :
print ( ' \n Compiling ' + args . version + ' Windows ' )
subprocess . check_call ( [ ' bin/gbuild ' , ' -j ' , args . jobs , ' -m ' , args . memory , ' --commit ' , ' bitcoin= ' + args . commit , ' --url ' , ' bitcoin= ' + args . url , ' ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ' ] )
subprocess . check_call ( [ ' bin/gsign ' , ' -p ' , args . sign_prog , ' --signer ' , args . signer , ' --release ' , args . version + ' -win-unsigned ' , ' --destination ' , ' ../gitian.sigs/ ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ' ] )
2018-09-26 04:07:33 +02:00
subprocess . check_call ( ' mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/ ' , shell = True )
2019-01-23 21:30:20 +01:00
subprocess . check_call ( ' mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/ ' + args . version , shell = True )
2018-07-16 15:01:09 +02:00
if args . macos :
print ( ' \n Compiling ' + args . version + ' MacOS ' )
subprocess . check_call ( [ ' bin/gbuild ' , ' -j ' , args . jobs , ' -m ' , args . memory , ' --commit ' , ' bitcoin= ' + args . commit , ' --url ' , ' bitcoin= ' + args . url , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ' ] )
subprocess . check_call ( [ ' bin/gsign ' , ' -p ' , args . sign_prog , ' --signer ' , args . signer , ' --release ' , args . version + ' -osx-unsigned ' , ' --destination ' , ' ../gitian.sigs/ ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ' ] )
2018-09-26 04:07:33 +02:00
subprocess . check_call ( ' mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/ ' , shell = True )
2019-01-23 21:30:20 +01:00
subprocess . check_call ( ' mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/ ' + args . version , shell = True )
2018-07-16 15:01:09 +02:00
os . chdir ( workdir )
if args . commit_files :
print ( ' \n Committing ' + args . version + ' Unsigned Sigs \n ' )
os . chdir ( ' gitian.sigs ' )
subprocess . check_call ( [ ' git ' , ' add ' , args . version + ' -linux/ ' + args . signer ] )
subprocess . check_call ( [ ' git ' , ' add ' , args . version + ' -win-unsigned/ ' + args . signer ] )
subprocess . check_call ( [ ' git ' , ' add ' , args . version + ' -osx-unsigned/ ' + args . signer ] )
2018-07-17 21:53:12 +02:00
subprocess . check_call ( [ ' git ' , ' commit ' , ' -m ' , ' Add ' + args . version + ' unsigned sigs for ' + args . signer ] )
2018-07-16 15:01:09 +02:00
os . chdir ( workdir )
def sign ( ) :
global args , workdir
os . chdir ( ' gitian-builder ' )
if args . windows :
print ( ' \n Signing ' + args . version + ' Windows ' )
2018-09-26 04:07:33 +02:00
subprocess . check_call ( ' cp inputs/bitcoin- ' + args . version + ' -win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz ' , shell = True )
2019-04-07 15:16:05 +02:00
subprocess . check_call ( [ ' bin/gbuild ' , ' --skip-image ' , ' --upgrade ' , ' --commit ' , ' signature= ' + args . commit , ' ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml ' ] )
2018-07-16 15:01:09 +02:00
subprocess . check_call ( [ ' bin/gsign ' , ' -p ' , args . sign_prog , ' --signer ' , args . signer , ' --release ' , args . version + ' -win-signed ' , ' --destination ' , ' ../gitian.sigs/ ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml ' ] )
subprocess . check_call ( ' mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/ ' + args . version , shell = True )
if args . macos :
print ( ' \n Signing ' + args . version + ' MacOS ' )
2018-09-26 04:07:33 +02:00
subprocess . check_call ( ' cp inputs/bitcoin- ' + args . version + ' -osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz ' , shell = True )
2019-04-07 15:16:05 +02:00
subprocess . check_call ( [ ' bin/gbuild ' , ' --skip-image ' , ' --upgrade ' , ' --commit ' , ' signature= ' + args . commit , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml ' ] )
2018-07-16 15:01:09 +02:00
subprocess . check_call ( [ ' bin/gsign ' , ' -p ' , args . sign_prog , ' --signer ' , args . signer , ' --release ' , args . version + ' -osx-signed ' , ' --destination ' , ' ../gitian.sigs/ ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml ' ] )
subprocess . check_call ( ' mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/ ' + args . version + ' /bitcoin- ' + args . version + ' -osx.dmg ' , shell = True )
os . chdir ( workdir )
if args . commit_files :
print ( ' \n Committing ' + args . version + ' Signed Sigs \n ' )
os . chdir ( ' gitian.sigs ' )
subprocess . check_call ( [ ' git ' , ' add ' , args . version + ' -win-signed/ ' + args . signer ] )
subprocess . check_call ( [ ' git ' , ' add ' , args . version + ' -osx-signed/ ' + args . signer ] )
subprocess . check_call ( [ ' git ' , ' commit ' , ' -a ' , ' -m ' , ' Add ' + args . version + ' signed binary sigs for ' + args . signer ] )
os . chdir ( workdir )
def verify ( ) :
global args , workdir
2018-10-20 19:58:11 +02:00
rc = 0
2018-07-16 15:01:09 +02:00
os . chdir ( ' gitian-builder ' )
print ( ' \n Verifying v ' + args . version + ' Linux \n ' )
2018-10-20 19:58:11 +02:00
if subprocess . call ( [ ' bin/gverify ' , ' -v ' , ' -d ' , ' ../gitian.sigs/ ' , ' -r ' , args . version + ' -linux ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ' ] ) :
print ( ' Verifying v ' + args . version + ' Linux FAILED \n ' )
rc = 1
2018-07-16 15:01:09 +02:00
print ( ' \n Verifying v ' + args . version + ' Windows \n ' )
2018-10-20 19:58:11 +02:00
if subprocess . call ( [ ' bin/gverify ' , ' -v ' , ' -d ' , ' ../gitian.sigs/ ' , ' -r ' , args . version + ' -win-unsigned ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ' ] ) :
print ( ' Verifying v ' + args . version + ' Windows FAILED \n ' )
rc = 1
2018-07-16 15:01:09 +02:00
print ( ' \n Verifying v ' + args . version + ' MacOS \n ' )
2018-10-20 19:58:11 +02:00
if subprocess . call ( [ ' bin/gverify ' , ' -v ' , ' -d ' , ' ../gitian.sigs/ ' , ' -r ' , args . version + ' -osx-unsigned ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ' ] ) :
print ( ' Verifying v ' + args . version + ' MacOS FAILED \n ' )
rc = 1
2018-07-16 15:01:09 +02:00
print ( ' \n Verifying v ' + args . version + ' Signed Windows \n ' )
2018-10-20 19:58:11 +02:00
if subprocess . call ( [ ' bin/gverify ' , ' -v ' , ' -d ' , ' ../gitian.sigs/ ' , ' -r ' , args . version + ' -win-signed ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml ' ] ) :
print ( ' Verifying v ' + args . version + ' Signed Windows FAILED \n ' )
rc = 1
2018-07-16 15:01:09 +02:00
print ( ' \n Verifying v ' + args . version + ' Signed MacOS \n ' )
2018-10-20 19:58:11 +02:00
if subprocess . call ( [ ' bin/gverify ' , ' -v ' , ' -d ' , ' ../gitian.sigs/ ' , ' -r ' , args . version + ' -osx-signed ' , ' ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml ' ] ) :
print ( ' Verifying v ' + args . version + ' Signed MacOS FAILED \n ' )
rc = 1
2018-07-16 15:01:09 +02:00
os . chdir ( workdir )
2018-10-20 19:58:11 +02:00
return rc
2018-07-16 15:01:09 +02:00
def main ( ) :
global args , workdir
2019-01-23 10:23:24 +01:00
parser = argparse . ArgumentParser ( description = ' Script for running full Gitian builds. ' )
2018-07-16 15:01:09 +02:00
parser . add_argument ( ' -c ' , ' --commit ' , action = ' store_true ' , dest = ' commit ' , help = ' Indicate that the version argument is for a commit or branch ' )
2018-07-26 11:35:52 +02:00
parser . add_argument ( ' -p ' , ' --pull ' , action = ' store_true ' , dest = ' pull ' , help = ' Indicate that the version argument is the number of a github repository pull request ' )
2018-07-16 15:01:09 +02:00
parser . add_argument ( ' -u ' , ' --url ' , dest = ' url ' , default = ' https://github.com/bitcoin/bitcoin ' , help = ' Specify the URL of the repository. Default is %(default)s ' )
parser . add_argument ( ' -v ' , ' --verify ' , action = ' store_true ' , dest = ' verify ' , help = ' Verify the Gitian build ' )
parser . add_argument ( ' -b ' , ' --build ' , action = ' store_true ' , dest = ' build ' , help = ' Do a Gitian build ' )
parser . add_argument ( ' -s ' , ' --sign ' , action = ' store_true ' , dest = ' sign ' , help = ' Make signed binaries for Windows and MacOS ' )
parser . add_argument ( ' -B ' , ' --buildsign ' , action = ' store_true ' , dest = ' buildsign ' , help = ' Build both signed and unsigned binaries ' )
parser . add_argument ( ' -o ' , ' --os ' , dest = ' os ' , default = ' lwm ' , help = ' Specify which Operating Systems the build is for. Default is %(default)s . l for Linux, w for Windows, m for MacOS ' )
parser . add_argument ( ' -j ' , ' --jobs ' , dest = ' jobs ' , default = ' 2 ' , help = ' Number of processes to use. Default %(default)s ' )
parser . add_argument ( ' -m ' , ' --memory ' , dest = ' memory ' , default = ' 2000 ' , help = ' Memory to allocate in MiB. Default %(default)s ' )
parser . add_argument ( ' -k ' , ' --kvm ' , action = ' store_true ' , dest = ' kvm ' , help = ' Use KVM instead of LXC ' )
parser . add_argument ( ' -d ' , ' --docker ' , action = ' store_true ' , dest = ' docker ' , help = ' Use Docker instead of LXC ' )
2019-05-18 16:29:35 +02:00
parser . add_argument ( ' -S ' , ' --setup ' , action = ' store_true ' , dest = ' setup ' , help = ' Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian) ' )
2018-07-16 15:01:09 +02:00
parser . add_argument ( ' -D ' , ' --detach-sign ' , action = ' store_true ' , dest = ' detach_sign ' , help = ' Create the assert file for detached signing. Will not commit anything. ' )
parser . add_argument ( ' -n ' , ' --no-commit ' , action = ' store_false ' , dest = ' commit_files ' , help = ' Do not commit anything to git ' )
2019-01-23 10:23:24 +01:00
parser . add_argument ( ' signer ' , nargs = ' ? ' , help = ' GPG signer to sign each build assert file ' )
parser . add_argument ( ' version ' , nargs = ' ? ' , help = ' Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified ' )
2018-07-16 15:01:09 +02:00
args = parser . parse_args ( )
workdir = os . getcwd ( )
2018-07-19 11:30:47 +02:00
args . is_bionic = b ' bionic ' in subprocess . check_output ( [ ' lsb_release ' , ' -cs ' ] )
2018-07-16 15:01:09 +02:00
if args . kvm and args . docker :
raise Exception ( ' Error: cannot have both kvm and docker ' )
2019-05-18 16:29:35 +02:00
# Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they
# can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm).
os . environ [ ' USE_LXC ' ] = ' '
os . environ [ ' USE_VBOX ' ] = ' '
os . environ [ ' USE_DOCKER ' ] = ' '
2018-07-16 15:01:09 +02:00
if args . docker :
os . environ [ ' USE_DOCKER ' ] = ' 1 '
elif not args . kvm :
os . environ [ ' USE_LXC ' ] = ' 1 '
2019-05-18 16:29:35 +02:00
if ' GITIAN_HOST_IP ' not in os . environ . keys ( ) :
2018-07-19 11:30:47 +02:00
os . environ [ ' GITIAN_HOST_IP ' ] = ' 10.0.3.1 '
2019-05-18 16:29:35 +02:00
if ' LXC_GUEST_IP ' not in os . environ . keys ( ) :
2018-07-19 11:30:47 +02:00
os . environ [ ' LXC_GUEST_IP ' ] = ' 10.0.3.5 '
2018-07-16 15:01:09 +02:00
2019-01-23 10:23:24 +01:00
if args . setup :
setup ( )
if args . buildsign :
args . build = True
args . sign = True
if not args . build and not args . sign and not args . verify :
sys . exit ( 0 )
args . linux = ' l ' in args . os
args . windows = ' w ' in args . os
args . macos = ' m ' in args . os
2018-07-16 15:01:09 +02:00
# Disable for MacOS if no SDK found
if args . macos and not os . path . isfile ( ' gitian-builder/inputs/MacOSX10.11.sdk.tar.gz ' ) :
print ( ' Cannot build for MacOS, SDK does not exist. Will build for other OSes ' )
args . macos = False
2019-01-23 10:23:24 +01:00
args . sign_prog = ' true ' if args . detach_sign else ' gpg --detach-sign '
2018-07-16 15:01:09 +02:00
script_name = os . path . basename ( sys . argv [ 0 ] )
2019-01-23 10:23:24 +01:00
if not args . signer :
print ( script_name + ' : Missing signer ' )
2018-07-16 15:01:09 +02:00
print ( ' Try ' + script_name + ' --help for more information ' )
2019-05-18 15:40:39 +02:00
sys . exit ( 1 )
2019-01-23 10:23:24 +01:00
if not args . version :
print ( script_name + ' : Missing version ' )
2018-07-16 15:01:09 +02:00
print ( ' Try ' + script_name + ' --help for more information ' )
2019-05-18 15:40:39 +02:00
sys . exit ( 1 )
2018-07-16 15:01:09 +02:00
# Add leading 'v' for tags
2018-07-26 11:35:52 +02:00
if args . commit and args . pull :
raise Exception ( ' Cannot have both commit and pull ' )
2018-07-16 15:01:09 +02:00
args . commit = ( ' ' if args . commit else ' v ' ) + args . version
os . chdir ( ' bitcoin ' )
2018-07-26 11:35:52 +02:00
if args . pull :
subprocess . check_call ( [ ' git ' , ' fetch ' , args . url , ' refs/pull/ ' + args . version + ' /merge ' ] )
os . chdir ( ' ../gitian-builder/inputs/bitcoin ' )
subprocess . check_call ( [ ' git ' , ' fetch ' , args . url , ' refs/pull/ ' + args . version + ' /merge ' ] )
2018-09-02 00:40:56 +02:00
args . commit = subprocess . check_output ( [ ' git ' , ' show ' , ' -s ' , ' --format= % H ' , ' FETCH_HEAD ' ] , universal_newlines = True , encoding = ' utf8 ' ) . strip ( )
2018-07-26 11:35:52 +02:00
args . version = ' pull- ' + args . version
print ( args . commit )
2018-07-16 15:01:09 +02:00
subprocess . check_call ( [ ' git ' , ' fetch ' ] )
subprocess . check_call ( [ ' git ' , ' checkout ' , args . commit ] )
os . chdir ( workdir )
2019-04-21 08:52:48 +02:00
os . chdir ( ' gitian-builder ' )
subprocess . check_call ( [ ' git ' , ' pull ' ] )
os . chdir ( workdir )
2018-07-16 15:01:09 +02:00
if args . build :
build ( )
if args . sign :
sign ( )
if args . verify :
2019-04-21 08:52:48 +02:00
os . chdir ( ' gitian.sigs ' )
subprocess . check_call ( [ ' git ' , ' pull ' ] )
os . chdir ( workdir )
2018-10-20 19:58:11 +02:00
sys . exit ( verify ( ) )
2018-07-16 15:01:09 +02:00
if __name__ == ' __main__ ' :
main ( )