dotnet-core/tools/dotnet-bootstrap/dotnet.bootstrap.py
2016-11-28 14:26:51 -08:00

582 lines
25 KiB
Python
Executable file

#!/usr/bin/env python
import os
import json
import platform
import argparse
import sys
import traceback
# for readability
from subprocess import call
from subprocess import check_output
from subprocess import check_call
from subprocess import CalledProcessError
from os import path
from os import makedirs
from os.path import normpath
from string import find
from urllib import urlretrieve
# ROVER BASE #
class RoverMods:
PIPE_TO_STDOUT = not sys.stdout.isatty()
HEADER = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
WHITE = '\033[97m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
ENDC = '\033[0m'
@staticmethod
def Header(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.HEADER + line + RoverMods.ENDC
@staticmethod
def Blue(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.BLUE + line + RoverMods.ENDC
@staticmethod
def Green(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.GREEN + line + RoverMods.ENDC
@staticmethod
def Yellow(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.YELLOW + line + RoverMods.ENDC
@staticmethod
def White(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.WHITE + line + RoverMods.ENDC
@staticmethod
def Red(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.RED + line + RoverMods.ENDC
@staticmethod
def Bold(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.BOLD + line + RoverMods.ENDC
@staticmethod
def Underline(line):
if(RoverMods.PIPE_TO_STDOUT):
return line
return RoverMods.UNDERLINE + line + RoverMods.ENDC
def RoverPrint(line):
print(RoverMods.Bold(RoverMods.Header('** ' + RoverMods.Underline('ROVER'))) + ' ' + (str(line)))
def UnexpectedRoverException(exc_info):
RoverPrint(RoverMods.Red('CAUGHT AN UNEXPECTED EXCEPTION: \"' + RoverMods.White('%s'%(str(exc_info[1]))) + '\" of type: %s'%(str(exc_info[0]))))
RoverPrint(RoverMods.White('%s'%(str(traceback.print_tb(exc_info[2])))))
os._exit(1) # bail out immediately to avoid possibly futzing up the state, or printing unhelpful messages.
# probably a pretty shaky interpretation of the semantic versioning 2.0.0 standard (http://semver.org/)
# I really focused on clauses 9, 10 and 11
class SemanticVersion:
# this is python overloading the '>' operator
def __gt__(self, other):
# Major, minor, and patch versions are always compared numerically.
# Example: 1.0.0 < 2.0.0 < 2.1.0 < 2.1.1.
for index, val in enumerate(self.VersionTuple[0]):
if self.VersionTuple[0][index] > other.VersionTuple[0][index]:
return True
# When major, minor, and patch are equal, a pre-release version has lower precedence than a normal version.
# Example: 1.0.0-alpha < 1.0.0
if(len(self.VersionTuple) < len(other.VersionTuple)):
return True
# Precedence for two pre-release versions with the same major, minor, and patch version MUST be determined
# by comparing each dot separated identifier from left to right until a difference is found as follows:
# identifiers consisting of only digits are compared numerically and identifiers with letters or hyphens are compared
# lexically in ASCII sort order.
# Numeric identifiers always have lower precedence than non-numeric identifiers.
# A larger set of pre-release fields has a higher precedence than a smaller set, if all of the preceding identifiers are equal.
# Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0.
# Maybe a tad hacky - but I treat the remainder of the version (the non numerical piece) as being lex-sorted.
# assuming, of course, that we have two to compare.
if(len(self.VersionTuple) >= 2 and len(self.VersionTuple) >= 2):
if(self.VersionTuple[1] > other.VersionTuple[1]):
return True
return False
def GetVersionTuple(self, versionStr):
# a version string potentially looks like this:
# 0.0.0-alpha-00000
# the first part is canonical: 0.0.0 - it is ordered by the 'ol intuitive manner
# the second part is 'build metadata' - it is ordered lexically
middleIndex = versionStr.find('-')
# if we do not have a middle
if middleIndex == -1:
array = versionStr.split('.')
versionTuple = ([array])
else:
# otherwise, we'll slice in two
versionTuple = (versionStr[0:middleIndex].split('.'), versionStr[middleIndex:len(versionStr)])
return versionTuple
def __str__(self):
return self.VersionString
def __init__(self, versionStr):
self.VersionTuple = self.GetVersionTuple(versionStr)
self.VersionString = versionStr
# END ROVER BASE #
class RoverSettings:
# Setting dev mode to True means that we keep a folder around.
# By Design DevMode is triggered if there is a pre-existing working directory
# By Design, when in DevMode we do not run any git commands.
# By Design, when in DevMode we do not clean up anything.
_DevMode=False
# This function needs to be here, because otherwise we wouldnt be able to change DevMode.
def FetchOSVariables():
try:
os_release_path = '/etc/os-release'
# according to the man page, we should fall back here if the canonical os-release is missing.
if not path.exists(os_release_path):
os_release_path = '/usr/lib/os-release'
os_vars = {}
with open(os_release_path) as f:
for line in f.readlines():
line = line.strip()
if not line: # skip blank lines
continue
data = line.split('=')
os_vars[str(data[0]).strip('\n\"')] = str(data[1]).strip('\n\"')
return os_vars
except IOError:
RoverPrint(RoverMods.Red('requires \'/etc/os-release/\' to exist. For more information, try ' + RoverMods.White('man os-release')))
except:
RoverSettings._DevMode = True # to prevent cleanup (to support investigation)
RoverPrint(RoverMods.Red('CAUGHT AN UNEXPECTED EXCEPTION: \"' + RoverMods.White("%s") + '\" of type: %s'%(str(sys.exc_info()[1]), str(sys.exc_info()[0]))))
RoverPrint(RoverMods.Red(RoverMods.White('%s')%(str(sys.exc_info()[3]))))
_OsVars = FetchOSVariables()
_Rid = '%s.%s-x64'%(_OsVars['ID'], _OsVars['VERSION_ID'])
_Moniker = '%s-dotnet'%(_Rid)
_ScriptDirectory = str(path.dirname(path.abspath(__file__)))
_LaunchedFromDirectory = os.getcwd()
_WorkingDirectory = path.join(_LaunchedFromDirectory, _Moniker)
_srcDirectory = path.join(_WorkingDirectory, "src")
_objDirectory = path.join(_WorkingDirectory, "obj")
_binDirectory = path.join(_WorkingDirectory, "bin")
@staticmethod
def SetWorkingDirectory(working_dir):
RoverSettings._WorkingDirectory = working_dir
RoverSettings._srcDirectory = path.join(working_dir, "src")
RoverSettings._objDirectory = path.join(working_dir, "obj")
RoverSettings._binDirectory = path.join(working_dir, "bin")
PayloadPath = str('')
PatchTargetPath = _binDirectory
BuildSet = []
Patch = True
CoreCLRBinDirectory = ''
CoreFXBinDirectory = ''
CoreSetupBinDirectory = ''
LibUVBinDirectory = ''
PatchTarget_Shared = ''
PatchTarget_SDK = ''
PatchTarget_Host = ''
DotNetCommitHash = ''
@staticmethod
def MaxPrecedence(versionStrA, versionStrB):
versionA = SemanticVersion(versionStrA)
versionB = SemanticVersion(versionStrB)
if(versionA > versionB):
return versionA
return versionB
@staticmethod
def SelectGreatestPrecendenceDirectory(containerDirectory):
maxVersion = '0.0.0-alpha-00000'
for root, dirs, files in os.walk(containerDirectory):
for dirName in dirs:
maxVersion = RoverSettings.MaxPrecedence(dirName, maxVersion)
break # just 'walk' the top level.
return str(maxVersion)
@staticmethod
def SetPatchTargetPath(pathToFolder):
if path.exists(pathToFolder):
RoverSettings.PatchTargetPath = pathToFolder
# require there to be a Microsoft.NETCore.App in the shared
# we will locate the highest version;
shared_containerFolder = path.join(pathToFolder, path.join('shared', 'Microsoft.NETCore.App'))
RoverSettings.PatchTarget_Shared = path.join(shared_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(shared_containerFolder))
# will locate the highest version and patch that
sdk_containerFolder = path.join(pathToFolder, path.join('sdk'))
RoverSettings.PatchTarget_SDK = path.join(sdk_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(sdk_containerFolder))
# require the host to be 'fxr', then we take the highest version.
host_containerFolder = path.join(pathToFolder, path.join('host', 'fxr'))
RoverSettings.PatchTarget_Host = path.join(host_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(host_containerFolder))
if path.exists(RoverSettings._WorkingDirectory):
RoverPrint(RoverMods.Header(RoverMods.Red('FORCED SETTINGS CHANGE: DEV MODE \'ON\' ')))
RoverPrint(RoverMods.Yellow(('will skip all git commands.')))
RoverPrint(RoverMods.Yellow(('requires the deletion of the directory \'%s\' to reset the dev-mode trigger.'%(RoverSettings._WorkingDirectory))))
RoverSettings._DevMode=True
# A 'Rover Shell Call' is a shell call that we want to be reproduceable in the event of a failure.
# namely, something that a developer can go in and 'drill in' on without running the entirety of the
# build again.
def RoverShellCall(cmd, cwd = None):
if not cwd:
cwd = os.getcwd()
try:
check_call(cmd, shell=True, cwd=cwd)
except CalledProcessError as repro_data:
RoverSettings._DevMode = True
repro_filename = 'rover_failure-repro.sh'
repro_destination = path.join(cwd, repro_filename)
# when the call fails, print a repro to the working directory.
with open(repro_destination, 'w') as repro_file:
repro_file.writelines(['#!/usr/bin/env bash\n', repro_data.cmd + '\n'])
if os.getuid() == 0:
call('chmod +x %s'%(repro_filename), shell=True, cwd=cwd)
RoverPrint(RoverMods.Red('has detected a failure. A repro shell script has been placed at ') + RoverMods.Yellow(repro_destination))
RoverPrint(RoverMods.White('To reproduce the failure:\n\tcd %s\n\t./%s'%(cwd, repro_filename)))
RoverPrint(RoverMods.Red('is forcefully closing. Note that re-running Rover will execute it with DevMode enabled (no git commands will be run)'))
os._exit(1) # if we fail a check_call then we want to bail out asap so the dev can investigate.
##
## ROVER FUNCTION DEFINITIONS
##
# detination_folder is expected to be relative to the _ScriptDirectory.
# payload path is expected to be a dotnet-cli tarball.
def SpawnPatchTarget(destination_folder, payload_path):
try:
if payload_path and not path.isabs(payload_path):
payload_path = path.join(RoverSettings._LaunchedFromDirectory, payload_path)
if not path.isabs(destination_folder):
destination_folder = path.join(RoverSettings._LaunchedFromDirectory, destination_folder)
if not path.exists(str(payload_path)):
fallback_url = 'https://dotnetcli.blob.core.windows.net/dotnet/Sdk/rel-1.0.0/dotnet-dev-debian-x64.latest.tar.gz'
payload_filename = 'dotnet.latest.tar.gz'
payload_path = path.join(RoverSettings._objDirectory, payload_filename)
if not path.exists(payload_path):
RoverPrint(RoverMods.Blue('is downloading latest .NET CLI for bootstrapping (%s)'%(payload_filename)))
urlretrieve(fallback_url, payload_path)
# lets force the path to be made absolute - assuming that the payload path is relative to the directory we launched the script from.
# otherwise if we have an abs path already - fantastic.
RoverShellCall('tar xf %s -C %s'%(payload_path, destination_folder))
except:
RoverSettings._DevMode = True
UnexpectedRoverException(sys.exc_info())
RoverSettings.SetPatchTargetPath(path.join(RoverSettings._ScriptDirectory, destination_folder))
def CloneRepositories(cwd,
coreclr_commit_hash,
corefx_commit_hash,
dotnet_commit_hash):
try:
if not path.exists(path.join(cwd, 'coreclr')):
RoverShellCall('git clone http://www.github.com/dotnet/coreclr', cwd=cwd)
RoverShellCall('git checkout %s'%(coreclr_commit_hash), cwd=path.join(cwd, 'coreclr'))
if not path.exists(path.join(cwd, 'corefx')):
RoverShellCall('git clone http://www.github.com/dotnet/corefx', cwd=cwd)
RoverShellCall('git checkout %s'%(corefx_commit_hash), cwd=path.join(cwd, 'corefx'))
if not path.exists(path.join(cwd, 'core-setup')):
RoverShellCall('git clone http://www.github.com/dotnet/core-setup', cwd=cwd)
RoverShellCall('git checkout %s'%(dotnet_commit_hash), cwd=path.join(cwd, 'core-setup'))
if not path.exists(path.join(cwd, 'libuv')):
RoverShellCall('git clone http://www.github.com/libuv/libuv', cwd=cwd)
# we are fixed to using libuv 1.9.0 - this is the commit hash for that (https://github.com/libuv/libuv/commit/229b3a4cc150aebd6561e6bd43076eafa7a03756)
RoverShellCall('git checkout %s'%('229b3a4cc150aebd6561e6bd43076eafa7a03756'), cwd=path.join(cwd, 'libuv'))
else:
RoverPrint(RoverMods.Yellow(('DEVMODE IS ON. Skipping all git calls : I.e. you must manually control git your self.')))
except:
RoverSettings._DevMode = True
UnexpectedRoverException(sys.exc_info())
def BuildNativeComponents( coreclr_git_directory,
corefx_git_directory,
core_setup_git_directory,
libuv_git_directory):
try:
RoverPrint(RoverMods.Blue('is building the .NET GitHub repositories.'))
# Build CoreCLR
# skipping non-essential for bootstrapping.
if 'coreclr' in RoverSettings.BuildSet:
RoverShellCall('./build.sh x64 release skiptests skipnuget', cwd=coreclr_git_directory)
# Build CoreFX Native Pieces
if 'corefx' in RoverSettings.BuildSet:
# at different points in the history of CoreFX there have been differing build behaviors, these conditionals
# cover that. However, if we find these differences, we will need to adapt.s
if path.exists(path.join(corefx_git_directory, 'src', 'Native', 'build-native.sh')):
RoverShellCall('./build-native.sh x64 release Linux --numProc 1', cwd="%s/src/Native"%(corefx_git_directory))
else:
RoverShellCall('./build.sh native x64 release', cwd=corefx_git_directory)
# Build corehost from core-setup
# TODO: Pull versions from the runtimes.
if 'core-setup' in RoverSettings.BuildSet:
RoverShellCall('./build.sh --arch x64 --rid %s --hostver 0.0.0 --fxrver 0.0.0 --policyver 0.0.0 --commithash %s'%(RoverSettings._Rid, RoverSettings.DotNetCommitHash), cwd="%s/src/corehost"%(core_setup_git_directory))
# Build libUV
if 'libuv' in RoverSettings.BuildSet:
RoverShellCall('./autogen.sh', cwd=libuv_git_directory)
RoverShellCall('./configure', cwd=libuv_git_directory)
RoverShellCall('make', cwd=libuv_git_directory)
except:
RoverSettings._DevMode = True
UnexpectedRoverException(sys.exc_info())
def PatchTarget(patchTarget_folder,
coreclr_bin_directory,
corefx_native_bin_directory,
core_setup_cli_bin_directory,
libuv_bin_directory):
try:
if RoverSettings.Patch:
RoverPrint(RoverMods.Blue('is patching %s'%(RoverMods.Yellow(patchTarget_folder))))
# replace native dotnet in the base directory
# from core_setup
RoverShellCall('cp dotnet %s'%(path.join(patchTarget_folder)), cwd='%s/exe/'%(core_setup_cli_bin_directory))
# replace native files in 'shared' folder.
# from coreclr
RoverShellCall('cp *so %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory)
RoverShellCall('cp corerun %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory)
RoverShellCall('cp crossgen %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory)
# from core_setup
RoverShellCall('cp dotnet %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/exe/'%(core_setup_cli_bin_directory))
RoverShellCall('cp libhostpolicy.so %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/dll/'%(core_setup_cli_bin_directory))
RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/fxr/'%(core_setup_cli_bin_directory))
# from corefxcd
RoverShellCall('cp System.* %s'%(RoverSettings.PatchTarget_Shared), cwd=corefx_native_bin_directory)
# from libuv
RoverShellCall('cp libuv.so %s'%(RoverSettings.PatchTarget_Shared), cwd=libuv_bin_directory)
# replace native files in 'sdk' folder.
# from core_setup
RoverShellCall('cp libhostpolicy.so %s'%(RoverSettings.PatchTarget_SDK), cwd='%s/dll/'%(core_setup_cli_bin_directory))
RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_SDK), cwd='%s/fxr/'%(core_setup_cli_bin_directory))
# replace native files in 'host' folder.
# from core_setup
RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_Host), cwd='%s/fxr/'%(core_setup_cli_bin_directory))
RoverPrint(RoverMods.Blue('has finished patching %s'%(RoverMods.Yellow(patchTarget_folder))))
except:
RoverSettings._DevMode = True
UnexpectedRoverException(sys.exc_info())
##
## END ROVER FUNCTION DEFINITIONS
##
if __name__ == "__main__":
##
## COMMAND-LINE BEHAVIOR
##
parser = argparse.ArgumentParser(description = 'This is the .NET CLI bootstrapping tool.')
parser.add_argument('-build', metavar='b', nargs='*', default = ['coreclr', 'corefx', 'core-setup', 'libuv'],help='\'Builds\' all native components if no arguments are specified. Otherwise, specify one or more (space separated) arguments from the following : {'
+ '%s, %s, %s, %s'%(RoverMods.Red('coreclr'), RoverMods.Blue('corefx'), RoverMods.Green('core-setup'), RoverMods.Yellow('libuv') +'}'))
parser.add_argument('-nopatch', action='store_true', default=False, help='prevents the copying of specific native binaries from the pre-built repositories in to the destination directory.')
parser.add_argument('-payload', nargs=1, help='Specify a path to a tarball (something that we can tar xf) that contains a version of the dotnet CLI.')
parser.add_argument('-to', type=str, default='%s'%(RoverSettings._Moniker), help='allows you to overwrite the default staging directory (default is %s)'%(RoverSettings._Moniker))
args = parser.parse_args()
if args.payload:
RoverPrint('is using payload from \'' + RoverMods.White(str(args.payload)) + '\'')
RoverPrint('Building: ' + RoverMods.White(str(args.build)))
RoverPrint('Patching? ' + RoverMods.White(str(not args.nopatch)))
RoverSettings.SetWorkingDirectory(normpath(str(args.to)))
RoverPrint('Staging in %s'%(RoverSettings._WorkingDirectory))
RoverSettings.BuildSet = args.build
# I am guessing that users are more inclined to want patching to happen whenever it can, and so I ask
# for specificity in the instances that they do not want patching.
RoverSettings.Patch = not args.nopatch
if args.payload:
RoverSettings.PayloadPath = args.payload[0]
##
## END COMMAND-LINE BEHAVIOR
##
##
## BEGIN DECLARATIONS
##
coreclr_working_git_directory = path.join(RoverSettings._srcDirectory, 'coreclr')
corefx_working_git_directory = path.join(RoverSettings._srcDirectory, 'corefx')
core_setup_working_git_directory = path.join(RoverSettings._srcDirectory, 'core-setup')
libuv_working_git_directory = path.join(RoverSettings._srcDirectory, 'libuv')
default_coreclr_bin_directory = '%s/bin/Product/Linux.x64.Release/'%(coreclr_working_git_directory)
default_corefx_native_bin_directory = '%s/bin/Linux.x64.Release/Native'%(corefx_working_git_directory)
default_core_setup_cli_bin_directory= '%s/src/corehost/cli'%(core_setup_working_git_directory)
default_libuv_bin_directory = '%s/.libs'%(libuv_working_git_directory)
RoverSettings.CoreCLRBinDirectory = default_coreclr_bin_directory
RoverSettings.CoreFXBinDirectory = default_corefx_native_bin_directory
RoverSettings.CoreSetupBinDirectory = default_core_setup_cli_bin_directory
RoverSettings.LibUVBinDirectory = default_libuv_bin_directory
platform_info = platform.uname()
this_distro_name = str(platform.linux_distribution()[0]).lower()
##
## END DECLARATIONS
##
##
## BEGIN PROCEDURE
##
try:
os.putenv('ID', RoverSettings._OsVars['ID'])
os.putenv('VERSION_ID', RoverSettings._OsVars['VERSION_ID'])
RoverPrint(RoverMods.Blue('RID: %s'%(RoverMods.Green(RoverSettings._Rid))))
# Spawn our working directory
if not path.exists(RoverSettings._WorkingDirectory):
makedirs(RoverSettings._WorkingDirectory)
if not path.exists(RoverSettings._srcDirectory):
makedirs(RoverSettings._srcDirectory)
if not path.exists(RoverSettings._objDirectory):
makedirs(RoverSettings._objDirectory)
if not path.exists(RoverSettings._binDirectory):
makedirs(RoverSettings._binDirectory)
SpawnPatchTarget(RoverSettings._binDirectory, RoverSettings.PayloadPath)
# Fetch the commit hashes from the native files.
coreclr_output = check_output('strings libcoreclr.so | grep @\(#\)', shell = True, cwd=RoverSettings.PatchTarget_Shared)
corefx_output = check_output('strings System.Native.so | grep @\(#\)', shell = True, cwd=RoverSettings.PatchTarget_Shared)
dotnet_commit_hash = check_output('strings dotnet | grep "[a-f0-9]\{40\}"', shell = True, cwd=RoverSettings.PatchTarget_Shared)
coreclr_commit_hash = coreclr_output[find(coreclr_output, 'Commit Hash: ') + len('Commit Hash: '):len(coreclr_output)]
corefx_commit_hash = corefx_output[find(corefx_output, 'Commit Hash: ') + len('Commit Hash: '):len(corefx_output)]
RoverSettings.DotNetCommitHash = dotnet_commit_hash
CloneRepositories(RoverSettings._srcDirectory,
coreclr_commit_hash,
corefx_commit_hash,
dotnet_commit_hash)
BuildNativeComponents(coreclr_working_git_directory,
corefx_working_git_directory,
core_setup_working_git_directory,
libuv_working_git_directory)
PatchTarget(RoverSettings.PatchTargetPath,
RoverSettings.CoreCLRBinDirectory,
RoverSettings.CoreFXBinDirectory,
RoverSettings.CoreSetupBinDirectory,
RoverSettings.LibUVBinDirectory)
RoverPrint(RoverMods.Green('spawned a \'dotnet\' in %s'%(RoverMods.Yellow('./' + path.relpath(RoverSettings.PatchTargetPath) + '/'))) + RoverMods.Green('(enjoy!)'))
except:
RoverSettings._DevMode = True
UnexpectedRoverException(sys.exc_info())
##
## END PROCEDURE
##