diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..a1e1e97a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,63 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary + +############################################################################### +# behavior for image files +# +# image files are treated as binary by default. +############################################################################### +#*.jpg binary +#*.png binary +#*.gif binary + +############################################################################### +# diff behavior for common document formats +# +# Convert binary document formats to text before diffing them. This feature +# is only available from the command line. Turn it on by uncommenting the +# entries below. +############################################################################### +#*.doc diff=astextplain +#*.DOC diff=astextplain +#*.docx diff=astextplain +#*.DOCX diff=astextplain +#*.dot diff=astextplain +#*.DOT diff=astextplain +#*.pdf diff=astextplain +#*.PDF diff=astextplain +#*.rtf diff=astextplain +#*.RTF diff=astextplain \ No newline at end of file diff --git a/.gitignore b/.gitignore index d38d4c72..ba9ed9d4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin/ obj/ project.lock.json .DS_Store +*.pyc \ No newline at end of file diff --git a/netci.groovy b/netci.groovy index 426cb02e..c830db4c 100644 --- a/netci.groovy +++ b/netci.groovy @@ -16,9 +16,6 @@ def branch = GithubBranchName // is a boolean indicating whether the job will be a PR job. If true, the // suffix _prtest will be appended. def newJobName = Utilities.getFullJobName(project, configuration, isPR) - - // Define build string - def buildString = """echo Configuration: ${configuration} isPR: ${isPR}""" // Create a new job with the specified name. The brace opens a new closure // and calls made within that closure apply to the newly created job. @@ -28,7 +25,10 @@ def branch = GithubBranchName // Indicates that a batch script should be run with the build string (see above) // Also available is: // shell (for unix scripting) - shell(buildString) + shell("ls") + + shell("python tools/dotnet-bootstrap/base/lab/containers.py bake") + shell("python tools/dotnet-bootstrap/base/lab/cases.py run") } } diff --git a/tools/dotnet-bootstrap/README.md b/tools/dotnet-bootstrap/README.md new file mode 100644 index 00000000..fdc94a1d --- /dev/null +++ b/tools/dotnet-bootstrap/README.md @@ -0,0 +1,85 @@ +DOTNET.BOOTSTRAP(1) +### NAME +.NET CLI Bootstrapping Tool - A tool to help you bootstrap the .NET Command Line Tool on unsupported platforms. + +### SYNOPSIS +python dotnet.bootstrap.py [-b __build_set__] [-nopatch] [-payload __tarball_path__] + +### DESCRIPTION +dotnet.bootstrap.py is the .NET CLI bootstrapping script (written for Python 2.7) that intends to help developers move to new platforms and "bring up" the required pieces. + +There are default settings that intend to hit the 'most cases' scenario. If all is going to plan, you should never need to specify any additional parameters. However +many situations arise where it becomes a necessity or a nice-to-have. Namely, when things go wrong, you want to be able to get in there and fix +up the build, make changes, etc. This is 'development mode' or DevMode for short. In DevMode, NO git commands are executed (ONE EXCEPTION: if the expected repo directories (coreclr, corefx, core-setup, libuv) do not exist, +we will clone them in). This is to prevent the script from stomping out any changes you have made in the working directory. Additionally, when things do go wrong (inevitably they will), this tool places a shell/batch +script within the working directory that contains the command line that failed. This is to enable the scenario where you want to 'drill into' a problem. + +### EXAMPLES +Intended use, + +``` +./dotnet.bootstrap.py +``` +This will spawn a directory next to the bootstrap script, named after its runtime identifier (the runtime identifier is currently picked from the /os/release and we concatenate the +VERSION and VERSION_ID values). So on an AMD64 Ubuntu 16.04 machine, the RID is ubuntu.16.04-x64-dotnet + +Any additional runs can be controlled via these command lines, and consequently, any additional runs are going to be in DevMode. + +``` +./dotnet.bootstrap.py -b corefx libuv +``` +This will build only the corefx binaries and coreclr binaries, then patch the files. + +If you want to prevent patching (for example, just to re-run a build): + +``` +./dotnet.bootstrap.py -b corefx coreclr -nopatch +``` + +Additionally, if you have a tarball of the files that you'd like to produce, consider: + +``` +./dotnet.bootstrap.py -b corefx coreclr -nopatch -payload ~/downloads/dotnet-dev-build.tar.gz +``` + +### DEFAULTS + +By default, running ./dotnet.bootstrap.py with no additional parameters is equivalent to this command line: +``` +./dotnet.bootstrap.py -b corefx coreclr libuv core-setup -payload __dotnet_cli_repository_url__ +``` + +### OPTIONS +*-b __build_set__* + +    __build_set__ is the space-delimited set of repositories to place. At the moment it is one of these {coreclr, corefx, core-setup, libuv}. + +*-nopatch* + +    As part of the bootstrapping process, we "patch" (overwrite/replace native binaries) a pre-built version of the CLI. + +*-payload __tar_filepath__* + +    By default dotnet.bootstrap will pull in a pre-built tar file from the CLI repository and patch this. If you want to provide your own binaries to patch + (from dev builds or something), then specify a path (relative or absolute). + +### OVERVIEW +After you run the dotnet.bootstrap, you'll see a directory named after the Runtime Identifier (RID) next to the script, the directory tree looks like this (for example), + +``` + +├── bin +│   ├── dotnet +│   ├── host +│   ├── LICENSE.txt +│   ├── sdk +│   ├── shared +│   └── ThirdPartyNotices.txt +├── obj +│   └── dotnet-dev-debian-x64.latest.tar.gz +└── src + ├── coreclr + ├── corefx + ├── core-setup + └── libuv +``` \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/README.md b/tools/dotnet-bootstrap/base/README.md new file mode 100644 index 00000000..8a208a9b --- /dev/null +++ b/tools/dotnet-bootstrap/base/README.md @@ -0,0 +1,6 @@ +# BASE + +## Lab +Go to the lab for testing information. + + diff --git a/tools/dotnet-bootstrap/base/lab/.gitignore b/tools/dotnet-bootstrap/base/lab/.gitignore new file mode 100644 index 00000000..16d3fd20 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/.gitignore @@ -0,0 +1,9 @@ +**/bin/* +**/src/* +**/obj/* +**/testing/* +dotnet.bootstrap.py +*~ +*-dotnet +shellcall_failure-repro.sh +*.deb \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/README.md b/tools/dotnet-bootstrap/base/lab/README.md new file mode 100644 index 00000000..fd38b4cc --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/README.md @@ -0,0 +1,31 @@ +# LAB + +## Test Cases +Intended interface for access is the class `Cases` in `cases.py`. + +Listing all test cases, +``` +./cases.py list +``` + +Running all test cases in all containers, +``` +./cases.py run +``` + +Running specific testcase in all containers, +``` +./cases.py run e2e +``` + +Running specific testcase in specific container, +``` +./cases.py run e2e -in debian8 +``` + +## Environments +Docker containers are used as the 'unit of environment.' + +``` +./containers.py list +``` \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/cases.py b/tools/dotnet-bootstrap/base/lab/cases.py new file mode 100755 index 00000000..3e6f74bc --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/cases.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +import os +import sys + +from shellcall import ShellCall +from shellcall import ContinueOnError +from sys import argv +from os import getcwd + +from os.path import join, exists, dirname, realpath + +from globals import g_override + +# interface + data binding for managing the testcases. +class Cases: + _labPath = dirname(realpath(__file__)) + _supported_containers = join(_labPath, 'containers/') # our 'list' of current supported platforms are the directories in this directory + _testcases = join(_labPath, 'cases/') + _continueOnError = True + _lenient = True + + + # if current_working_directory = None, then we use the working dir dictated by the dockerfile + # if none is specified in the dockerfile, then docker uses '/' + def _docker_compose(self, identifier, local_volume, current_working_directory = None): + wdir_parameter = None + + if current_working_directory: + wdir_parameter = '-w "%s"'%(current_working_directory) + + return 'docker run %s -v %s:/env/dotnet-bootstrap dotnet-bootstrap:%s'%(wdir_parameter, local_volume, identifier) + + # Runs a select case + def RunIn(self, container_name, casename): + local_mount_location = join(self._supported_containers, container_name) + testing_destination = join(local_mount_location, "testing/") + + ShellCall("echo \"running 'dotnet-bootstrap:%s - testcase: %s'\""%(container_name, casename), lenient = self._lenient) + + # copy the bootstrap and test source in to the container working directory (next to the Dockerfile) + ShellCall('cp %s %s'%(join(self._labPath, '../../dotnet.bootstrap.py'), join(self._supported_containers, container_name)), lenient = self._lenient) + ShellCall('mkdir -p %s'%(join(testing_destination, casename)), lenient=self._lenient) + ShellCall('cp -R %s %s'%(join(self._testcases, casename), join(testing_destination, casename)), lenient = self._lenient) + + docker_run_cmd = 'docker run -v %s:/env/dotnet-bootstrap dotnet-bootstrap:%s'%(local_mount_location, str(container_name)) + # ^ : This runs docker using the current container directory (with the Dockerfile) as the current working directory. + # so that anything placed in that directory becomes accessible. + # eventually we will copy the tests in to this directory as well (see below) + + # run the bootstrap + ShellCall('%s python /env/dotnet-bootstrap/dotnet.bootstrap.py -to /env/dotnet-bootstrap/'%(docker_run_cmd), lenient = self._lenient) # this will generate the src, obj, and bin directory here. + + # create whatever project file is the latest and greatest (was project.json, and is now named after the directory.csproj) + ShellCall('%s /env/dotnet-bootstrap/bin/dotnet new -t Console'%(self._docker_compose(container_name, local_mount_location, join("/env/dotnet-bootstrap/testing/", casename))), lenient= self._lenient) + #ShellCall('ls', cwd=join(testing_destination, casename, casename + '.csproj')) + + # confirm that it exists. + if exists(join(testing_destination, casename, casename + '.csproj')): + ShellCall('mkdir -p %s'%join(testing_destination, casename, "result")) + ShellCall('touch %s'%(join(testing_destination, casename, "result", "pass"))) # spawn a result; a failure is when this doesn't exist. If this exists, this is a passing testcase. + + ShellCall('cp -R %s/* %s'%(join(self._testcases, casename), join(testing_destination, casename)), lenient= self._lenient) + # ShellCall('%s /env/dotnet-bootstrap/bin/dotnet restore .'%(self._docker_compose(container_name, local_mount_location, join("/env/dotnet-bootstrap/testing/", casename))), lenient=self._lenient) + # ShellCall('%s /env/dotnet-bootstrap/bin/dotnet run'%(self._docker_compose(container_name, local_mount_location, join("/env/dotnet-bootstrap/testing/", casename))), lenient=self._lenient) + + self.Report() + + def _runOverride(self): + for container in g_override["containers"]: + for case in g_override["cases"]: + try: + self.RunIn(container, case) + except ContinueOnError: # we threw this up with the intention of being OK with moving on. + continue + + def _status(self, container, case): + print("CONTAINER - CASE - STATUS") + target = "%s - %s"%(container, case) + testing_destination = join(self._supported_containers, container, "testing/") + if exists(join(testing_destination, case, "result", "pass")): + print("%s - pass"%(target)) + else: + print("%s - fail"%(target)) + + + def Report(self): + for container in g_override["containers"]: + for case in g_override["cases"]: + self._status(container, case) + + # runs the full matrix of tests + def RunAll(self): + if g_override: + self._runOverride() + return + + for root, containers, files in os.walk(self._supported_containers): + for container in containers: # we keep it explicitly the case that there are no other directories in the cases or containers directories. + for root, cases, files in os.walk(self._testcases): + for case in cases: + try: + self.RunIn(container, case) # runs the full matrix of environments and cases + except ContinueOnError: + continue + break # just walk the top level + break # just walk the top level. + + def List(self): + ShellCall('ls -1 %s'%(self._testcases), lenient = self._lenient) + + + def __init__(self): + if not exists(self._supported_containers): + print('no such directory: %s\n'%(self._supported_containers)) + sys.exit() + + if not exists(self._testcases): + print('no such directory: %s\n'%(self._testcases)) + sys.exit() + +def PrintUsage(): + print("TODO: Usage") + +if __name__ == '__main__': + testcases = Cases() + + if len(argv) <= 1: + PrintUsage() + exit() + + dictionary = { + "run": testcases.RunAll, + "list": testcases.List, + "report": testcases.Report + } + + dictionary[argv[1]]() diff --git a/tools/dotnet-bootstrap/base/lab/cases/e2e/Program.cs b/tools/dotnet-bootstrap/base/lab/cases/e2e/Program.cs new file mode 100644 index 00000000..37cf47ff --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/cases/e2e/Program.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; + +public static class Program +{ + public static int Main(string[] args) + { + // Draw a dotnet bot. + Bot(QuoteOfTheDay()); + + return 42; // test pass code + } + + public static string QuoteOfTheDay() + { + var quotes = File.ReadAllLines("quotes.txt"); + + return quotes[new Random().Next(0, quotes.Length - 1)]; + } + + public static void Bot(string message) + { + var _tempForeground = Console.ForegroundColor; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(message); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(@" + __________________ + \\ + \\"); + Console.ForegroundColor = ConsoleColor.Magenta; + Console.WriteLine(@" + .... + ....' + .... + .......... + .............'..'.. + ................'..'..... + .......'..........'..'..'.... + ........'..........'..'..'..... + .'....'..'..........'..'.......'. + .'..................'... ...... + . ......'......... ..... + . ...... + .. . .. ...... + .... . ....... + ...... ....... ............ + ................ ...................... + ........................'................ + ......................'..'...... ....... + .........................'..'..... ....... + ........ ..'.............'..'.... .......... + ..'..'... ...............'....... .......... + ...'...... ...... .......... ...... ....... + ........... ....... ........ ...... + ....... '...'.'. '.'.'.' .... + ....... .....'.. ..'..... + .. .......... ..'........ + ............ .............. + ............. '.............. + ...........'.. .'.'............ + ............... .'.'............. + .............'.. ..'..'........... + ............... .'.............. + ......... .............. + ....."); + + + + Console.ForegroundColor = _tempForeground; + } +} \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/cases/e2e/quotes.txt b/tools/dotnet-bootstrap/base/lab/cases/e2e/quotes.txt new file mode 100644 index 00000000..1fc1274f --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/cases/e2e/quotes.txt @@ -0,0 +1,59 @@ +We have a strange illusion that mere time cancels sin. But mere time does nothing either to the fact or to the guilt of a sin. +There can be no deep disappointment where there is not deep love. +My therapist told me the way to achieve true inner peace is to finish what I start. So far today, I have finished 2 bags of M&M`s and a chocolate cake. I feel better already. +Half a truth is often a great lie. +The survival of liberty in our land increasingly depends on the success of liberty in other lands. +The Constitution is not an instrument for the government to restrain the people, it is an instrument for the people to restrain the government -- lest it come to dominate our lives and interests. +From the prodigious hilltops of New Hampshire, let freedom ring. From the mighty mountains of New York, let freedom ring. From the heightening Alleghenies of Pennsylvania, let freedom ring. But not only that: Let freedom ring from every hill and molehill of Mississippi. +Without music, life would be a mistake. +Prosperity is the measure or touchstone of virtue, for it is less difficult to bear misfortune than to remain uncorrupted by pleasure. +The more you read about politics, you got to admit that each party is worse than the other. +Just the omission of Jane Austen`s books alone would make a fairly good library out of a library that hadn`t a book in it. +An unexciting truth may be eclipsed by a thrilling lie. +Beware how you take away hope from another human being. +Marriage is a duel to the death which no man of honour should decline. +If liberty and equality, as is thought by some, are chiefly to be found in democracy, they will be best attained when all persons alike share in government to the utmost. +Ignore death up to the last moment; then, when it can`t be ignored any longer, have yourself squirted full of morphia and shuffle off in a coma. Thoroughly sensible, humane and scientific, eh? +A good novel tells us the truth about its hero; but a bad novel tells us the truth about its author. +I have a dream that my four little children will one day live in a nation where they will not be judged by the color of their skin, but by the content of their character. +Take from the church the miraculous, the supernatural, the incomprehensible, the unreasonable, the impossible, the unknowable, the absurd, and nothing but a vacuum remains. +Power over a man`s subsistence is power over his will. +The puritanism of Christianity has played havoc with the moderation that an enlightened and tolerant critical spirit would have produced. I`ve noticed that in whatever country, county, town, or other region there is a regulation enjoining temperance, the population seems to be entirely composed of teetotallers and drunkards. There`s a Bible on that shelf there. But I keep it next to Voltaire - poison and antidote. +Never pick a fight with people who buy ink by the barrel. +There can be no liberty unless there is economic liberty. +Do not dwell in the past, do not dream of the future, concentrate the mind on the present moment. +If there were no God, there would be no Atheists. +It is impossible to travel faster than the speed of light, and certainly not desirable, as one`s hat keeps blowing off. +God is really only another artist. He invented the giraffe, the elephant and the cat. He has no real style, He just goes on trying other things. +Saturate yourself with your subject and the camera will all but take you by the hand. +Familiarity breeds contempt - and children. +Heaven have mercy on us all--Presbyterians and Pagans alike--for we are all dreadfully cracked about the head and desperately in need of mending. +The difference between Los Angeles and yogurt is that yogurt comes with less fruit. +I`m afraid that if you look at a thing long enough, it loses all of its meaning. +The difference between sex and death is that with death you can do it alone and no one is going to make fun of you. +Money buys you everything except the chance to do it again. +Never worry about the size of your Christmas tree. In the eyes of children, they are all thirty feet tall. +I think, at a child`s birth, if a mother could ask a fairy godmother to endow it with the most useful gift, that gift should be curiosity. +An unexamined life is not worth living. +It`s a phonetic language. Anything can make sense. How do you think Dr. Seuss wrote any of that sh*t? +Make everything as simple as possible, but not simpler. +There are lots of people who mistake their imagination for their memory. +We are what we repeatedly do. Excellence, therefore, is not an act, but a habit. +Wise men argue causes, and fools decide them. +A pint of sweat, saves a gallon of blood. +It ain`t the parts of the Bible that I can`t understand that bother me, it is the parts that I do understand. +A `geek` by definition is someone who eats live animals....I`ve never eaten live animals. +No virtuous act is quite as virtuous from the standpoint of our friend or foe, as from our own. Therefore, we are saved by the final form of love which is forgiveness. +The ladder of success is best climbed by stepping on the rungs of opportunity. +My best friend is the man who in wishing me well wishes it for my sake. +The best way to win an argument is to begin by being right. +Christmas is the gentlest, loveliest festival of the revolving year. And yet, for all that, when it speaks, its voice has strong authority. +Whenever science makes a discovery, the devil grabs it while the angels are debating the best way to use it. +Don`t you wish you had a job like mine? All you have to do is think up a certain number of words! Plus, you can repeat words! And they don`t even have to be true! +When angry, count ten before you speak; if very angry, a hundred. +People have a way of becoming what you encourage them to be, not what you nag them to be. +A man never tells you anything until you contradict him. +There are very few monsters who warrant the fear we have of them. +Perspective is worth 80 IQ Points - Alan Kay +The best way to predict the future is to invent it. - Alan Kay +The Internet was done so well that most people think of it as a natural resource like the Pacific Ocean, rather than something that was man-made. When was the last time a technology with a scale like that was so error-free? The Web, in comparison, is a joke. The Web was done by amateurs. - Alan Kay \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers.py b/tools/dotnet-bootstrap/base/lab/containers.py new file mode 100755 index 00000000..7b52a276 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +# This shell script 'bakes' the docker containers that we depend on. Essentially, it sets up the docker containers in a certain manner where we +# are enabled to run a test case within them. + +# specify a single directory name to bake just a single container + +from shellcall import ShellCall +from shellcall import ContinueOnError +import sys +import os + +from sys import argv +from os.path import join +from os.path import dirname +from os.path import realpath +from os import getcwd + +from globals import g_override + +# interface + data binding for managing the containers. +class Containers: + _supported_platforms = join(dirname(realpath(__file__)), 'containers') + '/' # our 'list' of current supported platforms are the directories in this directory + + def Bake(self, selected_platform): + ShellCall("echo baking 'dotnet-bootstrap:%s'"%(selected_platform), lenient=True) + ShellCall("docker build -t \"dotnet-bootstrap:%s\" ."%(selected_platform), join(self._supported_platforms, selected_platform), lenient=True) + + def CleanContainerFolder(self, container, folderName): + ShellCall("rm -R -f %s"%(join(self._supported_platforms, container, folderName)), lenient=True) + + def CleanAll(self): + for root, platforms, files in os.walk(self._supported_platforms): + for platform in platforms: + self.CleanContainerFolder(platform, "src") + self.CleanContainerFolder(platform, "obj") + self.CleanContainerFolder(platform, "bin") + self.CleanContainerFolder(platform, "testing") + + break + + def _bakeOverride(self): + for container in g_override["containers"]: + try: + self.Bake(container) + except ContinueOnError: + continue + + + def BakeAll(self): + if g_override: + self._bakeOverride() + return + + for root, platforms, files in os.walk(self._supported_platforms): + for platform in platforms: # we keep it explicitly the case that there are no other directories in the cases or containers directories. + try: + self.Bake(platform) + except ContinueOnError: + continue + break + + def List(self): + ShellCall('ls -1 %s'%(self._supported_platforms)) + + +def PrintUsage(): + print("TODO: Usage") + +if __name__ == '__main__': + containers = Containers() + + dictionary = { + "bake": containers.BakeAll, + "list": containers.List, + "clean": containers.CleanAll + } + + if len(argv) <= 1: + PrintUsage() + exit() + + dictionary[argv[1]]() + diff --git a/tools/dotnet-bootstrap/base/lab/containers/centos71/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/centos71/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/centos71/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/centos71/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/centos71/Dockerfile new file mode 100644 index 00000000..e53c26fa --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/centos71/Dockerfile @@ -0,0 +1,26 @@ +FROM centos:7.1.1503 +MAINTAINER Bryan P. Arant +RUN yum -q -y install deltarpm +RUN yum -q -y install epel-release +RUN yum -q -y update +RUN yum -q -y install unzip libunwind8 libunwind8-dev gettext libcurl-devel openssl-devel zlib libicu-devel liblttng-ust lttng-ust-devel libunwind-devel libuuid-devel automake autoconf libtool +RUN yum -q -y install binutils +RUN yum install -y http://mirror.symnds.com/distributions/gf/el/7/plus/x86_64/cmake-3.3.2-1.gf.el7.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/clang-3.6.2-1.el7.centos.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/clang-libs-3.6.2-1.el7.centos.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/lldb-3.6.2-1.el7.centos.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/lldb-devel-3.6.2-1.el7.centos.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/llvm-3.6.2-1.el7.centos.x86_64.rpm \ + https://matell.blob.core.windows.net/rpms/llvm-libs-3.6.2-1.el7.centos.x86_64.rpm \ + which \ + make + +RUN yum -q -y install tar git + +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 +RUN update-alternatives --set c++ /usr/bin/clang++ + +RUN yum -q -y install sudo + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap diff --git a/tools/dotnet-bootstrap/base/lab/containers/debian8/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/debian8/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/debian8/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/debian8/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/debian8/Dockerfile new file mode 100644 index 00000000..677618f0 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/debian8/Dockerfile @@ -0,0 +1,73 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM debian:8.0 + +# Misc Dependencies for build +RUN apt-get update && \ + apt-get -qqy install \ + llvm-3.5 \ + lldb-3.5 \ + lldb-3.5-dev \ + curl \ + uuid-dev \ + unzip \ + gettext \ + sudo && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# This could become a "microsoft/coreclr" image, since it just installs the dependencies for CoreCLR (and stdlib) +RUN apt-get update &&\ + apt-get -qqy install \ + libunwind8 \ + libunwind8-dev \ + libkrb5-3 \ + libicu52 \ + libicu-dev \ + liblttng-ust0 \ + liblttng-ust-dev \ + libssl1.0.0 \ + zlib1g \ + libssl-dev \ + zlib1g-dev \ + libuuid1 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install Build Prereqs +RUN apt-get update && \ + apt-get -qqy install \ + autogen \ + binutils \ + debhelper \ + build-essential \ + devscripts \ + git \ + automake \ + cmake \ + libkrb5-dev \ + libtool \ + libcurl4-openssl-dev \ + clang-3.5 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +# Set working directory +WORKDIR /opt/code + +# Work around https://github.com/dotnet/cli/issues/1582 until Docker releases a +# fix (https://github.com/docker/docker/issues/20818). This workaround allows +# the container to be run with the default seccomp Docker settings by avoiding +# the restart_syscall made by LTTng which causes a failed assertion. +ENV LTTNG_UST_REGISTER_TIMEOUT 0 + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/fedora23/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/fedora23/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/fedora23/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/fedora23/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/fedora23/Dockerfile new file mode 100644 index 00000000..1ee44a50 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/fedora23/Dockerfile @@ -0,0 +1,47 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM fedora:23 + +# Install the base toolchain we need to build anything (clang, cmake, make and the like) +# this does not include libraries that we need to compile different projects, we'd like +# them in a different layer. +RUN dnf install -y cmake \ + clang \ + lldb-devel \ + make \ + which && \ + dnf clean all + +# Install tools used by the VSO build automation. +RUN dnf install -y git \ + zip \ + tar \ + nodejs \ + findutils \ + npm && \ + dnf clean all && \ + npm install -g azure-cli && \ + npm cache clean + +# Dependencies of CoreCLR and CoreFX. +RUN dnf install -y libicu-devel \ + libuuid-devel \ + libtool \ + libcurl-devel \ + openssl-devel \ + libunwind-devel \ + lttng-ust-devel && \ + dnf clean all + +# Upgrade NSS, used for SSL, to avoid NuGet restore timeouts. +RUN dnf upgrade -y nss +RUN dnf clean all + + +# Set working directory +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap diff --git a/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/Dockerfile new file mode 100644 index 00000000..e2bccc5f --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/opensuse13.2/Dockerfile @@ -0,0 +1,52 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +FROM opensuse:13.2 + +# Install the base toolchain we need to build anything (clang, cmake, make and the like) +# this does not include libraries that we need to compile different projects, we'd like +# them in a different layer. +RUN zypper -n update + +RUN zypper -n install binutils \ + python \ + cmake \ + which \ + gcc \ + libtool \ + python-xml \ + libicu-devel \ + lldb \ + lldb-devel \ + libunwind-devel \ + libuuid-devel \ + llvm-clang \ + tar \ + ncurses-utils \ + curl \ + git \ + sudo && \ + ln -s /usr/bin/clang++ /usr/bin/clang++-3.5 && \ + zypper clean -a + +# Dependencies of CoreCLR and CoreFX. We need to add an additional repository to get lttng. +RUN zypper addrepo -G http://download.opensuse.org/repositories/devel:/tools:/lttng/openSUSE_13.2/devel:tools:lttng.repo && \ + zypper refresh && \ + zypper -n remove krb5-mini && \ + zypper -n install libunwind \ + libicu \ + krb5-devel \ + lttng-ust-devel \ + libuuid1 \ + libopenssl-devel \ + libopenssl1_0_0 \ + libcurl-devel && \ + zypper clean -a + +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 +RUN update-alternatives --set c++ /usr/bin/clang++ + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/Dockerfile new file mode 100644 index 00000000..08e64d8b --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/opensuse42.1/Dockerfile @@ -0,0 +1,56 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +FROM opensuse:42.1 + +# Install the base toolchain we need to build anything (clang, cmake, make and the like) +# this does not include libraries that we need to compile different projects, we'd like +# them in a different layer. +RUN zypper -n update + +RUN zypper -n install binutils \ + python \ + cmake \ + which \ + gcc \ + libtool \ + python-xml \ + libicu-devel \ + lldb \ + lldb-devel \ + libunwind-devel \ + libuuid-devel \ + llvm-clang \ + tar \ + ncurses-utils \ + curl \ + git \ + sudo && \ + ln -s /usr/bin/clang++ /usr/bin/clang++-3.5 && \ + zypper clean -a + +# Dependencies of CoreCLR and CoreFX. We need to add an additional repository to get lttng. +RUN zypper addrepo -G http://download.opensuse.org/repositories/devel:/tools:/lttng/openSUSE_13.2/devel:tools:lttng.repo +RUN whereis zypper +RUN zypper refresh +RUN whereis zypper +RUN whereis zypper +RUN zypper -n -q install libunwind +RUN whereis zypper +# we have to force zypper to resolve this dependency because it, +# itself, depends on krb5-mini which needs to be uninstalled (possibly +# uninstalling zypper) +RUN zypper -n -q install --force-resolution krb5-devel +RUN zypper -n -q install lttng-ust-devel +RUN zypper -n -q install libuuid1 +RUN zypper -n -q install libopenssl-devel +RUN zypper -n -q install libopenssl1_0_0 +RUN zypper -n -q install libcurl-devel + +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 +RUN update-alternatives --set c++ /usr/bin/clang++ + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/Dockerfile new file mode 100644 index 00000000..acb94466 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1404/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:14.04 +MAINTAINER Bryan P. Arant +RUN apt-get -qqy update +RUN apt-get -qqy install libkrb5-dev build-essential libtool git binutils python automake cmake llvm-3.5 clang-3.5 lldb-3.6 lldb-3.6-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libcurl4-openssl-dev libssl-dev uuid-dev + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/Dockerfile new file mode 100644 index 00000000..2cdf723d --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1604/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:16.04 +MAINTAINER Bryan P. Arant +RUN apt-get -qqy update +RUN apt-get -qqy install libkrb5-dev build-essential libtool git binutils python automake cmake llvm-3.5 clang-3.5 lldb-3.6 lldb-3.6-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libcurl4-openssl-dev libssl-dev uuid-dev + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/.dockerignore b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/.dockerignore new file mode 100644 index 00000000..6d1fc225 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/.dockerignore @@ -0,0 +1,5 @@ +src/* +bin/* +obj/* +testing/* +dotnet.bootstrap.py \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/Dockerfile b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/Dockerfile new file mode 100644 index 00000000..3d037797 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/containers/ubuntu1610/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:16.10 +MAINTAINER Bryan P. Arant +RUN apt-get -qqy update +RUN apt-get -qqy install libkrb5-dev build-essential libtool git binutils python automake cmake llvm-3.5 clang-3.5 lldb-3.6 lldb-3.6-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libcurl4-openssl-dev libssl-dev uuid-dev + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.5 100 +RUN update-alternatives --set c++ /usr/bin/clang++-3.5 + +VOLUME /env/dotnet-bootstrap +WORKDIR /env/dotnet-bootstrap diff --git a/tools/dotnet-bootstrap/base/lab/dockercleanup.py b/tools/dotnet-bootstrap/base/lab/dockercleanup.py new file mode 100755 index 00000000..8eeea300 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/dockercleanup.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + + +from shellcall import ShellCall + +# Delete all containers +ShellCall("docker rm $(docker ps -a -q)") +# Delete all images +ShellCall("docker rmi $(docker images -q)") \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/globals.py b/tools/dotnet-bootstrap/base/lab/globals.py new file mode 100755 index 00000000..717ab257 --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/globals.py @@ -0,0 +1,8 @@ +g_override = { + "containers": [ + "ubuntu1404" + ], + "cases" : [ + "e2e" + ] +} \ No newline at end of file diff --git a/tools/dotnet-bootstrap/base/lab/shellcall.py b/tools/dotnet-bootstrap/base/lab/shellcall.py new file mode 100755 index 00000000..e561db8c --- /dev/null +++ b/tools/dotnet-bootstrap/base/lab/shellcall.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# A 'Shell Call' is a shell call (duh) that we want to be reproduceable in the event of a failure. +# namely, something that a developer can go in and 'drill in' on without running the entirety of the +# build again. + +import os +from subprocess import call +from subprocess import check_call +from subprocess import CalledProcessError +from os import path + +class ContinueOnError(Exception): + def __init__(self, working_directory, reprofile): + self.working_directory = working_directory + self.reprofile = reprofile + + def __str__(self): + return '%s - %s'%(working_directory, reprofile) + +def ShellCall(cmd, cwd = None, lenient=False): + if not cwd: + cwd = os.getcwd() + + try: + check_call(cmd, shell=True, cwd=cwd) + + except CalledProcessError as repro_data: + repro_filename = 'shellcall_failure-repro.sh' + repro_destination = path.join(cwd, repro_filename) + + # when the call fails, print a repro to the working directory. + with open(repro_destination, 'w') as repro_file: + repro_file.writelines(['#!/usr/bin/env bash\n', repro_data.cmd + '\n']) + + # if we're rooted + if os.getuid() == 0: + call('chmod +x %s'%(repro_filename), shell=True, cwd=cwd) + + # prints "Rover has detected a failure" + print("a reproduction script was placed at : %s"%(repro_destination)) + print("To reproduce the failure:\n\tcd %s\n\t./%s"%(cwd, repro_filename)) + + # meh, lets just try to keep building everything. + if lenient: + raise ContinueOnError(cwd, repro_filename) # if we're feeling lenient, then we will raise up this opportunity to continue. + + os._exit(1) # if we fail a check_call then we want to bail out asap so the dev can investigate. \ No newline at end of file diff --git a/tools/dotnet-bootstrap/dotnet.bootstrap.py b/tools/dotnet-bootstrap/dotnet.bootstrap.py new file mode 100755 index 00000000..92749bc0 --- /dev/null +++ b/tools/dotnet-bootstrap/dotnet.bootstrap.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python + +import os +import json +import platform +import argparse +import sys +import traceback + +# for readability +from subprocess import call +from subprocess import check_output +from subprocess import check_call +from subprocess import CalledProcessError + +from os import path +from os import makedirs +from os.path import normpath + +from string import find +from urllib import urlretrieve + + +# ROVER BASE # +class RoverMods: + PIPE_TO_STDOUT = not sys.stdout.isatty() + + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + WHITE = '\033[97m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + ENDC = '\033[0m' + + @staticmethod + def Header(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.HEADER + line + RoverMods.ENDC + + @staticmethod + def Blue(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.BLUE + line + RoverMods.ENDC + + @staticmethod + def Green(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.GREEN + line + RoverMods.ENDC + + @staticmethod + def Yellow(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.YELLOW + line + RoverMods.ENDC + + @staticmethod + def White(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.WHITE + line + RoverMods.ENDC + + @staticmethod + def Red(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.RED + line + RoverMods.ENDC + + @staticmethod + def Bold(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.BOLD + line + RoverMods.ENDC + + @staticmethod + def Underline(line): + if(RoverMods.PIPE_TO_STDOUT): + return line + + return RoverMods.UNDERLINE + line + RoverMods.ENDC + +def RoverPrint(line): + print(RoverMods.Bold(RoverMods.Header('** ' + RoverMods.Underline('ROVER'))) + ' ' + (str(line))) + +def UnexpectedRoverException(exc_info): + RoverPrint(RoverMods.Red('CAUGHT AN UNEXPECTED EXCEPTION: \"' + RoverMods.White('%s'%(str(exc_info[1]))) + '\" of type: %s'%(str(exc_info[0])))) + RoverPrint(RoverMods.White('%s'%(str(traceback.print_tb(exc_info[2]))))) + os._exit(1) # bail out immediately to avoid possibly futzing up the state, or printing unhelpful messages. + + +# probably a pretty shaky interpretation of the semantic versioning 2.0.0 standard (http://semver.org/) +# I really focused on clauses 9, 10 and 11 +class SemanticVersion: + # this is python overloading the '>' operator + def __gt__(self, other): + # Major, minor, and patch versions are always compared numerically. + # Example: 1.0.0 < 2.0.0 < 2.1.0 < 2.1.1. + for index, val in enumerate(self.VersionTuple[0]): + if self.VersionTuple[0][index] > other.VersionTuple[0][index]: + return True + + # When major, minor, and patch are equal, a pre-release version has lower precedence than a normal version. + # Example: 1.0.0-alpha < 1.0.0 + if(len(self.VersionTuple) < len(other.VersionTuple)): + return True + + # Precedence for two pre-release versions with the same major, minor, and patch version MUST be determined + # by comparing each dot separated identifier from left to right until a difference is found as follows: + # identifiers consisting of only digits are compared numerically and identifiers with letters or hyphens are compared + # lexically in ASCII sort order. + + # Numeric identifiers always have lower precedence than non-numeric identifiers. + # A larger set of pre-release fields has a higher precedence than a smaller set, if all of the preceding identifiers are equal. + # Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0. + + # Maybe a tad hacky - but I treat the remainder of the version (the non numerical piece) as being lex-sorted. + # assuming, of course, that we have two to compare. + if(len(self.VersionTuple) >= 2 and len(self.VersionTuple) >= 2): + if(self.VersionTuple[1] > other.VersionTuple[1]): + return True + + return False + + def GetVersionTuple(self, versionStr): + # a version string potentially looks like this: + # 0.0.0-alpha-00000 + # the first part is canonical: 0.0.0 - it is ordered by the 'ol intuitive manner + # the second part is 'build metadata' - it is ordered lexically + middleIndex = versionStr.find('-') + # if we do not have a middle + if middleIndex == -1: + array = versionStr.split('.') + versionTuple = ([array]) + else: + # otherwise, we'll slice in two + versionTuple = (versionStr[0:middleIndex].split('.'), versionStr[middleIndex:len(versionStr)]) + + return versionTuple + + def __str__(self): + return self.VersionString + + def __init__(self, versionStr): + self.VersionTuple = self.GetVersionTuple(versionStr) + self.VersionString = versionStr + +# END ROVER BASE # + +class RoverSettings: + # Setting dev mode to True means that we keep a folder around. + # By Design DevMode is triggered if there is a pre-existing working directory + # By Design, when in DevMode we do not run any git commands. + # By Design, when in DevMode we do not clean up anything. + _DevMode=False + + # This function needs to be here, because otherwise we wouldnt be able to change DevMode. + def FetchOSVariables(): + try: + os_release_path = '/etc/os-release' + + # according to the man page, we should fall back here if the canonical os-release is missing. + if not path.exists(os_release_path): + os_release_path = '/usr/lib/os-release' + + os_vars = {} + with open(os_release_path) as f: + for line in f.readlines(): + line = line.strip() + + if not line: # skip blank lines + continue + + data = line.split('=') + os_vars[str(data[0]).strip('\n\"')] = str(data[1]).strip('\n\"') + + return os_vars + except IOError: + RoverPrint(RoverMods.Red('requires \'/etc/os-release/\' to exist. For more information, try ' + RoverMods.White('man os-release'))) + except: + RoverSettings._DevMode = True # to prevent cleanup (to support investigation) + RoverPrint(RoverMods.Red('CAUGHT AN UNEXPECTED EXCEPTION: \"' + RoverMods.White("%s") + '\" of type: %s'%(str(sys.exc_info()[1]), str(sys.exc_info()[0])))) + RoverPrint(RoverMods.Red(RoverMods.White('%s')%(str(sys.exc_info()[3])))) + + _OsVars = FetchOSVariables() + _Rid = '%s.%s-x64'%(_OsVars['ID'], _OsVars['VERSION_ID']) + _Moniker = '%s-dotnet'%(_Rid) + _ScriptDirectory = str(path.dirname(path.abspath(__file__))) + _LaunchedFromDirectory = os.getcwd() + + _WorkingDirectory = path.join(_LaunchedFromDirectory, _Moniker) + _srcDirectory = path.join(_WorkingDirectory, "src") + _objDirectory = path.join(_WorkingDirectory, "obj") + _binDirectory = path.join(_WorkingDirectory, "bin") + + @staticmethod + def SetWorkingDirectory(working_dir): + RoverSettings._WorkingDirectory = working_dir + RoverSettings._srcDirectory = path.join(working_dir, "src") + RoverSettings._objDirectory = path.join(working_dir, "obj") + RoverSettings._binDirectory = path.join(working_dir, "bin") + + + PayloadPath = str('') + PatchTargetPath = _binDirectory + BuildSet = [] + Patch = True + + CoreCLRBinDirectory = '' + CoreFXBinDirectory = '' + CoreSetupBinDirectory = '' + LibUVBinDirectory = '' + + PatchTarget_Shared = '' + PatchTarget_SDK = '' + PatchTarget_Host = '' + + DotNetCommitHash = '' + + @staticmethod + def MaxPrecedence(versionStrA, versionStrB): + versionA = SemanticVersion(versionStrA) + versionB = SemanticVersion(versionStrB) + + if(versionA > versionB): + return versionA + + return versionB + + + @staticmethod + def SelectGreatestPrecendenceDirectory(containerDirectory): + maxVersion = '0.0.0-alpha-00000' + + for root, dirs, files in os.walk(containerDirectory): + for dirName in dirs: + maxVersion = RoverSettings.MaxPrecedence(dirName, maxVersion) + + break # just 'walk' the top level. + + return str(maxVersion) + + @staticmethod + def SetPatchTargetPath(pathToFolder): + if path.exists(pathToFolder): + RoverSettings.PatchTargetPath = pathToFolder + # require there to be a Microsoft.NETCore.App in the shared + # we will locate the highest version; + shared_containerFolder = path.join(pathToFolder, path.join('shared', 'Microsoft.NETCore.App')) + RoverSettings.PatchTarget_Shared = path.join(shared_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(shared_containerFolder)) + + # will locate the highest version and patch that + sdk_containerFolder = path.join(pathToFolder, path.join('sdk')) + RoverSettings.PatchTarget_SDK = path.join(sdk_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(sdk_containerFolder)) + + # require the host to be 'fxr', then we take the highest version. + host_containerFolder = path.join(pathToFolder, path.join('host', 'fxr')) + RoverSettings.PatchTarget_Host = path.join(host_containerFolder, RoverSettings.SelectGreatestPrecendenceDirectory(host_containerFolder)) + + +if path.exists(RoverSettings._WorkingDirectory): + RoverPrint(RoverMods.Header(RoverMods.Red('FORCED SETTINGS CHANGE: DEV MODE \'ON\' '))) + RoverPrint(RoverMods.Yellow(('will skip all git commands.'))) + RoverPrint(RoverMods.Yellow(('requires the deletion of the directory \'%s\' to reset the dev-mode trigger.'%(RoverSettings._WorkingDirectory)))) + + RoverSettings._DevMode=True + +# A 'Rover Shell Call' is a shell call that we want to be reproduceable in the event of a failure. +# namely, something that a developer can go in and 'drill in' on without running the entirety of the +# build again. +def RoverShellCall(cmd, cwd = None): + if not cwd: + cwd = os.getcwd() + + try: + check_call(cmd, shell=True, cwd=cwd) + + except CalledProcessError as repro_data: + RoverSettings._DevMode = True + + repro_filename = 'rover_failure-repro.sh' + repro_destination = path.join(cwd, repro_filename) + + # when the call fails, print a repro to the working directory. + with open(repro_destination, 'w') as repro_file: + repro_file.writelines(['#!/usr/bin/env bash\n', repro_data.cmd + '\n']) + + if os.getuid() == 0: + call('chmod +x %s'%(repro_filename), shell=True, cwd=cwd) + + RoverPrint(RoverMods.Red('has detected a failure. A repro shell script has been placed at ') + RoverMods.Yellow(repro_destination)) + RoverPrint(RoverMods.White('To reproduce the failure:\n\tcd %s\n\t./%s'%(cwd, repro_filename))) + RoverPrint(RoverMods.Red('is forcefully closing. Note that re-running Rover will execute it with DevMode enabled (no git commands will be run)')) + + os._exit(1) # if we fail a check_call then we want to bail out asap so the dev can investigate. + + +## +## ROVER FUNCTION DEFINITIONS +## + +# detination_folder is expected to be relative to the _ScriptDirectory. +# payload path is expected to be a dotnet-cli tarball. +def SpawnPatchTarget(destination_folder, payload_path): + try: + if payload_path and not path.isabs(payload_path): + payload_path = path.join(RoverSettings._LaunchedFromDirectory, payload_path) + + if not path.isabs(destination_folder): + destination_folder = path.join(RoverSettings._LaunchedFromDirectory, destination_folder) + + if not path.exists(str(payload_path)): + fallback_url = 'https://dotnetcli.blob.core.windows.net/dotnet/Sdk/rel-1.0.0/dotnet-dev-debian-x64.latest.tar.gz' + + payload_filename = 'dotnet.latest.tar.gz' + payload_path = path.join(RoverSettings._objDirectory, payload_filename) + + if not path.exists(payload_path): + RoverPrint(RoverMods.Blue('is downloading latest .NET CLI for bootstrapping (%s)'%(payload_filename))) + + urlretrieve(fallback_url, payload_path) + + # lets force the path to be made absolute - assuming that the payload path is relative to the directory we launched the script from. + # otherwise if we have an abs path already - fantastic. + + RoverShellCall('tar xf %s -C %s'%(payload_path, destination_folder)) + except: + RoverSettings._DevMode = True + UnexpectedRoverException(sys.exc_info()) + + RoverSettings.SetPatchTargetPath(path.join(RoverSettings._ScriptDirectory, destination_folder)) + +def CloneRepositories(cwd, + coreclr_commit_hash, + corefx_commit_hash, + dotnet_commit_hash): + try: + if not path.exists(path.join(cwd, 'coreclr')): + RoverShellCall('git clone http://www.github.com/dotnet/coreclr', cwd=cwd) + RoverShellCall('git checkout %s'%(coreclr_commit_hash), cwd=path.join(cwd, 'coreclr')) + + if not path.exists(path.join(cwd, 'corefx')): + RoverShellCall('git clone http://www.github.com/dotnet/corefx', cwd=cwd) + RoverShellCall('git checkout %s'%(corefx_commit_hash), cwd=path.join(cwd, 'corefx')) + + if not path.exists(path.join(cwd, 'core-setup')): + RoverShellCall('git clone http://www.github.com/dotnet/core-setup', cwd=cwd) + RoverShellCall('git checkout %s'%(dotnet_commit_hash), cwd=path.join(cwd, 'core-setup')) + + if not path.exists(path.join(cwd, 'libuv')): + RoverShellCall('git clone http://www.github.com/libuv/libuv', cwd=cwd) + # we are fixed to using libuv 1.9.0 - this is the commit hash for that (https://github.com/libuv/libuv/commit/229b3a4cc150aebd6561e6bd43076eafa7a03756) + RoverShellCall('git checkout %s'%('229b3a4cc150aebd6561e6bd43076eafa7a03756'), cwd=path.join(cwd, 'libuv')) + + else: + RoverPrint(RoverMods.Yellow(('DEVMODE IS ON. Skipping all git calls : I.e. you must manually control git your self.'))) + + except: + RoverSettings._DevMode = True + UnexpectedRoverException(sys.exc_info()) + + + +def BuildNativeComponents( coreclr_git_directory, + corefx_git_directory, + core_setup_git_directory, + libuv_git_directory): + try: + RoverPrint(RoverMods.Blue('is building the .NET GitHub repositories.')) + + # Build CoreCLR + # skipping non-essential for bootstrapping. + if 'coreclr' in RoverSettings.BuildSet: + RoverShellCall('./build.sh x64 release skiptests skipnuget', cwd=coreclr_git_directory) + + # Build CoreFX Native Pieces + if 'corefx' in RoverSettings.BuildSet: + # at different points in the history of CoreFX there have been differing build behaviors, these conditionals + # cover that. However, if we find these differences, we will need to adapt.s + if path.exists(path.join(corefx_git_directory, 'src', 'Native', 'build-native.sh')): + RoverShellCall('./build-native.sh x64 release Linux --numProc 1', cwd="%s/src/Native"%(corefx_git_directory)) + else: + RoverShellCall('./build.sh native x64 release', cwd=corefx_git_directory) + + # Build corehost from core-setup + # TODO: Pull versions from the runtimes. + if 'core-setup' in RoverSettings.BuildSet: + RoverShellCall('./build.sh --arch x64 --rid %s --hostver 0.0.0 --fxrver 0.0.0 --policyver 0.0.0 --commithash %s'%(RoverSettings._Rid, RoverSettings.DotNetCommitHash), cwd="%s/src/corehost"%(core_setup_git_directory)) + + # Build libUV + if 'libuv' in RoverSettings.BuildSet: + RoverShellCall('./autogen.sh', cwd=libuv_git_directory) + RoverShellCall('./configure', cwd=libuv_git_directory) + RoverShellCall('make', cwd=libuv_git_directory) + + except: + RoverSettings._DevMode = True + UnexpectedRoverException(sys.exc_info()) + + +def PatchTarget(patchTarget_folder, + coreclr_bin_directory, + corefx_native_bin_directory, + core_setup_cli_bin_directory, + libuv_bin_directory): + try: + if RoverSettings.Patch: + RoverPrint(RoverMods.Blue('is patching %s'%(RoverMods.Yellow(patchTarget_folder)))) + + # replace native dotnet in the base directory + # from core_setup + RoverShellCall('cp dotnet %s'%(path.join(patchTarget_folder)), cwd='%s/exe/'%(core_setup_cli_bin_directory)) + + # replace native files in 'shared' folder. + # from coreclr + RoverShellCall('cp *so %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory) + RoverShellCall('cp corerun %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory) + RoverShellCall('cp crossgen %s'%(RoverSettings.PatchTarget_Shared), cwd=coreclr_bin_directory) + + # from core_setup + RoverShellCall('cp dotnet %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/exe/'%(core_setup_cli_bin_directory)) + RoverShellCall('cp libhostpolicy.so %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/dll/'%(core_setup_cli_bin_directory)) + RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_Shared), cwd='%s/fxr/'%(core_setup_cli_bin_directory)) + + # from corefxcd + RoverShellCall('cp System.* %s'%(RoverSettings.PatchTarget_Shared), cwd=corefx_native_bin_directory) + + # from libuv + RoverShellCall('cp libuv.so %s'%(RoverSettings.PatchTarget_Shared), cwd=libuv_bin_directory) + + # replace native files in 'sdk' folder. + # from core_setup + RoverShellCall('cp libhostpolicy.so %s'%(RoverSettings.PatchTarget_SDK), cwd='%s/dll/'%(core_setup_cli_bin_directory)) + RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_SDK), cwd='%s/fxr/'%(core_setup_cli_bin_directory)) + + # replace native files in 'host' folder. + # from core_setup + RoverShellCall('cp libhostfxr.so %s'%(RoverSettings.PatchTarget_Host), cwd='%s/fxr/'%(core_setup_cli_bin_directory)) + + RoverPrint(RoverMods.Blue('has finished patching %s'%(RoverMods.Yellow(patchTarget_folder)))) + except: + RoverSettings._DevMode = True + UnexpectedRoverException(sys.exc_info()) + +## +## END ROVER FUNCTION DEFINITIONS +## + +if __name__ == "__main__": + ## + ## COMMAND-LINE BEHAVIOR + ## + + parser = argparse.ArgumentParser(description = 'This is the .NET CLI bootstrapping tool.') + + parser.add_argument('-build', metavar='b', nargs='*', default = ['coreclr', 'corefx', 'core-setup', 'libuv'],help='\'Builds\' all native components if no arguments are specified. Otherwise, specify one or more (space separated) arguments from the following : {' + + '%s, %s, %s, %s'%(RoverMods.Red('coreclr'), RoverMods.Blue('corefx'), RoverMods.Green('core-setup'), RoverMods.Yellow('libuv') +'}')) + parser.add_argument('-nopatch', action='store_true', default=False, help='prevents the copying of specific native binaries from the pre-built repositories in to the destination directory.') + parser.add_argument('-payload', nargs=1, help='Specify a path to a tarball (something that we can tar xf) that contains a version of the dotnet CLI.') + parser.add_argument('-to', type=str, default='%s'%(RoverSettings._Moniker), help='allows you to overwrite the default staging directory (default is %s)'%(RoverSettings._Moniker)) + + args = parser.parse_args() + + if args.payload: + RoverPrint('is using payload from \'' + RoverMods.White(str(args.payload)) + '\'') + + RoverPrint('Building: ' + RoverMods.White(str(args.build))) + RoverPrint('Patching? ' + RoverMods.White(str(not args.nopatch))) + + RoverSettings.SetWorkingDirectory(normpath(str(args.to))) + + RoverPrint('Staging in %s'%(RoverSettings._WorkingDirectory)) + RoverSettings.BuildSet = args.build + + # I am guessing that users are more inclined to want patching to happen whenever it can, and so I ask + # for specificity in the instances that they do not want patching. + RoverSettings.Patch = not args.nopatch + + if args.payload: + RoverSettings.PayloadPath = args.payload[0] + ## + ## END COMMAND-LINE BEHAVIOR + ## + + ## + ## BEGIN DECLARATIONS + ## + + coreclr_working_git_directory = path.join(RoverSettings._srcDirectory, 'coreclr') + corefx_working_git_directory = path.join(RoverSettings._srcDirectory, 'corefx') + core_setup_working_git_directory = path.join(RoverSettings._srcDirectory, 'core-setup') + libuv_working_git_directory = path.join(RoverSettings._srcDirectory, 'libuv') + + default_coreclr_bin_directory = '%s/bin/Product/Linux.x64.Release/'%(coreclr_working_git_directory) + default_corefx_native_bin_directory = '%s/bin/Linux.x64.Release/Native'%(corefx_working_git_directory) + default_core_setup_cli_bin_directory= '%s/src/corehost/cli'%(core_setup_working_git_directory) + default_libuv_bin_directory = '%s/.libs'%(libuv_working_git_directory) + + RoverSettings.CoreCLRBinDirectory = default_coreclr_bin_directory + RoverSettings.CoreFXBinDirectory = default_corefx_native_bin_directory + RoverSettings.CoreSetupBinDirectory = default_core_setup_cli_bin_directory + RoverSettings.LibUVBinDirectory = default_libuv_bin_directory + + platform_info = platform.uname() + this_distro_name = str(platform.linux_distribution()[0]).lower() + + ## + ## END DECLARATIONS + ## + + ## + ## BEGIN PROCEDURE + ## + + try: + os.putenv('ID', RoverSettings._OsVars['ID']) + os.putenv('VERSION_ID', RoverSettings._OsVars['VERSION_ID']) + + RoverPrint(RoverMods.Blue('RID: %s'%(RoverMods.Green(RoverSettings._Rid)))) + + # Spawn our working directory + if not path.exists(RoverSettings._WorkingDirectory): + makedirs(RoverSettings._WorkingDirectory) + + if not path.exists(RoverSettings._srcDirectory): + makedirs(RoverSettings._srcDirectory) + + if not path.exists(RoverSettings._objDirectory): + makedirs(RoverSettings._objDirectory) + + if not path.exists(RoverSettings._binDirectory): + makedirs(RoverSettings._binDirectory) + + SpawnPatchTarget(RoverSettings._binDirectory, RoverSettings.PayloadPath) + + # Fetch the commit hashes from the native files. + coreclr_output = check_output('strings libcoreclr.so | grep @\(#\)', shell = True, cwd=RoverSettings.PatchTarget_Shared) + corefx_output = check_output('strings System.Native.so | grep @\(#\)', shell = True, cwd=RoverSettings.PatchTarget_Shared) + + dotnet_commit_hash = check_output('strings dotnet | grep "[a-f0-9]\{40\}"', shell = True, cwd=RoverSettings.PatchTarget_Shared) + coreclr_commit_hash = coreclr_output[find(coreclr_output, 'Commit Hash: ') + len('Commit Hash: '):len(coreclr_output)] + corefx_commit_hash = corefx_output[find(corefx_output, 'Commit Hash: ') + len('Commit Hash: '):len(corefx_output)] + + RoverSettings.DotNetCommitHash = dotnet_commit_hash + + CloneRepositories(RoverSettings._srcDirectory, + coreclr_commit_hash, + corefx_commit_hash, + dotnet_commit_hash) + + BuildNativeComponents(coreclr_working_git_directory, + corefx_working_git_directory, + core_setup_working_git_directory, + libuv_working_git_directory) + + PatchTarget(RoverSettings.PatchTargetPath, + RoverSettings.CoreCLRBinDirectory, + RoverSettings.CoreFXBinDirectory, + RoverSettings.CoreSetupBinDirectory, + RoverSettings.LibUVBinDirectory) + + RoverPrint(RoverMods.Green('spawned a \'dotnet\' in %s'%(RoverMods.Yellow('./' + path.relpath(RoverSettings.PatchTargetPath) + '/'))) + RoverMods.Green('(enjoy!)')) + except: + RoverSettings._DevMode = True + UnexpectedRoverException(sys.exc_info()) + + + ## + ## END PROCEDURE + ##