Use newer versions of the build-sdk.sh scripts
This commit is contained in:
Matt Ellis
parent
4ef4eba01a
commit
213c9c3c08
|
|
@ -1,181 +1,2 @@
|
|||
@echo off
|
||||
setlocal EnableDelayedExpansion
|
||||
|
||||
set s3_bucket_root=s3://eng.pulumi.com/
|
||||
set s3_publish_folder=%s3_bucket_root%releases/sdk/
|
||||
set s3_rootfolder_pulumi_windows=s3://eng.pulumi.com/releases/pulumi/windows/amd64/
|
||||
set s3_rootfolder_pulumi_aws_windows=s3://eng.pulumi.com/releases/pulumi-aws/windows/amd64/
|
||||
set s3_rootfolder_pulumi_azure_windows=s3://eng.pulumi.com/releases/pulumi-azure/windows/amd64/
|
||||
set s3_rootfolder_pulumi_kubernetes_windows=s3://eng.pulumi.com/releases/pulumi-kubernetes/windows/amd64/
|
||||
set s3_rootfolder_pulumi_cloud_windows=s3://eng.pulumi.com/releases/pulumi-cloud/windows/amd64/
|
||||
set versionInput=
|
||||
set refName=master
|
||||
|
||||
if [%1] NEQ [] (
|
||||
set versionInput=%1
|
||||
) else (
|
||||
set file_date=%DATE:~-4,4%%DATE:~-7,2%%DATE:~-10,2%
|
||||
set file_time=%TIME:~0,2%%TIME:~3,2%%TIME:~6,2%
|
||||
set versionInput=!file_date!_!file_time!
|
||||
)
|
||||
|
||||
if [%2] NEQ [] (
|
||||
set refName=%2
|
||||
)
|
||||
|
||||
set sdkfilename=pulumi-%versionInput%-windows.x64.zip
|
||||
|
||||
echo Upon completion package: %sdkfilename% will be uploaded to S3 location: %s3_publish_folder%
|
||||
|
||||
where npm.cmd >nul 2>nul
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Please install npm ^(and add them to your %%PATH%%^) before running this script
|
||||
exit /B
|
||||
)
|
||||
|
||||
where 7z.exe >nul 2>nul
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Please install 7z ^(and add them to your %%PATH%%^) before running this script
|
||||
exit /B
|
||||
)
|
||||
|
||||
where aws.exe >nul 2>nul
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Please install AWS CLI ^(and add them to your %%PATH%%^) before running this script
|
||||
exit /B
|
||||
)
|
||||
|
||||
set pulumi_folder=%TEMP%\pulumi_%random%
|
||||
echo %pulumi_folder%
|
||||
|
||||
IF EXIST %pulumi_folder% (
|
||||
echo "Deleting existing Pulumi folder"
|
||||
rd /s /q %pulumi_folder%
|
||||
)
|
||||
|
||||
echo Creating Temporary Pulumi folder
|
||||
mkdir %pulumi_folder%
|
||||
|
||||
echo Switching to Temporary folder
|
||||
pushd .
|
||||
cd /d %pulumi_folder%
|
||||
|
||||
mkdir Pulumi
|
||||
cd Pulumi
|
||||
|
||||
echo copying install.cmd from dist
|
||||
copy "%~dp0\..\dist\install.cmd" .
|
||||
|
||||
call:processPackage pulumi .zip %s3_rootfolder_pulumi_windows%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to process Package for pulumi
|
||||
goto:eof
|
||||
)
|
||||
call:processPackage pulumi-aws .tgz %s3_rootfolder_pulumi_aws_windows%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to process Package for pulumi-aws
|
||||
goto:eof
|
||||
)
|
||||
|
||||
call:processPackage pulumi-azure .tgz %s3_rootfolder_pulumi_azure_windows%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to process Package for pulumi-azure
|
||||
goto:eof
|
||||
)
|
||||
|
||||
call:processPackage pulumi-kubernetes .tgz %s3_rootfolder_pulumi_kubernetes_windows%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to process Package for pulumi-kubernetes
|
||||
goto:eof
|
||||
)
|
||||
|
||||
call:processPackage pulumi-cloud .tgz %s3_rootfolder_pulumi_cloud_windows%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to process Package for pulumi-cloud
|
||||
goto:eof
|
||||
)
|
||||
|
||||
echo npm install pulumi and its components
|
||||
|
||||
call:npmInstall node_modules\pulumi
|
||||
|
||||
call:npmInstall node_modules\@pulumi\aws
|
||||
|
||||
call:npmInstall node_modules\@pulumi\azurerm
|
||||
|
||||
call:npmInstall node_modules\@pulumi\kubernetes
|
||||
|
||||
call:npmInstall node_modules\@pulumi\cloud
|
||||
|
||||
call:npmInstall node_modules\@pulumi\cloud-aws
|
||||
|
||||
cd ..
|
||||
|
||||
echo zip contents of the folder in a package
|
||||
|
||||
7z a -tzip %sdkfilename%
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to zip the contents of the folder to file: %sdkfilename%
|
||||
exit /b
|
||||
)
|
||||
|
||||
echo Upload %sdkfilename% to aws s3 bucket %s3_publish_folder%
|
||||
aws s3 cp --only-show-errors %sdkfilename% %s3_publish_folder%
|
||||
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to upload package: %sdkfilename% to S3 Bucket: %s3_publish_folder%
|
||||
exit /b
|
||||
)
|
||||
|
||||
echo Successfully created Windows Installation Package: %sdkfilename% and uploaded to S3 Bucket: %s3_publish_folder%
|
||||
|
||||
popd
|
||||
echo Deleting Temporary folder
|
||||
rd /s /q %pulumi_folder%
|
||||
echo Success.
|
||||
goto:eof
|
||||
|
||||
:processPackage
|
||||
@echo off
|
||||
echo Getting Packages for repo: %~1 pkg type: %~2
|
||||
setlocal
|
||||
|
||||
set gitrepo=https://github.com/pulumi/%~1.git
|
||||
|
||||
for /f "tokens=1" %%A IN ('git ls-remote -h -t %gitrepo% %refName%') DO set commit=%%A
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to get last commit hash for %gitrepo%
|
||||
exit /b
|
||||
)
|
||||
set file=%commit%%~2%
|
||||
set s3-file=%~3%file%
|
||||
aws s3 cp --only-show-errors %s3-file% .
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to get package: %s3-file% from AWS S3
|
||||
exit /b
|
||||
)
|
||||
echo unzipping %file%
|
||||
if %~2 NEQ .zip (
|
||||
7z x %file% -tgzip -so | 7z x -ttar -si
|
||||
) else (
|
||||
7z x %file%
|
||||
)
|
||||
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo Failed to unzip package: %file%
|
||||
exit /b
|
||||
) else (
|
||||
echo deleting the package: %file%
|
||||
del /q %file%
|
||||
)
|
||||
goto:eof
|
||||
|
||||
:npmInstall
|
||||
echo Installing Node modules from %~1
|
||||
pushd %~1
|
||||
CALL npm install --only=production
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo npm install failed for Module: %~1
|
||||
)
|
||||
popd
|
||||
goto:eof
|
||||
powershell -noprofile -executionPolicy Unrestricted -file "%~dpn0.ps1" %*
|
||||
82
scripts/build-sdk.ps1
Normal file
82
scripts/build-sdk.ps1
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
param (
|
||||
$VersionTag,
|
||||
$PulumiRef
|
||||
)
|
||||
|
||||
Set-StrictMode -Version 2.0
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$S3ProdBucketRoot="s3://rel.pulumi.com/releases/"
|
||||
$S3EngBucketRoot="s3://eng.pulumi.com/releases/"
|
||||
$S3PublishFolderSdk="${S3ProdBucketRoot}sdk/"
|
||||
|
||||
function New-TemporaryDirectory {
|
||||
New-Item -ItemType Directory -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()))
|
||||
}
|
||||
|
||||
function New-TemporaryFile {
|
||||
New-Item -ItemType File -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()))
|
||||
}
|
||||
|
||||
function Download-Release ($repoName, $repoCommit, [ValidateSet("zip", "tgz")]$ext) {
|
||||
Write-Host "downloading $repoName@$repoCommit"
|
||||
|
||||
$file="${repoCommit}.${ext}"
|
||||
$s3File="${S3EngBucketRoot}${repoName}/windows/amd64/${file}"
|
||||
|
||||
aws s3 cp --only-show-errors "$s3File" ".\$file"
|
||||
|
||||
switch($ext) {
|
||||
"zip" { 7z x ${file} }
|
||||
"tgz" { cmd /C "7z x ${file} -tgzip -so | 7z x -ttar -si" }
|
||||
default { Write-Error "Unknown extention type $ext" }
|
||||
}
|
||||
|
||||
Remove-Item -Force "$file"
|
||||
}
|
||||
|
||||
if (!$VersionTag) { $VersionTag=Get-Date -UFormat '%Y%m%d_%H%M%S' }
|
||||
if (!$PulumiRef) { $PulumiRef="master" }
|
||||
|
||||
$SdkFileName="pulumi-$VersionTag-windows.x64.zip"
|
||||
|
||||
$PulumiFolder=(Join-Path (New-TemporaryDirectory) "Pulumi")
|
||||
|
||||
New-Item -ItemType Directory -Path $PulumiFolder | Out-Null
|
||||
|
||||
Push-Location "$PulumiFolder" | Out-Null
|
||||
|
||||
Write-Host "pulumi: $PulumiRef"
|
||||
Write-Host ""
|
||||
|
||||
Download-Release "pulumi" $PulumiRef "zip"
|
||||
|
||||
Remove-Item -Recurse -Force -Path "$PulumiFolder\node_modules"
|
||||
|
||||
$SdkPackagePath=(Join-Path ([System.IO.Path]::GetTempPath()) $SdkFileName)
|
||||
|
||||
if (Test-Path $SdkPackagePath) {
|
||||
Remove-Item -Force -Path $SdkPackagePath
|
||||
}
|
||||
|
||||
7z a -tzip "$SdkPackagePath" "$(Join-Path (Split-Path -Parent $PulumiFolder) '.')"
|
||||
|
||||
Write-Host "uploading SDK to ${S3PublishFolderSdk}${SdkFileName}"
|
||||
|
||||
$AWSCreds=((aws sts assume-role `
|
||||
--role-arn "arn:aws:iam::058607598222:role/UploadPulumiReleases" `
|
||||
--role-session-name "upload-sdk" `
|
||||
--external-id "upload-pulumi-release") | ConvertFrom-Json)
|
||||
|
||||
$env:AWS_ACCESS_KEY_ID=$AWSCreds.Credentials.AccessKeyId
|
||||
$env:AWS_SECRET_ACCESS_KEY=$AWSCreds.Credentials.SecretAccessKey
|
||||
$env:AWS_SECURITY_TOKEN=$AWSCreds.Credentials.SessionToken
|
||||
|
||||
aws s3 cp --only-show-errors "$SdkPackagePath" "${S3PublishFolderSdk}${SdkFileName}"
|
||||
|
||||
Pop-Location | Out-Null
|
||||
|
||||
Remove-Item -Path $SdkPackagePath
|
||||
Remove-Item -Path (Split-Path -Parent $PulumiFolder) -Force -Recurse
|
||||
|
||||
Write-Host "done"
|
||||
|
|
@ -1,144 +1,104 @@
|
|||
#!/bin/bash
|
||||
# Usage CreatePkg.sh [version-tag] [ref-name]
|
||||
# CreatePkg.sh will package the latest packages from S3 release bucket based on the git commit associated with ref-name
|
||||
# for main components namely pulumi, pulumi-aws, pulumi-azure, pulumi-kubernetes, pulumi-cloud and then package them together along with install.sh to
|
||||
# create a distributable SDK package.
|
||||
# Usage build-sdk.sh [version-tag] [pulumi-cloud-ref-name]
|
||||
#
|
||||
# version-tag defaults to current date and time
|
||||
# ref-name defaults to master (can be a branch or tag name)
|
||||
set -o nounset -o errexit -o pipefail
|
||||
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
|
||||
|
||||
# This function does npm install of the extracted packages
|
||||
# usage npm_install path_to_module
|
||||
function npm_install()
|
||||
S3_PROD_BUCKET_ROOT="s3://rel.pulumi.com/releases/"
|
||||
S3_ENG_BUCKET_ROOT="s3://eng.pulumi.com/releases/"
|
||||
S3_PUBLISH_FOLDER_SDK="${S3_PROD_BUCKET_ROOT}sdk/"
|
||||
|
||||
# This function downloads a specific release and into the current working directory
|
||||
# usage: download_release <repo-name> <commitish>
|
||||
function download_release()
|
||||
{
|
||||
echo Installing Node modules from ${1}
|
||||
( cd ${1} ; npm install --only=production )
|
||||
}
|
||||
local repo_name=${1}
|
||||
local repo_commit=${2}
|
||||
|
||||
# This function does the following
|
||||
# Usage process_package <component_name> <s3_bucket_package_folder>
|
||||
# 1. Gets the last commit for the component identified by the repo
|
||||
# 2. Get the tgz package corresponding to the commit from the S3 bucket
|
||||
# 3. Extracts the package locally
|
||||
# 4. Deletes the package after successful extraction
|
||||
echo "downloading ${repo_name}@${repo_commit}"
|
||||
|
||||
function process_package()
|
||||
{
|
||||
echo Getting Packages for repo: ${1}
|
||||
commit=$(git ls-remote -h -t git@github.com:pulumi/$1.git ${ref_name} | cut -f1)
|
||||
|
||||
# check for empty commit hash
|
||||
if [ -z "${commit}" ]; then
|
||||
echo Failed to get last commit hash for ${1}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
file=${commit}.tgz
|
||||
s3_file=${s3_bucket_root}${1}/${OS}/${platform}/${file}
|
||||
local file=${repo_commit}.tgz
|
||||
local s3_file=${S3_ENG_BUCKET_ROOT}${repo_name}/${OS}/amd64/${file}
|
||||
|
||||
# Use AWS CLI to download the package corresponding to the component from S3 bucket
|
||||
if aws s3 cp --only-show-errors "${s3_file}" './'"${file}" 2>/dev/null; then
|
||||
echo Successfully downloaded: ${s3_file}
|
||||
else
|
||||
echo Failed to get package: ${s3_file} from AWS S3
|
||||
if ! aws s3 cp --only-show-errors "${s3_file}" "./${file}" 2>/dev/null; then
|
||||
>&2 echo "failed to download ${s3_file}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if tar -xzf "${file}" 2>/dev/null; then
|
||||
echo Successfully unzipped file:${file}. Now deleting
|
||||
rm -f ${file}
|
||||
else
|
||||
echo Failed to unzip package: ${file}
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main script processing the workload. Following steps are performed
|
||||
# 1. Get the OS to find out if we should build for Darwin(Mac) or Linux
|
||||
# 2. Check if a Version Input is passed. If not, get current date/time to create a version stamp
|
||||
# 3. Check if a tag or head name was passed, otherwise use master
|
||||
# 4. Verify dependencies such as npm, tar and aws cli are installed
|
||||
# 5. Process the generated packages for each component (pulumi, pulumi-aws, pulumi-cloud) based on the last
|
||||
# commit sha from master branch and extract them to the temporary folder
|
||||
# 6. run npm for each component
|
||||
# 7. Build the tar file and upload to S3 bucket
|
||||
# This function downloads and extracts a specific release and into the current working directory
|
||||
# usage: download_and_extract_release <repo-name> <commitish>
|
||||
function download_and_extract_release()
|
||||
{
|
||||
local repo_name=${1}
|
||||
local repo_commit=${2}
|
||||
|
||||
platform=amd64
|
||||
s3_bucket_root="s3://eng.pulumi.com/releases/"
|
||||
s3_publish_folder="${s3_bucket_root}sdk/"
|
||||
components=("pulumi" "pulumi-aws" "pulumi-azure" "pulumi-kubernetes" "pulumi-cloud")
|
||||
dependencies=("npm" "tar" "aws")
|
||||
local file=${repo_commit}.tgz
|
||||
|
||||
# Get the OS version
|
||||
download_release "${repo_name}" "${repo_commit}"
|
||||
|
||||
if ! tar -xzf "${file}" 2>/dev/null; then
|
||||
>&2 echo "failed to untar ${file}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm "./${file}"
|
||||
}
|
||||
|
||||
# get the OS version
|
||||
OS=""
|
||||
case $(uname) in
|
||||
"Linux") OS="linux";;
|
||||
"Darwin") OS="darwin";;
|
||||
"Linux") OS=linux;;
|
||||
"Darwin") OS=darwin;;
|
||||
*) echo "error: unknown host os $(uname)" ; exit 1;;
|
||||
esac
|
||||
|
||||
sdkfilename=pulumi-${1:-$(date +"%Y%m%d_%H%M%S")}-${OS}.x64.tar.gz
|
||||
ref_name="${2:-master}"
|
||||
|
||||
echo Upon completion package: ${sdkfilename} will be uploaded to S3 location: ${s3_publish_folder}
|
||||
|
||||
# Verify the required dependencies are already installed
|
||||
for dependency in ${dependencies[@]}
|
||||
do
|
||||
if ! command -v ${dependency} >/dev/null; then
|
||||
echo "required dependency '${dependency}' is not installed"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
SDK_FILENAME=pulumi-${1:-$(date +"%Y%m%d_%H%M%S")}-${OS}.x64.tar.gz
|
||||
PULUMI_REF=${2:-master}
|
||||
|
||||
# setup temporary folder to process the package
|
||||
pulumi_folder=$(mktemp -d)/pulumi
|
||||
mkdir -p ${pulumi_folder}
|
||||
PULUMI_FOLDER=$(mktemp -d)/pulumi
|
||||
mkdir -p "${PULUMI_FOLDER}"
|
||||
|
||||
echo ${pulumi_folder}
|
||||
cd ${pulumi_folder}
|
||||
cd "${PULUMI_FOLDER}"
|
||||
|
||||
cp ${SCRIPT_DIR}/../dist/install.sh .
|
||||
cp "${SCRIPT_DIR}/../dist/install.sh" .
|
||||
|
||||
# Process each component from the components list
|
||||
for component in ${components[@]}
|
||||
do
|
||||
echo Processing Component: $component
|
||||
process_package $component
|
||||
done
|
||||
download_and_extract_release pulumi "${PULUMI_REF}"
|
||||
|
||||
echo npm install pulumi and its components
|
||||
if [ -d "${pulumi_folder}/node_modules/pulumi" ]; then
|
||||
npm_install "${pulumi_folder}/node_modules/pulumi"
|
||||
fi
|
||||
# All node packages are now delivered via npm, so remove the node_modules folder.
|
||||
rm -rf "${PULUMI_FOLDER}/node_modules"
|
||||
|
||||
if [ -d "${pulumi_folder}/node_modules/@pulumi" ]; then
|
||||
for packageDir in "${pulumi_folder}/node_modules/@pulumi"/*; do
|
||||
npm_install "$packageDir"
|
||||
done
|
||||
fi
|
||||
SDK_PACKAGE_PATH=$(mktemp)
|
||||
|
||||
echo zip contents of the folder in a package
|
||||
sdk_package_path=$(mktemp)
|
||||
echo "compressing package to ${SDK_PACKAGE_PATH}"
|
||||
|
||||
cd ..
|
||||
|
||||
if tar -zcf ${sdk_package_path} pulumi; then
|
||||
echo Successfully created the package: ${sdk_package_path}
|
||||
else
|
||||
echo Failed to zip the contents of the folder to file: ${sdk_package_path}
|
||||
if ! tar -zcf ${SDK_PACKAGE_PATH} pulumi; then
|
||||
>&2 echo "failed to compress package"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo Upload ${sdk_package_path} to aws s3 bucket ${s3_publish_folder}${sdkfilename}
|
||||
aws s3 cp --only-show-errors ${sdk_package_path} ${s3_publish_folder}${sdkfilename}
|
||||
echo "uploading SDK to ${S3_PUBLISH_FOLDER_SDK}${SDK_FILENAME}"
|
||||
echo "${SDK_PACKAGE_PATH}"
|
||||
# rel.pulumi.com is in our production account, so assume that role first
|
||||
CREDS_JSON=$(aws sts assume-role \
|
||||
--role-arn "arn:aws:iam::058607598222:role/UploadPulumiReleases" \
|
||||
--role-session-name "upload-sdk" \
|
||||
--external-id "upload-pulumi-release")
|
||||
|
||||
echo Successfully created: ${s3_publish_folder}${sdkfilename}
|
||||
# Use these new credentials to create the PPC user account.
|
||||
export AWS_ACCESS_KEY_ID=$(echo ${CREDS_JSON} | jq ".Credentials.AccessKeyId" --raw-output)
|
||||
export AWS_SECRET_ACCESS_KEY=$(echo ${CREDS_JSON} | jq ".Credentials.SecretAccessKey" --raw-output)
|
||||
export AWS_SECURITY_TOKEN=$(echo ${CREDS_JSON} | jq ".Credentials.SessionToken" --raw-output)
|
||||
|
||||
echo Deleting Temporary folder
|
||||
rm ${sdk_package_path}
|
||||
rm -rf ${pulumi_folder}/
|
||||
aws s3 cp --only-show-errors "${SDK_PACKAGE_PATH}" "${S3_PUBLISH_FOLDER_SDK}${SDK_FILENAME}"
|
||||
|
||||
echo Success.
|
||||
rm "${SDK_PACKAGE_PATH}"
|
||||
rm -rf "${PULUMI_FOLDER}/"
|
||||
|
||||
echo "done"
|
||||
|
|
|
|||
Loading…
Reference in a new issue