2020-01-07 17:50:08 +01:00
#!/bin/groovy
// This job effectively has two SCM configurations:
// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job
// one for elasticsearch, used to check out the elasticsearch source before building it
// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same
// 'branch_specifier' is for kibana / the job itself
// ES_BRANCH is for elasticsearch
library 'kibana-pipeline-library'
kibanaLibrary.load()
def ES_BRANCH = params.ES_BRANCH
if (!ES_BRANCH) {
error "Parameter 'ES_BRANCH' must be specified."
}
currentBuild.displayName += " - ${ES_BRANCH}"
currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}"
def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
2020-07-02 21:41:23 +02:00
node(workers.label('l')) {
2020-03-02 21:04:27 +01:00
catchErrors {
2020-01-07 17:50:08 +01:00
def VERSION
def SNAPSHOT_ID
def DESTINATION
def scmVars = checkoutEs(ES_BRANCH)
def GIT_COMMIT = scmVars.GIT_COMMIT
def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()
buildArchives('to-archive')
dir('to-archive') {
def now = new Date()
def date = now.format("yyyyMMdd-HHmmss")
def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
def files = filesRaw
.split("\n")
.collect { filename ->
// Filename examples
// elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
// elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
def parts = filename.replace("elasticsearch-oss", "oss").split("-")
VERSION = VERSION ?: parts[1]
SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"
return [
filename: filename,
checksum: filename + '.sha512',
url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
version: parts[1],
platform: parts[3],
architecture: parts[4].split('\\.')[0],
license: parts[0] == 'oss' ? 'oss' : 'default',
]
}
sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'
def manifest = [
bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
branch: ES_BRANCH,
sha: GIT_COMMIT,
sha_short: GIT_COMMIT_SHORT,
version: VERSION,
generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
archives: files,
]
def manifestJson = toJSON(manifest).toString()
writeFile file: 'manifest.json', text: manifestJson
upload(DESTINATION, '*.*')
sh "cp manifest.json manifest-latest.json"
upload(VERSION, 'manifest-latest.json')
}
if (PROMOTE_WITHOUT_VERIFY) {
esSnapshots.promote(VERSION, SNAPSHOT_ID)
emailext(
to: 'build-kibana@elastic.co',
subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
body: '${SCRIPT,template="groovy-html.template"}',
mimeType: 'text/html',
)
} else {
build(
propagate: false,
wait: false,
job: 'elasticsearch+snapshots+verify',
parameters: [
string(name: 'branch_specifier', value: branch_specifier),
string(name: 'SNAPSHOT_VERSION', value: VERSION),
string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
]
)
}
}
kibanaPipeline.sendMail()
}
}
}
}
def checkoutEs(branch) {
retryWithDelay(8, 15) {
return checkout([
$class: 'GitSCM',
branches: [[name: branch]],
doGenerateSubmoduleConfigurations: false,
extensions: [],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
url: 'git@github.com:elastic/elasticsearch',
]],
])
}
}
def upload(destination, pattern) {
return googleStorageUpload(
credentialsId: 'kibana-ci-gcs-plugin',
bucket: "gs://kibana-ci-es-snapshots-daily/${destination}",
pattern: pattern,
sharedPublicly: false,
showInline: false,
)
}
def buildArchives(destination) {
def props = readProperties file: '.ci/java-versions.properties'
withEnv([
// Select the correct JDK for this branch
"PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}",
// These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want
"BUILD_NUMBER=",
"JENKINS_URL=",
"BUILD_URL=",
"JOB_NAME=",
"NODE_NAME=",
]) {
sh """
2020-07-02 21:41:23 +02:00
./gradlew -Dbuild.docker=true assemble --parallel
2020-01-07 17:50:08 +01:00
mkdir -p ${destination}
2020-07-02 21:41:23 +02:00
find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
2020-01-07 17:50:08 +01:00
"""
}
}