* [jenkins] refer to sizes in most pipeline code * switch back to `linux && immutable` for small instances Co-authored-by: spalger <spalger@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
163 lines
5.4 KiB
Groovy
163 lines
5.4 KiB
Groovy
#!/bin/groovy
|
|
|
|
// This job effectively has two SCM configurations:
|
|
// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job
|
|
// one for elasticsearch, used to check out the elasticsearch source before building it
|
|
|
|
// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same
|
|
// 'branch_specifier' is for kibana / the job itself
|
|
// ES_BRANCH is for elasticsearch
|
|
|
|
library 'kibana-pipeline-library'
|
|
kibanaLibrary.load()
|
|
|
|
def ES_BRANCH = params.ES_BRANCH
|
|
|
|
if (!ES_BRANCH) {
|
|
error "Parameter 'ES_BRANCH' must be specified."
|
|
}
|
|
|
|
currentBuild.displayName += " - ${ES_BRANCH}"
|
|
currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}"
|
|
|
|
def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
|
|
|
|
timeout(time: 120, unit: 'MINUTES') {
|
|
timestamps {
|
|
ansiColor('xterm') {
|
|
node(workers.label('s')) {
|
|
catchErrors {
|
|
def VERSION
|
|
def SNAPSHOT_ID
|
|
def DESTINATION
|
|
|
|
def scmVars = checkoutEs(ES_BRANCH)
|
|
def GIT_COMMIT = scmVars.GIT_COMMIT
|
|
def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()
|
|
|
|
buildArchives('to-archive')
|
|
|
|
dir('to-archive') {
|
|
def now = new Date()
|
|
def date = now.format("yyyyMMdd-HHmmss")
|
|
|
|
def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
|
|
def files = filesRaw
|
|
.split("\n")
|
|
.collect { filename ->
|
|
// Filename examples
|
|
// elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
|
|
// elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
|
|
def parts = filename.replace("elasticsearch-oss", "oss").split("-")
|
|
|
|
VERSION = VERSION ?: parts[1]
|
|
SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
|
|
DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"
|
|
|
|
return [
|
|
filename: filename,
|
|
checksum: filename + '.sha512',
|
|
url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
|
|
version: parts[1],
|
|
platform: parts[3],
|
|
architecture: parts[4].split('\\.')[0],
|
|
license: parts[0] == 'oss' ? 'oss' : 'default',
|
|
]
|
|
}
|
|
|
|
sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'
|
|
|
|
def manifest = [
|
|
bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
|
|
branch: ES_BRANCH,
|
|
sha: GIT_COMMIT,
|
|
sha_short: GIT_COMMIT_SHORT,
|
|
version: VERSION,
|
|
generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
|
|
archives: files,
|
|
]
|
|
def manifestJson = toJSON(manifest).toString()
|
|
writeFile file: 'manifest.json', text: manifestJson
|
|
|
|
upload(DESTINATION, '*.*')
|
|
|
|
sh "cp manifest.json manifest-latest.json"
|
|
upload(VERSION, 'manifest-latest.json')
|
|
}
|
|
|
|
if (PROMOTE_WITHOUT_VERIFY) {
|
|
esSnapshots.promote(VERSION, SNAPSHOT_ID)
|
|
|
|
emailext(
|
|
to: 'build-kibana@elastic.co',
|
|
subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
|
|
body: '${SCRIPT,template="groovy-html.template"}',
|
|
mimeType: 'text/html',
|
|
)
|
|
} else {
|
|
build(
|
|
propagate: false,
|
|
wait: false,
|
|
job: 'elasticsearch+snapshots+verify',
|
|
parameters: [
|
|
string(name: 'branch_specifier', value: branch_specifier),
|
|
string(name: 'SNAPSHOT_VERSION', value: VERSION),
|
|
string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
|
|
]
|
|
)
|
|
}
|
|
}
|
|
|
|
kibanaPipeline.sendMail()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def checkoutEs(branch) {
|
|
retryWithDelay(8, 15) {
|
|
return checkout([
|
|
$class: 'GitSCM',
|
|
branches: [[name: branch]],
|
|
doGenerateSubmoduleConfigurations: false,
|
|
extensions: [],
|
|
submoduleCfg: [],
|
|
userRemoteConfigs: [[
|
|
credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
|
|
url: 'git@github.com:elastic/elasticsearch',
|
|
]],
|
|
])
|
|
}
|
|
}
|
|
|
|
def upload(destination, pattern) {
|
|
return googleStorageUpload(
|
|
credentialsId: 'kibana-ci-gcs-plugin',
|
|
bucket: "gs://kibana-ci-es-snapshots-daily/${destination}",
|
|
pattern: pattern,
|
|
sharedPublicly: false,
|
|
showInline: false,
|
|
)
|
|
}
|
|
|
|
def buildArchives(destination) {
|
|
def props = readProperties file: '.ci/java-versions.properties'
|
|
withEnv([
|
|
// Select the correct JDK for this branch
|
|
"PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}",
|
|
|
|
// These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want
|
|
"BUILD_NUMBER=",
|
|
"JENKINS_URL=",
|
|
"BUILD_URL=",
|
|
"JOB_NAME=",
|
|
"NODE_NAME=",
|
|
]) {
|
|
sh """
|
|
./gradlew -p distribution/archives assemble --parallel
|
|
mkdir -p ${destination}
|
|
find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
|
|
"""
|
|
}
|
|
}
|