kibana/.ci/es-snapshots/Jenkinsfile_build_es

#!/bin/groovy

// This job effectively has two SCM configurations:
// one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job
// one for elasticsearch, used to check out the elasticsearch source before building it

// There are two parameters that drive which branch is checked out for each of these, but they will typically be the same
// 'branch_specifier' is for kibana / the job itself
// ES_BRANCH is for elasticsearch

library 'kibana-pipeline-library'
kibanaLibrary.load()

def ES_BRANCH = params.ES_BRANCH

if (!ES_BRANCH) {
  error "Parameter 'ES_BRANCH' must be specified."
}

currentBuild.displayName += " - ${ES_BRANCH}"
currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}"

def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION

timeout(time: 120, unit: 'MINUTES') {
  timestamps {
    ansiColor('xterm') {
      node(workers.label('l')) {
        catchErrors {
          def VERSION
          def SNAPSHOT_ID
          def DESTINATION

          def scmVars = checkoutEs(ES_BRANCH)
          def GIT_COMMIT = scmVars.GIT_COMMIT
          def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim()

          buildArchives('to-archive')

          dir('to-archive') {
            def now = new Date()
            def date = now.format("yyyyMMdd-HHmmss")

            def filesRaw = sh(script: "ls -1", returnStdout: true).trim()
            def files = filesRaw
              .split("\n")
              .collect { filename ->
                // Filename examples
                // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
                // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
                def parts = filename.replace("elasticsearch-oss", "oss").split("-")

                VERSION = VERSION ?: parts[1]
                SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}"
                DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}"

                return [
                  filename: filename,
                  checksum: filename + '.sha512',
                  url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(),
                  version: parts[1],
                  platform: parts[3],
                  architecture: parts[4].split('\\.')[0],
                  license: parts[0] == 'oss' ? 'oss' : 'default',
                ]
              }

            sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;'

            def manifest = [
              bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(),
              branch: ES_BRANCH,
              sha: GIT_COMMIT,
              sha_short: GIT_COMMIT_SHORT,
              version: VERSION,
              generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")),
              archives: files,
            ]
            def manifestJson = toJSON(manifest).toString()
            writeFile file: 'manifest.json', text: manifestJson

            upload(DESTINATION, '*.*')

            sh "cp manifest.json manifest-latest.json"
            upload(VERSION, 'manifest-latest.json')
          }

          if (PROMOTE_WITHOUT_VERIFY) {
            esSnapshots.promote(VERSION, SNAPSHOT_ID)

            emailext(
              to: 'build-kibana@elastic.co',
              subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}",
              body: '${SCRIPT,template="groovy-html.template"}',
              mimeType: 'text/html',
            )
          } else {
            build(
              propagate: false,
              wait: false,
              job: 'elasticsearch+snapshots+verify',
              parameters: [
                string(name: 'branch_specifier', value: branch_specifier),
                string(name: 'SNAPSHOT_VERSION', value: VERSION),
                string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID),
              ]
            )
          }
        }

        kibanaPipeline.sendMail()
      }
    }
  }
}

def checkoutEs(branch) {
  retryWithDelay(8, 15) {
    return checkout([
      $class: 'GitSCM',
      branches: [[name: branch]],
      doGenerateSubmoduleConfigurations: false,
      extensions: [],
      submoduleCfg: [],
      userRemoteConfigs: [[
        credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
        url: 'git@github.com:elastic/elasticsearch',
      ]],
    ])
  }
}

def upload(destination, pattern) {
  return googleStorageUpload(
    credentialsId: 'kibana-ci-gcs-plugin',
    bucket: "gs://kibana-ci-es-snapshots-daily/${destination}",
    pattern: pattern,
    sharedPublicly: false,
    showInline: false,
  )
}

def buildArchives(destination) {
  def props = readProperties file: '.ci/java-versions.properties'
  withEnv([
    // Select the correct JDK for this branch
    "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}",

    // These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want
    "BUILD_NUMBER=",
    "JENKINS_URL=",
    "BUILD_URL=",
    "JOB_NAME=",
    "NODE_NAME=",
  ]) {
    sh """
      ./gradlew -Dbuild.docker=true assemble --parallel
      mkdir -p ${destination}
      find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
      docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
    """
  }
}