[CI] Buildkite ES Snapshots Pipelines (#100843)

.buildkite/pipelines/es_snapshots/build.yml

@@ -0,0 +1,5 @@
+steps:
+  - command: .buildkite/scripts/steps/es_snapshots/build.sh
+    label: Build ES Snapshot
+    agents:
+      queue: c2-8

.buildkite/pipelines/es_snapshots/promote.yml

@@ -0,0 +1,12 @@
+steps:
+  - block: 'Promote'
+    prompt: "Enter the details for the snapshot you'd like to promote"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+  - label: Promote Snapshot
+    command: .buildkite/scripts/steps/es_snapshots/promote.sh

.buildkite/pipelines/es_snapshots/verify.yml

@@ -0,0 +1,102 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
+steps:
+  - block: 'Verify'
+    prompt: "Enter the details for the snapshot you'd like to verify"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: c2-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - command: .buildkite/scripts/steps/functional/xpack-cigroup.sh
+    label: 'Default CI Group'
+    parallelism: 13
+    agents:
+      queue: ci-group-6
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: default-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack-cigroup.sh
+    label: 'Docker CI Group'
+    agents:
+      queue: ci-group-6
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: default-cigroup-docker
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss-cigroup.sh
+    label: 'OSS CI Group'
+    parallelism: 12
+    agents:
+      queue: ci-group-4d
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: oss-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/jest_integration.sh
+    label: 'Jest Integration Tests'
+    agents:
+      queue: jest
+    artifact_paths: target/junit/**/*.xml
+    key: jest-integration
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/api_integration.sh
+    label: 'API Integration Tests'
+    agents:
+      queue: jest
+    artifact_paths: target/junit/**/*.xml
+    key: api-integration
+
+  - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
+    label: Trigger promotion
+    depends_on:
+      - default-cigroup
+      - default-cigroup-docker
+      - oss-cigroup
+      - jest-integration
+      - api-integration
+
+  - wait: ~
+    continue_on_failure: true
+
+  - plugins:
+      - junit-annotate#v1.9.0:
+          artifacts: target/junit/**/*.xml
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/scripts/download_build_artifacts.sh

@@ -7,8 +7,8 @@ if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
 
   cd "$WORKSPACE"
 
-  buildkite-agent artifact download kibana-default.tar.gz .
-  buildkite-agent artifact download kibana-default-plugins.tar.gz .
+  buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
   mkdir -p "$KIBANA_BUILD_LOCATION"
   tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

.buildkite/scripts/lifecycle/pre_command.sh

@@ -72,3 +72,8 @@ if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
     source .buildkite/scripts/common/setup_bazel.sh
   fi
 fi
+
+PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
+if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
+  source "$PIPELINE_PRE_COMMAND"
+fi

.buildkite/scripts/steps/build_kibana.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana.sh
+.buildkite/scripts/build_kibana_plugins.sh
+.buildkite/scripts/post_build_kibana.sh

.buildkite/scripts/steps/es_snapshots/bucket_config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  BASE_BUCKET_DAILY: 'kibana-ci-es-snapshots-daily',
+  BASE_BUCKET_PERMANENT: 'kibana-ci-es-snapshots-permanent',
+};

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo "--- Cloning Elasticsearch and preparing workspace"
+
+cd ..
+destination="$(pwd)/es-build"
+rm -rf "$destination"
+mkdir -p "$destination"
+
+mkdir -p elasticsearch && cd elasticsearch
+
+export ELASTICSEARCH_BRANCH="${ELASTICSEARCH_BRANCH:-$BUILDKITE_BRANCH}"
+
+if [[ ! -d .git ]]; then
+  git init
+  git remote add origin https://github.com/elastic/elasticsearch.git
+fi
+git fetch origin --depth 1 "$ELASTICSEARCH_BRANCH"
+git reset --hard FETCH_HEAD
+
+ELASTICSEARCH_GIT_COMMIT="$(git rev-parse HEAD)"
+export ELASTICSEARCH_GIT_COMMIT
+
+ELASTICSEARCH_GIT_COMMIT_SHORT="$(git rev-parse --short HEAD)"
+export ELASTICSEARCH_GIT_COMMIT_SHORT
+
+# These turn off automation in the Elasticsearch repo
+export BUILD_NUMBER=""
+export JENKINS_URL=""
+export BUILD_URL=""
+export JOB_NAME=""
+export NODE_NAME=""
+export DOCKER_BUILDKIT=""
+
+# Reads the ES_BUILD_JAVA env var out of .ci/java-versions.properties and exports it
+export "$(grep '^ES_BUILD_JAVA' .ci/java-versions.properties | xargs)"
+
+export PATH="$HOME/.java/$ES_BUILD_JAVA/bin:$PATH"
+export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+
+# The Elasticsearch Dockerfile needs to be built with root privileges, but Docker on our servers is running using a non-root user
+# So, let's use docker-in-docker to temporarily create a privileged docker daemon to run `docker build` on
+# We have to do this, because there's no `docker build --privileged` or similar
+
+echo "--- Setting up Docker-in-Docker for Elasticsearch"
+
+docker rm -f dind || true # If there's an old daemon running that somehow didn't get cleaned up, lets remove it first
+CERTS_DIR="$HOME/dind-certs"
+rm -rf "$CERTS_DIR"
+docker run -d --rm --privileged --name dind --userns host -p 2377:2376 -e DOCKER_TLS_CERTDIR=/certs -v "$CERTS_DIR":/certs docker:dind
+
+trap "docker rm -f dind" EXIT
+
+export DOCKER_TLS_VERIFY=true
+export DOCKER_CERT_PATH="$CERTS_DIR/client"
+export DOCKER_TLS_CERTDIR="$CERTS_DIR"
+export DOCKER_HOST=localhost:2377
+
+echo "--- Build Elasticsearch"
+./gradlew -Dbuild.docker=true assemble --parallel
+
+echo "--- Create distribution archives"
+find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \;
+
+ls -alh "$destination"
+
+echo "--- Create docker image archives"
+docker images "docker.elastic.co/elasticsearch/elasticsearch"
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+
+echo "--- Create checksums for snapshot files"
+cd "$destination"
+find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;
+
+cd "$BUILDKITE_BUILD_CHECKOUT_PATH"
+node "$(dirname "${0}")/create_manifest.js" "$destination"
+
+ES_SNAPSHOT_MANIFEST="$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)"
+
+cat << EOF | buildkite-agent annotate --style "info"
+  - \`ELASTICSEARCH_BRANCH\` - \`$ELASTICSEARCH_BRANCH\`
+  - \`ELASTICSEARCH_GIT_COMMIT\` - \`$ELASTICSEARCH_GIT_COMMIT\`
+  - \`ES_SNAPSHOT_MANIFEST\` - \`$ES_SNAPSHOT_MANIFEST\`
+  - \`ES_SNAPSHOT_VERSION\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_VERSION)\`
+  - \`ES_SNAPSHOT_ID\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_ID)\`
+EOF
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - trigger: 'kibana-elasticsearch-snapshot-verify'
+    async: true
+    build:
+      env:
+        ES_SNAPSHOT_MANIFEST: '$ES_SNAPSHOT_MANIFEST'
+      branch: '$BUILDKITE_BRANCH'
+EOF

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -0,0 +1,90 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
+
+(async () => {
+  console.log('--- Create ES Snapshot Manifest');
+
+  const destination = process.argv[2] || __dirname + '/test';
+
+  const ES_BRANCH = process.env.ELASTICSEARCH_BRANCH;
+  const GIT_COMMIT = process.env.ELASTICSEARCH_GIT_COMMIT;
+  const GIT_COMMIT_SHORT = process.env.ELASTICSEARCH_GIT_COMMIT_SHORT;
+
+  let VERSION = '';
+  let SNAPSHOT_ID = '';
+  let DESTINATION = '';
+
+  const now = new Date();
+
+  // format: yyyyMMdd-HHmmss
+  const date = [
+    now.getFullYear(),
+    (now.getMonth() + 1).toString().padStart(2, '0'),
+    now.getDate().toString().padStart(2, '0'),
+    '-',
+    now.getHours().toString().padStart(2, '0'),
+    now.getMinutes().toString().padStart(2, '0'),
+    now.getSeconds().toString().padStart(2, '0'),
+  ].join('');
+
+  try {
+    const files = fs.readdirSync(destination);
+    const manifestEntries = files
+      .filter((filename) => !filename.match(/.sha512$/))
+      .filter((filename) => !filename.match(/.json$/))
+      .map((filename) => {
+        const parts = filename.replace('elasticsearch-oss', 'oss').split('-');
+
+        VERSION = VERSION || parts[1];
+        SNAPSHOT_ID = SNAPSHOT_ID || `${date}_${GIT_COMMIT_SHORT}`;
+        DESTINATION = DESTINATION || `${VERSION}/archives/${SNAPSHOT_ID}`;
+
+        return {
+          filename: filename,
+          checksum: filename + '.sha512',
+          url: `https://storage.googleapis.com/${BASE_BUCKET_DAILY}/${DESTINATION}/${filename}`,
+          version: parts[1],
+          platform: parts[3],
+          architecture: parts[4].split('.')[0],
+          license: parts[0] == 'oss' ? 'oss' : 'default',
+        };
+      });
+
+    const manifest = {
+      id: SNAPSHOT_ID,
+      bucket: `${BASE_BUCKET_DAILY}/${DESTINATION}`.toString(),
+      branch: ES_BRANCH,
+      sha: GIT_COMMIT,
+      sha_short: GIT_COMMIT_SHORT,
+      version: VERSION,
+      generated: now.toISOString(),
+      archives: manifestEntries,
+    };
+
+    const manifestJSON = JSON.stringify(manifest, null, 2);
+    fs.writeFileSync(`${destination}/manifest.json`, manifestJSON);
+
+    console.log('Manifest:', manifestJSON);
+
+    execSync(
+      `
+      set -euo pipefail
+
+      echo '--- Upload files to GCS'
+      cd "${destination}"
+      gsutil -m cp -r *.* gs://${BASE_BUCKET_DAILY}/${DESTINATION}
+      cp manifest.json manifest-latest.json
+      gsutil cp manifest-latest.json gs://${BASE_BUCKET_DAILY}/${VERSION}
+
+      buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST 'https://storage.googleapis.com/${BASE_BUCKET_DAILY}/${DESTINATION}/manifest.json'
+      buildkite-agent meta-data set ES_SNAPSHOT_VERSION '${VERSION}'
+      buildkite-agent meta-data set ES_SNAPSHOT_ID '${SNAPSHOT_ID}'
+    `,
+      { shell: '/bin/bash' }
+    );
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/es_snapshots/promote.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export ES_SNAPSHOT_MANIFEST="${ES_SNAPSHOT_MANIFEST:-"$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)"}"
+
+cat << EOF | buildkite-agent annotate --style "info"
+  This promotion is for the following snapshot manifest:
+
+  $ES_SNAPSHOT_MANIFEST
+EOF
+
+node "$(dirname "${0}")/promote_manifest.js" "$ES_SNAPSHOT_MANIFEST"

.buildkite/scripts/steps/es_snapshots/promote_manifest.js

@@ -0,0 +1,46 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+const { BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT } = require('./bucket_config.js');
+
+(async () => {
+  try {
+    const MANIFEST_URL = process.argv[2];
+
+    if (!MANIFEST_URL) {
+      throw Error('Manifest URL missing');
+    }
+
+    const tempDir = fs.mkdtempSync('snapshot-promotion');
+    process.chdir(tempDir);
+
+    execSync(`curl '${MANIFEST_URL}' > manifest.json`);
+
+    const manifestJson = fs.readFileSync('manifest.json').toString();
+    const manifest = JSON.parse(manifestJson);
+    const { id, bucket, version } = manifest;
+
+    const manifestPermanentJson = manifestJson
+      .split(BASE_BUCKET_DAILY)
+      .join(BASE_BUCKET_PERMANENT)
+      .split(`${version}/archives/${id}`)
+      .join(version); // e.g. replaceAll
+
+    fs.writeFileSync('manifest-permanent.json', manifestPermanentJson);
+
+    execSync(
+      `
+      set -euo pipefail
+      cp manifest.json manifest-latest-verified.json
+      gsutil cp manifest-latest-verified.json gs://${BASE_BUCKET_DAILY}/${version}/
+      rm manifest.json
+      cp manifest-permanent.json manifest.json
+      gsutil -m cp -r gs://${bucket}/* gs://${BASE_BUCKET_PERMANENT}/${version}/
+      gsutil cp manifest.json gs://${BASE_BUCKET_PERMANENT}/${version}/
+    `,
+      { shell: '/bin/bash' }
+    );
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/es_snapshots/trigger_promote.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# If ES_SNAPSHOT_MANIFEST is set dynamically during the verify job, rather than provided during the trigger,
+# such as if you provide it as input during a manual build,
+# the ES_SNAPSHOT_MANIFEST env var will be empty in the context of the pipeline.
+# So, we'll trigger with a script instead, so that we can ensure ES_SNAPSHOT_MANIFEST is populated.
+
+export ES_SNAPSHOT_MANIFEST="${ES_SNAPSHOT_MANIFEST:-"$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)"}"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - trigger: 'kibana-elasticsearch-snapshot-promote'
+    async: true
+    build:
+      env:
+        ES_SNAPSHOT_MANIFEST: '$ES_SNAPSHOT_MANIFEST'
+      branch: '$BUILDKITE_BRANCH'
+EOF

.buildkite/scripts/steps/functional/oss-cigroup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
+export JOB=kibana-oss-ciGroup${CI_GROUP}
+
+echo "--- OSS CI Group $CI_GROUP"
+
+node scripts/functional_tests \
+  --bail \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+  --include-tag "ciGroup$CI_GROUP"

.buildkite/scripts/steps/functional/xpack-cigroup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
+export JOB=kibana-default-ciGroup${CI_GROUP}
+
+echo "--- Default CI Group $CI_GROUP"
+
+cd "$XPACK_DIR"
+
+node scripts/functional_tests \
+  --bail \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+  --include-tag "ciGroup$CI_GROUP"

.buildkite/scripts/steps/test/api_integration.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- API Integration Tests'
+node scripts/functional_tests \
+  --config test/api_integration/config.js \
+  --bail \
+  --debug

.buildkite/scripts/steps/test/jest.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest'
+node scripts/jest --ci --verbose --maxWorkers=13

.buildkite/scripts/steps/test/jest_integration.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest Integration Tests'
+node scripts/jest_integration --ci --verbose