From 3ad7814934abcd1f814611f953fe1c321ba17fff Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 6 Feb 2021 01:33:08 +0000 Subject: [PATCH] chore(NA): integrate build buddy with our bazel setup and remote cache for ci (#90116) (#90534) * chore(NA): simple changes on bazelrc * chore(NA): integrate bazel tools with BuildBuddy and remote cache service * chore(NA) fix bazelrc line config * chore(NA): move non auth settings out of bazelrc.auth * chore(NA): output home dir * chore(NA): load .bazelrc-ci.auth from /Users/tiagocosta dir * chore(NA): remove bazelrc auth file and append directly into home bazelrc * chore(NA): comment announce option * chore(NA): integrate build buddy metadata * chore(NA): update src/dev/ci_setup/.bazelrc-ci Co-authored-by: Tyler Smalley * chore(NA): move build metadata integation to common confdig * chore(NA): fix problem on bazel file location * chore(NA): correct sh file permissions * chore(NA): only get host on CI * chore(NA): add cores into host info on CI * chore(NA): sync with last settings to setup bazelisk tools on ci * chore(NA): sync last changes on ci setup env * chore(NA): sync settings on ci setup with the other PR * chore(NA): remove yarn export Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tyler Smalley Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tyler Smalley --- .bazelrc | 9 +++++ src/dev/bazel_workspace_status.sh | 57 +++++++++++++++++++++++++++++ src/dev/ci_setup/.bazelrc-ci | 12 ++++-- src/dev/ci_setup/.bazelrc-ci.common | 3 -- src/dev/ci_setup/load_env_keys.sh | 3 ++ src/dev/ci_setup/setup.sh | 11 ++++++ 6 files changed, 89 insertions(+), 6 deletions(-) create mode 100755 src/dev/bazel_workspace_status.sh diff --git a/.bazelrc b/.bazelrc index 741067e4ff18..158338ec5f09 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,8 +2,17 @@ # Import shared settings first so we can override below import %workspace%/.bazelrc.common +## Disabled for now # Remote cache settings for local env # build --remote_cache=https://storage.googleapis.com/kibana-bazel-cache # build --incompatible_remote_results_ignore_disk=true # build --remote_accept_cached=true # build --remote_upload_local_results=false + +# BuildBuddy +## Metadata settings +build --workspace_status_command=$(pwd)/src/dev/bazel_workspace_status.sh +# Enable this in case you want to share your build info +# build --build_metadata=VISIBILITY=PUBLIC +build --build_metadata=TEST_GROUPS=//packages + diff --git a/src/dev/bazel_workspace_status.sh b/src/dev/bazel_workspace_status.sh new file mode 100755 index 000000000000..efaca4bb9884 --- /dev/null +++ b/src/dev/bazel_workspace_status.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Inspired on https://github.com/buildbuddy-io/buildbuddy/blob/master/workspace_status.sh +# This script will be run bazel when building process starts to +# generate key-value information that represents the status of the +# workspace. The output should be like +# +# KEY1 VALUE1 +# KEY2 VALUE2 +# +# If the script exits with non-zero code, it's considered as a failure +# and the output will be discarded. + +# Git repo +repo_url=$(git config --get remote.origin.url) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "REPO_URL ${repo_url}" + +# Commit SHA +commit_sha=$(git rev-parse HEAD) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "COMMIT_SHA ${commit_sha}" + +# Git branch +repo_url=$(git rev-parse --abbrev-ref HEAD) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "GIT_BRANCH ${repo_url}" + +# Tree status +git diff-index --quiet HEAD -- +if [[ $? == 0 ]]; +then + tree_status="Clean" +else + tree_status="Modified" +fi +echo "GIT_TREE_STATUS ${tree_status}" + +# Host +if [ "$CI" = "true" ]; then + host=$(hostname | sed 's|\(.*\)-.*|\1|') + cores=$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}' ) + if [[ $? != 0 ]]; + then + exit 1 + fi + echo "HOST ${host}-${cores}" +fi diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index 5b345d3c9e20..ef6fab3a3059 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -5,6 +5,12 @@ # Import and load bazelrc common settings for ci env try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common -# Remote cache settings for ci env -# build --google_default_credentials -# build --remote_upload_local_results=true +# BuildBuddy settings +## Remote settings including cache +build --bes_results_url=https://app.buildbuddy.io/invocation/ +build --bes_backend=grpcs://cloud.buildbuddy.io +build --remote_cache=grpcs://cloud.buildbuddy.io +build --remote_timeout=3600 + +## Metadata settings +build --build_metadata=ROLE=CI diff --git a/src/dev/ci_setup/.bazelrc-ci.common b/src/dev/ci_setup/.bazelrc-ci.common index 3f58e4e03a17..9d00ee563974 100644 --- a/src/dev/ci_setup/.bazelrc-ci.common +++ b/src/dev/ci_setup/.bazelrc-ci.common @@ -4,8 +4,5 @@ # Don't be spammy in the logs build --noshow_progress -# Print all the options that apply to the build. -build --announce_rc - # More details on failures build --verbose_failures=true diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh index 62d29db232ea..5f7a6c26bab2 100644 --- a/src/dev/ci_setup/load_env_keys.sh +++ b/src/dev/ci_setup/load_env_keys.sh @@ -34,6 +34,9 @@ else PERCY_TOKEN=$(retry 5 vault read -field=value secret/kibana-issues/dev/percy) export PERCY_TOKEN + KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) + export KIBANA_BUILDBUDDY_CI_API_KEY + # remove vault related secrets unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR fi diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index 61f578ba3397..0b24f0b22b81 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -10,6 +10,17 @@ echo " -- PARENT_DIR='$PARENT_DIR'" echo " -- KIBANA_PKG_BRANCH='$KIBANA_PKG_BRANCH'" echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'" +### +### copy .bazelrc-ci into $HOME/.bazelrc +### +cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; + +### +### append auth token to buildbuddy into "$HOME/.bazelrc"; +### +echo "# Appended by src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc" +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" + ### ### install dependencies ###