maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

chore(NA): build bazel projects all at once in the distributable build process (#90328)

Browse source 
* chore(NA): build bazel projects all at once in the distributable build process

* chore(NA): make sure bazelisk is installed

* chore(NA): install bazelisk using npm

* chore(NA): remove extra spac

* chore(NA): test yarn path exports

* chore(NA): add direct global dir

* chore(NA): some more debug steps

* chore(NA): remove one statement

* chore(NA): comment one more line out for testing purposes

* chore(NA): export the correct yarn bin location into the PATH

* chore(NA): cleaning implementation

* chore(NA): move installation process of bazelisk into npm

* chore(NA): add missing type
This commit is contained in:
Tiago Costa 2021-02-05 21:55:09 +00:00 committed by GitHub
parent fc516bacbd
commit feda8a0785
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 76 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
packages/kbn-pm/dist/index.js vendored
View file

@ -48106,23 +48106,34 @@ async function isBazelBinAvailable() {
}
}
async function isBazeliskInstalled(bazeliskVersion) {
try {
const {
stdout: bazeliskPkgInstallStdout
} = await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('npm', ['ls', '--global', '--parseable', '--long', `@bazel/bazelisk@${bazeliskVersion}`], {
stdio: 'pipe'
});
return bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`);
} catch {
return false;
}
}
async function installBazelTools(repoRootPath) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] reading bazel tools versions from version files`);
const bazeliskVersion = await readBazelToolsVersionFile(repoRootPath, '.bazeliskversion');
const bazelVersion = await readBazelToolsVersionFile(repoRootPath, '.bazelversion'); // Check what globals are installed
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`);
const {
stdout: bazeliskPkgInstallStdout
} = await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('yarn', ['global', 'list'], {
stdio: 'pipe'
});
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`); // Test if bazelisk is already installed in the correct version
const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion); // Test if bazel bin is available
const isBazelBinAlreadyAvailable = await isBazelBinAvailable(); // Install bazelisk if not installed
if (!bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`) || !isBazelBinAlreadyAvailable) {
if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] installing Bazel tools`);
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] bazelisk is not installed. Installing @bazel/bazelisk@${bazeliskVersion} and bazel@${bazelVersion}`);
await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('yarn', ['global', 'add', `@bazel/bazelisk@${bazeliskVersion}`], {
await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('npm', ['install', '--global', `@bazel/bazelisk@${bazeliskVersion}`], {
env: {
USE_BAZEL_VERSION: bazelVersion
},
@ -48132,7 +48143,7 @@ async function installBazelTools(repoRootPath) {
if (!isBazelBinAvailableAfterInstall) {
throw new Error(dedent__WEBPACK_IMPORTED_MODULE_0___default.a`
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure 'yarn global bin' is on your $PATH, otherwise just add it there
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure you have access to npm globally installed modules on your $PATH
`);
}
}
@ -59771,10 +59782,11 @@ __webpack_require__.r(__webpack_exports__);
/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(4);
/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__);
/* harmony import */ var _build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(745);
/* harmony import */ var _utils_fs__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(131);
/* harmony import */ var _utils_log__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(246);
/* harmony import */ var _utils_package_json__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(251);
/* harmony import */ var _utils_projects__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(248);
/* harmony import */ var _utils_bazel_run__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(374);
/* harmony import */ var _utils_fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(131);
/* harmony import */ var _utils_log__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(246);
/* harmony import */ var _utils_package_json__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(251);
/* harmony import */ var _utils_projects__WEBPACK_IMPORTED_MODULE_8__ = __webpack_require__(248);
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
@ -59790,17 +59802,19 @@ __webpack_require__.r(__webpack_exports__);
async function buildBazelProductionProjects({
kibanaRoot,
buildRoot,
onlyOSS
}) {
const projects = await Object(_utils_projects__WEBPACK_IMPORTED_MODULE_7__["getBazelProjectsOnly"])(await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["getProductionProjects"])(kibanaRoot, onlyOSS));
const projects = await Object(_utils_projects__WEBPACK_IMPORTED_MODULE_8__["getBazelProjectsOnly"])(await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["getProductionProjects"])(kibanaRoot, onlyOSS));
const projectNames = [...projects.values()].map(project => project.name);
_utils_log__WEBPACK_IMPORTED_MODULE_5__["log"].info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
_utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
await Object(_utils_bazel_run__WEBPACK_IMPORTED_MODULE_4__["runBazel"])(['build', '//packages:build']);
_utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`);
for (const project of projects.values()) {
await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["buildProject"])(project);
await copyToBuild(project, kibanaRoot, buildRoot);
await applyCorrectPermissions(project, kibanaRoot, buildRoot);
}
@ -59835,9 +59849,9 @@ async function copyToBuild(project, kibanaRoot, buildRoot) {
// the intermediate build, we fall back to using the project's already defined
// `package.json`.
const packageJson = (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isFile"])(Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(buildProjectPath, 'package.json'))) ? await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["readPackageJson"])(buildProjectPath) : project.json;
const preparedPackageJson = Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["createProductionPackageJson"])(packageJson);
await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["writePackageJson"])(buildProjectPath, preparedPackageJson);
const packageJson = (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(buildProjectPath, 'package.json'))) ? await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["readPackageJson"])(buildProjectPath) : project.json;
const preparedPackageJson = Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["createProductionPackageJson"])(packageJson);
await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["writePackageJson"])(buildProjectPath, preparedPackageJson);
}
async function applyCorrectPermissions(project, kibanaRoot, buildRoot) {
@ -59852,12 +59866,12 @@ async function applyCorrectPermissions(project, kibanaRoot, buildRoot) {
for (const pluginPath of allPluginPaths) {
const resolvedPluginPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildRoot, pluginPath);
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isFile"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["chmod"])(resolvedPluginPath, 0o644);
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["chmod"])(resolvedPluginPath, 0o644);
}
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isDirectory"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["chmod"])(resolvedPluginPath, 0o755);
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isDirectory"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["chmod"])(resolvedPluginPath, 0o755);
}
}
}

7
packages/kbn-pm/src/production/build_bazel_production_projects.ts
View file

@ -10,7 +10,8 @@ import copy from 'cpy';
import globby from 'globby';
import { basename, join, relative, resolve } from 'path';
import { buildProject, getProductionProjects } from './build_non_bazel_production_projects';
import { getProductionProjects } from './build_non_bazel_production_projects';
import { runBazel } from '../utils/bazel/run';
import { chmod, isFile, isDirectory } from '../utils/fs';
import { log } from '../utils/log';
import {
@ -35,8 +36,10 @@ export async function buildBazelProductionProjects({
const projectNames = [...projects.values()].map((project) => project.name);
log.info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
await runBazel(['build', '//packages:build']);
log.info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`);
for (const project of projects.values()) {
await buildProject(project);
await copyToBuild(project, kibanaRoot, buildRoot);
await applyCorrectPermissions(project, kibanaRoot, buildRoot);
}

32
packages/kbn-pm/src/utils/bazel/install_tools.ts
View file

@ -36,6 +36,22 @@ async function isBazelBinAvailable() {
}
}
async function isBazeliskInstalled(bazeliskVersion: string) {
try {
const { stdout: bazeliskPkgInstallStdout } = await spawn(
'npm',
['ls', '--global', '--parseable', '--long', `@bazel/bazelisk@${bazeliskVersion}`],
{
stdio: 'pipe',
}
);
return bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`);
} catch {
return false;
}
}
export async function installBazelTools(repoRootPath: string) {
log.debug(`[bazel_tools] reading bazel tools versions from version files`);
const bazeliskVersion = await readBazelToolsVersionFile(repoRootPath, '.bazeliskversion');
@ -43,23 +59,21 @@ export async function installBazelTools(repoRootPath: string) {
// Check what globals are installed
log.debug(`[bazel_tools] verify if bazelisk is installed`);
const { stdout: bazeliskPkgInstallStdout } = await spawn('yarn', ['global', 'list'], {
stdio: 'pipe',
});
// Test if bazelisk is already installed in the correct version
const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion);
// Test if bazel bin is available
const isBazelBinAlreadyAvailable = await isBazelBinAvailable();
// Install bazelisk if not installed
if (
!bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`) ||
!isBazelBinAlreadyAvailable
) {
if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) {
log.info(`[bazel_tools] installing Bazel tools`);
log.debug(
`[bazel_tools] bazelisk is not installed. Installing @bazel/bazelisk@${bazeliskVersion} and bazel@${bazelVersion}`
);
await spawn('yarn', ['global', 'add', `@bazel/bazelisk@${bazeliskVersion}`], {
await spawn('npm', ['install', '--global', `@bazel/bazelisk@${bazeliskVersion}`], {
env: {
USE_BAZEL_VERSION: bazelVersion,
},
@ -69,7 +83,7 @@ export async function installBazelTools(repoRootPath: string) {
const isBazelBinAvailableAfterInstall = await isBazelBinAvailable();
if (!isBazelBinAvailableAfterInstall) {
throw new Error(dedent`
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure 'yarn global bin' is on your $PATH, otherwise just add it there
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure you have access to npm globally installed modules on your $PATH
`);
}
}

5
src/dev/ci_setup/setup.sh
View file

@ -65,8 +65,3 @@ if [ "$GIT_CHANGES" ]; then
echo -e "$GIT_CHANGES\n"
exit 1
fi
###
### copy .bazelrc-ci into $HOME/.bazelrc
###
cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc";

11
src/dev/ci_setup/setup_env.sh
View file

@ -175,4 +175,15 @@ if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then
export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
fi
###
### copy .bazelrc-ci into $HOME/.bazelrc
###
cp -f "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc";
###
### append auth token to buildbuddy into "$HOME/.bazelrc";
###
echo "# Appended by $KIBANA_DIR/src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc"
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"
export CI_ENV_SETUP=true