chore(NA): manage npm dependencies within bazel (#92864)
* chore(NA): full WORKSPACE.bazel logic plus manage yarn dependencies with Bazel * chore(NA): update BUILD.bazel files comments on root and packages * chore(NA): add workspace file with useful data * chore(NA): install deps through bazel * chore(NA): update workspace file * chore(NA): update into last rules nodejs * chore(NA): ensure bazel always run yarn install * chore(NA): support offline mode * chore(NA): remove elastic-datemath * chore(NA): restore bazel 4.0.0 * chore(NA): update kbn pm dist * chore(NA): introduce force-install command * docs(NA): update docs with new yarn kbn bootstrap flags * chore(NA): use path.resolve on kbn bootstrap integrity check verification * chore(NA): update .yarnrc Co-authored-by: Tyler Smalley <tylersmalley@me.com> * chore(NA): change cli argument typo * chore(NA): fix spacing on kbn pm cli Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tyler Smalley <tylersmalley@me.com>
This commit is contained in:
Tiago Costa
GitHub
parent
a8d45e7457
commit
86f1684076
9
.bazelrc
9
.bazelrc
|
|
@ -16,3 +16,12 @@ build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
|
|||
# build --build_metadata=VISIBILITY=PUBLIC
|
||||
build --build_metadata=TEST_GROUPS=//packages
|
||||
|
||||
###############################
|
||||
# Offline Support #
|
||||
# Turn on these settings with #
|
||||
# --config=offline #
|
||||
###############################
|
||||
|
||||
## Reset remote cache and backend support
|
||||
build:offline --bes_backend="" --remote_cache=""
|
||||
run:offline --bes_backend="" --remote_cache=""
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
|
||||
# Local Cache Settings
|
||||
## Avoid cache results from being corrupt when changing source during build
|
||||
common --experimental_guard_against_concurrent_changes
|
||||
build --experimental_guard_against_concurrent_changes
|
||||
run --experimental_guard_against_concurrent_changes
|
||||
test --experimental_guard_against_concurrent_changes
|
||||
|
||||
## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
|
||||
build --disk_cache=~/.bazel-cache/disk-cache
|
||||
|
|
|
|||
6
.yarnrc
6
.yarnrc
|
|
@ -3,3 +3,9 @@ yarn-offline-mirror ".yarn-local-mirror"
|
|||
|
||||
# Always look into the cache first before fetching online
|
||||
--install.prefer-offline true
|
||||
|
||||
# Disable interactive and progress logs as yarn install is now
|
||||
# managed by Bazel and we are piping the logs from the underlying
|
||||
# process running bazel into the parent one running kbn
|
||||
--install.non-interactive true
|
||||
--install.no-progress true
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# Expose those targets as they are required as part of
|
||||
# other packages builds and need to be included as inputs
|
||||
exports_files(
|
||||
[
|
||||
"tsconfig.json",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,68 @@
|
|||
# Define the workspace base name and a managed directory by bazel
|
||||
# that will hold the node_modules called @npm
|
||||
workspace(
|
||||
name = "kibana",
|
||||
name = "kibana",
|
||||
managed_directories = {"@npm": ["node_modules"]},
|
||||
)
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
# Fetch Node.js rules
|
||||
http_archive(
|
||||
name = "build_bazel_rules_nodejs",
|
||||
sha256 = "bfacf15161d96a6a39510e7b3d3b522cf61cb8b82a31e79400a84c5abcab5347",
|
||||
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.2.1/rules_nodejs-3.2.1.tar.gz"],
|
||||
)
|
||||
|
||||
# Now that we have the rules let's import from them to complete the work
|
||||
load("@build_bazel_rules_nodejs//:index.bzl", "check_rules_nodejs_version", "node_repositories", "yarn_install")
|
||||
|
||||
# Assure we have at least a given rules_nodejs version
|
||||
check_rules_nodejs_version(minimum_version_string = "3.2.1")
|
||||
|
||||
# Setup the Node.js toolchain for the architectures we want to support
|
||||
#
|
||||
# NOTE: darwin-arm64 is not being installed because bazel is not yet available on that architecture.
|
||||
# The PR for it was merged and should be available in the next release of bazel and bazelisk. As soon as they have it
|
||||
# we can update that rule.
|
||||
node_repositories(
|
||||
node_repositories = {
|
||||
"14.16.0-darwin_amd64": ("node-v14.16.0-darwin-x64.tar.gz", "node-v14.16.0-darwin-x64", "14ec767e376d1e2e668f997065926c5c0086ec46516d1d45918af8ae05bd4583"),
|
||||
"14.16.0-linux_arm64": ("node-v14.16.0-linux-arm64.tar.xz", "node-v14.16.0-linux-arm64", "440489a08bfd020e814c9e65017f58d692299ac3f150c8e78d01abb1104c878a"),
|
||||
"14.16.0-linux_s390x": ("node-v14.16.0-linux-s390x.tar.xz", "node-v14.16.0-linux-s390x", "335348e46f45284b6356416ef58f85602d2dee99094588b65900f6c8839df77e"),
|
||||
"14.16.0-linux_amd64": ("node-v14.16.0-linux-x64.tar.xz", "node-v14.16.0-linux-x64", "2e079cf638766fedd720d30ec8ffef5d6ceada4e8b441fc2a093cb9a865f4087"),
|
||||
"14.16.0-windows_amd64": ("node-v14.16.0-win-x64.zip", "node-v14.16.0-win-x64", "716045c2f16ea10ca97bd04cf2e5ef865f9c4d6d677a9bc25e2ea522b594af4f"),
|
||||
},
|
||||
node_version = "14.16.0",
|
||||
node_urls = [
|
||||
"https://nodejs.org/dist/v{version}/{filename}",
|
||||
],
|
||||
yarn_repositories = {
|
||||
"1.21.1": ("yarn-v1.21.1.tar.gz", "yarn-v1.21.1", "d1d9f4a0f16f5ed484e814afeb98f39b82d4728c6c8beaafb5abc99c02db6674"),
|
||||
},
|
||||
yarn_version = "1.21.1",
|
||||
yarn_urls = [
|
||||
"https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}",
|
||||
],
|
||||
package_json = ["//:package.json"],
|
||||
)
|
||||
|
||||
# Run yarn_install rule to take care of dependencies
|
||||
#
|
||||
# NOTE: FORCE_COLOR env var forces colors on non tty mode
|
||||
yarn_install(
|
||||
name = "npm",
|
||||
environment = {
|
||||
"FORCE_COLOR": "True",
|
||||
},
|
||||
package_json = "//:package.json",
|
||||
yarn_lock = "//:yarn.lock",
|
||||
data = [
|
||||
"//:.yarnrc",
|
||||
"//:preinstall_check.js",
|
||||
"//:node_modules/.yarn-integrity",
|
||||
],
|
||||
symlink_node_modules = True,
|
||||
quiet = False,
|
||||
frozen_lockfile = False,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -47,6 +47,23 @@ https://github.com/nodejs/node-gyp#installation[https://github.com/nodejs/node-g
|
|||
and follow the guide according your platform.
|
||||
____
|
||||
|
||||
In case you don't have an internet connection, the `yarn kbn bootstrap` command will
|
||||
fail. As it is likely you have the required node_modules in the
|
||||
offline mirror, you can try to run the step in offline mode by using:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
yarn kbn bootstrap --offline
|
||||
----
|
||||
|
||||
In any other circumstance where you want to force the node_modules install step
|
||||
you can use:
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
yarn kbn bootstrap --force-install
|
||||
----
|
||||
|
||||
(You can also run `yarn kbn` to see the other available commands. For
|
||||
more info about this tool, see
|
||||
{kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}/packages/kbn-pm].)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# Call each package final target
|
||||
# Grouping target to call all underlying packages build
|
||||
# targets so we can build them all at once
|
||||
filegroup(
|
||||
name = "build",
|
||||
srcs = [],
|
||||
|
|
|
|||
1999
packages/kbn-pm/dist/index.js
vendored
1999
packages/kbn-pm/dist/index.js
vendored
File diff suppressed because one or more lines are too long
|
|
@ -37,6 +37,8 @@ function help() {
|
|||
--skip-kibana-plugins Filter all plugins in ./plugins and ../kibana-extra when running command.
|
||||
--no-cache Disable the kbn packages bootstrap cache
|
||||
--no-validate Disable the bootstrap yarn.lock validation
|
||||
--force-install Forces yarn install to run on bootstrap
|
||||
--offline Run in offline mode
|
||||
--verbose Set log level to verbose
|
||||
--debug Set log level to debug
|
||||
--quiet Set log level to error
|
||||
|
|
@ -73,9 +75,11 @@ export async function run(argv: string[]) {
|
|||
},
|
||||
default: {
|
||||
cache: true,
|
||||
'force-install': true,
|
||||
offline: false,
|
||||
validate: true,
|
||||
},
|
||||
boolean: ['cache', 'validate'],
|
||||
boolean: ['cache', 'force-install', 'offline', 'validate'],
|
||||
});
|
||||
|
||||
const args = options._;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { sep } from 'path';
|
||||
import { resolve, sep } from 'path';
|
||||
import { linkProjectExecutables } from '../utils/link_project_executables';
|
||||
import { log } from '../utils/log';
|
||||
import { parallelizeBatches } from '../utils/parallelize';
|
||||
|
|
@ -17,7 +17,7 @@ import { getAllChecksums } from '../utils/project_checksums';
|
|||
import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
|
||||
import { readYarnLock } from '../utils/yarn_lock';
|
||||
import { validateDependencies } from '../utils/validate_dependencies';
|
||||
import { installBazelTools, runBazel } from '../utils/bazel';
|
||||
import { ensureYarnIntegrityFileExists, installBazelTools, runBazel } from '../utils/bazel';
|
||||
|
||||
export const BootstrapCommand: ICommand = {
|
||||
description: 'Install dependencies and crosslink projects',
|
||||
|
|
@ -26,12 +26,36 @@ export const BootstrapCommand: ICommand = {
|
|||
async run(projects, projectGraph, { options, kbn, rootPath }) {
|
||||
const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
|
||||
const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
|
||||
const kibanaProjectPath = projects.get('kibana')?.path;
|
||||
const kibanaProjectPath = projects.get('kibana')?.path || '';
|
||||
const runOffline = options?.offline === true;
|
||||
const forceInstall = !!options && options['force-install'] === true;
|
||||
|
||||
// Ensure we have a `node_modules/.yarn-integrity` file as we depend on it
|
||||
// for bazel to know it has to re-install the node_modules after a reset or a clean
|
||||
await ensureYarnIntegrityFileExists(resolve(kibanaProjectPath, 'node_modules'));
|
||||
|
||||
// Install bazel machinery tools if needed
|
||||
await installBazelTools(rootPath);
|
||||
|
||||
// Install monorepo npm dependencies
|
||||
// Bootstrap process for Bazel packages
|
||||
// Bazel is now managing dependencies so yarn install
|
||||
// will happen as part of this
|
||||
//
|
||||
// NOTE: Bazel projects will be introduced incrementally
|
||||
// And should begin from the ones with none dependencies forward.
|
||||
// That way non bazel projects could depend on bazel projects but not the other way around
|
||||
// That is only intended during the migration process while non Bazel projects are not removed at all.
|
||||
//
|
||||
// Until we have our first package build within Bazel we will always need to directly call the yarn rule
|
||||
// otherwise yarn install won't trigger as we don't have any npm dependency within Bazel
|
||||
// TODO: Change CLI default in order to not force install as soon as we have our first Bazel package being built
|
||||
if (forceInstall) {
|
||||
await runBazel(['run', '@nodejs//:yarn'], runOffline);
|
||||
}
|
||||
|
||||
await runBazel(['build', '//packages:build'], runOffline);
|
||||
|
||||
// Install monorepo npm dependencies outside of the Bazel managed ones
|
||||
for (const batch of batchedNonBazelProjects) {
|
||||
for (const project of batch) {
|
||||
const isExternalPlugin = project.path.includes(`${kibanaProjectPath}${sep}plugins`);
|
||||
|
|
@ -40,12 +64,16 @@ export const BootstrapCommand: ICommand = {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (project.isSinglePackageJsonProject || isExternalPlugin) {
|
||||
if (isExternalPlugin) {
|
||||
await project.installDependencies();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!project.isEveryDependencyLocal() && !isExternalPlugin) {
|
||||
if (
|
||||
!project.isSinglePackageJsonProject &&
|
||||
!project.isEveryDependencyLocal() &&
|
||||
!isExternalPlugin
|
||||
) {
|
||||
throw new Error(
|
||||
`[${project.name}] is not eligible to hold non local dependencies. Move the non local dependencies into the top level package.json.`
|
||||
);
|
||||
|
|
@ -61,15 +89,9 @@ export const BootstrapCommand: ICommand = {
|
|||
|
||||
// Assure all kbn projects with bin defined scripts
|
||||
// copy those scripts into the top level node_modules folder
|
||||
await linkProjectExecutables(projects, projectGraph);
|
||||
|
||||
// Bootstrap process for Bazel packages
|
||||
//
|
||||
// NOTE: Bazel projects will be introduced incrementally
|
||||
// And should begin from the ones with none dependencies forward.
|
||||
// That way non bazel projects could depend on bazel projects but not the other way around
|
||||
// That is only intended during the migration process while non Bazel projects are not removed at all.
|
||||
await runBazel(['build', '//packages:build']);
|
||||
// NOTE: We don't probably need this anymore, is actually not being used
|
||||
await linkProjectExecutables(projects, projectGraph);
|
||||
|
||||
// Bootstrap process for non Bazel packages
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { join } from 'path';
|
||||
import { writeFile } from '../fs';
|
||||
|
||||
export async function ensureYarnIntegrityFileExists(nodeModulesPath: string) {
|
||||
try {
|
||||
await writeFile(join(nodeModulesPath, '.yarn-integrity'), '', { flag: 'wx' });
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export * from './ensure_yarn_integrity_exists';
|
||||
export * from './get_cache_folders';
|
||||
export * from './install_tools';
|
||||
export * from './run';
|
||||
|
|
|
|||
|
|
@ -14,13 +14,21 @@ import { observeLines } from '@kbn/dev-utils/stdio';
|
|||
import { spawn } from '../child_process';
|
||||
import { log } from '../log';
|
||||
|
||||
export async function runBazel(bazelArgs: string[], runOpts: execa.Options = {}) {
|
||||
export async function runBazel(
|
||||
bazelArgs: string[],
|
||||
offline: boolean = false,
|
||||
runOpts: execa.Options = {}
|
||||
) {
|
||||
// Force logs to pipe in order to control the output of them
|
||||
const bazelOpts: execa.Options = {
|
||||
...runOpts,
|
||||
stdio: 'pipe',
|
||||
};
|
||||
|
||||
if (offline) {
|
||||
bazelArgs.push('--config=offline');
|
||||
}
|
||||
|
||||
const bazelProc = spawn('bazel', bazelArgs, bazelOpts);
|
||||
|
||||
const bazelLogs$ = new Rx.Subject<string>();
|
||||
|
|
|
|||
Loading…
Reference in a new issue