chore(NA): introduce new yarn kbn reset command to support bazel workflow (#89597)

* chore(NA): introduce new yarn kbn destroy command to support bazel workflow

* chore(NA): update wording for new kbn pm changes

* chore(NA): update .bazelrc.common

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update .bazelrc

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/destroy.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/destroy.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): rename destroy command into reset

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): restore old behaviour on kbn clean

* chore(NA): update reset command to delete bazel caches on disk

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update prefix to match bazel defined one for cache settings

* chore(NA): update kbn pm dist file

* chore(NA): update kbn pm dist

* chore(NA): update gitignore

* chore(NA): add new ignore files after changed bazel aggregated folder

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tyler Smalley <tylersmalley@me.com>
This commit is contained in:
Tiago Costa 2021-02-03 22:02:34 +00:00 committed by GitHub
parent b7b9ca75d7
commit a0d5d5614d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 7257 additions and 6736 deletions

View file

@ -8,8 +8,7 @@
.idea
.teamcity
.yarn-local-mirror
bazel-cache
bazel-dist
/bazel
build
node_modules
target

View file

@ -1,4 +1,4 @@
# Inspired on from https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelrc
# Inspired by https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelrc
# Import shared settings first so we can override below
import %workspace%/.bazelrc.common

View file

@ -5,18 +5,22 @@
#
# The full list of Bazel options: https://docs.bazel.build/versions/master/command-line-reference.html
# Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
build --disk_cache=bazel-cache/disk-cache
# Local Cache Settings
## Avoid cache results from being corrupt when changing source during build
common --experimental_guard_against_concurrent_changes
# Bazel repo cache settings
build --repository_cache=bazel-cache/repository-cache
## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
build --disk_cache=~/.bazel-cache/disk-cache
## Bazel repo cache settings
build --repository_cache=~/.bazel-cache/repository-cache
# Bazel will create symlinks from the workspace directory to output artifacts.
# Build results will be placed in a directory called "bazel-dist/bin"
# Build results will be placed in a directory called "bazel/bin"
# This will still create a bazel-out symlink in
# the project directory, which must be excluded from the
# editor's search path.
build --symlink_prefix=bazel-dist/
build --symlink_prefix=bazel/
# To disable the symlinks altogether (including bazel-out) we can use
# build --symlink_prefix=/
# however this makes it harder to find outputs.
@ -37,9 +41,7 @@ common --color=yes
build --show_task_finish
build --noshow_progress
build --noshow_loading_progress
## enforced default values
build --show_result=1
build --show_result=0
# Specifies desired output mode for running tests.
# Valid values are
@ -78,7 +80,8 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999
# The node process will break before user code starts and wait for the debugger to connect.
run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk
# The following option will change the build output of certain rules such as terser and may not be desirable in all cases
build:debug --compilation_mode=dbg
# It will also output both the repo cache and action cache to a folder inside the repo
build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel/disk-cache --repository_cache=bazel/repository-cache
# Turn off legacy external runfiles
# This prevents accidentally depending on this feature, which Bazel will remove.

View file

@ -46,4 +46,4 @@ snapshots.js
/packages/kbn-monaco/src/painless/antlr
# Bazel
/bazel-*
/bazel

2
.gitignore vendored
View file

@ -78,5 +78,5 @@ report.asciidoc
.yarn-local-mirror
# Bazel
/bazel-*
/bazel
/.bazelrc.user

View file

@ -0,0 +1,3 @@
{
"main": "../target/stdio"
}

File diff suppressed because one or more lines are too long

View file

@ -6,19 +6,27 @@
* Public License, v 1.
*/
import dedent from 'dedent';
import del from 'del';
import ora from 'ora';
import { join, relative } from 'path';
import { runBazel } from '../utils/bazel';
import { isDirectory } from '../utils/fs';
import { log } from '../utils/log';
import { ICommand } from './';
export const CleanCommand: ICommand = {
description: 'Remove the node_modules and target directories from all projects.',
description: 'Deletes output directories, node_modules and resets internal caches.',
name: 'clean',
async run(projects) {
log.warning(dedent`
This command is only necessary for the rare circumstance where you need to recover a consistent
state when problems arise. If you need to run this command often, please let us know by
filling out this form: https://ela.st/yarn-kbn-clean
`);
const toDelete = [];
for (const project of projects.values()) {
if (await isDirectory(project.nodeModulesLocation)) {
@ -44,6 +52,10 @@ export const CleanCommand: ICommand = {
}
}
// Runs Bazel soft clean
await runBazel(['clean']);
log.success('Soft cleaned bazel');
if (toDelete.length === 0) {
log.success('Nothing to delete');
} else {

View file

@ -24,6 +24,7 @@ export interface ICommand {
import { BootstrapCommand } from './bootstrap';
import { CleanCommand } from './clean';
import { ResetCommand } from './reset';
import { RunCommand } from './run';
import { WatchCommand } from './watch';
import { Kibana } from '../utils/kibana';
@ -31,6 +32,7 @@ import { Kibana } from '../utils/kibana';
export const commands: { [key: string]: ICommand } = {
bootstrap: BootstrapCommand,
clean: CleanCommand,
reset: ResetCommand,
run: RunCommand,
watch: WatchCommand,
};

View file

@ -0,0 +1,94 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* and the Server Side Public License, v 1; you may not use this file except in
* compliance with, at your election, the Elastic License or the Server Side
* Public License, v 1.
*/
import dedent from 'dedent';
import del from 'del';
import ora from 'ora';
import { join, relative } from 'path';
import { getBazelDiskCacheFolder, getBazelRepositoryCacheFolder, runBazel } from '../utils/bazel';
import { isDirectory } from '../utils/fs';
import { log } from '../utils/log';
import { ICommand } from './';
export const ResetCommand: ICommand = {
description:
'Deletes node_modules and output directories, resets internal and disk caches, and stops Bazel server',
name: 'reset',
async run(projects) {
log.warning(dedent`
In most cases, 'yarn kbn clean' is all that should be needed to recover a consistent state when
problems arise. If you need to use this command, please let us know, as it should not be necessary.
`);
const toDelete = [];
for (const project of projects.values()) {
if (await isDirectory(project.nodeModulesLocation)) {
toDelete.push({
cwd: project.path,
pattern: relative(project.path, project.nodeModulesLocation),
});
}
if (await isDirectory(project.targetLocation)) {
toDelete.push({
cwd: project.path,
pattern: relative(project.path, project.targetLocation),
});
}
const { extraPatterns } = project.getCleanConfig();
if (extraPatterns) {
toDelete.push({
cwd: project.path,
pattern: extraPatterns,
});
}
}
// Runs Bazel hard clean
await runBazel(['clean', '--expunge']);
log.success('Hard cleaned bazel');
// Deletes Bazel Cache Folders
await del([await getBazelDiskCacheFolder(), await getBazelRepositoryCacheFolder()], {
force: true,
});
log.success('Removed disk caches');
if (toDelete.length === 0) {
return;
}
/**
* In order to avoid patterns like `/build` in packages from accidentally
* impacting files outside the package we use `process.chdir()` to change
* the cwd to the package and execute `del()` without the `force` option
* so it will check that each file being deleted is within the package.
*
* `del()` does support a `cwd` option, but it's only for resolving the
* patterns and does not impact the cwd check.
*/
const originalCwd = process.cwd();
try {
for (const { pattern, cwd } of toDelete) {
process.chdir(cwd);
const promise = del(pattern);
if (log.wouldLogLevel('info')) {
ora.promise(promise, relative(originalCwd, join(cwd, String(pattern))));
}
await promise;
}
} finally {
process.chdir(originalCwd);
}
},
};

View file

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* and the Server Side Public License, v 1; you may not use this file except in
* compliance with, at your election, the Elastic License or the Server Side
* Public License, v 1.
*/
import { dirname, resolve } from 'path';
import { spawn } from '../child_process';
async function rawRunBazelInfoRepoCache() {
const { stdout: bazelRepositoryCachePath } = await spawn('bazel', ['info', 'repository_cache'], {
stdio: 'pipe',
});
return bazelRepositoryCachePath;
}
export async function getBazelDiskCacheFolder() {
return resolve(dirname(await rawRunBazelInfoRepoCache()), 'disk-cache');
}
export async function getBazelRepositoryCacheFolder() {
return await rawRunBazelInfoRepoCache();
}

View file

@ -6,4 +6,6 @@
* Public License, v 1.
*/
export * from './get_cache_folders';
export * from './install_tools';
export * from './run';

View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* and the Server Side Public License, v 1; you may not use this file except in
* compliance with, at your election, the Elastic License or the Server Side
* Public License, v 1.
*/
import chalk from 'chalk';
import execa from 'execa';
import * as Rx from 'rxjs';
import { tap } from 'rxjs/operators';
import { observeLines } from '@kbn/dev-utils/stdio';
import { spawn } from '../child_process';
import { log } from '../log';
export async function runBazel(bazelArgs: string[], runOpts: execa.Options = {}) {
// Force logs to pipe in order to control the output of them
const bazelOpts: execa.Options = {
...runOpts,
stdio: 'pipe',
};
const bazelProc = spawn('bazel', bazelArgs, bazelOpts);
const bazelLogs$ = new Rx.Subject<string>();
// Bazel outputs machine readable output into stdout and human readable output goes to stderr.
// Therefore we need to get both. In order to get errors we need to parse the actual text line
const bazelLogSubscription = Rx.merge(
observeLines(bazelProc.stdout!).pipe(
tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
),
observeLines(bazelProc.stderr!).pipe(
tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
)
).subscribe(bazelLogs$);
// Wait for process and logs to finish, unsubscribing in the end
await bazelProc;
await bazelLogs$.toPromise();
await bazelLogSubscription.unsubscribe();
}