[dev/build] scan node_modules rather than lots of deleteAll() calls (#24692)

2018-10-29 13:02:04 -07:00 · 2018-10-29 13:02:04 -07:00 · dc89a9dc61
parent 460a297bc9
commit dc89a9dc61
7 changed files with 272 additions and 91 deletions
--- a/package.json
+++ b/package.json
@ -231,6 +231,7 @@
    "@types/classnames": "^2.2.3",
    "@types/d3": "^5.0.0",
    "@types/dedent": "^0.7.0",
+    "@types/del": "^3.0.1",
    "@types/elasticsearch": "^5.0.26",
    "@types/enzyme": "^3.1.12",
    "@types/eslint": "^4.16.2",
--- a/src/dev/build/lib/fs.js
+++ b/src/dev/build/lib/fs.js
@ -39,7 +39,7 @@ const readFileAsync = promisify(fs.readFile);
 const readdirAsync = promisify(fs.readdir);
 const utimesAsync = promisify(fs.utimes);

-function assertAbsolute(path) {
+export function assertAbsolute(path) {
  if (!isAbsolute(path)) {
    throw new TypeError(
      'Please use absolute paths to keep things explicit. You probably want to use `build.resolvePath()` or `config.resolveFromRepo()`.'
--- a/src/dev/build/lib/index.js
+++ b/src/dev/build/lib/index.js
@ -31,3 +31,4 @@ export {
  untar,
  deleteAll,
 } from './fs';
+export { scanDelete } from './scan_delete';
--- a/src/dev/build/lib/scan_delete.test.ts
+++ b/src/dev/build/lib/scan_delete.test.ts
@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readdirSync } from 'fs';
+import { relative, resolve } from 'path';
+
+import del from 'del';
+
+// @ts-ignore
+import { mkdirp, write } from './fs';
+import { scanDelete } from './scan_delete';
+
+const TMP = resolve(__dirname, '__tests__/__tmp__');
+
+// clean and recreate TMP directory
+beforeEach(async () => {
+  await del(TMP);
+  await mkdirp(resolve(TMP, 'foo/bar/baz'));
+  await mkdirp(resolve(TMP, 'foo/bar/box'));
+  await mkdirp(resolve(TMP, 'a/b/c/d/e'));
+  await write(resolve(TMP, 'a/bar'), 'foo');
+});
+
+// cleanup TMP directory
+afterAll(async () => {
+  await del(TMP);
+});
+
+it('requires an absolute directory', async () => {
+  await expect(
+    scanDelete({
+      directory: relative(process.cwd(), TMP),
+      regularExpressions: [],
+    })
+  ).rejects.toMatchInlineSnapshot(
+    `[TypeError: Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`.]`
+  );
+});
+
+it('deletes files/folders matching regular expression', async () => {
+  await scanDelete({
+    directory: TMP,
+    regularExpressions: [/^.*[\/\\](bar|c)([\/\\]|$)/],
+  });
+  expect(readdirSync(resolve(TMP, 'foo'))).toEqual([]);
+  expect(readdirSync(resolve(TMP, 'a'))).toEqual(['b']);
+  expect(readdirSync(resolve(TMP, 'a/b'))).toEqual([]);
+});
--- a/src/dev/build/lib/scan_delete.ts
+++ b/src/dev/build/lib/scan_delete.ts
@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import del from 'del';
+import { join } from 'path';
+import * as Rx from 'rxjs';
+import { count, map, mergeAll, mergeMap } from 'rxjs/operators';
+
+// @ts-ignore
+import { assertAbsolute } from './fs';
+
+const getStat$ = Rx.bindNodeCallback(Fs.stat);
+const getReadDir$ = Rx.bindNodeCallback(Fs.readdir);
+
+interface Options {
+  directory: string;
+  regularExpressions: RegExp[];
+  concurrency?: 20;
+}
+
+/**
+ * Scan the files in a directory and delete the directories/files that
+ * are matched by an array of regular expressions.
+ *
+ * @param options.directory the directory to scan, all files including dot files will be checked
+ * @param options.regularExpressions an array of regular expressions, if any matches the file/directory will be deleted
+ * @param options.concurrency optional concurrency to run deletes, defaults to 20
+ */
+export async function scanDelete(options: Options) {
+  const { directory, regularExpressions, concurrency = 20 } = options;
+
+  assertAbsolute(directory);
+
+  // get an observable of absolute paths within a directory
+  const getChildPath$ = (path: string) =>
+    getReadDir$(path).pipe(
+      mergeAll(),
+      map(name => join(path, name))
+    );
+
+  // get an observable of all paths to be deleted, by starting with the arg
+  // and recursively iterating through all children, unless a child matches
+  // one of the supplied regular expressions
+  const getPathsToDelete$ = (path: string): Rx.Observable<string> => {
+    if (regularExpressions.some(re => re.test(path))) {
+      return Rx.of(path);
+    }
+
+    return getStat$(path).pipe(
+      mergeMap(stat => (stat.isDirectory() ? getChildPath$(path) : Rx.EMPTY)),
+      mergeMap(getPathsToDelete$)
+    );
+  };
+
+  return await Rx.of(directory)
+    .pipe(
+      mergeMap(getPathsToDelete$),
+      mergeMap(async path => await del(path), concurrency),
+      count()
+    )
+    .toPromise();
+}
--- a/src/dev/build/tasks/clean_tasks.js
+++ b/src/dev/build/tasks/clean_tasks.js
@ -17,7 +17,9 @@
 * under the License.
 */

-import { deleteAll } from '../lib';
+import minimatch from 'minimatch';
+
+import { deleteAll, scanDelete } from '../lib';

 export const CleanTask = {
  global: true,
@ -49,10 +51,13 @@ export const CleanTypescriptTask = {
    'Cleaning typescript source files that have been transpiled to JS',

  async run(config, log, build) {
-    await deleteAll(log, [
-      build.resolvePath('**/*.{ts,tsx,d.ts}'),
-      build.resolvePath('**/tsconfig*.json'),
-    ]);
+    log.info('Deleted %d files', await scanDelete({
+      directory: build.resolvePath(),
+      regularExpressions: [
+        /\.(ts|tsx|d\.ts)$/,
+        /tsconfig.*\.json$/
+      ]
+    }));
  },
 };

@ -60,94 +65,108 @@ export const CleanExtraFilesFromModulesTask = {
  description: 'Cleaning tests, examples, docs, etc. from node_modules',

  async run(config, log, build) {
-    const deleteFromNodeModules = globs => {
-      return deleteAll(
-        log,
-        globs.map(p => build.resolvePath(`node_modules/**/${p}`))
+    const makeRegexps = patterns =>
+      patterns.map(pattern =>
+        minimatch.makeRe(pattern, { nocase: true })
      );
-    };

-    const tests = [
-      'test',
-      'tests',
-      '__tests__',
-      'mocha.opts',
-      '*.test.js',
-      '*.snap',
-      'coverage',
-    ];
-    const docs = [
-      'doc',
-      'docs',
-      'CONTRIBUTING.md',
-      'Contributing.md',
-      'contributing.md',
-      'History.md',
-      'HISTORY.md',
-      'history.md',
-      'CHANGELOG.md',
-      'Changelog.md',
-      'changelog.md',
-    ];
-    const examples = ['example', 'examples', 'demo', 'samples'];
-    const bins = ['.bin'];
-    const linters = [
-      '.eslintrc',
-      '.eslintrc.js',
-      '.eslintrc.yml',
-      '.prettierrc',
-      '.jshintrc',
-      '.babelrc',
-      '.jscs.json',
-      '.lint',
-    ];
-    const hints = ['*.flow', '*.webidl', '*.map', '@types'];
-    const scripts = [
-      '*.sh',
-      '*.bat',
-      '*.exe',
-      'Gruntfile.js',
-      'gulpfile.js',
-      'Makefile',
-    ];
-    const untranspiledSources = ['*.coffee', '*.scss', '*.sass', '.ts', '.tsx'];
-    const editors = ['.editorconfig', '.vscode'];
-    const git = [
-      '.gitattributes',
-      '.gitkeep',
-      '.gitempty',
-      '.gitmodules',
-      '.keep',
-      '.empty',
-    ];
-    const ci = [
-      '.travis.yml',
-      '.coveralls.yml',
-      '.instanbul.yml',
-      'appveyor.yml',
-      '.zuul.yml',
-    ];
-    const meta = [
-      'package-lock.json',
-      'component.json',
-      'bower.json',
-      'yarn.lock',
-    ];
-    const misc = ['.*ignore', '.DS_Store', 'Dockerfile', 'docker-compose.yml'];
+    log.info('Deleted %d files', await scanDelete({
+      directory: build.resolvePath('node_modules'),
+      regularExpressions: makeRegexps([
+        // tests
+        '**/test',
+        '**/tests',
+        '**/__tests__',
+        '**/mocha.opts',
+        '**/*.test.js',
+        '**/*.snap',
+        '**/coverage',

-    await deleteFromNodeModules(tests);
-    await deleteFromNodeModules(docs);
-    await deleteFromNodeModules(examples);
-    await deleteFromNodeModules(bins);
-    await deleteFromNodeModules(linters);
-    await deleteFromNodeModules(hints);
-    await deleteFromNodeModules(scripts);
-    await deleteFromNodeModules(untranspiledSources);
-    await deleteFromNodeModules(editors);
-    await deleteFromNodeModules(git);
-    await deleteFromNodeModules(ci);
-    await deleteFromNodeModules(meta);
-    await deleteFromNodeModules(misc);
+        // docs
+        '**/doc',
+        '**/docs',
+        '**/CONTRIBUTING.md',
+        '**/Contributing.md',
+        '**/contributing.md',
+        '**/History.md',
+        '**/HISTORY.md',
+        '**/history.md',
+        '**/CHANGELOG.md',
+        '**/Changelog.md',
+        '**/changelog.md',
+
+        // examples
+        '**/example',
+        '**/examples',
+        '**/demo',
+        '**/samples',
+
+        // bins
+        '**/.bin',
+
+        // linters
+        '**/.eslintrc',
+        '**/.eslintrc.js',
+        '**/.eslintrc.yml',
+        '**/.prettierrc',
+        '**/.jshintrc',
+        '**/.babelrc',
+        '**/.jscs.json',
+        '**/.lint',
+
+        // hints
+        '**/*.flow',
+        '**/*.webidl',
+        '**/*.map',
+        '**/@types',
+
+        // scripts
+        '**/*.sh',
+        '**/*.bat',
+        '**/*.exe',
+        '**/Gruntfile.js',
+        '**/gulpfile.js',
+        '**/Makefile',
+
+        // untranspiled sources
+        '**/*.coffee',
+        '**/*.scss',
+        '**/*.sass',
+        '**/.ts',
+        '**/.tsx',
+
+        // editors
+        '**/.editorconfig',
+        '**/.vscode',
+
+        // git
+        '**/.gitattributes',
+        '**/.gitkeep',
+        '**/.gitempty',
+        '**/.gitmodules',
+        '**/.keep',
+        '**/.empty',
+
+        // ci
+        '**/.travis.yml',
+        '**/.coveralls.yml',
+        '**/.instanbul.yml',
+        '**/appveyor.yml',
+        '**/.zuul.yml',
+
+        // metadata
+        '**/package-lock.json',
+        '**/component.json',
+        '**/bower.json',
+        '**/yarn.lock',
+
+        // misc
+        '**/.*ignore',
+        '**/.DS_Store',
+        '**/Dockerfile',
+        '**/docker-compose.yml'
+      ])
+    }));
  },
 };

--- a/yarn.lock
+++ b/yarn.lock
@ -651,6 +651,13 @@
  resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050"
  integrity sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A==

+"@types/del@^3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@types/del/-/del-3.0.1.tgz#4712da8c119873cbbf533ad8dbf1baac5940ac5d"
+  integrity sha512-y6qRq6raBuu965clKgx6FHuiPu3oHdtmzMPXi8Uahsjdq1L6DL5fS/aY5/s71YwM7k6K1QIWvem5vNwlnNGIkQ==
+  dependencies:
+    "@types/glob" "*"
+
 "@types/delay@^2.0.1":
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/@types/delay/-/delay-2.0.1.tgz#61bcf318a74b61e79d1658fbf054f984c90ef901"
@ -716,6 +723,15 @@
  resolved "https://registry.yarnpkg.com/@types/getopts/-/getopts-2.0.0.tgz#8a603370cb367d3192bd8012ad39ab2320b5b476"
  integrity sha512-/WJ73/6+Ffulo6LDm0P11Y0uGDaitJBJyVhXr4Eg+/Bqi0epRLOnGDNOgplhMBFy7NLBMlZ5UQcukSABqaV5Kg==

+"@types/glob@*":
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
+  integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
+  dependencies:
+    "@types/events" "*"
+    "@types/minimatch" "*"
+    "@types/node" "*"
+
 "@types/glob@^5.0.35":
  version "5.0.35"
  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-5.0.35.tgz#1ae151c802cece940443b5ac246925c85189f32a"