diff --git a/package.json b/package.json index 4ed784a6a0fb..5d6af46b32d8 100644 --- a/package.json +++ b/package.json @@ -231,6 +231,7 @@ "@types/classnames": "^2.2.3", "@types/d3": "^5.0.0", "@types/dedent": "^0.7.0", + "@types/del": "^3.0.1", "@types/elasticsearch": "^5.0.26", "@types/enzyme": "^3.1.12", "@types/eslint": "^4.16.2", diff --git a/src/dev/build/lib/fs.js b/src/dev/build/lib/fs.js index d6809afded31..1386cbe4c7da 100644 --- a/src/dev/build/lib/fs.js +++ b/src/dev/build/lib/fs.js @@ -39,7 +39,7 @@ const readFileAsync = promisify(fs.readFile); const readdirAsync = promisify(fs.readdir); const utimesAsync = promisify(fs.utimes); -function assertAbsolute(path) { +export function assertAbsolute(path) { if (!isAbsolute(path)) { throw new TypeError( 'Please use absolute paths to keep things explicit. You probably want to use `build.resolvePath()` or `config.resolveFromRepo()`.' diff --git a/src/dev/build/lib/index.js b/src/dev/build/lib/index.js index bf7388417ecf..952519773d6b 100644 --- a/src/dev/build/lib/index.js +++ b/src/dev/build/lib/index.js @@ -31,3 +31,4 @@ export { untar, deleteAll, } from './fs'; +export { scanDelete } from './scan_delete'; diff --git a/src/dev/build/lib/scan_delete.test.ts b/src/dev/build/lib/scan_delete.test.ts new file mode 100644 index 000000000000..815119630ef4 --- /dev/null +++ b/src/dev/build/lib/scan_delete.test.ts @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { readdirSync } from 'fs'; +import { relative, resolve } from 'path'; + +import del from 'del'; + +// @ts-ignore +import { mkdirp, write } from './fs'; +import { scanDelete } from './scan_delete'; + +const TMP = resolve(__dirname, '__tests__/__tmp__'); + +// clean and recreate TMP directory +beforeEach(async () => { + await del(TMP); + await mkdirp(resolve(TMP, 'foo/bar/baz')); + await mkdirp(resolve(TMP, 'foo/bar/box')); + await mkdirp(resolve(TMP, 'a/b/c/d/e')); + await write(resolve(TMP, 'a/bar'), 'foo'); +}); + +// cleanup TMP directory +afterAll(async () => { + await del(TMP); +}); + +it('requires an absolute directory', async () => { + await expect( + scanDelete({ + directory: relative(process.cwd(), TMP), + regularExpressions: [], + }) + ).rejects.toMatchInlineSnapshot( + `[TypeError: Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`.]` + ); +}); + +it('deletes files/folders matching regular expression', async () => { + await scanDelete({ + directory: TMP, + regularExpressions: [/^.*[\/\\](bar|c)([\/\\]|$)/], + }); + expect(readdirSync(resolve(TMP, 'foo'))).toEqual([]); + expect(readdirSync(resolve(TMP, 'a'))).toEqual(['b']); + expect(readdirSync(resolve(TMP, 'a/b'))).toEqual([]); +}); diff --git a/src/dev/build/lib/scan_delete.ts b/src/dev/build/lib/scan_delete.ts new file mode 100644 index 000000000000..854e82ed169a --- /dev/null +++ b/src/dev/build/lib/scan_delete.ts @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; + +import del from 'del'; +import { join } from 'path'; +import * as Rx from 'rxjs'; +import { count, map, mergeAll, mergeMap } from 'rxjs/operators'; + +// @ts-ignore +import { assertAbsolute } from './fs'; + +const getStat$ = Rx.bindNodeCallback(Fs.stat); +const getReadDir$ = Rx.bindNodeCallback(Fs.readdir); + +interface Options { + directory: string; + regularExpressions: RegExp[]; + concurrency?: 20; +} + +/** + * Scan the files in a directory and delete the directories/files that + * are matched by an array of regular expressions. + * + * @param options.directory the directory to scan, all files including dot files will be checked + * @param options.regularExpressions an array of regular expressions, if any matches the file/directory will be deleted + * @param options.concurrency optional concurrency to run deletes, defaults to 20 + */ +export async function scanDelete(options: Options) { + const { directory, regularExpressions, concurrency = 20 } = options; + + assertAbsolute(directory); + + // get an observable of absolute paths within a directory + const getChildPath$ = (path: string) => + getReadDir$(path).pipe( + mergeAll(), + map(name => join(path, name)) + ); + + // get an observable of all paths to be deleted, by starting with the arg + // and recursively iterating through all children, unless a child matches + // one of the supplied regular expressions + const getPathsToDelete$ = (path: string): Rx.Observable => { + if (regularExpressions.some(re => re.test(path))) { + return Rx.of(path); + } + + return getStat$(path).pipe( + mergeMap(stat => (stat.isDirectory() ? getChildPath$(path) : Rx.EMPTY)), + mergeMap(getPathsToDelete$) + ); + }; + + return await Rx.of(directory) + .pipe( + mergeMap(getPathsToDelete$), + mergeMap(async path => await del(path), concurrency), + count() + ) + .toPromise(); +} diff --git a/src/dev/build/tasks/clean_tasks.js b/src/dev/build/tasks/clean_tasks.js index f2b66e506b25..cb6625c00311 100644 --- a/src/dev/build/tasks/clean_tasks.js +++ b/src/dev/build/tasks/clean_tasks.js @@ -17,7 +17,9 @@ * under the License. */ -import { deleteAll } from '../lib'; +import minimatch from 'minimatch'; + +import { deleteAll, scanDelete } from '../lib'; export const CleanTask = { global: true, @@ -49,10 +51,13 @@ export const CleanTypescriptTask = { 'Cleaning typescript source files that have been transpiled to JS', async run(config, log, build) { - await deleteAll(log, [ - build.resolvePath('**/*.{ts,tsx,d.ts}'), - build.resolvePath('**/tsconfig*.json'), - ]); + log.info('Deleted %d files', await scanDelete({ + directory: build.resolvePath(), + regularExpressions: [ + /\.(ts|tsx|d\.ts)$/, + /tsconfig.*\.json$/ + ] + })); }, }; @@ -60,94 +65,108 @@ export const CleanExtraFilesFromModulesTask = { description: 'Cleaning tests, examples, docs, etc. from node_modules', async run(config, log, build) { - const deleteFromNodeModules = globs => { - return deleteAll( - log, - globs.map(p => build.resolvePath(`node_modules/**/${p}`)) + const makeRegexps = patterns => + patterns.map(pattern => + minimatch.makeRe(pattern, { nocase: true }) ); - }; - const tests = [ - 'test', - 'tests', - '__tests__', - 'mocha.opts', - '*.test.js', - '*.snap', - 'coverage', - ]; - const docs = [ - 'doc', - 'docs', - 'CONTRIBUTING.md', - 'Contributing.md', - 'contributing.md', - 'History.md', - 'HISTORY.md', - 'history.md', - 'CHANGELOG.md', - 'Changelog.md', - 'changelog.md', - ]; - const examples = ['example', 'examples', 'demo', 'samples']; - const bins = ['.bin']; - const linters = [ - '.eslintrc', - '.eslintrc.js', - '.eslintrc.yml', - '.prettierrc', - '.jshintrc', - '.babelrc', - '.jscs.json', - '.lint', - ]; - const hints = ['*.flow', '*.webidl', '*.map', '@types']; - const scripts = [ - '*.sh', - '*.bat', - '*.exe', - 'Gruntfile.js', - 'gulpfile.js', - 'Makefile', - ]; - const untranspiledSources = ['*.coffee', '*.scss', '*.sass', '.ts', '.tsx']; - const editors = ['.editorconfig', '.vscode']; - const git = [ - '.gitattributes', - '.gitkeep', - '.gitempty', - '.gitmodules', - '.keep', - '.empty', - ]; - const ci = [ - '.travis.yml', - '.coveralls.yml', - '.instanbul.yml', - 'appveyor.yml', - '.zuul.yml', - ]; - const meta = [ - 'package-lock.json', - 'component.json', - 'bower.json', - 'yarn.lock', - ]; - const misc = ['.*ignore', '.DS_Store', 'Dockerfile', 'docker-compose.yml']; + log.info('Deleted %d files', await scanDelete({ + directory: build.resolvePath('node_modules'), + regularExpressions: makeRegexps([ + // tests + '**/test', + '**/tests', + '**/__tests__', + '**/mocha.opts', + '**/*.test.js', + '**/*.snap', + '**/coverage', - await deleteFromNodeModules(tests); - await deleteFromNodeModules(docs); - await deleteFromNodeModules(examples); - await deleteFromNodeModules(bins); - await deleteFromNodeModules(linters); - await deleteFromNodeModules(hints); - await deleteFromNodeModules(scripts); - await deleteFromNodeModules(untranspiledSources); - await deleteFromNodeModules(editors); - await deleteFromNodeModules(git); - await deleteFromNodeModules(ci); - await deleteFromNodeModules(meta); - await deleteFromNodeModules(misc); + // docs + '**/doc', + '**/docs', + '**/CONTRIBUTING.md', + '**/Contributing.md', + '**/contributing.md', + '**/History.md', + '**/HISTORY.md', + '**/history.md', + '**/CHANGELOG.md', + '**/Changelog.md', + '**/changelog.md', + + // examples + '**/example', + '**/examples', + '**/demo', + '**/samples', + + // bins + '**/.bin', + + // linters + '**/.eslintrc', + '**/.eslintrc.js', + '**/.eslintrc.yml', + '**/.prettierrc', + '**/.jshintrc', + '**/.babelrc', + '**/.jscs.json', + '**/.lint', + + // hints + '**/*.flow', + '**/*.webidl', + '**/*.map', + '**/@types', + + // scripts + '**/*.sh', + '**/*.bat', + '**/*.exe', + '**/Gruntfile.js', + '**/gulpfile.js', + '**/Makefile', + + // untranspiled sources + '**/*.coffee', + '**/*.scss', + '**/*.sass', + '**/.ts', + '**/.tsx', + + // editors + '**/.editorconfig', + '**/.vscode', + + // git + '**/.gitattributes', + '**/.gitkeep', + '**/.gitempty', + '**/.gitmodules', + '**/.keep', + '**/.empty', + + // ci + '**/.travis.yml', + '**/.coveralls.yml', + '**/.instanbul.yml', + '**/appveyor.yml', + '**/.zuul.yml', + + // metadata + '**/package-lock.json', + '**/component.json', + '**/bower.json', + '**/yarn.lock', + + // misc + '**/.*ignore', + '**/.DS_Store', + '**/Dockerfile', + '**/docker-compose.yml' + ]) + })); }, }; diff --git a/yarn.lock b/yarn.lock index ab07df6f81a5..ab9b413c1810 100644 --- a/yarn.lock +++ b/yarn.lock @@ -651,6 +651,13 @@ resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050" integrity sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A== +"@types/del@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@types/del/-/del-3.0.1.tgz#4712da8c119873cbbf533ad8dbf1baac5940ac5d" + integrity sha512-y6qRq6raBuu965clKgx6FHuiPu3oHdtmzMPXi8Uahsjdq1L6DL5fS/aY5/s71YwM7k6K1QIWvem5vNwlnNGIkQ== + dependencies: + "@types/glob" "*" + "@types/delay@^2.0.1": version "2.0.1" resolved "https://registry.yarnpkg.com/@types/delay/-/delay-2.0.1.tgz#61bcf318a74b61e79d1658fbf054f984c90ef901" @@ -716,6 +723,15 @@ resolved "https://registry.yarnpkg.com/@types/getopts/-/getopts-2.0.0.tgz#8a603370cb367d3192bd8012ad39ab2320b5b476" integrity sha512-/WJ73/6+Ffulo6LDm0P11Y0uGDaitJBJyVhXr4Eg+/Bqi0epRLOnGDNOgplhMBFy7NLBMlZ5UQcukSABqaV5Kg== +"@types/glob@*": + version "7.1.1" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" + integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w== + dependencies: + "@types/events" "*" + "@types/minimatch" "*" + "@types/node" "*" + "@types/glob@^5.0.35": version "5.0.35" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-5.0.35.tgz#1ae151c802cece940443b5ac246925c85189f32a"