Watch optimizer cache invalidation (#24172)

* chore(NA): cherry pick work from spencer on impleting the cache invalidation system and merging it with current master. * feat(NA): add support for dlls bundle into the cache state invalidation system. * chore(NA): merge with master. * feat(NA): first working version for the watch cache. * feat(NA): added logger, correct cache delete and removed last todos. * feat(NA): remove some useless features for the time being. * refact(NA): just pass kibanaHapiServer.log function directly instead of an anonimous function that calls the kibanaHapiServer.log one. * refact(NA): move everything to async. * refact(NA): remove dll mentions. * chore(NA): removed types/mkdirp as we dont use mkdirp into typescript.
2018-11-28 04:15:54 +00:00 · 2018-11-28 04:15:54 +00:00 · afb581fa14
parent 3909d8bed6
commit afb581fa14
6 changed files with 225 additions and 1 deletions
--- a/package.json
+++ b/package.json
@ -259,6 +259,7 @@
    "@types/d3": "^3.5.41",
    "@types/dedent": "^0.7.0",
    "@types/del": "^3.0.1",
+    "@types/delete-empty": "^2.0.0",
    "@types/elasticsearch": "^5.0.26",
    "@types/enzyme": "^3.1.12",
    "@types/eslint": "^4.16.2",
@ -266,6 +267,7 @@
    "@types/fetch-mock": "^5.12.2",
    "@types/getopts": "^2.0.0",
    "@types/glob": "^5.0.35",
+    "@types/globby": "^8.0.0",
    "@types/graphql": "^0.13.1",
    "@types/hapi": "^17.0.18",
    "@types/has-ansi": "^3.0.0",
@ -312,6 +314,7 @@
    "chromedriver": "2.42.1",
    "classnames": "2.2.5",
    "dedent": "^0.7.0",
+    "delete-empty": "^2.0.0",
    "enzyme": "3.2.0",
    "enzyme-adapter-react-16": "^1.1.1",
    "enzyme-to-json": "3.3.1",
--- a/src/optimize/watch/optmzr_role.js
+++ b/src/optimize/watch/optmzr_role.js
@ -17,17 +17,27 @@
 * under the License.
 */

+import { resolve } from 'path';
+
 import WatchServer from './watch_server';
 import WatchOptimizer, { STATUS } from './watch_optimizer';
+import { WatchCache } from './watch_cache';

 export default async (kbnServer, kibanaHapiServer, config) => {
+  const log = (tags, data) => kibanaHapiServer.log(tags, data);
+
  const watchOptimizer = new WatchOptimizer({
-    log: (tags, data) => kibanaHapiServer.log(tags, data),
+    log,
    uiBundles: kbnServer.uiBundles,
    profile: config.get('optimize.profile'),
    sourceMaps: config.get('optimize.sourceMaps'),
    prebuild: config.get('optimize.watchPrebuild'),
    unsafeCache: config.get('optimize.unsafeCache'),
+    watchCache: new WatchCache({
+      log,
+      outputPath: config.get('path.data'),
+      cachePath: resolve(kbnServer.uiBundles.getCacheDirectory(), '../'),
+    })
  });

  const server = new WatchServer(
--- a/src/optimize/watch/watch_cache.ts
+++ b/src/optimize/watch/watch_cache.ts
@ -0,0 +1,157 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createHash } from 'crypto';
+import { readFile, writeFile } from 'fs';
+import { resolve } from 'path';
+import { promisify } from 'util';
+
+import del from 'del';
+import deleteEmpty from 'delete-empty';
+import globby from 'globby';
+
+const readAsync = promisify(readFile);
+const writeAsync = promisify(writeFile);
+
+interface Params {
+  log: (tags: string[], data: string) => void;
+  outputPath: string;
+  cachePath: string;
+}
+
+interface WatchCacheStateContent {
+  optimizerConfigSha?: string;
+  yarnLockSha?: string;
+}
+
+export class WatchCache {
+  private readonly log: Params['log'];
+  private readonly outputPath: Params['outputPath'];
+  private readonly cachePath: Params['cachePath'];
+  private readonly cacheState: WatchCacheStateContent;
+  private statePath: string;
+  private diskCacheState: WatchCacheStateContent;
+  private isInitialized: boolean;
+
+  constructor(params: Params) {
+    this.log = params.log;
+    this.outputPath = params.outputPath;
+    this.cachePath = params.cachePath;
+
+    this.isInitialized = false;
+    this.statePath = '';
+    this.cacheState = {};
+    this.diskCacheState = {};
+    this.cacheState.yarnLockSha = '';
+    this.cacheState.optimizerConfigSha = '';
+  }
+
+  public async tryInit() {
+    if (!this.isInitialized) {
+      this.statePath = resolve(this.outputPath, 'watch_optimizer_cache_state.json');
+      this.diskCacheState = await this.read();
+      this.cacheState.yarnLockSha = await this.buildYarnLockSha();
+      this.cacheState.optimizerConfigSha = await this.buildOptimizerConfigSha();
+      this.isInitialized = true;
+    }
+  }
+
+  public async tryReset() {
+    await this.tryInit();
+
+    if (!this.isResetNeeded()) {
+      return;
+    }
+
+    await this.reset();
+  }
+
+  public async reset() {
+    this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache will reset');
+
+    // start by deleting the state file to lower the
+    // amount of time that another process might be able to
+    // successfully read it once we decide to delete it
+    await del(this.statePath);
+
+    // delete everything in optimize/.cache directory
+    // except ts-node
+    await del(await globby([this.cachePath, `!${this.cachePath}/ts-node/**`], { dot: true }));
+
+    // delete some empty folder that could be left
+    // from the previous cache path reset action
+    await deleteEmpty(this.cachePath);
+
+    // re-write new cache state file
+    await this.write();
+
+    this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache has reset');
+  }
+
+  private async buildShaWithMultipleFiles(filePaths: string[]) {
+    const shaHash = createHash('sha1');
+
+    for (const filePath of filePaths) {
+      try {
+        shaHash.update(await readAsync(filePath), 'utf8');
+      } catch (e) {
+        /* no-op */
+      }
+    }
+
+    return shaHash.digest('hex');
+  }
+
+  private async buildYarnLockSha() {
+    const kibanaYarnLock = resolve(__dirname, '../../../yarn.lock');
+
+    return await this.buildShaWithMultipleFiles([kibanaYarnLock]);
+  }
+
+  private async buildOptimizerConfigSha() {
+    const baseOptimizer = resolve(__dirname, '../base_optimizer.js');
+
+    return await this.buildShaWithMultipleFiles([baseOptimizer]);
+  }
+
+  private isResetNeeded() {
+    return this.hasYarnLockChanged() || this.hasOptimizerConfigChanged();
+  }
+
+  private hasYarnLockChanged() {
+    return this.cacheState.yarnLockSha !== this.diskCacheState.yarnLockSha;
+  }
+
+  private hasOptimizerConfigChanged() {
+    return this.cacheState.optimizerConfigSha !== this.diskCacheState.optimizerConfigSha;
+  }
+
+  private async write() {
+    await writeAsync(this.statePath, JSON.stringify(this.cacheState, null, 2), 'utf8');
+    this.diskCacheState = this.cacheState;
+  }
+
+  private async read(): Promise<WatchCacheStateContent> {
+    try {
+      return JSON.parse(await readAsync(this.statePath, 'utf8'));
+    } catch (error) {
+      return {};
+    }
+  }
+}
--- a/src/optimize/watch/watch_optimizer.js
+++ b/src/optimize/watch/watch_optimizer.js
@ -36,6 +36,7 @@ export default class WatchOptimizer extends BaseOptimizer {
    super(opts);
    this.log = opts.log || (() => null);
    this.prebuild = opts.prebuild || false;
+    this.watchCache = opts.watchCache;
    this.status$ = new Rx.ReplaySubject(1);
  }

@ -43,6 +44,9 @@ export default class WatchOptimizer extends BaseOptimizer {
    this.initializing = true;
    this.initialBuildComplete = false;

+    // try reset the watch optimizer cache
+    await this.watchCache.tryReset();
+
    // log status changes
    this.status$.subscribe(this.onStatusChangeHandler);
    await this.uiBundles.resetBundleDir();
--- a/src/server/logging/log_format_string.js
+++ b/src/server/logging/log_format_string.js
@ -49,6 +49,7 @@ const typeColors = {
  optmzr: 'white',
  manager: 'green',
  optimize: 'magentaBright',
+  'optimize:watch_cache': 'magentaBright',
  listening: 'magentaBright',
  scss: 'magentaBright',
 };
--- a/yarn.lock
+++ b/yarn.lock
@ -1095,6 +1095,11 @@
  resolved "https://registry.yarnpkg.com/@types/delay/-/delay-2.0.1.tgz#61bcf318a74b61e79d1658fbf054f984c90ef901"
  integrity sha512-D1/YuYOcdOIdaQnaiUJ77VcilVvESkynw79CtGqpjkXyv4OUezEVZtdXnSOwXL8Zcelu66QbyC8QQcVQ/ZPdig==

+"@types/delete-empty@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@types/delete-empty/-/delete-empty-2.0.0.tgz#1647ae9e68f708a6ba778531af667ec55bc61964"
+  integrity sha512-sq+kwx8zA9BSugT9N+Jr8/uWjbHMZ+N/meJEzRyT3gmLq/WMtx/iSIpvdpmBUi/cvXl6Kzpvve8G2ESkabFwmg==
+
 "@types/elasticsearch@^5.0.26":
  version "5.0.28"
  resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.28.tgz#0e4cdf7d9c9a3fe901c0da4fb9ad824c6d3b4091"
@ -1178,6 +1183,14 @@
  dependencies:
    "@types/glob" "*"

+"@types/globby@^8.0.0":
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/@types/globby/-/globby-8.0.0.tgz#7bd10eaf802e1e11afdb1e5436cf472ddf4c0dd2"
+  integrity sha512-xDtsX5tlctxJzvg29r/LN12z30oJpoFP9cE8eJ8nY5cbSvN0c0RdRHrVlEq4LRh362Sd+JsqxJ3QWw0Wnyto8w==
+  dependencies:
+    "@types/glob" "*"
+    fast-glob "^2.0.2"
+
 "@types/got@^7.1.7":
  version "7.1.8"
  resolved "https://registry.yarnpkg.com/@types/got/-/got-7.1.8.tgz#c5f421b25770689bf8948b1241f710d71a00d7dd"
@ -2022,6 +2035,13 @@ ansi-gray@^0.1.1:
  dependencies:
    ansi-wrap "0.1.0"

+ansi-green@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/ansi-green/-/ansi-green-0.1.1.tgz#8a5d9a979e458d57c40e33580b37390b8e10d0f7"
+  integrity sha1-il2al55FjVfEDjNYCzc5C44Q0Pc=
+  dependencies:
+    ansi-wrap "0.1.0"
+
 ansi-html@0.0.7:
  version "0.0.7"
  resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e"
@ -6714,6 +6734,15 @@ delegates@^1.0.0:
  resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
  integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=

+delete-empty@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/delete-empty/-/delete-empty-2.0.0.tgz#dcf7c4f93a98445119acd57b137d13e7af78fa39"
+  integrity sha512-voZ8OiMkVR9MOTTHZ5P0DaMDtIW6xEbXZeADp6U8uwxIJFhs2hRwyIlUZIs5hR4YIp9VYBURqZrV6Yz0ozhVpg==
+  dependencies:
+    log-ok "^0.1.1"
+    relative "^3.0.2"
+    rimraf "^2.6.2"
+
 depd@1.1.1:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359"
@ -13623,6 +13652,14 @@ lodash@~4.3.0:
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.3.0.tgz#efd9c4a6ec53f3b05412429915c3e4824e4d25a4"
  integrity sha1-79nEpuxT87BUEkKZFcPkgk5NJaQ=

+log-ok@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334"
+  integrity sha1-vqPdNqzQuKckDXhza1uXxlREozQ=
+  dependencies:
+    ansi-green "^0.1.1"
+    success-symbol "^0.1.0"
+
 log-symbols@^1.0.1, log-symbols@^1.0.2:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18"
@ -18025,6 +18062,13 @@ relateurl@0.2.x:
  resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9"
  integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk=

+relative@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f"
+  integrity sha1-Dc2OxUpdNaPBXhBFA9ZTdbWlNn8=
+  dependencies:
+    isobject "^2.0.0"
+
 remark-parse@^5.0.0:
  version "5.0.0"
  resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-5.0.0.tgz#4c077f9e499044d1d5c13f80d7a98cf7b9285d95"
@ -19982,6 +20026,11 @@ subtext@6.x.x:
    pez "4.x.x"
    wreck "14.x.x"

+success-symbol@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/success-symbol/-/success-symbol-0.1.0.tgz#24022e486f3bf1cdca094283b769c472d3b72897"
+  integrity sha1-JAIuSG878c3KCUKDt2nEctO3KJc=
+
 sudo-block@^1.1.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/sudo-block/-/sudo-block-1.2.0.tgz#cc539bf8191624d4f507d83eeb45b4cea27f3463"