maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[7.x] [kbn/optimizer/node] properly separate lmdb databases, log better (#83849) (#83941)

Browse source 
Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Spencer 2020-11-20 11:39:55 -07:00 committed by GitHub
parent 447722e3d2
commit 79d4646bdd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 218 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -720,7 +720,7 @@
"less": "npm:@elastic/less@2.7.3-kibana",
"license-checker": "^16.0.0",
"listr": "^0.14.1",
"lmdb-store": "^0.6.10",
"lmdb-store": "^0.8.15",
"load-grunt-config": "^3.0.1",
"loader-utils": "^1.2.3",
"log-symbols": "^2.2.0",

154
packages/kbn-optimizer/src/node/cache.ts
View file

@ -17,91 +17,67 @@
* under the License.
*/
import Path from 'path';
import Fs from 'fs';
import { Writable } from 'stream';
// @ts-expect-error no types available
import chalk from 'chalk';
import * as LmdbStore from 'lmdb-store';
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
const LMDB_PKG = JSON.parse(
Fs.readFileSync(Path.resolve(REPO_ROOT, 'node_modules/lmdb-store/package.json'), 'utf8')
);
const CACHE_DIR = Path.resolve(
REPO_ROOT,
`data/node_auto_transpilation_cache/lmdb-${LMDB_PKG.version}/${UPSTREAM_BRANCH}`
);
const reportError = () => {
// right now I'm not sure we need to worry about errors, the cache isn't actually
// necessary, and if the cache is broken it should just rebuild on the next restart
// of the process. We don't know how often errors occur though and what types of
// things might fail on different machines so we probably want some way to signal
// to users that something is wrong
};
const GLOBAL_ATIME = `${Date.now()}`;
const MINUTE = 1000 * 60;
const HOUR = MINUTE * 60;
const DAY = HOUR * 24;
interface Lmdb<T> {
name: string;
get(key: string): T | undefined;
put(key: string, value: T, version?: number, ifVersion?: number): Promise<boolean>;
remove(key: string, ifVersion?: number): Promise<boolean>;
removeSync(key: string): void;
openDB<T2>(options: {
name: string;
encoding: 'msgpack' | 'string' | 'json' | 'binary';
}): Lmdb<T2>;
getRange(options?: {
start?: T;
end?: T;
reverse?: boolean;
limit?: number;
versions?: boolean;
}): Iterable<{ key: string; value: T }>;
}
const dbName = (db: LmdbStore.Database) =>
// @ts-expect-error db.name is not a documented/typed property
db.name;
export class Cache {
private readonly codes: Lmdb<string>;
private readonly atimes: Lmdb<string>;
private readonly mtimes: Lmdb<string>;
private readonly sourceMaps: Lmdb<any>;
private readonly codes: LmdbStore.RootDatabase<string, string>;
private readonly atimes: LmdbStore.Database<string, string>;
private readonly mtimes: LmdbStore.Database<string, string>;
private readonly sourceMaps: LmdbStore.Database<string, string>;
private readonly prefix: string;
private readonly log?: Writable;
private readonly timer: NodeJS.Timer;
constructor(config: { prefix: string }) {
constructor(config: { dir: string; prefix: string; log?: Writable }) {
this.prefix = config.prefix;
this.log = config.log;
this.codes = LmdbStore.open({
this.codes = LmdbStore.open(config.dir, {
name: 'codes',
path: CACHE_DIR,
encoding: 'string',
maxReaders: 500,
});
this.atimes = this.codes.openDB({
// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
this.atimes = this.codes.openDB('atimes', {
name: 'atimes',
encoding: 'string',
});
this.mtimes = this.codes.openDB({
this.mtimes = this.codes.openDB('mtimes', {
name: 'mtimes',
encoding: 'string',
});
this.sourceMaps = this.codes.openDB({
this.sourceMaps = this.codes.openDB('sourceMaps', {
name: 'sourceMaps',
encoding: 'msgpack',
encoding: 'string',
});
// after the process has been running for 30 minutes prune the
// keys which haven't been used in 30 days. We use `unref()` to
// make sure this timer doesn't hold other processes open
// unexpectedly
setTimeout(() => {
this.timer = setTimeout(() => {
this.pruneOldKeys();
}, 30 * MINUTE).unref();
}, 30 * MINUTE);
// timer.unref is not defined in jest which emulates the dom by default
if (typeof this.timer.unref === 'function') {
this.timer.unref();
}
}
getMtime(path: string) {
@ -110,45 +86,78 @@ export class Cache {
getCode(path: string) {
const key = this.getKey(path);
const code = this.safeGet(this.codes, key);
// when we use a file from the cache set the "atime" of that cache entry
// so that we know which cache items we use and which haven't been
// touched in a long time (currently 30 days)
this.atimes.put(key, GLOBAL_ATIME).catch(reportError);
if (code !== undefined) {
// when we use a file from the cache set the "atime" of that cache entry
// so that we know which cache items we use and which haven't been
// touched in a long time (currently 30 days)
this.safePut(this.atimes, key, GLOBAL_ATIME);
}
return this.safeGet(this.codes, key);
return code;
}
getSourceMap(path: string) {
return this.safeGet(this.sourceMaps, this.getKey(path));
const map = this.safeGet(this.sourceMaps, this.getKey(path));
if (typeof map === 'string') {
return JSON.parse(map);
}
}
update(path: string, file: { mtime: string; code: string; map: any }) {
async update(path: string, file: { mtime: string; code: string; map: any }) {
const key = this.getKey(path);
Promise.all([
this.atimes.put(key, GLOBAL_ATIME),
this.mtimes.put(key, file.mtime),
this.codes.put(key, file.code),
this.sourceMaps.put(key, file.map),
]).catch(reportError);
await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
]);
}
close() {
clearTimeout(this.timer);
}
private getKey(path: string) {
return `${this.prefix}${path}`;
}
private safeGet<V>(db: Lmdb<V>, key: string) {
private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {
try {
return db.get(key);
const value = db.get(key);
this.debug(value === undefined ? 'MISS' : 'HIT', db, key);
return value;
} catch (error) {
process.stderr.write(
`failed to read node transpilation [${db.name}] cache for [${key}]: ${error.stack}\n`
);
db.removeSync(key);
this.logError('GET', db, key, error);
}
}
private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
try {
await db.put(key, value);
this.debug('PUT', db, key);
} catch (error) {
this.logError('PUT', db, key, error);
}
}
private debug(type: string, db: LmdbStore.Database, key: LmdbStore.Key) {
if (this.log) {
this.log.write(`${type} [${dbName(db)}] ${String(key)}\n`);
}
}
private logError(type: 'GET' | 'PUT', db: LmdbStore.Database, key: LmdbStore.Key, error: Error) {
this.debug(`ERROR/${type}`, db, `${String(key)}: ${error.stack}`);
process.stderr.write(
chalk.red(
`[@kbn/optimizer/node] ${type} error [${dbName(db)}/${String(key)}]: ${error.stack}\n`
)
);
}
private async pruneOldKeys() {
try {
const ATIME_LIMIT = Date.now() - 30 * DAY;
@ -157,9 +166,10 @@ export class Cache {
const validKeys: string[] = [];
const invalidKeys: string[] = [];
// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
for (const { key, value } of this.atimes.getRange()) {
const atime = parseInt(value, 10);
if (atime < ATIME_LIMIT) {
const atime = parseInt(`${value}`, 10);
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
invalidKeys.push(key);
} else {
validKeys.push(key);

97
packages/kbn-optimizer/src/node/integration_tests/cache.test.ts Normal file
View file

@ -0,0 +1,97 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Path from 'path';
import { Writable } from 'stream';
import del from 'del';
import { Cache } from '../cache';
const DIR = Path.resolve(__dirname, '../__tmp__/cache');
const makeTestLog = () => {
const log = Object.assign(
new Writable({
write(chunk, enc, cb) {
log.output += chunk;
cb();
},
}),
{
output: '',
}
);
return log;
};
const instances: Cache[] = [];
const makeCache = (...options: ConstructorParameters<typeof Cache>) => {
const instance = new Cache(...options);
instances.push(instance);
return instance;
};
beforeEach(async () => await del(DIR));
afterEach(async () => {
await del(DIR);
for (const instance of instances) {
instance.close();
}
instances.length = 0;
});
it('returns undefined until values are set', async () => {
const path = '/foo/bar.js';
const mtime = new Date().toJSON();
const log = makeTestLog();
const cache = makeCache({
dir: DIR,
prefix: 'foo',
log,
});
expect(cache.getMtime(path)).toBe(undefined);
expect(cache.getCode(path)).toBe(undefined);
expect(cache.getSourceMap(path)).toBe(undefined);
await cache.update(path, {
mtime,
code: 'var x = 1',
map: { foo: 'bar' },
});
expect(cache.getMtime(path)).toBe(mtime);
expect(cache.getCode(path)).toBe('var x = 1');
expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' });
expect(log.output).toMatchInlineSnapshot(`
"MISS [mtimes] foo/foo/bar.js
MISS [codes] foo/foo/bar.js
MISS [sourceMaps] foo/foo/bar.js
PUT [atimes] foo/foo/bar.js
PUT [mtimes] foo/foo/bar.js
PUT [codes] foo/foo/bar.js
PUT [sourceMaps] foo/foo/bar.js
HIT [mtimes] foo/foo/bar.js
HIT [codes] foo/foo/bar.js
HIT [sourceMaps] foo/foo/bar.js
"
`);
});

8
packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
View file

@ -39,7 +39,7 @@ import Crypto from 'crypto';
import * as babel from '@babel/core';
import { addHook } from 'pirates';
import { REPO_ROOT } from '@kbn/dev-utils';
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
import sourceMapSupport from 'source-map-support';
import { Cache } from './cache';
@ -134,7 +134,13 @@ export function registerNodeAutoTranspilation() {
installed = true;
const cache = new Cache({
dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache', UPSTREAM_BRANCH),
prefix: determineCachePrefix(),
log: process.env.DEBUG_NODE_TRANSPILER_CACHE
? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), {
flags: 'a',
})
: undefined,
});
sourceMapSupport.install({

50
yarn.lock
View file

@ -18945,16 +18945,28 @@ livereload-js@^2.3.0:
resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.4.0.tgz#447c31cf1ea9ab52fc20db615c5ddf678f78009c"
integrity sha512-XPQH8Z2GDP/Hwz2PCDrh2mth4yFejwA1OZ/81Ti3LgKyhDcEjsSsqFWZojHG0va/duGd+WyosY7eXLDoOyqcPw==
lmdb-store@^0.6.10:
version "0.6.10"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.6.10.tgz#db8efde6e052aabd17ebc63c8a913e1f31694129"
integrity sha512-ZLvp3qbBQ5VlBmaWa4EUAPyYEZ8qdUHsW69HmxkDi84pFQ37WMxYhFaF/7PQkdtxS/vyiKkZigd9TFgHjek1Nw==
lmdb-store-0.9@0.7.3:
version "0.7.3"
resolved "https://registry.yarnpkg.com/lmdb-store-0.9/-/lmdb-store-0.9-0.7.3.tgz#c2cb27dfa916ab966cceed692c67e4236813104a"
integrity sha512-t8iCnN6T3NZPFelPmjYIjCg+nhGbOdc0xcHCG40v01AWRTN49OINSt2k/u+16/2/HrI+b6Ssb8WByXUhbyHz6w==
dependencies:
fs-extra "^9.0.1"
msgpackr "^0.5.0"
msgpackr "^0.5.3"
nan "^2.14.1"
node-gyp-build "^4.2.3"
weak-lru-cache "^0.2.0"
weak-lru-cache "^0.3.9"
lmdb-store@^0.8.15:
version "0.8.15"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.8.15.tgz#4efb0341c2df505dd6f3a7f26f834f0a142a80a2"
integrity sha512-4Q0WZh2FmcJC6esZRUWMfkCmNiz0WU9cOgrxt97ZMTnVfHyOdZhtrt0oOF5EQPfetxxJf/BorKY28aX92R6G6g==
dependencies:
fs-extra "^9.0.1"
lmdb-store-0.9 "0.7.3"
msgpackr "^0.5.4"
nan "^2.14.1"
node-gyp-build "^4.2.3"
weak-lru-cache "^0.3.9"
load-bmfont@^1.3.1, load-bmfont@^1.4.0:
version "1.4.0"
@ -20477,20 +20489,20 @@ ms@2.1.1, ms@^2.0.0, ms@^2.1.1:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
msgpackr-extract@^0.3.4:
version "0.3.4"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.4.tgz#8ee5e73d1135340e564c498e8c593134365eb060"
integrity sha512-d3+qwTJzgqqsq2L2sQuH0SoO4StvpUhMqMAKy6tMimn7XdBaRtDlquFzRJsp0iMGt2hnU4UOqD8Tz9mb0KglTA==
msgpackr-extract@^0.3.5:
version "0.3.5"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.5.tgz#0f206da058bd3dad0f8605d324de001a8f4de967"
integrity sha512-zHhstybu+m/j3H6CVBMcILVIzATK6dWRGtlePJjsnSAj8kLT5joMa9i0v21Uc80BPNDcwFsnG/dz2318tfI81w==
dependencies:
nan "^2.14.1"
node-gyp-build "^4.2.3"
msgpackr@^0.5.0:
version "0.5.1"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.1.tgz#7eecbf342645b7718dd2e3386894368d06732b3f"
integrity sha512-nK2uJl67Q5KU3MWkYBUlYynqKS1UUzJ5M1h6TQejuJtJzD3hW2Suv2T1pf01E9lUEr93xaLokf/xC+jwBShMPQ==
msgpackr@^0.5.3, msgpackr@^0.5.4:
version "0.5.4"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.4.tgz#c21c03d5e132d2e54d0b9ced02a75b1f48413380"
integrity sha512-ILEWtIWwd5ESWHKoVjJ4GP7JWkpuAUJ20qi2j2qEC6twecBmK4E6YG3QW847OpmvdAhMJGq2LoDJRn/kNERTeQ==
optionalDependencies:
msgpackr-extract "^0.3.4"
msgpackr-extract "^0.3.5"
multicast-dns-service-types@^1.1.0:
version "1.1.0"
@ -29326,10 +29338,10 @@ wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"
weak-lru-cache@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.2.0.tgz#447379ccff6dfda1b7a9566c9ef168260be859d1"
integrity sha512-M1l5CzKvM7maa7tCbtL0NW6sOnp8gqup853+9Aq7GL0XNWKNnFOkeE3v3Z5X2IeMzedPwQyPbi4RlFvD6rxs7A==
weak-lru-cache@^0.3.9:
version "0.3.9"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.3.9.tgz#9e56920d4115e8542625d8ef8cc278cbd97f7624"
integrity sha512-WqAu3wzbHQvjSi/vgYhidZkf2p7L3Z8iDEIHnqvE31EQQa7Vh7PDOphrRJ1oxlW8JIjgr2HvMcRe9Q1GhW2NPw==
web-namespaces@^1.0.0:
version "1.1.4"