* [Stats] Add metrics collector and stats API * uptime_ms in the process namespace * make uptime_in_millis always equal process.uptime_ms * fix api integration test * fix api integration test better * fix false positive with last change * change object detection, add fallbacks to return undefined
This commit is contained in:
parent
88eed1a0aa
commit
b4c226bbb7
|
@ -1,19 +1,28 @@
|
|||
import ServerStatus from './server_status';
|
||||
import { MetricsCollector } from './metrics_collector';
|
||||
import { Metrics } from './metrics_collector/metrics';
|
||||
import { registerStatusPage, registerStatusApi } from './routes';
|
||||
import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
|
||||
|
||||
export function statusMixin(kbnServer, server, config) {
|
||||
const collector = new MetricsCollector(server, config);
|
||||
kbnServer.status = new ServerStatus(kbnServer.server);
|
||||
|
||||
if (server.plugins['even-better']) {
|
||||
const { ['even-better']: evenBetter } = server.plugins;
|
||||
|
||||
if (evenBetter) {
|
||||
const metrics = new Metrics(config, server);
|
||||
|
||||
server.plugins['even-better'].monitor.on('ops', event => {
|
||||
evenBetter.monitor.on('ops', event => {
|
||||
// for status API (to deprecate in next major)
|
||||
metrics.capture(event).then(data => { kbnServer.metrics = data; });
|
||||
|
||||
// for metrics API (replacement API)
|
||||
collector.collect(event); // collect() is async, but here we aren't depending on the return value
|
||||
});
|
||||
}
|
||||
|
||||
// init routes
|
||||
registerStatusPage(kbnServer, server, config);
|
||||
registerStatusApi(kbnServer, server, config);
|
||||
registerStatsApi(kbnServer, server, config, collector);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,269 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`Metrics Collector collection should accumulate counter metrics 1`] = `
|
||||
Object {
|
||||
"collection_interval_in_millis": "test-123",
|
||||
"concurrent_connections": 0,
|
||||
"event_loop_delay": 0.3764979839324951,
|
||||
"last_updated": "2018-04-19T21:50:54.366Z",
|
||||
"name": "test-123",
|
||||
"os": Object {
|
||||
"cpu": Object {
|
||||
"load_average": Object {
|
||||
"15m": 1.81201171875,
|
||||
"1m": 1.97119140625,
|
||||
"5m": 1.90283203125,
|
||||
},
|
||||
},
|
||||
"mem": Object {
|
||||
"free_in_bytes": 12,
|
||||
"total_in_bytes": 24,
|
||||
},
|
||||
},
|
||||
"process": Object {
|
||||
"mem": Object {
|
||||
"external_in_bytes": 25028,
|
||||
"heap_max_in_bytes": 15548416,
|
||||
"heap_used_in_bytes": 12996392,
|
||||
"resident_set_size_in_bytes": 36085760,
|
||||
},
|
||||
"pid": 7777,
|
||||
"uptime_ms": 6666000,
|
||||
},
|
||||
"requests": Object {
|
||||
"disconnects": 0,
|
||||
"status_codes": Object {
|
||||
"200": 8,
|
||||
},
|
||||
"total": 8,
|
||||
},
|
||||
"response_times": Object {
|
||||
"avg_in_millis": 19,
|
||||
"max_in_millis": 19,
|
||||
},
|
||||
"sockets": Object {
|
||||
"http": Object {
|
||||
"total": 0,
|
||||
},
|
||||
"https": Object {
|
||||
"total": 0,
|
||||
},
|
||||
},
|
||||
"uptime_in_millis": 6666000,
|
||||
"uuid": "test-123",
|
||||
"version": Object {
|
||||
"build_hash": "test-123",
|
||||
"build_number": "test-123",
|
||||
"build_snapshot": false,
|
||||
"number": "test-123",
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Metrics Collector collection should accumulate counter metrics 2`] = `
|
||||
Object {
|
||||
"collection_interval_in_millis": "test-123",
|
||||
"concurrent_connections": 0,
|
||||
"event_loop_delay": 0.7529959678649902,
|
||||
"last_updated": "2018-04-19T21:50:54.366Z",
|
||||
"name": "test-123",
|
||||
"os": Object {
|
||||
"cpu": Object {
|
||||
"load_average": Object {
|
||||
"15m": 1.81201171875,
|
||||
"1m": 1.97119140625,
|
||||
"5m": 1.90283203125,
|
||||
},
|
||||
},
|
||||
"mem": Object {
|
||||
"free_in_bytes": 12,
|
||||
"total_in_bytes": 24,
|
||||
},
|
||||
},
|
||||
"process": Object {
|
||||
"mem": Object {
|
||||
"external_in_bytes": 25028,
|
||||
"heap_max_in_bytes": 15548416,
|
||||
"heap_used_in_bytes": 12996392,
|
||||
"resident_set_size_in_bytes": 36085760,
|
||||
},
|
||||
"pid": 7777,
|
||||
"uptime_ms": 6666000,
|
||||
},
|
||||
"requests": Object {
|
||||
"disconnects": 0,
|
||||
"status_codes": Object {
|
||||
"200": 16,
|
||||
},
|
||||
"total": 16,
|
||||
},
|
||||
"response_times": Object {
|
||||
"avg_in_millis": 38,
|
||||
"max_in_millis": 38,
|
||||
},
|
||||
"sockets": Object {
|
||||
"http": Object {
|
||||
"total": 0,
|
||||
},
|
||||
"https": Object {
|
||||
"total": 0,
|
||||
},
|
||||
},
|
||||
"uptime_in_millis": 6666000,
|
||||
"uuid": "test-123",
|
||||
"version": Object {
|
||||
"build_hash": "test-123",
|
||||
"build_number": "test-123",
|
||||
"build_snapshot": false,
|
||||
"number": "test-123",
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Metrics Collector collection should accumulate counter metrics 3`] = `
|
||||
Object {
|
||||
"collection_interval_in_millis": "test-123",
|
||||
"concurrent_connections": 0,
|
||||
"event_loop_delay": 1.1294939517974854,
|
||||
"last_updated": "2018-04-19T21:50:54.366Z",
|
||||
"name": "test-123",
|
||||
"os": Object {
|
||||
"cpu": Object {
|
||||
"load_average": Object {
|
||||
"15m": 1.81201171875,
|
||||
"1m": 1.97119140625,
|
||||
"5m": 1.90283203125,
|
||||
},
|
||||
},
|
||||
"mem": Object {
|
||||
"free_in_bytes": 12,
|
||||
"total_in_bytes": 24,
|
||||
},
|
||||
},
|
||||
"process": Object {
|
||||
"mem": Object {
|
||||
"external_in_bytes": 25028,
|
||||
"heap_max_in_bytes": 15548416,
|
||||
"heap_used_in_bytes": 12996392,
|
||||
"resident_set_size_in_bytes": 36085760,
|
||||
},
|
||||
"pid": 7777,
|
||||
"uptime_ms": 6666000,
|
||||
},
|
||||
"requests": Object {
|
||||
"disconnects": 0,
|
||||
"status_codes": Object {
|
||||
"200": 24,
|
||||
},
|
||||
"total": 24,
|
||||
},
|
||||
"response_times": Object {
|
||||
"avg_in_millis": 57,
|
||||
"max_in_millis": 57,
|
||||
},
|
||||
"sockets": Object {
|
||||
"http": Object {
|
||||
"total": 0,
|
||||
},
|
||||
"https": Object {
|
||||
"total": 0,
|
||||
},
|
||||
},
|
||||
"uptime_in_millis": 6666000,
|
||||
"uuid": "test-123",
|
||||
"version": Object {
|
||||
"build_hash": "test-123",
|
||||
"build_number": "test-123",
|
||||
"build_snapshot": false,
|
||||
"number": "test-123",
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Metrics Collector collection should update stats with new data 1`] = `
|
||||
Object {
|
||||
"collection_interval_in_millis": "test-123",
|
||||
"concurrent_connections": 0,
|
||||
"event_loop_delay": 0.33843398094177246,
|
||||
"last_updated": "2018-04-19T21:50:54.366Z",
|
||||
"name": "test-123",
|
||||
"os": Object {
|
||||
"cpu": Object {
|
||||
"load_average": Object {
|
||||
"15m": 1.8154296875,
|
||||
"1m": 1.68017578125,
|
||||
"5m": 1.7685546875,
|
||||
},
|
||||
},
|
||||
"mem": Object {
|
||||
"free_in_bytes": 12,
|
||||
"total_in_bytes": 24,
|
||||
},
|
||||
},
|
||||
"process": Object {
|
||||
"mem": Object {
|
||||
"external_in_bytes": 25028,
|
||||
"heap_max_in_bytes": 15548416,
|
||||
"heap_used_in_bytes": 12911128,
|
||||
"resident_set_size_in_bytes": 35307520,
|
||||
},
|
||||
"pid": 7777,
|
||||
"uptime_ms": 6666000,
|
||||
},
|
||||
"requests": Object {
|
||||
"disconnects": 0,
|
||||
"status_codes": Object {
|
||||
"200": 4,
|
||||
},
|
||||
"total": 4,
|
||||
},
|
||||
"response_times": Object {
|
||||
"avg_in_millis": 13,
|
||||
"max_in_millis": 13,
|
||||
},
|
||||
"sockets": Object {
|
||||
"http": Object {
|
||||
"total": 0,
|
||||
},
|
||||
"https": Object {
|
||||
"total": 0,
|
||||
},
|
||||
},
|
||||
"uptime_in_millis": 6666000,
|
||||
"uuid": "test-123",
|
||||
"version": Object {
|
||||
"build_hash": "test-123",
|
||||
"build_number": "test-123",
|
||||
"build_snapshot": false,
|
||||
"number": "test-123",
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Metrics Collector initialize should return stub metrics 1`] = `
|
||||
Object {
|
||||
"name": "test-123",
|
||||
"os": Object {
|
||||
"cpu": Object {},
|
||||
"mem": Object {},
|
||||
},
|
||||
"process": Object {
|
||||
"mem": Object {},
|
||||
},
|
||||
"requests": Object {
|
||||
"status_codes": Object {},
|
||||
},
|
||||
"response_times": Object {},
|
||||
"sockets": Object {
|
||||
"http": Object {},
|
||||
"https": Object {},
|
||||
},
|
||||
"uuid": "test-123",
|
||||
"version": Object {
|
||||
"build_hash": "test-123",
|
||||
"build_number": "test-123",
|
||||
"build_snapshot": false,
|
||||
"number": "test-123",
|
||||
},
|
||||
}
|
||||
`;
|
1
src/server/status/metrics_collector/index.js
Normal file
1
src/server/status/metrics_collector/index.js
Normal file
|
@ -0,0 +1 @@
|
|||
export { MetricsCollector } from './metrics_collector';
|
|
@ -1,3 +1,4 @@
|
|||
import os from 'os';
|
||||
import { get, isObject, merge } from 'lodash';
|
||||
import { keysToSnakeCaseShallow } from '../../../utils/case_conversion';
|
||||
import { getAllStats as cGroupStats } from './cgroup';
|
||||
|
@ -9,6 +10,26 @@ export class Metrics {
|
|||
this.checkCGroupStats = true;
|
||||
}
|
||||
|
||||
static getStubMetrics() {
|
||||
return {
|
||||
process: {
|
||||
mem: {}
|
||||
},
|
||||
os: {
|
||||
cpu: {},
|
||||
mem: {}
|
||||
},
|
||||
response_times: {},
|
||||
requests: {
|
||||
status_codes: {}
|
||||
},
|
||||
sockets: {
|
||||
http: {},
|
||||
https: {}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async capture(hapiEvent) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const event = this.captureEvent(hapiEvent);
|
||||
|
@ -17,7 +38,7 @@ export class Metrics {
|
|||
const metrics = {
|
||||
last_updated: timestamp,
|
||||
collection_interval_in_millis: this.config.get('ops.interval'),
|
||||
uptime_in_millis: process.uptime() * 1000,
|
||||
uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
|
||||
};
|
||||
|
||||
return merge(metrics, event, cgroup);
|
||||
|
@ -26,12 +47,20 @@ export class Metrics {
|
|||
captureEvent(hapiEvent) {
|
||||
const port = this.config.get('server.port');
|
||||
|
||||
const avgInMillis = get(hapiEvent, ['responseTimes', port, 'avg']); // sadly, it's possible for this to be NaN
|
||||
const maxInMillis = get(hapiEvent, ['responseTimes', port, 'max']);
|
||||
|
||||
return {
|
||||
process: {
|
||||
mem: {
|
||||
// https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage
|
||||
heap_max_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
|
||||
heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed')
|
||||
}
|
||||
heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
|
||||
resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'),
|
||||
external_in_bytes: get(hapiEvent, 'psmem.external')
|
||||
},
|
||||
pid: process.pid,
|
||||
uptime_ms: process.uptime() * 1000
|
||||
},
|
||||
os: {
|
||||
cpu: {
|
||||
|
@ -40,14 +69,21 @@ export class Metrics {
|
|||
'5m': get(hapiEvent, 'osload.1'),
|
||||
'15m': get(hapiEvent, 'osload.2')
|
||||
}
|
||||
},
|
||||
mem: {
|
||||
free_in_bytes: os.freemem(),
|
||||
total_in_bytes: os.totalmem()
|
||||
}
|
||||
},
|
||||
response_times: {
|
||||
avg_in_millis: get(hapiEvent, ['responseTimes', port, 'avg']),
|
||||
max_in_millis: get(hapiEvent, ['responseTimes', port, 'max'])
|
||||
// TODO: rename to use `_ms` suffix per beats naming conventions
|
||||
avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined
|
||||
max_in_millis: maxInMillis
|
||||
},
|
||||
requests: keysToSnakeCaseShallow(get(hapiEvent, ['requests', port])),
|
||||
concurrent_connections: get(hapiEvent, ['concurrents', port])
|
||||
concurrent_connections: get(hapiEvent, ['concurrents', port]),
|
||||
sockets: get(hapiEvent, 'sockets'),
|
||||
event_loop_delay: get(hapiEvent, 'psdelay')
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
@ -1,10 +1,24 @@
|
|||
jest.mock('fs', () => ({
|
||||
readFile: jest.fn()
|
||||
}));
|
||||
|
||||
jest.mock('os', () => ({
|
||||
freemem: jest.fn(),
|
||||
totalmem: jest.fn()
|
||||
}));
|
||||
|
||||
jest.mock('process');
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import _ from 'lodash';
|
||||
import sinon from 'sinon';
|
||||
import mockFs from 'mock-fs';
|
||||
import { cGroups as cGroupsFsStub } from './__mocks__/_fs_stubs';
|
||||
import { cGroups as cGroupsFsStub, setMockFiles, readFileMock } from './__mocks__/_fs_stubs';
|
||||
import { Metrics } from './metrics';
|
||||
|
||||
describe('Metrics', function () {
|
||||
fs.readFile.mockImplementation(readFileMock);
|
||||
|
||||
const sampleConfig = {
|
||||
ops: {
|
||||
interval: 5000
|
||||
|
@ -24,30 +38,38 @@ describe('Metrics', function () {
|
|||
});
|
||||
|
||||
afterEach(() => {
|
||||
mockFs.restore();
|
||||
setMockFiles();
|
||||
});
|
||||
|
||||
|
||||
describe('capture', () => {
|
||||
it('merges all metrics', async () => {
|
||||
mockFs();
|
||||
sinon.stub(metrics, 'captureEvent').returns({ 'a': [{ 'b': 2 }, { 'd': 4 }] });
|
||||
setMockFiles();
|
||||
sinon.stub(metrics, 'captureEvent').returns({ 'a': [{ 'b': 2 }, { 'd': 4 }], process: { uptime_ms: 1980 } });
|
||||
sinon.stub(metrics, 'captureCGroupsIfAvailable').returns({ 'a': [{ 'c': 3 }, { 'e': 5 }] });
|
||||
sinon.stub(Date.prototype, 'toISOString').returns('2017-04-14T18:35:41.534Z');
|
||||
sinon.stub(process, 'uptime').returns(5000);
|
||||
|
||||
const capturedMetrics = await metrics.capture();
|
||||
expect(capturedMetrics).toEqual({
|
||||
last_updated: '2017-04-14T18:35:41.534Z',
|
||||
collection_interval_in_millis: 5000,
|
||||
uptime_in_millis: 5000000,
|
||||
a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ]
|
||||
uptime_in_millis: 1980,
|
||||
a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ], process: { uptime_ms: 1980 }
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('captureEvent', () => {
|
||||
it('parses the hapi event', () => {
|
||||
sinon.stub(process, 'uptime').returns(5000);
|
||||
|
||||
os.freemem.mockImplementation(() => 12);
|
||||
os.totalmem.mockImplementation(() => 24);
|
||||
|
||||
const pidMock = jest.fn();
|
||||
pidMock.mockReturnValue(8675309);
|
||||
Object.defineProperty(process, 'pid', { get: pidMock }); //
|
||||
|
||||
const hapiEvent = {
|
||||
'requests': { '5603': { 'total': 22, 'disconnects': 0, 'statusCodes': { '200': 22 } } },
|
||||
'responseTimes': { '5603': { 'avg': 1.8636363636363635, 'max': 4 } },
|
||||
|
@ -59,14 +81,15 @@ describe('Metrics', function () {
|
|||
'osmem': { 'total': 17179869184, 'free': 102318080 },
|
||||
'osup': 1008991,
|
||||
'psup': 7.168,
|
||||
'psmem': { 'rss': 193716224, 'heapTotal': 168194048, 'heapUsed': 130553400 },
|
||||
'psmem': { 'rss': 193716224, 'heapTotal': 168194048, 'heapUsed': 130553400, 'external': 1779619 },
|
||||
'concurrents': { '5603': 0 },
|
||||
'psdelay': 1.6091690063476562,
|
||||
'host': '123'
|
||||
'host': 'blahblah.local'
|
||||
};
|
||||
|
||||
expect(metrics.captureEvent(hapiEvent)).toEqual({
|
||||
'concurrent_connections': 0,
|
||||
'event_loop_delay': 1.6091690063476562,
|
||||
'os': {
|
||||
'cpu': {
|
||||
'load_average': {
|
||||
|
@ -74,13 +97,21 @@ describe('Metrics', function () {
|
|||
'1m': 2.20751953125,
|
||||
'5m': 2.02294921875
|
||||
}
|
||||
}
|
||||
},
|
||||
'mem': {
|
||||
'free_in_bytes': 12,
|
||||
'total_in_bytes': 24,
|
||||
},
|
||||
},
|
||||
'process': {
|
||||
'mem': {
|
||||
'external_in_bytes': 1779619,
|
||||
'heap_max_in_bytes': 168194048,
|
||||
'heap_used_in_bytes': 130553400
|
||||
}
|
||||
'heap_used_in_bytes': 130553400,
|
||||
'resident_set_size_in_bytes': 193716224,
|
||||
},
|
||||
'pid': 8675309,
|
||||
'uptime_ms': 5000000
|
||||
},
|
||||
'requests': {
|
||||
'disconnects': 0,
|
||||
|
@ -92,18 +123,46 @@ describe('Metrics', function () {
|
|||
'response_times': {
|
||||
'avg_in_millis': 1.8636363636363635,
|
||||
'max_in_millis': 4
|
||||
},
|
||||
'sockets': {
|
||||
'http': {
|
||||
'total': 0
|
||||
},
|
||||
'https': {
|
||||
'total': 0
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('parses event with missing fields / NaN for responseTimes.avg', () => {
|
||||
const hapiEvent = {
|
||||
requests: {
|
||||
'5603': { total: 22, disconnects: 0, statusCodes: { '200': 22 } },
|
||||
},
|
||||
responseTimes: { '5603': { avg: NaN, max: 4 } },
|
||||
host: 'blahblah.local',
|
||||
};
|
||||
|
||||
expect(metrics.captureEvent(hapiEvent)).toEqual({
|
||||
process: { mem: {}, pid: 8675309, uptime_ms: 5000000 },
|
||||
os: {
|
||||
cpu: { load_average: {} },
|
||||
mem: { free_in_bytes: 12, total_in_bytes: 24 },
|
||||
},
|
||||
response_times: { max_in_millis: 4 },
|
||||
requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('captureCGroups', () => {
|
||||
afterEach(() => {
|
||||
mockFs.restore();
|
||||
setMockFiles();
|
||||
});
|
||||
|
||||
it('returns undefined if cgroups do not exist', async () => {
|
||||
mockFs();
|
||||
setMockFiles();
|
||||
|
||||
const stats = await metrics.captureCGroups();
|
||||
|
||||
|
@ -112,7 +171,7 @@ describe('Metrics', function () {
|
|||
|
||||
it('returns cgroups', async () => {
|
||||
const fsStub = cGroupsFsStub();
|
||||
mockFs(fsStub.files);
|
||||
setMockFiles(fsStub.files);
|
||||
|
||||
const capturedMetrics = await metrics.captureCGroups();
|
||||
|
||||
|
@ -141,11 +200,11 @@ describe('Metrics', function () {
|
|||
|
||||
describe('captureCGroupsIfAvailable', () => {
|
||||
afterEach(() => {
|
||||
mockFs.restore();
|
||||
setMockFiles();
|
||||
});
|
||||
|
||||
it('marks cgroups as unavailable and prevents subsequent calls', async () => {
|
||||
mockFs();
|
||||
setMockFiles();
|
||||
sinon.spy(metrics, 'captureCGroups');
|
||||
|
||||
expect(metrics.checkCGroupStats).toBe(true);
|
||||
|
@ -159,7 +218,7 @@ describe('Metrics', function () {
|
|||
|
||||
it('allows subsequent calls if cgroups are available', async () => {
|
||||
const fsStub = cGroupsFsStub();
|
||||
mockFs(fsStub.files);
|
||||
setMockFiles(fsStub.files);
|
||||
sinon.spy(metrics, 'captureCGroups');
|
||||
|
||||
expect(metrics.checkCGroupStats).toBe(true);
|
||||
|
|
94
src/server/status/metrics_collector/metrics_collector.js
Normal file
94
src/server/status/metrics_collector/metrics_collector.js
Normal file
|
@ -0,0 +1,94 @@
|
|||
import { Metrics } from './metrics';
|
||||
|
||||
const matchSnapshot = /-SNAPSHOT$/;
|
||||
|
||||
/*
|
||||
* Persist operational data for machine reading
|
||||
* sets the latest guage values
|
||||
* sums the latest accumulative values
|
||||
*/
|
||||
export class MetricsCollector {
|
||||
constructor(server, config) {
|
||||
|
||||
// NOTE we need access to config every time this is used because uuid is managed by the kibana core_plugin, which is initialized AFTER kbn_server
|
||||
this._getBaseStats = () => ({
|
||||
name: config.get('server.name'),
|
||||
uuid: config.get('server.uuid'),
|
||||
version: {
|
||||
number: config.get('pkg.version').replace(matchSnapshot, ''),
|
||||
build_hash: config.get('pkg.buildSha'),
|
||||
build_number: config.get('pkg.buildNum'),
|
||||
build_snapshot: matchSnapshot.test(config.get('pkg.version'))
|
||||
}
|
||||
});
|
||||
|
||||
this._stats = Metrics.getStubMetrics();
|
||||
this._metrics = new Metrics(config, server); // TODO: deprecate status API that uses Metrics class, move it this module, fix the order of its constructor params
|
||||
}
|
||||
|
||||
/*
|
||||
* Accumulate metrics by summing values in an accumulutor object with the next values
|
||||
*
|
||||
* @param {String} property The property of the objects to roll up
|
||||
* @param {Object} accum The accumulator object
|
||||
* @param {Object} next The object containing next values
|
||||
*/
|
||||
static sumAccumulate(property, accum, next) {
|
||||
const accumValue = accum[property];
|
||||
const nextValue = next[property];
|
||||
|
||||
if (nextValue === null || nextValue === undefined) {
|
||||
return; // do not accumulate null/undefined since it can't be part of a sum
|
||||
} else if (nextValue.constructor === Object) { // nested structure, object
|
||||
const newProps = {};
|
||||
for (const innerKey in nextValue) {
|
||||
if (nextValue.hasOwnProperty(innerKey)) {
|
||||
const tempAccumValue = accumValue || {};
|
||||
newProps[innerKey] = MetricsCollector.sumAccumulate(innerKey, tempAccumValue, nextValue);
|
||||
}
|
||||
}
|
||||
return { // merge the newly summed nested values
|
||||
...accumValue,
|
||||
...newProps,
|
||||
};
|
||||
} else if (nextValue.constructor === Number) {
|
||||
// leaf value
|
||||
if (nextValue || nextValue === 0) {
|
||||
const tempAccumValue = accumValue || 0; // treat null / undefined as 0
|
||||
const tempNextValue = nextValue || 0;
|
||||
return tempAccumValue + tempNextValue; // perform sum
|
||||
}
|
||||
} else {
|
||||
return; // drop unknown type
|
||||
}
|
||||
}
|
||||
|
||||
async collect(event) {
|
||||
const capturedEvent = await this._metrics.capture(event); // wait for cgroup measurement
|
||||
const { process, os, ...metrics } = capturedEvent;
|
||||
|
||||
const stats = {
|
||||
// guage values
|
||||
...metrics,
|
||||
process,
|
||||
os,
|
||||
|
||||
// accumulative counters
|
||||
response_times: MetricsCollector.sumAccumulate('response_times', this._stats, metrics),
|
||||
requests: MetricsCollector.sumAccumulate('requests', this._stats, metrics),
|
||||
concurrent_connections: MetricsCollector.sumAccumulate('concurrent_connections', this._stats, metrics),
|
||||
sockets: MetricsCollector.sumAccumulate('sockets', this._stats, metrics),
|
||||
event_loop_delay: MetricsCollector.sumAccumulate('event_loop_delay', this._stats, metrics),
|
||||
};
|
||||
|
||||
this._stats = stats;
|
||||
return stats;
|
||||
}
|
||||
|
||||
getStats() {
|
||||
return {
|
||||
...this._getBaseStats(),
|
||||
...this._stats
|
||||
};
|
||||
}
|
||||
}
|
145
src/server/status/metrics_collector/metrics_collector.test.js
Normal file
145
src/server/status/metrics_collector/metrics_collector.test.js
Normal file
|
@ -0,0 +1,145 @@
|
|||
jest.mock('os', () => ({
|
||||
freemem: jest.fn(),
|
||||
totalmem: jest.fn()
|
||||
}));
|
||||
|
||||
const mockProcessUptime = jest.fn().mockImplementation(() => 6666);
|
||||
jest.mock('process', () => ({
|
||||
uptime: mockProcessUptime
|
||||
}));
|
||||
|
||||
import os from 'os';
|
||||
import sinon from 'sinon';
|
||||
import { MetricsCollector } from './';
|
||||
|
||||
const mockServer = {};
|
||||
const mockConfig = {
|
||||
get: sinon.stub(),
|
||||
};
|
||||
mockConfig.get.returns('test-123');
|
||||
mockConfig.get.withArgs('server.port').returns(3000);
|
||||
|
||||
describe('Metrics Collector', () => {
|
||||
describe('initialize', () => {
|
||||
it('should return stub metrics', () => {
|
||||
const collector = new MetricsCollector(mockServer, mockConfig);
|
||||
expect(collector.getStats()).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe('collection', () => {
|
||||
os.freemem.mockImplementation(() => 12);
|
||||
os.totalmem.mockImplementation(() => 24);
|
||||
|
||||
Object.defineProperty(process, 'pid', { value: 7777 });
|
||||
Object.defineProperty(process, 'uptime', { value: mockProcessUptime });
|
||||
|
||||
let sandbox;
|
||||
let clock;
|
||||
beforeAll(() => {
|
||||
sandbox = sinon.sandbox.create();
|
||||
clock = sinon.useFakeTimers(1524174654366);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
clock.restore();
|
||||
sandbox.restore();
|
||||
});
|
||||
|
||||
it('should update stats with new data', async () => {
|
||||
const collector = new MetricsCollector(mockServer, mockConfig);
|
||||
|
||||
await collector.collect({
|
||||
requests: {
|
||||
'3000': { total: 4, disconnects: 0, statusCodes: { '200': 4 } },
|
||||
},
|
||||
responseTimes: { '3000': { avg: 13, max: 13 } },
|
||||
sockets: { http: { total: 0 }, https: { total: 0 } },
|
||||
osload: [1.68017578125, 1.7685546875, 1.8154296875],
|
||||
osmem: { total: 17179869184, free: 3984404480 },
|
||||
psmem: {
|
||||
rss: 35307520,
|
||||
heapTotal: 15548416,
|
||||
heapUsed: 12911128,
|
||||
external: 25028,
|
||||
},
|
||||
concurrents: { '3000': 0 },
|
||||
osup: 965002,
|
||||
psup: 29.466,
|
||||
psdelay: 0.33843398094177246,
|
||||
host: 'spicy.local',
|
||||
});
|
||||
expect(collector.getStats()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should accumulate counter metrics', async () => {
|
||||
const collector = new MetricsCollector(mockServer, mockConfig);
|
||||
|
||||
await collector.collect({
|
||||
requests: {
|
||||
'3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
|
||||
},
|
||||
responseTimes: { '3000': { avg: 19, max: 19 } },
|
||||
sockets: { http: { total: 0 }, https: { total: 0 } },
|
||||
osload: [1.97119140625, 1.90283203125, 1.81201171875],
|
||||
osmem: { total: 17179869184, free: 3987533824 },
|
||||
psmem: {
|
||||
rss: 36085760,
|
||||
heapTotal: 15548416,
|
||||
heapUsed: 12996392,
|
||||
external: 25028,
|
||||
},
|
||||
concurrents: { '3000': 0 },
|
||||
osup: 965606,
|
||||
psup: 22.29,
|
||||
psdelay: 0.3764979839324951,
|
||||
host: 'spicy.local',
|
||||
});
|
||||
expect(collector.getStats()).toMatchSnapshot();
|
||||
|
||||
await collector.collect({
|
||||
requests: {
|
||||
'3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
|
||||
},
|
||||
responseTimes: { '3000': { avg: 19, max: 19 } },
|
||||
sockets: { http: { total: 0 }, https: { total: 0 } },
|
||||
osload: [1.97119140625, 1.90283203125, 1.81201171875],
|
||||
osmem: { total: 17179869184, free: 3987533824 },
|
||||
psmem: {
|
||||
rss: 36085760,
|
||||
heapTotal: 15548416,
|
||||
heapUsed: 12996392,
|
||||
external: 25028,
|
||||
},
|
||||
concurrents: { '3000': 0 },
|
||||
osup: 965606,
|
||||
psup: 22.29,
|
||||
psdelay: 0.3764979839324951,
|
||||
host: 'spicy.local',
|
||||
});
|
||||
expect(collector.getStats()).toMatchSnapshot();
|
||||
|
||||
await collector.collect({
|
||||
requests: {
|
||||
'3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
|
||||
},
|
||||
responseTimes: { '3000': { avg: 19, max: 19 } },
|
||||
sockets: { http: { total: 0 }, https: { total: 0 } },
|
||||
osload: [1.97119140625, 1.90283203125, 1.81201171875],
|
||||
osmem: { total: 17179869184, free: 3987533824 },
|
||||
psmem: {
|
||||
rss: 36085760,
|
||||
heapTotal: 15548416,
|
||||
heapUsed: 12996392,
|
||||
external: 25028,
|
||||
},
|
||||
concurrents: { '3000': 0 },
|
||||
osup: 965606,
|
||||
psup: 22.29,
|
||||
psdelay: 0.3764979839324951,
|
||||
host: 'spicy.local',
|
||||
});
|
||||
expect(collector.getStats()).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
});
|
82
src/server/status/metrics_collector/sum_accumulate.test.js
Normal file
82
src/server/status/metrics_collector/sum_accumulate.test.js
Normal file
|
@ -0,0 +1,82 @@
|
|||
import { MetricsCollector } from './';
|
||||
|
||||
const { sumAccumulate } = MetricsCollector;
|
||||
|
||||
describe('Accumulate By Summing Metrics', function () {
|
||||
it('should accumulate empty object with nothing as nothing', () => {
|
||||
const accum = { blues: {} };
|
||||
const current = sumAccumulate('blues', accum, {});
|
||||
expect(current).toEqual(undefined);
|
||||
});
|
||||
|
||||
it('should return data to merge with initial empty data', () => {
|
||||
let accum = { blues: {} };
|
||||
const next = { blues: { total: 1 } };
|
||||
const accumulated = sumAccumulate('blues', accum, next);
|
||||
accum = { ...accum, blues: accumulated };
|
||||
expect(accum).toEqual({ blues: { total: 1 } });
|
||||
});
|
||||
|
||||
it('should return data to merge with already accumulated data', () => {
|
||||
let currentProp;
|
||||
let accumulated;
|
||||
|
||||
// initial
|
||||
let accum = {
|
||||
reds: 1,
|
||||
oranges: { total: 2 },
|
||||
yellows: { total: 3 },
|
||||
greens: { total: 4 },
|
||||
blues: { dislikes: 2, likes: 3, total: 5 },
|
||||
indigos: { total: 6 },
|
||||
violets: { total: 7 },
|
||||
};
|
||||
|
||||
// first accumulation - existing nested object
|
||||
currentProp = 'blues';
|
||||
accumulated = sumAccumulate(currentProp, accum, {
|
||||
[currentProp]: { likes: 2, total: 2 },
|
||||
});
|
||||
accum = { ...accum, [currentProp]: accumulated };
|
||||
expect(accum).toEqual({
|
||||
reds: 1,
|
||||
oranges: { total: 2 },
|
||||
yellows: { total: 3 },
|
||||
greens: { total: 4 },
|
||||
blues: { dislikes: 2, likes: 5, total: 7 },
|
||||
indigos: { total: 6 },
|
||||
violets: { total: 7 },
|
||||
});
|
||||
|
||||
// second accumulation - existing non-nested object
|
||||
currentProp = 'reds';
|
||||
accumulated = sumAccumulate(currentProp, accum, { [currentProp]: 2 });
|
||||
accum = { ...accum, [currentProp]: accumulated };
|
||||
expect(accum).toEqual({
|
||||
reds: 3,
|
||||
oranges: { total: 2 },
|
||||
yellows: { total: 3 },
|
||||
greens: { total: 4 },
|
||||
blues: { dislikes: 2, likes: 5, total: 7 },
|
||||
indigos: { total: 6 },
|
||||
violets: { total: 7 },
|
||||
});
|
||||
|
||||
// third accumulation - new nested object prop
|
||||
currentProp = 'ultraviolets';
|
||||
accumulated = sumAccumulate(currentProp, accum, {
|
||||
[currentProp]: { total: 1, likes: 1, dislikes: 0 },
|
||||
});
|
||||
accum = { ...accum, [currentProp]: accumulated };
|
||||
expect(accum).toEqual({
|
||||
reds: 3,
|
||||
oranges: { total: 2 },
|
||||
yellows: { total: 3 },
|
||||
greens: { total: 4 },
|
||||
blues: { dislikes: 2, likes: 5, total: 7 },
|
||||
indigos: { total: 6 },
|
||||
violets: { total: 7 },
|
||||
ultraviolets: { dislikes: 0, likes: 1, total: 1 },
|
||||
});
|
||||
});
|
||||
});
|
52
src/server/status/routes/api/register_stats.js
Normal file
52
src/server/status/routes/api/register_stats.js
Normal file
|
@ -0,0 +1,52 @@
|
|||
import Joi from 'joi';
|
||||
import { wrapAuthConfig } from '../../wrap_auth_config';
|
||||
|
||||
/*
|
||||
* API for Kibana meta info and accumulated operations stats
|
||||
* Including ?extended in the query string fetches Elasticsearch cluster_uuid
|
||||
* - Requests to set isExtended = true
|
||||
* GET /api/stats?extended=true
|
||||
* GET /api/stats?extended
|
||||
* - No value or 'false' is isExtended = false
|
||||
* - Any other value causes a statusCode 400 response (Bad Request)
|
||||
*/
|
||||
export function registerStatsApi(kbnServer, server, config, collector) {
|
||||
const wrapAuth = wrapAuthConfig(config.get('status.allowAnonymous'));
|
||||
server.route(
|
||||
wrapAuth({
|
||||
method: 'GET',
|
||||
path: '/api/stats',
|
||||
config: {
|
||||
validate: {
|
||||
query: {
|
||||
extended: Joi.string().valid('', 'true', 'false')
|
||||
}
|
||||
},
|
||||
tags: ['api'],
|
||||
},
|
||||
async handler(req, reply) {
|
||||
const { extended } = req.query;
|
||||
const isExtended = extended !== undefined && extended !== 'false';
|
||||
|
||||
let clusterUuid;
|
||||
if (isExtended) {
|
||||
try {
|
||||
const { callWithRequest, } = server.plugins.elasticsearch.getCluster('data');
|
||||
const { cluster_uuid: uuid } = await callWithRequest(req, 'info', { filterPath: 'cluster_uuid', });
|
||||
clusterUuid = uuid;
|
||||
} catch (err) {
|
||||
clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
|
||||
}
|
||||
}
|
||||
|
||||
const stats = {
|
||||
cluster_uuid: clusterUuid, // serialization makes an undefined get stripped out, as undefined isn't a JSON type
|
||||
status: kbnServer.status.toJSON(),
|
||||
...collector.getStats(),
|
||||
};
|
||||
|
||||
reply(stats);
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
|
@ -1,2 +1,3 @@
|
|||
export { registerStatusPage } from './page/register_status';
|
||||
export { registerStatusApi } from './api/register_status';
|
||||
export { registerStatsApi } from './api/register_stats';
|
||||
|
|
|
@ -9,5 +9,6 @@ export default function ({ loadTestFile }) {
|
|||
loadTestFile(require.resolve('./shorten'));
|
||||
loadTestFile(require.resolve('./suggestions'));
|
||||
loadTestFile(require.resolve('./status'));
|
||||
loadTestFile(require.resolve('./stats'));
|
||||
});
|
||||
}
|
||||
|
|
6
test/api_integration/apis/stats/index.js
Normal file
6
test/api_integration/apis/stats/index.js
Normal file
|
@ -0,0 +1,6 @@
|
|||
export default function ({ loadTestFile }) {
|
||||
describe('stats', () => {
|
||||
loadTestFile(require.resolve('./stats'));
|
||||
});
|
||||
}
|
||||
|
86
test/api_integration/apis/stats/stats.js
Normal file
86
test/api_integration/apis/stats/stats.js
Normal file
|
@ -0,0 +1,86 @@
|
|||
import expect from 'expect.js';
|
||||
|
||||
const assertStatsAndMetrics = body => {
|
||||
expect(body.status.overall.state).to.be('green');
|
||||
expect(body.status.statuses).to.be.an('array');
|
||||
const kibanaPlugin = body.status.statuses.find(s => {
|
||||
return s.id.indexOf('plugin:kibana') === 0;
|
||||
});
|
||||
expect(kibanaPlugin.state).to.be('green');
|
||||
|
||||
expect(body.name).to.be.a('string');
|
||||
expect(body.uuid).to.be.a('string');
|
||||
|
||||
expect(body.version.number).to.be.a('string');
|
||||
|
||||
expect(body.process.mem.external_in_bytes).to.be.an('number');
|
||||
expect(body.process.mem.heap_max_in_bytes).to.be.an('number');
|
||||
expect(body.process.mem.heap_used_in_bytes).to.be.an('number');
|
||||
expect(body.process.mem.resident_set_size_in_bytes).to.be.an('number');
|
||||
expect(body.process.pid).to.be.an('number');
|
||||
expect(body.process.uptime_ms).to.be.an('number');
|
||||
|
||||
expect(body.os.cpu.load_average['1m']).to.be.a('number');
|
||||
|
||||
expect(body.response_times.avg_in_millis).not.to.be(null); // ok if is undefined
|
||||
expect(body.response_times.max_in_millis).not.to.be(null); // ok if is undefined
|
||||
|
||||
expect(body.requests.status_codes).to.be.an('object');
|
||||
|
||||
expect(body.sockets.http).to.be.an('object');
|
||||
expect(body.sockets.https).to.be.an('object');
|
||||
|
||||
expect(body.concurrent_connections).to.be.a('number');
|
||||
|
||||
expect(body.event_loop_delay).to.be.an('number');
|
||||
};
|
||||
|
||||
export default function ({ getService }) {
|
||||
const supertest = getService('supertest');
|
||||
|
||||
describe('kibana stats api', () => {
|
||||
it('should return the stats and metric fields without cluster_uuid when extended param is not present', () => {
|
||||
return supertest
|
||||
.get('/api/stats')
|
||||
.expect('Content-Type', /json/)
|
||||
.expect(200)
|
||||
.then(({ body }) => {
|
||||
expect(body.cluster_uuid).to.be(undefined);
|
||||
assertStatsAndMetrics(body);
|
||||
});
|
||||
});
|
||||
it('should return the stats and metric fields without cluster_uuid when extended param is given as false', () => {
|
||||
return supertest
|
||||
.get('/api/stats?extended=false')
|
||||
.expect('Content-Type', /json/)
|
||||
.expect(200)
|
||||
.then(({ body }) => {
|
||||
expect(body.cluster_uuid).to.be(undefined);
|
||||
assertStatsAndMetrics(body);
|
||||
});
|
||||
});
|
||||
|
||||
it('should return the stats and metric fields with cluster_uuid when extended param is present', () => {
|
||||
return supertest
|
||||
.get('/api/stats?extended')
|
||||
.expect('Content-Type', /json/)
|
||||
.expect(200)
|
||||
.then(({ body }) => {
|
||||
expect(body.cluster_uuid).to.be.a('string');
|
||||
assertStatsAndMetrics(body);
|
||||
});
|
||||
});
|
||||
|
||||
it('should return the stats and metric fields with cluster_uuid when extended param is given as true', () => {
|
||||
return supertest
|
||||
.get('/api/stats?extended=true')
|
||||
.expect('Content-Type', /json/)
|
||||
.expect(200)
|
||||
.then(({ body }) => {
|
||||
expect(body.cluster_uuid).to.be.a('string');
|
||||
assertStatsAndMetrics(body);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -34,9 +34,8 @@ export default function ({ getService }) {
|
|||
expect(body.metrics.os.cpu.load_average['5m']).to.be.a('number');
|
||||
expect(body.metrics.os.cpu.load_average['15m']).to.be.a('number');
|
||||
|
||||
// TODO: fix this in the status/metrics class so this is always defined
|
||||
// expect(body.metrics.response_times.avg_in_millis).not.to.be(undefined); // a number, but is null if no measurements have yet been collected for averaging
|
||||
expect(body.metrics.response_times.max_in_millis).to.be.a('number');
|
||||
expect(body.metrics.response_times.avg_in_millis).not.to.be(null); // ok if undefined
|
||||
expect(body.metrics.response_times.max_in_millis).not.to.be(null); // ok if undefined
|
||||
|
||||
expect(body.metrics.requests.total).to.be.a('number');
|
||||
expect(body.metrics.requests.disconnects).to.be.a('number');
|
||||
|
|
Loading…
Reference in a new issue