[APM] Optimize synthtrace (#116091)
This commit is contained in:
Dario Gieselaar
GitHub
parent
d442213c0e
commit
9d9eeb0e8c
|
|
@ -93,23 +93,24 @@ const esEvents = toElasticsearchOutput([
|
|||
|
||||
Via the CLI, you can upload scenarios, either using a fixed time range or continuously generating data. Some examples are available in in `src/scripts/examples`. Here's an example for live data:
|
||||
|
||||
`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
|
||||
`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
|
||||
|
||||
For a fixed time window:
|
||||
`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
|
||||
`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
|
||||
|
||||
The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
|
||||
The script will try to automatically find bootstrapped APM indices. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
|
||||
|
||||
The following options are supported:
|
||||
| Option | Description | Default |
|
||||
| -------------- | ------------------------------------------------------- | ------------ |
|
||||
| `--from` | The start of the time window. | `now - 15m` |
|
||||
| `--to` | The end of the time window. | `now` |
|
||||
| `--live` | Continously ingest data | `false` |
|
||||
| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
|
||||
| `--clean` | Clean APM indices before indexing new data. | `false` |
|
||||
| `--interval` | The interval at which to index data. | `10s` |
|
||||
| `--logLevel` | Log level. | `info` |
|
||||
| `--lookback` | The lookback window for which data should be generated. | `15m` |
|
||||
| `--target` | Elasticsearch target, including username/password. | **Required** |
|
||||
| `--workers` | Amount of simultaneously connected ES clients. | `1` |
|
||||
| Option | Description | Default |
|
||||
| ------------------| ------------------------------------------------------- | ------------ |
|
||||
| `--target` | Elasticsearch target, including username/password. | **Required** |
|
||||
| `--from` | The start of the time window. | `now - 15m` |
|
||||
| `--to` | The end of the time window. | `now` |
|
||||
| `--live` | Continously ingest data | `false` |
|
||||
| `--clean` | Clean APM indices before indexing new data. | `false` |
|
||||
| `--workers` | Amount of Node.js worker threads | `5` |
|
||||
| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
|
||||
| `--interval` | The interval at which to index data. | `10s` |
|
||||
| `--clientWorkers` | Number of simultaneously connected ES clients | `5` |
|
||||
| `--batchSize` | Number of documents per bulk index request | `1000` |
|
||||
| `--logLevel` | Log level. | `info` |
|
||||
|
|
|
|||
|
|
@ -22,6 +22,18 @@ export interface ElasticsearchOutputWriteTargets {
|
|||
metric: string;
|
||||
}
|
||||
|
||||
const observerDefaults = getObserverDefaults();
|
||||
|
||||
const esDocumentDefaults = {
|
||||
ecs: {
|
||||
version: '1.4',
|
||||
},
|
||||
};
|
||||
|
||||
// eslint-disable-next-line guard-for-in
|
||||
for (const key in observerDefaults) {
|
||||
set(esDocumentDefaults, key, observerDefaults[key as keyof typeof observerDefaults]);
|
||||
}
|
||||
export function toElasticsearchOutput({
|
||||
events,
|
||||
writeTargets,
|
||||
|
|
@ -30,22 +42,25 @@ export function toElasticsearchOutput({
|
|||
writeTargets: ElasticsearchOutputWriteTargets;
|
||||
}): ElasticsearchOutput[] {
|
||||
return events.map((event) => {
|
||||
const values = {
|
||||
...event,
|
||||
...getObserverDefaults(),
|
||||
const values = {};
|
||||
|
||||
Object.assign(values, event, {
|
||||
'@timestamp': new Date(event['@timestamp']!).toISOString(),
|
||||
'timestamp.us': event['@timestamp']! * 1000,
|
||||
'ecs.version': '1.4',
|
||||
'service.node.name':
|
||||
event['service.node.name'] || event['container.id'] || event['host.name'],
|
||||
};
|
||||
});
|
||||
|
||||
const document = {};
|
||||
|
||||
Object.assign(document, esDocumentDefaults);
|
||||
|
||||
// eslint-disable-next-line guard-for-in
|
||||
for (const key in values) {
|
||||
const val = values[key as keyof typeof values];
|
||||
set(document, key, val);
|
||||
}
|
||||
|
||||
return {
|
||||
_index: writeTargets[event['processor.event'] as keyof ElasticsearchOutputWriteTargets],
|
||||
_source: document,
|
||||
|
|
|
|||
|
|
@ -6,20 +6,17 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import uuidv5 from 'uuid/v5';
|
||||
|
||||
let seq = 0;
|
||||
|
||||
const namespace = 'f38d5b83-8eee-4f5b-9aa6-2107e15a71e3';
|
||||
|
||||
function generateId(seed?: string) {
|
||||
return uuidv5(seed ?? String(seq++), namespace).replace(/-/g, '');
|
||||
function generateId(seed?: string, length: number = 32) {
|
||||
const str = seed ?? String(seq++);
|
||||
return str.padStart(length, '0');
|
||||
}
|
||||
|
||||
export function generateShortId(seed?: string) {
|
||||
return generateId(seed).substr(0, 16);
|
||||
return generateId(seed, 16);
|
||||
}
|
||||
|
||||
export function generateLongId(seed?: string) {
|
||||
return generateId(seed).substr(0, 32);
|
||||
return generateId(seed, 32);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,16 +10,20 @@ import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics }
|
|||
import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics';
|
||||
|
||||
export default function ({ from, to }: { from: number; to: number }) {
|
||||
const instance = service('opbeans-go', 'production', 'go').instance('instance');
|
||||
const numServices = 3;
|
||||
|
||||
const range = timerange(from, to);
|
||||
|
||||
const transactionName = '240rpm/75% 1000ms';
|
||||
|
||||
const successfulTraceEvents = range
|
||||
.interval('1s')
|
||||
.rate(3)
|
||||
.flatMap((timestamp) =>
|
||||
const successfulTimestamps = range.interval('1s').rate(3);
|
||||
|
||||
const failedTimestamps = range.interval('1s').rate(1);
|
||||
|
||||
return new Array(numServices).fill(undefined).flatMap((_, index) => {
|
||||
const instance = service(`opbeans-go-${index}`, 'production', 'go').instance('instance');
|
||||
|
||||
const successfulTraceEvents = successfulTimestamps.flatMap((timestamp) =>
|
||||
instance
|
||||
.transaction(transactionName)
|
||||
.timestamp(timestamp)
|
||||
|
|
@ -37,10 +41,7 @@ export default function ({ from, to }: { from: number; to: number }) {
|
|||
.serialize()
|
||||
);
|
||||
|
||||
const failedTraceEvents = range
|
||||
.interval('1s')
|
||||
.rate(1)
|
||||
.flatMap((timestamp) =>
|
||||
const failedTraceEvents = failedTimestamps.flatMap((timestamp) =>
|
||||
instance
|
||||
.transaction(transactionName)
|
||||
.timestamp(timestamp)
|
||||
|
|
@ -52,27 +53,28 @@ export default function ({ from, to }: { from: number; to: number }) {
|
|||
.serialize()
|
||||
);
|
||||
|
||||
const metricsets = range
|
||||
.interval('30s')
|
||||
.rate(1)
|
||||
.flatMap((timestamp) =>
|
||||
instance
|
||||
.appMetrics({
|
||||
'system.memory.actual.free': 800,
|
||||
'system.memory.total': 1000,
|
||||
'system.cpu.total.norm.pct': 0.6,
|
||||
'system.process.cpu.total.norm.pct': 0.7,
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
.serialize()
|
||||
);
|
||||
const events = successfulTraceEvents.concat(failedTraceEvents);
|
||||
const metricsets = range
|
||||
.interval('30s')
|
||||
.rate(1)
|
||||
.flatMap((timestamp) =>
|
||||
instance
|
||||
.appMetrics({
|
||||
'system.memory.actual.free': 800,
|
||||
'system.memory.total': 1000,
|
||||
'system.cpu.total.norm.pct': 0.6,
|
||||
'system.process.cpu.total.norm.pct': 0.7,
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
.serialize()
|
||||
);
|
||||
const events = successfulTraceEvents.concat(failedTraceEvents);
|
||||
|
||||
return [
|
||||
...events,
|
||||
...metricsets,
|
||||
...getTransactionMetrics(events),
|
||||
...getSpanDestinationMetrics(events),
|
||||
...getBreakdownMetrics(events),
|
||||
];
|
||||
return [
|
||||
...events,
|
||||
...metricsets,
|
||||
...getTransactionMetrics(events),
|
||||
...getSpanDestinationMetrics(events),
|
||||
...getBreakdownMetrics(events),
|
||||
];
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,15 +8,6 @@
|
|||
import datemath from '@elastic/datemath';
|
||||
import yargs from 'yargs/yargs';
|
||||
import { cleanWriteTargets } from './utils/clean_write_targets';
|
||||
import {
|
||||
bucketSizeOption,
|
||||
cleanOption,
|
||||
fileOption,
|
||||
intervalOption,
|
||||
targetOption,
|
||||
workerOption,
|
||||
logLevelOption,
|
||||
} from './utils/common_options';
|
||||
import { intervalToMs } from './utils/interval_to_ms';
|
||||
import { getCommonResources } from './utils/get_common_resources';
|
||||
import { startHistoricalDataUpload } from './utils/start_historical_data_upload';
|
||||
|
|
@ -28,13 +19,16 @@ yargs(process.argv.slice(2))
|
|||
'Generate data and index into Elasticsearch',
|
||||
(y) => {
|
||||
return y
|
||||
.positional('file', fileOption)
|
||||
.option('bucketSize', bucketSizeOption)
|
||||
.option('workers', workerOption)
|
||||
.option('interval', intervalOption)
|
||||
.option('clean', cleanOption)
|
||||
.option('target', targetOption)
|
||||
.option('logLevel', logLevelOption)
|
||||
.positional('file', {
|
||||
describe: 'File that contains the trace scenario',
|
||||
demandOption: true,
|
||||
string: true,
|
||||
})
|
||||
.option('target', {
|
||||
describe: 'Elasticsearch target, including username/password',
|
||||
demandOption: true,
|
||||
string: true,
|
||||
})
|
||||
.option('from', {
|
||||
description: 'The start of the time window',
|
||||
})
|
||||
|
|
@ -45,20 +39,47 @@ yargs(process.argv.slice(2))
|
|||
description: 'Generate and index data continuously',
|
||||
boolean: true,
|
||||
})
|
||||
.option('clean', {
|
||||
describe: 'Clean APM indices before indexing new data',
|
||||
default: false,
|
||||
boolean: true,
|
||||
})
|
||||
.option('workers', {
|
||||
describe: 'Amount of Node.js worker threads',
|
||||
default: 5,
|
||||
})
|
||||
.option('bucketSize', {
|
||||
describe: 'Size of bucket for which to generate data',
|
||||
default: '15m',
|
||||
})
|
||||
.option('interval', {
|
||||
describe: 'The interval at which to index data',
|
||||
default: '10s',
|
||||
})
|
||||
.option('clientWorkers', {
|
||||
describe: 'Number of concurrently connected ES clients',
|
||||
default: 5,
|
||||
})
|
||||
.option('batchSize', {
|
||||
describe: 'Number of documents per bulk index request',
|
||||
default: 1000,
|
||||
})
|
||||
.option('logLevel', {
|
||||
describe: 'Log level',
|
||||
default: 'info',
|
||||
})
|
||||
.conflicts('to', 'live');
|
||||
},
|
||||
async (argv) => {
|
||||
const {
|
||||
scenario,
|
||||
intervalInMs,
|
||||
bucketSizeInMs,
|
||||
target,
|
||||
workers,
|
||||
clean,
|
||||
logger,
|
||||
writeTargets,
|
||||
client,
|
||||
} = await getCommonResources(argv);
|
||||
const file = String(argv.file || argv._[0]);
|
||||
|
||||
const { target, workers, clean, clientWorkers, batchSize } = argv;
|
||||
|
||||
const { scenario, intervalInMs, bucketSizeInMs, logger, writeTargets, client, logLevel } =
|
||||
await getCommonResources({
|
||||
...argv,
|
||||
file,
|
||||
});
|
||||
|
||||
if (clean) {
|
||||
await cleanWriteTargets({ writeTargets, client, logger });
|
||||
|
|
@ -91,13 +112,16 @@ yargs(process.argv.slice(2))
|
|||
startHistoricalDataUpload({
|
||||
from,
|
||||
to,
|
||||
scenario,
|
||||
intervalInMs,
|
||||
file,
|
||||
bucketSizeInMs,
|
||||
client,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
logger,
|
||||
logLevel,
|
||||
target,
|
||||
});
|
||||
|
||||
if (live) {
|
||||
|
|
@ -108,7 +132,8 @@ yargs(process.argv.slice(2))
|
|||
logger,
|
||||
scenario,
|
||||
start: to,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
const fileOption = {
|
||||
describe: 'File that contains the trace scenario',
|
||||
demandOption: true,
|
||||
};
|
||||
|
||||
const intervalOption = {
|
||||
describe: 'The interval at which to index data',
|
||||
default: '10s',
|
||||
};
|
||||
|
||||
const targetOption = {
|
||||
describe: 'Elasticsearch target, including username/password',
|
||||
demandOption: true,
|
||||
};
|
||||
|
||||
const bucketSizeOption = {
|
||||
describe: 'Size of bucket for which to generate data',
|
||||
default: '15m',
|
||||
};
|
||||
|
||||
const workerOption = {
|
||||
describe: 'Amount of simultaneously connected ES clients',
|
||||
default: 1,
|
||||
};
|
||||
|
||||
const cleanOption = {
|
||||
describe: 'Clean APM indices before indexing new data',
|
||||
default: false,
|
||||
boolean: true as const,
|
||||
};
|
||||
|
||||
const logLevelOption = {
|
||||
describe: 'Log level',
|
||||
default: 'info',
|
||||
};
|
||||
|
||||
export {
|
||||
fileOption,
|
||||
intervalOption,
|
||||
targetOption,
|
||||
bucketSizeOption,
|
||||
workerOption,
|
||||
cleanOption,
|
||||
logLevelOption,
|
||||
};
|
||||
|
|
@ -16,21 +16,21 @@ export async function getCommonResources({
|
|||
file,
|
||||
interval,
|
||||
bucketSize,
|
||||
workers,
|
||||
target,
|
||||
clean,
|
||||
logLevel,
|
||||
}: {
|
||||
file: unknown;
|
||||
interval: unknown;
|
||||
bucketSize: unknown;
|
||||
workers: unknown;
|
||||
target: unknown;
|
||||
clean: boolean;
|
||||
logLevel: unknown;
|
||||
file: string;
|
||||
interval: string;
|
||||
bucketSize: string;
|
||||
target: string;
|
||||
logLevel: string;
|
||||
}) {
|
||||
let parsedLogLevel = LogLevel.info;
|
||||
switch (logLevel) {
|
||||
case 'trace':
|
||||
parsedLogLevel = LogLevel.trace;
|
||||
break;
|
||||
|
||||
case 'info':
|
||||
parsedLogLevel = LogLevel.info;
|
||||
break;
|
||||
|
|
@ -39,8 +39,8 @@ export async function getCommonResources({
|
|||
parsedLogLevel = LogLevel.debug;
|
||||
break;
|
||||
|
||||
case 'quiet':
|
||||
parsedLogLevel = LogLevel.quiet;
|
||||
case 'error':
|
||||
parsedLogLevel = LogLevel.error;
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ export async function getCommonResources({
|
|||
}
|
||||
|
||||
const client = new Client({
|
||||
node: String(target),
|
||||
node: target,
|
||||
});
|
||||
|
||||
const [scenario, writeTargets] = await Promise.all([
|
||||
|
|
@ -73,8 +73,6 @@ export async function getCommonResources({
|
|||
client,
|
||||
intervalInMs,
|
||||
bucketSizeInMs,
|
||||
workers: Number(workers),
|
||||
target: String(target),
|
||||
clean,
|
||||
logLevel: parsedLogLevel,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,8 @@ export async function getWriteTargets({
|
|||
)?.[0],
|
||||
};
|
||||
})
|
||||
.find(({ key }) => key.includes(filter))?.writeIndexAlias!;
|
||||
.find(({ key, writeIndexAlias }) => writeIndexAlias && key.includes(filter))
|
||||
?.writeIndexAlias!;
|
||||
}
|
||||
|
||||
const targets = {
|
||||
|
|
|
|||
|
|
@ -6,24 +6,59 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { isPromise } from 'util/types';
|
||||
|
||||
export enum LogLevel {
|
||||
debug = 0,
|
||||
info = 1,
|
||||
quiet = 2,
|
||||
trace = 0,
|
||||
debug = 1,
|
||||
info = 2,
|
||||
error = 3,
|
||||
}
|
||||
|
||||
function getTimeString() {
|
||||
return `[${new Date().toLocaleTimeString()}]`;
|
||||
}
|
||||
|
||||
export function createLogger(logLevel: LogLevel) {
|
||||
function logPerf(name: string, start: bigint) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug(
|
||||
getTimeString(),
|
||||
`${name}: ${Number(process.hrtime.bigint() - start) / 1000000}ms`
|
||||
);
|
||||
}
|
||||
return {
|
||||
perf: <T extends any>(name: string, cb: () => T): T => {
|
||||
if (logLevel <= LogLevel.trace) {
|
||||
const start = process.hrtime.bigint();
|
||||
const val = cb();
|
||||
if (isPromise(val)) {
|
||||
val.then(() => {
|
||||
logPerf(name, start);
|
||||
});
|
||||
} else {
|
||||
logPerf(name, start);
|
||||
}
|
||||
return val;
|
||||
}
|
||||
return cb();
|
||||
},
|
||||
debug: (...args: any[]) => {
|
||||
if (logLevel <= LogLevel.debug) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug(...args);
|
||||
console.debug(getTimeString(), ...args);
|
||||
}
|
||||
},
|
||||
info: (...args: any[]) => {
|
||||
if (logLevel <= LogLevel.info) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(...args);
|
||||
console.log(getTimeString(), ...args);
|
||||
}
|
||||
},
|
||||
error: (...args: any[]) => {
|
||||
if (logLevel <= LogLevel.error) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(getTimeString(), ...args);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,60 +5,105 @@
|
|||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import pLimit from 'p-limit';
|
||||
import Path from 'path';
|
||||
import { Worker } from 'worker_threads';
|
||||
import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
|
||||
import { Scenario } from './get_scenario';
|
||||
import { Logger } from './logger';
|
||||
import { uploadEvents } from './upload_events';
|
||||
import { Logger, LogLevel } from './logger';
|
||||
|
||||
export async function startHistoricalDataUpload({
|
||||
from,
|
||||
to,
|
||||
scenario,
|
||||
intervalInMs,
|
||||
bucketSizeInMs,
|
||||
client,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
logLevel,
|
||||
logger,
|
||||
target,
|
||||
file,
|
||||
}: {
|
||||
from: number;
|
||||
to: number;
|
||||
scenario: Scenario;
|
||||
intervalInMs: number;
|
||||
bucketSizeInMs: number;
|
||||
client: Client;
|
||||
workers: number;
|
||||
clientWorkers: number;
|
||||
batchSize: number;
|
||||
writeTargets: ElasticsearchOutputWriteTargets;
|
||||
logger: Logger;
|
||||
logLevel: LogLevel;
|
||||
target: string;
|
||||
file: string;
|
||||
}) {
|
||||
let requestedUntil: number = from;
|
||||
function uploadNextBatch() {
|
||||
|
||||
function processNextBatch() {
|
||||
const bucketFrom = requestedUntil;
|
||||
const bucketTo = Math.min(to, bucketFrom + bucketSizeInMs);
|
||||
|
||||
const events = scenario({ from: bucketFrom, to: bucketTo });
|
||||
|
||||
logger.info(
|
||||
`Uploading: ${new Date(bucketFrom).toISOString()} to ${new Date(bucketTo).toISOString()}`
|
||||
);
|
||||
|
||||
uploadEvents({
|
||||
events,
|
||||
client,
|
||||
workers,
|
||||
writeTargets,
|
||||
logger,
|
||||
}).then(() => {
|
||||
if (bucketTo >= to) {
|
||||
return;
|
||||
}
|
||||
uploadNextBatch();
|
||||
});
|
||||
if (bucketFrom === bucketTo) {
|
||||
return;
|
||||
}
|
||||
|
||||
requestedUntil = bucketTo;
|
||||
|
||||
logger.info(
|
||||
`Starting worker for ${new Date(bucketFrom).toISOString()} to ${new Date(
|
||||
bucketTo
|
||||
).toISOString()}`
|
||||
);
|
||||
|
||||
const worker = new Worker(Path.join(__dirname, './upload_next_batch.js'), {
|
||||
workerData: {
|
||||
bucketFrom,
|
||||
bucketTo,
|
||||
logLevel,
|
||||
writeTargets,
|
||||
target,
|
||||
file,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
},
|
||||
});
|
||||
|
||||
logger.perf('created_worker', () => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
worker.on('online', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
logger.perf('completed_worker', () => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
worker.on('exit', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
worker.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
reject(new Error(`Worker stopped: exit code ${code}`));
|
||||
return;
|
||||
}
|
||||
logger.debug('Worker completed');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return uploadNextBatch();
|
||||
const numBatches = Math.ceil((to - from) / bucketSizeInMs);
|
||||
|
||||
const limiter = pLimit(workers);
|
||||
|
||||
return Promise.all(new Array(numBatches).fill(undefined).map((_) => limiter(processNextBatch)));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ export function startLiveDataUpload({
|
|||
start,
|
||||
bucketSizeInMs,
|
||||
intervalInMs,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
scenario,
|
||||
client,
|
||||
|
|
@ -27,7 +28,8 @@ export function startLiveDataUpload({
|
|||
start: number;
|
||||
bucketSizeInMs: number;
|
||||
intervalInMs: number;
|
||||
workers: number;
|
||||
clientWorkers: number;
|
||||
batchSize: number;
|
||||
writeTargets: ElasticsearchOutputWriteTargets;
|
||||
scenario: Scenario;
|
||||
client: Client;
|
||||
|
|
@ -63,7 +65,8 @@ export function startLiveDataUpload({
|
|||
uploadEvents({
|
||||
events: eventsToUpload,
|
||||
client,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
logger,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -19,20 +19,24 @@ import { Logger } from './logger';
|
|||
export function uploadEvents({
|
||||
events,
|
||||
client,
|
||||
workers,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
logger,
|
||||
}: {
|
||||
events: Fields[];
|
||||
client: Client;
|
||||
workers: number;
|
||||
clientWorkers: number;
|
||||
batchSize: number;
|
||||
writeTargets: ElasticsearchOutputWriteTargets;
|
||||
logger: Logger;
|
||||
}) {
|
||||
const esDocuments = toElasticsearchOutput({ events, writeTargets });
|
||||
const fn = pLimit(workers);
|
||||
const esDocuments = logger.perf('to_elasticsearch_output', () => {
|
||||
return toElasticsearchOutput({ events, writeTargets });
|
||||
});
|
||||
const fn = pLimit(clientWorkers);
|
||||
|
||||
const batches = chunk(esDocuments, 5000);
|
||||
const batches = chunk(esDocuments, batchSize);
|
||||
|
||||
logger.debug(`Uploading ${esDocuments.length} in ${batches.length} batches`);
|
||||
|
||||
|
|
@ -41,12 +45,15 @@ export function uploadEvents({
|
|||
return Promise.all(
|
||||
batches.map((batch) =>
|
||||
fn(() => {
|
||||
return client.bulk({
|
||||
require_alias: true,
|
||||
body: batch.flatMap((doc) => {
|
||||
return [{ index: { _index: doc._index } }, doc._source];
|
||||
}),
|
||||
});
|
||||
return logger.perf('bulk_upload', () =>
|
||||
client.bulk({
|
||||
require_alias: true,
|
||||
refresh: false,
|
||||
body: batch.flatMap((doc) => {
|
||||
return [{ index: { _index: doc._index } }, doc._source];
|
||||
}),
|
||||
})
|
||||
);
|
||||
})
|
||||
)
|
||||
)
|
||||
|
|
@ -57,8 +64,7 @@ export function uploadEvents({
|
|||
.map((item) => item.index?.error);
|
||||
|
||||
if (errors.length) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(inspect(errors.slice(0, 10), { depth: null }));
|
||||
logger.error(inspect(errors.slice(0, 10), { depth: null }));
|
||||
throw new Error('Failed to upload some items');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-var-requires*/
|
||||
require('@babel/register')({
|
||||
extensions: ['.ts', '.js'],
|
||||
presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
|
||||
});
|
||||
|
||||
require('./upload_next_batch.ts');
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
// add this to workerExample.js file.
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { workerData } from 'worker_threads';
|
||||
import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
|
||||
import { getScenario } from './get_scenario';
|
||||
import { createLogger, LogLevel } from './logger';
|
||||
import { uploadEvents } from './upload_events';
|
||||
|
||||
const { bucketFrom, bucketTo, file, logLevel, target, writeTargets, clientWorkers, batchSize } =
|
||||
workerData as {
|
||||
bucketFrom: number;
|
||||
bucketTo: number;
|
||||
file: string;
|
||||
logLevel: LogLevel;
|
||||
target: string;
|
||||
writeTargets: ElasticsearchOutputWriteTargets;
|
||||
clientWorkers: number;
|
||||
batchSize: number;
|
||||
};
|
||||
|
||||
async function uploadNextBatch() {
|
||||
if (bucketFrom === bucketTo) {
|
||||
return;
|
||||
}
|
||||
|
||||
const logger = createLogger(logLevel);
|
||||
const client = new Client({
|
||||
node: target,
|
||||
});
|
||||
|
||||
const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
|
||||
|
||||
const events = logger.perf('execute_scenario', () =>
|
||||
scenario({ from: bucketFrom, to: bucketTo })
|
||||
);
|
||||
|
||||
return uploadEvents({
|
||||
events,
|
||||
client,
|
||||
clientWorkers,
|
||||
batchSize,
|
||||
writeTargets,
|
||||
logger,
|
||||
});
|
||||
}
|
||||
|
||||
uploadNextBatch()
|
||||
.then(() => {
|
||||
process.exit(0);
|
||||
})
|
||||
.catch(() => {
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -75,9 +75,9 @@ describe('simple trace', () => {
|
|||
'service.environment': 'production',
|
||||
'service.name': 'opbeans-java',
|
||||
'service.node.name': 'instance-1',
|
||||
'trace.id': 'f6eb2f1cbba2597e89d2a63771c4344d',
|
||||
'trace.id': '00000000000000000000000000000241',
|
||||
'transaction.duration.us': 1000000,
|
||||
'transaction.id': 'e9ece67cbacb52bf',
|
||||
'transaction.id': '0000000000000240',
|
||||
'transaction.name': 'GET /api/product/list',
|
||||
'transaction.type': 'request',
|
||||
'transaction.sampled': true,
|
||||
|
|
@ -92,19 +92,19 @@ describe('simple trace', () => {
|
|||
'agent.name': 'java',
|
||||
'container.id': 'instance-1',
|
||||
'event.outcome': 'success',
|
||||
'parent.id': 'e7433020f2745625',
|
||||
'parent.id': '0000000000000300',
|
||||
'processor.event': 'span',
|
||||
'processor.name': 'transaction',
|
||||
'service.environment': 'production',
|
||||
'service.name': 'opbeans-java',
|
||||
'service.node.name': 'instance-1',
|
||||
'span.duration.us': 900000,
|
||||
'span.id': '21a776b44b9853dd',
|
||||
'span.id': '0000000000000302',
|
||||
'span.name': 'GET apm-*/_search',
|
||||
'span.subtype': 'elasticsearch',
|
||||
'span.type': 'db',
|
||||
'trace.id': '048a0647263853abb94649ec0b92bdb4',
|
||||
'transaction.id': 'e7433020f2745625',
|
||||
'trace.id': '00000000000000000000000000000301',
|
||||
'transaction.id': '0000000000000300',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -61,6 +61,6 @@ describe('transactions with errors', () => {
|
|||
.serialize();
|
||||
|
||||
expect(error['error.grouping_name']).toEqual('test error');
|
||||
expect(error['error.grouping_key']).toMatchInlineSnapshot(`"8b96fa10a7f85a5d960198627bf50840"`);
|
||||
expect(error['error.grouping_key']).toMatchInlineSnapshot(`"0000000000000000000000test error"`);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
|
||||
"trace.id": "00000000000000000000000000000001",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "36c16f18e75058f8",
|
||||
"transaction.id": "0000000000000000",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -24,19 +24,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "36c16f18e75058f8",
|
||||
"parent.id": "0000000000000000",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "fe778a305e6d57dd",
|
||||
"span.id": "0000000000000002",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
|
||||
"transaction.id": "36c16f18e75058f8",
|
||||
"trace.id": "00000000000000000000000000000001",
|
||||
"transaction.id": "0000000000000000",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459260000,
|
||||
|
|
@ -48,9 +48,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
|
||||
"trace.id": "00000000000000000000000000000005",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "65ce74106eb050be",
|
||||
"transaction.id": "0000000000000004",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -60,19 +60,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "65ce74106eb050be",
|
||||
"parent.id": "0000000000000004",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "ad8c5e249a8658ec",
|
||||
"span.id": "0000000000000006",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
|
||||
"transaction.id": "65ce74106eb050be",
|
||||
"trace.id": "00000000000000000000000000000005",
|
||||
"transaction.id": "0000000000000004",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459320000,
|
||||
|
|
@ -84,9 +84,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
|
||||
"trace.id": "00000000000000000000000000000009",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "91fa709d90625fff",
|
||||
"transaction.id": "0000000000000008",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -96,19 +96,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "91fa709d90625fff",
|
||||
"parent.id": "0000000000000008",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "228b569c530c52ac",
|
||||
"span.id": "0000000000000010",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
|
||||
"transaction.id": "91fa709d90625fff",
|
||||
"trace.id": "00000000000000000000000000000009",
|
||||
"transaction.id": "0000000000000008",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459380000,
|
||||
|
|
@ -120,9 +120,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "6e8da4beb752589a86d53287c9d902de",
|
||||
"trace.id": "00000000000000000000000000000013",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "6c500d1d19835e68",
|
||||
"transaction.id": "0000000000000012",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -132,19 +132,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "6c500d1d19835e68",
|
||||
"parent.id": "0000000000000012",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "5eb13f140bde5334",
|
||||
"span.id": "0000000000000014",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "6e8da4beb752589a86d53287c9d902de",
|
||||
"transaction.id": "6c500d1d19835e68",
|
||||
"trace.id": "00000000000000000000000000000013",
|
||||
"transaction.id": "0000000000000012",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459440000,
|
||||
|
|
@ -156,9 +156,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
|
||||
"trace.id": "00000000000000000000000000000017",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "1b3246cc83595869",
|
||||
"transaction.id": "0000000000000016",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -168,19 +168,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "1b3246cc83595869",
|
||||
"parent.id": "0000000000000016",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "582221c79fd75a76",
|
||||
"span.id": "0000000000000018",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
|
||||
"transaction.id": "1b3246cc83595869",
|
||||
"trace.id": "00000000000000000000000000000017",
|
||||
"transaction.id": "0000000000000016",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459500000,
|
||||
|
|
@ -192,9 +192,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
|
||||
"trace.id": "00000000000000000000000000000021",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "12b49e3c83fe58d5",
|
||||
"transaction.id": "0000000000000020",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -204,19 +204,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "12b49e3c83fe58d5",
|
||||
"parent.id": "0000000000000020",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "526d186996835c09",
|
||||
"span.id": "0000000000000022",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
|
||||
"transaction.id": "12b49e3c83fe58d5",
|
||||
"trace.id": "00000000000000000000000000000021",
|
||||
"transaction.id": "0000000000000020",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459560000,
|
||||
|
|
@ -228,9 +228,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "c17c414c0b51564ca30e2ad839393180",
|
||||
"trace.id": "00000000000000000000000000000025",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "d9272009dd4354a1",
|
||||
"transaction.id": "0000000000000024",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -240,19 +240,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "d9272009dd4354a1",
|
||||
"parent.id": "0000000000000024",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "7582541fcbfc5dc6",
|
||||
"span.id": "0000000000000026",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "c17c414c0b51564ca30e2ad839393180",
|
||||
"transaction.id": "d9272009dd4354a1",
|
||||
"trace.id": "00000000000000000000000000000025",
|
||||
"transaction.id": "0000000000000024",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459620000,
|
||||
|
|
@ -264,9 +264,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
|
||||
"trace.id": "00000000000000000000000000000029",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "bc52ca08063c505b",
|
||||
"transaction.id": "0000000000000028",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -276,19 +276,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "bc52ca08063c505b",
|
||||
"parent.id": "0000000000000028",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "37ab978487935abb",
|
||||
"span.id": "0000000000000030",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
|
||||
"transaction.id": "bc52ca08063c505b",
|
||||
"trace.id": "00000000000000000000000000000029",
|
||||
"transaction.id": "0000000000000028",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459680000,
|
||||
|
|
@ -300,9 +300,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "6fb5191297fb59cebdb6a0196e273676",
|
||||
"trace.id": "00000000000000000000000000000033",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "186858dd88b75d59",
|
||||
"transaction.id": "0000000000000032",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -312,19 +312,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "186858dd88b75d59",
|
||||
"parent.id": "0000000000000032",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "5ab56f27d0ae569b",
|
||||
"span.id": "0000000000000034",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "6fb5191297fb59cebdb6a0196e273676",
|
||||
"transaction.id": "186858dd88b75d59",
|
||||
"trace.id": "00000000000000000000000000000033",
|
||||
"transaction.id": "0000000000000032",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459740000,
|
||||
|
|
@ -336,9 +336,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
|
||||
"trace.id": "00000000000000000000000000000037",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "0d5f44d48189546c",
|
||||
"transaction.id": "0000000000000036",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -348,19 +348,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "0d5f44d48189546c",
|
||||
"parent.id": "0000000000000036",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "80e94b0847cd5104",
|
||||
"span.id": "0000000000000038",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
|
||||
"transaction.id": "0d5f44d48189546c",
|
||||
"trace.id": "00000000000000000000000000000037",
|
||||
"transaction.id": "0000000000000036",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459800000,
|
||||
|
|
@ -372,9 +372,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
|
||||
"trace.id": "00000000000000000000000000000041",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "7483e0606e435c83",
|
||||
"transaction.id": "0000000000000040",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -384,19 +384,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "7483e0606e435c83",
|
||||
"parent.id": "0000000000000040",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "2e99d193e0f954c1",
|
||||
"span.id": "0000000000000042",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
|
||||
"transaction.id": "7483e0606e435c83",
|
||||
"trace.id": "00000000000000000000000000000041",
|
||||
"transaction.id": "0000000000000040",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459860000,
|
||||
|
|
@ -408,9 +408,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "5d91a6cde6015897935e413bc500f211",
|
||||
"trace.id": "00000000000000000000000000000045",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "f142c4cbc7f3568e",
|
||||
"transaction.id": "0000000000000044",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -420,19 +420,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "f142c4cbc7f3568e",
|
||||
"parent.id": "0000000000000044",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "1fc52f16e2f551ea",
|
||||
"span.id": "0000000000000046",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "5d91a6cde6015897935e413bc500f211",
|
||||
"transaction.id": "f142c4cbc7f3568e",
|
||||
"trace.id": "00000000000000000000000000000045",
|
||||
"transaction.id": "0000000000000044",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459920000,
|
||||
|
|
@ -444,9 +444,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "c097c19d884d52579bb11a601b8a98b3",
|
||||
"trace.id": "00000000000000000000000000000049",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "2e3a47fa2d905519",
|
||||
"transaction.id": "0000000000000048",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -456,19 +456,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "2e3a47fa2d905519",
|
||||
"parent.id": "0000000000000048",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "7c7828c850685337",
|
||||
"span.id": "0000000000000050",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "c097c19d884d52579bb11a601b8a98b3",
|
||||
"transaction.id": "2e3a47fa2d905519",
|
||||
"trace.id": "00000000000000000000000000000049",
|
||||
"transaction.id": "0000000000000048",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609459980000,
|
||||
|
|
@ -480,9 +480,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "4591e57f4d7f5986bdd7892561224e0f",
|
||||
"trace.id": "00000000000000000000000000000053",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "de5eaa1e47dc56b1",
|
||||
"transaction.id": "0000000000000052",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -492,19 +492,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "de5eaa1e47dc56b1",
|
||||
"parent.id": "0000000000000052",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "8f62257f4a41546a",
|
||||
"span.id": "0000000000000054",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "4591e57f4d7f5986bdd7892561224e0f",
|
||||
"transaction.id": "de5eaa1e47dc56b1",
|
||||
"trace.id": "00000000000000000000000000000053",
|
||||
"transaction.id": "0000000000000052",
|
||||
},
|
||||
Object {
|
||||
"@timestamp": 1609460040000,
|
||||
|
|
@ -516,9 +516,9 @@ Array [
|
|||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"trace.id": "85ee8e618433577b9316a1e14961aa89",
|
||||
"trace.id": "00000000000000000000000000000057",
|
||||
"transaction.duration.us": 1000000,
|
||||
"transaction.id": "af7eac7ae61e576a",
|
||||
"transaction.id": "0000000000000056",
|
||||
"transaction.name": "GET /api/product/list",
|
||||
"transaction.sampled": true,
|
||||
"transaction.type": "request",
|
||||
|
|
@ -528,19 +528,19 @@ Array [
|
|||
"agent.name": "java",
|
||||
"container.id": "instance-1",
|
||||
"event.outcome": "success",
|
||||
"parent.id": "af7eac7ae61e576a",
|
||||
"parent.id": "0000000000000056",
|
||||
"processor.event": "span",
|
||||
"processor.name": "transaction",
|
||||
"service.environment": "production",
|
||||
"service.name": "opbeans-java",
|
||||
"service.node.name": "instance-1",
|
||||
"span.duration.us": 900000,
|
||||
"span.id": "cc88b4cd921e590e",
|
||||
"span.id": "0000000000000058",
|
||||
"span.name": "GET apm-*/_search",
|
||||
"span.subtype": "elasticsearch",
|
||||
"span.type": "db",
|
||||
"trace.id": "85ee8e618433577b9316a1e14961aa89",
|
||||
"transaction.id": "af7eac7ae61e576a",
|
||||
"trace.id": "00000000000000000000000000000057",
|
||||
"transaction.id": "0000000000000056",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ describe('output to elasticsearch', () => {
|
|||
'@timestamp': new Date('2020-12-31T23:00:00.000Z').getTime(),
|
||||
'processor.event': 'transaction',
|
||||
'processor.name': 'transaction',
|
||||
'service.node.name': 'instance-a',
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -41,4 +42,33 @@ describe('output to elasticsearch', () => {
|
|||
name: 'transaction',
|
||||
});
|
||||
});
|
||||
|
||||
it('formats all fields consistently', () => {
|
||||
const doc = toElasticsearchOutput({ events: [event], writeTargets })[0] as any;
|
||||
|
||||
expect(doc._source).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"@timestamp": "2020-12-31T23:00:00.000Z",
|
||||
"ecs": Object {
|
||||
"version": "1.4",
|
||||
},
|
||||
"observer": Object {
|
||||
"version": "7.16.0",
|
||||
"version_major": 7,
|
||||
},
|
||||
"processor": Object {
|
||||
"event": "transaction",
|
||||
"name": "transaction",
|
||||
},
|
||||
"service": Object {
|
||||
"node": Object {
|
||||
"name": "instance-a",
|
||||
},
|
||||
},
|
||||
"timestamp": Object {
|
||||
"us": 1609455600000000,
|
||||
},
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue