[APM] Optimize synthtrace (#116091)

packages/elastic-apm-synthtrace/README.md

@@ -93,23 +93,24 @@ const esEvents = toElasticsearchOutput([
 
 Via the CLI, you can upload scenarios, either using a fixed time range or continuously generating data. Some examples are available in in `src/scripts/examples`. Here's an example for live data:
 
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
 
 For a fixed time window:
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
 
-The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
+The script will try to automatically find bootstrapped APM indices. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
 
 The following options are supported:
-| Option | Description | Default |
+| Option            | Description                                             | Default      |
-| -------------- | ------------------------------------------------------- | ------------ |
+| ------------------| ------------------------------------------------------- | ------------ |
-| `--from` | The start of the time window. | `now - 15m` |
+| `--target`        | Elasticsearch target, including username/password.      | **Required** |
-| `--to` | The end of the time window. | `now` |
+| `--from`          | The start of the time window.                           | `now - 15m`  |
-| `--live` | Continously ingest data | `false` |
+| `--to`            | The end of the time window.                             | `now`        |
-| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
+| `--live`          | Continously ingest data                                 | `false`      |
-| `--clean` | Clean APM indices before indexing new data. | `false` |
+| `--clean`         | Clean APM indices before indexing new data.             | `false`      |
-| `--interval` | The interval at which to index data. | `10s` |
+| `--workers`       | Amount of Node.js worker threads                        | `5`          |
-| `--logLevel` | Log level. | `info` |
+| `--bucketSize`    | Size of bucket for which to generate data.              | `15m`        |
-| `--lookback` | The lookback window for which data should be generated. | `15m` |
+| `--interval`      | The interval at which to index data.                    | `10s`        |
-| `--target` | Elasticsearch target, including username/password. | **Required** |
+| `--clientWorkers` | Number of simultaneously connected ES clients           | `5`          |
-| `--workers` | Amount of simultaneously connected ES clients. | `1` |
+| `--batchSize`     | Number of documents per bulk index request              | `1000`       |
+| `--logLevel`      | Log level.                                              | `info`       |

packages/elastic-apm-synthtrace/src/lib/output/to_elasticsearch_output.ts

@@ -22,6 +22,18 @@ export interface ElasticsearchOutputWriteTargets {
   metric: string;
 }
 
+const observerDefaults = getObserverDefaults();
+
+const esDocumentDefaults = {
+  ecs: {
+    version: '1.4',
+  },
+};
+
+// eslint-disable-next-line guard-for-in
+for (const key in observerDefaults) {
+  set(esDocumentDefaults, key, observerDefaults[key as keyof typeof observerDefaults]);
+}
 export function toElasticsearchOutput({
   events,
   writeTargets,
@@ -30,22 +42,25 @@ export function toElasticsearchOutput({
   writeTargets: ElasticsearchOutputWriteTargets;
 }): ElasticsearchOutput[] {
   return events.map((event) => {
-    const values = {
+    const values = {};
-      ...event,
+
-      ...getObserverDefaults(),
+    Object.assign(values, event, {
       '@timestamp': new Date(event['@timestamp']!).toISOString(),
       'timestamp.us': event['@timestamp']! * 1000,
-      'ecs.version': '1.4',
       'service.node.name':
         event['service.node.name'] || event['container.id'] || event['host.name'],
-    };
+    });
 
     const document = {};
+
+    Object.assign(document, esDocumentDefaults);
+
     // eslint-disable-next-line guard-for-in
     for (const key in values) {
       const val = values[key as keyof typeof values];
       set(document, key, val);
     }
+
     return {
       _index: writeTargets[event['processor.event'] as keyof ElasticsearchOutputWriteTargets],
       _source: document,

packages/elastic-apm-synthtrace/src/lib/utils/generate_id.ts

@@ -6,20 +6,17 @@
  * Side Public License, v 1.
  */
 
-import uuidv5 from 'uuid/v5';
-
 let seq = 0;
 
-const namespace = 'f38d5b83-8eee-4f5b-9aa6-2107e15a71e3';
+function generateId(seed?: string, length: number = 32) {
-
+  const str = seed ?? String(seq++);
-function generateId(seed?: string) {
+  return str.padStart(length, '0');
-  return uuidv5(seed ?? String(seq++), namespace).replace(/-/g, '');
 }
 
 export function generateShortId(seed?: string) {
-  return generateId(seed).substr(0, 16);
+  return generateId(seed, 16);
 }
 
 export function generateLongId(seed?: string) {
-  return generateId(seed).substr(0, 32);
+  return generateId(seed, 32);
 }

packages/elastic-apm-synthtrace/src/scripts/examples/01_simple_trace.ts

@@ -10,16 +10,20 @@ import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics }
 import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics';
 
 export default function ({ from, to }: { from: number; to: number }) {
-  const instance = service('opbeans-go', 'production', 'go').instance('instance');
+  const numServices = 3;
 
   const range = timerange(from, to);
 
   const transactionName = '240rpm/75% 1000ms';
 
-  const successfulTraceEvents = range
+  const successfulTimestamps = range.interval('1s').rate(3);
-    .interval('1s')
+
-    .rate(3)
+  const failedTimestamps = range.interval('1s').rate(1);
-    .flatMap((timestamp) =>
+
+  return new Array(numServices).fill(undefined).flatMap((_, index) => {
+    const instance = service(`opbeans-go-${index}`, 'production', 'go').instance('instance');
+
+    const successfulTraceEvents = successfulTimestamps.flatMap((timestamp) =>
       instance
         .transaction(transactionName)
         .timestamp(timestamp)
@@ -37,10 +41,7 @@ export default function ({ from, to }: { from: number; to: number }) {
         .serialize()
     );
 
-  const failedTraceEvents = range
+    const failedTraceEvents = failedTimestamps.flatMap((timestamp) =>
-    .interval('1s')
-    .rate(1)
-    .flatMap((timestamp) =>
       instance
         .transaction(transactionName)
         .timestamp(timestamp)
@@ -52,27 +53,28 @@ export default function ({ from, to }: { from: number; to: number }) {
         .serialize()
     );
 
-  const metricsets = range
+    const metricsets = range
-    .interval('30s')
+      .interval('30s')
-    .rate(1)
+      .rate(1)
-    .flatMap((timestamp) =>
+      .flatMap((timestamp) =>
-      instance
+        instance
-        .appMetrics({
+          .appMetrics({
-          'system.memory.actual.free': 800,
+            'system.memory.actual.free': 800,
-          'system.memory.total': 1000,
+            'system.memory.total': 1000,
-          'system.cpu.total.norm.pct': 0.6,
+            'system.cpu.total.norm.pct': 0.6,
-          'system.process.cpu.total.norm.pct': 0.7,
+            'system.process.cpu.total.norm.pct': 0.7,
-        })
+          })
-        .timestamp(timestamp)
+          .timestamp(timestamp)
-        .serialize()
+          .serialize()
-    );
+      );
-  const events = successfulTraceEvents.concat(failedTraceEvents);
+    const events = successfulTraceEvents.concat(failedTraceEvents);
 
-  return [
+    return [
-    ...events,
+      ...events,
-    ...metricsets,
+      ...metricsets,
-    ...getTransactionMetrics(events),
+      ...getTransactionMetrics(events),
-    ...getSpanDestinationMetrics(events),
+      ...getSpanDestinationMetrics(events),
-    ...getBreakdownMetrics(events),
+      ...getBreakdownMetrics(events),
-  ];
+    ];
+  });
 }

packages/elastic-apm-synthtrace/src/scripts/run.ts

@@ -8,15 +8,6 @@
 import datemath from '@elastic/datemath';
 import yargs from 'yargs/yargs';
 import { cleanWriteTargets } from './utils/clean_write_targets';
-import {
-  bucketSizeOption,
-  cleanOption,
-  fileOption,
-  intervalOption,
-  targetOption,
-  workerOption,
-  logLevelOption,
-} from './utils/common_options';
 import { intervalToMs } from './utils/interval_to_ms';
 import { getCommonResources } from './utils/get_common_resources';
 import { startHistoricalDataUpload } from './utils/start_historical_data_upload';
@@ -28,13 +19,16 @@ yargs(process.argv.slice(2))
     'Generate data and index into Elasticsearch',
     (y) => {
       return y
-        .positional('file', fileOption)
+        .positional('file', {
-        .option('bucketSize', bucketSizeOption)
+          describe: 'File that contains the trace scenario',
-        .option('workers', workerOption)
+          demandOption: true,
-        .option('interval', intervalOption)
+          string: true,
-        .option('clean', cleanOption)
+        })
-        .option('target', targetOption)
+        .option('target', {
-        .option('logLevel', logLevelOption)
+          describe: 'Elasticsearch target, including username/password',
+          demandOption: true,
+          string: true,
+        })
         .option('from', {
           description: 'The start of the time window',
         })
@@ -45,20 +39,47 @@ yargs(process.argv.slice(2))
           description: 'Generate and index data continuously',
           boolean: true,
         })
+        .option('clean', {
+          describe: 'Clean APM indices before indexing new data',
+          default: false,
+          boolean: true,
+        })
+        .option('workers', {
+          describe: 'Amount of Node.js worker threads',
+          default: 5,
+        })
+        .option('bucketSize', {
+          describe: 'Size of bucket for which to generate data',
+          default: '15m',
+        })
+        .option('interval', {
+          describe: 'The interval at which to index data',
+          default: '10s',
+        })
+        .option('clientWorkers', {
+          describe: 'Number of concurrently connected ES clients',
+          default: 5,
+        })
+        .option('batchSize', {
+          describe: 'Number of documents per bulk index request',
+          default: 1000,
+        })
+        .option('logLevel', {
+          describe: 'Log level',
+          default: 'info',
+        })
         .conflicts('to', 'live');
     },
     async (argv) => {
-      const {
+      const file = String(argv.file || argv._[0]);
-        scenario,
+
-        intervalInMs,
+      const { target, workers, clean, clientWorkers, batchSize } = argv;
-        bucketSizeInMs,
+
-        target,
+      const { scenario, intervalInMs, bucketSizeInMs, logger, writeTargets, client, logLevel } =
-        workers,
+        await getCommonResources({
-        clean,
+          ...argv,
-        logger,
+          file,
-        writeTargets,
+        });
-        client,
-      } = await getCommonResources(argv);
 
       if (clean) {
         await cleanWriteTargets({ writeTargets, client, logger });
@@ -91,13 +112,16 @@ yargs(process.argv.slice(2))
       startHistoricalDataUpload({
         from,
         to,
-        scenario,
+        file,
-        intervalInMs,
         bucketSizeInMs,
         client,
         workers,
+        clientWorkers,
+        batchSize,
         writeTargets,
         logger,
+        logLevel,
+        target,
       });
 
       if (live) {
@@ -108,7 +132,8 @@ yargs(process.argv.slice(2))
           logger,
           scenario,
           start: to,
-          workers,
+          clientWorkers,
+          batchSize,
           writeTargets,
         });
       }

packages/elastic-apm-synthtrace/src/scripts/utils/common_options.ts

@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-const fileOption = {
-  describe: 'File that contains the trace scenario',
-  demandOption: true,
-};
-
-const intervalOption = {
-  describe: 'The interval at which to index data',
-  default: '10s',
-};
-
-const targetOption = {
-  describe: 'Elasticsearch target, including username/password',
-  demandOption: true,
-};
-
-const bucketSizeOption = {
-  describe: 'Size of bucket for which to generate data',
-  default: '15m',
-};
-
-const workerOption = {
-  describe: 'Amount of simultaneously connected ES clients',
-  default: 1,
-};
-
-const cleanOption = {
-  describe: 'Clean APM indices before indexing new data',
-  default: false,
-  boolean: true as const,
-};
-
-const logLevelOption = {
-  describe: 'Log level',
-  default: 'info',
-};
-
-export {
-  fileOption,
-  intervalOption,
-  targetOption,
-  bucketSizeOption,
-  workerOption,
-  cleanOption,
-  logLevelOption,
-};

packages/elastic-apm-synthtrace/src/scripts/utils/get_common_resources.ts

@@ -16,21 +16,21 @@ export async function getCommonResources({
   file,
   interval,
   bucketSize,
-  workers,
   target,
-  clean,
   logLevel,
 }: {
-  file: unknown;
+  file: string;
-  interval: unknown;
+  interval: string;
-  bucketSize: unknown;
+  bucketSize: string;
-  workers: unknown;
+  target: string;
-  target: unknown;
+  logLevel: string;
-  clean: boolean;
-  logLevel: unknown;
 }) {
   let parsedLogLevel = LogLevel.info;
   switch (logLevel) {
+    case 'trace':
+      parsedLogLevel = LogLevel.trace;
+      break;
+
     case 'info':
       parsedLogLevel = LogLevel.info;
       break;
@@ -39,8 +39,8 @@ export async function getCommonResources({
       parsedLogLevel = LogLevel.debug;
       break;
 
-    case 'quiet':
+    case 'error':
-      parsedLogLevel = LogLevel.quiet;
+      parsedLogLevel = LogLevel.error;
       break;
   }
 
@@ -58,7 +58,7 @@ export async function getCommonResources({
   }
 
   const client = new Client({
-    node: String(target),
+    node: target,
   });
 
   const [scenario, writeTargets] = await Promise.all([
@@ -73,8 +73,6 @@ export async function getCommonResources({
     client,
     intervalInMs,
     bucketSizeInMs,
-    workers: Number(workers),
+    logLevel: parsedLogLevel,
-    target: String(target),
-    clean,
   };
 }

packages/elastic-apm-synthtrace/src/scripts/utils/get_write_targets.ts

@@ -38,7 +38,8 @@ export async function getWriteTargets({
           )?.[0],
         };
       })
-      .find(({ key }) => key.includes(filter))?.writeIndexAlias!;
+      .find(({ key, writeIndexAlias }) => writeIndexAlias && key.includes(filter))
+      ?.writeIndexAlias!;
   }
 
   const targets = {

packages/elastic-apm-synthtrace/src/scripts/utils/logger.ts

@@ -6,24 +6,59 @@
  * Side Public License, v 1.
  */
 
+import { isPromise } from 'util/types';
+
 export enum LogLevel {
-  debug = 0,
+  trace = 0,
-  info = 1,
+  debug = 1,
-  quiet = 2,
+  info = 2,
+  error = 3,
+}
+
+function getTimeString() {
+  return `[${new Date().toLocaleTimeString()}]`;
 }
 
 export function createLogger(logLevel: LogLevel) {
+  function logPerf(name: string, start: bigint) {
+    // eslint-disable-next-line no-console
+    console.debug(
+      getTimeString(),
+      `${name}: ${Number(process.hrtime.bigint() - start) / 1000000}ms`
+    );
+  }
   return {
+    perf: <T extends any>(name: string, cb: () => T): T => {
+      if (logLevel <= LogLevel.trace) {
+        const start = process.hrtime.bigint();
+        const val = cb();
+        if (isPromise(val)) {
+          val.then(() => {
+            logPerf(name, start);
+          });
+        } else {
+          logPerf(name, start);
+        }
+        return val;
+      }
+      return cb();
+    },
     debug: (...args: any[]) => {
       if (logLevel <= LogLevel.debug) {
         // eslint-disable-next-line no-console
-        console.debug(...args);
+        console.debug(getTimeString(), ...args);
       }
     },
     info: (...args: any[]) => {
       if (logLevel <= LogLevel.info) {
         // eslint-disable-next-line no-console
-        console.log(...args);
+        console.log(getTimeString(), ...args);
+      }
+    },
+    error: (...args: any[]) => {
+      if (logLevel <= LogLevel.error) {
+        // eslint-disable-next-line no-console
+        console.log(getTimeString(), ...args);
       }
     },
   };

packages/elastic-apm-synthtrace/src/scripts/utils/start_historical_data_upload.ts

@@ -5,60 +5,105 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
 import { Client } from '@elastic/elasticsearch';
+import pLimit from 'p-limit';
+import Path from 'path';
+import { Worker } from 'worker_threads';
 import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
-import { Scenario } from './get_scenario';
+import { Logger, LogLevel } from './logger';
-import { Logger } from './logger';
-import { uploadEvents } from './upload_events';
 
 export async function startHistoricalDataUpload({
   from,
   to,
-  scenario,
-  intervalInMs,
   bucketSizeInMs,
-  client,
   workers,
+  clientWorkers,
+  batchSize,
   writeTargets,
+  logLevel,
   logger,
+  target,
+  file,
 }: {
   from: number;
   to: number;
-  scenario: Scenario;
-  intervalInMs: number;
   bucketSizeInMs: number;
   client: Client;
   workers: number;
+  clientWorkers: number;
+  batchSize: number;
   writeTargets: ElasticsearchOutputWriteTargets;
   logger: Logger;
+  logLevel: LogLevel;
+  target: string;
+  file: string;
 }) {
   let requestedUntil: number = from;
-  function uploadNextBatch() {
+
+  function processNextBatch() {
     const bucketFrom = requestedUntil;
     const bucketTo = Math.min(to, bucketFrom + bucketSizeInMs);
 
-    const events = scenario({ from: bucketFrom, to: bucketTo });
+    if (bucketFrom === bucketTo) {
-
+      return;
-    logger.info(
+    }
-      `Uploading: ${new Date(bucketFrom).toISOString()} to ${new Date(bucketTo).toISOString()}`
-    );
-
-    uploadEvents({
-      events,
-      client,
-      workers,
-      writeTargets,
-      logger,
-    }).then(() => {
-      if (bucketTo >= to) {
-        return;
-      }
-      uploadNextBatch();
-    });
 
     requestedUntil = bucketTo;
+
+    logger.info(
+      `Starting worker for ${new Date(bucketFrom).toISOString()} to ${new Date(
+        bucketTo
+      ).toISOString()}`
+    );
+
+    const worker = new Worker(Path.join(__dirname, './upload_next_batch.js'), {
+      workerData: {
+        bucketFrom,
+        bucketTo,
+        logLevel,
+        writeTargets,
+        target,
+        file,
+        clientWorkers,
+        batchSize,
+      },
+    });
+
+    logger.perf('created_worker', () => {
+      return new Promise<void>((resolve, reject) => {
+        worker.on('online', () => {
+          resolve();
+        });
+      });
+    });
+
+    logger.perf('completed_worker', () => {
+      return new Promise<void>((resolve, reject) => {
+        worker.on('exit', () => {
+          resolve();
+        });
+      });
+    });
+
+    return new Promise<void>((resolve, reject) => {
+      worker.on('error', (err) => {
+        reject(err);
+      });
+
+      worker.on('exit', (code) => {
+        if (code !== 0) {
+          reject(new Error(`Worker stopped: exit code ${code}`));
+          return;
+        }
+        logger.debug('Worker completed');
+        resolve();
+      });
+    });
   }
 
-  return uploadNextBatch();
+  const numBatches = Math.ceil((to - from) / bucketSizeInMs);
+
+  const limiter = pLimit(workers);
+
+  return Promise.all(new Array(numBatches).fill(undefined).map((_) => limiter(processNextBatch)));
 }

packages/elastic-apm-synthtrace/src/scripts/utils/start_live_data_upload.ts

@@ -18,7 +18,8 @@ export function startLiveDataUpload({
   start,
   bucketSizeInMs,
   intervalInMs,
-  workers,
+  clientWorkers,
+  batchSize,
   writeTargets,
   scenario,
   client,
@@ -27,7 +28,8 @@ export function startLiveDataUpload({
   start: number;
   bucketSizeInMs: number;
   intervalInMs: number;
-  workers: number;
+  clientWorkers: number;
+  batchSize: number;
   writeTargets: ElasticsearchOutputWriteTargets;
   scenario: Scenario;
   client: Client;
@@ -63,7 +65,8 @@ export function startLiveDataUpload({
     uploadEvents({
       events: eventsToUpload,
       client,
-      workers,
+      clientWorkers,
+      batchSize,
       writeTargets,
       logger,
     });

packages/elastic-apm-synthtrace/src/scripts/utils/upload_events.ts

@@ -19,20 +19,24 @@ import { Logger } from './logger';
 export function uploadEvents({
   events,
   client,
-  workers,
+  clientWorkers,
+  batchSize,
   writeTargets,
   logger,
 }: {
   events: Fields[];
   client: Client;
-  workers: number;
+  clientWorkers: number;
+  batchSize: number;
   writeTargets: ElasticsearchOutputWriteTargets;
   logger: Logger;
 }) {
-  const esDocuments = toElasticsearchOutput({ events, writeTargets });
+  const esDocuments = logger.perf('to_elasticsearch_output', () => {
-  const fn = pLimit(workers);
+    return toElasticsearchOutput({ events, writeTargets });
+  });
+  const fn = pLimit(clientWorkers);
 
-  const batches = chunk(esDocuments, 5000);
+  const batches = chunk(esDocuments, batchSize);
 
   logger.debug(`Uploading ${esDocuments.length} in ${batches.length} batches`);
 
@@ -41,12 +45,15 @@ export function uploadEvents({
   return Promise.all(
     batches.map((batch) =>
       fn(() => {
-        return client.bulk({
+        return logger.perf('bulk_upload', () =>
-          require_alias: true,
+          client.bulk({
-          body: batch.flatMap((doc) => {
+            require_alias: true,
-            return [{ index: { _index: doc._index } }, doc._source];
+            refresh: false,
-          }),
+            body: batch.flatMap((doc) => {
-        });
+              return [{ index: { _index: doc._index } }, doc._source];
+            }),
+          })
+        );
       })
     )
   )
@@ -57,8 +64,7 @@ export function uploadEvents({
         .map((item) => item.index?.error);
 
       if (errors.length) {
-        // eslint-disable-next-line no-console
+        logger.error(inspect(errors.slice(0, 10), { depth: null }));
-        console.error(inspect(errors.slice(0, 10), { depth: null }));
         throw new Error('Failed to upload some items');
       }
 

packages/elastic-apm-synthtrace/src/scripts/utils/upload_next_batch.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/* eslint-disable @typescript-eslint/no-var-requires*/
+require('@babel/register')({
+  extensions: ['.ts', '.js'],
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
+});
+
+require('./upload_next_batch.ts');

packages/elastic-apm-synthtrace/src/scripts/utils/upload_next_batch.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// add this to workerExample.js file.
+import { Client } from '@elastic/elasticsearch';
+import { workerData } from 'worker_threads';
+import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
+import { getScenario } from './get_scenario';
+import { createLogger, LogLevel } from './logger';
+import { uploadEvents } from './upload_events';
+
+const { bucketFrom, bucketTo, file, logLevel, target, writeTargets, clientWorkers, batchSize } =
+  workerData as {
+    bucketFrom: number;
+    bucketTo: number;
+    file: string;
+    logLevel: LogLevel;
+    target: string;
+    writeTargets: ElasticsearchOutputWriteTargets;
+    clientWorkers: number;
+    batchSize: number;
+  };
+
+async function uploadNextBatch() {
+  if (bucketFrom === bucketTo) {
+    return;
+  }
+
+  const logger = createLogger(logLevel);
+  const client = new Client({
+    node: target,
+  });
+
+  const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
+
+  const events = logger.perf('execute_scenario', () =>
+    scenario({ from: bucketFrom, to: bucketTo })
+  );
+
+  return uploadEvents({
+    events,
+    client,
+    clientWorkers,
+    batchSize,
+    writeTargets,
+    logger,
+  });
+}
+
+uploadNextBatch()
+  .then(() => {
+    process.exit(0);
+  })
+  .catch(() => {
+    process.exit(1);
+  });

packages/elastic-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts

@@ -75,9 +75,9 @@ describe('simple trace', () => {
       'service.environment': 'production',
       'service.name': 'opbeans-java',
       'service.node.name': 'instance-1',
-      'trace.id': 'f6eb2f1cbba2597e89d2a63771c4344d',
+      'trace.id': '00000000000000000000000000000241',
       'transaction.duration.us': 1000000,
-      'transaction.id': 'e9ece67cbacb52bf',
+      'transaction.id': '0000000000000240',
       'transaction.name': 'GET /api/product/list',
       'transaction.type': 'request',
       'transaction.sampled': true,
@@ -92,19 +92,19 @@ describe('simple trace', () => {
       'agent.name': 'java',
       'container.id': 'instance-1',
       'event.outcome': 'success',
-      'parent.id': 'e7433020f2745625',
+      'parent.id': '0000000000000300',
       'processor.event': 'span',
       'processor.name': 'transaction',
       'service.environment': 'production',
       'service.name': 'opbeans-java',
       'service.node.name': 'instance-1',
       'span.duration.us': 900000,
-      'span.id': '21a776b44b9853dd',
+      'span.id': '0000000000000302',
       'span.name': 'GET apm-*/_search',
       'span.subtype': 'elasticsearch',
       'span.type': 'db',
-      'trace.id': '048a0647263853abb94649ec0b92bdb4',
+      'trace.id': '00000000000000000000000000000301',
-      'transaction.id': 'e7433020f2745625',
+      'transaction.id': '0000000000000300',
     });
   });
 });

packages/elastic-apm-synthtrace/src/test/scenarios/05_transactions_with_errors.test.ts

@@ -61,6 +61,6 @@ describe('transactions with errors', () => {
       .serialize();
 
     expect(error['error.grouping_name']).toEqual('test error');
-    expect(error['error.grouping_key']).toMatchInlineSnapshot(`"8b96fa10a7f85a5d960198627bf50840"`);
+    expect(error['error.grouping_key']).toMatchInlineSnapshot(`"0000000000000000000000test error"`);
   });
 });

packages/elastic-apm-synthtrace/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap

@@ -12,9 +12,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
+    "trace.id": "00000000000000000000000000000001",
     "transaction.duration.us": 1000000,
-    "transaction.id": "36c16f18e75058f8",
+    "transaction.id": "0000000000000000",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -24,19 +24,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "36c16f18e75058f8",
+    "parent.id": "0000000000000000",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "fe778a305e6d57dd",
+    "span.id": "0000000000000002",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
+    "trace.id": "00000000000000000000000000000001",
-    "transaction.id": "36c16f18e75058f8",
+    "transaction.id": "0000000000000000",
   },
   Object {
     "@timestamp": 1609459260000,
@@ -48,9 +48,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
+    "trace.id": "00000000000000000000000000000005",
     "transaction.duration.us": 1000000,
-    "transaction.id": "65ce74106eb050be",
+    "transaction.id": "0000000000000004",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -60,19 +60,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "65ce74106eb050be",
+    "parent.id": "0000000000000004",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "ad8c5e249a8658ec",
+    "span.id": "0000000000000006",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
+    "trace.id": "00000000000000000000000000000005",
-    "transaction.id": "65ce74106eb050be",
+    "transaction.id": "0000000000000004",
   },
   Object {
     "@timestamp": 1609459320000,
@@ -84,9 +84,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
+    "trace.id": "00000000000000000000000000000009",
     "transaction.duration.us": 1000000,
-    "transaction.id": "91fa709d90625fff",
+    "transaction.id": "0000000000000008",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -96,19 +96,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "91fa709d90625fff",
+    "parent.id": "0000000000000008",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "228b569c530c52ac",
+    "span.id": "0000000000000010",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
+    "trace.id": "00000000000000000000000000000009",
-    "transaction.id": "91fa709d90625fff",
+    "transaction.id": "0000000000000008",
   },
   Object {
     "@timestamp": 1609459380000,
@@ -120,9 +120,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "6e8da4beb752589a86d53287c9d902de",
+    "trace.id": "00000000000000000000000000000013",
     "transaction.duration.us": 1000000,
-    "transaction.id": "6c500d1d19835e68",
+    "transaction.id": "0000000000000012",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -132,19 +132,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "6c500d1d19835e68",
+    "parent.id": "0000000000000012",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "5eb13f140bde5334",
+    "span.id": "0000000000000014",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "6e8da4beb752589a86d53287c9d902de",
+    "trace.id": "00000000000000000000000000000013",
-    "transaction.id": "6c500d1d19835e68",
+    "transaction.id": "0000000000000012",
   },
   Object {
     "@timestamp": 1609459440000,
@@ -156,9 +156,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
+    "trace.id": "00000000000000000000000000000017",
     "transaction.duration.us": 1000000,
-    "transaction.id": "1b3246cc83595869",
+    "transaction.id": "0000000000000016",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -168,19 +168,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "1b3246cc83595869",
+    "parent.id": "0000000000000016",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "582221c79fd75a76",
+    "span.id": "0000000000000018",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
+    "trace.id": "00000000000000000000000000000017",
-    "transaction.id": "1b3246cc83595869",
+    "transaction.id": "0000000000000016",
   },
   Object {
     "@timestamp": 1609459500000,
@@ -192,9 +192,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
+    "trace.id": "00000000000000000000000000000021",
     "transaction.duration.us": 1000000,
-    "transaction.id": "12b49e3c83fe58d5",
+    "transaction.id": "0000000000000020",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -204,19 +204,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "12b49e3c83fe58d5",
+    "parent.id": "0000000000000020",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "526d186996835c09",
+    "span.id": "0000000000000022",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
+    "trace.id": "00000000000000000000000000000021",
-    "transaction.id": "12b49e3c83fe58d5",
+    "transaction.id": "0000000000000020",
   },
   Object {
     "@timestamp": 1609459560000,
@@ -228,9 +228,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "c17c414c0b51564ca30e2ad839393180",
+    "trace.id": "00000000000000000000000000000025",
     "transaction.duration.us": 1000000,
-    "transaction.id": "d9272009dd4354a1",
+    "transaction.id": "0000000000000024",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -240,19 +240,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "d9272009dd4354a1",
+    "parent.id": "0000000000000024",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "7582541fcbfc5dc6",
+    "span.id": "0000000000000026",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "c17c414c0b51564ca30e2ad839393180",
+    "trace.id": "00000000000000000000000000000025",
-    "transaction.id": "d9272009dd4354a1",
+    "transaction.id": "0000000000000024",
   },
   Object {
     "@timestamp": 1609459620000,
@@ -264,9 +264,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
+    "trace.id": "00000000000000000000000000000029",
     "transaction.duration.us": 1000000,
-    "transaction.id": "bc52ca08063c505b",
+    "transaction.id": "0000000000000028",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -276,19 +276,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "bc52ca08063c505b",
+    "parent.id": "0000000000000028",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "37ab978487935abb",
+    "span.id": "0000000000000030",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
+    "trace.id": "00000000000000000000000000000029",
-    "transaction.id": "bc52ca08063c505b",
+    "transaction.id": "0000000000000028",
   },
   Object {
     "@timestamp": 1609459680000,
@@ -300,9 +300,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "6fb5191297fb59cebdb6a0196e273676",
+    "trace.id": "00000000000000000000000000000033",
     "transaction.duration.us": 1000000,
-    "transaction.id": "186858dd88b75d59",
+    "transaction.id": "0000000000000032",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -312,19 +312,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "186858dd88b75d59",
+    "parent.id": "0000000000000032",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "5ab56f27d0ae569b",
+    "span.id": "0000000000000034",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "6fb5191297fb59cebdb6a0196e273676",
+    "trace.id": "00000000000000000000000000000033",
-    "transaction.id": "186858dd88b75d59",
+    "transaction.id": "0000000000000032",
   },
   Object {
     "@timestamp": 1609459740000,
@@ -336,9 +336,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
+    "trace.id": "00000000000000000000000000000037",
     "transaction.duration.us": 1000000,
-    "transaction.id": "0d5f44d48189546c",
+    "transaction.id": "0000000000000036",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -348,19 +348,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "0d5f44d48189546c",
+    "parent.id": "0000000000000036",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "80e94b0847cd5104",
+    "span.id": "0000000000000038",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
+    "trace.id": "00000000000000000000000000000037",
-    "transaction.id": "0d5f44d48189546c",
+    "transaction.id": "0000000000000036",
   },
   Object {
     "@timestamp": 1609459800000,
@@ -372,9 +372,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
+    "trace.id": "00000000000000000000000000000041",
     "transaction.duration.us": 1000000,
-    "transaction.id": "7483e0606e435c83",
+    "transaction.id": "0000000000000040",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -384,19 +384,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "7483e0606e435c83",
+    "parent.id": "0000000000000040",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "2e99d193e0f954c1",
+    "span.id": "0000000000000042",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
+    "trace.id": "00000000000000000000000000000041",
-    "transaction.id": "7483e0606e435c83",
+    "transaction.id": "0000000000000040",
   },
   Object {
     "@timestamp": 1609459860000,
@@ -408,9 +408,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "5d91a6cde6015897935e413bc500f211",
+    "trace.id": "00000000000000000000000000000045",
     "transaction.duration.us": 1000000,
-    "transaction.id": "f142c4cbc7f3568e",
+    "transaction.id": "0000000000000044",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -420,19 +420,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "f142c4cbc7f3568e",
+    "parent.id": "0000000000000044",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "1fc52f16e2f551ea",
+    "span.id": "0000000000000046",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "5d91a6cde6015897935e413bc500f211",
+    "trace.id": "00000000000000000000000000000045",
-    "transaction.id": "f142c4cbc7f3568e",
+    "transaction.id": "0000000000000044",
   },
   Object {
     "@timestamp": 1609459920000,
@@ -444,9 +444,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "c097c19d884d52579bb11a601b8a98b3",
+    "trace.id": "00000000000000000000000000000049",
     "transaction.duration.us": 1000000,
-    "transaction.id": "2e3a47fa2d905519",
+    "transaction.id": "0000000000000048",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -456,19 +456,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "2e3a47fa2d905519",
+    "parent.id": "0000000000000048",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "7c7828c850685337",
+    "span.id": "0000000000000050",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "c097c19d884d52579bb11a601b8a98b3",
+    "trace.id": "00000000000000000000000000000049",
-    "transaction.id": "2e3a47fa2d905519",
+    "transaction.id": "0000000000000048",
   },
   Object {
     "@timestamp": 1609459980000,
@@ -480,9 +480,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "4591e57f4d7f5986bdd7892561224e0f",
+    "trace.id": "00000000000000000000000000000053",
     "transaction.duration.us": 1000000,
-    "transaction.id": "de5eaa1e47dc56b1",
+    "transaction.id": "0000000000000052",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -492,19 +492,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "de5eaa1e47dc56b1",
+    "parent.id": "0000000000000052",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "8f62257f4a41546a",
+    "span.id": "0000000000000054",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "4591e57f4d7f5986bdd7892561224e0f",
+    "trace.id": "00000000000000000000000000000053",
-    "transaction.id": "de5eaa1e47dc56b1",
+    "transaction.id": "0000000000000052",
   },
   Object {
     "@timestamp": 1609460040000,
@@ -516,9 +516,9 @@ Array [
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
-    "trace.id": "85ee8e618433577b9316a1e14961aa89",
+    "trace.id": "00000000000000000000000000000057",
     "transaction.duration.us": 1000000,
-    "transaction.id": "af7eac7ae61e576a",
+    "transaction.id": "0000000000000056",
     "transaction.name": "GET /api/product/list",
     "transaction.sampled": true,
     "transaction.type": "request",
@@ -528,19 +528,19 @@ Array [
     "agent.name": "java",
     "container.id": "instance-1",
     "event.outcome": "success",
-    "parent.id": "af7eac7ae61e576a",
+    "parent.id": "0000000000000056",
     "processor.event": "span",
     "processor.name": "transaction",
     "service.environment": "production",
     "service.name": "opbeans-java",
     "service.node.name": "instance-1",
     "span.duration.us": 900000,
-    "span.id": "cc88b4cd921e590e",
+    "span.id": "0000000000000058",
     "span.name": "GET apm-*/_search",
     "span.subtype": "elasticsearch",
     "span.type": "db",
-    "trace.id": "85ee8e618433577b9316a1e14961aa89",
+    "trace.id": "00000000000000000000000000000057",
-    "transaction.id": "af7eac7ae61e576a",
+    "transaction.id": "0000000000000056",
   },
 ]
 `;

packages/elastic-apm-synthtrace/src/test/to_elasticsearch_output.test.ts

@@ -24,6 +24,7 @@ describe('output to elasticsearch', () => {
       '@timestamp': new Date('2020-12-31T23:00:00.000Z').getTime(),
       'processor.event': 'transaction',
       'processor.name': 'transaction',
+      'service.node.name': 'instance-a',
     };
   });
 
@@ -41,4 +42,33 @@ describe('output to elasticsearch', () => {
       name: 'transaction',
     });
   });
+
+  it('formats all fields consistently', () => {
+    const doc = toElasticsearchOutput({ events: [event], writeTargets })[0] as any;
+
+    expect(doc._source).toMatchInlineSnapshot(`
+      Object {
+        "@timestamp": "2020-12-31T23:00:00.000Z",
+        "ecs": Object {
+          "version": "1.4",
+        },
+        "observer": Object {
+          "version": "7.16.0",
+          "version_major": 7,
+        },
+        "processor": Object {
+          "event": "transaction",
+          "name": "transaction",
+        },
+        "service": Object {
+          "node": Object {
+            "name": "instance-a",
+          },
+        },
+        "timestamp": Object {
+          "us": 1609455600000000,
+        },
+      }
+    `);
+  });
 });