maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[APM] Remove value_count aggregations (#89408)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dario Gieselaar 2021-02-01 15:14:17 +01:00 committed by GitHub
parent 53637d0158
commit 3255f905c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 63 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts
View file

@ -44,9 +44,7 @@ export async function getTransactionErrorRateChartPreview({
},
};
const outcomes = getOutcomeAggregation({
searchAggregatedTransactions: false,
});
const outcomes = getOutcomeAggregation();
const { intervalString } = getBucketSize({ start, end, numBuckets: 20 });

31
x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { isEmpty, omit } from 'lodash';
import { isEmpty, omit, merge } from 'lodash';
import { EventOutcome } from '../../../../common/event_outcome';
import {
processSignificantTermAggs,
@ -134,8 +134,7 @@ export async function getErrorRateTimeSeries({
extended_bounds: { min: start, max: end },
},
aggs: {
// TODO: add support for metrics
outcomes: getOutcomeAggregation({ searchAggregatedTransactions: false }),
outcomes: getOutcomeAggregation(),
},
};
@ -147,13 +146,12 @@ export async function getErrorRateTimeSeries({
};
return acc;
},
{} as Record<
string,
{
{} as {
[key: string]: {
filter: AggregationOptionsByType['filter'];
aggs: { timeseries: typeof timeseriesAgg };
}
>
};
}
);
const params = {
@ -162,32 +160,25 @@ export async function getErrorRateTimeSeries({
body: {
size: 0,
query: { bool: { filter: backgroundFilters } },
aggs: {
// overall aggs
timeseries: timeseriesAgg,
// per term aggs
...perTermAggs,
},
aggs: merge({ timeseries: timeseriesAgg }, perTermAggs),
},
};
const response = await apmEventClient.search(params);
type Agg = NonNullable<typeof response.aggregations>;
const { aggregations } = response;
if (!response.aggregations) {
if (!aggregations) {
return {};
}
return {
overall: {
timeseries: getTransactionErrorRateTimeSeries(
response.aggregations.timeseries.buckets
aggregations.timeseries.buckets
),
},
significantTerms: topSigTerms.map((topSig, index) => {
// @ts-expect-error
const agg = response.aggregations[`term_${index}`] as Agg;
const agg = aggregations[`term_${index}`]!;
return {
...topSig,

41
x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts
View file

@ -10,40 +10,21 @@ import {
AggregationOptionsByType,
AggregationResultOf,
} from '../../../../../typings/elasticsearch/aggregations';
import { getTransactionDurationFieldForAggregatedTransactions } from './aggregated_transactions';
export function getOutcomeAggregation({
searchAggregatedTransactions,
}: {
searchAggregatedTransactions: boolean;
}) {
return {
terms: {
field: EVENT_OUTCOME,
include: [EventOutcome.failure, EventOutcome.success],
},
aggs: {
// simply using the doc count to get the number of requests is not possible for transaction metrics (histograms)
// to work around this we get the number of transactions by counting the number of latency values
count: {
value_count: {
field: getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
),
},
},
},
};
}
export const getOutcomeAggregation = () => ({
terms: {
field: EVENT_OUTCOME,
include: [EventOutcome.failure, EventOutcome.success],
},
});
type OutcomeAggregation = ReturnType<typeof getOutcomeAggregation>;
export function calculateTransactionErrorPercentage(
outcomeResponse: AggregationResultOf<
ReturnType<typeof getOutcomeAggregation>,
{}
>
outcomeResponse: AggregationResultOf<OutcomeAggregation, {}>
) {
const outcomes = Object.fromEntries(
outcomeResponse.buckets.map(({ key, count }) => [key, count.value])
outcomeResponse.buckets.map(({ key, doc_count: count }) => [key, count])
);
const failedTransactions = outcomes[EventOutcome.failure] ?? 0;
@ -56,7 +37,7 @@ export function getTransactionErrorRateTimeSeries(
buckets: AggregationResultOf<
{
date_histogram: AggregationOptionsByType['date_histogram'];
aggs: { outcomes: ReturnType<typeof getOutcomeAggregation> };
aggs: { outcomes: OutcomeAggregation };
},
{}
>['buckets']

16
x-pack/plugins/apm/server/lib/observability_overview/get_transaction_coordinates.ts
View file

@ -11,10 +11,7 @@
import { rangeFilter } from '../../../common/utils/range_filter';
import { Coordinates } from '../../../../observability/typings/common';
import { Setup, SetupTimeRange } from '../helpers/setup_request';
import {
getProcessorEventForAggregatedTransactions,
getTransactionDurationFieldForAggregatedTransactions,
} from '../helpers/aggregated_transactions';
import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions';
export async function getTransactionCoordinates({
setup,
@ -49,15 +46,6 @@ export async function getTransactionCoordinates({
fixed_interval: bucketSize,
min_doc_count: 0,
},
aggs: {
count: {
value_count: {
field: getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
),
},
},
},
},
},
},
@ -68,7 +56,7 @@ export async function getTransactionCoordinates({
return (
aggregations?.distribution.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.count.value / deltaAsMinutes,
y: bucket.doc_count / deltaAsMinutes,
})) || []
);
}

4
x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.test.ts
View file

@ -52,8 +52,10 @@ describe('getServiceMapServiceNodeInfo', () => {
apmEventClient: {
search: () =>
Promise.resolve({
hits: {
total: { value: 1 },
},
aggregations: {
count: { value: 1 },
duration: { value: null },
avgCpuUsage: { value: null },
avgMemoryUsage: { value: null },

9
x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts
View file

@ -162,19 +162,12 @@ async function getTransactionStats({
),
},
},
count: {
value_count: {
field: getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
),
},
},
},
},
};
const response = await apmEventClient.search(params);
const totalRequests = response.aggregations?.count.value ?? 0;
const totalRequests = response.hits.total.value;
return {
avgTransactionDuration: response.aggregations?.duration.value ?? null,

27
x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap generated
View file

@ -122,13 +122,6 @@ Array [
},
},
"outcomes": Object {
"aggs": Object {
"count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
},
"terms": Object {
"field": "event.outcome",
"include": Array [
@ -137,11 +130,6 @@ Array [
],
},
},
"real_document_count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
"timeseries": Object {
"aggs": Object {
"avg_duration": Object {
@ -150,13 +138,6 @@ Array [
},
},
"outcomes": Object {
"aggs": Object {
"count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
},
"terms": Object {
"field": "event.outcome",
"include": Array [
@ -165,11 +146,6 @@ Array [
],
},
},
"real_document_count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
},
"date_histogram": Object {
"extended_bounds": Object {
@ -184,9 +160,6 @@ Array [
},
"terms": Object {
"field": "transaction.type",
"order": Object {
"real_document_count": "desc",
},
},
},
},

29
x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instance_transaction_stats.ts
View file

@ -30,18 +30,17 @@ export async function getServiceInstanceTransactionStats({
}: ServiceInstanceParams) {
const { apmEventClient, start, end, esFilter } = setup;
const { intervalString } = getBucketSize({ start, end, numBuckets });
const { intervalString, bucketSize } = getBucketSize({
start,
end,
numBuckets,
});
const field = getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
);
const subAggs = {
count: {
value_count: {
field,
},
},
avg_transaction_duration: {
avg: {
field,
@ -53,13 +52,6 @@ export async function getServiceInstanceTransactionStats({
[EVENT_OUTCOME]: EventOutcome.failure,
},
},
aggs: {
count: {
value_count: {
field,
},
},
},
},
};
@ -113,12 +105,13 @@ export async function getServiceInstanceTransactionStats({
});
const deltaAsMinutes = (end - start) / 60 / 1000;
const bucketSizeInMinutes = bucketSize / 60;
return (
response.aggregations?.[SERVICE_NODE_NAME].buckets.map(
(serviceNodeBucket) => {
const {
count,
doc_count: count,
avg_transaction_duration: avgTransactionDuration,
key,
failures,
@ -128,17 +121,17 @@ export async function getServiceInstanceTransactionStats({
return {
serviceNodeName: String(key),
errorRate: {
value: failures.count.value / count.value,
value: failures.doc_count / count,
timeseries: timeseries.buckets.map((dateBucket) => ({
x: dateBucket.key,
y: dateBucket.failures.count.value / dateBucket.count.value,
y: dateBucket.failures.doc_count / dateBucket.doc_count,
})),
},
throughput: {
value: count.value / deltaAsMinutes,
value: count / deltaAsMinutes,
timeseries: timeseries.buckets.map((dateBucket) => ({
x: dateBucket.key,
y: dateBucket.count.value / deltaAsMinutes,
y: dateBucket.doc_count / bucketSizeInMinutes,
})),
},
latency: {

6
x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_timeseries_data_for_transaction_groups.ts
View file

@ -17,6 +17,7 @@ import {
import { ESFilter } from '../../../../../../typings/elasticsearch';
import {
getDocumentTypeFilterForAggregatedTransactions,
getProcessorEventForAggregatedTransactions,
getTransactionDurationFieldForAggregatedTransactions,
} from '../../helpers/aggregated_transactions';
@ -76,6 +77,9 @@ export async function getTimeseriesDataForTransactionGroups({
{ term: { [SERVICE_NAME]: serviceName } },
{ term: { [TRANSACTION_TYPE]: transactionType } },
{ range: rangeFilter(start, end) },
...getDocumentTypeFilterForAggregatedTransactions(
searchAggregatedTransactions
),
...esFilter,
],
},
@ -99,10 +103,8 @@ export async function getTimeseriesDataForTransactionGroups({
},
aggs: {
...getLatencyAggregation(latencyAggregationType, field),
transaction_count: { value_count: { field } },
[EVENT_OUTCOME]: {
filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } },
aggs: { transaction_count: { value_count: { field } } },
},
},
},

9
x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_transaction_groups_for_page.ts
View file

@ -99,10 +99,8 @@ export async function getTransactionGroupsForPage({
},
aggs: {
...getLatencyAggregation(latencyAggregationType, field),
transaction_count: { value_count: { field } },
[EVENT_OUTCOME]: {
filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } },
aggs: { transaction_count: { value_count: { field } } },
},
},
},
@ -113,9 +111,8 @@ export async function getTransactionGroupsForPage({
const transactionGroups =
response.aggregations?.transaction_groups.buckets.map((bucket) => {
const errorRate =
bucket.transaction_count.value > 0
? (bucket[EVENT_OUTCOME].transaction_count.value ?? 0) /
bucket.transaction_count.value
bucket.doc_count > 0
? bucket[EVENT_OUTCOME].doc_count / bucket.doc_count
: null;
return {
@ -124,7 +121,7 @@ export async function getTransactionGroupsForPage({
latencyAggregationType,
aggregation: bucket.latency,
}),
throughput: bucket.transaction_count.value / deltaAsMinutes,
throughput: bucket.doc_count / deltaAsMinutes,
errorRate,
};
}) ?? [];

8
x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/merge_transaction_group_data.ts
View file

@ -52,18 +52,14 @@ export function mergeTransactionGroupData({
...acc.throughput,
timeseries: acc.throughput.timeseries.concat({
x: point.key,
y: point.transaction_count.value / deltaAsMinutes,
y: point.doc_count / deltaAsMinutes,
}),
},
errorRate: {
...acc.errorRate,
timeseries: acc.errorRate.timeseries.concat({
x: point.key,
y:
point.transaction_count.value > 0
? (point[EVENT_OUTCOME].transaction_count.value ?? 0) /
point.transaction_count.value
: null,
y: point[EVENT_OUTCOME].doc_count / point.doc_count,
}),
},
};

14
x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts
View file

@ -51,16 +51,9 @@ export async function getServiceTransactionStats({
}: AggregationParams) {
const { apmEventClient, start, end, esFilter } = setup;
const outcomes = getOutcomeAggregation({ searchAggregatedTransactions });
const outcomes = getOutcomeAggregation();
const metrics = {
real_document_count: {
value_count: {
field: getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
),
},
},
avg_duration: {
avg: {
field: getTransactionDurationFieldForAggregatedTransactions(
@ -102,7 +95,6 @@ export async function getServiceTransactionStats({
transactionType: {
terms: {
field: TRANSACTION_TYPE,
order: { real_document_count: 'desc' },
},
aggs: {
...metrics,
@ -180,14 +172,14 @@ export async function getServiceTransactionStats({
},
transactionsPerMinute: {
value: calculateAvgDuration({
value: topTransactionTypeBucket.real_document_count.value,
value: topTransactionTypeBucket.doc_count,
deltaAsMinutes,
}),
timeseries: topTransactionTypeBucket.timeseries.buckets.map(
(dateBucket) => ({
x: dateBucket.key,
y: calculateAvgDuration({
value: dateBucket.real_document_count.value,
value: dateBucket.doc_count,
deltaAsMinutes,
}),
})

10
x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/queries.test.ts.snap generated
View file

@ -12,11 +12,6 @@ Array [
"aggs": Object {
"transaction_groups": Object {
"aggs": Object {
"count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
"transaction_type": Object {
"top_hits": Object {
"_source": Array [
@ -226,11 +221,6 @@ Array [
"aggs": Object {
"transaction_groups": Object {
"aggs": Object {
"count": Object {
"value_count": Object {
"field": "transaction.duration.us",
},
},
"transaction_type": Object {
"top_hits": Object {
"_source": Array [

10
x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts
View file

@ -14,7 +14,10 @@ import {
} from '../../../common/elasticsearch_fieldnames';
import { EventOutcome } from '../../../common/event_outcome';
import { rangeFilter } from '../../../common/utils/range_filter';
import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions';
import {
getDocumentTypeFilterForAggregatedTransactions,
getProcessorEventForAggregatedTransactions,
} from '../helpers/aggregated_transactions';
import { getBucketSize } from '../helpers/get_bucket_size';
import { Setup, SetupTimeRange } from '../helpers/setup_request';
import {
@ -55,12 +58,15 @@ export async function getErrorRate({
{
terms: { [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success] },
},
...getDocumentTypeFilterForAggregatedTransactions(
searchAggregatedTransactions
),
...transactionNamefilter,
...transactionTypefilter,
...esFilter,
];
const outcomes = getOutcomeAggregation({ searchAggregatedTransactions });
const outcomes = getOutcomeAggregation();
const params = {
apm: {

9
x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts
View file

@ -66,13 +66,6 @@ export async function getCounts({
searchAggregatedTransactions,
}: MetricParams) {
const params = mergeRequestWithAggs(request, {
count: {
value_count: {
field: getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
),
},
},
transaction_type: {
top_hits: {
size: 1,
@ -92,7 +85,7 @@ export async function getCounts({
return {
key: bucket.key as BucketKey,
count: bucket.count.value,
count: bucket.doc_count,
transactionType: source.transaction.type,
};
});

6
x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts
View file

@ -15,7 +15,6 @@ import { rangeFilter } from '../../../../common/utils/range_filter';
import {
getDocumentTypeFilterForAggregatedTransactions,
getProcessorEventForAggregatedTransactions,
getTransactionDurationFieldForAggregatedTransactions,
} from '../../../lib/helpers/aggregated_transactions';
import { getBucketSize } from '../../../lib/helpers/get_bucket_size';
import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request';
@ -56,10 +55,6 @@ async function searchThroughput({
filter.push({ term: { [TRANSACTION_NAME]: transactionName } });
}
const field = getTransactionDurationFieldForAggregatedTransactions(
searchAggregatedTransactions
);
const params = {
apm: {
events: [
@ -82,7 +77,6 @@ async function searchThroughput({
min_doc_count: 0,
extended_bounds: { min: start, max: end },
},
aggs: { count: { value_count: { field } } },
},
},
},

4
x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/transform.ts
View file

@ -25,7 +25,7 @@ export function getThroughputBuckets({
return {
x: bucket.key,
// divide by minutes
y: bucket.count.value / (bucketSize / 60),
y: bucket.doc_count / (bucketSize / 60),
};
});
@ -34,7 +34,7 @@ export function getThroughputBuckets({
resultKey === '' ? NOT_AVAILABLE_LABEL : (resultKey as string);
const docCountTotal = timeseries.buckets
.map((bucket) => bucket.count.value)
.map((bucket) => bucket.doc_count)
.reduce((a, b) => a + b, 0);
// calculate average throughput