maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[APM] Optimize service map query (#60412)

Browse source 
* [APM] Optimize service map query

Closes #60411.

- Chunk trace lookup
- Remove pagination, move dedupe logic to server

* Fix imports

* Fix imports again

Co-authored-by: Nathan L Smith <smith@nlsmith.com>
This commit is contained in:
Dario Gieselaar 2020-03-19 08:37:58 +01:00 committed by GitHub
parent 01571b6739
commit 9cd0a36740
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 291 additions and 313 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx
View file

@ -13,7 +13,7 @@ import { getCytoscapeElements } from './get_cytoscape_elements';
import serviceMapResponse from './cytoscape-layout-test-response.json';
import { iconForNode } from './icons';
const elementsFromResponses = getCytoscapeElements([serviceMapResponse], '');
const elementsFromResponses = getCytoscapeElements(serviceMapResponse, '');
storiesOf('app/ServiceMap/Cytoscape', module).add(
'example',

175
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts
View file

@ -4,166 +4,63 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ValuesType } from 'utility-types';
import { sortBy, isEqual } from 'lodash';
import {
Connection,
ConnectionNode
} from '../../../../../../../plugins/apm/common/service_map';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map';
import { getAPMHref } from '../../shared/Links/apm/APMLink';
function getConnectionNodeId(node: ConnectionNode): string {
if ('destination.address' in node) {
// use a prefix to distinguish exernal destination ids from services
return `>${node['destination.address']}`;
}
return node['service.name'];
}
function getConnectionId(connection: Connection) {
return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId(
connection.destination
)}`;
}
export function getCytoscapeElements(
responses: ServiceMapAPIResponse[],
response: ServiceMapAPIResponse,
search: string
) {
const discoveredServices = responses.flatMap(
response => response.discoveredServices
);
const serviceNodes = responses
.flatMap(response => response.services)
.map(service => ({
...service,
id: service['service.name']
}));
// maps destination.address to service.name if possible
function getConnectionNode(node: ConnectionNode) {
let mappedNode: ConnectionNode | undefined;
if ('destination.address' in node) {
mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to;
}
if (!mappedNode) {
mappedNode = node;
}
return {
...mappedNode,
id: getConnectionNodeId(mappedNode)
};
}
// build connections with mapped nodes
const connections = responses
.flatMap(response => response.connections)
.map(connection => {
const source = getConnectionNode(connection.source);
const destination = getConnectionNode(connection.destination);
return {
source,
destination,
id: getConnectionId({ source, destination })
};
})
.filter(connection => connection.source.id !== connection.destination.id);
const nodes = connections
.flatMap(connection => [connection.source, connection.destination])
.concat(serviceNodes);
type ConnectionWithId = ValuesType<typeof connections>;
type ConnectionNodeWithId = ValuesType<typeof nodes>;
const connectionsById = connections.reduce((connectionMap, connection) => {
return {
...connectionMap,
[connection.id]: connection
};
}, {} as Record<string, ConnectionWithId>);
const { nodes, connections } = response;
const nodesById = nodes.reduce((nodeMap, node) => {
return {
...nodeMap,
[node.id]: node
};
}, {} as Record<string, ConnectionNodeWithId>);
}, {} as Record<string, ValuesType<typeof nodes>>);
const cyNodes = (Object.values(nodesById) as ConnectionNodeWithId[]).map(
node => {
let data = {};
const cyNodes = (Object.values(nodesById) as Array<
ValuesType<typeof nodes>
>).map(node => {
let data = {};
if ('service.name' in node) {
data = {
href: getAPMHref(
`/services/${node['service.name']}/service-map`,
search
),
agentName: node['agent.name'],
frameworkName: node['service.framework.name'],
type: 'service'
};
}
if ('span.type' in node) {
data = {
// For nodes with span.type "db", convert it to "database". Otherwise leave it as-is.
type: node['span.type'] === 'db' ? 'database' : node['span.type'],
// Externals should not have a subtype so make it undefined if the type is external.
subtype: node['span.type'] !== 'external' && node['span.subtype']
};
}
return {
group: 'nodes' as const,
data: {
id: node.id,
label:
'service.name' in node
? node['service.name']
: node['destination.address'],
...data
}
if ('service.name' in node) {
data = {
href: getAPMHref(
`/services/${node['service.name']}/service-map`,
search
),
agentName: node['agent.name'],
frameworkName: node['service.framework.name'],
type: 'service'
};
}
);
// instead of adding connections in two directions,
// we add a `bidirectional` flag to use in styling
// and hide the inverse edge when rendering
const dedupedConnections = (sortBy(
Object.values(connectionsById),
// make sure that order is stable
'id'
) as ConnectionWithId[]).reduce<
Array<
ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean }
>
>((prev, connection) => {
const reversedConnection = prev.find(
c =>
c.destination.id === connection.source.id &&
c.source.id === connection.destination.id
);
if (reversedConnection) {
reversedConnection.bidirectional = true;
return prev.concat({
...connection,
isInverseEdge: true
});
if ('span.type' in node) {
data = {
// For nodes with span.type "db", convert it to "database". Otherwise leave it as-is.
type: node['span.type'] === 'db' ? 'database' : node['span.type'],
// Externals should not have a subtype so make it undefined if the type is external.
subtype: node['span.type'] !== 'external' && node['span.subtype']
};
}
return prev.concat(connection);
}, []);
return {
group: 'nodes' as const,
data: {
id: node.id,
label:
'service.name' in node
? node['service.name']
: node['destination.address'],
...data
}
};
});
const cyEdges = dedupedConnections.map(connection => {
const cyEdges = connections.map(connection => {
return {
group: 'edges' as const,
classes: connection.isInverseEdge ? 'invisible' : undefined,

124
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx
View file

@ -4,26 +4,15 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { EuiBetaBadge } from '@elastic/eui';
import theme from '@elastic/eui/dist/eui_theme_light.json';
import { i18n } from '@kbn/i18n';
import { ElementDefinition } from 'cytoscape';
import { find, isEqual } from 'lodash';
import React, {
useCallback,
useEffect,
useMemo,
useRef,
useState
} from 'react';
import { EuiBetaBadge } from '@elastic/eui';
import React, { useMemo } from 'react';
import styled from 'styled-components';
import { isValidPlatinumLicense } from '../../../../../../../plugins/apm/common/service_map';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map';
import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
import { useDeepObjectIdentity } from '../../../hooks/useDeepObjectIdentity';
import { useFetcher } from '../../../hooks/useFetcher';
import { useLicense } from '../../../hooks/useLicense';
import { useLoadingIndicator } from '../../../hooks/useLoadingIndicator';
import { useLocation } from '../../../hooks/useLocation';
import { useUrlParams } from '../../../hooks/useUrlParams';
import { callApmApi } from '../../../services/rest/createCallApmApi';
@ -64,13 +53,11 @@ const BetaBadgeContainer = styled.div`
top: ${theme.gutterTypes.gutterSmall};
z-index: 1; /* The element containing the cytoscape canvas has z-index = 0. */
`;
const MAX_REQUESTS = 5;
export function ServiceMap({ serviceName }: ServiceMapProps) {
const license = useLicense();
const { search } = useLocation();
const { urlParams, uiFilters } = useUrlParams();
const { notifications } = useApmPluginContext().core;
const params = useDeepObjectIdentity({
start: urlParams.start,
end: urlParams.end,
@ -82,95 +69,28 @@ export function ServiceMap({ serviceName }: ServiceMapProps) {
}
});
const renderedElements = useRef<ElementDefinition[]>([]);
const [responses, setResponses] = useState<ServiceMapAPIResponse[]>([]);
const { setIsLoading } = useLoadingIndicator();
const [, _setUnusedState] = useState(false);
const elements = useMemo(() => getCytoscapeElements(responses, search), [
responses,
search
]);
const forceUpdate = useCallback(() => _setUnusedState(value => !value), []);
const getNext = useCallback(
async (input: { reset?: boolean; after?: string | undefined }) => {
const { start, end, uiFilters: strippedUiFilters, ...query } = params;
if (input.reset) {
renderedElements.current = [];
setResponses([]);
}
if (start && end) {
setIsLoading(true);
try {
const data = await callApmApi({
pathname: '/api/apm/service-map',
params: {
query: {
...query,
start,
end,
uiFilters: JSON.stringify(strippedUiFilters),
after: input.after
}
}
});
setResponses(resp => resp.concat(data));
const shouldGetNext =
responses.length + 1 < MAX_REQUESTS && data.after;
if (shouldGetNext) {
await getNext({ after: data.after });
} else {
setIsLoading(false);
const { data } = useFetcher(() => {
const { start, end } = params;
if (start && end) {
return callApmApi({
pathname: '/api/apm/service-map',
params: {
query: {
...params,
start,
end,
uiFilters: JSON.stringify(params.uiFilters)
}
} catch (error) {
setIsLoading(false);
notifications.toasts.addError(error, {
title: i18n.translate('xpack.apm.errorServiceMapData', {
defaultMessage: `Error loading service connections`
})
});
}
}
},
[params, setIsLoading, responses.length, notifications.toasts]
);
useEffect(() => {
const loadServiceMaps = async () => {
await getNext({ reset: true });
};
loadServiceMaps();
// eslint-disable-next-line react-hooks/exhaustive-deps
});
}
}, [params]);
useEffect(() => {
if (renderedElements.current.length === 0) {
renderedElements.current = elements;
return;
}
const elements = useMemo(() => {
return data ? getCytoscapeElements(data as any, search) : [];
}, [data, search]);
const newElements = elements.filter(element => {
return !find(renderedElements.current, el => isEqual(el, element));
});
if (newElements.length > 0 && renderedElements.current.length > 0) {
renderedElements.current = elements;
forceUpdate();
}
}, [elements, forceUpdate]);
const { ref: wrapperRef, width, height } = useRefDimensions();
const { ref, height, width } = useRefDimensions();
if (!license) {
return null;
@ -179,10 +99,10 @@ export function ServiceMap({ serviceName }: ServiceMapProps) {
return isValidPlatinumLicense(license) ? (
<div
style={{ height: height - parseInt(theme.gutterTypes.gutterLarge, 10) }}
ref={wrapperRef}
ref={ref}
>
<Cytoscape
elements={renderedElements.current}
elements={elements}
serviceName={serviceName}
height={height}
width={width}

17
x-pack/plugins/apm/server/index.ts
View file

@ -17,6 +17,13 @@ export const config = {
schema: schema.object({
enabled: schema.boolean({ defaultValue: true }),
serviceMapEnabled: schema.boolean({ defaultValue: true }),
serviceMapFingerprintBucketSize: schema.number({ defaultValue: 100 }),
serviceMapTraceIdBucketSize: schema.number({ defaultValue: 65 }),
serviceMapFingerprintGlobalBucketSize: schema.number({
defaultValue: 1000
}),
serviceMapTraceIdGlobalBucketSize: schema.number({ defaultValue: 6 }),
serviceMapMaxTracesPerRequest: schema.number({ defaultValue: 50 }),
autocreateApmIndexPattern: schema.boolean({ defaultValue: true }),
ui: schema.object({
enabled: schema.boolean({ defaultValue: true }),
@ -41,6 +48,16 @@ export function mergeConfigs(
'apm_oss.onboardingIndices': apmOssConfig.onboardingIndices,
'apm_oss.indexPattern': apmOssConfig.indexPattern,
'xpack.apm.serviceMapEnabled': apmConfig.serviceMapEnabled,
'xpack.apm.serviceMapFingerprintBucketSize':
apmConfig.serviceMapFingerprintBucketSize,
'xpack.apm.serviceMapTraceIdBucketSize':
apmConfig.serviceMapTraceIdBucketSize,
'xpack.apm.serviceMapFingerprintGlobalBucketSize':
apmConfig.serviceMapFingerprintGlobalBucketSize,
'xpack.apm.serviceMapTraceIdGlobalBucketSize':
apmConfig.serviceMapTraceIdGlobalBucketSize,
'xpack.apm.serviceMapMaxTracesPerRequest':
apmConfig.serviceMapMaxTracesPerRequest,
'xpack.apm.ui.enabled': apmConfig.ui.enabled,
'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems,
'xpack.apm.ui.transactionGroupBucketSize':

1
x-pack/plugins/apm/server/lib/helpers/setup_request.ts
View file

@ -45,6 +45,7 @@ export interface SetupTimeRange {
start: number;
end: number;
}
export interface SetupUIFilters {
uiFiltersES: ESFilter[];
}

123
x-pack/plugins/apm/server/lib/service_map/dedupe_connections.ts Normal file
View file

@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { isEqual, sortBy } from 'lodash';
import { ValuesType } from 'utility-types';
import { ConnectionNode, Connection } from '../../../common/service_map';
import { ConnectionsResponse, ServicesResponse } from './get_service_map';
function getConnectionNodeId(node: ConnectionNode): string {
if ('destination.address' in node) {
// use a prefix to distinguish exernal destination ids from services
return `>${node['destination.address']}`;
}
return node['service.name'];
}
function getConnectionId(connection: Connection) {
return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId(
connection.destination
)}`;
}
type ServiceMapResponse = ConnectionsResponse & { services: ServicesResponse };
export function dedupeConnections(response: ServiceMapResponse) {
const { discoveredServices, services, connections } = response;
const serviceNodes = services.map(service => ({
...service,
id: service['service.name']
}));
// maps destination.address to service.name if possible
function getConnectionNode(node: ConnectionNode) {
let mappedNode: ConnectionNode | undefined;
if ('destination.address' in node) {
mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to;
}
if (!mappedNode) {
mappedNode = node;
}
return {
...mappedNode,
id: getConnectionNodeId(mappedNode)
};
}
// build connections with mapped nodes
const mappedConnections = connections
.map(connection => {
const source = getConnectionNode(connection.source);
const destination = getConnectionNode(connection.destination);
return {
source,
destination,
id: getConnectionId({ source, destination })
};
})
.filter(connection => connection.source.id !== connection.destination.id);
const nodes = mappedConnections
.flatMap(connection => [connection.source, connection.destination])
.concat(serviceNodes);
const dedupedNodes: typeof nodes = [];
nodes.forEach(node => {
if (!dedupedNodes.find(dedupedNode => isEqual(node, dedupedNode))) {
dedupedNodes.push(node);
}
});
type ConnectionWithId = ValuesType<typeof mappedConnections>;
const connectionsById = mappedConnections.reduce(
(connectionMap, connection) => {
return {
...connectionMap,
[connection.id]: connection
};
},
{} as Record<string, ConnectionWithId>
);
// instead of adding connections in two directions,
// we add a `bidirectional` flag to use in styling
const dedupedConnections = (sortBy(
Object.values(connectionsById),
// make sure that order is stable
'id'
) as ConnectionWithId[]).reduce<
Array<
ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean }
>
>((prev, connection) => {
const reversedConnection = prev.find(
c =>
c.destination.id === connection.source.id &&
c.source.id === connection.destination.id
);
if (reversedConnection) {
reversedConnection.bidirectional = true;
return prev.concat({
...connection,
isInverseEdge: true
});
}
return prev.concat(connection);
}, []);
return {
nodes: dedupedNodes,
connections: dedupedConnections
};
}

61
x-pack/plugins/apm/server/lib/service_map/get_service_map.ts
View file

@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { chunk } from 'lodash';
import { PromiseReturnType } from '../../../typings/common';
import {
Setup,
@ -19,48 +19,61 @@ import {
SERVICE_NAME,
SERVICE_FRAMEWORK_NAME
} from '../../../common/elasticsearch_fieldnames';
import { dedupeConnections } from './dedupe_connections';
export interface IEnvOptions {
setup: Setup & SetupTimeRange & SetupUIFilters;
serviceName?: string;
environment?: string;
after?: string;
}
async function getConnectionData({
setup,
serviceName,
environment,
after
environment
}: IEnvOptions) {
const { traceIds, after: nextAfter } = await getTraceSampleIds({
const { traceIds } = await getTraceSampleIds({
setup,
serviceName,
environment,
after
environment
});
const serviceMapData = traceIds.length
? await getServiceMapFromTraceIds({
const chunks = chunk(
traceIds,
setup.config['xpack.apm.serviceMapMaxTracesPerRequest']
);
const init = {
connections: [],
discoveredServices: []
};
if (!traceIds.length) {
return init;
}
const chunkedResponses = await Promise.all(
chunks.map(traceIdsChunk =>
getServiceMapFromTraceIds({
setup,
serviceName,
environment,
traceIds
traceIds: traceIdsChunk
})
: { connections: [], discoveredServices: [] };
)
);
return {
after: nextAfter,
...serviceMapData
};
return chunkedResponses.reduce((prev, current) => {
return {
connections: prev.connections.concat(current.connections),
discoveredServices: prev.discoveredServices.concat(
current.discoveredServices
)
};
});
}
async function getServicesData(options: IEnvOptions) {
// only return services on the first request for the global service map
if (options.after) {
return [];
}
const { setup } = options;
const projection = getServicesProjection({ setup });
@ -125,15 +138,19 @@ async function getServicesData(options: IEnvOptions) {
);
}
export type ConnectionsResponse = PromiseReturnType<typeof getConnectionData>;
export type ServicesResponse = PromiseReturnType<typeof getServicesData>;
export type ServiceMapAPIResponse = PromiseReturnType<typeof getServiceMap>;
export async function getServiceMap(options: IEnvOptions) {
const [connectionData, servicesData] = await Promise.all([
getConnectionData(options),
getServicesData(options)
]);
return {
return dedupeConnections({
...connectionData,
services: servicesData
};
});
}

89
x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts
View file

@ -15,27 +15,24 @@ import {
PROCESSOR_EVENT,
SERVICE_NAME,
SERVICE_ENVIRONMENT,
SPAN_TYPE,
SPAN_SUBTYPE,
TRACE_ID,
DESTINATION_ADDRESS,
TRACE_ID
SPAN_TYPE,
SPAN_SUBTYPE
} from '../../../common/elasticsearch_fieldnames';
const MAX_CONNECTIONS_PER_REQUEST = 1000;
const MAX_TRACES_TO_INSPECT = 1000;
export async function getTraceSampleIds({
after,
serviceName,
environment,
setup
}: {
after?: string;
serviceName?: string;
environment?: string;
setup: Setup & SetupTimeRange & SetupUIFilters;
}) {
const { start, end, client, indices } = setup;
const { start, end, client, indices, config } = setup;
const rangeQuery = { range: rangeFilter(start, end) };
@ -65,9 +62,15 @@ export async function getTraceSampleIds({
query.bool.filter.push({ term: { [SERVICE_ENVIRONMENT]: environment } });
}
const afterObj = after
? { after: JSON.parse(Buffer.from(after, 'base64').toString()) }
: {};
const fingerprintBucketSize = serviceName
? config['xpack.apm.serviceMapFingerprintBucketSize']
: config['xpack.apm.serviceMapFingerprintGlobalBucketSize'];
const traceIdBucketSize = serviceName
? config['xpack.apm.serviceMapTraceIdBucketSize']
: config['xpack.apm.serviceMapTraceIdGlobalBucketSize'];
const samplerShardSize = traceIdBucketSize * 10;
const params = {
index: [indices['apm_oss.spanIndices']],
@ -77,42 +80,57 @@ export async function getTraceSampleIds({
aggs: {
connections: {
composite: {
size: MAX_CONNECTIONS_PER_REQUEST,
...afterObj,
sources: [
{ [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } },
{
[DESTINATION_ADDRESS]: {
terms: {
field: DESTINATION_ADDRESS
}
}
},
{
[SERVICE_NAME]: {
terms: {
field: SERVICE_NAME
}
}
},
{
[SERVICE_ENVIRONMENT]: {
terms: { field: SERVICE_ENVIRONMENT, missing_bucket: true }
terms: {
field: SERVICE_ENVIRONMENT,
missing_bucket: true
}
}
},
{
[SPAN_TYPE]: {
terms: { field: SPAN_TYPE, missing_bucket: true }
terms: {
field: SPAN_TYPE
}
}
},
{
[SPAN_SUBTYPE]: {
terms: { field: SPAN_SUBTYPE, missing_bucket: true }
}
},
{
[DESTINATION_ADDRESS]: {
terms: { field: DESTINATION_ADDRESS }
terms: {
field: SPAN_SUBTYPE,
missing_bucket: true
}
}
}
]
],
size: fingerprintBucketSize
},
aggs: {
sample: {
sampler: {
shard_size: 30
shard_size: samplerShardSize
},
aggs: {
trace_ids: {
terms: {
field: TRACE_ID,
size: 10,
size: traceIdBucketSize,
execution_hint: 'map' as const,
// remove bias towards large traces by sorting on trace.id
// which will be random-esque
@ -129,25 +147,9 @@ export async function getTraceSampleIds({
}
};
const tracesSampleResponse = await client.search<
{ trace: { id: string } },
typeof params
>(params);
let nextAfter: string | undefined;
const receivedAfterKey =
tracesSampleResponse.aggregations?.connections.after_key;
if (
receivedAfterKey &&
(tracesSampleResponse.aggregations?.connections.buckets.length ?? 0) >=
MAX_CONNECTIONS_PER_REQUEST
) {
nextAfter = Buffer.from(JSON.stringify(receivedAfterKey)).toString(
'base64'
);
}
const tracesSampleResponse = await client.search<unknown, typeof params>(
params
);
// make sure at least one trace per composite/connection bucket
// is queried
@ -167,7 +169,6 @@ export async function getTraceSampleIds({
);
return {
after: nextAfter,
traceIds
};
}

12
x-pack/plugins/apm/server/routes/service_map.ts
View file

@ -20,10 +20,12 @@ export const serviceMapRoute = createRoute(() => ({
path: '/api/apm/service-map',
params: {
query: t.intersection([
t.partial({ environment: t.string, serviceName: t.string }),
t.partial({
environment: t.string,
serviceName: t.string
}),
uiFiltersRt,
rangeRt,
t.partial({ after: t.string })
rangeRt
])
},
handler: async ({ context, request }) => {
@ -36,9 +38,9 @@ export const serviceMapRoute = createRoute(() => ({
const setup = await setupRequest(context, request);
const {
query: { serviceName, environment, after }
query: { serviceName, environment }
} = context.params;
return getServiceMap({ setup, serviceName, environment, after });
return getServiceMap({ setup, serviceName, environment });
}
}));