[APM] Optimize service map query (#60412)
* [APM] Optimize service map query Closes #60411. - Chunk trace lookup - Remove pagination, move dedupe logic to server * Fix imports * Fix imports again Co-authored-by: Nathan L Smith <smith@nlsmith.com>
This commit is contained in:
parent
01571b6739
commit
9cd0a36740
|
@ -13,7 +13,7 @@ import { getCytoscapeElements } from './get_cytoscape_elements';
|
|||
import serviceMapResponse from './cytoscape-layout-test-response.json';
|
||||
import { iconForNode } from './icons';
|
||||
|
||||
const elementsFromResponses = getCytoscapeElements([serviceMapResponse], '');
|
||||
const elementsFromResponses = getCytoscapeElements(serviceMapResponse, '');
|
||||
|
||||
storiesOf('app/ServiceMap/Cytoscape', module).add(
|
||||
'example',
|
||||
|
|
|
@ -4,166 +4,63 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { ValuesType } from 'utility-types';
|
||||
import { sortBy, isEqual } from 'lodash';
|
||||
import {
|
||||
Connection,
|
||||
ConnectionNode
|
||||
} from '../../../../../../../plugins/apm/common/service_map';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map';
|
||||
import { getAPMHref } from '../../shared/Links/apm/APMLink';
|
||||
|
||||
function getConnectionNodeId(node: ConnectionNode): string {
|
||||
if ('destination.address' in node) {
|
||||
// use a prefix to distinguish exernal destination ids from services
|
||||
return `>${node['destination.address']}`;
|
||||
}
|
||||
return node['service.name'];
|
||||
}
|
||||
|
||||
function getConnectionId(connection: Connection) {
|
||||
return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId(
|
||||
connection.destination
|
||||
)}`;
|
||||
}
|
||||
export function getCytoscapeElements(
|
||||
responses: ServiceMapAPIResponse[],
|
||||
response: ServiceMapAPIResponse,
|
||||
search: string
|
||||
) {
|
||||
const discoveredServices = responses.flatMap(
|
||||
response => response.discoveredServices
|
||||
);
|
||||
|
||||
const serviceNodes = responses
|
||||
.flatMap(response => response.services)
|
||||
.map(service => ({
|
||||
...service,
|
||||
id: service['service.name']
|
||||
}));
|
||||
|
||||
// maps destination.address to service.name if possible
|
||||
function getConnectionNode(node: ConnectionNode) {
|
||||
let mappedNode: ConnectionNode | undefined;
|
||||
|
||||
if ('destination.address' in node) {
|
||||
mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to;
|
||||
}
|
||||
|
||||
if (!mappedNode) {
|
||||
mappedNode = node;
|
||||
}
|
||||
|
||||
return {
|
||||
...mappedNode,
|
||||
id: getConnectionNodeId(mappedNode)
|
||||
};
|
||||
}
|
||||
|
||||
// build connections with mapped nodes
|
||||
const connections = responses
|
||||
.flatMap(response => response.connections)
|
||||
.map(connection => {
|
||||
const source = getConnectionNode(connection.source);
|
||||
const destination = getConnectionNode(connection.destination);
|
||||
|
||||
return {
|
||||
source,
|
||||
destination,
|
||||
id: getConnectionId({ source, destination })
|
||||
};
|
||||
})
|
||||
.filter(connection => connection.source.id !== connection.destination.id);
|
||||
|
||||
const nodes = connections
|
||||
.flatMap(connection => [connection.source, connection.destination])
|
||||
.concat(serviceNodes);
|
||||
|
||||
type ConnectionWithId = ValuesType<typeof connections>;
|
||||
type ConnectionNodeWithId = ValuesType<typeof nodes>;
|
||||
|
||||
const connectionsById = connections.reduce((connectionMap, connection) => {
|
||||
return {
|
||||
...connectionMap,
|
||||
[connection.id]: connection
|
||||
};
|
||||
}, {} as Record<string, ConnectionWithId>);
|
||||
const { nodes, connections } = response;
|
||||
|
||||
const nodesById = nodes.reduce((nodeMap, node) => {
|
||||
return {
|
||||
...nodeMap,
|
||||
[node.id]: node
|
||||
};
|
||||
}, {} as Record<string, ConnectionNodeWithId>);
|
||||
}, {} as Record<string, ValuesType<typeof nodes>>);
|
||||
|
||||
const cyNodes = (Object.values(nodesById) as ConnectionNodeWithId[]).map(
|
||||
node => {
|
||||
let data = {};
|
||||
const cyNodes = (Object.values(nodesById) as Array<
|
||||
ValuesType<typeof nodes>
|
||||
>).map(node => {
|
||||
let data = {};
|
||||
|
||||
if ('service.name' in node) {
|
||||
data = {
|
||||
href: getAPMHref(
|
||||
`/services/${node['service.name']}/service-map`,
|
||||
search
|
||||
),
|
||||
agentName: node['agent.name'],
|
||||
frameworkName: node['service.framework.name'],
|
||||
type: 'service'
|
||||
};
|
||||
}
|
||||
|
||||
if ('span.type' in node) {
|
||||
data = {
|
||||
// For nodes with span.type "db", convert it to "database". Otherwise leave it as-is.
|
||||
type: node['span.type'] === 'db' ? 'database' : node['span.type'],
|
||||
// Externals should not have a subtype so make it undefined if the type is external.
|
||||
subtype: node['span.type'] !== 'external' && node['span.subtype']
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
group: 'nodes' as const,
|
||||
data: {
|
||||
id: node.id,
|
||||
label:
|
||||
'service.name' in node
|
||||
? node['service.name']
|
||||
: node['destination.address'],
|
||||
...data
|
||||
}
|
||||
if ('service.name' in node) {
|
||||
data = {
|
||||
href: getAPMHref(
|
||||
`/services/${node['service.name']}/service-map`,
|
||||
search
|
||||
),
|
||||
agentName: node['agent.name'],
|
||||
frameworkName: node['service.framework.name'],
|
||||
type: 'service'
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// instead of adding connections in two directions,
|
||||
// we add a `bidirectional` flag to use in styling
|
||||
// and hide the inverse edge when rendering
|
||||
const dedupedConnections = (sortBy(
|
||||
Object.values(connectionsById),
|
||||
// make sure that order is stable
|
||||
'id'
|
||||
) as ConnectionWithId[]).reduce<
|
||||
Array<
|
||||
ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean }
|
||||
>
|
||||
>((prev, connection) => {
|
||||
const reversedConnection = prev.find(
|
||||
c =>
|
||||
c.destination.id === connection.source.id &&
|
||||
c.source.id === connection.destination.id
|
||||
);
|
||||
|
||||
if (reversedConnection) {
|
||||
reversedConnection.bidirectional = true;
|
||||
return prev.concat({
|
||||
...connection,
|
||||
isInverseEdge: true
|
||||
});
|
||||
if ('span.type' in node) {
|
||||
data = {
|
||||
// For nodes with span.type "db", convert it to "database". Otherwise leave it as-is.
|
||||
type: node['span.type'] === 'db' ? 'database' : node['span.type'],
|
||||
// Externals should not have a subtype so make it undefined if the type is external.
|
||||
subtype: node['span.type'] !== 'external' && node['span.subtype']
|
||||
};
|
||||
}
|
||||
|
||||
return prev.concat(connection);
|
||||
}, []);
|
||||
return {
|
||||
group: 'nodes' as const,
|
||||
data: {
|
||||
id: node.id,
|
||||
label:
|
||||
'service.name' in node
|
||||
? node['service.name']
|
||||
: node['destination.address'],
|
||||
...data
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
const cyEdges = dedupedConnections.map(connection => {
|
||||
const cyEdges = connections.map(connection => {
|
||||
return {
|
||||
group: 'edges' as const,
|
||||
classes: connection.isInverseEdge ? 'invisible' : undefined,
|
||||
|
|
|
@ -4,26 +4,15 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { EuiBetaBadge } from '@elastic/eui';
|
||||
import theme from '@elastic/eui/dist/eui_theme_light.json';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { ElementDefinition } from 'cytoscape';
|
||||
import { find, isEqual } from 'lodash';
|
||||
import React, {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState
|
||||
} from 'react';
|
||||
import { EuiBetaBadge } from '@elastic/eui';
|
||||
import React, { useMemo } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import { isValidPlatinumLicense } from '../../../../../../../plugins/apm/common/service_map';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map';
|
||||
import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
|
||||
import { useDeepObjectIdentity } from '../../../hooks/useDeepObjectIdentity';
|
||||
import { useFetcher } from '../../../hooks/useFetcher';
|
||||
import { useLicense } from '../../../hooks/useLicense';
|
||||
import { useLoadingIndicator } from '../../../hooks/useLoadingIndicator';
|
||||
import { useLocation } from '../../../hooks/useLocation';
|
||||
import { useUrlParams } from '../../../hooks/useUrlParams';
|
||||
import { callApmApi } from '../../../services/rest/createCallApmApi';
|
||||
|
@ -64,13 +53,11 @@ const BetaBadgeContainer = styled.div`
|
|||
top: ${theme.gutterTypes.gutterSmall};
|
||||
z-index: 1; /* The element containing the cytoscape canvas has z-index = 0. */
|
||||
`;
|
||||
const MAX_REQUESTS = 5;
|
||||
|
||||
export function ServiceMap({ serviceName }: ServiceMapProps) {
|
||||
const license = useLicense();
|
||||
const { search } = useLocation();
|
||||
const { urlParams, uiFilters } = useUrlParams();
|
||||
const { notifications } = useApmPluginContext().core;
|
||||
const params = useDeepObjectIdentity({
|
||||
start: urlParams.start,
|
||||
end: urlParams.end,
|
||||
|
@ -82,95 +69,28 @@ export function ServiceMap({ serviceName }: ServiceMapProps) {
|
|||
}
|
||||
});
|
||||
|
||||
const renderedElements = useRef<ElementDefinition[]>([]);
|
||||
|
||||
const [responses, setResponses] = useState<ServiceMapAPIResponse[]>([]);
|
||||
|
||||
const { setIsLoading } = useLoadingIndicator();
|
||||
|
||||
const [, _setUnusedState] = useState(false);
|
||||
|
||||
const elements = useMemo(() => getCytoscapeElements(responses, search), [
|
||||
responses,
|
||||
search
|
||||
]);
|
||||
|
||||
const forceUpdate = useCallback(() => _setUnusedState(value => !value), []);
|
||||
|
||||
const getNext = useCallback(
|
||||
async (input: { reset?: boolean; after?: string | undefined }) => {
|
||||
const { start, end, uiFilters: strippedUiFilters, ...query } = params;
|
||||
|
||||
if (input.reset) {
|
||||
renderedElements.current = [];
|
||||
setResponses([]);
|
||||
}
|
||||
|
||||
if (start && end) {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const data = await callApmApi({
|
||||
pathname: '/api/apm/service-map',
|
||||
params: {
|
||||
query: {
|
||||
...query,
|
||||
start,
|
||||
end,
|
||||
uiFilters: JSON.stringify(strippedUiFilters),
|
||||
after: input.after
|
||||
}
|
||||
}
|
||||
});
|
||||
setResponses(resp => resp.concat(data));
|
||||
|
||||
const shouldGetNext =
|
||||
responses.length + 1 < MAX_REQUESTS && data.after;
|
||||
|
||||
if (shouldGetNext) {
|
||||
await getNext({ after: data.after });
|
||||
} else {
|
||||
setIsLoading(false);
|
||||
const { data } = useFetcher(() => {
|
||||
const { start, end } = params;
|
||||
if (start && end) {
|
||||
return callApmApi({
|
||||
pathname: '/api/apm/service-map',
|
||||
params: {
|
||||
query: {
|
||||
...params,
|
||||
start,
|
||||
end,
|
||||
uiFilters: JSON.stringify(params.uiFilters)
|
||||
}
|
||||
} catch (error) {
|
||||
setIsLoading(false);
|
||||
notifications.toasts.addError(error, {
|
||||
title: i18n.translate('xpack.apm.errorServiceMapData', {
|
||||
defaultMessage: `Error loading service connections`
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
[params, setIsLoading, responses.length, notifications.toasts]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const loadServiceMaps = async () => {
|
||||
await getNext({ reset: true });
|
||||
};
|
||||
|
||||
loadServiceMaps();
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
});
|
||||
}
|
||||
}, [params]);
|
||||
|
||||
useEffect(() => {
|
||||
if (renderedElements.current.length === 0) {
|
||||
renderedElements.current = elements;
|
||||
return;
|
||||
}
|
||||
const elements = useMemo(() => {
|
||||
return data ? getCytoscapeElements(data as any, search) : [];
|
||||
}, [data, search]);
|
||||
|
||||
const newElements = elements.filter(element => {
|
||||
return !find(renderedElements.current, el => isEqual(el, element));
|
||||
});
|
||||
|
||||
if (newElements.length > 0 && renderedElements.current.length > 0) {
|
||||
renderedElements.current = elements;
|
||||
forceUpdate();
|
||||
}
|
||||
}, [elements, forceUpdate]);
|
||||
|
||||
const { ref: wrapperRef, width, height } = useRefDimensions();
|
||||
const { ref, height, width } = useRefDimensions();
|
||||
|
||||
if (!license) {
|
||||
return null;
|
||||
|
@ -179,10 +99,10 @@ export function ServiceMap({ serviceName }: ServiceMapProps) {
|
|||
return isValidPlatinumLicense(license) ? (
|
||||
<div
|
||||
style={{ height: height - parseInt(theme.gutterTypes.gutterLarge, 10) }}
|
||||
ref={wrapperRef}
|
||||
ref={ref}
|
||||
>
|
||||
<Cytoscape
|
||||
elements={renderedElements.current}
|
||||
elements={elements}
|
||||
serviceName={serviceName}
|
||||
height={height}
|
||||
width={width}
|
||||
|
|
|
@ -17,6 +17,13 @@ export const config = {
|
|||
schema: schema.object({
|
||||
enabled: schema.boolean({ defaultValue: true }),
|
||||
serviceMapEnabled: schema.boolean({ defaultValue: true }),
|
||||
serviceMapFingerprintBucketSize: schema.number({ defaultValue: 100 }),
|
||||
serviceMapTraceIdBucketSize: schema.number({ defaultValue: 65 }),
|
||||
serviceMapFingerprintGlobalBucketSize: schema.number({
|
||||
defaultValue: 1000
|
||||
}),
|
||||
serviceMapTraceIdGlobalBucketSize: schema.number({ defaultValue: 6 }),
|
||||
serviceMapMaxTracesPerRequest: schema.number({ defaultValue: 50 }),
|
||||
autocreateApmIndexPattern: schema.boolean({ defaultValue: true }),
|
||||
ui: schema.object({
|
||||
enabled: schema.boolean({ defaultValue: true }),
|
||||
|
@ -41,6 +48,16 @@ export function mergeConfigs(
|
|||
'apm_oss.onboardingIndices': apmOssConfig.onboardingIndices,
|
||||
'apm_oss.indexPattern': apmOssConfig.indexPattern,
|
||||
'xpack.apm.serviceMapEnabled': apmConfig.serviceMapEnabled,
|
||||
'xpack.apm.serviceMapFingerprintBucketSize':
|
||||
apmConfig.serviceMapFingerprintBucketSize,
|
||||
'xpack.apm.serviceMapTraceIdBucketSize':
|
||||
apmConfig.serviceMapTraceIdBucketSize,
|
||||
'xpack.apm.serviceMapFingerprintGlobalBucketSize':
|
||||
apmConfig.serviceMapFingerprintGlobalBucketSize,
|
||||
'xpack.apm.serviceMapTraceIdGlobalBucketSize':
|
||||
apmConfig.serviceMapTraceIdGlobalBucketSize,
|
||||
'xpack.apm.serviceMapMaxTracesPerRequest':
|
||||
apmConfig.serviceMapMaxTracesPerRequest,
|
||||
'xpack.apm.ui.enabled': apmConfig.ui.enabled,
|
||||
'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems,
|
||||
'xpack.apm.ui.transactionGroupBucketSize':
|
||||
|
|
|
@ -45,6 +45,7 @@ export interface SetupTimeRange {
|
|||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
export interface SetupUIFilters {
|
||||
uiFiltersES: ESFilter[];
|
||||
}
|
||||
|
|
123
x-pack/plugins/apm/server/lib/service_map/dedupe_connections.ts
Normal file
123
x-pack/plugins/apm/server/lib/service_map/dedupe_connections.ts
Normal file
|
@ -0,0 +1,123 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { isEqual, sortBy } from 'lodash';
|
||||
import { ValuesType } from 'utility-types';
|
||||
import { ConnectionNode, Connection } from '../../../common/service_map';
|
||||
import { ConnectionsResponse, ServicesResponse } from './get_service_map';
|
||||
|
||||
function getConnectionNodeId(node: ConnectionNode): string {
|
||||
if ('destination.address' in node) {
|
||||
// use a prefix to distinguish exernal destination ids from services
|
||||
return `>${node['destination.address']}`;
|
||||
}
|
||||
return node['service.name'];
|
||||
}
|
||||
|
||||
function getConnectionId(connection: Connection) {
|
||||
return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId(
|
||||
connection.destination
|
||||
)}`;
|
||||
}
|
||||
|
||||
type ServiceMapResponse = ConnectionsResponse & { services: ServicesResponse };
|
||||
|
||||
export function dedupeConnections(response: ServiceMapResponse) {
|
||||
const { discoveredServices, services, connections } = response;
|
||||
|
||||
const serviceNodes = services.map(service => ({
|
||||
...service,
|
||||
id: service['service.name']
|
||||
}));
|
||||
|
||||
// maps destination.address to service.name if possible
|
||||
function getConnectionNode(node: ConnectionNode) {
|
||||
let mappedNode: ConnectionNode | undefined;
|
||||
|
||||
if ('destination.address' in node) {
|
||||
mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to;
|
||||
}
|
||||
|
||||
if (!mappedNode) {
|
||||
mappedNode = node;
|
||||
}
|
||||
|
||||
return {
|
||||
...mappedNode,
|
||||
id: getConnectionNodeId(mappedNode)
|
||||
};
|
||||
}
|
||||
|
||||
// build connections with mapped nodes
|
||||
const mappedConnections = connections
|
||||
.map(connection => {
|
||||
const source = getConnectionNode(connection.source);
|
||||
const destination = getConnectionNode(connection.destination);
|
||||
|
||||
return {
|
||||
source,
|
||||
destination,
|
||||
id: getConnectionId({ source, destination })
|
||||
};
|
||||
})
|
||||
.filter(connection => connection.source.id !== connection.destination.id);
|
||||
|
||||
const nodes = mappedConnections
|
||||
.flatMap(connection => [connection.source, connection.destination])
|
||||
.concat(serviceNodes);
|
||||
|
||||
const dedupedNodes: typeof nodes = [];
|
||||
|
||||
nodes.forEach(node => {
|
||||
if (!dedupedNodes.find(dedupedNode => isEqual(node, dedupedNode))) {
|
||||
dedupedNodes.push(node);
|
||||
}
|
||||
});
|
||||
|
||||
type ConnectionWithId = ValuesType<typeof mappedConnections>;
|
||||
|
||||
const connectionsById = mappedConnections.reduce(
|
||||
(connectionMap, connection) => {
|
||||
return {
|
||||
...connectionMap,
|
||||
[connection.id]: connection
|
||||
};
|
||||
},
|
||||
{} as Record<string, ConnectionWithId>
|
||||
);
|
||||
|
||||
// instead of adding connections in two directions,
|
||||
// we add a `bidirectional` flag to use in styling
|
||||
const dedupedConnections = (sortBy(
|
||||
Object.values(connectionsById),
|
||||
// make sure that order is stable
|
||||
'id'
|
||||
) as ConnectionWithId[]).reduce<
|
||||
Array<
|
||||
ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean }
|
||||
>
|
||||
>((prev, connection) => {
|
||||
const reversedConnection = prev.find(
|
||||
c =>
|
||||
c.destination.id === connection.source.id &&
|
||||
c.source.id === connection.destination.id
|
||||
);
|
||||
|
||||
if (reversedConnection) {
|
||||
reversedConnection.bidirectional = true;
|
||||
return prev.concat({
|
||||
...connection,
|
||||
isInverseEdge: true
|
||||
});
|
||||
}
|
||||
|
||||
return prev.concat(connection);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
nodes: dedupedNodes,
|
||||
connections: dedupedConnections
|
||||
};
|
||||
}
|
|
@ -3,7 +3,7 @@
|
|||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { chunk } from 'lodash';
|
||||
import { PromiseReturnType } from '../../../typings/common';
|
||||
import {
|
||||
Setup,
|
||||
|
@ -19,48 +19,61 @@ import {
|
|||
SERVICE_NAME,
|
||||
SERVICE_FRAMEWORK_NAME
|
||||
} from '../../../common/elasticsearch_fieldnames';
|
||||
import { dedupeConnections } from './dedupe_connections';
|
||||
|
||||
export interface IEnvOptions {
|
||||
setup: Setup & SetupTimeRange & SetupUIFilters;
|
||||
serviceName?: string;
|
||||
environment?: string;
|
||||
after?: string;
|
||||
}
|
||||
|
||||
async function getConnectionData({
|
||||
setup,
|
||||
serviceName,
|
||||
environment,
|
||||
after
|
||||
environment
|
||||
}: IEnvOptions) {
|
||||
const { traceIds, after: nextAfter } = await getTraceSampleIds({
|
||||
const { traceIds } = await getTraceSampleIds({
|
||||
setup,
|
||||
serviceName,
|
||||
environment,
|
||||
after
|
||||
environment
|
||||
});
|
||||
|
||||
const serviceMapData = traceIds.length
|
||||
? await getServiceMapFromTraceIds({
|
||||
const chunks = chunk(
|
||||
traceIds,
|
||||
setup.config['xpack.apm.serviceMapMaxTracesPerRequest']
|
||||
);
|
||||
|
||||
const init = {
|
||||
connections: [],
|
||||
discoveredServices: []
|
||||
};
|
||||
|
||||
if (!traceIds.length) {
|
||||
return init;
|
||||
}
|
||||
|
||||
const chunkedResponses = await Promise.all(
|
||||
chunks.map(traceIdsChunk =>
|
||||
getServiceMapFromTraceIds({
|
||||
setup,
|
||||
serviceName,
|
||||
environment,
|
||||
traceIds
|
||||
traceIds: traceIdsChunk
|
||||
})
|
||||
: { connections: [], discoveredServices: [] };
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
after: nextAfter,
|
||||
...serviceMapData
|
||||
};
|
||||
return chunkedResponses.reduce((prev, current) => {
|
||||
return {
|
||||
connections: prev.connections.concat(current.connections),
|
||||
discoveredServices: prev.discoveredServices.concat(
|
||||
current.discoveredServices
|
||||
)
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function getServicesData(options: IEnvOptions) {
|
||||
// only return services on the first request for the global service map
|
||||
if (options.after) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { setup } = options;
|
||||
|
||||
const projection = getServicesProjection({ setup });
|
||||
|
@ -125,15 +138,19 @@ async function getServicesData(options: IEnvOptions) {
|
|||
);
|
||||
}
|
||||
|
||||
export type ConnectionsResponse = PromiseReturnType<typeof getConnectionData>;
|
||||
export type ServicesResponse = PromiseReturnType<typeof getServicesData>;
|
||||
|
||||
export type ServiceMapAPIResponse = PromiseReturnType<typeof getServiceMap>;
|
||||
|
||||
export async function getServiceMap(options: IEnvOptions) {
|
||||
const [connectionData, servicesData] = await Promise.all([
|
||||
getConnectionData(options),
|
||||
getServicesData(options)
|
||||
]);
|
||||
|
||||
return {
|
||||
return dedupeConnections({
|
||||
...connectionData,
|
||||
services: servicesData
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
|
@ -15,27 +15,24 @@ import {
|
|||
PROCESSOR_EVENT,
|
||||
SERVICE_NAME,
|
||||
SERVICE_ENVIRONMENT,
|
||||
SPAN_TYPE,
|
||||
SPAN_SUBTYPE,
|
||||
TRACE_ID,
|
||||
DESTINATION_ADDRESS,
|
||||
TRACE_ID
|
||||
SPAN_TYPE,
|
||||
SPAN_SUBTYPE
|
||||
} from '../../../common/elasticsearch_fieldnames';
|
||||
|
||||
const MAX_CONNECTIONS_PER_REQUEST = 1000;
|
||||
const MAX_TRACES_TO_INSPECT = 1000;
|
||||
|
||||
export async function getTraceSampleIds({
|
||||
after,
|
||||
serviceName,
|
||||
environment,
|
||||
setup
|
||||
}: {
|
||||
after?: string;
|
||||
serviceName?: string;
|
||||
environment?: string;
|
||||
setup: Setup & SetupTimeRange & SetupUIFilters;
|
||||
}) {
|
||||
const { start, end, client, indices } = setup;
|
||||
const { start, end, client, indices, config } = setup;
|
||||
|
||||
const rangeQuery = { range: rangeFilter(start, end) };
|
||||
|
||||
|
@ -65,9 +62,15 @@ export async function getTraceSampleIds({
|
|||
query.bool.filter.push({ term: { [SERVICE_ENVIRONMENT]: environment } });
|
||||
}
|
||||
|
||||
const afterObj = after
|
||||
? { after: JSON.parse(Buffer.from(after, 'base64').toString()) }
|
||||
: {};
|
||||
const fingerprintBucketSize = serviceName
|
||||
? config['xpack.apm.serviceMapFingerprintBucketSize']
|
||||
: config['xpack.apm.serviceMapFingerprintGlobalBucketSize'];
|
||||
|
||||
const traceIdBucketSize = serviceName
|
||||
? config['xpack.apm.serviceMapTraceIdBucketSize']
|
||||
: config['xpack.apm.serviceMapTraceIdGlobalBucketSize'];
|
||||
|
||||
const samplerShardSize = traceIdBucketSize * 10;
|
||||
|
||||
const params = {
|
||||
index: [indices['apm_oss.spanIndices']],
|
||||
|
@ -77,42 +80,57 @@ export async function getTraceSampleIds({
|
|||
aggs: {
|
||||
connections: {
|
||||
composite: {
|
||||
size: MAX_CONNECTIONS_PER_REQUEST,
|
||||
...afterObj,
|
||||
sources: [
|
||||
{ [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } },
|
||||
{
|
||||
[DESTINATION_ADDRESS]: {
|
||||
terms: {
|
||||
field: DESTINATION_ADDRESS
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
[SERVICE_NAME]: {
|
||||
terms: {
|
||||
field: SERVICE_NAME
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
[SERVICE_ENVIRONMENT]: {
|
||||
terms: { field: SERVICE_ENVIRONMENT, missing_bucket: true }
|
||||
terms: {
|
||||
field: SERVICE_ENVIRONMENT,
|
||||
missing_bucket: true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
[SPAN_TYPE]: {
|
||||
terms: { field: SPAN_TYPE, missing_bucket: true }
|
||||
terms: {
|
||||
field: SPAN_TYPE
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
[SPAN_SUBTYPE]: {
|
||||
terms: { field: SPAN_SUBTYPE, missing_bucket: true }
|
||||
}
|
||||
},
|
||||
{
|
||||
[DESTINATION_ADDRESS]: {
|
||||
terms: { field: DESTINATION_ADDRESS }
|
||||
terms: {
|
||||
field: SPAN_SUBTYPE,
|
||||
missing_bucket: true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
],
|
||||
size: fingerprintBucketSize
|
||||
},
|
||||
aggs: {
|
||||
sample: {
|
||||
sampler: {
|
||||
shard_size: 30
|
||||
shard_size: samplerShardSize
|
||||
},
|
||||
aggs: {
|
||||
trace_ids: {
|
||||
terms: {
|
||||
field: TRACE_ID,
|
||||
size: 10,
|
||||
size: traceIdBucketSize,
|
||||
execution_hint: 'map' as const,
|
||||
// remove bias towards large traces by sorting on trace.id
|
||||
// which will be random-esque
|
||||
|
@ -129,25 +147,9 @@ export async function getTraceSampleIds({
|
|||
}
|
||||
};
|
||||
|
||||
const tracesSampleResponse = await client.search<
|
||||
{ trace: { id: string } },
|
||||
typeof params
|
||||
>(params);
|
||||
|
||||
let nextAfter: string | undefined;
|
||||
|
||||
const receivedAfterKey =
|
||||
tracesSampleResponse.aggregations?.connections.after_key;
|
||||
|
||||
if (
|
||||
receivedAfterKey &&
|
||||
(tracesSampleResponse.aggregations?.connections.buckets.length ?? 0) >=
|
||||
MAX_CONNECTIONS_PER_REQUEST
|
||||
) {
|
||||
nextAfter = Buffer.from(JSON.stringify(receivedAfterKey)).toString(
|
||||
'base64'
|
||||
);
|
||||
}
|
||||
const tracesSampleResponse = await client.search<unknown, typeof params>(
|
||||
params
|
||||
);
|
||||
|
||||
// make sure at least one trace per composite/connection bucket
|
||||
// is queried
|
||||
|
@ -167,7 +169,6 @@ export async function getTraceSampleIds({
|
|||
);
|
||||
|
||||
return {
|
||||
after: nextAfter,
|
||||
traceIds
|
||||
};
|
||||
}
|
||||
|
|
|
@ -20,10 +20,12 @@ export const serviceMapRoute = createRoute(() => ({
|
|||
path: '/api/apm/service-map',
|
||||
params: {
|
||||
query: t.intersection([
|
||||
t.partial({ environment: t.string, serviceName: t.string }),
|
||||
t.partial({
|
||||
environment: t.string,
|
||||
serviceName: t.string
|
||||
}),
|
||||
uiFiltersRt,
|
||||
rangeRt,
|
||||
t.partial({ after: t.string })
|
||||
rangeRt
|
||||
])
|
||||
},
|
||||
handler: async ({ context, request }) => {
|
||||
|
@ -36,9 +38,9 @@ export const serviceMapRoute = createRoute(() => ({
|
|||
|
||||
const setup = await setupRequest(context, request);
|
||||
const {
|
||||
query: { serviceName, environment, after }
|
||||
query: { serviceName, environment }
|
||||
} = context.params;
|
||||
return getServiceMap({ setup, serviceName, environment, after });
|
||||
return getServiceMap({ setup, serviceName, environment });
|
||||
}
|
||||
}));
|
||||
|
||||
|
|
Loading…
Reference in a new issue