[APM] Show warning about unmigrated legacy data (#34164)

This commit is contained in:
Søren Louv-Jansen 2019-04-01 22:18:01 +02:00 committed by GitHub
parent 93b2fca25d
commit 433bb2d3fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 326 additions and 255 deletions

View file

@ -12,7 +12,6 @@ import { makeApmUsageCollector } from './server/lib/apm_telemetry';
import { initErrorsApi } from './server/routes/errors';
import { initMetricsApi } from './server/routes/metrics';
import { initServicesApi } from './server/routes/services';
import { initStatusApi } from './server/routes/status_check';
import { initTracesApi } from './server/routes/traces';
import { initTransactionGroupsApi } from './server/routes/transaction_groups';
@ -79,7 +78,6 @@ export function apm(kibana: any) {
initTracesApi(server);
initServicesApi(server);
initErrorsApi(server);
initStatusApi(server);
initMetricsApi(server);
makeApmUsageCollector(server);
}

View file

@ -10,13 +10,13 @@ import React from 'react';
import styled from 'styled-components';
import { NOT_AVAILABLE_LABEL } from 'x-pack/plugins/apm/common/i18n';
import { KibanaLink } from 'x-pack/plugins/apm/public/components/shared/Links/KibanaLink';
import { IServiceListItem } from 'x-pack/plugins/apm/server/lib/services/get_services';
import { ServiceListAPIResponse } from 'x-pack/plugins/apm/server/lib/services/get_services';
import { fontSizes, truncate } from '../../../../style/variables';
import { asDecimal, asMillis } from '../../../../utils/formatters';
import { ITableColumn, ManagedTable } from '../../../shared/ManagedTable';
interface Props {
items?: IServiceListItem[];
items?: ServiceListAPIResponse['items'];
noItemsMessage?: React.ReactNode;
}
@ -39,7 +39,9 @@ const AppLink = styled(KibanaLink)`
${truncate('100%')};
`;
export const SERVICE_COLUMNS: Array<ITableColumn<IServiceListItem>> = [
export const SERVICE_COLUMNS: Array<
ITableColumn<ServiceListAPIResponse['items'][0]>
> = [
{
field: 'serviceName',
name: i18n.translate('xpack.apm.servicesTable.nameColumnLabel', {

View file

@ -8,16 +8,16 @@ import React from 'react';
import { Provider } from 'react-redux';
import { render, wait, waitForElement } from 'react-testing-library';
import 'react-testing-library/cleanup-after-each';
import { toastNotifications } from 'ui/notify';
import * as apmRestServices from 'x-pack/plugins/apm/public/services/rest/apm/services';
// @ts-ignore
import configureStore from 'x-pack/plugins/apm/public/store/config/configureStore';
import * as statusCheck from '../../../../services/rest/apm/status_check';
import { ServiceOverview } from '../view';
function Comp() {
function renderServiceOverview() {
const store = configureStore();
return (
return render(
<Provider store={store}>
<ServiceOverview urlParams={{}} />
</Provider>
@ -41,60 +41,51 @@ describe('Service Overview -> View', () => {
it('should render services, when list is not empty', async () => {
// mock rest requests
const spy1 = jest
.spyOn(statusCheck, 'loadAgentStatus')
.mockResolvedValue(true);
const spy2 = jest
const dataFetchingSpy = jest
.spyOn(apmRestServices, 'loadServiceList')
.mockResolvedValue([
{
serviceName: 'My Python Service',
agentName: 'python',
transactionsPerMinute: 100,
errorsPerMinute: 200,
avgResponseTime: 300
},
{
serviceName: 'My Go Service',
agentName: 'go',
transactionsPerMinute: 400,
errorsPerMinute: 500,
avgResponseTime: 600
}
]);
.mockResolvedValue({
hasLegacyData: false,
hasHistoricalData: true,
items: [
{
serviceName: 'My Python Service',
agentName: 'python',
transactionsPerMinute: 100,
errorsPerMinute: 200,
avgResponseTime: 300
},
{
serviceName: 'My Go Service',
agentName: 'go',
transactionsPerMinute: 400,
errorsPerMinute: 500,
avgResponseTime: 600
}
]
});
const { container, getByText } = render(<Comp />);
const { container, getByText } = renderServiceOverview();
// wait for requests to be made
await wait(
() =>
expect(spy1).toHaveBeenCalledTimes(1) &&
expect(spy2).toHaveBeenCalledTimes(1)
);
await wait(() => expect(dataFetchingSpy).toHaveBeenCalledTimes(1));
await waitForElement(() => getByText('My Python Service'));
expect(container.querySelectorAll('.euiTableRow')).toMatchSnapshot();
});
it('should render getting started message, when list is empty and no historical data is found', async () => {
// mock rest requests
const spy1 = jest
.spyOn(statusCheck, 'loadAgentStatus')
.mockResolvedValue(false);
const spy2 = jest
const dataFetchingSpy = jest
.spyOn(apmRestServices, 'loadServiceList')
.mockResolvedValue([]);
.mockResolvedValue({
hasLegacyData: false,
hasHistoricalData: false,
items: []
});
const { container, getByText } = render(<Comp />);
const { container, getByText } = renderServiceOverview();
// wait for requests to be made
await wait(
() =>
expect(spy1).toHaveBeenCalledTimes(1) &&
expect(spy2).toHaveBeenCalledTimes(1)
);
await wait(() => expect(dataFetchingSpy).toHaveBeenCalledTimes(1));
// wait for elements to be rendered
await waitForElement(() =>
@ -107,26 +98,62 @@ describe('Service Overview -> View', () => {
});
it('should render empty message, when list is empty and historical data is found', async () => {
// mock rest requests
const spy1 = jest
.spyOn(statusCheck, 'loadAgentStatus')
.mockResolvedValue(true);
const spy2 = jest
const dataFetchingSpy = jest
.spyOn(apmRestServices, 'loadServiceList')
.mockResolvedValue([]);
.mockResolvedValue({
hasLegacyData: false,
hasHistoricalData: true,
items: []
});
const { container, getByText } = render(<Comp />);
const { container, getByText } = renderServiceOverview();
// wait for requests to be made
await wait(
() =>
expect(spy1).toHaveBeenCalledTimes(1) &&
expect(spy2).toHaveBeenCalledTimes(1)
);
// wait for elements to be rendered
await wait(() => expect(dataFetchingSpy).toHaveBeenCalledTimes(1));
await waitForElement(() => getByText('No services found'));
expect(container.querySelectorAll('.euiTableRow')).toMatchSnapshot();
});
it('should render upgrade migration notification when legacy data is found, ', async () => {
// create spies
const toastSpy = jest.spyOn(toastNotifications, 'addWarning');
const dataFetchingSpy = jest
.spyOn(apmRestServices, 'loadServiceList')
.mockResolvedValue({
hasLegacyData: true,
hasHistoricalData: true,
items: []
});
renderServiceOverview();
// wait for requests to be made
await wait(() => expect(dataFetchingSpy).toHaveBeenCalledTimes(1));
expect(toastSpy).toHaveBeenLastCalledWith(
expect.objectContaining({
title: 'Legacy data was detected within the selected time range'
})
);
});
it('should not render upgrade migration notification when legacy data is not found, ', async () => {
// create spies
const toastSpy = jest.spyOn(toastNotifications, 'addWarning');
const dataFetchingSpy = jest
.spyOn(apmRestServices, 'loadServiceList')
.mockResolvedValue({
hasLegacyData: false,
hasHistoricalData: true,
items: []
});
renderServiceOverview();
// wait for requests to be made
await wait(() => expect(dataFetchingSpy).toHaveBeenCalledTimes(1));
expect(toastSpy).not.toHaveBeenCalled();
});
});

View file

@ -5,11 +5,15 @@
*/
import { EuiPanel } from '@elastic/eui';
import React from 'react';
import { EuiLink } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import React, { useEffect } from 'react';
import chrome from 'ui/chrome';
import { toastNotifications } from 'ui/notify';
import url from 'url';
import { IUrlParams } from 'x-pack/plugins/apm/public/store/urlParams';
import { useFetcher } from '../../../hooks/useFetcher';
import { loadServiceList } from '../../../services/rest/apm/services';
import { loadAgentStatus } from '../../../services/rest/apm/status_check';
import { NoServicesMessage } from './NoServicesMessage';
import { ServiceList } from './ServiceList';
@ -17,19 +21,65 @@ interface Props {
urlParams: IUrlParams;
}
const initalData = {
items: [],
hasHistoricalData: true,
hasLegacyData: false
};
let hasDisplayedToast = false;
export function ServiceOverview({ urlParams }: Props) {
const { start, end, kuery } = urlParams;
const { data: agentStatus = true } = useFetcher(() => loadAgentStatus(), []);
const { data: serviceListData } = useFetcher(
const { data = initalData } = useFetcher(
() => loadServiceList({ start, end, kuery }),
[start, end, kuery]
);
useEffect(
() => {
if (data.hasLegacyData && !hasDisplayedToast) {
hasDisplayedToast = true;
toastNotifications.addWarning({
title: i18n.translate('xpack.apm.serviceOverview.toastTitle', {
defaultMessage:
'Legacy data was detected within the selected time range'
}),
text: (
<p>
{i18n.translate('xpack.apm.serviceOverview.toastText', {
defaultMessage:
"You're running Elastic Stack 7.0+ and we've detected incompatible data from a previous 6.x version. If you want to view this data in APM, you should migrate it. See more in "
})}
<EuiLink
href={url.format({
pathname: chrome.addBasePath('/app/kibana'),
hash: '/management/elasticsearch/upgrade_assistant'
})}
>
{i18n.translate(
'xpack.apm.serviceOverview.upgradeAssistantLink',
{
defaultMessage: 'the upgrade assistant'
}
)}
</EuiLink>
</p>
)
});
}
},
[data.hasLegacyData]
);
return (
<EuiPanel>
<ServiceList
items={serviceListData}
noItemsMessage={<NoServicesMessage historicalDataFound={agentStatus} />}
items={data.items}
noItemsMessage={
<NoServicesMessage historicalDataFound={data.hasHistoricalData} />
}
/>
</EuiPanel>
);

View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { callApi } from '../callApi';
export async function loadServerStatus() {
return callApi({
pathname: `/api/apm/status/server`
});
}
export async function loadAgentStatus() {
const res = await callApi<{ dataFound: boolean }>({
pathname: `/api/apm/status/agent`
});
return res.dataFound;
}

View file

@ -29,7 +29,7 @@ describe('setupRequest', () => {
};
});
it('should call callWithRequest with correct args', async () => {
it('should call callWithRequest with default args', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', { body: { foo: 'bar' } });
expect(callWithRequestSpy).toHaveBeenCalledWith(mockReq, 'myType', {
@ -46,6 +46,70 @@ describe('setupRequest', () => {
});
});
describe('omitLegacyData', () => {
it('should add `observer.version_major` filter if `omitLegacyData=true` ', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {
omitLegacyData: true,
body: { query: { bool: { filter: [{ term: 'someTerm' }] } } }
});
expect(callWithRequestSpy.mock.calls[0][2].body).toEqual({
query: {
bool: {
filter: [
{ term: 'someTerm' },
{ range: { 'observer.version_major': { gte: 7 } } }
]
}
}
});
});
it('should not add `observer.version_major` filter if `omitLegacyData=false` ', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {
omitLegacyData: false,
body: { query: { bool: { filter: [{ term: 'someTerm' }] } } }
});
expect(callWithRequestSpy.mock.calls[0][2].body).toEqual({
query: { bool: { filter: [{ term: 'someTerm' }] } }
});
});
it('should set filter if none exists', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {});
const params = callWithRequestSpy.mock.calls[0][2];
expect(params.body).toEqual({
query: {
bool: {
filter: [{ range: { 'observer.version_major': { gte: 7 } } }]
}
}
});
});
it('should have `omitLegacyData=true` as default and merge boolean filters', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {
body: {
query: { bool: { filter: [{ term: 'someTerm' }] } }
}
});
const params = callWithRequestSpy.mock.calls[0][2];
expect(params.body).toEqual({
query: {
bool: {
filter: [
{ term: 'someTerm' },
{ range: { 'observer.version_major': { gte: 7 } } }
]
}
}
});
});
});
it('should set ignore_throttled to false if includeFrozen is true', async () => {
// mock includeFrozen to return true
mockReq.getUiSettingsService.mockImplementation(() => ({
@ -56,35 +120,4 @@ describe('setupRequest', () => {
const params = callWithRequestSpy.mock.calls[0][2];
expect(params.ignore_throttled).toBe(false);
});
it('should set filter if none exists', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {});
const params = callWithRequestSpy.mock.calls[0][2];
expect(params.body).toEqual({
query: {
bool: { filter: [{ range: { 'observer.version_major': { gte: 7 } } }] }
}
});
});
it('should merge filters if one exists', async () => {
const setup = setupRequest(mockReq);
await setup.client('myType', {
body: {
query: { bool: { filter: [{ term: 'someTerm' }] } }
}
});
const params = callWithRequestSpy.mock.calls[0][2];
expect(params.body).toEqual({
query: {
bool: {
filter: [
{ term: 'someTerm' },
{ range: { 'observer.version_major': { gte: 7 } } }
]
}
}
});
});
});

View file

@ -11,7 +11,7 @@ import {
SearchParams
} from 'elasticsearch';
import { Legacy } from 'kibana';
import { merge } from 'lodash';
import { cloneDeep, has, set } from 'lodash';
import moment from 'moment';
import { OBSERVER_VERSION_MAJOR } from 'x-pack/plugins/apm/common/elasticsearch_fieldnames';
@ -19,9 +19,13 @@ function decodeEsQuery(esQuery?: string) {
return esQuery ? JSON.parse(decodeURIComponent(esQuery)) : null;
}
export interface APMSearchParams extends SearchParams {
omitLegacyData?: boolean;
}
export type ESClient = <T = void, U = void>(
type: string,
params: SearchParams
params: APMSearchParams
) => Promise<AggregationSearchResponse<T, U>>;
export interface Setup {
@ -39,13 +43,21 @@ interface APMRequestQuery {
esFilterQuery: string;
}
function addFilterForLegacyData(params: SearchParams) {
// ensure a filter exists
const nextParams = merge({}, params, {
body: { query: { bool: { filter: [] } } }
});
function addFilterForLegacyData({
omitLegacyData = true,
...params
}: APMSearchParams): SearchParams {
// search across all data (including data)
if (!omitLegacyData) {
return params;
}
// add to filter
const nextParams = cloneDeep(params);
if (!has(nextParams, 'body.query.bool.filter')) {
set(nextParams, 'body.query.bool.filter', []);
}
// add filter for omitting pre-7.x data
nextParams.body.query.bool.filter.push({
range: { [OBSERVER_VERSION_MAJOR]: { gte: 7 } }
});

View file

@ -5,16 +5,19 @@
*/
import { SearchParams } from 'elasticsearch';
import { PROCESSOR_EVENT } from '../../../common/elasticsearch_fieldnames';
import { Setup } from '../helpers/setup_request';
import { PROCESSOR_EVENT } from 'x-pack/plugins/apm/common/elasticsearch_fieldnames';
import { Setup } from '../../helpers/setup_request';
// Note: this logic is duplicated in tutorials/apm/envs/on_prem
export async function getAgentStatus({ setup }: { setup: Setup }) {
export async function getAgentStatus(setup: Setup) {
const { client, config } = setup;
const params: SearchParams = {
terminateAfter: 1,
index: [
config.get('apm_oss.errorIndices'),
config.get('apm_oss.metricsIndices'),
config.get('apm_oss.sourcemapIndices'),
config.get('apm_oss.transactionIndices')
],
body: {
@ -26,9 +29,9 @@ export async function getAgentStatus({ setup }: { setup: Setup }) {
terms: {
[PROCESSOR_EVENT]: [
'error',
'transaction',
'metric',
'sourcemap'
'sourcemap',
'transaction'
]
}
}
@ -39,8 +42,6 @@ export async function getAgentStatus({ setup }: { setup: Setup }) {
};
const resp = await client('search', params);
return {
dataFound: resp.hits.total > 0
};
const hasHistorialAgentData = resp.hits.total > 0;
return hasHistorialAgentData;
}

View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
OBSERVER_VERSION_MAJOR,
PROCESSOR_EVENT
} from 'x-pack/plugins/apm/common/elasticsearch_fieldnames';
import { APMSearchParams, Setup } from '../../helpers/setup_request';
// returns true if 6.x data is found
export async function getLegacyDataStatus(setup: Setup) {
const { client, config } = setup;
const params: APMSearchParams = {
omitLegacyData: false,
terminateAfter: 1,
index: [config.get('apm_oss.transactionIndices')],
body: {
size: 0,
query: {
bool: {
filter: [
{ terms: { [PROCESSOR_EVENT]: ['transaction'] } },
{ range: { [OBSERVER_VERSION_MAJOR]: { lt: 7 } } }
]
}
}
}
};
const resp = await client('search', params);
const hasLegacyData = resp.hits.total > 0;
return hasLegacyData;
}

View file

@ -12,23 +12,11 @@ import {
SERVICE_AGENT_NAME,
SERVICE_NAME,
TRANSACTION_DURATION
} from '../../../common/elasticsearch_fieldnames';
import { rangeFilter } from '../helpers/range_filter';
import { Setup } from '../helpers/setup_request';
} from '../../../../common/elasticsearch_fieldnames';
import { rangeFilter } from '../../helpers/range_filter';
import { Setup } from '../../helpers/setup_request';
export interface IServiceListItem {
serviceName: string;
agentName: string | undefined;
transactionsPerMinute: number;
errorsPerMinute: number;
avgResponseTime: number;
}
export type ServiceListAPIResponse = IServiceListItem[];
export async function getServices(
setup: Setup
): Promise<ServiceListAPIResponse> {
export async function getServicesItems(setup: Setup) {
const { start, end, esFilterQuery, client, config } = setup;
const filter: ESFilter[] = [
@ -97,7 +85,7 @@ export async function getServices(
const aggs = resp.aggregations;
const serviceBuckets = idx(aggs, _ => _.services.buckets) || [];
return serviceBuckets.map(bucket => {
const items = serviceBuckets.map(bucket => {
const eventTypes = bucket.events.buckets;
const transactions = eventTypes.find(e => e.key === 'transaction');
const totalTransactions = idx(transactions, _ => _.doc_count) || 0;
@ -117,4 +105,6 @@ export async function getServices(
avgResponseTime: bucket.avg.value
};
});
return items;
}

View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { isEmpty } from 'lodash';
import { PromiseReturnType } from 'x-pack/plugins/apm/typings/common';
import { Setup } from '../../helpers/setup_request';
import { getAgentStatus } from './get_agent_status';
import { getLegacyDataStatus } from './get_legacy_data_status';
import { getServicesItems } from './get_services_items';
export type ServiceListAPIResponse = PromiseReturnType<typeof getServices>;
export async function getServices(setup: Setup) {
const items = await getServicesItems(setup);
const hasLegacyData = await getLegacyDataStatus(setup);
// conditionally check for historical data if no services were found in the current time range
const noDataInCurrentTimeRange = isEmpty(items);
let hasHistorialAgentData = true;
if (noDataInCurrentTimeRange) {
hasHistorialAgentData = await getAgentStatus(setup);
}
return {
items,
hasHistoricalData: hasHistorialAgentData,
hasLegacyData
};
}

View file

@ -1,36 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchParams } from 'elasticsearch';
import { OBSERVER_LISTENING } from '../../../common/elasticsearch_fieldnames';
import { Setup } from '../helpers/setup_request';
// Note: this logic is duplicated in tutorials/apm/envs/on_prem
export async function getServerStatus({ setup }: { setup: Setup }) {
const { client, config } = setup;
const params: SearchParams = {
index: config.get('apm_oss.onboardingIndices'),
body: {
size: 0,
query: {
bool: {
filter: {
exists: {
field: OBSERVER_LISTENING
}
}
}
}
}
};
const resp = await client('search', params);
return {
dataFound: resp.hits.total >= 1
};
}

View file

@ -6,11 +6,8 @@
import { Server } from 'hapi';
import { flatten } from 'lodash';
// @ts-ignore
import { initErrorsApi } from '../errors';
import { initServicesApi } from '../services';
// @ts-ignore
import { initStatusApi } from '../status_check';
import { initTracesApi } from '../traces';
describe('route handlers should fail with a Boom error', () => {
@ -71,10 +68,6 @@ describe('route handlers should fail with a Boom error', () => {
await testRouteFailures(initServicesApi);
});
describe('status check routes', async () => {
await testRouteFailures(initStatusApi);
});
describe('trace routes', async () => {
await testRouteFailures(initTracesApi);
});

View file

@ -34,7 +34,7 @@ export function initServicesApi(server: Server) {
const services = await getServices(setup).catch(defaultErrorHandler);
// Store telemetry data derived from services
const agentNames = services.map(
const agentNames = services.items.map(
({ agentName }) => agentName as AgentName
);
const apmTelemetry = createApmTelementry(agentNames);

View file

@ -1,53 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import { Server } from 'hapi';
import Joi from 'joi';
import { setupRequest } from '../lib/helpers/setup_request';
import { getAgentStatus } from '../lib/status_check/agent_check';
import { getServerStatus } from '../lib/status_check/server_check';
const ROOT = '/api/apm/status';
const defaultErrorHandler = (err: Error) => {
// tslint:disable-next-line
console.error(err.stack);
throw Boom.boomify(err, { statusCode: 400 });
};
export function initStatusApi(server: Server) {
server.route({
method: 'GET',
path: `${ROOT}/server`,
options: {
validate: {
query: Joi.object().keys({
_debug: Joi.bool()
})
}
},
handler: req => {
const setup = setupRequest(req);
return getServerStatus({ setup }).catch(defaultErrorHandler);
}
});
server.route({
method: 'GET',
path: `${ROOT}/agent`,
options: {
validate: {
query: Joi.object().keys({
_debug: Joi.bool()
})
}
},
handler: req => {
const setup = setupRequest(req);
return getAgentStatus({ setup }).catch(defaultErrorHandler);
}
});
}

View file

@ -9,8 +9,14 @@ export interface StringMap<T = unknown> {
}
// Allow unknown properties in an object
export type AllowUnknownProperties<T> = T extends object
? { [P in keyof T]: AllowUnknownProperties<T[P]> } & {
export type AllowUnknownProperties<Obj> = Obj extends object
? { [Prop in keyof Obj]: AllowUnknownProperties<Obj[Prop]> } & {
[key: string]: unknown;
}
: T;
: Obj;
export type PromiseReturnType<Func> = Func extends (
...args: any[]
) => Promise<infer Value>
? Value
: Func;

View file

@ -7,4 +7,5 @@
export const toastNotifications = {
addSuccess: () => {},
addDanger: () => {},
addWarning: () => {},
};