From cb5979ad51abdd160ab2fa8829764bcc98ee8e0d Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 8 Sep 2020 17:29:26 -0400 Subject: [PATCH] [Lens] Show field stats for IP fields and scripted fields (#76457) Co-authored-by: Elastic Machine --- .../indexpattern_datasource/field_item.tsx | 9 +- .../indexpattern_datasource/loader.test.ts | 13 +- .../public/indexpattern_datasource/loader.ts | 30 ++-- .../public/indexpattern_datasource/mocks.ts | 12 ++ .../public/indexpattern_datasource/types.ts | 11 +- .../plugins/lens/server/routes/field_stats.ts | 50 ++++--- .../api_integration/apis/lens/field_stats.ts | 133 ++++++++++++++++++ 7 files changed, 215 insertions(+), 43 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index a0cc5ec35213..cf15c2984405 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -117,14 +117,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { ); function fetchData() { - if ( - state.isLoading || - (field.type !== 'number' && - field.type !== 'string' && - field.type !== 'date' && - field.type !== 'boolean' && - field.type !== 'ip') - ) { + if (state.isLoading) { return; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts index 660be9514a92..19213d4afc9b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts @@ -93,6 +93,16 @@ const indexPattern1 = ({ searchable: true, esTypes: ['keyword'], }, + { + name: 'scripted', + displayName: 'Scripted', + type: 'string', + searchable: true, + aggregatable: true, + scripted: true, + lang: 'painless', + script: '1234', + }, documentField, ], } as unknown) as IndexPattern; @@ -156,12 +166,13 @@ const indexPattern2 = ({ aggregatable: true, searchable: true, scripted: true, + lang: 'painless', + script: '1234', aggregationRestrictions: { terms: { agg: 'terms', }, }, - esTypes: ['keyword'], }, documentField, ], diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index 585a1281cbf5..0ab658b96133 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -55,15 +55,27 @@ export async function loadIndexPatterns({ !indexPatternsUtils.isNestedField(field) && (!!field.aggregatable || !!field.scripted) ) .map( - (field): IndexPatternField => ({ - name: field.name, - displayName: field.displayName, - type: field.type, - aggregatable: field.aggregatable, - searchable: field.searchable, - scripted: field.scripted, - esTypes: field.esTypes, - }) + (field): IndexPatternField => { + // Convert the getters on the index pattern service into plain JSON + const base = { + name: field.name, + displayName: field.displayName, + type: field.type, + aggregatable: field.aggregatable, + searchable: field.searchable, + esTypes: field.esTypes, + scripted: field.scripted, + }; + + // Simplifies tests by hiding optional properties instead of undefined + return base.scripted + ? { + ...base, + lang: field.lang, + script: field.script, + } + : base; + } ) .concat(documentField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts index 31e6240993d3..21ed23321cf5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts @@ -64,6 +64,16 @@ export const createMockedIndexPattern = (): IndexPattern => ({ searchable: true, esTypes: ['keyword'], }, + { + name: 'scripted', + displayName: 'Scripted', + type: 'string', + searchable: true, + aggregatable: true, + scripted: true, + lang: 'painless', + script: '1234', + }, ], }); @@ -95,6 +105,8 @@ export const createMockedRestrictedIndexPattern = () => ({ searchable: true, scripted: true, esTypes: ['keyword'], + lang: 'painless', + script: '1234', }, ], typeMeta: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index c101f1354b70..21ca41234fdf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { IFieldType } from 'src/plugins/data/common'; import { IndexPatternColumn } from './operations'; import { IndexPatternAggRestrictions } from '../../../../../src/plugins/data/public'; @@ -22,16 +23,10 @@ export interface IndexPattern { hasRestrictions: boolean; } -export interface IndexPatternField { - name: string; +export type IndexPatternField = IFieldType & { displayName: string; - type: string; - esTypes?: string[]; - aggregatable: boolean; - scripted?: boolean; - searchable: boolean; aggregationRestrictions?: Partial; -} +}; export interface IndexPatternLayer { columnOrder: string[]; diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 20d3e2b4164c..a7368a12f0e2 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -8,6 +8,7 @@ import Boom from 'boom'; import DateMath from '@elastic/datemath'; import { schema } from '@kbn/config-schema'; import { CoreSetup } from 'src/core/server'; +import { IFieldType } from 'src/plugins/data/common'; import { ESSearchResponse } from '../../../apm/typings/elasticsearch'; import { FieldStatsResponse, BASE_API_URL } from '../../common'; @@ -33,6 +34,9 @@ export async function initFieldsRoute(setup: CoreSetup) { name: schema.string(), type: schema.string(), esTypes: schema.maybe(schema.arrayOf(schema.string())), + scripted: schema.maybe(schema.boolean()), + lang: schema.maybe(schema.string()), + script: schema.maybe(schema.string()), }, { unknowns: 'allow' } ), @@ -83,21 +87,15 @@ export async function initFieldsRoute(setup: CoreSetup) { return res.ok({ body: await getNumberHistogram(search, field), }); - } else if (field.type === 'string') { - return res.ok({ - body: await getStringSamples(search, field), - }); } else if (field.type === 'date') { return res.ok({ body: await getDateHistogram(search, field, { fromDate, toDate }), }); - } else if (field.type === 'boolean') { - return res.ok({ - body: await getStringSamples(search, field), - }); } - return res.ok({}); + return res.ok({ + body: await getStringSamples(search, field), + }); } catch (e) { if (e.status === 404) { return res.notFound(); @@ -119,8 +117,10 @@ export async function initFieldsRoute(setup: CoreSetup) { export async function getNumberHistogram( aggSearchWithBody: (body: unknown) => Promise, - field: { name: string; type: string; esTypes?: string[] } + field: IFieldType ): Promise { + const fieldRef = getFieldRef(field); + const searchBody = { sample: { sampler: { shard_size: SHARD_SIZE }, @@ -131,9 +131,9 @@ export async function getNumberHistogram( max_value: { max: { field: field.name }, }, - sample_count: { value_count: { field: field.name } }, + sample_count: { value_count: { ...fieldRef } }, top_values: { - terms: { field: field.name, size: 10 }, + terms: { ...fieldRef, size: 10 }, }, }, }, @@ -206,15 +206,20 @@ export async function getNumberHistogram( export async function getStringSamples( aggSearchWithBody: (body: unknown) => unknown, - field: { name: string; type: string } + field: IFieldType ): Promise { + const fieldRef = getFieldRef(field); + const topValuesBody = { sample: { sampler: { shard_size: SHARD_SIZE }, aggs: { - sample_count: { value_count: { field: field.name } }, + sample_count: { value_count: { ...fieldRef } }, top_values: { - terms: { field: field.name, size: 10 }, + terms: { + ...fieldRef, + size: 10, + }, }, }, }, @@ -241,7 +246,7 @@ export async function getStringSamples( // This one is not sampled so that it returns the full date range export async function getDateHistogram( aggSearchWithBody: (body: unknown) => unknown, - field: { name: string; type: string }, + field: IFieldType, range: { fromDate: string; toDate: string } ): Promise { const fromDate = DateMath.parse(range.fromDate); @@ -265,7 +270,7 @@ export async function getDateHistogram( const fixedInterval = `${interval}ms`; const histogramBody = { - histo: { date_histogram: { field: field.name, fixed_interval: fixedInterval } }, + histo: { date_histogram: { ...getFieldRef(field), fixed_interval: fixedInterval } }, }; const results = (await aggSearchWithBody(histogramBody)) as ESSearchResponse< unknown, @@ -283,3 +288,14 @@ export async function getDateHistogram( }, }; } + +function getFieldRef(field: IFieldType) { + return field.scripted + ? { + script: { + lang: field.lang as string, + source: field.script as string, + }, + } + : { field: field.name }; +} diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index 87c9d97be9b6..ccaea03691f0 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -279,6 +279,139 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should return top values for ip fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'ip', + type: 'ip', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4633, + topValues: { + buckets: [ + { + count: 13, + key: '177.194.175.66', + }, + { + count: 12, + key: '18.55.141.62', + }, + { + count: 12, + key: '53.55.251.105', + }, + { + count: 11, + key: '21.111.249.239', + }, + { + count: 11, + key: '97.63.84.25', + }, + { + count: 11, + key: '100.99.207.174', + }, + { + count: 11, + key: '112.34.138.226', + }, + { + count: 11, + key: '194.68.89.92', + }, + { + count: 11, + key: '235.186.79.201', + }, + { + count: 10, + key: '57.79.108.136', + }, + ], + }, + }); + }); + + it('should return histograms for scripted date fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'scripted date', + type: 'date', + scripted: true, + script: '1234', + lang: 'painless', + }, + }) + .expect(200); + + expect(body).to.eql({ + histogram: { + buckets: [ + { + count: 4634, + key: 0, + }, + ], + }, + totalDocuments: 4634, + }); + }); + + it('should return top values for scripted string fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'scripted string', + type: 'string', + scripted: true, + script: 'return "hello"', + lang: 'painless', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4634, + topValues: { + buckets: [ + { + count: 4634, + key: 'hello', + }, + ], + }, + }); + }); + it('should apply filters and queries', async () => { const { body } = await supertest .post('/api/lens/index_stats/logstash-2015.09.22/field')