maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Lens] Show field stats for IP fields and scripted fields (#76457)

Browse source 
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Wylie Conlon 2020-09-08 17:29:26 -04:00 committed by GitHub
parent 02233e740e
commit cb5979ad51
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 215 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx
View file

@ -117,14 +117,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
);
function fetchData() {
if (
state.isLoading ||
(field.type !== 'number' &&
field.type !== 'string' &&
field.type !== 'date' &&
field.type !== 'boolean' &&
field.type !== 'ip')
) {
if (state.isLoading) {
return;
}

13
x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts
View file

@ -93,6 +93,16 @@ const indexPattern1 = ({
searchable: true,
esTypes: ['keyword'],
},
{
name: 'scripted',
displayName: 'Scripted',
type: 'string',
searchable: true,
aggregatable: true,
scripted: true,
lang: 'painless',
script: '1234',
},
documentField,
],
} as unknown) as IndexPattern;
@ -156,12 +166,13 @@ const indexPattern2 = ({
aggregatable: true,
searchable: true,
scripted: true,
lang: 'painless',
script: '1234',
aggregationRestrictions: {
terms: {
agg: 'terms',
},
},
esTypes: ['keyword'],
},
documentField,
],

30
x-pack/plugins/lens/public/indexpattern_datasource/loader.ts
View file

@ -55,15 +55,27 @@ export async function loadIndexPatterns({
!indexPatternsUtils.isNestedField(field) && (!!field.aggregatable || !!field.scripted)
)
.map(
(field): IndexPatternField => ({
name: field.name,
displayName: field.displayName,
type: field.type,
aggregatable: field.aggregatable,
searchable: field.searchable,
scripted: field.scripted,
esTypes: field.esTypes,
})
(field): IndexPatternField => {
// Convert the getters on the index pattern service into plain JSON
const base = {
name: field.name,
displayName: field.displayName,
type: field.type,
aggregatable: field.aggregatable,
searchable: field.searchable,
esTypes: field.esTypes,
scripted: field.scripted,
};
// Simplifies tests by hiding optional properties instead of undefined
return base.scripted
? {
...base,
lang: field.lang,
script: field.script,
}
: base;
}
)
.concat(documentField);

12
x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts
View file

@ -64,6 +64,16 @@ export const createMockedIndexPattern = (): IndexPattern => ({
searchable: true,
esTypes: ['keyword'],
},
{
name: 'scripted',
displayName: 'Scripted',
type: 'string',
searchable: true,
aggregatable: true,
scripted: true,
lang: 'painless',
script: '1234',
},
],
});
@ -95,6 +105,8 @@ export const createMockedRestrictedIndexPattern = () => ({
searchable: true,
scripted: true,
esTypes: ['keyword'],
lang: 'painless',
script: '1234',
},
],
typeMeta: {

11
x-pack/plugins/lens/public/indexpattern_datasource/types.ts
View file

@ -4,6 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { IFieldType } from 'src/plugins/data/common';
import { IndexPatternColumn } from './operations';
import { IndexPatternAggRestrictions } from '../../../../../src/plugins/data/public';
@ -22,16 +23,10 @@ export interface IndexPattern {
hasRestrictions: boolean;
}
export interface IndexPatternField {
name: string;
export type IndexPatternField = IFieldType & {
displayName: string;
type: string;
esTypes?: string[];
aggregatable: boolean;
scripted?: boolean;
searchable: boolean;
aggregationRestrictions?: Partial<IndexPatternAggRestrictions>;
}
};
export interface IndexPatternLayer {
columnOrder: string[];

50
x-pack/plugins/lens/server/routes/field_stats.ts
View file

@ -8,6 +8,7 @@ import Boom from 'boom';
import DateMath from '@elastic/datemath';
import { schema } from '@kbn/config-schema';
import { CoreSetup } from 'src/core/server';
import { IFieldType } from 'src/plugins/data/common';
import { ESSearchResponse } from '../../../apm/typings/elasticsearch';
import { FieldStatsResponse, BASE_API_URL } from '../../common';
@ -33,6 +34,9 @@ export async function initFieldsRoute(setup: CoreSetup) {
name: schema.string(),
type: schema.string(),
esTypes: schema.maybe(schema.arrayOf(schema.string())),
scripted: schema.maybe(schema.boolean()),
lang: schema.maybe(schema.string()),
script: schema.maybe(schema.string()),
},
{ unknowns: 'allow' }
),
@ -83,21 +87,15 @@ export async function initFieldsRoute(setup: CoreSetup) {
return res.ok({
body: await getNumberHistogram(search, field),
});
} else if (field.type === 'string') {
return res.ok({
body: await getStringSamples(search, field),
});
} else if (field.type === 'date') {
return res.ok({
body: await getDateHistogram(search, field, { fromDate, toDate }),
});
} else if (field.type === 'boolean') {
return res.ok({
body: await getStringSamples(search, field),
});
}
return res.ok({});
return res.ok({
body: await getStringSamples(search, field),
});
} catch (e) {
if (e.status === 404) {
return res.notFound();
@ -119,8 +117,10 @@ export async function initFieldsRoute(setup: CoreSetup) {
export async function getNumberHistogram(
aggSearchWithBody: (body: unknown) => Promise<unknown>,
field: { name: string; type: string; esTypes?: string[] }
field: IFieldType
): Promise<FieldStatsResponse> {
const fieldRef = getFieldRef(field);
const searchBody = {
sample: {
sampler: { shard_size: SHARD_SIZE },
@ -131,9 +131,9 @@ export async function getNumberHistogram(
max_value: {
max: { field: field.name },
},
sample_count: { value_count: { field: field.name } },
sample_count: { value_count: { ...fieldRef } },
top_values: {
terms: { field: field.name, size: 10 },
terms: { ...fieldRef, size: 10 },
},
},
},
@ -206,15 +206,20 @@ export async function getNumberHistogram(
export async function getStringSamples(
aggSearchWithBody: (body: unknown) => unknown,
field: { name: string; type: string }
field: IFieldType
): Promise<FieldStatsResponse> {
const fieldRef = getFieldRef(field);
const topValuesBody = {
sample: {
sampler: { shard_size: SHARD_SIZE },
aggs: {
sample_count: { value_count: { field: field.name } },
sample_count: { value_count: { ...fieldRef } },
top_values: {
terms: { field: field.name, size: 10 },
terms: {
...fieldRef,
size: 10,
},
},
},
},
@ -241,7 +246,7 @@ export async function getStringSamples(
// This one is not sampled so that it returns the full date range
export async function getDateHistogram(
aggSearchWithBody: (body: unknown) => unknown,
field: { name: string; type: string },
field: IFieldType,
range: { fromDate: string; toDate: string }
): Promise<FieldStatsResponse> {
const fromDate = DateMath.parse(range.fromDate);
@ -265,7 +270,7 @@ export async function getDateHistogram(
const fixedInterval = `${interval}ms`;
const histogramBody = {
histo: { date_histogram: { field: field.name, fixed_interval: fixedInterval } },
histo: { date_histogram: { ...getFieldRef(field), fixed_interval: fixedInterval } },
};
const results = (await aggSearchWithBody(histogramBody)) as ESSearchResponse<
unknown,
@ -283,3 +288,14 @@ export async function getDateHistogram(
},
};
}
function getFieldRef(field: IFieldType) {
return field.scripted
? {
script: {
lang: field.lang as string,
source: field.script as string,
},
}
: { field: field.name };
}

133
x-pack/test/api_integration/apis/lens/field_stats.ts
View file

@ -279,6 +279,139 @@ export default ({ getService }: FtrProviderContext) => {
});
});
it('should return top values for ip fields', async () => {
const { body } = await supertest
.post('/api/lens/index_stats/logstash-2015.09.22/field')
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
timeFieldName: '@timestamp',
field: {
name: 'ip',
type: 'ip',
},
})
.expect(200);
expect(body).to.eql({
totalDocuments: 4634,
sampledDocuments: 4634,
sampledValues: 4633,
topValues: {
buckets: [
{
count: 13,
key: '177.194.175.66',
},
{
count: 12,
key: '18.55.141.62',
},
{
count: 12,
key: '53.55.251.105',
},
{
count: 11,
key: '21.111.249.239',
},
{
count: 11,
key: '97.63.84.25',
},
{
count: 11,
key: '100.99.207.174',
},
{
count: 11,
key: '112.34.138.226',
},
{
count: 11,
key: '194.68.89.92',
},
{
count: 11,
key: '235.186.79.201',
},
{
count: 10,
key: '57.79.108.136',
},
],
},
});
});
it('should return histograms for scripted date fields', async () => {
const { body } = await supertest
.post('/api/lens/index_stats/logstash-2015.09.22/field')
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
timeFieldName: '@timestamp',
field: {
name: 'scripted date',
type: 'date',
scripted: true,
script: '1234',
lang: 'painless',
},
})
.expect(200);
expect(body).to.eql({
histogram: {
buckets: [
{
count: 4634,
key: 0,
},
],
},
totalDocuments: 4634,
});
});
it('should return top values for scripted string fields', async () => {
const { body } = await supertest
.post('/api/lens/index_stats/logstash-2015.09.22/field')
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
timeFieldName: '@timestamp',
field: {
name: 'scripted string',
type: 'string',
scripted: true,
script: 'return "hello"',
lang: 'painless',
},
})
.expect(200);
expect(body).to.eql({
totalDocuments: 4634,
sampledDocuments: 4634,
sampledValues: 4634,
topValues: {
buckets: [
{
count: 4634,
key: 'hello',
},
],
},
});
});
it('should apply filters and queries', async () => {
const { body } = await supertest
.post('/api/lens/index_stats/logstash-2015.09.22/field')