From 3fd7dbe4555d7bbdaa5b6b2c8390d84e49e5f00c Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Mon, 13 Sep 2021 10:45:34 -0500 Subject: [PATCH] [data views] allow fields that start with an underscore in the field list (#111238) * stop filtering out fields that start with underscore --- .../index_patterns/flatten_hit.test.ts | 3 ++- .../index_patterns/index_patterns/flatten_hit.ts | 1 - .../fetcher/index_patterns_fetcher.ts | 1 + .../field_capabilities/field_capabilities.test.js | 11 +++++++++++ .../lib/field_capabilities/field_capabilities.ts | 14 ++++++++------ .../field_capabilities/field_caps_response.test.js | 1 + .../lib/field_capabilities/field_caps_response.ts | 4 ++++ .../fields_for_time_pattern_route/pattern.js | 9 +++++++++ .../fields_for_wildcard_route/conflicts.js | 4 ++++ .../fields_for_wildcard_route/response.js | 13 +++++++++++++ 10 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts index c9bb7d974997..5cb531a97ff7 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts @@ -57,9 +57,10 @@ describe('flattenHit', () => { fields: { date: ['1'], zzz: ['z'], + _abc: ['a'], }, }); - const expectedOrder = ['date', 'name', 'zzz', '_id', '_routing', '_score', '_type']; + const expectedOrder = ['_abc', 'date', 'name', 'zzz', '_id', '_routing', '_score', '_type']; expect(Object.keys(response)).toEqual(expectedOrder); expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder); }); diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts index bcdcca3a4daa..421994a9e02f 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts @@ -66,7 +66,6 @@ function decorateFlattenedWrapper(hit: Record, metaFields: Record { }); describe('response order', () => { + it('supports fields that start with an underscore', async () => { + const fields = ['_field_a', '_field_b']; + + stubDeps({ + fieldsFromFieldCaps: fields.map((name) => ({ name })), + }); + + const fieldNames = (await getFieldCapabilities()).map((field) => field.name); + expect(fieldNames).toEqual(fields); + }); + it('always returns fields in alphabetical order', async () => { const letters = 'ambcdfjopngihkel'.split(''); const sortedLetters = sortBy(letters); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index 56f666c424ed..a587bbbc39ca 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -34,20 +34,21 @@ export async function getFieldCapabilities( const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name'); const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) - .filter((name) => !name.startsWith('_')) + // not all meta fields are provided, so remove and manually add + .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field) .concat(metaFields) - .reduce<{ names: string[]; hash: Record }>( + .reduce<{ names: string[]; map: Map }>( (agg, value) => { - // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes - if (agg.hash[value] != null) { + // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data + if (agg.map.get(value) != null) { return agg; } else { - agg.hash[value] = value; + agg.map.set(value, value); agg.names.push(value); return agg; } }, - { names: [], hash: {} } + { names: [], map: new Map() } ) .names.map((name) => defaults({}, fieldsFromFieldCapsByName[name], { @@ -56,6 +57,7 @@ export async function getFieldCapabilities( searchable: false, aggregatable: false, readFromDocValues: false, + metadata_field: metaFields.includes(name), }) ) .map(mergeOverrides); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js index 5ac356f9e461..c12eff1b5a37 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js @@ -48,6 +48,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => { 'searchable', 'aggregatable', 'readFromDocValues', + 'metadata_field', ]); }); } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts index 6db9254b8152..3f83fd71b74e 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts @@ -116,6 +116,8 @@ export function readFieldCapsResponse( }), {} ), + // @ts-expect-error + metadata_field: capsByType[types[0]].metadata_field, }; // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes agg.array.push(field); @@ -131,6 +133,8 @@ export function readFieldCapsResponse( searchable: isSearchable, aggregatable: isAggregatable, readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType), + // @ts-expect-error + metadata_field: capsByType[types[0]].metadata_field, }; // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes agg.array.push(field); diff --git a/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js b/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js index 8d72cbd2fb37..da04e7c9f4b7 100644 --- a/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js +++ b/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js @@ -38,6 +38,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'Jan01', @@ -46,6 +47,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'Jan02', @@ -54,6 +56,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, ], }); @@ -77,6 +80,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'Jan02', @@ -85,6 +89,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, ], }); @@ -109,6 +114,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'Jan02', @@ -117,6 +123,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'meta1', @@ -124,6 +131,7 @@ export default function ({ getService }) { aggregatable: false, searchable: false, readFromDocValues: false, + metadata_field: true, }, { name: 'meta2', @@ -131,6 +139,7 @@ export default function ({ getService }) { aggregatable: false, searchable: false, readFromDocValues: false, + metadata_field: true, }, ], }); diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js index f050e2a0626d..2e7cc4a76340 100644 --- a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js +++ b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js @@ -35,6 +35,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'number_conflict', @@ -43,6 +44,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: true, + metadata_field: false, }, { name: 'string_conflict', @@ -51,6 +53,7 @@ export default function ({ getService }) { aggregatable: true, searchable: true, readFromDocValues: false, + metadata_field: false, }, { name: 'success', @@ -63,6 +66,7 @@ export default function ({ getService }) { boolean: ['logs-2017.01.02'], keyword: ['logs-2017.01.01'], }, + metadata_field: false, }, ], }); diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js index a3ae8f55135b..07603be1de9a 100644 --- a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js +++ b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js @@ -25,6 +25,7 @@ export default function ({ getService }) { aggregatable: true, name: 'bar', readFromDocValues: true, + metadata_field: false, }, { type: 'string', @@ -33,6 +34,7 @@ export default function ({ getService }) { aggregatable: false, name: 'baz', readFromDocValues: false, + metadata_field: false, }, { type: 'string', @@ -42,6 +44,7 @@ export default function ({ getService }) { name: 'baz.keyword', readFromDocValues: true, subType: { multi: { parent: 'baz' } }, + metadata_field: false, }, { type: 'number', @@ -50,6 +53,7 @@ export default function ({ getService }) { aggregatable: true, name: 'foo', readFromDocValues: true, + metadata_field: false, }, { aggregatable: true, @@ -63,6 +67,7 @@ export default function ({ getService }) { }, }, type: 'string', + metadata_field: false, }, ]; @@ -100,6 +105,7 @@ export default function ({ getService }) { readFromDocValues: false, searchable: true, type: 'string', + metadata_field: true, }, { aggregatable: false, @@ -108,6 +114,7 @@ export default function ({ getService }) { readFromDocValues: false, searchable: false, type: '_source', + metadata_field: true, }, { type: 'boolean', @@ -116,6 +123,7 @@ export default function ({ getService }) { aggregatable: true, name: 'bar', readFromDocValues: true, + metadata_field: false, }, { aggregatable: false, @@ -124,6 +132,7 @@ export default function ({ getService }) { readFromDocValues: false, searchable: true, type: 'string', + metadata_field: false, }, { type: 'string', @@ -133,6 +142,7 @@ export default function ({ getService }) { name: 'baz.keyword', readFromDocValues: true, subType: { multi: { parent: 'baz' } }, + metadata_field: false, }, { aggregatable: false, @@ -140,6 +150,7 @@ export default function ({ getService }) { readFromDocValues: false, searchable: false, type: 'string', + metadata_field: true, }, { type: 'number', @@ -148,6 +159,7 @@ export default function ({ getService }) { aggregatable: true, name: 'foo', readFromDocValues: true, + metadata_field: false, }, { aggregatable: true, @@ -161,6 +173,7 @@ export default function ({ getService }) { }, }, type: 'string', + metadata_field: false, }, ], })