[Lens] Add support for scripted fields and aliases to the existence API (#54064)
* Add support for scripted fields and default index pattern * Add scripted fields and aliases to existence API * Fix TypeScript errors. * Fix mappings parsing * Default to the index pattern timeFieldName Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Chris Davies
Wylie Conlon
parent
8c0440f29d
commit
79054afb5a
|
|
@ -282,7 +282,7 @@ describe('IndexPattern Data Panel', () => {
|
|||
const parts = url.split('/');
|
||||
const indexPatternTitle = parts[parts.length - 1];
|
||||
return {
|
||||
indexPatternTitle,
|
||||
indexPatternTitle: `${indexPatternTitle}_testtitle`,
|
||||
existingFieldNames: ['field_1', 'field_2'].map(
|
||||
fieldName => `${indexPatternTitle}_${fieldName}`
|
||||
),
|
||||
|
|
@ -352,9 +352,9 @@ describe('IndexPattern Data Panel', () => {
|
|||
});
|
||||
|
||||
expect(nextState.existingFields).toEqual({
|
||||
aaa: {
|
||||
aaa_field_1: true,
|
||||
aaa_field_2: true,
|
||||
a_testtitle: {
|
||||
a_field_1: true,
|
||||
a_field_2: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
@ -369,13 +369,13 @@ describe('IndexPattern Data Panel', () => {
|
|||
});
|
||||
|
||||
expect(nextState.existingFields).toEqual({
|
||||
aaa: {
|
||||
aaa_field_1: true,
|
||||
aaa_field_2: true,
|
||||
a_testtitle: {
|
||||
a_field_1: true,
|
||||
a_field_2: true,
|
||||
},
|
||||
bbb: {
|
||||
bbb_field_1: true,
|
||||
bbb_field_2: true,
|
||||
b_testtitle: {
|
||||
b_field_1: true,
|
||||
b_field_2: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
@ -397,7 +397,7 @@ describe('IndexPattern Data Panel', () => {
|
|||
expect(setState).toHaveBeenCalledTimes(2);
|
||||
expect(core.http.get).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/aaa', {
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
|
||||
query: {
|
||||
fromDate: '2019-01-01',
|
||||
toDate: '2020-01-01',
|
||||
|
|
@ -405,7 +405,7 @@ describe('IndexPattern Data Panel', () => {
|
|||
},
|
||||
});
|
||||
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/aaa', {
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
|
||||
query: {
|
||||
fromDate: '2019-01-01',
|
||||
toDate: '2020-01-02',
|
||||
|
|
@ -418,9 +418,9 @@ describe('IndexPattern Data Panel', () => {
|
|||
});
|
||||
|
||||
expect(nextState.existingFields).toEqual({
|
||||
aaa: {
|
||||
aaa_field_1: true,
|
||||
aaa_field_2: true,
|
||||
a_testtitle: {
|
||||
a_field_1: true,
|
||||
a_field_2: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
@ -436,7 +436,7 @@ describe('IndexPattern Data Panel', () => {
|
|||
|
||||
expect(setState).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/aaa', {
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
|
||||
query: {
|
||||
fromDate: '2019-01-01',
|
||||
toDate: '2020-01-01',
|
||||
|
|
@ -444,7 +444,7 @@ describe('IndexPattern Data Panel', () => {
|
|||
},
|
||||
});
|
||||
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/bbb', {
|
||||
expect(core.http.get).toHaveBeenCalledWith('/api/lens/existing_fields/b', {
|
||||
query: {
|
||||
fromDate: '2019-01-01',
|
||||
toDate: '2020-01-01',
|
||||
|
|
@ -457,13 +457,13 @@ describe('IndexPattern Data Panel', () => {
|
|||
});
|
||||
|
||||
expect(nextState.existingFields).toEqual({
|
||||
aaa: {
|
||||
aaa_field_1: true,
|
||||
aaa_field_2: true,
|
||||
a_testtitle: {
|
||||
a_field_1: true,
|
||||
a_field_2: true,
|
||||
},
|
||||
bbb: {
|
||||
bbb_field_1: true,
|
||||
bbb_field_2: true,
|
||||
b_testtitle: {
|
||||
b_field_1: true,
|
||||
b_field_2: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ export function IndexPatternDataPanel({
|
|||
.sort((a, b) => a.localeCompare(b))
|
||||
.filter(id => !!indexPatterns[id])
|
||||
.map(id => ({
|
||||
id,
|
||||
title: indexPatterns[id].title,
|
||||
timeFieldName: indexPatterns[id].timeFieldName,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -551,7 +551,7 @@ describe('loader', () => {
|
|||
dateRange: { fromDate: '1900-01-01', toDate: '2000-01-01' },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
fetchJson: fetchJson as any,
|
||||
indexPatterns: [{ title: 'a' }, { title: 'b' }, { title: 'c' }],
|
||||
indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
|
||||
setState,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
IndexPatternPersistedState,
|
||||
IndexPatternPrivateState,
|
||||
IndexPatternField,
|
||||
AggregationRestrictions,
|
||||
} from './types';
|
||||
import { updateLayerIndexPattern } from './state_helpers';
|
||||
import { DateRange, ExistingFields } from '../../common/types';
|
||||
|
|
@ -30,19 +31,7 @@ interface SavedIndexPatternAttributes extends SavedObjectAttributes {
|
|||
}
|
||||
|
||||
interface SavedRestrictionsObject {
|
||||
aggs: Record<
|
||||
string,
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
agg: string;
|
||||
fixed_interval?: string;
|
||||
calendar_interval?: string;
|
||||
delay?: string;
|
||||
time_zone?: string;
|
||||
}
|
||||
>
|
||||
>;
|
||||
aggs: Record<string, AggregationRestrictions>;
|
||||
}
|
||||
|
||||
type SetState = StateSetter<IndexPatternPrivateState>;
|
||||
|
|
@ -230,7 +219,7 @@ export async function syncExistingFields({
|
|||
setState,
|
||||
}: {
|
||||
dateRange: DateRange;
|
||||
indexPatterns: Array<{ title: string; timeFieldName?: string | null }>;
|
||||
indexPatterns: Array<{ id: string; timeFieldName?: string | null }>;
|
||||
fetchJson: HttpSetup['get'];
|
||||
setState: SetState;
|
||||
}) {
|
||||
|
|
@ -245,7 +234,7 @@ export async function syncExistingFields({
|
|||
query.timeFieldName = pattern.timeFieldName;
|
||||
}
|
||||
|
||||
return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.title}`, {
|
||||
return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, {
|
||||
query,
|
||||
}) as Promise<ExistingFields>;
|
||||
})
|
||||
|
|
@ -301,8 +290,9 @@ function fromSavedObject(
|
|||
newFields.forEach((field, index) => {
|
||||
const restrictionsObj: IndexPatternField['aggregationRestrictions'] = {};
|
||||
aggs.forEach(agg => {
|
||||
if (typeMeta.aggs[agg] && typeMeta.aggs[agg][field.name]) {
|
||||
restrictionsObj[agg] = typeMeta.aggs[agg][field.name];
|
||||
const restriction = typeMeta.aggs[agg] && typeMeta.aggs[agg][field.name];
|
||||
if (restriction) {
|
||||
restrictionsObj[agg] = restriction;
|
||||
}
|
||||
});
|
||||
if (Object.keys(restrictionsObj).length) {
|
||||
|
|
|
|||
|
|
@ -322,7 +322,7 @@ function parseInterval(currentInterval: string) {
|
|||
};
|
||||
}
|
||||
|
||||
function restrictedInterval(aggregationRestrictions?: AggregationRestrictions) {
|
||||
function restrictedInterval(aggregationRestrictions?: Partial<AggregationRestrictions>) {
|
||||
if (!aggregationRestrictions || !aggregationRestrictions.date_histogram) {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,18 +20,16 @@ export interface IndexPattern {
|
|||
>;
|
||||
}
|
||||
|
||||
export type AggregationRestrictions = Partial<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
agg: string;
|
||||
interval?: number;
|
||||
fixed_interval?: string;
|
||||
calendar_interval?: string;
|
||||
delay?: string;
|
||||
time_zone?: string;
|
||||
}
|
||||
>
|
||||
export type AggregationRestrictions = Record<
|
||||
string,
|
||||
{
|
||||
agg?: string;
|
||||
interval?: number;
|
||||
fixed_interval?: string;
|
||||
calendar_interval?: string;
|
||||
delay?: string;
|
||||
time_zone?: string;
|
||||
}
|
||||
>;
|
||||
|
||||
export interface IndexPatternField {
|
||||
|
|
@ -41,7 +39,7 @@ export interface IndexPatternField {
|
|||
aggregatable: boolean;
|
||||
scripted?: boolean;
|
||||
searchable: boolean;
|
||||
aggregationRestrictions?: AggregationRestrictions;
|
||||
aggregationRestrictions?: Partial<AggregationRestrictions>;
|
||||
}
|
||||
|
||||
export interface IndexPatternLayer {
|
||||
|
|
|
|||
|
|
@ -4,24 +4,29 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { existingFields } from './existing_fields';
|
||||
import { existingFields, Field, buildFieldList } from './existing_fields';
|
||||
|
||||
describe('existingFields', () => {
|
||||
function field(name: string, parent?: string) {
|
||||
function field(opts: string | Partial<Field>): Field {
|
||||
const obj = typeof opts === 'object' ? opts : {};
|
||||
const name = (typeof opts === 'string' ? opts : opts.name) || 'test';
|
||||
|
||||
return {
|
||||
name,
|
||||
subType: parent ? { multi: { parent } } : undefined,
|
||||
aggregatable: true,
|
||||
esTypes: [],
|
||||
readFromDocValues: true,
|
||||
searchable: true,
|
||||
type: 'string',
|
||||
isScript: false,
|
||||
isAlias: false,
|
||||
path: name.split('.'),
|
||||
...obj,
|
||||
};
|
||||
}
|
||||
|
||||
function indexPattern(_source: unknown, fields: unknown = {}) {
|
||||
return { _source, fields };
|
||||
}
|
||||
|
||||
it('should handle root level fields', () => {
|
||||
const result = existingFields(
|
||||
[{ _source: { foo: 'bar' } }, { _source: { baz: 0 } }],
|
||||
[indexPattern({ foo: 'bar' }), indexPattern({ baz: 0 })],
|
||||
[field('foo'), field('bar'), field('baz')]
|
||||
);
|
||||
|
||||
|
|
@ -30,7 +35,7 @@ describe('existingFields', () => {
|
|||
|
||||
it('should handle arrays of objects', () => {
|
||||
const result = existingFields(
|
||||
[{ _source: { stuff: [{ foo: 'bar' }, { baz: 0 }] } }],
|
||||
[indexPattern({ stuff: [{ foo: 'bar' }, { baz: 0 }] })],
|
||||
[field('stuff.foo'), field('stuff.bar'), field('stuff.baz')]
|
||||
);
|
||||
|
||||
|
|
@ -38,14 +43,14 @@ describe('existingFields', () => {
|
|||
});
|
||||
|
||||
it('should handle basic arrays', () => {
|
||||
const result = existingFields([{ _source: { stuff: ['heyo', 'there'] } }], [field('stuff')]);
|
||||
const result = existingFields([indexPattern({ stuff: ['heyo', 'there'] })], [field('stuff')]);
|
||||
|
||||
expect(result).toEqual(['stuff']);
|
||||
});
|
||||
|
||||
it('should handle deep object structures', () => {
|
||||
const result = existingFields(
|
||||
[{ _source: { geo: { coordinates: { lat: 40, lon: -77 } } } }],
|
||||
[indexPattern({ geo: { coordinates: { lat: 40, lon: -77 } } })],
|
||||
[field('geo.coordinates')]
|
||||
);
|
||||
|
||||
|
|
@ -54,19 +59,97 @@ describe('existingFields', () => {
|
|||
|
||||
it('should be false if it hits a positive leaf before the end of the path', () => {
|
||||
const result = existingFields(
|
||||
[{ _source: { geo: { coordinates: 32 } } }],
|
||||
[indexPattern({ geo: { coordinates: 32 } })],
|
||||
[field('geo.coordinates.lat')]
|
||||
);
|
||||
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
|
||||
it('should prefer parent to name', () => {
|
||||
it('should use path, not name', () => {
|
||||
const result = existingFields(
|
||||
[{ _source: { stuff: [{ foo: 'bar' }, { baz: 0 }] } }],
|
||||
[field('goober', 'stuff.foo'), field('soup', 'stuff.bar'), field('pea', 'stuff.baz')]
|
||||
[indexPattern({ stuff: [{ foo: 'bar' }, { baz: 0 }] })],
|
||||
[field({ name: 'goober', path: ['stuff', 'foo'] })]
|
||||
);
|
||||
|
||||
expect(result).toEqual(['goober', 'pea']);
|
||||
expect(result).toEqual(['goober']);
|
||||
});
|
||||
|
||||
it('supports scripted fields', () => {
|
||||
const result = existingFields(
|
||||
[indexPattern({}, { bar: 'scriptvalue' })],
|
||||
[field({ name: 'baz', isScript: true, path: ['bar'] })]
|
||||
);
|
||||
|
||||
expect(result).toEqual(['baz']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFieldList', () => {
|
||||
const indexPattern = {
|
||||
id: '',
|
||||
type: 'indexpattern',
|
||||
attributes: {
|
||||
title: 'testpattern',
|
||||
fields: JSON.stringify([
|
||||
{ name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
|
||||
{ name: 'bar' },
|
||||
{ name: '@bar' },
|
||||
{ name: 'baz' },
|
||||
]),
|
||||
},
|
||||
references: [],
|
||||
};
|
||||
|
||||
const mappings = {
|
||||
testpattern: {
|
||||
mappings: {
|
||||
properties: {
|
||||
'@bar': {
|
||||
type: 'alias',
|
||||
path: 'bar',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const fieldDescriptors = [
|
||||
{
|
||||
name: 'baz',
|
||||
subType: { multi: { parent: 'a.b.c' } },
|
||||
},
|
||||
];
|
||||
|
||||
it('uses field descriptors to determine the path', () => {
|
||||
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
|
||||
expect(fields.find(f => f.name === 'baz')).toMatchObject({
|
||||
isAlias: false,
|
||||
isScript: false,
|
||||
name: 'baz',
|
||||
path: ['a', 'b', 'c'],
|
||||
});
|
||||
});
|
||||
|
||||
it('uses aliases to determine the path', () => {
|
||||
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
|
||||
expect(fields.find(f => f.isAlias)).toMatchObject({
|
||||
isAlias: true,
|
||||
isScript: false,
|
||||
name: '@bar',
|
||||
path: ['bar'],
|
||||
});
|
||||
});
|
||||
|
||||
it('supports scripted fields', () => {
|
||||
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
|
||||
expect(fields.find(f => f.isScript)).toMatchObject({
|
||||
isAlias: false,
|
||||
isScript: true,
|
||||
name: 'foo',
|
||||
path: ['foo'],
|
||||
lang: 'painless',
|
||||
script: '2+2',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,28 +6,50 @@
|
|||
|
||||
import Boom from 'boom';
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { SearchResponse } from 'elasticsearch';
|
||||
import _ from 'lodash';
|
||||
import { IScopedClusterClient } from 'src/core/server';
|
||||
import { IScopedClusterClient, SavedObject, RequestHandlerContext } from 'src/core/server';
|
||||
import { CoreSetup } from 'src/core/server';
|
||||
import { BASE_API_URL } from '../../common';
|
||||
import { FieldDescriptor, IndexPatternsFetcher } from '../../../../../../src/plugins/data/server';
|
||||
import { IndexPatternsFetcher } from '../../../../../../src/plugins/data/server';
|
||||
|
||||
/**
|
||||
* The number of docs to sample to determine field empty status.
|
||||
*/
|
||||
const SAMPLE_SIZE = 500;
|
||||
|
||||
type Document = Record<string, unknown>;
|
||||
interface MappingResult {
|
||||
[indexPatternTitle: string]: {
|
||||
mappings: {
|
||||
properties: Record<string, { type: string; path: string }>;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
interface FieldDescriptor {
|
||||
name: string;
|
||||
subType?: { multi?: { parent?: string } };
|
||||
}
|
||||
|
||||
export interface Field {
|
||||
name: string;
|
||||
isScript: boolean;
|
||||
isAlias: boolean;
|
||||
path: string[];
|
||||
lang?: string;
|
||||
script?: string;
|
||||
}
|
||||
|
||||
// TODO: Pull this from kibana advanced settings
|
||||
const metaFields = ['_source', '_id', '_type', '_index', '_score'];
|
||||
|
||||
export async function existingFieldsRoute(setup: CoreSetup) {
|
||||
const router = setup.http.createRouter();
|
||||
router.get(
|
||||
{
|
||||
path: `${BASE_API_URL}/existing_fields/{indexPatternTitle}`,
|
||||
path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
|
||||
validate: {
|
||||
params: schema.object({
|
||||
indexPatternTitle: schema.string(),
|
||||
indexPatternId: schema.string(),
|
||||
}),
|
||||
query: schema.object({
|
||||
fromDate: schema.maybe(schema.string()),
|
||||
|
|
@ -37,31 +59,13 @@ export async function existingFieldsRoute(setup: CoreSetup) {
|
|||
},
|
||||
},
|
||||
async (context, req, res) => {
|
||||
const { indexPatternTitle } = req.params;
|
||||
const requestClient = context.core.elasticsearch.dataClient;
|
||||
const indexPatternsFetcher = new IndexPatternsFetcher(requestClient.callAsCurrentUser);
|
||||
const { fromDate, toDate, timeFieldName } = req.query;
|
||||
|
||||
try {
|
||||
const fields = await indexPatternsFetcher.getFieldsForWildcard({
|
||||
pattern: indexPatternTitle,
|
||||
// TODO: Pull this from kibana advanced settings
|
||||
metaFields: ['_source', '_id', '_type', '_index', '_score'],
|
||||
});
|
||||
|
||||
const results = await fetchIndexPatternStats({
|
||||
fromDate,
|
||||
toDate,
|
||||
client: requestClient,
|
||||
index: indexPatternTitle,
|
||||
timeFieldName,
|
||||
});
|
||||
|
||||
return res.ok({
|
||||
body: {
|
||||
indexPatternTitle,
|
||||
existingFieldNames: existingFields(results.hits.hits, fields),
|
||||
},
|
||||
body: await fetchFieldExistence({
|
||||
...req.query,
|
||||
...req.params,
|
||||
context,
|
||||
}),
|
||||
});
|
||||
} catch (e) {
|
||||
if (e.status === 404) {
|
||||
|
|
@ -82,6 +86,166 @@ export async function existingFieldsRoute(setup: CoreSetup) {
|
|||
);
|
||||
}
|
||||
|
||||
async function fetchFieldExistence({
|
||||
context,
|
||||
indexPatternId,
|
||||
fromDate,
|
||||
toDate,
|
||||
timeFieldName,
|
||||
}: {
|
||||
indexPatternId: string;
|
||||
context: RequestHandlerContext;
|
||||
fromDate?: string;
|
||||
toDate?: string;
|
||||
timeFieldName?: string;
|
||||
}) {
|
||||
const {
|
||||
indexPattern,
|
||||
indexPatternTitle,
|
||||
mappings,
|
||||
fieldDescriptors,
|
||||
} = await fetchIndexPatternDefinition(indexPatternId, context);
|
||||
|
||||
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
|
||||
|
||||
const docs = await fetchIndexPatternStats({
|
||||
fromDate,
|
||||
toDate,
|
||||
client: context.core.elasticsearch.dataClient,
|
||||
index: indexPatternTitle,
|
||||
timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
|
||||
fields,
|
||||
});
|
||||
|
||||
return {
|
||||
indexPatternTitle,
|
||||
existingFieldNames: existingFields(docs, fields),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchIndexPatternDefinition(indexPatternId: string, context: RequestHandlerContext) {
|
||||
const savedObjectsClient = context.core.savedObjects.client;
|
||||
const requestClient = context.core.elasticsearch.dataClient;
|
||||
const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternId);
|
||||
const indexPatternTitle = indexPattern.attributes.title;
|
||||
// TODO: maybe don't use IndexPatternsFetcher at all, since we're only using it
|
||||
// to look up field values in the resulting documents. We can accomplish the same
|
||||
// using the mappings which we're also fetching here.
|
||||
const indexPatternsFetcher = new IndexPatternsFetcher(requestClient.callAsCurrentUser);
|
||||
const [mappings, fieldDescriptors] = await Promise.all([
|
||||
requestClient.callAsCurrentUser('indices.getMapping', {
|
||||
index: indexPatternTitle,
|
||||
}),
|
||||
|
||||
indexPatternsFetcher.getFieldsForWildcard({
|
||||
pattern: indexPatternTitle,
|
||||
// TODO: Pull this from kibana advanced settings
|
||||
metaFields,
|
||||
}),
|
||||
]);
|
||||
|
||||
return {
|
||||
indexPattern,
|
||||
indexPatternTitle,
|
||||
mappings,
|
||||
fieldDescriptors,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Exported only for unit tests.
|
||||
*/
|
||||
export function buildFieldList(
|
||||
indexPattern: SavedObject,
|
||||
mappings: MappingResult,
|
||||
fieldDescriptors: FieldDescriptor[]
|
||||
): Field[] {
|
||||
const aliasMap = Object.entries(Object.values(mappings)[0].mappings.properties)
|
||||
.map(([name, v]) => ({ ...v, name }))
|
||||
.filter(f => f.type === 'alias')
|
||||
.reduce((acc, f) => {
|
||||
acc[f.name] = f.path;
|
||||
return acc;
|
||||
}, {} as Record<string, string>);
|
||||
|
||||
const descriptorMap = fieldDescriptors.reduce((acc, f) => {
|
||||
acc[f.name] = f;
|
||||
return acc;
|
||||
}, {} as Record<string, FieldDescriptor>);
|
||||
|
||||
return JSON.parse(indexPattern.attributes.fields).map(
|
||||
(field: { name: string; lang: string; scripted?: boolean; script?: string }) => {
|
||||
const path =
|
||||
aliasMap[field.name] || descriptorMap[field.name]?.subType?.multi?.parent || field.name;
|
||||
return {
|
||||
name: field.name,
|
||||
isScript: !!field.scripted,
|
||||
isAlias: !!aliasMap[field.name],
|
||||
path: path.split('.'),
|
||||
lang: field.lang,
|
||||
script: field.script,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchIndexPatternStats({
|
||||
client,
|
||||
index,
|
||||
timeFieldName,
|
||||
fromDate,
|
||||
toDate,
|
||||
fields,
|
||||
}: {
|
||||
client: IScopedClusterClient;
|
||||
index: string;
|
||||
timeFieldName?: string;
|
||||
fromDate?: string;
|
||||
toDate?: string;
|
||||
fields: Field[];
|
||||
}) {
|
||||
if (!timeFieldName || !fromDate || !toDate) {
|
||||
return [];
|
||||
}
|
||||
const viableFields = fields.filter(
|
||||
f => !f.isScript && !f.isAlias && !metaFields.includes(f.name)
|
||||
);
|
||||
const scriptedFields = fields.filter(f => f.isScript);
|
||||
|
||||
const result = await client.callAsCurrentUser('search', {
|
||||
index,
|
||||
body: {
|
||||
size: SAMPLE_SIZE,
|
||||
_source: viableFields.map(f => f.name),
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
[timeFieldName]: {
|
||||
gte: fromDate,
|
||||
lte: toDate,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
script_fields: scriptedFields.reduce((acc, field) => {
|
||||
acc[field.name] = {
|
||||
script: {
|
||||
lang: field.lang,
|
||||
source: field.script,
|
||||
},
|
||||
};
|
||||
return acc;
|
||||
}, {} as Record<string, unknown>),
|
||||
},
|
||||
});
|
||||
|
||||
return result.hits.hits;
|
||||
}
|
||||
|
||||
function exists(obj: unknown, path: string[], i = 0): boolean {
|
||||
if (obj == null) {
|
||||
return false;
|
||||
|
|
@ -103,21 +267,13 @@ function exists(obj: unknown, path: string[], i = 0): boolean {
|
|||
}
|
||||
|
||||
/**
|
||||
* Exported for testing purposes only.
|
||||
* Exported only for unit tests.
|
||||
*/
|
||||
export function existingFields(
|
||||
docs: Array<{ _source: Document }>,
|
||||
fields: FieldDescriptor[]
|
||||
docs: Array<{ _source: unknown; fields: unknown }>,
|
||||
fields: Field[]
|
||||
): string[] {
|
||||
const allFields = fields.map(field => {
|
||||
const parent = field.subType && field.subType.multi && field.subType.multi.parent;
|
||||
return {
|
||||
name: field.name,
|
||||
parent,
|
||||
path: (parent || field.name).split('.'),
|
||||
};
|
||||
});
|
||||
const missingFields = new Set(allFields);
|
||||
const missingFields = new Set(fields);
|
||||
|
||||
for (const doc of docs) {
|
||||
if (missingFields.size === 0) {
|
||||
|
|
@ -125,53 +281,11 @@ export function existingFields(
|
|||
}
|
||||
|
||||
missingFields.forEach(field => {
|
||||
if (exists(doc._source, field.path)) {
|
||||
if (exists(field.isScript ? doc.fields : doc._source, field.path)) {
|
||||
missingFields.delete(field);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return allFields.filter(field => !missingFields.has(field)).map(f => f.name);
|
||||
}
|
||||
|
||||
async function fetchIndexPatternStats({
|
||||
client,
|
||||
fromDate,
|
||||
index,
|
||||
toDate,
|
||||
timeFieldName,
|
||||
}: {
|
||||
client: IScopedClusterClient;
|
||||
fromDate?: string;
|
||||
index: string;
|
||||
toDate?: string;
|
||||
timeFieldName?: string;
|
||||
}) {
|
||||
const body =
|
||||
!timeFieldName || !fromDate || !toDate
|
||||
? {}
|
||||
: {
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
[timeFieldName]: {
|
||||
gte: fromDate,
|
||||
lte: toDate,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return (await client.callAsCurrentUser('search', {
|
||||
index,
|
||||
body: {
|
||||
...body,
|
||||
size: SAMPLE_SIZE,
|
||||
},
|
||||
})) as SearchResponse<Document>;
|
||||
return fields.filter(field => !missingFields.has(field)).map(f => f.name);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,13 +114,13 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
const { body } = await supertest
|
||||
.get(
|
||||
`/api/lens/existing_fields/${encodeURIComponent(
|
||||
'logstash-2015.09.22'
|
||||
'logstash-*'
|
||||
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
|
||||
)
|
||||
.set(COMMON_HEADERS)
|
||||
.expect(200);
|
||||
|
||||
expect(body.indexPatternTitle).to.eql('logstash-2015.09.22');
|
||||
expect(body.indexPatternTitle).to.eql('logstash-*');
|
||||
expect(body.existingFieldNames.sort()).to.eql(fieldsWithData.sort());
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue