From c129f93083197876a8dd2000a715e23205a4efb5 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Sat, 19 Dec 2020 21:56:06 -0600 Subject: [PATCH] [index patterns] Fleep app - Keep saved object field list until field caps provides fields (#85370) --- ...s-data-public.indexpattern.allownoindex.md | 13 +++++ ...ublic.indexpattern.getassavedobjectbody.md | 2 + ...plugin-plugins-data-public.indexpattern.md | 1 + ...lic.indexpatternattributes.allownoindex.md | 13 +++++ ...gins-data-public.indexpatternattributes.md | 1 + ...ta-public.indexpatternspec.allownoindex.md | 11 ++++ ...in-plugins-data-public.indexpatternspec.md | 1 + ...s-data-server.indexpattern.allownoindex.md | 13 +++++ ...erver.indexpattern.getassavedobjectbody.md | 2 + ...plugin-plugins-data-server.indexpattern.md | 1 + ...ver.indexpatternattributes.allownoindex.md | 13 +++++ ...gins-data-server.indexpatternattributes.md | 1 + .../__snapshots__/index_pattern.test.ts.snap | 1 + .../__snapshots__/index_patterns.test.ts.snap | 1 + .../index_patterns/index_pattern.ts | 7 +++ .../index_patterns/index_patterns.test.ts | 15 ++++++ .../index_patterns/index_patterns.ts | 19 ++++++- .../data/common/index_patterns/types.ts | 6 +++ .../index_patterns_api_client.ts | 5 +- src/plugins/data/public/public.api.md | 7 ++- .../index_patterns_api_client.ts | 10 +++- .../data/server/index_patterns/routes.ts | 12 ++++- .../routes/create_index_pattern.ts | 1 + .../routes/update_index_pattern.ts | 1 + .../index_pattern_migrations.test.ts | 51 +++++++++++++++++++ .../saved_objects/index_pattern_migrations.ts | 9 ++++ src/plugins/data/server/server.api.md | 5 +- x-pack/plugins/fleet/common/constants/epm.ts | 1 - .../plugins/fleet/server/constants/index.ts | 1 - .../__snapshots__/install.test.ts.snap | 3 +- .../epm/kibana/index_pattern/install.ts | 37 ++------------ x-pack/plugins/fleet/server/services/setup.ts | 2 - .../test/fleet_api_integration/apis/index.js | 2 - .../test/fleet_api_integration/apis/setup.ts | 32 ------------ .../apps/endpoint/resolver.ts | 28 +++++++--- .../apis/resolver/events.ts | 2 +- .../apis/resolver/tree.ts | 2 +- 37 files changed, 241 insertions(+), 91 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md delete mode 100644 x-pack/test/fleet_api_integration/apis/setup.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md new file mode 100644 index 000000000000..5e397d11b0a8 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) > [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md) + +## IndexPattern.allowNoIndex property + +prevents errors when index pattern exists before indices + +Signature: + +```typescript +readonly allowNoIndex: boolean; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md index a37034100096..b318427012c0 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md @@ -19,6 +19,7 @@ getAsSavedObjectBody(): { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }; ``` Returns: @@ -33,5 +34,6 @@ getAsSavedObjectBody(): { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md index 179148265e68..b640ef1b8960 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md @@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern | Property | Modifiers | Type | Description | | --- | --- | --- | --- | +| [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md) | | boolean | prevents errors when index pattern exists before indices | | [deleteFieldFormat](./kibana-plugin-plugins-data-public.indexpattern.deletefieldformat.md) | | (fieldName: string) => void | | | [fieldAttrs](./kibana-plugin-plugins-data-public.indexpattern.fieldattrs.md) | | FieldAttrs | | | [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpattern.fieldformatmap.md) | | Record<string, any> | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md new file mode 100644 index 000000000000..9438f3819449 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [IndexPatternAttributes](./kibana-plugin-plugins-data-public.indexpatternattributes.md) > [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md) + +## IndexPatternAttributes.allowNoIndex property + +prevents errors when index pattern exists before indices + +Signature: + +```typescript +allowNoIndex?: boolean; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md index c5ea38278e82..1bbede565894 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md @@ -14,6 +14,7 @@ export interface IndexPatternAttributes | Property | Type | Description | | --- | --- | --- | +| [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md) | boolean | prevents errors when index pattern exists before indices | | [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldattrs.md) | string | | | [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldformatmap.md) | string | | | [fields](./kibana-plugin-plugins-data-public.indexpatternattributes.fields.md) | string | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md new file mode 100644 index 000000000000..50adef826869 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [IndexPatternSpec](./kibana-plugin-plugins-data-public.indexpatternspec.md) > [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md) + +## IndexPatternSpec.allowNoIndex property + +Signature: + +```typescript +allowNoIndex?: boolean; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md index 06917fcac1b4..9357ad7d5077 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md @@ -14,6 +14,7 @@ export interface IndexPatternSpec | Property | Type | Description | | --- | --- | --- | +| [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md) | boolean | | | [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternspec.fieldattrs.md) | FieldAttrs | | | [fieldFormats](./kibana-plugin-plugins-data-public.indexpatternspec.fieldformats.md) | Record<string, SerializedFieldFormat> | | | [fields](./kibana-plugin-plugins-data-public.indexpatternspec.fields.md) | IndexPatternFieldMap | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md new file mode 100644 index 000000000000..fe7bec70196c --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IndexPattern](./kibana-plugin-plugins-data-server.indexpattern.md) > [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md) + +## IndexPattern.allowNoIndex property + +prevents errors when index pattern exists before indices + +Signature: + +```typescript +readonly allowNoIndex: boolean; +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md index 274a475872b0..7d70af4b535f 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md @@ -19,6 +19,7 @@ getAsSavedObjectBody(): { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }; ``` Returns: @@ -33,5 +34,6 @@ getAsSavedObjectBody(): { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md index b2cb217fecaa..54f020e57cf4 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md @@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern | Property | Modifiers | Type | Description | | --- | --- | --- | --- | +| [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md) | | boolean | prevents errors when index pattern exists before indices | | [deleteFieldFormat](./kibana-plugin-plugins-data-server.indexpattern.deletefieldformat.md) | | (fieldName: string) => void | | | [fieldAttrs](./kibana-plugin-plugins-data-server.indexpattern.fieldattrs.md) | | FieldAttrs | | | [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpattern.fieldformatmap.md) | | Record<string, any> | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md new file mode 100644 index 000000000000..1255a6fe9f0c --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [IndexPatternAttributes](./kibana-plugin-plugins-data-server.indexpatternattributes.md) > [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md) + +## IndexPatternAttributes.allowNoIndex property + +prevents errors when index pattern exists before indices + +Signature: + +```typescript +allowNoIndex?: boolean; +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md index 6559b4d7110b..b9b9f955c7ab 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md @@ -14,6 +14,7 @@ export interface IndexPatternAttributes | Property | Type | Description | | --- | --- | --- | +| [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md) | boolean | prevents errors when index pattern exists before indices | | [fieldAttrs](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldattrs.md) | string | | | [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldformatmap.md) | string | | | [fields](./kibana-plugin-plugins-data-server.indexpatternattributes.fields.md) | string | | diff --git a/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_pattern.test.ts.snap b/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_pattern.test.ts.snap index 19ec286307a0..76de2b2662bb 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_pattern.test.ts.snap +++ b/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_pattern.test.ts.snap @@ -2,6 +2,7 @@ exports[`IndexPattern toSpec should match snapshot 1`] = ` Object { + "allowNoIndex": false, "fieldAttrs": Object {}, "fieldFormats": Object {}, "fields": Object { diff --git a/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap b/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap index c020e7595c56..bad74430b896 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap +++ b/src/plugins/data/common/index_patterns/index_patterns/__snapshots__/index_patterns.test.ts.snap @@ -2,6 +2,7 @@ exports[`IndexPatterns savedObjectToSpec 1`] = ` Object { + "allowNoIndex": undefined, "fieldAttrs": Object {}, "fieldFormats": Object { "field": Object {}, diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts index 4c89cbeb446a..590ff872f3bf 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts @@ -74,6 +74,10 @@ export class IndexPattern implements IIndexPattern { private fieldFormats: FieldFormatsStartCommon; // make private once manual field refresh is removed public fieldAttrs: FieldAttrs; + /** + * prevents errors when index pattern exists before indices + */ + public readonly allowNoIndex: boolean = false; constructor({ spec = {}, @@ -110,6 +114,7 @@ export class IndexPattern implements IIndexPattern { this.typeMeta = spec.typeMeta; this.fieldAttrs = spec.fieldAttrs || {}; this.intervalName = spec.intervalName; + this.allowNoIndex = spec.allowNoIndex || false; } /** @@ -204,6 +209,7 @@ export class IndexPattern implements IIndexPattern { fieldFormats: this.fieldFormatMap, fieldAttrs: this.fieldAttrs, intervalName: this.intervalName, + allowNoIndex: this.allowNoIndex, }; } @@ -309,6 +315,7 @@ export class IndexPattern implements IIndexPattern { fieldFormatMap, type: this.type, typeMeta: this.typeMeta ? JSON.stringify(this.typeMeta) : undefined, + allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined, }; } diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts index 2a203b57d201..3d32742c168a 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts @@ -114,6 +114,21 @@ describe('IndexPatterns', () => { SOClientGetDelay = 0; }); + test('allowNoIndex flag preserves existing fields when index is missing', async () => { + const id = '2'; + setDocsourcePayload(id, { + id: 'foo', + version: 'foo', + attributes: { + title: 'something', + allowNoIndex: true, + fields: '[{"name":"field"}]', + }, + }); + + expect((await indexPatterns.get(id)).fields.length).toBe(1); + }); + test('savedObjectCache pre-fetches only title', async () => { expect(await indexPatterns.getIds()).toEqual(['id']); expect(savedObjectsClient.find).toHaveBeenCalledWith({ diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index 0235f748ec1e..3333dba36fe6 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -222,6 +222,7 @@ export class IndexPatternsService { metaFields, type: options.type, rollupIndex: options.rollupIndex, + allowNoIndex: options.allowNoIndex, }); }; @@ -281,10 +282,21 @@ export class IndexPatternsService { options: GetFieldsOptions, fieldAttrs: FieldAttrs = {} ) => { - const scriptedFields = Object.values(fields).filter((field) => field.scripted); + const fieldsAsArr = Object.values(fields); + const scriptedFields = fieldsAsArr.filter((field) => field.scripted); try { + let updatedFieldList: FieldSpec[]; const newFields = (await this.getFieldsForWildcard(options)) as FieldSpec[]; - return this.fieldArrayToMap([...newFields, ...scriptedFields], fieldAttrs); + + // If allowNoIndex, only update field list if field caps finds fields. To support + // beats creating index pattern and dashboard before docs + if (!options.allowNoIndex || (newFields && newFields.length > 5)) { + updatedFieldList = [...newFields, ...scriptedFields]; + } else { + updatedFieldList = fieldsAsArr; + } + + return this.fieldArrayToMap(updatedFieldList, fieldAttrs); } catch (err) { if (err instanceof IndexPatternMissingIndices) { this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' }); @@ -334,6 +346,7 @@ export class IndexPatternsService { typeMeta, type, fieldAttrs, + allowNoIndex, }, } = savedObject; @@ -355,6 +368,7 @@ export class IndexPatternsService { type, fieldFormats: parsedFieldFormatMap, fieldAttrs: parsedFieldAttrs, + allowNoIndex, }; }; @@ -384,6 +398,7 @@ export class IndexPatternsService { metaFields: await this.config.get(UI_SETTINGS.META_FIELDS), type, rollupIndex: typeMeta?.params?.rollup_index, + allowNoIndex: spec.allowNoIndex, }, spec.fieldAttrs ); diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts index 8d9b29175162..12496b07d348 100644 --- a/src/plugins/data/common/index_patterns/types.ts +++ b/src/plugins/data/common/index_patterns/types.ts @@ -49,6 +49,10 @@ export interface IndexPatternAttributes { sourceFilters?: string; fieldFormatMap?: string; fieldAttrs?: string; + /** + * prevents errors when index pattern exists before indices + */ + allowNoIndex?: boolean; } export interface FieldAttrs { @@ -101,6 +105,7 @@ export interface GetFieldsOptions { lookBack?: boolean; metaFields?: string[]; rollupIndex?: string; + allowNoIndex?: boolean; } export interface GetFieldsOptionsTimePattern { @@ -193,6 +198,7 @@ export interface IndexPatternSpec { type?: string; fieldFormats?: Record; fieldAttrs?: FieldAttrs; + allowNoIndex?: boolean; } export interface SourceFilter { diff --git a/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts b/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts index ca0f35d6612b..36a193a4f6f9 100644 --- a/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts +++ b/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts @@ -64,12 +64,13 @@ export class IndexPatternsApiClient implements IIndexPatternsApiClient { }).then((resp: any) => resp.fields); } - getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) { + getFieldsForWildcard({ pattern, metaFields, type, rollupIndex, allowNoIndex }: GetFieldsOptions) { return this._request(this._getUrl(['_fields_for_wildcard']), { pattern, meta_fields: metaFields, type, rollup_index: rollupIndex, - }).then((resp: any) => resp.fields); + allow_no_index: allowNoIndex, + }).then((resp: any) => resp.fields || []); } } diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 120540ddb92e..3493844a71ac 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1256,6 +1256,7 @@ export class IndexPattern implements IIndexPattern { // Warning: (ae-forgotten-export) The symbol "IndexPatternDeps" needs to be exported by the entry point index.d.ts constructor({ spec, fieldFormats, shortDotsEnable, metaFields, }: IndexPatternDeps); addScriptedField(name: string, script: string, fieldType?: string): Promise; + readonly allowNoIndex: boolean; // (undocumented) readonly deleteFieldFormat: (fieldName: string) => void; // Warning: (ae-forgotten-export) The symbol "FieldAttrs" needs to be exported by the entry point index.d.ts @@ -1296,6 +1297,7 @@ export class IndexPattern implements IIndexPattern { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }; // (undocumented) getComputedFields(): { @@ -1388,6 +1390,7 @@ export type IndexPatternAggRestrictions = Record, 'isLo // // @public (undocumented) export interface IndexPatternSpec { + // (undocumented) + allowNoIndex?: boolean; // (undocumented) fieldAttrs?: FieldAttrs; // (undocumented) @@ -2564,7 +2569,7 @@ export const UI_SETTINGS: { // src/plugins/data/common/es_query/filters/phrase_filter.ts:33:3 - (ae-forgotten-export) The symbol "PhraseFilterMeta" needs to be exported by the entry point index.d.ts // src/plugins/data/common/es_query/filters/phrases_filter.ts:31:3 - (ae-forgotten-export) The symbol "PhrasesFilterMeta" needs to be exported by the entry point index.d.ts // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts -// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:128:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts +// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:133:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts // src/plugins/data/common/search/aggs/types.ts:150:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts // src/plugins/data/common/search/search_source/search_source.ts:197:7 - (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts // src/plugins/data/public/field_formats/field_formats_service.ts:67:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/server/index_patterns/index_patterns_api_client.ts b/src/plugins/data/server/index_patterns/index_patterns_api_client.ts index 21a3bf6e73e6..9023044184df 100644 --- a/src/plugins/data/server/index_patterns/index_patterns_api_client.ts +++ b/src/plugins/data/server/index_patterns/index_patterns_api_client.ts @@ -30,8 +30,14 @@ export class IndexPatternsApiServer implements IIndexPatternsApiClient { constructor(elasticsearchClient: ElasticsearchClient) { this.esClient = elasticsearchClient; } - async getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) { - const indexPatterns = new IndexPatternsFetcher(this.esClient); + async getFieldsForWildcard({ + pattern, + metaFields, + type, + rollupIndex, + allowNoIndex, + }: GetFieldsOptions) { + const indexPatterns = new IndexPatternsFetcher(this.esClient, allowNoIndex); return await indexPatterns.getFieldsForWildcard({ pattern, metaFields, diff --git a/src/plugins/data/server/index_patterns/routes.ts b/src/plugins/data/server/index_patterns/routes.ts index e9dbc2e972c6..f0b51e456337 100644 --- a/src/plugins/data/server/index_patterns/routes.ts +++ b/src/plugins/data/server/index_patterns/routes.ts @@ -75,13 +75,20 @@ export function registerRoutes( }), type: schema.maybe(schema.string()), rollup_index: schema.maybe(schema.string()), + allow_no_index: schema.maybe(schema.boolean()), }), }, }, async (context, request, response) => { const { asCurrentUser } = context.core.elasticsearch.client; const indexPatterns = new IndexPatternsFetcher(asCurrentUser); - const { pattern, meta_fields: metaFields, type, rollup_index: rollupIndex } = request.query; + const { + pattern, + meta_fields: metaFields, + type, + rollup_index: rollupIndex, + allow_no_index: allowNoIndex, + } = request.query; let parsedFields: string[] = []; try { @@ -96,6 +103,9 @@ export function registerRoutes( metaFields: parsedFields, type, rollupIndex, + fieldCapsOptions: { + allow_no_indices: allowNoIndex || false, + }, }); return response.ok({ diff --git a/src/plugins/data/server/index_patterns/routes/create_index_pattern.ts b/src/plugins/data/server/index_patterns/routes/create_index_pattern.ts index 57a745b19748..1163fd2dc995 100644 --- a/src/plugins/data/server/index_patterns/routes/create_index_pattern.ts +++ b/src/plugins/data/server/index_patterns/routes/create_index_pattern.ts @@ -50,6 +50,7 @@ const indexPatternSpecSchema = schema.object({ }) ) ), + allowNoIndex: schema.maybe(schema.boolean()), }); export const registerCreateIndexPatternRoute = ( diff --git a/src/plugins/data/server/index_patterns/routes/update_index_pattern.ts b/src/plugins/data/server/index_patterns/routes/update_index_pattern.ts index 10567544af6e..8bd59e47730f 100644 --- a/src/plugins/data/server/index_patterns/routes/update_index_pattern.ts +++ b/src/plugins/data/server/index_patterns/routes/update_index_pattern.ts @@ -38,6 +38,7 @@ const indexPatternUpdateSchema = schema.object({ ), fieldFormats: schema.maybe(schema.recordOf(schema.string(), serializedFieldFormatSchema)), fields: schema.maybe(schema.recordOf(schema.string(), fieldSpecSchema)), + allowNoIndex: schema.maybe(schema.boolean()), }); export const registerUpdateIndexPatternRoute = ( diff --git a/src/plugins/data/server/saved_objects/index_pattern_migrations.test.ts b/src/plugins/data/server/saved_objects/index_pattern_migrations.test.ts index b1410e249866..3b223e6fdb9b 100644 --- a/src/plugins/data/server/saved_objects/index_pattern_migrations.test.ts +++ b/src/plugins/data/server/saved_objects/index_pattern_migrations.test.ts @@ -94,4 +94,55 @@ Object { expect(migrationFn(input, savedObjectMigrationContext)).toEqual(expected); }); }); + + describe('7.11.0', () => { + const migrationFn = indexPatternSavedObjectTypeMigrations['7.11.0']; + + test('should set allowNoIndex', () => { + const input = { + type: 'index-pattern', + id: 'logs-*', + attributes: {}, + }; + const expected = { + type: 'index-pattern', + id: 'logs-*', + attributes: { + allowNoIndex: true, + }, + }; + + expect(migrationFn(input, savedObjectMigrationContext)).toEqual(expected); + + const input2 = { + type: 'index-pattern', + id: 'metrics-*', + attributes: {}, + }; + const expected2 = { + type: 'index-pattern', + id: 'metrics-*', + attributes: { + allowNoIndex: true, + }, + }; + + expect(migrationFn(input2, savedObjectMigrationContext)).toEqual(expected2); + + const input3 = { + type: 'index-pattern', + id: 'xxx', + attributes: {}, + }; + const expected3 = { + type: 'index-pattern', + id: 'xxx', + attributes: { + allowNoIndex: undefined, + }, + }; + + expect(migrationFn(input3, savedObjectMigrationContext)).toEqual(expected3); + }); + }); }); diff --git a/src/plugins/data/server/saved_objects/index_pattern_migrations.ts b/src/plugins/data/server/saved_objects/index_pattern_migrations.ts index 768041a376ad..4650aeefba05 100644 --- a/src/plugins/data/server/saved_objects/index_pattern_migrations.ts +++ b/src/plugins/data/server/saved_objects/index_pattern_migrations.ts @@ -54,7 +54,16 @@ const migrateSubTypeAndParentFieldProperties: SavedObjectMigrationFn = }; }; +const addAllowNoIndex: SavedObjectMigrationFn = (doc) => ({ + ...doc, + attributes: { + ...doc.attributes, + allowNoIndex: doc.id === 'logs-*' || doc.id === 'metrics-*' || undefined, + }, +}); + export const indexPatternSavedObjectTypeMigrations = { '6.5.0': flow(migrateAttributeTypeAndAttributeTypeMeta), '7.6.0': flow(migrateSubTypeAndParentFieldProperties), + '7.11.0': flow(addAllowNoIndex), }; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 60d5e167921c..cd3527d5ad7a 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -689,6 +689,7 @@ export class IndexPattern implements IIndexPattern { // Warning: (ae-forgotten-export) The symbol "IndexPatternDeps" needs to be exported by the entry point index.d.ts constructor({ spec, fieldFormats, shortDotsEnable, metaFields, }: IndexPatternDeps); addScriptedField(name: string, script: string, fieldType?: string): Promise; + readonly allowNoIndex: boolean; // (undocumented) readonly deleteFieldFormat: (fieldName: string) => void; // Warning: (ae-forgotten-export) The symbol "FieldAttrs" needs to be exported by the entry point index.d.ts @@ -731,6 +732,7 @@ export class IndexPattern implements IIndexPattern { fieldFormatMap: string | undefined; type: string | undefined; typeMeta: string | undefined; + allowNoIndex: true | undefined; }; // (undocumented) getComputedFields(): { @@ -819,6 +821,7 @@ export class IndexPattern implements IIndexPattern { // // @public (undocumented) export interface IndexPatternAttributes { + allowNoIndex?: boolean; // (undocumented) fieldAttrs?: string; // (undocumented) @@ -1388,7 +1391,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/common/es_query/filters/meta_filter.ts:54:3 - (ae-forgotten-export) The symbol "FilterMeta" needs to be exported by the entry point index.d.ts // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:58:45 - (ae-forgotten-export) The symbol "IndexPatternFieldMap" needs to be exported by the entry point index.d.ts // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts -// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:128:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts +// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:133:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildCustomFilter" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildFilter" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:57:23 - (ae-forgotten-export) The symbol "datatableToCSV" needs to be exported by the entry point index.d.ts diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 287b7ccdb88e..b94c2cd12cd5 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -6,7 +6,6 @@ export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; -export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder'; export const MAX_TIME_COMPLETE_INSTALL = 60000; export const requiredPackages = { diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index b1d7318ff510..5d00b9663421 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -14,7 +14,6 @@ export { AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL, AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS, AGENT_UPDATE_ACTIONS_INTERVAL_MS, - INDEX_PATTERN_PLACEHOLDER_SUFFIX, MAX_TIME_COMPLETE_INSTALL, // Routes LIMITED_CONCURRENCY_ROUTE_TAG, diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap index d8495840453f..862e82589b9b 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap @@ -41,7 +41,8 @@ exports[`creating index patterns from yaml fields createIndexPattern function cr "title": "logs-*", "timeFieldName": "@timestamp", "fields": "[{\\"name\\":\\"coredns.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.allParams\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.length\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.class\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.flags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.dnssec_ok\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"boolean\\"},{\\"name\\":\\"@timestamp\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"date\\"},{\\"name\\":\\"labels\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"message\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"tags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.ephemeral_id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"as.number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"as.organization.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.remote_ip_list\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.body_sent.bytes\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.method\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.url\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.http_version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.response_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.referrer\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.agent\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.device\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.original\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.geoip.country_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.location\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.city_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"source.geo.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.keyword\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.text\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"}]", - "fieldFormatMap": "{\\"coredns.allParams\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQueryWeight\\",\\"inputFormat\\":\\"inputFormatVal,\\",\\"outputFormat\\":\\"outputFormalVal,\\",\\"outputPrecision\\":\\"3,\\",\\"labelTemplate\\":\\"labelTemplateVal,\\",\\"urlTemplate\\":\\"urlTemplateVal,\\"}},\\"coredns.query.length\\":{\\"params\\":{\\"pattern\\":\\"patternValQueryLength\\"}},\\"coredns.query.size\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQuerySize\\"}},\\"coredns.response.size\\":{\\"id\\":\\"bytes\\"}}" + "fieldFormatMap": "{\\"coredns.allParams\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQueryWeight\\",\\"inputFormat\\":\\"inputFormatVal,\\",\\"outputFormat\\":\\"outputFormalVal,\\",\\"outputPrecision\\":\\"3,\\",\\"labelTemplate\\":\\"labelTemplateVal,\\",\\"urlTemplate\\":\\"urlTemplateVal,\\"}},\\"coredns.query.length\\":{\\"params\\":{\\"pattern\\":\\"patternValQueryLength\\"}},\\"coredns.query.size\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQuerySize\\"}},\\"coredns.response.size\\":{\\"id\\":\\"bytes\\"}}", + "allowNoIndex": true } `; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts index bdb6744745c9..d5077308a530 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts @@ -5,15 +5,11 @@ */ import { SavedObjectsClientContract } from 'src/core/server'; -import { - INDEX_PATTERN_SAVED_OBJECT_TYPE, - INDEX_PATTERN_PLACEHOLDER_SUFFIX, -} from '../../../../constants'; +import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../constants'; import { loadFieldsFromYaml, Fields, Field } from '../../fields/field'; import { dataTypes, installationStatuses } from '../../../../../common/constants'; import { ArchivePackage, InstallSource, ValueOf } from '../../../../../common/types'; -import { RegistryPackage, CallESAsCurrentUser, DataType } from '../../../../types'; -import { appContextService } from '../../../../services'; +import { RegistryPackage, DataType } from '../../../../types'; import { getPackageFromSource, getPackageSavedObjects } from '../../packages/get'; interface FieldFormatMap { @@ -172,6 +168,7 @@ export const createIndexPattern = (indexPatternType: string, fields: Fields) => timeFieldName: '@timestamp', fields: JSON.stringify(indexPatternFields), fieldFormatMap: JSON.stringify(fieldFormatMap), + allowNoIndex: true, }; }; @@ -382,31 +379,3 @@ const getFieldFormatParams = (field: Field): FieldFormatParams => { if (field.open_link_in_current_tab) params.openLinkInCurrentTab = field.open_link_in_current_tab; return params; }; - -export const ensureDefaultIndices = async (callCluster: CallESAsCurrentUser) => { - // create placeholder indices to supress errors in the kibana Dashboards app - // that no matching indices exist https://github.com/elastic/kibana/issues/62343 - const logger = appContextService.getLogger(); - return Promise.all( - Object.values(dataTypes).map(async (indexPattern) => { - const defaultIndexPatternName = indexPattern + INDEX_PATTERN_PLACEHOLDER_SUFFIX; - const indexExists = await callCluster('indices.exists', { index: defaultIndexPatternName }); - if (!indexExists) { - try { - await callCluster('indices.create', { - index: defaultIndexPatternName, - body: { - mappings: { - properties: { - '@timestamp': { type: 'date' }, - }, - }, - }, - }); - } catch (putErr) { - logger.error(`${defaultIndexPatternName} could not be created`); - } - } - }) - ); -}; diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index a3df7bc3dcdc..f514f1ecb9ae 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -13,7 +13,6 @@ import { ensureInstalledDefaultPackages, ensurePackagesCompletedInstall, } from './epm/packages/install'; -import { ensureDefaultIndices } from './epm/kibana/index_pattern/install'; import { packageToPackagePolicy, PackagePolicy, @@ -58,7 +57,6 @@ async function createSetupSideEffects( ensureInstalledDefaultPackages(soClient, callCluster), outputService.ensureDefaultOutput(soClient), agentPolicyService.ensureDefaultAgentPolicy(soClient), - ensureDefaultIndices(callCluster), settingsService.getSettings(soClient).catch((e: any) => { if (e.isBoom && e.output.statusCode === 404) { const defaultSettings = createDefaultSettings(); diff --git a/x-pack/test/fleet_api_integration/apis/index.js b/x-pack/test/fleet_api_integration/apis/index.js index 5b230e5a179a..0d634f60e282 100644 --- a/x-pack/test/fleet_api_integration/apis/index.js +++ b/x-pack/test/fleet_api_integration/apis/index.js @@ -7,8 +7,6 @@ export default function ({ loadTestFile }) { describe('Fleet Endpoints', function () { this.tags('ciGroup10'); - // Fleet setup - loadTestFile(require.resolve('./setup')); // Agent setup loadTestFile(require.resolve('./agents_setup')); // Agents diff --git a/x-pack/test/fleet_api_integration/apis/setup.ts b/x-pack/test/fleet_api_integration/apis/setup.ts deleted file mode 100644 index 4d1562e70377..000000000000 --- a/x-pack/test/fleet_api_integration/apis/setup.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; - -export default function (providerContext: FtrProviderContext) { - const { getService } = providerContext; - const supertest = getService('supertest'); - const es = getService('es'); - describe('Fleet setup', async () => { - before(async () => { - await supertest.post(`/api/fleet/setup`).set('kbn-xsrf', 'xxx').send(); - }); - - it('should have installed placeholder indices', async function () { - const resLogsIndexPatternPlaceholder = await es.transport.request({ - method: 'GET', - path: `/logs-index_pattern_placeholder`, - }); - expect(resLogsIndexPatternPlaceholder.statusCode).equal(200); - const resMetricsIndexPatternPlaceholder = await es.transport.request({ - method: 'GET', - path: `/metrics-index_pattern_placeholder`, - }); - expect(resMetricsIndexPatternPlaceholder.statusCode).equal(200); - }); - }); -} diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/resolver.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/resolver.ts index 7a30d7acf5f8..1ccab64a5856 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/resolver.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/resolver.ts @@ -13,19 +13,32 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const browser = getService('browser'); - describe('Endpoint Event Resolver', function () { + /** + * Navigating to the hosts page must be done after data is loaded into ES otherwise + * the hosts page will display the empty default page and if we load data after that + * we'd have to set the source filter on the page. + */ + const navigateToHostsAndSetDate = async () => { + await pageObjects.hosts.navigateToSecurityHostsPage(); + await pageObjects.common.dismissBanner(); + const fromTime = 'Jan 1, 2018 @ 00:00:00.000'; + const toTime = 'now'; + await pageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + }; + + describe.skip('Endpoint Event Resolver', function () { before(async () => { - await pageObjects.hosts.navigateToSecurityHostsPage(); - await pageObjects.common.dismissBanner(); - const fromTime = 'Jan 1, 2018 @ 00:00:00.000'; - const toTime = 'now'; - await pageObjects.timePicker.setAbsoluteRange(fromTime, toTime); await browser.setWindowSize(1800, 1200); }); - describe.skip('Endpoint Resolver Tree', function () { + after(async () => { + await pageObjects.hosts.deleteDataStreams(); + }); + + describe('Endpoint Resolver Tree', function () { before(async () => { await esArchiver.load('empty_kibana'); await esArchiver.load('endpoint/resolver_tree/functions', { useCreate: true }); + await navigateToHostsAndSetDate(); await pageObjects.hosts.executeQueryAndOpenResolver('event.dataset : endpoint.events.file'); }); after(async () => { @@ -213,6 +226,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { before(async () => { await esArchiver.load('empty_kibana'); await esArchiver.load('endpoint/resolver_tree/alert_events', { useCreate: true }); + await navigateToHostsAndSetDate(); }); after(async () => { await pageObjects.hosts.deleteDataStreams(); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts index 220d932787ff..3f27d1868461 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts @@ -277,7 +277,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'xxx') .send({ filter: entityIDFilter, - indexPatterns: ['metrics-*'], + indexPatterns: ['doesnotexist-*'], timeRange: { from: tree.startTime, to: tree.endTime, diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts index 9a731f1d5aee..ab6cac7f357a 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts @@ -281,7 +281,7 @@ export default function ({ getService }: FtrProviderContext) { from: tree.startTime.toISOString(), to: tree.endTime.toISOString(), }, - indexPatterns: ['metrics-*'], + indexPatterns: ['doesnotexist-*'], }) .expect(200); expect(body).to.be.empty();