Merge branch 'index-pattern/allow-no-index' into epm/missing-index-patterns

src/plugins/data/common/index_patterns/index_patterns/index_pattern.test.ts

@@ -17,7 +17,7 @@ import { stubbedSavedObjectIndexPattern } from './fixtures/stubbed_saved_object_
 import { IndexPatternField } from '../fields';
 
 import { fieldFormatsMock } from '../../field_formats/mocks';
-import { FieldFormat } from '../..';
+import { FieldFormat, IndexPatternSpec } from '../..';
 import { RuntimeField } from '../types';
 
 class MockFieldFormatter {}
@@ -42,7 +42,7 @@ const runtimeField = {
 fieldFormatsMock.getInstance = jest.fn().mockImplementation(() => new MockFieldFormatter()) as any;
 
 // helper function to create index patterns
-function create(id: string) {
+function create(id: string, spec?: Partial<IndexPatternSpec>) {
   const {
     type,
     version,
@@ -58,6 +58,7 @@ function create(id: string) {
       fields: { ...fields, runtime_field: runtimeField },
       title,
       runtimeFieldMap,
+      ...(spec || {}),
     },
     fieldFormats: fieldFormatsMock,
     shortDotsEnable: false,
@@ -314,4 +315,15 @@ describe('IndexPattern', () => {
       expect(restoredPattern.fields.length).toEqual(indexPattern.fields.length);
     });
   });
+
+  describe('getAsSavedObjectBody', () => {
+    test('should match snapshot', () => {
+      expect(indexPattern.getAsSavedObjectBody()).toMatchSnapshot();
+    });
+
+    test('allowNoIndex perserves all fields', () => {
+      const allowNoIndexIndexPattern = create('test-no-index-pattern', { allowNoIndex: true });
+      expect(allowNoIndexIndexPattern.getAsSavedObjectBody()).toMatchSnapshot();
+    });
+  });
 });

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -322,7 +322,9 @@ export class IndexPattern implements IIndexPattern {
       intervalName: this.intervalName,
       sourceFilters: this.sourceFilters ? JSON.stringify(this.sourceFilters) : undefined,
       fields: this.fields
-        ? JSON.stringify(this.fields.filter((field) => field.scripted))
+        ? JSON.stringify(
+            this.allowNoIndex ? this.fields : this.fields.filter((field) => field.scripted)
+          )
         : undefined,
       fieldFormatMap,
       type: this.type,

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -283,25 +283,11 @@ export class IndexPatternsService {
     fields: IndexPatternFieldMap,
     id: string,
     title: string,
-    options: GetFieldsOptions,
     fieldAttrs: FieldAttrs = {}
   ) => {
     const fieldsAsArr = Object.values(fields);
-    const scriptedFields = fieldsAsArr.filter((field) => field.scripted);
     try {
-      let updatedFieldList: FieldSpec[];
-      const newFields = (await this.getFieldsForWildcard(options)) as FieldSpec[];
-      newFields.forEach((field) => (field.isMapped = true));
-
-      // If allowNoIndex, only update field list if field caps finds fields. To support
-      // beats creating index pattern and dashboard before docs
-      if (!options.allowNoIndex || (newFields && newFields.length > 5)) {
-        updatedFieldList = [...newFields, ...scriptedFields];
-      } else {
-        updatedFieldList = fieldsAsArr;
-      }
-
-      return this.fieldArrayToMap(updatedFieldList, fieldAttrs);
+      return this.fieldArrayToMap(fieldsAsArr, fieldAttrs);
     } catch (err) {
       if (err instanceof IndexPatternMissingIndices) {
         this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' });
@@ -396,7 +382,7 @@ export class IndexPatternsService {
     }
 
     const spec = this.savedObjectToSpec(savedObject);
-    const { title, type, typeMeta, runtimeFieldMap } = spec;
+    const { title, runtimeFieldMap } = spec;
     spec.fieldAttrs = savedObject.attributes.fieldAttrs
       ? JSON.parse(savedObject.attributes.fieldAttrs)
       : {};
@@ -406,13 +392,6 @@ export class IndexPatternsService {
         spec.fields || {},
         id,
         spec.title as string,
-        {
-          pattern: title as string,
-          metaFields: await this.config.get(UI_SETTINGS.META_FIELDS),
-          type,
-          rollupIndex: typeMeta?.params?.rollup_index,
-          allowNoIndex: spec.allowNoIndex,
-        },
         spec.fieldAttrs
       );
       // CREATE RUNTIME FIELDS

src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts

@@ -9,6 +9,8 @@
 import { ElasticsearchClient } from 'kibana/server';
 import { keyBy } from 'lodash';
 
+import { IndexPatternFieldMap } from '../../../common/index_patterns/types';
+import { IndexPatternsCommonService } from '../..';
 import {
   getFieldCapabilities,
   resolveTimePattern,
@@ -35,10 +37,16 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
   private elasticsearchClient: ElasticsearchClient;
   private allowNoIndices: boolean;
+  private indexPatternService: IndexPatternsCommonService | undefined;
 
-  constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
+  constructor(
+    elasticsearchClient: ElasticsearchClient,
+    allowNoIndices: boolean = false,
+    indexPatternService?: IndexPatternsCommonService
+  ) {
     this.elasticsearchClient = elasticsearchClient;
     this.allowNoIndices = allowNoIndices;
+    this.indexPatternService = indexPatternService;
   }
 
   /**
@@ -58,24 +66,25 @@ export class IndexPatternsFetcher {
     rollupIndex?: string;
   }): Promise<FieldDescriptor[]> {
     const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
+    const allowNoIndices = fieldCapsOptions
+      ? fieldCapsOptions.allow_no_indices
+      : this.allowNoIndices;
     const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
     let patternListActive: string[] = patternList;
     // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
     if (patternList.length > 1) {
       patternListActive = await this.validatePatternListActive(patternList);
     }
+
     const fieldCapsResponse = await getFieldCapabilities(
       this.elasticsearchClient,
       // if none of the patterns are active, pass the original list to get an error
       patternListActive.length > 0 ? patternListActive : patternList,
       metaFields,
-      {
-        allow_no_indices: fieldCapsOptions
-          ? fieldCapsOptions.allow_no_indices
-          : this.allowNoIndices,
-      }
+      { allow_no_indices: allowNoIndices }
     );
 
+    // Get rollup information for rollup indices
     if (type === 'rollup' && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
       const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -99,6 +108,40 @@ export class IndexPatternsFetcher {
         rollupFields
       );
     }
+    // If we allow no indices, fetch fields defined on the saved index pattern(s) and merge with file caps
+    // For now, this is mutually exclusive from the rollup handling for simplicity
+    else if (allowNoIndices && this.indexPatternService) {
+      const savedIndexPatterns = await this.indexPatternService.find(patternList.join(','));
+      const savedIndexPatternsFields = savedIndexPatterns.reduce((acc, indexPattern) => {
+        return {
+          ...acc,
+          ...indexPattern.fields.toSpec(),
+        };
+      }, {} as IndexPatternFieldMap);
+      return [
+        ...Object.values(savedIndexPatternsFields).map(
+          ({
+            name,
+            type: fieldType,
+            esTypes = [],
+            searchable,
+            aggregatable,
+            readFromDocValues = false,
+          }) => {
+            return {
+              name,
+              type: fieldType,
+              esTypes,
+              searchable,
+              aggregatable,
+              readFromDocValues,
+            };
+          }
+        ),
+        ...fieldCapsResponse,
+      ];
+    }
+
     return fieldCapsResponse;
   }
 

src/plugins/data/server/index_patterns/routes.ts

@@ -69,8 +69,13 @@ export function registerRoutes(
       },
     },
     async (context, request, response) => {
-      const { asCurrentUser } = context.core.elasticsearch.client;
-      const indexPatterns = new IndexPatternsFetcher(asCurrentUser);
+      const savedObjectsClient = context.core.savedObjects.client;
+      const elasticsearchClient = context.core.elasticsearch.client.asCurrentUser;
+      const [, , pluginStart] = await getStartServices();
+      const indexPatternsService = await pluginStart.indexPatterns.indexPatternsServiceFactory(
+        savedObjectsClient,
+        elasticsearchClient
+      );
       const {
         pattern,
         meta_fields: metaFields,
@@ -78,6 +83,11 @@ export function registerRoutes(
         rollup_index: rollupIndex,
         allow_no_index: allowNoIndex,
       } = request.query;
+      const indexPatterns = new IndexPatternsFetcher(
+        elasticsearchClient,
+        allowNoIndex,
+        indexPatternsService
+      );
 
       let parsedFields: string[] = [];
       try {