[Lens] Use index pattern through service instead of reading saved object (#84432) (#84737)

x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx

@@ -563,7 +563,8 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
                 '{availableFields} available {availableFields, plural, one {field} other {fields}}. {emptyFields} empty {emptyFields, plural, one {field} other {fields}}. {metaFields} meta {metaFields, plural, one {field} other {fields}}.',
               values: {
                 availableFields: fieldGroups.AvailableFields.fields.length,
-                emptyFields: fieldGroups.EmptyFields.fields.length,
+                // empty fields can be undefined if there is no existence information to be fetched
+                emptyFields: fieldGroups.EmptyFields?.fields.length || 0,
                 metaFields: fieldGroups.MetaFields.fields.length,
               },
             })}

x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx

@@ -14,13 +14,6 @@ import { IndexPatternField } from './types';
 import { FieldItemSharedProps, FieldsAccordion } from './fields_accordion';
 const PAGINATION_SIZE = 50;
 
-export interface FieldsGroup {
-  specialFields: IndexPatternField[];
-  availableFields: IndexPatternField[];
-  emptyFields: IndexPatternField[];
-  metaFields: IndexPatternField[];
-}
-
 export type FieldGroups = Record<
   string,
   {

x-pack/plugins/lens/server/plugin.tsx

@@ -7,6 +7,7 @@
 import { Plugin, CoreSetup, CoreStart, PluginInitializerContext, Logger } from 'src/core/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { Observable } from 'rxjs';
+import { PluginStart as DataPluginStart } from 'src/plugins/data/server';
 import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
 import { setupRoutes } from './routes';
 import {
@@ -23,6 +24,7 @@ export interface PluginSetupContract {
 
 export interface PluginStartContract {
   taskManager?: TaskManagerStartContract;
+  data: DataPluginStart;
 }
 
 export class LensServerPlugin implements Plugin<{}, {}, {}, {}> {

x-pack/plugins/lens/server/routes/existing_fields.test.ts

@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { IndexPattern } from 'src/plugins/data/common';
 import { existingFields, Field, buildFieldList } from './existing_fields';
 
 describe('existingFields', () => {
@@ -71,25 +72,20 @@ describe('existingFields', () => {
 
 describe('buildFieldList', () => {
   const indexPattern = {
-    id: '',
-    type: 'indexpattern',
-    attributes: {
-      title: 'testpattern',
-      type: 'type',
-      typeMeta: 'typemeta',
-      fields: JSON.stringify([
-        { name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
-        { name: 'bar' },
-        { name: '@bar' },
-        { name: 'baz' },
-        { name: '_mymeta' },
-      ]),
-    },
-    references: [],
+    title: 'testpattern',
+    type: 'type',
+    typeMeta: 'typemeta',
+    fields: [
+      { name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
+      { name: 'bar' },
+      { name: '@bar' },
+      { name: 'baz' },
+      { name: '_mymeta' },
+    ],
   };
 
   it('supports scripted fields', () => {
-    const fields = buildFieldList(indexPattern, []);
+    const fields = buildFieldList((indexPattern as unknown) as IndexPattern, []);
     expect(fields.find((f) => f.isScript)).toMatchObject({
       isScript: true,
       name: 'foo',
@@ -99,7 +95,7 @@ describe('buildFieldList', () => {
   });
 
   it('supports meta fields', () => {
-    const fields = buildFieldList(indexPattern, ['_mymeta']);
+    const fields = buildFieldList((indexPattern as unknown) as IndexPattern, ['_mymeta']);
     expect(fields.find((f) => f.isMeta)).toMatchObject({
       isScript: false,
       isMeta: true,

x-pack/plugins/lens/server/routes/existing_fields.ts

@@ -6,10 +6,12 @@
 
 import Boom from '@hapi/boom';
 import { schema } from '@kbn/config-schema';
-import { ILegacyScopedClusterClient, SavedObject, RequestHandlerContext } from 'src/core/server';
+import { ILegacyScopedClusterClient, RequestHandlerContext } from 'src/core/server';
 import { CoreSetup, Logger } from 'src/core/server';
+import { IndexPattern, IndexPatternsService } from 'src/plugins/data/common';
 import { BASE_API_URL } from '../../common';
-import { IndexPatternAttributes, UI_SETTINGS } from '../../../../../src/plugins/data/server';
+import { UI_SETTINGS } from '../../../../../src/plugins/data/server';
+import { PluginStartContract } from '../plugin';
 
 export function isBoomError(error: { isBoom?: boolean }): error is Boom {
   return error.isBoom === true;
@@ -28,7 +30,7 @@ export interface Field {
   script?: string;
 }
 
-export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) {
+export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>, logger: Logger) {
   const router = setup.http.createRouter();
 
   router.post(
@@ -47,11 +49,18 @@ export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) {
       },
     },
     async (context, req, res) => {
+      const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices();
+      const savedObjectsClient = savedObjects.getScopedClient(req);
+      const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
       try {
         return res.ok({
           body: await fetchFieldExistence({
             ...req.params,
             ...req.body,
+            indexPatternsService: await data.indexPatterns.indexPatternsServiceFactory(
+              savedObjectsClient,
+              esClient
+            ),
             context,
           }),
         });
@@ -80,6 +89,7 @@ export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) {
 async function fetchFieldExistence({
   context,
   indexPatternId,
+  indexPatternsService,
   dslQuery = { match_all: {} },
   fromDate,
   toDate,
@@ -87,16 +97,14 @@ async function fetchFieldExistence({
 }: {
   indexPatternId: string;
   context: RequestHandlerContext;
+  indexPatternsService: IndexPatternsService;
   dslQuery: object;
   fromDate?: string;
   toDate?: string;
   timeFieldName?: string;
 }) {
   const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
-  const { indexPattern, indexPatternTitle } = await fetchIndexPatternDefinition(
-    indexPatternId,
-    context
-  );
+  const indexPattern = await indexPatternsService.get(indexPatternId);
 
   const fields = buildFieldList(indexPattern, metaFields);
   const docs = await fetchIndexPatternStats({
@@ -104,51 +112,32 @@ async function fetchFieldExistence({
     toDate,
     dslQuery,
     client: context.core.elasticsearch.legacy.client,
-    index: indexPatternTitle,
-    timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
+    index: indexPattern.title,
+    timeFieldName: timeFieldName || indexPattern.timeFieldName,
     fields,
   });
 
   return {
-    indexPatternTitle,
+    indexPatternTitle: indexPattern.title,
     existingFieldNames: existingFields(docs, fields),
   };
 }
 
-async function fetchIndexPatternDefinition(indexPatternId: string, context: RequestHandlerContext) {
-  const savedObjectsClient = context.core.savedObjects.client;
-  const indexPattern = await savedObjectsClient.get<IndexPatternAttributes>(
-    'index-pattern',
-    indexPatternId
-  );
-  const indexPatternTitle = indexPattern.attributes.title;
-
-  return {
-    indexPattern,
-    indexPatternTitle,
-  };
-}
-
 /**
  * Exported only for unit tests.
  */
-export function buildFieldList(
-  indexPattern: SavedObject<IndexPatternAttributes>,
-  metaFields: string[]
-): Field[] {
-  return JSON.parse(indexPattern.attributes.fields).map(
-    (field: { name: string; lang: string; scripted?: boolean; script?: string }) => {
-      return {
-        name: field.name,
-        isScript: !!field.scripted,
-        lang: field.lang,
-        script: field.script,
-        // id is a special case - it doesn't show up in the meta field list,
-        // but as it's not part of source, it has to be handled separately.
-        isMeta: metaFields.includes(field.name) || field.name === '_id',
-      };
-    }
-  );
+export function buildFieldList(indexPattern: IndexPattern, metaFields: string[]): Field[] {
+  return indexPattern.fields.map((field) => {
+    return {
+      name: field.name,
+      isScript: !!field.scripted,
+      lang: field.lang,
+      script: field.script,
+      // id is a special case - it doesn't show up in the meta field list,
+      // but as it's not part of source, it has to be handled separately.
+      isMeta: metaFields.includes(field.name) || field.name === '_id',
+    };
+  });
 }
 
 async function fetchIndexPatternStats({

x-pack/plugins/lens/server/routes/field_stats.ts

@@ -11,10 +11,11 @@ import { CoreSetup } from 'src/core/server';
 import { IFieldType } from 'src/plugins/data/common';
 import { ESSearchResponse } from '../../../../typings/elasticsearch';
 import { FieldStatsResponse, BASE_API_URL } from '../../common';
+import { PluginStartContract } from '../plugin';
 
 const SHARD_SIZE = 5000;
 
-export async function initFieldsRoute(setup: CoreSetup) {
+export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
   const router = setup.http.createRouter();
   router.post(
     {

x-pack/plugins/lens/server/routes/index.ts

@@ -5,11 +5,12 @@
  */
 
 import { CoreSetup, Logger } from 'src/core/server';
+import { PluginStartContract } from '../plugin';
 import { existingFieldsRoute } from './existing_fields';
 import { initFieldsRoute } from './field_stats';
 import { initLensUsageRoute } from './telemetry';
 
-export function setupRoutes(setup: CoreSetup, logger: Logger) {
+export function setupRoutes(setup: CoreSetup<PluginStartContract>, logger: Logger) {
   existingFieldsRoute(setup, logger);
   initFieldsRoute(setup);
   initLensUsageRoute(setup);

x-pack/plugins/lens/server/routes/telemetry.ts

@@ -8,10 +8,11 @@ import Boom from '@hapi/boom';
 import { CoreSetup } from 'src/core/server';
 import { schema } from '@kbn/config-schema';
 import { BASE_API_URL } from '../../common';
+import { PluginStartContract } from '../plugin';
 
 // This route is responsible for taking a batch of click events from the browser
 // and writing them to saved objects
-export async function initLensUsageRoute(setup: CoreSetup) {
+export async function initLensUsageRoute(setup: CoreSetup<PluginStartContract>) {
   const router = setup.http.createRouter();
   router.post(
     {

x-pack/test/api_integration/apis/lens/existing_fields.ts

@@ -104,26 +104,46 @@ const metricBeatData = [
   '_index',
   '_type',
   'agent.ephemeral_id',
+  'agent.ephemeral_id.keyword',
   'agent.hostname',
+  'agent.hostname.keyword',
   'agent.id',
+  'agent.id.keyword',
   'agent.type',
+  'agent.type.keyword',
   'agent.version',
+  'agent.version.keyword',
   'ecs.version',
+  'ecs.version.keyword',
   'event.dataset',
+  'event.dataset.keyword',
   'event.duration',
   'event.module',
+  'event.module.keyword',
   'host.architecture',
+  'host.architecture.keyword',
   'host.hostname',
+  'host.hostname.keyword',
   'host.id',
+  'host.id.keyword',
   'host.name',
+  'host.name.keyword',
   'host.os.build',
+  'host.os.build.keyword',
   'host.os.family',
+  'host.os.family.keyword',
   'host.os.kernel',
+  'host.os.kernel.keyword',
   'host.os.name',
+  'host.os.name.keyword',
   'host.os.platform',
+  'host.os.platform.keyword',
   'host.os.version',
+  'host.os.version.keyword',
   'metricset.name',
+  'metricset.name.keyword',
   'service.type',
+  'service.type.keyword',
   'system.cpu.cores',
   'system.cpu.idle.pct',
   'system.cpu.iowait.pct',