Use new terms enum API for autocomplete value suggestions (#100174)

* Migrate kibana.autocomplete config to data plugin

* Fix CI

* Fix tests

* Use new terms enum API for autocomplete value suggestions

* Add tiers to config

* Re-introduce terms agg and add config/tests for swapping algorithms

* Add data_content and data_cold tiers by default

* Fix types

* Fix maps test

* Update tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/data/config.ts

@@ -15,6 +15,21 @@ export const configSchema = schema.object({
     }),
     valueSuggestions: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
+      method: schema.oneOf([schema.literal('terms_enum'), schema.literal('terms_agg')], {
+        defaultValue: 'terms_enum',
+      }),
+      tiers: schema.arrayOf(
+        schema.oneOf([
+          schema.literal('data_content'),
+          schema.literal('data_hot'),
+          schema.literal('data_warm'),
+          schema.literal('data_cold'),
+          schema.literal('data_frozen'),
+        ]),
+        {
+          defaultValue: ['data_hot', 'data_warm', 'data_content', 'data_cold'],
+        }
+      ),
       terminateAfter: schema.duration({ defaultValue: 100000 }),
       timeout: schema.duration({ defaultValue: 1000 }),
     }),

src/plugins/data/server/autocomplete/terms_agg.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { coreMock } from '../../../../core/server/mocks';
+import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
+import { ConfigSchema } from '../../config';
+import type { DeeplyMockedKeys } from '@kbn/utility-types/jest';
+import type { ApiResponse } from '@elastic/elasticsearch';
+import { termsAggSuggestions } from './terms_agg';
+import { SearchResponse } from 'elasticsearch';
+import { duration } from 'moment';
+
+let savedObjectsClientMock: jest.Mocked<SavedObjectsClientContract>;
+let esClientMock: DeeplyMockedKeys<ElasticsearchClient>;
+const configMock = ({
+  autocomplete: {
+    valueSuggestions: { timeout: duration(4513), terminateAfter: duration(98430) },
+  },
+} as unknown) as ConfigSchema;
+const mockResponse = {
+  body: {
+    aggregations: {
+      suggestions: {
+        buckets: [{ key: 'whoa' }, { key: 'amazing' }],
+      },
+    },
+  },
+} as ApiResponse<SearchResponse<any>>;
+
+describe('terms agg suggestions', () => {
+  beforeEach(() => {
+    const requestHandlerContext = coreMock.createRequestHandlerContext();
+    savedObjectsClientMock = requestHandlerContext.savedObjects.client;
+    esClientMock = requestHandlerContext.elasticsearch.client.asCurrentUser;
+    esClientMock.search.mockResolvedValue(mockResponse);
+  });
+
+  it('calls the _search API with a terms agg with the given args', async () => {
+    const result = await termsAggSuggestions(
+      configMock,
+      savedObjectsClientMock,
+      esClientMock,
+      'index',
+      'fieldName',
+      'query',
+      [],
+      { name: 'field_name', type: 'string' }
+    );
+
+    const [[args]] = esClientMock.search.mock.calls;
+
+    expect(args).toMatchInlineSnapshot(`
+      Object {
+        "body": Object {
+          "aggs": Object {
+            "suggestions": Object {
+              "terms": Object {
+                "execution_hint": "map",
+                "field": "field_name",
+                "include": "query.*",
+                "shard_size": 10,
+              },
+            },
+          },
+          "query": Object {
+            "bool": Object {
+              "filter": Array [],
+            },
+          },
+          "size": 0,
+          "terminate_after": 98430,
+          "timeout": "4513ms",
+        },
+        "index": "index",
+      }
+    `);
+    expect(result).toMatchInlineSnapshot(`
+      Array [
+        "whoa",
+        "amazing",
+      ]
+    `);
+  });
+});

src/plugins/data/server/autocomplete/terms_agg.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { get, map } from 'lodash';
+import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
+import { estypes } from '@elastic/elasticsearch';
+import { ConfigSchema } from '../../config';
+import { IFieldType } from '../../common';
+import { findIndexPatternById, getFieldByName } from '../index_patterns';
+import { shimAbortSignal } from '../search';
+
+export async function termsAggSuggestions(
+  config: ConfigSchema,
+  savedObjectsClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  index: string,
+  fieldName: string,
+  query: string,
+  filters?: estypes.QueryDslQueryContainer[],
+  field?: IFieldType,
+  abortSignal?: AbortSignal
+) {
+  const autocompleteSearchOptions = {
+    timeout: `${config.autocomplete.valueSuggestions.timeout.asMilliseconds()}ms`,
+    terminate_after: config.autocomplete.valueSuggestions.terminateAfter.asMilliseconds(),
+  };
+
+  if (!field?.name && !field?.type) {
+    const indexPattern = await findIndexPatternById(savedObjectsClient, index);
+
+    field = indexPattern && getFieldByName(fieldName, indexPattern);
+  }
+
+  const body = await getBody(autocompleteSearchOptions, field ?? fieldName, query, filters);
+
+  const promise = esClient.search({ index, body });
+  const result = await shimAbortSignal(promise, abortSignal);
+
+  const buckets =
+    get(result.body, 'aggregations.suggestions.buckets') ||
+    get(result.body, 'aggregations.nestedSuggestions.suggestions.buckets');
+
+  return map(buckets ?? [], 'key');
+}
+
+async function getBody(
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  { timeout, terminate_after }: Record<string, any>,
+  field: IFieldType | string,
+  query: string,
+  filters: estypes.QueryDslQueryContainer[] = []
+) {
+  const isFieldObject = (f: any): f is IFieldType => Boolean(f && f.name);
+
+  // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html#_standard_operators
+  const getEscapedQuery = (q: string = '') =>
+    q.replace(/[.?+*|{}[\]()"\\#@&<>~]/g, (match) => `\\${match}`);
+
+  // Helps ensure that the regex is not evaluated eagerly against the terms dictionary
+  const executionHint = 'map' as const;
+
+  // We don't care about the accuracy of the counts, just the content of the terms, so this reduces
+  // the amount of information that needs to be transmitted to the coordinating node
+  const shardSize = 10;
+  const body = {
+    size: 0,
+    timeout,
+    terminate_after,
+    query: {
+      bool: {
+        filter: filters,
+      },
+    },
+    aggs: {
+      suggestions: {
+        terms: {
+          field: isFieldObject(field) ? field.name : field,
+          include: `${getEscapedQuery(query)}.*`,
+          execution_hint: executionHint,
+          shard_size: shardSize,
+        },
+      },
+    },
+  };
+
+  if (isFieldObject(field) && field.subType && field.subType.nested) {
+    return {
+      ...body,
+      aggs: {
+        nestedSuggestions: {
+          nested: {
+            path: field.subType.nested.path,
+          },
+          aggs: body.aggs,
+        },
+      },
+    };
+  }
+
+  return body;
+}

src/plugins/data/server/autocomplete/terms_enum.test.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { termsEnumSuggestions } from './terms_enum';
+import { coreMock } from '../../../../core/server/mocks';
+import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
+import { ConfigSchema } from '../../config';
+import type { DeeplyMockedKeys } from '@kbn/utility-types/jest';
+import type { ApiResponse } from '@elastic/elasticsearch';
+
+let savedObjectsClientMock: jest.Mocked<SavedObjectsClientContract>;
+let esClientMock: DeeplyMockedKeys<ElasticsearchClient>;
+const configMock = {
+  autocomplete: { valueSuggestions: { tiers: ['data_hot', 'data_warm', 'data_content'] } },
+} as ConfigSchema;
+const mockResponse = {
+  body: { terms: ['whoa', 'amazing'] },
+};
+
+describe('_terms_enum suggestions', () => {
+  beforeEach(() => {
+    const requestHandlerContext = coreMock.createRequestHandlerContext();
+    savedObjectsClientMock = requestHandlerContext.savedObjects.client;
+    esClientMock = requestHandlerContext.elasticsearch.client.asCurrentUser;
+    esClientMock.transport.request.mockResolvedValue((mockResponse as unknown) as ApiResponse);
+  });
+
+  it('calls the _terms_enum API with the field, query, filters, and config tiers', async () => {
+    const result = await termsEnumSuggestions(
+      configMock,
+      savedObjectsClientMock,
+      esClientMock,
+      'index',
+      'fieldName',
+      'query',
+      [],
+      { name: 'field_name', type: 'string' }
+    );
+
+    const [[args]] = esClientMock.transport.request.mock.calls;
+
+    expect(args).toMatchInlineSnapshot(`
+      Object {
+        "body": Object {
+          "field": "field_name",
+          "index_filter": Object {
+            "bool": Object {
+              "must": Array [
+                Object {
+                  "terms": Object {
+                    "_tier": Array [
+                      "data_hot",
+                      "data_warm",
+                      "data_content",
+                    ],
+                  },
+                },
+              ],
+            },
+          },
+          "string": "query",
+        },
+        "method": "POST",
+        "path": "/index/_terms_enum",
+      }
+    `);
+    expect(result).toEqual(mockResponse.body.terms);
+  });
+});

src/plugins/data/server/autocomplete/terms_enum.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
+import { estypes } from '@elastic/elasticsearch';
+import { IFieldType } from '../../common';
+import { findIndexPatternById, getFieldByName } from '../index_patterns';
+import { shimAbortSignal } from '../search';
+import { getKbnServerError } from '../../../kibana_utils/server';
+import { ConfigSchema } from '../../config';
+
+export async function termsEnumSuggestions(
+  config: ConfigSchema,
+  savedObjectsClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  index: string,
+  fieldName: string,
+  query: string,
+  filters?: estypes.QueryDslQueryContainer[],
+  field?: IFieldType,
+  abortSignal?: AbortSignal
+) {
+  const { tiers } = config.autocomplete.valueSuggestions;
+  if (!field?.name && !field?.type) {
+    const indexPattern = await findIndexPatternById(savedObjectsClient, index);
+    field = indexPattern && getFieldByName(fieldName, indexPattern);
+  }
+
+  try {
+    const promise = esClient.transport.request({
+      method: 'POST',
+      path: encodeURI(`/${index}/_terms_enum`),
+      body: {
+        field: field?.name ?? field,
+        string: query,
+        index_filter: {
+          bool: {
+            must: [
+              ...(filters ?? []),
+              {
+                terms: {
+                  _tier: tiers,
+                },
+              },
+            ],
+          },
+        },
+      },
+    });
+
+    const result = await shimAbortSignal(promise, abortSignal);
+
+    return result.body.terms;
+  } catch (e) {
+    throw getKbnServerError(e);
+  }
+}

src/plugins/data/server/autocomplete/value_suggestions_route.ts

@@ -6,17 +6,15 @@
  * Side Public License, v 1.
  */
 
-import { get, map } from 'lodash';
 import { schema } from '@kbn/config-schema';
 import { IRouter } from 'kibana/server';
-
 import { Observable } from 'rxjs';
 import { first } from 'rxjs/operators';
-import type { estypes } from '@elastic/elasticsearch';
-import type { IFieldType } from '../index';
-import { findIndexPatternById, getFieldByName } from '../index_patterns';
 import { getRequestAbortedSignal } from '../lib';
-import { ConfigSchema } from '../../config';
+import { getKbnServerError } from '../../../kibana_utils/server';
+import type { ConfigSchema } from '../../config';
+import { termsEnumSuggestions } from './terms_enum';
+import { termsAggSuggestions } from './terms_agg';
 
 export function registerValueSuggestionsRoute(router: IRouter, config$: Observable<ConfigSchema>) {
   router.post(
@@ -44,88 +42,28 @@ export function registerValueSuggestionsRoute(router: IRouter, config$: Observab
       const config = await config$.pipe(first()).toPromise();
       const { field: fieldName, query, filters, fieldMeta } = request.body;
       const { index } = request.params;
-      const { client } = context.core.elasticsearch.legacy;
-      const signal = getRequestAbortedSignal(request.events.aborted$);
+      const abortSignal = getRequestAbortedSignal(request.events.aborted$);
 
-      const autocompleteSearchOptions = {
-        timeout: `${config.autocomplete.valueSuggestions.timeout.asMilliseconds()}ms`,
-        terminate_after: config.autocomplete.valueSuggestions.terminateAfter.asMilliseconds(),
-      };
-
-      let field: IFieldType | undefined = fieldMeta;
-
-      if (!field?.name && !field?.type) {
-        const indexPattern = await findIndexPatternById(context.core.savedObjects.client, index);
-
-        field = indexPattern && getFieldByName(fieldName, indexPattern);
+      try {
+        const fn =
+          config.autocomplete.valueSuggestions.method === 'terms_enum'
+            ? termsEnumSuggestions
+            : termsAggSuggestions;
+        const body = await fn(
+          config,
+          context.core.savedObjects.client,
+          context.core.elasticsearch.client.asCurrentUser,
+          index,
+          fieldName,
+          query,
+          filters,
+          fieldMeta,
+          abortSignal
+        );
+        return response.ok({ body });
+      } catch (e) {
+        throw getKbnServerError(e);
       }
-
-      const body = await getBody(autocompleteSearchOptions, field || fieldName, query, filters);
-
-      const result = await client.callAsCurrentUser('search', { index, body }, { signal });
-
-      const buckets: any[] =
-        get(result, 'aggregations.suggestions.buckets') ||
-        get(result, 'aggregations.nestedSuggestions.suggestions.buckets');
-
-      return response.ok({ body: map(buckets || [], 'key') });
     }
   );
 }
-
-async function getBody(
-  // eslint-disable-next-line @typescript-eslint/naming-convention
-  { timeout, terminate_after }: Record<string, any>,
-  field: IFieldType | string,
-  query: string,
-  filters: estypes.QueryDslQueryContainer[] = []
-) {
-  const isFieldObject = (f: any): f is IFieldType => Boolean(f && f.name);
-
-  // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html#_standard_operators
-  const getEscapedQuery = (q: string = '') =>
-    q.replace(/[.?+*|{}[\]()"\\#@&<>~]/g, (match) => `\\${match}`);
-
-  // Helps ensure that the regex is not evaluated eagerly against the terms dictionary
-  const executionHint = 'map' as const;
-
-  // We don't care about the accuracy of the counts, just the content of the terms, so this reduces
-  // the amount of information that needs to be transmitted to the coordinating node
-  const shardSize = 10;
-  const body = {
-    size: 0,
-    timeout,
-    terminate_after,
-    query: {
-      bool: {
-        filter: filters,
-      },
-    },
-    aggs: {
-      suggestions: {
-        terms: {
-          field: isFieldObject(field) ? field.name : field,
-          include: `${getEscapedQuery(query)}.*`,
-          execution_hint: executionHint,
-          shard_size: shardSize,
-        },
-      },
-    },
-  };
-
-  if (isFieldObject(field) && field.subType && field.subType.nested) {
-    return {
-      ...body,
-      aggs: {
-        nestedSuggestions: {
-          nested: {
-            path: field.subType.nested.path,
-          },
-          aggs: body.aggs,
-        },
-      },
-    };
-  }
-
-  return body;
-}

x-pack/test/functional/apps/maps/vector_styling.js

@@ -34,7 +34,7 @@ export default function ({ getService, getPageObjects }) {
         await PageObjects.maps.setStyleByValue('fillColor', 'machine.os.raw');
         await PageObjects.maps.selectCustomColorRamp('fillColor');
         const suggestions = await PageObjects.maps.getCategorySuggestions();
-        expect(suggestions.trim().split('\n').join()).to.equal('win 8,win xp,win 7,ios,osx');
+        expect(suggestions.trim().split('\n').join()).to.equal('ios,osx,win 7,win 8,win xp');
       });
     });
   });