Add scoring support to KQL (#103727)

* Add ability to generate KQL filters in the "must" clause
Also defaults search source to generate filters in the must clause if _score is one of the sort fields

* Update docs

* Review feedback

* Fix tests

* update tests

* Fix merge error

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-es-query/src/es_query/build_es_query.ts

@@ -12,17 +12,17 @@ import { buildQueryFromFilters } from './from_filters';
 import { buildQueryFromLucene } from './from_lucene';
 import { Filter, Query } from '../filters';
 import { IndexPatternBase } from './types';
+import { KueryQueryOptions } from '../kuery';
 
 /**
  * Configurations to be used while constructing an ES query.
  * @public
  */
-export interface EsQueryConfig {
+export type EsQueryConfig = KueryQueryOptions & {
   allowLeadingWildcards: boolean;
   queryStringOptions: Record<string, any>;
   ignoreFilterIfFieldNotInIndex: boolean;
-  dateFormatTZ?: string;
-}
+};
 
 function removeMatchAll<T>(filters: T[]) {
   return filters.filter(
@@ -59,7 +59,8 @@ export function buildEsQuery(
     indexPattern,
     queriesByLanguage.kuery,
     config.allowLeadingWildcards,
-    config.dateFormatTZ
+    config.dateFormatTZ,
+    config.filtersInMustClause
   );
   const luceneQuery = buildQueryFromLucene(
     queriesByLanguage.lucene,

packages/kbn-es-query/src/es_query/from_kuery.ts

@@ -15,13 +15,14 @@ export function buildQueryFromKuery(
   indexPattern: IndexPatternBase | undefined,
   queries: Query[] = [],
   allowLeadingWildcards: boolean = false,
-  dateFormatTZ?: string
+  dateFormatTZ?: string,
+  filtersInMustClause: boolean = false
 ) {
   const queryASTs = queries.map((query) => {
     return fromKueryExpression(query.query, { allowLeadingWildcards });
   });
 
-  return buildQuery(indexPattern, queryASTs, { dateFormatTZ });
+  return buildQuery(indexPattern, queryASTs, { dateFormatTZ, filtersInMustClause });
 }
 
 function buildQuery(

packages/kbn-es-query/src/kuery/functions/and.test.ts

@@ -55,6 +55,24 @@ describe('kuery functions', () => {
           )
         );
       });
+
+      test("should wrap subqueries in an ES bool query's must clause for scoring if enabled", () => {
+        const node = nodeTypes.function.buildNode('and', [childNode1, childNode2]);
+        const result = and.toElasticsearchQuery(node, indexPattern, {
+          filtersInMustClause: true,
+        });
+
+        expect(result).toHaveProperty('bool');
+        expect(Object.keys(result).length).toBe(1);
+        expect(result.bool).toHaveProperty('must');
+        expect(Object.keys(result.bool).length).toBe(1);
+
+        expect(result.bool.must).toEqual(
+          [childNode1, childNode2].map((childNode) =>
+            ast.toElasticsearchQuery(childNode, indexPattern)
+          )
+        );
+      });
     });
   });
 });

packages/kbn-es-query/src/kuery/functions/and.ts

@@ -7,7 +7,7 @@
  */
 
 import * as ast from '../ast';
-import { IndexPatternBase, KueryNode } from '../..';
+import { IndexPatternBase, KueryNode, KueryQueryOptions } from '../..';
 
 export function buildNodeParams(children: KueryNode[]) {
   return {
@@ -18,14 +18,16 @@ export function buildNodeParams(children: KueryNode[]) {
 export function toElasticsearchQuery(
   node: KueryNode,
   indexPattern?: IndexPatternBase,
-  config: Record<string, any> = {},
+  config: KueryQueryOptions = {},
   context: Record<string, any> = {}
 ) {
+  const { filtersInMustClause } = config;
   const children = node.arguments || [];
+  const key = filtersInMustClause ? 'must' : 'filter';
 
   return {
     bool: {
-      filter: children.map((child: KueryNode) => {
+      [key]: children.map((child: KueryNode) => {
         return ast.toElasticsearchQuery(child, indexPattern, config, context);
       }),
     },

packages/kbn-es-query/src/kuery/index.ts

@@ -9,4 +9,4 @@
 export { KQLSyntaxError } from './kuery_syntax_error';
 export { nodeTypes, nodeBuilder } from './node_types';
 export { fromKueryExpression, toElasticsearchQuery } from './ast';
-export { DslQuery, KueryNode } from './types';
+export { DslQuery, KueryNode, KueryQueryOptions } from './types';

packages/kbn-es-query/src/kuery/types.ts

@@ -32,3 +32,9 @@ export interface KueryParseOptions {
 }
 
 export { nodeTypes } from './node_types';
+
+/** @public */
+export interface KueryQueryOptions {
+  filtersInMustClause?: boolean;
+  dateFormatTZ?: string;
+}

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -359,6 +359,69 @@ describe('SearchSource', () => {
         expect(request.fields).toEqual(['*']);
         expect(request._source).toEqual(false);
       });
+
+      test('includes queries in the "filter" clause by default', async () => {
+        searchSource.setField('query', {
+          query: 'agent.keyword : "Mozilla" ',
+          language: 'kuery',
+        });
+        const request = searchSource.getSearchRequestBody();
+        expect(request.query).toMatchInlineSnapshot(`
+          Object {
+            "bool": Object {
+              "filter": Array [
+                Object {
+                  "bool": Object {
+                    "minimum_should_match": 1,
+                    "should": Array [
+                      Object {
+                        "match_phrase": Object {
+                          "agent.keyword": "Mozilla",
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+              "must": Array [],
+              "must_not": Array [],
+              "should": Array [],
+            },
+          }
+        `);
+      });
+
+      test('includes queries in the "must" clause if sorting by _score', async () => {
+        searchSource.setField('query', {
+          query: 'agent.keyword : "Mozilla" ',
+          language: 'kuery',
+        });
+        searchSource.setField('sort', [{ _score: SortDirection.asc }]);
+        const request = searchSource.getSearchRequestBody();
+        expect(request.query).toMatchInlineSnapshot(`
+          Object {
+            "bool": Object {
+              "filter": Array [],
+              "must": Array [
+                Object {
+                  "bool": Object {
+                    "minimum_should_match": 1,
+                    "should": Array [
+                      Object {
+                        "match_phrase": Object {
+                          "agent.keyword": "Mozilla",
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+              "must_not": Array [],
+              "should": Array [],
+            },
+          }
+        `);
+      });
     });
 
     describe('source filters handling', () => {
@@ -943,27 +1006,27 @@ describe('SearchSource', () => {
         expect(next).toBeCalledTimes(2);
         expect(complete).toBeCalledTimes(1);
         expect(next.mock.calls[0]).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "isPartial": true,
-            "isRunning": true,
-            "rawResponse": Object {
-              "test": 1,
-            },
-          },
-        ]
-        `);
+                  Array [
+                    Object {
+                      "isPartial": true,
+                      "isRunning": true,
+                      "rawResponse": Object {
+                        "test": 1,
+                      },
+                    },
+                  ]
+                `);
         expect(next.mock.calls[1]).toMatchInlineSnapshot(`
-        Array [
-          Object {
-            "isPartial": false,
-            "isRunning": false,
-            "rawResponse": Object {
-              "test": 2,
-            },
-          },
-        ]
-        `);
+                  Array [
+                    Object {
+                      "isPartial": false,
+                      "isRunning": false,
+                      "rawResponse": Object {
+                        "test": 2,
+                      },
+                    },
+                  ]
+                `);
       });
 
       test('shareReplays result', async () => {

src/plugins/data/common/search/search_source/search_source.ts

@@ -79,6 +79,7 @@ import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patt
 import {
   AggConfigs,
   ES_SEARCH_STRATEGY,
+  EsQuerySortValue,
   IEsSearchResponse,
   ISearchGeneric,
   ISearchOptions,
@@ -833,7 +834,14 @@ export class SearchSource {
       body.fields = filteredDocvalueFields;
     }
 
-    const esQueryConfigs = getEsQueryConfig({ get: getConfig });
+    // If sorting by _score, build queries in the "must" clause instead of "filter" clause to enable scoring
+    const filtersInMustClause = (body.sort ?? []).some((sort: EsQuerySortValue[]) =>
+      sort.hasOwnProperty('_score')
+    );
+    const esQueryConfigs = {
+      ...getEsQueryConfig({ get: getConfig }),
+      filtersInMustClause,
+    };
     body.query = buildEsQuery(index, query, filters, esQueryConfigs);
 
     if (highlightAll && body.query) {

src/plugins/discover/public/application/apps/main/components/doc_table/components/table_header/score_sort_warning.tsx

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React from 'react';
+import { EuiIconTip } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+export function DocViewTableScoreSortWarning() {
+  const tooltipContent = i18n.translate('discover.docViews.table.scoreSortWarningTooltip', {
+    defaultMessage: 'In order to retrieve values for _score, you must sort by it.',
+  });
+
+  return <EuiIconTip content={tooltipContent} color="warning" size="s" type="alert" />;
+}

src/plugins/discover/public/application/apps/main/components/doc_table/components/table_header/table_header_column.tsx

@@ -10,6 +10,7 @@ import React from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiButtonIcon, EuiToolTip } from '@elastic/eui';
 import { SortOrder } from './helpers';
+import { DocViewTableScoreSortWarning } from './score_sort_warning';
 
 interface Props {
   colLeftIdx: number; // idx of the column to the left, -1 if moving is not possible
@@ -64,6 +65,10 @@ export function TableHeaderColumn({
   const curColSort = sortOrder.find((pair) => pair[0] === name);
   const curColSortDir = (curColSort && curColSort[1]) || '';
 
+  // If this is the _score column, and _score is not one of the columns inside the sort, show a
+  // warning that the _score will not be retrieved from Elasticsearch
+  const showScoreSortWarning = name === '_score' && !curColSort;
+
   const handleChangeSortOrder = () => {
     if (!onChangeSortOrder) return;
 
@@ -177,6 +182,7 @@ export function TableHeaderColumn({
   return (
     <th data-test-subj="docTableHeaderField">
       <span data-test-subj={`docTableHeader-${name}`} className="kbnDocTableHeader__actions">
+        {showScoreSortWarning && <DocViewTableScoreSortWarning />}
         {displayName}
         {buttons
           .filter((button) => button.active)