Add scoring support to KQL (#103727)
* Add ability to generate KQL filters in the "must" clause Also defaults search source to generate filters in the must clause if _score is one of the sort fields * Update docs * Review feedback * Fix tests * update tests * Fix merge error Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
parent
7860c2aac3
commit
a2347b2d77
|
@ -12,17 +12,17 @@ import { buildQueryFromFilters } from './from_filters';
|
|||
import { buildQueryFromLucene } from './from_lucene';
|
||||
import { Filter, Query } from '../filters';
|
||||
import { IndexPatternBase } from './types';
|
||||
import { KueryQueryOptions } from '../kuery';
|
||||
|
||||
/**
|
||||
* Configurations to be used while constructing an ES query.
|
||||
* @public
|
||||
*/
|
||||
export interface EsQueryConfig {
|
||||
export type EsQueryConfig = KueryQueryOptions & {
|
||||
allowLeadingWildcards: boolean;
|
||||
queryStringOptions: Record<string, any>;
|
||||
ignoreFilterIfFieldNotInIndex: boolean;
|
||||
dateFormatTZ?: string;
|
||||
}
|
||||
};
|
||||
|
||||
function removeMatchAll<T>(filters: T[]) {
|
||||
return filters.filter(
|
||||
|
@ -59,7 +59,8 @@ export function buildEsQuery(
|
|||
indexPattern,
|
||||
queriesByLanguage.kuery,
|
||||
config.allowLeadingWildcards,
|
||||
config.dateFormatTZ
|
||||
config.dateFormatTZ,
|
||||
config.filtersInMustClause
|
||||
);
|
||||
const luceneQuery = buildQueryFromLucene(
|
||||
queriesByLanguage.lucene,
|
||||
|
|
|
@ -15,13 +15,14 @@ export function buildQueryFromKuery(
|
|||
indexPattern: IndexPatternBase | undefined,
|
||||
queries: Query[] = [],
|
||||
allowLeadingWildcards: boolean = false,
|
||||
dateFormatTZ?: string
|
||||
dateFormatTZ?: string,
|
||||
filtersInMustClause: boolean = false
|
||||
) {
|
||||
const queryASTs = queries.map((query) => {
|
||||
return fromKueryExpression(query.query, { allowLeadingWildcards });
|
||||
});
|
||||
|
||||
return buildQuery(indexPattern, queryASTs, { dateFormatTZ });
|
||||
return buildQuery(indexPattern, queryASTs, { dateFormatTZ, filtersInMustClause });
|
||||
}
|
||||
|
||||
function buildQuery(
|
||||
|
|
|
@ -55,6 +55,24 @@ describe('kuery functions', () => {
|
|||
)
|
||||
);
|
||||
});
|
||||
|
||||
test("should wrap subqueries in an ES bool query's must clause for scoring if enabled", () => {
|
||||
const node = nodeTypes.function.buildNode('and', [childNode1, childNode2]);
|
||||
const result = and.toElasticsearchQuery(node, indexPattern, {
|
||||
filtersInMustClause: true,
|
||||
});
|
||||
|
||||
expect(result).toHaveProperty('bool');
|
||||
expect(Object.keys(result).length).toBe(1);
|
||||
expect(result.bool).toHaveProperty('must');
|
||||
expect(Object.keys(result.bool).length).toBe(1);
|
||||
|
||||
expect(result.bool.must).toEqual(
|
||||
[childNode1, childNode2].map((childNode) =>
|
||||
ast.toElasticsearchQuery(childNode, indexPattern)
|
||||
)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
*/
|
||||
|
||||
import * as ast from '../ast';
|
||||
import { IndexPatternBase, KueryNode } from '../..';
|
||||
import { IndexPatternBase, KueryNode, KueryQueryOptions } from '../..';
|
||||
|
||||
export function buildNodeParams(children: KueryNode[]) {
|
||||
return {
|
||||
|
@ -18,14 +18,16 @@ export function buildNodeParams(children: KueryNode[]) {
|
|||
export function toElasticsearchQuery(
|
||||
node: KueryNode,
|
||||
indexPattern?: IndexPatternBase,
|
||||
config: Record<string, any> = {},
|
||||
config: KueryQueryOptions = {},
|
||||
context: Record<string, any> = {}
|
||||
) {
|
||||
const { filtersInMustClause } = config;
|
||||
const children = node.arguments || [];
|
||||
const key = filtersInMustClause ? 'must' : 'filter';
|
||||
|
||||
return {
|
||||
bool: {
|
||||
filter: children.map((child: KueryNode) => {
|
||||
[key]: children.map((child: KueryNode) => {
|
||||
return ast.toElasticsearchQuery(child, indexPattern, config, context);
|
||||
}),
|
||||
},
|
||||
|
|
|
@ -9,4 +9,4 @@
|
|||
export { KQLSyntaxError } from './kuery_syntax_error';
|
||||
export { nodeTypes, nodeBuilder } from './node_types';
|
||||
export { fromKueryExpression, toElasticsearchQuery } from './ast';
|
||||
export { DslQuery, KueryNode } from './types';
|
||||
export { DslQuery, KueryNode, KueryQueryOptions } from './types';
|
||||
|
|
|
@ -32,3 +32,9 @@ export interface KueryParseOptions {
|
|||
}
|
||||
|
||||
export { nodeTypes } from './node_types';
|
||||
|
||||
/** @public */
|
||||
export interface KueryQueryOptions {
|
||||
filtersInMustClause?: boolean;
|
||||
dateFormatTZ?: string;
|
||||
}
|
||||
|
|
|
@ -359,6 +359,69 @@ describe('SearchSource', () => {
|
|||
expect(request.fields).toEqual(['*']);
|
||||
expect(request._source).toEqual(false);
|
||||
});
|
||||
|
||||
test('includes queries in the "filter" clause by default', async () => {
|
||||
searchSource.setField('query', {
|
||||
query: 'agent.keyword : "Mozilla" ',
|
||||
language: 'kuery',
|
||||
});
|
||||
const request = searchSource.getSearchRequestBody();
|
||||
expect(request.query).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"bool": Object {
|
||||
"filter": Array [
|
||||
Object {
|
||||
"bool": Object {
|
||||
"minimum_should_match": 1,
|
||||
"should": Array [
|
||||
Object {
|
||||
"match_phrase": Object {
|
||||
"agent.keyword": "Mozilla",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
"must": Array [],
|
||||
"must_not": Array [],
|
||||
"should": Array [],
|
||||
},
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('includes queries in the "must" clause if sorting by _score', async () => {
|
||||
searchSource.setField('query', {
|
||||
query: 'agent.keyword : "Mozilla" ',
|
||||
language: 'kuery',
|
||||
});
|
||||
searchSource.setField('sort', [{ _score: SortDirection.asc }]);
|
||||
const request = searchSource.getSearchRequestBody();
|
||||
expect(request.query).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"bool": Object {
|
||||
"filter": Array [],
|
||||
"must": Array [
|
||||
Object {
|
||||
"bool": Object {
|
||||
"minimum_should_match": 1,
|
||||
"should": Array [
|
||||
Object {
|
||||
"match_phrase": Object {
|
||||
"agent.keyword": "Mozilla",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
"must_not": Array [],
|
||||
"should": Array [],
|
||||
},
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('source filters handling', () => {
|
||||
|
@ -943,27 +1006,27 @@ describe('SearchSource', () => {
|
|||
expect(next).toBeCalledTimes(2);
|
||||
expect(complete).toBeCalledTimes(1);
|
||||
expect(next.mock.calls[0]).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"isPartial": true,
|
||||
"isRunning": true,
|
||||
"rawResponse": Object {
|
||||
"test": 1,
|
||||
},
|
||||
},
|
||||
]
|
||||
`);
|
||||
Array [
|
||||
Object {
|
||||
"isPartial": true,
|
||||
"isRunning": true,
|
||||
"rawResponse": Object {
|
||||
"test": 1,
|
||||
},
|
||||
},
|
||||
]
|
||||
`);
|
||||
expect(next.mock.calls[1]).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"isPartial": false,
|
||||
"isRunning": false,
|
||||
"rawResponse": Object {
|
||||
"test": 2,
|
||||
},
|
||||
},
|
||||
]
|
||||
`);
|
||||
Array [
|
||||
Object {
|
||||
"isPartial": false,
|
||||
"isRunning": false,
|
||||
"rawResponse": Object {
|
||||
"test": 2,
|
||||
},
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('shareReplays result', async () => {
|
||||
|
|
|
@ -79,6 +79,7 @@ import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patt
|
|||
import {
|
||||
AggConfigs,
|
||||
ES_SEARCH_STRATEGY,
|
||||
EsQuerySortValue,
|
||||
IEsSearchResponse,
|
||||
ISearchGeneric,
|
||||
ISearchOptions,
|
||||
|
@ -833,7 +834,14 @@ export class SearchSource {
|
|||
body.fields = filteredDocvalueFields;
|
||||
}
|
||||
|
||||
const esQueryConfigs = getEsQueryConfig({ get: getConfig });
|
||||
// If sorting by _score, build queries in the "must" clause instead of "filter" clause to enable scoring
|
||||
const filtersInMustClause = (body.sort ?? []).some((sort: EsQuerySortValue[]) =>
|
||||
sort.hasOwnProperty('_score')
|
||||
);
|
||||
const esQueryConfigs = {
|
||||
...getEsQueryConfig({ get: getConfig }),
|
||||
filtersInMustClause,
|
||||
};
|
||||
body.query = buildEsQuery(index, query, filters, esQueryConfigs);
|
||||
|
||||
if (highlightAll && body.query) {
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { EuiIconTip } from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
export function DocViewTableScoreSortWarning() {
|
||||
const tooltipContent = i18n.translate('discover.docViews.table.scoreSortWarningTooltip', {
|
||||
defaultMessage: 'In order to retrieve values for _score, you must sort by it.',
|
||||
});
|
||||
|
||||
return <EuiIconTip content={tooltipContent} color="warning" size="s" type="alert" />;
|
||||
}
|
|
@ -10,6 +10,7 @@ import React from 'react';
|
|||
import { i18n } from '@kbn/i18n';
|
||||
import { EuiButtonIcon, EuiToolTip } from '@elastic/eui';
|
||||
import { SortOrder } from './helpers';
|
||||
import { DocViewTableScoreSortWarning } from './score_sort_warning';
|
||||
|
||||
interface Props {
|
||||
colLeftIdx: number; // idx of the column to the left, -1 if moving is not possible
|
||||
|
@ -64,6 +65,10 @@ export function TableHeaderColumn({
|
|||
const curColSort = sortOrder.find((pair) => pair[0] === name);
|
||||
const curColSortDir = (curColSort && curColSort[1]) || '';
|
||||
|
||||
// If this is the _score column, and _score is not one of the columns inside the sort, show a
|
||||
// warning that the _score will not be retrieved from Elasticsearch
|
||||
const showScoreSortWarning = name === '_score' && !curColSort;
|
||||
|
||||
const handleChangeSortOrder = () => {
|
||||
if (!onChangeSortOrder) return;
|
||||
|
||||
|
@ -177,6 +182,7 @@ export function TableHeaderColumn({
|
|||
return (
|
||||
<th data-test-subj="docTableHeaderField">
|
||||
<span data-test-subj={`docTableHeader-${name}`} className="kbnDocTableHeader__actions">
|
||||
{showScoreSortWarning && <DocViewTableScoreSortWarning />}
|
||||
{displayName}
|
||||
{buttons
|
||||
.filter((button) => button.active)
|
||||
|
|
Loading…
Reference in a new issue