From a708d69f50acc667e081d81466b5b9e6bb1a4aac Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 18 Mar 2020 18:06:59 +0300 Subject: [PATCH] [Visualize] Duplicated query filters in es request (#60106) * [Visualize] Duplicated query filters in es request Closes: #59630 * Fix CI * fix CI * move uniq_filters to common * fix scripts/check_published_api_changes Co-authored-by: Elastic Machine --- ...na-plugin-plugins-data-public.esfilters.md | 4 +- .../filter_manager}/compare_filters.test.ts | 2 +- .../query/filter_manager}/compare_filters.ts | 2 +- .../filter_manager}/dedup_filters.test.ts | 10 +---- .../query/filter_manager}/dedup_filters.ts | 2 +- .../data/common/query/filter_manager/index.ts | 22 ++++++++++ .../filter_manager}/uniq_filters.test.ts | 2 +- .../query/filter_manager}/uniq_filters.ts | 2 +- src/plugins/data/common/query/index.ts | 1 + src/plugins/data/public/index.ts | 4 +- src/plugins/data/public/public.api.md | 8 ++-- .../query/filter_manager/filter_manager.ts | 12 ++++-- .../data/public/query/filter_manager/index.ts | 2 - .../query/filter_manager/lib/only_disabled.ts | 3 +- .../state_sync/connect_to_query_state.ts | 3 +- .../create_global_query_observable.ts | 4 +- .../specs/kibana_context.ts | 43 ++++++++++--------- 17 files changed, 73 insertions(+), 53 deletions(-) rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/compare_filters.test.ts (99%) rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/compare_filters.ts (98%) rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/dedup_filters.test.ts (95%) rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/dedup_filters.ts (97%) create mode 100644 src/plugins/data/common/query/filter_manager/index.ts rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/uniq_filters.test.ts (99%) rename src/plugins/data/{public/query/filter_manager/lib => common/query/filter_manager}/uniq_filters.ts (96%) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md index e03072f9a41c..7fd65e5db35f 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md @@ -44,8 +44,8 @@ esFilters: { getPhraseFilterField: (filter: import("../common").PhraseFilter) => string; getPhraseFilterValue: (filter: import("../common").PhraseFilter) => string | number | boolean; getDisplayValueFromFilter: typeof getDisplayValueFromFilter; - compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions) => boolean; - COMPARE_ALL_OPTIONS: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions; + compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("../common").FilterCompareOptions) => boolean; + COMPARE_ALL_OPTIONS: import("../common").FilterCompareOptions; generateFilters: typeof generateFilters; onlyDisabledFiltersChanged: (newFilters?: import("../common").Filter[] | undefined, oldFilters?: import("../common").Filter[] | undefined) => boolean; changeTimeFilter: typeof changeTimeFilter; diff --git a/src/plugins/data/public/query/filter_manager/lib/compare_filters.test.ts b/src/plugins/data/common/query/filter_manager/compare_filters.test.ts similarity index 99% rename from src/plugins/data/public/query/filter_manager/lib/compare_filters.test.ts rename to src/plugins/data/common/query/filter_manager/compare_filters.test.ts index da8f5b356494..b0bb2f754d6c 100644 --- a/src/plugins/data/public/query/filter_manager/lib/compare_filters.test.ts +++ b/src/plugins/data/common/query/filter_manager/compare_filters.test.ts @@ -18,7 +18,7 @@ */ import { compareFilters, COMPARE_ALL_OPTIONS } from './compare_filters'; -import { buildEmptyFilter, buildQueryFilter, FilterStateStore } from '../../../../common'; +import { buildEmptyFilter, buildQueryFilter, FilterStateStore } from '../../es_query'; describe('filter manager utilities', () => { describe('compare filters', () => { diff --git a/src/plugins/data/public/query/filter_manager/lib/compare_filters.ts b/src/plugins/data/common/query/filter_manager/compare_filters.ts similarity index 98% rename from src/plugins/data/public/query/filter_manager/lib/compare_filters.ts rename to src/plugins/data/common/query/filter_manager/compare_filters.ts index a2105fdc1d3e..e047d5e0665d 100644 --- a/src/plugins/data/public/query/filter_manager/lib/compare_filters.ts +++ b/src/plugins/data/common/query/filter_manager/compare_filters.ts @@ -18,7 +18,7 @@ */ import { defaults, isEqual, omit, map } from 'lodash'; -import { FilterMeta, Filter } from '../../../../common'; +import { FilterMeta, Filter } from '../../es_query'; export interface FilterCompareOptions { disabled?: boolean; diff --git a/src/plugins/data/public/query/filter_manager/lib/dedup_filters.test.ts b/src/plugins/data/common/query/filter_manager/dedup_filters.test.ts similarity index 95% rename from src/plugins/data/public/query/filter_manager/lib/dedup_filters.test.ts rename to src/plugins/data/common/query/filter_manager/dedup_filters.test.ts index ecc0ec94e07c..228489de37da 100644 --- a/src/plugins/data/public/query/filter_manager/lib/dedup_filters.test.ts +++ b/src/plugins/data/common/query/filter_manager/dedup_filters.test.ts @@ -18,14 +18,8 @@ */ import { dedupFilters } from './dedup_filters'; -import { - Filter, - IIndexPattern, - IFieldType, - buildRangeFilter, - buildQueryFilter, - FilterStateStore, -} from '../../../../common'; +import { Filter, buildRangeFilter, buildQueryFilter, FilterStateStore } from '../../es_query'; +import { IIndexPattern, IFieldType } from '../../index_patterns'; describe('filter manager utilities', () => { let indexPattern: IIndexPattern; diff --git a/src/plugins/data/public/query/filter_manager/lib/dedup_filters.ts b/src/plugins/data/common/query/filter_manager/dedup_filters.ts similarity index 97% rename from src/plugins/data/public/query/filter_manager/lib/dedup_filters.ts rename to src/plugins/data/common/query/filter_manager/dedup_filters.ts index d5d0e70504b4..7d1b00ac10c0 100644 --- a/src/plugins/data/public/query/filter_manager/lib/dedup_filters.ts +++ b/src/plugins/data/common/query/filter_manager/dedup_filters.ts @@ -19,7 +19,7 @@ import { filter, find } from 'lodash'; import { compareFilters, FilterCompareOptions } from './compare_filters'; -import { Filter } from '../../../../common'; +import { Filter } from '../../es_query'; /** * Combine 2 filter collections, removing duplicates diff --git a/src/plugins/data/common/query/filter_manager/index.ts b/src/plugins/data/common/query/filter_manager/index.ts new file mode 100644 index 000000000000..315c124f083a --- /dev/null +++ b/src/plugins/data/common/query/filter_manager/index.ts @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { dedupFilters } from './dedup_filters'; +export { uniqFilters } from './uniq_filters'; +export { compareFilters, COMPARE_ALL_OPTIONS, FilterCompareOptions } from './compare_filters'; diff --git a/src/plugins/data/public/query/filter_manager/lib/uniq_filters.test.ts b/src/plugins/data/common/query/filter_manager/uniq_filters.test.ts similarity index 99% rename from src/plugins/data/public/query/filter_manager/lib/uniq_filters.test.ts rename to src/plugins/data/common/query/filter_manager/uniq_filters.test.ts index 8b525a3d2a2e..5a35e85c95ea 100644 --- a/src/plugins/data/public/query/filter_manager/lib/uniq_filters.test.ts +++ b/src/plugins/data/common/query/filter_manager/uniq_filters.test.ts @@ -18,7 +18,7 @@ */ import { uniqFilters } from './uniq_filters'; -import { buildQueryFilter, Filter, FilterStateStore } from '../../../../common'; +import { buildQueryFilter, Filter, FilterStateStore } from '../../es_query'; describe('filter manager utilities', () => { describe('niqFilter', () => { diff --git a/src/plugins/data/public/query/filter_manager/lib/uniq_filters.ts b/src/plugins/data/common/query/filter_manager/uniq_filters.ts similarity index 96% rename from src/plugins/data/public/query/filter_manager/lib/uniq_filters.ts rename to src/plugins/data/common/query/filter_manager/uniq_filters.ts index 44c102d7ab15..683cbf7c78a8 100644 --- a/src/plugins/data/public/query/filter_manager/lib/uniq_filters.ts +++ b/src/plugins/data/common/query/filter_manager/uniq_filters.ts @@ -17,8 +17,8 @@ * under the License. */ import { each, union } from 'lodash'; +import { Filter } from '../../es_query'; import { dedupFilters } from './dedup_filters'; -import { Filter } from '../../../../common'; /** * Remove duplicate filters from an array of filters diff --git a/src/plugins/data/common/query/index.ts b/src/plugins/data/common/query/index.ts index d8f7b5091eb8..421cc4f63e4e 100644 --- a/src/plugins/data/common/query/index.ts +++ b/src/plugins/data/common/query/index.ts @@ -17,4 +17,5 @@ * under the License. */ +export * from './filter_manager'; export * from './types'; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 58bd9a5ab05d..339a5fea91c5 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -47,13 +47,13 @@ import { isQueryStringFilter, isRangeFilter, toggleFilterNegated, + compareFilters, + COMPARE_ALL_OPTIONS, } from '../common'; import { FilterLabel } from './ui/filter_bar'; import { - compareFilters, - COMPARE_ALL_OPTIONS, generateFilters, onlyDisabledFiltersChanged, changeTimeFilter, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 783411bbf27e..07d8d302bc18 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -362,8 +362,8 @@ export const esFilters: { getPhraseFilterField: (filter: import("../common").PhraseFilter) => string; getPhraseFilterValue: (filter: import("../common").PhraseFilter) => string | number | boolean; getDisplayValueFromFilter: typeof getDisplayValueFromFilter; - compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions) => boolean; - COMPARE_ALL_OPTIONS: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions; + compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("../common").FilterCompareOptions) => boolean; + COMPARE_ALL_OPTIONS: import("../common").FilterCompareOptions; generateFilters: typeof generateFilters; onlyDisabledFiltersChanged: (newFilters?: import("../common").Filter[] | undefined, oldFilters?: import("../common").Filter[] | undefined) => boolean; changeTimeFilter: typeof changeTimeFilter; @@ -1843,8 +1843,8 @@ export type TSearchStrategyProvider = (context: ISearc // src/plugins/data/public/index.ts:405:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:406:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:409:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:33 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:38:1 - (ae-forgotten-export) The symbol "QueryStateChange" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:33:33 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:37:1 - (ae-forgotten-export) The symbol "QueryStateChange" needs to be exported by the entry point index.d.ts // src/plugins/data/public/types.ts:52:5 - (ae-forgotten-export) The symbol "createFiltersFromEvent" needs to be exported by the entry point index.d.ts // src/plugins/data/public/types.ts:60:5 - (ae-forgotten-export) The symbol "IndexPatternSelectProps" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/public/query/filter_manager/filter_manager.ts b/src/plugins/data/public/query/filter_manager/filter_manager.ts index c951953b2655..fba1866ebd61 100644 --- a/src/plugins/data/public/query/filter_manager/filter_manager.ts +++ b/src/plugins/data/public/query/filter_manager/filter_manager.ts @@ -22,13 +22,19 @@ import { Subject } from 'rxjs'; import { IUiSettingsClient } from 'src/core/public'; -import { COMPARE_ALL_OPTIONS, compareFilters } from './lib/compare_filters'; import { sortFilters } from './lib/sort_filters'; import { mapAndFlattenFilters } from './lib/map_and_flatten_filters'; -import { uniqFilters } from './lib/uniq_filters'; import { onlyDisabledFiltersChanged } from './lib/only_disabled'; import { PartitionedFilters } from './types'; -import { FilterStateStore, Filter, isFilterPinned } from '../../../common'; + +import { + FilterStateStore, + Filter, + uniqFilters, + isFilterPinned, + compareFilters, + COMPARE_ALL_OPTIONS, +} from '../../../common'; export class FilterManager { private filters: Filter[] = []; diff --git a/src/plugins/data/public/query/filter_manager/index.ts b/src/plugins/data/public/query/filter_manager/index.ts index 09990adacde4..be512c503d53 100644 --- a/src/plugins/data/public/query/filter_manager/index.ts +++ b/src/plugins/data/public/query/filter_manager/index.ts @@ -19,8 +19,6 @@ export { FilterManager } from './filter_manager'; -export { uniqFilters } from './lib/uniq_filters'; export { mapAndFlattenFilters } from './lib/map_and_flatten_filters'; export { onlyDisabledFiltersChanged } from './lib/only_disabled'; export { generateFilters } from './lib/generate_filters'; -export { compareFilters, COMPARE_ALL_OPTIONS } from './lib/compare_filters'; diff --git a/src/plugins/data/public/query/filter_manager/lib/only_disabled.ts b/src/plugins/data/public/query/filter_manager/lib/only_disabled.ts index 34e1ac38ae95..18c51ebeabe5 100644 --- a/src/plugins/data/public/query/filter_manager/lib/only_disabled.ts +++ b/src/plugins/data/public/query/filter_manager/lib/only_disabled.ts @@ -18,8 +18,7 @@ */ import { filter } from 'lodash'; -import { Filter } from '../../../../common'; -import { compareFilters, COMPARE_ALL_OPTIONS } from './compare_filters'; +import { Filter, compareFilters, COMPARE_ALL_OPTIONS } from '../../../../common'; const isEnabled = (f: Filter) => f && f.meta && !f.meta.disabled; diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts index a22e66860c76..331d8969f248 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts @@ -21,10 +21,9 @@ import { Subscription } from 'rxjs'; import { filter, map } from 'rxjs/operators'; import _ from 'lodash'; import { BaseStateContainer } from '../../../../kibana_utils/public'; -import { COMPARE_ALL_OPTIONS, compareFilters } from '../filter_manager/lib/compare_filters'; import { QuerySetup, QueryStart } from '../query_service'; import { QueryState, QueryStateChange } from './types'; -import { FilterStateStore } from '../../../common/es_query/filters'; +import { FilterStateStore, COMPARE_ALL_OPTIONS, compareFilters } from '../../../common'; /** * Helper to setup two-way syncing of global data and a state container diff --git a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts index d0d97bfaaeb3..dd075f9be7d9 100644 --- a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts +++ b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts @@ -20,10 +20,10 @@ import { Observable, Subscription } from 'rxjs'; import { map, tap } from 'rxjs/operators'; import { TimefilterSetup } from '../timefilter'; -import { COMPARE_ALL_OPTIONS, compareFilters, FilterManager } from '../filter_manager'; +import { FilterManager } from '../filter_manager'; import { QueryState, QueryStateChange } from './index'; import { createStateContainer } from '../../../../kibana_utils/public'; -import { isFilterPinned } from '../../../common/es_query/filters'; +import { isFilterPinned, compareFilters, COMPARE_ALL_OPTIONS } from '../../../common'; export function createQueryStateObservable({ timefilter: { timefilter }, diff --git a/src/plugins/expressions/common/expression_functions/specs/kibana_context.ts b/src/plugins/expressions/common/expression_functions/specs/kibana_context.ts index 4092dfbba00d..b8be273d7bbd 100644 --- a/src/plugins/expressions/common/expression_functions/specs/kibana_context.ts +++ b/src/plugins/expressions/common/expression_functions/specs/kibana_context.ts @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ - +import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition } from '../../expression_functions'; import { KibanaContext } from '../../expression_types'; +import { Query, uniqFilters } from '../../../../data/common'; interface Arguments { q?: string | null; @@ -35,6 +36,15 @@ export type ExpressionFunctionKibanaContext = ExpressionFunctionDefinition< Promise >; +const getParsedValue = (data: any, defaultValue: any) => + typeof data === 'string' && data.length ? JSON.parse(data) || defaultValue : defaultValue; + +const mergeQueries = (first: Query | Query[] = [], second: Query | Query[]) => + uniq( + [...(Array.isArray(first) ? first : [first]), ...(Array.isArray(second) ? second : [second])], + (n: any) => JSON.stringify(n.query) + ); + export const kibanaContextFunction: ExpressionFunctionKibanaContext = { name: 'kibana_context', type: 'kibana_context', @@ -75,9 +85,9 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = { }, async fn(input, args, { getSavedObject }) { - const queryArg = args.q ? JSON.parse(args.q) : []; - let queries = Array.isArray(queryArg) ? queryArg : [queryArg]; - let filters = args.filters ? JSON.parse(args.filters) : []; + const timeRange = getParsedValue(args.timeRange, input?.timeRange); + let queries = mergeQueries(input?.query, getParsedValue(args?.q, [])); + let filters = [...(input?.filters || []), ...getParsedValue(args?.filters, [])]; if (args.savedSearchId) { if (typeof getSavedObject !== 'function') { @@ -89,29 +99,20 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = { } const obj = await getSavedObject('search', args.savedSearchId); const search = obj.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string }; - const data = JSON.parse(search.searchSourceJSON) as { query: string; filter: any[] }; - queries = queries.concat(data.query); - filters = filters.concat(data.filter); - } + const { query, filter } = getParsedValue(search.searchSourceJSON, {}); - if (input && input.query) { - queries = queries.concat(input.query); + if (query) { + queries = mergeQueries(queries, query); + } + if (filter) { + filters = [...filters, ...(Array.isArray(filter) ? filter : [filter])]; + } } - if (input && input.filters) { - filters = filters.concat(input.filters).filter((f: any) => !f.meta.disabled); - } - - const timeRange = args.timeRange - ? JSON.parse(args.timeRange) - : input - ? input.timeRange - : undefined; - return { type: 'kibana_context', query: queries, - filters, + filters: uniqFilters(filters).filter((f: any) => !f.meta?.disabled), timeRange, }; },