From da785eae041280581c4302676eb9319d1577c82c Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 3 May 2021 12:42:26 -0500 Subject: [PATCH] Revert "Revert "Add essql search strategy and integrate in canvas (#94754)" (#98841) * Revert "Revert "Add essql search strategy and integrate in canvas (#94754)"" This reverts commit 0f15a12420cb195f813471aed81c3c932fc33dcb. * Update squel usage to safe-squel Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/collectors/management/schema.ts | 4 + .../server/collectors/management/types.ts | 1 + src/plugins/presentation_util/common/labs.ts | 19 ++- src/plugins/telemetry/schema/oss_plugins.json | 6 + .../functions/browser/escount.ts | 94 ++++++++++++ .../functions/browser/esdocs.ts | 141 ++++++++++++++++++ .../functions/browser/essql.ts | 103 +++++++++++++ .../functions/browser/index.ts | 14 +- .../functions/server/escount.ts | 5 +- x-pack/plugins/canvas/common/lib/constants.ts | 1 + .../lib/request}/build_bool_array.ts | 2 +- .../lib/request}/build_es_request.js | 0 .../lib => common/lib/request}/filters.ts | 2 +- .../lib/request}/format_response.js | 0 .../lib/request}/get_es_filter.ts | 2 +- .../lib/request}/normalize_type.ts | 0 .../lib/request}/sanitize_name.ts | 0 .../canvas/i18n/functions/dict/escount.ts | 2 +- .../canvas/i18n/functions/dict/esdocs.ts | 2 +- .../canvas/i18n/functions/dict/essql.ts | 8 +- x-pack/plugins/canvas/public/plugin.tsx | 3 +- .../canvas/public/services/context.tsx | 2 + .../canvas/public/services/expressions.ts | 14 +- .../plugins/canvas/public/services/index.ts | 5 + .../plugins/canvas/public/services/search.ts | 24 +++ .../canvas/public/services/stubs/index.ts | 2 + .../canvas/public/services/stubs/search.ts | 11 ++ .../canvas/server/lib/essql_strategy.ts | 96 ++++++++++++ .../plugins/canvas/server/lib/query_es_sql.ts | 6 +- x-pack/plugins/canvas/server/plugin.ts | 18 ++- .../server/routes/es_fields/es_fields.ts | 2 +- x-pack/plugins/canvas/types/index.ts | 1 + x-pack/plugins/canvas/types/strategy.ts | 31 ++++ 33 files changed, 603 insertions(+), 18 deletions(-) create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts rename x-pack/plugins/canvas/{server/lib => common/lib/request}/build_bool_array.ts (92%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/build_es_request.js (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/filters.ts (98%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/format_response.js (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/get_es_filter.ts (93%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/normalize_type.ts (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/sanitize_name.ts (100%) create mode 100644 x-pack/plugins/canvas/public/services/search.ts create mode 100644 x-pack/plugins/canvas/public/services/stubs/search.ts create mode 100644 x-pack/plugins/canvas/server/lib/essql_strategy.ts create mode 100644 x-pack/plugins/canvas/types/strategy.ts diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index c5a255072381..06d1cd290ffd 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -440,4 +440,8 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'labs:canvas:useDataService': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 4dc1773ecfbe..dfbe6bd3e048 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -119,5 +119,6 @@ export interface UsageStats { 'banners:textColor': string; 'banners:backgroundColor': string; 'labs:canvas:enable_ui': boolean; + 'labs:canvas:useDataService': boolean; 'labs:presentation:timeToPresent': boolean; } diff --git a/src/plugins/presentation_util/common/labs.ts b/src/plugins/presentation_util/common/labs.ts index ce7855c516c8..d551b733ecb8 100644 --- a/src/plugins/presentation_util/common/labs.ts +++ b/src/plugins/presentation_util/common/labs.ts @@ -8,9 +8,10 @@ import { i18n } from '@kbn/i18n'; +export const USE_DATA_SERVICE = 'labs:canvas:useDataService'; export const TIME_TO_PRESENT = 'labs:presentation:timeToPresent'; -export const projectIDs = [TIME_TO_PRESENT] as const; +export const projectIDs = [TIME_TO_PRESENT, USE_DATA_SERVICE] as const; export const environmentNames = ['kibana', 'browser', 'session'] as const; export const solutionNames = ['canvas', 'dashboard', 'presentation'] as const; @@ -32,6 +33,22 @@ export const projects: { [ID in ProjectID]: ProjectConfig & { id: ID } } = { }), solutions: ['canvas'], }, + [USE_DATA_SERVICE]: { + id: USE_DATA_SERVICE, + isActive: true, + isDisplayed: true, + environments: ['kibana', 'browser', 'session'], + name: i18n.translate('presentationUtil.experiments.enableUseDataServiceExperimentName', { + defaultMessage: 'Use data service', + }), + description: i18n.translate( + 'presentationUtil.experiments.enableUseDataServiceExperimentDescription', + { + defaultMessage: 'An experiment of using the new data.search service for Canvas datasources', + } + ), + solutions: ['canvas'], + }, }; export type ProjectID = typeof projectIDs[number]; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 76460a57ee44..7cd66dc8eef3 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -8342,6 +8342,12 @@ "_meta": { "description": "Non-default value of setting." } + }, + "labs:canvas:useDataService": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } } } }, diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts new file mode 100644 index 000000000000..97aa93428041 --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; + +// @ts-expect-error untyped local +import { buildESRequest } from '../../../common/lib/request/build_es_request'; + +import { searchService } from '../../../public/services'; + +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + index: string | null; + query: string; +} + +export function escount(): ExpressionFunctionDefinition< + 'escount', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().escount; + + return { + name: 'escount', + type: 'number', + context: { + types: ['filter'], + }, + help, + args: { + query: { + types: ['string'], + aliases: ['_', 'q'], + help: argHelp.query, + default: '"-_index:.kibana"', + }, + index: { + types: ['string'], + default: '_all', + help: argHelp.index, + }, + }, + fn: (input, args, handlers) => { + input.and = input.and.concat([ + { + type: 'filter', + filterType: 'luceneQueryString', + query: args.query, + and: [], + }, + ]); + + const esRequest = buildESRequest( + { + index: args.index, + body: { + track_total_hits: true, + size: 0, + query: { + bool: { + must: [{ match_all: {} }], + }, + }, + }, + }, + input + ); + + const search = searchService.getService().search; + const req = { + params: { + ...esRequest, + }, + }; + + return search + .search(req) + .toPromise() + .then((resp: any) => { + return resp.rawResponse.hits.total; + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts new file mode 100644 index 000000000000..c40e1ffd6243 --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; + +// @ts-expect-error untyped local +import { buildESRequest } from '../../../common/lib/request/build_es_request'; + +import { searchService } from '../../../public/services'; +import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + index: string; + query: string; + sort: string; + fields: string; + metaFields: string; + count: number; +} + +export function esdocs(): ExpressionFunctionDefinition< + 'esdocs', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().esdocs; + + return { + name: 'esdocs', + type: 'datatable', + context: { + types: ['filter'], + }, + help, + args: { + query: { + types: ['string'], + aliases: ['_', 'q'], + help: argHelp.query, + default: '-_index:.kibana', + }, + count: { + types: ['number'], + default: 1000, + help: argHelp.count, + }, + fields: { + help: argHelp.fields, + types: ['string'], + }, + index: { + types: ['string'], + default: '_all', + help: argHelp.index, + }, + // TODO: This arg isn't being used in the function. + // We need to restore this functionality or remove it as an arg. + metaFields: { + help: argHelp.metaFields, + types: ['string'], + }, + sort: { + types: ['string'], + help: argHelp.sort, + }, + }, + fn: async (input, args, handlers) => { + const { count, index, fields, sort } = args; + + input.and = input.and.concat([ + { + type: 'filter', + filterType: 'luceneQueryString', + query: args.query, + and: [], + }, + ]); + + // Load ad-hoc to avoid adding to the page load bundle size + const squel = await import('safe-squel'); + + let query = squel.select({ + autoQuoteTableNames: true, + autoQuoteFieldNames: true, + autoQuoteAliasNames: true, + nameQuoteCharacter: '"', + }); + + if (index) { + query.from(index); + } + + if (fields) { + const allFields = fields.split(',').map((field) => field.trim()); + allFields.forEach((field) => (query = query.field(field))); + } + + if (sort) { + const [sortField, sortOrder] = sort.split(',').map((str) => str.trim()); + if (sortField) { + query.order(`"${sortField}"`, sortOrder === 'asc'); + } + } + + const search = searchService.getService().search; + + const req = { + count, + query: query.toString(), + filter: input.and, + }; + + // We're requesting the data using the ESSQL strategy because + // the SQL routes return type information with the result set + return search + .search(req, { + strategy: ESSQL_SEARCH_STRATEGY, + }) + .toPromise() + .then((resp: EssqlSearchStrategyResponse) => { + return { + type: 'datatable', + meta: { + type: 'essql', + }, + ...resp, + }; + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts new file mode 100644 index 000000000000..1339c93032ea --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; +import { searchService } from '../../../public/services'; +import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + query: string; + parameter: Array; + count: number; + timezone: string; +} + +export function essql(): ExpressionFunctionDefinition< + 'essql', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().essql; + + return { + name: 'essql', + type: 'datatable', + context: { + types: ['filter'], + }, + help, + args: { + query: { + aliases: ['_', 'q'], + types: ['string'], + help: argHelp.query, + }, + parameter: { + aliases: ['param'], + types: ['string', 'number', 'boolean'], + multi: true, + help: argHelp.parameter, + }, + count: { + types: ['number'], + help: argHelp.count, + default: 1000, + }, + timezone: { + aliases: ['tz'], + types: ['string'], + default: 'UTC', + help: argHelp.timezone, + }, + }, + fn: (input, args, handlers) => { + const search = searchService.getService().search; + const { parameter, ...restOfArgs } = args; + const req = { + ...restOfArgs, + params: parameter, + filter: input.and, + }; + + return search + .search(req, { + strategy: ESSQL_SEARCH_STRATEGY, + }) + .toPromise() + .then((resp: EssqlSearchStrategyResponse) => { + return { + type: 'datatable', + meta: { + type: 'essql', + }, + ...resp, + }; + }) + .catch((e) => { + let message = `Unexpected error from Elasticsearch: ${e.message}`; + if (e.err) { + const { type, reason } = e.err.attributes; + if (type === 'parsing_exception') { + message = `Couldn't parse Elasticsearch SQL query. You may need to add double quotes to names containing special characters. Check your query and try again. Error: ${reason}`; + } else { + message = `Unexpected error from Elasticsearch: ${type} - ${reason}`; + } + } + + // Re-write the error message before surfacing it up + e.message = message; + throw e; + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts index 6e7c43135f41..2cfdebafb70d 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts @@ -10,5 +10,17 @@ import { functions as externalFunctions } from '../external'; import { location } from './location'; import { markdown } from './markdown'; import { urlparam } from './urlparam'; +import { escount } from './escount'; +import { esdocs } from './esdocs'; +import { essql } from './essql'; -export const functions = [location, markdown, urlparam, ...commonFunctions, ...externalFunctions]; +export const functions = [ + location, + markdown, + urlparam, + escount, + esdocs, + essql, + ...commonFunctions, + ...externalFunctions, +]; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts index 206e47413ae5..95f5ef446a47 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts @@ -9,10 +9,9 @@ import { ExpressionFunctionDefinition, ExpressionValueFilter, } from 'src/plugins/expressions/common'; -/* eslint-disable */ // @ts-expect-error untyped local -import { buildESRequest } from '../../../server/lib/build_es_request'; -/* eslint-enable */ +import { buildESRequest } from '../../../common/lib/request/build_es_request'; + import { getFunctionHelp } from '../../../i18n'; interface Arguments { diff --git a/x-pack/plugins/canvas/common/lib/constants.ts b/x-pack/plugins/canvas/common/lib/constants.ts index 697389fe2ce7..2b916033ce55 100644 --- a/x-pack/plugins/canvas/common/lib/constants.ts +++ b/x-pack/plugins/canvas/common/lib/constants.ts @@ -44,3 +44,4 @@ export const API_ROUTE_SHAREABLE_RUNTIME_DOWNLOAD = `/public/canvas/${SHAREABLE_ export const CANVAS_EMBEDDABLE_CLASSNAME = `canvasEmbeddable`; export const CONTEXT_MENU_TOP_BORDER_CLASSNAME = 'canvasContextMenu--topBorder'; export const API_ROUTE_FUNCTIONS = `${API_ROUTE}/fns`; +export const ESSQL_SEARCH_STRATEGY = 'essql'; diff --git a/x-pack/plugins/canvas/server/lib/build_bool_array.ts b/x-pack/plugins/canvas/common/lib/request/build_bool_array.ts similarity index 92% rename from x-pack/plugins/canvas/server/lib/build_bool_array.ts rename to x-pack/plugins/canvas/common/lib/request/build_bool_array.ts index 826449ca6ad3..c0d630b4c405 100644 --- a/x-pack/plugins/canvas/server/lib/build_bool_array.ts +++ b/x-pack/plugins/canvas/common/lib/request/build_bool_array.ts @@ -6,7 +6,7 @@ */ import { getESFilter } from './get_es_filter'; -import { ExpressionValueFilter } from '../../types'; +import { ExpressionValueFilter } from '../../../types'; const compact = (arr: T[]) => (Array.isArray(arr) ? arr.filter((val) => Boolean(val)) : []); diff --git a/x-pack/plugins/canvas/server/lib/build_es_request.js b/x-pack/plugins/canvas/common/lib/request/build_es_request.js similarity index 100% rename from x-pack/plugins/canvas/server/lib/build_es_request.js rename to x-pack/plugins/canvas/common/lib/request/build_es_request.js diff --git a/x-pack/plugins/canvas/server/lib/filters.ts b/x-pack/plugins/canvas/common/lib/request/filters.ts similarity index 98% rename from x-pack/plugins/canvas/server/lib/filters.ts rename to x-pack/plugins/canvas/common/lib/request/filters.ts index 8c6b485c4cca..f1465fe48bdc 100644 --- a/x-pack/plugins/canvas/server/lib/filters.ts +++ b/x-pack/plugins/canvas/common/lib/request/filters.ts @@ -11,7 +11,7 @@ import { CanvasTimeFilter, CanvasLuceneFilter, CanvasExactlyFilter, -} from '../../types'; +} from '../../../types'; /* TODO: This could be pluggable diff --git a/x-pack/plugins/canvas/server/lib/format_response.js b/x-pack/plugins/canvas/common/lib/request/format_response.js similarity index 100% rename from x-pack/plugins/canvas/server/lib/format_response.js rename to x-pack/plugins/canvas/common/lib/request/format_response.js diff --git a/x-pack/plugins/canvas/server/lib/get_es_filter.ts b/x-pack/plugins/canvas/common/lib/request/get_es_filter.ts similarity index 93% rename from x-pack/plugins/canvas/server/lib/get_es_filter.ts rename to x-pack/plugins/canvas/common/lib/request/get_es_filter.ts index 85335a4be06d..353a793adcd1 100644 --- a/x-pack/plugins/canvas/server/lib/get_es_filter.ts +++ b/x-pack/plugins/canvas/common/lib/request/get_es_filter.ts @@ -12,7 +12,7 @@ */ import { filters } from './filters'; -import { ExpressionValueFilter } from '../../types'; +import { ExpressionValueFilter } from '../../../types'; export function getESFilter(filter: ExpressionValueFilter) { if (!filter.filterType || !filters[filter.filterType]) { diff --git a/x-pack/plugins/canvas/server/lib/normalize_type.ts b/x-pack/plugins/canvas/common/lib/request/normalize_type.ts similarity index 100% rename from x-pack/plugins/canvas/server/lib/normalize_type.ts rename to x-pack/plugins/canvas/common/lib/request/normalize_type.ts diff --git a/x-pack/plugins/canvas/server/lib/sanitize_name.ts b/x-pack/plugins/canvas/common/lib/request/sanitize_name.ts similarity index 100% rename from x-pack/plugins/canvas/server/lib/sanitize_name.ts rename to x-pack/plugins/canvas/common/lib/request/sanitize_name.ts diff --git a/x-pack/plugins/canvas/i18n/functions/dict/escount.ts b/x-pack/plugins/canvas/i18n/functions/dict/escount.ts index d88156ba32ce..af1337360ba6 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/escount.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/escount.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { escount } from '../../../canvas_plugin_src/functions/server/escount'; +import { escount } from '../../../canvas_plugin_src/functions/browser/escount'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, LUCENE } from '../../constants'; diff --git a/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts b/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts index b78425de144e..6be5acdb8bc9 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { esdocs } from '../../../canvas_plugin_src/functions/server/esdocs'; +import { esdocs } from '../../../canvas_plugin_src/functions/browser/esdocs'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, LUCENE } from '../../constants'; diff --git a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts index cfe848455dc3..6304db945fc3 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { essql } from '../../../canvas_plugin_src/functions/server/essql'; +import { essql } from '../../../canvas_plugin_src/functions/browser/essql'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, SQL, ISO8601, UTC } from '../../constants'; @@ -27,6 +27,12 @@ export const help: FunctionHelp> = { SQL, }, }), + parameter: i18n.translate('xpack.canvas.functions.essql.args.parameterHelpText', { + defaultMessage: 'A parameter to be passed to the {SQL} query.', + values: { + SQL, + }, + }), count: i18n.translate('xpack.canvas.functions.essql.args.countHelpText', { defaultMessage: 'The number of documents to retrieve. For better performance, use a smaller data set.', diff --git a/x-pack/plugins/canvas/public/plugin.tsx b/x-pack/plugins/canvas/public/plugin.tsx index 750b542116a7..d31a5a18cecc 100644 --- a/x-pack/plugins/canvas/public/plugin.tsx +++ b/x-pack/plugins/canvas/public/plugin.tsx @@ -22,7 +22,7 @@ import { getSessionStorage } from './lib/storage'; import { SESSIONSTORAGE_LASTPATH } from '../common/lib/constants'; import { featureCatalogueEntry } from './feature_catalogue_entry'; import { ExpressionsSetup, ExpressionsStart } from '../../../../src/plugins/expressions/public'; -import { DataPublicPluginSetup } from '../../../../src/plugins/data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public'; import { UiActionsStart } from '../../../../src/plugins/ui_actions/public'; import { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public'; @@ -54,6 +54,7 @@ export interface CanvasStartDeps { inspector: InspectorStart; uiActions: UiActionsStart; charts: ChartsPluginStart; + data: DataPublicPluginStart; presentationUtil: PresentationUtilPluginStart; } diff --git a/x-pack/plugins/canvas/public/services/context.tsx b/x-pack/plugins/canvas/public/services/context.tsx index 4c18aa68fb51..e078efe18b54 100644 --- a/x-pack/plugins/canvas/public/services/context.tsx +++ b/x-pack/plugins/canvas/public/services/context.tsx @@ -25,6 +25,7 @@ const defaultContextValue = { notify: {}, platform: {}, navLink: {}, + search: {}, }; const context = createContext(defaultContextValue as CanvasServices); @@ -54,6 +55,7 @@ export const ServicesProvider: FC<{ notify: specifiedProviders.notify.getService(), platform: specifiedProviders.platform.getService(), navLink: specifiedProviders.navLink.getService(), + search: specifiedProviders.search.getService(), reporting: specifiedProviders.reporting.getService(), labs: specifiedProviders.labs.getService(), }; diff --git a/x-pack/plugins/canvas/public/services/expressions.ts b/x-pack/plugins/canvas/public/services/expressions.ts index 131919e1eefe..fd733862c4b6 100644 --- a/x-pack/plugins/canvas/public/services/expressions.ts +++ b/x-pack/plugins/canvas/public/services/expressions.ts @@ -24,6 +24,11 @@ export const expressionsServiceFactory: CanvasServiceFactory const loadServerFunctionWrappers = async () => { if (!cached) { cached = (async () => { + const labService = startPlugins.presentationUtil.labsService; + const useDataSearchProject = labService.getProject('labs:canvas:useDataService'); + const hasDataSearch = useDataSearchProject.status.isEnabled; + const dataSearchFns = ['essql', 'esdocs', 'escount']; + const serverFunctionList = await coreSetup.http.get(API_ROUTE_FUNCTIONS); const batchedFunction = bfetch.batchedFunction({ url: API_ROUTE_FUNCTIONS }); const { serialize } = serializeProvider(expressions.getTypes()); @@ -32,9 +37,16 @@ export const expressionsServiceFactory: CanvasServiceFactory // function that matches its definition, but which simply // calls the server-side function endpoint. Object.keys(serverFunctionList).forEach((functionName) => { - if (expressions.getFunction(functionName)) { + // Allow function to be overwritten if we want to use + // the server-hosted essql, esdocs, and escount functions + if (dataSearchFns.includes(functionName)) { + if (hasDataSearch && expressions.getFunction(functionName)) { + return; + } + } else if (expressions.getFunction(functionName)) { return; } + const fn = () => ({ ...serverFunctionList[functionName], fn: (input: any, args: any) => { diff --git a/x-pack/plugins/canvas/public/services/index.ts b/x-pack/plugins/canvas/public/services/index.ts index 1566d6f28085..cbe7de43eff9 100644 --- a/x-pack/plugins/canvas/public/services/index.ts +++ b/x-pack/plugins/canvas/public/services/index.ts @@ -13,10 +13,12 @@ import { platformServiceFactory } from './platform'; import { navLinkServiceFactory } from './nav_link'; import { embeddablesServiceFactory } from './embeddables'; import { expressionsServiceFactory } from './expressions'; +import { searchServiceFactory } from './search'; import { labsServiceFactory } from './labs'; import { reportingServiceFactory } from './reporting'; export { NotifyService } from './notify'; +export { SearchService } from './search'; export { PlatformService } from './platform'; export { NavLinkService } from './nav_link'; export { EmbeddablesService } from './embeddables'; @@ -80,6 +82,7 @@ export const services = { notify: new CanvasServiceProvider(notifyServiceFactory), platform: new CanvasServiceProvider(platformServiceFactory), navLink: new CanvasServiceProvider(navLinkServiceFactory), + search: new CanvasServiceProvider(searchServiceFactory), reporting: new CanvasServiceProvider(reportingServiceFactory), labs: new CanvasServiceProvider(labsServiceFactory), }; @@ -92,6 +95,7 @@ export interface CanvasServices { notify: ServiceFromProvider; platform: ServiceFromProvider; navLink: ServiceFromProvider; + search: ServiceFromProvider; reporting: ServiceFromProvider; labs: ServiceFromProvider; } @@ -120,5 +124,6 @@ export const { platform: platformService, navLink: navLinkService, expressions: expressionsService, + search: searchService, reporting: reportingService, } = services; diff --git a/x-pack/plugins/canvas/public/services/search.ts b/x-pack/plugins/canvas/public/services/search.ts new file mode 100644 index 000000000000..0fe5c89c7709 --- /dev/null +++ b/x-pack/plugins/canvas/public/services/search.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataPublicPluginStart } from 'src/plugins/data/public'; +import { CanvasServiceFactory } from '.'; + +export interface SearchService { + search: DataPublicPluginStart['search']; +} + +export const searchServiceFactory: CanvasServiceFactory = ( + setup, + start, + canvasSetup, + canvasStart +) => { + return { + search: canvasStart.data.search, + }; +}; diff --git a/x-pack/plugins/canvas/public/services/stubs/index.ts b/x-pack/plugins/canvas/public/services/stubs/index.ts index 786582ed94bd..7246a34d7f49 100644 --- a/x-pack/plugins/canvas/public/services/stubs/index.ts +++ b/x-pack/plugins/canvas/public/services/stubs/index.ts @@ -13,6 +13,7 @@ import { navLinkService } from './nav_link'; import { notifyService } from './notify'; import { labsService } from './labs'; import { platformService } from './platform'; +import { searchService } from './search'; export const stubs: CanvasServices = { embeddables: embeddablesService, @@ -21,6 +22,7 @@ export const stubs: CanvasServices = { navLink: navLinkService, notify: notifyService, platform: platformService, + search: searchService, labs: labsService, }; diff --git a/x-pack/plugins/canvas/public/services/stubs/search.ts b/x-pack/plugins/canvas/public/services/stubs/search.ts new file mode 100644 index 000000000000..a4558a93e38a --- /dev/null +++ b/x-pack/plugins/canvas/public/services/stubs/search.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +const noop = (..._args: any[]): any => {}; + +export const searchService: any = { + search: noop, +}; diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts new file mode 100644 index 000000000000..795b4fedaaaa --- /dev/null +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { from } from 'rxjs'; +import { map, zipObject } from 'lodash'; + +import { ISearchStrategy, PluginStart } from 'src/plugins/data/server'; + +import { getKbnServerError } from '../../../../../src/plugins/kibana_utils/server'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types'; + +import { buildBoolArray } from '../../common/lib/request/build_bool_array'; +import { sanitizeName } from '../../common/lib/request/sanitize_name'; +import { normalizeType } from '../../common/lib/request/normalize_type'; + +export const essqlSearchStrategyProvider = ( + data: PluginStart +): ISearchStrategy => { + return { + search: (request, options, { esClient }) => { + const { count, query, filter, timezone, params } = request; + + const searchUntilEnd = async () => { + try { + let response = await esClient.asCurrentUser.sql.query({ + format: 'json', + body: { + query, + // @ts-expect-error `params` missing from `QuerySqlRequest` type + params, + field_multi_value_leniency: true, + time_zone: timezone, + fetch_size: count, + client_id: 'canvas', + filter: { + bool: { + must: [{ match_all: {} }, ...buildBoolArray(filter)], + }, + }, + }, + }); + + let body = response.body; + + const columns = body.columns!.map(({ name, type }) => { + return { + id: sanitizeName(name), + name: sanitizeName(name), + meta: { type: normalizeType(type) }, + }; + }); + const columnNames = map(columns, 'name'); + let rows = body.rows.map((row) => zipObject(columnNames, row)); + + // If we still have rows to retrieve, continue requesting data + // using the cursor until we have everything + while (rows.length < count && body.cursor !== undefined) { + response = await esClient.asCurrentUser.sql.query({ + format: 'json', + body: { + cursor: body.cursor, + }, + }); + + body = response.body; + + rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))]; + } + + // If we used a cursor, clean it up + if (body.cursor !== undefined) { + await esClient.asCurrentUser.sql.clearCursor({ + body: { + cursor: body.cursor, + }, + }); + } + + return { + columns, + rows, + rawResponse: response, + }; + } catch (e) { + throw getKbnServerError(e); + } + }; + + return from(searchUntilEnd()); + }, + }; +}; diff --git a/x-pack/plugins/canvas/server/lib/query_es_sql.ts b/x-pack/plugins/canvas/server/lib/query_es_sql.ts index a315657dadd2..2c4416094914 100644 --- a/x-pack/plugins/canvas/server/lib/query_es_sql.ts +++ b/x-pack/plugins/canvas/server/lib/query_es_sql.ts @@ -6,9 +6,9 @@ */ import { map, zipObject } from 'lodash'; -import { buildBoolArray } from './build_bool_array'; -import { sanitizeName } from './sanitize_name'; -import { normalizeType } from './normalize_type'; +import { buildBoolArray } from '../../common/lib/request/build_bool_array'; +import { sanitizeName } from '../../common/lib/request/sanitize_name'; +import { normalizeType } from '../../common/lib/request/normalize_type'; import { LegacyAPICaller } from '../../../../../src/core/server'; import { ExpressionValueFilter } from '../../types'; diff --git a/x-pack/plugins/canvas/server/plugin.ts b/x-pack/plugins/canvas/server/plugin.ts index 9360825830e5..9ccf3c251fec 100644 --- a/x-pack/plugins/canvas/server/plugin.ts +++ b/x-pack/plugins/canvas/server/plugin.ts @@ -6,10 +6,15 @@ */ import { CoreSetup, PluginInitializerContext, Plugin, Logger, CoreStart } from 'src/core/server'; +import { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from 'src/plugins/data/server'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { HomeServerPluginSetup } from 'src/plugins/home/server'; +import { ESSQL_SEARCH_STRATEGY } from '../common/lib/constants'; import { ReportingSetup } from '../../reporting/server'; import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; import { getCanvasFeature } from './feature'; @@ -19,6 +24,7 @@ import { loadSampleData } from './sample_data'; import { setupInterpreter } from './setup_interpreter'; import { customElementType, workpadType, workpadTemplateType } from './saved_objects'; import { initializeTemplates } from './templates'; +import { essqlSearchStrategyProvider } from './lib/essql_strategy'; import { getUISettings } from './ui_settings'; interface PluginsSetup { @@ -26,17 +32,22 @@ interface PluginsSetup { features: FeaturesPluginSetup; home: HomeServerPluginSetup; bfetch: BfetchServerSetup; + data: DataPluginSetup; reporting?: ReportingSetup; usageCollection?: UsageCollectionSetup; } +interface PluginsStart { + data: DataPluginStart; +} + export class CanvasPlugin implements Plugin { private readonly logger: Logger; constructor(public readonly initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); } - public setup(coreSetup: CoreSetup, plugins: PluginsSetup) { + public setup(coreSetup: CoreSetup, plugins: PluginsSetup) { coreSetup.uiSettings.register(getUISettings()); coreSetup.savedObjects.registerType(customElementType); coreSetup.savedObjects.registerType(workpadType); @@ -64,6 +75,11 @@ export class CanvasPlugin implements Plugin { registerCanvasUsageCollector(plugins.usageCollection, globalConfig.kibana.index); setupInterpreter(plugins.expressions); + + coreSetup.getStartServices().then(([_, depsStart]) => { + const strategy = essqlSearchStrategyProvider(depsStart.data); + plugins.data.search.registerSearchStrategy(ESSQL_SEARCH_STRATEGY, strategy); + }); } public start(coreStart: CoreStart) { diff --git a/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts b/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts index 7ceace8b84db..20a4775847c9 100644 --- a/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts +++ b/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts @@ -9,7 +9,7 @@ import { mapValues, keys } from 'lodash'; import { schema } from '@kbn/config-schema'; import { API_ROUTE } from '../../../common/lib'; import { catchErrorHandler } from '../catch_error_handler'; -import { normalizeType } from '../../lib/normalize_type'; +import { normalizeType } from '../../../common/lib/request/normalize_type'; import { RouteInitializerDeps } from '..'; const ESFieldsRequestSchema = schema.object({ diff --git a/x-pack/plugins/canvas/types/index.ts b/x-pack/plugins/canvas/types/index.ts index 80314cab0625..09ae1510be6d 100644 --- a/x-pack/plugins/canvas/types/index.ts +++ b/x-pack/plugins/canvas/types/index.ts @@ -14,5 +14,6 @@ export * from './functions'; export * from './renderers'; export * from './shortcuts'; export * from './state'; +export * from './strategy'; export * from './style'; export * from './telemetry'; diff --git a/x-pack/plugins/canvas/types/strategy.ts b/x-pack/plugins/canvas/types/strategy.ts new file mode 100644 index 000000000000..1c94059f0c9c --- /dev/null +++ b/x-pack/plugins/canvas/types/strategy.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; +import { QuerySqlResponse } from '@elastic/elasticsearch/api/types'; +import { IKibanaSearchRequest } from 'src/plugins/data/common'; +import { ExpressionValueFilter } from '.'; +export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest { + count: number; + query: string; + params?: Array; + timezone?: string; + filter: ExpressionValueFilter[]; +} + +export interface EssqlSearchStrategyResponse { + columns: Array<{ + id: string; + name: string; + meta: { + type: string; + }; + }>; + rows: any[]; + + rawResponse: ApiResponse; +}