From b2f44622fe0ea8b4b5e68ede886ca1cfbb3a81ab Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 2 Jan 2019 09:34:38 +0200 Subject: [PATCH] [ML] Remove use of types in datafeeds (#27572) This commit removes the use of types in datafeeds and a few more places where it was possible. Relates #26968 --- .../__mocks__/mock_job_config.json | 1 - .../mock_series_config_filebeat.json | 3 -- .../__mocks__/mock_series_config_rare.json | 1 - ...explorer_chart_config_builder.test.js.snap | 1 - ...orer_charts_container_service.test.js.snap | 2 -- .../explorer_charts_container_service.js | 2 -- .../new_job/advanced/new_job_controller.js | 30 +------------------ .../simple/components/watcher/watch.js | 1 - .../ml/public/services/results_service.js | 17 ++--------- .../forecasting_modal/forecasting_modal.js | 1 - .../timeseries_search_service.js | 2 -- .../ml/datafeed_high_mean_response_time.json | 1 - ...feed_docker_high_count_process_events.json | 1 - ...datafeed_docker_rare_process_activity.json | 1 - ...afeed_hosts_high_count_process_events.json | 1 - .../datafeed_hosts_rare_process_activity.json | 1 - 16 files changed, 4 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_job_config.json b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_job_config.json index 2750ad84a830..5f2714dfb5f1 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_job_config.json +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_job_config.json @@ -72,7 +72,6 @@ "indices": [ "farequote-2017" ], - "types": [], "query": { "match_all": { "boost": 1 diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_filebeat.json b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_filebeat.json index b2c974e737e4..e626535e83df 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_filebeat.json +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_filebeat.json @@ -12,9 +12,6 @@ "indices": [ "filebeat-7.0.0*" ], - "types": [ - "doc" - ], "query": { "match_all": { "boost": 1 diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_rare.json b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_rare.json index 07c13d7cd8b7..bf93979f745b 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_rare.json +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/__mocks__/mock_series_config_rare.json @@ -11,7 +11,6 @@ "indices": [ "filebeat-6.0.0-2017-nginx-elasticco-anon" ], - "types": [], "query": { "match_all": { "boost": 1 diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_chart_config_builder.test.js.snap b/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_chart_config_builder.test.js.snap index ca27cd106562..3713b28a103e 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_chart_config_builder.test.js.snap +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_chart_config_builder.test.js.snap @@ -20,7 +20,6 @@ Object { "query_delay": "86658ms", "scroll_size": 1000, "state": "stopped", - "types": Array [], }, "detectorIndex": 0, "detectorLabel": "mean(responsetime)", diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_charts_container_service.test.js.snap b/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_charts_container_service.test.js.snap index e9292ed69d6c..2c226f3bc5a7 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_charts_container_service.test.js.snap +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/__snapshots__/explorer_charts_container_service.test.js.snap @@ -32,7 +32,6 @@ Object { "query_delay": "86658ms", "scroll_size": 1000, "state": "stopped", - "types": Array [], }, "detectorIndex": 0, "detectorLabel": "mean(responsetime)", @@ -575,7 +574,6 @@ Object { "query_delay": "86658ms", "scroll_size": 1000, "state": "stopped", - "types": Array [], }, "detectorIndex": 0, "detectorLabel": "mean(responsetime)", diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js b/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js index 4c58095504d9..4076c9c0492a 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js @@ -105,7 +105,6 @@ export function explorerChartsContainerServiceFactory( const datafeedQuery = _.get(config, 'datafeedConfig.query', null); return mlResultsService.getMetricData( config.datafeedConfig.indices, - config.datafeedConfig.types, config.entityFields, datafeedQuery, config.metricFunction, @@ -165,7 +164,6 @@ export function explorerChartsContainerServiceFactory( const datafeedQuery = _.get(config, 'datafeedConfig.query', null); return mlResultsService.getEventDistributionData( config.datafeedConfig.indices, - config.datafeedConfig.types, splitField, filterField, datafeedQuery, diff --git a/x-pack/plugins/ml/public/jobs/new_job/advanced/new_job_controller.js b/x-pack/plugins/ml/public/jobs/new_job/advanced/new_job_controller.js index ac366227bb07..e598263a984a 100644 --- a/x-pack/plugins/ml/public/jobs/new_job/advanced/new_job_controller.js +++ b/x-pack/plugins/ml/public/jobs/new_job/advanced/new_job_controller.js @@ -130,7 +130,6 @@ module.controller('MlNewJob', $scope.mode = MODE.NEW; $scope.saveLock = false; $scope.indices = {}; - $scope.types = {}; $scope.fields = {}; $scope.dateFields = {}; $scope.catFields = {}; @@ -198,7 +197,6 @@ module.controller('MlNewJob', indexTextOk: false, fieldsUpToDate: false, indices: {}, - types: {}, isDatafeed: true, useDedicatedIndex: false, enableModelPlot: false, @@ -214,7 +212,6 @@ module.controller('MlNewJob', scrollSizeText: '', scrollSizeDefault: 1000, indicesText: '', - typesText: '', scriptFields: [], }, saveStatus: { @@ -826,11 +823,6 @@ module.controller('MlNewJob', scrollSize = ''; } - clear($scope.types); - _.each(datafeedConfig.types, (type) => { - $scope.types[type] = $scope.ui.types[type]; - }); - clear($scope.indices); _.each(datafeedConfig.indices, (index) => { $scope.indices[index] = $scope.ui.indices[index]; @@ -847,8 +839,6 @@ module.controller('MlNewJob', $scope.ui.fieldsUpToDate = fieldsUpToDate; - const types = Array.isArray(datafeedConfig.types) ? datafeedConfig.types : []; - $scope.ui.datafeed = { queryText: angular.toJson(datafeedConfig.query, true), queryDelayText: queryDelay, @@ -858,7 +848,6 @@ module.controller('MlNewJob', scrollSizeText: scrollSize, scrollSizeDefault: scrollSizeDefault, indicesText, - typesText: types.join(','), scriptFields, }; @@ -978,22 +967,6 @@ module.controller('MlNewJob', indices = df.indicesText.split(',').map(i => i.trim()); } - let types = []; - if (df.typesText) { - types = df.typesText.split(','); - for (let i = 0; i < types.length; i++) { - types[i] = types[i].trim(); - } - } - // if the selected types is different to all types - // the user must have edited the json, so use the types object - // otherwise, the types object is the same as all types, so set - // types to an empty array - const typeKeys = Object.keys($scope.ui.types); - if (_.difference(typeKeys, types).length === 0) { - types = []; - } - // create datafeedConfig if it doesn't already exist if (!$scope.job.datafeed_config) { $scope.job.datafeed_config = {}; @@ -1025,7 +998,6 @@ module.controller('MlNewJob', } config.indices = indices; - config.types = types; } } @@ -1264,7 +1236,7 @@ module.controller('MlNewJob', }); } - // using the selected indices and types, perform a search + // using the selected indices, perform a search // on the ES server and display the results in the Data preview tab function loadDataPreview() { createJSONText(); diff --git a/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js b/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js index b193f0cc55ed..aab27151036f 100644 --- a/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js +++ b/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js @@ -19,7 +19,6 @@ export const watch = { indices: [ ML_RESULTS_INDEX_PATTERN ], - types: [], body: { size: 0, query: { diff --git a/x-pack/plugins/ml/public/services/results_service.js b/x-pack/plugins/ml/public/services/results_service.js index 42b363acff9b..28aa2031c2c9 100644 --- a/x-pack/plugins/ml/public/services/results_service.js +++ b/x-pack/plugins/ml/public/services/results_service.js @@ -1160,7 +1160,6 @@ function getRecordsForCriteria(jobIds, criteriaFields, threshold, earliestMs, la // Queries Elasticsearch to obtain metric aggregation results. // index can be a String, or String[], of index names to search. -// types must be a String[] of types to search. // entityFields parameter must be an array, with each object in the array having 'fieldName' // and 'fieldValue' properties. // Extra query object can be supplied, or pass null if no additional query @@ -1168,7 +1167,6 @@ function getRecordsForCriteria(jobIds, criteriaFields, threshold, earliestMs, la // Returned response contains a results property containing the requested aggregation. function getMetricData( index, - types, entityFields, query, metricFunction, @@ -1181,15 +1179,11 @@ function getMetricData( const obj = { success: true, results: {} }; // Build the criteria to use in the bool filter part of the request. - // Add criteria for the types, time range, entity fields, + // Add criteria for the time range, entity fields, // plus any additional supplied query. const mustCriteria = []; const shouldCriteria = []; - if (types && types.length) { - mustCriteria.push({ terms: { _type: types } }); - } - mustCriteria.push({ range: { [timeFieldName]: { @@ -1336,7 +1330,7 @@ function getEventRateData( const obj = { success: true, results: {} }; // Build the criteria to use in the bool filter part of the request. - // Add criteria for the types, time range, entity fields, + // Add criteria for the time range, entity fields, // plus any additional supplied query. const mustCriteria = [{ range: { @@ -1407,7 +1401,6 @@ const ENTITY_AGGREGATION_SIZE = 10; const AGGREGATION_MIN_DOC_COUNT = 1; function getEventDistributionData( index, - types, splitField, filterField = null, query, @@ -1423,14 +1416,10 @@ function getEventDistributionData( } // Build the criteria to use in the bool filter part of the request. - // Add criteria for the types, time range, entity fields, + // Add criteria for the time range, entity fields, // plus any additional supplied query. const mustCriteria = []; - if (types && types.length) { - mustCriteria.push({ terms: { _type: types } }); - } - mustCriteria.push({ range: { [timeFieldName]: { diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/components/forecasting_modal/forecasting_modal.js b/x-pack/plugins/ml/public/timeseriesexplorer/components/forecasting_modal/forecasting_modal.js index 9fa30fe56ea7..9430c4e1d923 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/components/forecasting_modal/forecasting_modal.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/components/forecasting_modal/forecasting_modal.js @@ -284,7 +284,6 @@ class ForecastingModal extends Component { if (entityFieldNames.length > 0) { ml.getCardinalityOfFields({ index: job.datafeed_config.indices, - types: job.datafeed_config.types, fieldNames: entityFieldNames, query: job.datafeed_config.query, timeFieldName: job.data_description.time_field, diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_search_service.js b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_search_service.js index 54485016b51b..50bb1f48086a 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_search_service.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_search_service.js @@ -64,7 +64,6 @@ function getMetricData(job, detectorIndex, entityFields, earliestMs, latestMs, i mlResultsService.getMetricData( chartConfig.datafeedConfig.indices, - chartConfig.datafeedConfig.types, entityFields, chartConfig.datafeedConfig.query, chartConfig.metricFunction, @@ -121,7 +120,6 @@ function getChartDetails(job, detectorIndex, entityFields, earliestMs, latestMs) const entityFieldNames = _.map(blankEntityFields, 'fieldName'); ml.getCardinalityOfFields({ index: chartConfig.datafeedConfig.indices, - types: chartConfig.datafeedConfig.types, fieldNames: entityFieldNames, query: chartConfig.datafeedConfig.query, timeFieldName: chartConfig.timeField, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json index 69d5a8651a8f..cdc5ad507a0b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json @@ -3,7 +3,6 @@ "indexes": [ "INDEX_PATTERN_NAME" ], - "types": [], "query": { "match": { "processor.event": { diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_high_count_process_events.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_high_count_process_events.json index 84dea03a0771..3757877c7c7e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_high_count_process_events.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_high_count_process_events.json @@ -3,7 +3,6 @@ "indexes": [ "INDEX_PATTERN_NAME" ], - "types": [], "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_rare_process_activity.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_rare_process_activity.json index 73e42b296ead..cd559af1851e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_rare_process_activity.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_docker/ml/datafeed_docker_rare_process_activity.json @@ -3,7 +3,6 @@ "indexes": [ "INDEX_PATTERN_NAME" ], - "types": [], "query": { "bool": { "must": [ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_high_count_process_events.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_high_count_process_events.json index a9fab9b33f1a..111f08784751 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_high_count_process_events.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_high_count_process_events.json @@ -3,7 +3,6 @@ "indexes": [ "INDEX_PATTERN_NAME" ], - "types": [], "query":{ "bool": { "must":[ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_rare_process_activity.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_rare_process_activity.json index f760e7f2ab08..3f222f7a0616 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_rare_process_activity.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/auditbeat_process_hosts/ml/datafeed_hosts_rare_process_activity.json @@ -3,7 +3,6 @@ "indexes": [ "INDEX_PATTERN_NAME" ], - "types": [], "query":{ "bool": { "must":[