From fe79979328b38b1394a945ff47b8194188238f3f Mon Sep 17 00:00:00 2001 From: Robert Oskamp Date: Mon, 27 Apr 2020 15:13:13 +0200 Subject: [PATCH] [ML] Functional tests - stabilize job row details validations (#64503) This PR stabilizes the AD job row details check by only validating expected properties instead of the whole object. --- .../anomaly_detection/advanced_job.ts | 4 -- .../anomaly_detection/categorization_job.ts | 1 - .../anomaly_detection/date_nanos_job.ts | 2 - .../anomaly_detection/multi_metric_job.ts | 1 - .../anomaly_detection/population_job.ts | 1 - .../anomaly_detection/saved_search_job.ts | 6 --- .../anomaly_detection/single_metric_job.ts | 1 - .../functional/apps/machine_learning/pages.ts | 2 +- .../services/machine_learning/job_table.ts | 52 ++++++------------- 9 files changed, 17 insertions(+), 53 deletions(-) diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts index e238937ee845..a9133bb38017 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts @@ -195,7 +195,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '10.0 MB', total_by_field_count: '37', total_over_field_count: '92', total_partition_field_count: '8', @@ -262,7 +261,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '100.0 MB', total_by_field_count: '994', total_over_field_count: '0', total_partition_field_count: '2', @@ -549,7 +547,6 @@ export default function({ getService }: FtrProviderContext) { job_id: testData.jobId, result_type: testData.expected.modelSizeStats.result_type, model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded, - model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit, total_by_field_count: testData.expected.modelSizeStats.total_by_field_count, total_over_field_count: testData.expected.modelSizeStats.total_over_field_count, total_partition_field_count: @@ -813,7 +810,6 @@ export default function({ getService }: FtrProviderContext) { job_id: testData.jobIdClone, result_type: testData.expected.modelSizeStats.result_type, model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded, - model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit, total_by_field_count: testData.expected.modelSizeStats.total_by_field_count, total_over_field_count: testData.expected.modelSizeStats.total_over_field_count, total_partition_field_count: diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts index 5a39b06b4c15..b8ac646c3587 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts @@ -64,7 +64,6 @@ export default function({ getService }: FtrProviderContext) { job_id: expectedJobId, result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '15.0 MB', total_by_field_count: '30', total_over_field_count: '0', total_partition_field_count: '2', diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts index 96b69c10f4c2..d3934d674124 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts @@ -153,7 +153,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '10.0 MB', total_by_field_count: '3', total_over_field_count: '0', total_partition_field_count: '2', @@ -422,7 +421,6 @@ export default function({ getService }: FtrProviderContext) { job_id: testData.jobId, result_type: testData.expected.modelSizeStats.result_type, model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded, - model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit, total_by_field_count: testData.expected.modelSizeStats.total_by_field_count, total_over_field_count: testData.expected.modelSizeStats.total_over_field_count, total_partition_field_count: diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts index 45b03b3d179b..f6a9c96492f3 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts @@ -61,7 +61,6 @@ export default function({ getService }: FtrProviderContext) { job_id: expectedJobId, result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '59', total_over_field_count: '0', total_partition_field_count: '58', diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts index a9c94a650898..547c489411b5 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts @@ -75,7 +75,6 @@ export default function({ getService }: FtrProviderContext) { job_id: expectedJobId, result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '8.0 MB', total_by_field_count: '25', total_over_field_count: '92', total_partition_field_count: '3', diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts index 862904862cb1..b1fee1633641 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts @@ -54,7 +54,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '3', total_over_field_count: '0', total_partition_field_count: '2', @@ -105,7 +104,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '7', total_over_field_count: '0', total_partition_field_count: '6', @@ -156,7 +154,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '7', total_over_field_count: '0', total_partition_field_count: '6', @@ -208,7 +205,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '3', total_over_field_count: '0', total_partition_field_count: '2', @@ -259,7 +255,6 @@ export default function({ getService }: FtrProviderContext) { modelSizeStats: { result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '20.0 MB', total_by_field_count: '3', total_over_field_count: '0', total_partition_field_count: '2', @@ -464,7 +459,6 @@ export default function({ getService }: FtrProviderContext) { job_id: testData.jobId, result_type: testData.expected.modelSizeStats.result_type, model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded, - model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit, total_by_field_count: testData.expected.modelSizeStats.total_by_field_count, total_over_field_count: testData.expected.modelSizeStats.total_over_field_count, total_partition_field_count: diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts index 46fbc4013948..0f8655e3c6bb 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts @@ -60,7 +60,6 @@ export default function({ getService }: FtrProviderContext) { job_id: expectedJobId, result_type: 'model_size_stats', model_bytes_exceeded: '0.0 B', - model_bytes_memory_limit: '15.0 MB', total_by_field_count: '3', total_over_field_count: '0', total_partition_field_count: '2', diff --git a/x-pack/test/functional/apps/machine_learning/pages.ts b/x-pack/test/functional/apps/machine_learning/pages.ts index 3c3698792ff7..35536b4eeda1 100644 --- a/x-pack/test/functional/apps/machine_learning/pages.ts +++ b/x-pack/test/functional/apps/machine_learning/pages.ts @@ -10,7 +10,7 @@ export default function({ getService }: FtrProviderContext) { const ml = getService('ml'); describe('page navigation', function() { - this.tags(['includeFirefox', 'mlqa']); + this.tags(['skipFirefox', 'mlqa']); before(async () => { await ml.api.cleanMlIndices(); await ml.securityUI.loginAsMlPowerUser(); diff --git a/x-pack/test/functional/services/machine_learning/job_table.ts b/x-pack/test/functional/services/machine_learning/job_table.ts index 0e638963f236..e2451328ea94 100644 --- a/x-pack/test/functional/services/machine_learning/job_table.ts +++ b/x-pack/test/functional/services/machine_learning/job_table.ts @@ -187,44 +187,24 @@ export function MachineLearningJobTableProvider({ getService }: FtrProviderConte expectedCounts: object, expectedModelSizeStats: object ) { - const countDetails = await this.parseJobCounts(jobId); - const counts = countDetails.counts; + const { counts, modelSizeStats } = await this.parseJobCounts(jobId); - // fields that have changing values are only validated - // to be present and then removed so they don't make - // the object validation fail - expect(counts).to.have.property('last_data_time'); - delete counts.last_data_time; + // Only check for expected keys / values, ignore additional properties + // This way the tests stay stable when new properties are added on the ES side + for (const [key, value] of Object.entries(expectedCounts)) { + expect(counts) + .to.have.property(key) + .eql(value, `Expected counts property '${key}' to exist with value '${value}'`); + } - expect(counts).to.eql(expectedCounts); - - const modelSizeStats = countDetails.modelSizeStats; - - // fields that have changing values are only validated - // to be present and then removed so they don't make - // the object validation fail - expect(modelSizeStats).to.have.property('log_time'); - delete modelSizeStats.log_time; - expect(modelSizeStats).to.have.property('model_bytes'); - delete modelSizeStats.model_bytes; - - // remove categorization fields from validation until - // the ES version is updated - delete modelSizeStats.categorization_status; - delete modelSizeStats.categorized_doc_count; - delete modelSizeStats.dead_category_count; - delete modelSizeStats.frequent_category_count; - delete modelSizeStats.rare_category_count; - delete modelSizeStats.total_category_count; - - // MML during clone has changed in #61589 - // TODO: adjust test code to reflect the new behavior - expect(modelSizeStats).to.have.property('model_bytes_memory_limit'); - delete modelSizeStats.model_bytes_memory_limit; - // @ts-ignore - delete expectedModelSizeStats.model_bytes_memory_limit; - - expect(modelSizeStats).to.eql(expectedModelSizeStats); + for (const [key, value] of Object.entries(expectedModelSizeStats)) { + expect(modelSizeStats) + .to.have.property(key) + .eql( + value, + `Expected model size stats property '${key}' to exist with value '${value}')` + ); + } } public async clickActionsMenu(jobId: string) {