[Metrics UI] Add track_total_hits to Metric Threshold query to support alerts with over 10K documents (#115465)

* [Metrics UI] Add track_total_hits to Metric Threshold query

* Adding tests

* Making the esArchive smaller

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -120,6 +120,7 @@ export const getElasticsearchMetricQuery = (
   const parsedFilterQuery = getParsedFilterQuery(filterQuery);
 
   return {
+    track_total_hits: true,
     query: {
       bool: {
         filter: [

x-pack/test/api_integration/apis/metrics_ui/constants.ts

@@ -39,4 +39,8 @@ export const DATES = {
       max: 1609545900000, // '2021-01-02T00:05:00Z'
     },
   },
+  ten_thousand_plus: {
+    min: 1634604480001, // 2021-10-19T00:48:00.001Z
+    max: 1634604839997, // 2021-10-19T00:53:59.997Z
+  },
 };

x-pack/test/api_integration/apis/metrics_ui/metric_threshold_alert.ts

@@ -81,10 +81,95 @@ export default function ({ getService }: FtrProviderContext) {
   };
 
   describe('Metric Threshold Alerts Executor', () => {
-    before(() => esArchiver.load('x-pack/test/functional/es_archives/infra/alerts_test_data'));
-    after(() => esArchiver.unload('x-pack/test/functional/es_archives/infra/alerts_test_data'));
-
+    describe('with 10K plus docs', () => {
+      before(() => esArchiver.load('x-pack/test/functional/es_archives/infra/ten_thousand_plus'));
+      after(() => esArchiver.unload('x-pack/test/functional/es_archives/infra/ten_thousand_plus'));
+      describe('without group by', () => {
+        it('should alert on document count', async () => {
+          const params = {
+            ...baseParams,
+            criteria: [
+              {
+                timeSize: 5,
+                timeUnit: 'm',
+                threshold: [10000],
+                comparator: Comparator.LT_OR_EQ,
+                aggType: Aggregators.COUNT,
+              } as CountMetricExpressionParams,
+            ],
+          };
+          const config = {
+            ...configuration,
+            metricAlias: 'filebeat-*',
+          };
+          const timeFrame = { end: DATES.ten_thousand_plus.max };
+          const results = await evaluateAlert(esClient, params, config, [], timeFrame);
+          expect(results).to.eql([
+            {
+              '*': {
+                timeSize: 5,
+                timeUnit: 'm',
+                threshold: [10000],
+                comparator: '<=',
+                aggType: 'count',
+                metric: 'Document count',
+                currentValue: 20895,
+                timestamp: '2021-10-19T00:48:59.997Z',
+                shouldFire: [false],
+                shouldWarn: [false],
+                isNoData: [false],
+                isError: false,
+              },
+            },
+          ]);
+        });
+      });
+      describe('with group by', () => {
+        it('should alert on document count', async () => {
+          const params = {
+            ...baseParams,
+            groupBy: ['event.category'],
+            criteria: [
+              {
+                timeSize: 5,
+                timeUnit: 'm',
+                threshold: [10000],
+                comparator: Comparator.LT_OR_EQ,
+                aggType: Aggregators.COUNT,
+              } as CountMetricExpressionParams,
+            ],
+          };
+          const config = {
+            ...configuration,
+            metricAlias: 'filebeat-*',
+          };
+          const timeFrame = { end: DATES.ten_thousand_plus.max };
+          const results = await evaluateAlert(esClient, params, config, [], timeFrame);
+          expect(results).to.eql([
+            {
+              web: {
+                timeSize: 5,
+                timeUnit: 'm',
+                threshold: [10000],
+                comparator: '<=',
+                aggType: 'count',
+                metric: 'Document count',
+                currentValue: 20895,
+                timestamp: '2021-10-19T00:48:59.997Z',
+                shouldFire: [false],
+                shouldWarn: [false],
+                isNoData: [false],
+                isError: false,
+              },
+            },
+          ]);
+        });
+      });
+    });
     describe('with gauge data', () => {
+      before(() => esArchiver.load('x-pack/test/functional/es_archives/infra/alerts_test_data'));
+      after(() => esArchiver.unload('x-pack/test/functional/es_archives/infra/alerts_test_data'));
+
       describe('without groupBy', () => {
         it('should alert on document count', async () => {
           const params = {
@@ -285,6 +370,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     describe('with rate data', () => {
+      before(() => esArchiver.load('x-pack/test/functional/es_archives/infra/alerts_test_data'));
+      after(() => esArchiver.unload('x-pack/test/functional/es_archives/infra/alerts_test_data'));
       describe('without groupBy', () => {
         it('should alert on rate', async () => {
           const params = {