[Security Solution] Adding tests for endpoint package pipelines (#73703)

* Adding tests for endpoint package pipelines

* Removing content type check on types that can change based on docker image version

* Skipping ingest tests instead of remove expect

* Switching ingest tests over to use application/json

* Removing country names

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/test/ingest_manager_api_integration/apis/epm/file.ts

@@ -47,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) {
             '/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/visualization/sample_visualization.json'
           )
           .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'text/plain; charset=utf-8')
+          .expect('Content-Type', 'application/json; charset=utf-8')
           .expect(200);
       } else {
         warnAndSkipTest(this, log);
@@ -61,7 +61,7 @@ export default function ({ getService }: FtrProviderContext) {
             '/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/dashboard/sample_dashboard.json'
           )
           .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'text/plain; charset=utf-8')
+          .expect('Content-Type', 'application/json; charset=utf-8')
           .expect(200);
       } else {
         warnAndSkipTest(this, log);
@@ -73,7 +73,7 @@ export default function ({ getService }: FtrProviderContext) {
         await supertest
           .get('/api/ingest_manager/epm/packages/filetest/0.1.0/kibana/search/sample_search.json')
           .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'text/plain; charset=utf-8')
+          .expect('Content-Type', 'application/json; charset=utf-8')
           .expect(200);
       } else {
         warnAndSkipTest(this, log);

x-pack/test/ingest_manager_api_integration/config.ts

@@ -10,9 +10,9 @@ import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { defineDockerServersConfig } from '@kbn/test';
 
 // Docker image to use for Ingest Manager API integration tests.
-// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit/48f3935a72b0c5aacc6fec8ef36d559b089a238b
+// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit
 export const dockerImage =
-  'docker.elastic.co/package-registry/distribution:48f3935a72b0c5aacc6fec8ef36d559b089a238b';
+  'docker.elastic.co/package-registry/distribution:80e93ade87f65e18d487b1c407406825915daba8';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));

x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml

@@ -1,2 +1,4 @@
 package_paths:
   - /packages/production
+  - /packages/staging
+  - /packages/snapshot

x-pack/test/security_solution_endpoint_api_int/apis/index.ts

@@ -31,5 +31,6 @@ export default function endpointAPIIntegrationTests(providerContext: FtrProvider
     loadTestFile(require.resolve('./metadata'));
     loadTestFile(require.resolve('./policy'));
     loadTestFile(require.resolve('./artifacts'));
+    loadTestFile(require.resolve('./package'));
   });
 }

x-pack/test/security_solution_endpoint_api_int/apis/package.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import expect from '@kbn/expect';
+import { SearchResponse } from 'elasticsearch';
+import { eventsIndexPattern } from '../../../plugins/security_solution/common/endpoint/constants';
+import {
+  EndpointDocGenerator,
+  Event,
+} from '../../../plugins/security_solution/common/endpoint/generate_data';
+import { FtrProviderContext } from '../ftr_provider_context';
+import { InsertedEvents, processEventsIndex } from '../services/resolver';
+
+interface EventIngested {
+  event: {
+    ingested: number;
+  };
+}
+
+interface NetworkEvent {
+  source: {
+    geo?: {
+      country_name: string;
+    };
+  };
+  destination: {
+    geo?: {
+      country_name: string;
+    };
+  };
+}
+
+const networkIndex = 'logs-endpoint.events.network-default';
+
+export default function ({ getService }: FtrProviderContext) {
+  const resolver = getService('resolverGenerator');
+  const es = getService('es');
+  const generator = new EndpointDocGenerator('data');
+
+  const searchForID = async <T>(id: string) => {
+    return es.search<SearchResponse<T>>({
+      index: eventsIndexPattern,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {
+                ids: {
+                  values: id,
+                },
+              },
+            ],
+          },
+        },
+      },
+    });
+  };
+
+  describe('Endpoint package', () => {
+    describe('ingested processor', () => {
+      let event: Event;
+      let genData: InsertedEvents;
+
+      before(async () => {
+        event = generator.generateEvent();
+        genData = await resolver.insertEvents([event]);
+      });
+
+      after(async () => {
+        await resolver.deleteData(genData);
+      });
+
+      it('sets the event.ingested field', async () => {
+        const resp = await searchForID<EventIngested>(genData.eventsInfo[0]._id);
+        expect(resp.body.hits.hits[0]._source.event.ingested).to.not.be(undefined);
+      });
+    });
+
+    describe('geoip processor', () => {
+      let processIndexData: InsertedEvents;
+      let networkIndexData: InsertedEvents;
+
+      before(async () => {
+        // 46.239.193.5 should be in Iceland
+        // 8.8.8.8 should be in the US
+        const eventWithBothIPs = generator.generateEvent({
+          extensions: { source: { ip: '8.8.8.8' }, destination: { ip: '46.239.193.5' } },
+        });
+
+        const eventWithSourceOnly = generator.generateEvent({
+          extensions: { source: { ip: '8.8.8.8' } },
+        });
+        networkIndexData = await resolver.insertEvents(
+          [eventWithBothIPs, eventWithSourceOnly],
+          networkIndex
+        );
+
+        processIndexData = await resolver.insertEvents([eventWithBothIPs], processEventsIndex);
+      });
+
+      after(async () => {
+        await resolver.deleteData(networkIndexData);
+        await resolver.deleteData(processIndexData);
+      });
+
+      it('sets the geoip fields', async () => {
+        const eventWithBothIPs = await searchForID<NetworkEvent>(
+          networkIndexData.eventsInfo[0]._id
+        );
+        // Should be 'United States'
+        expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo?.country_name).to.not.be(
+          undefined
+        );
+        // should be 'Iceland'
+        expect(eventWithBothIPs.body.hits.hits[0]._source.destination.geo?.country_name).to.not.be(
+          undefined
+        );
+
+        const eventWithSourceOnly = await searchForID<NetworkEvent>(
+          networkIndexData.eventsInfo[1]._id
+        );
+        // Should be 'United States'
+        expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo?.country_name).to.not.be(
+          undefined
+        );
+        expect(eventWithSourceOnly.body.hits.hits[0]._source.destination?.geo).to.be(undefined);
+      });
+
+      it('does not set geoip fields for events in indices other than the network index', async () => {
+        const eventWithBothIPs = await searchForID<NetworkEvent>(
+          processIndexData.eventsInfo[0]._id
+        );
+        expect(eventWithBothIPs.body.hits.hits[0]._source.source.geo).to.be(undefined);
+        expect(eventWithBothIPs.body.hits.hits[0]._source.destination.geo).to.be(undefined);
+      });
+    });
+  });
+}

x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts

@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import expect from '@kbn/expect';
-import { SearchResponse } from 'elasticsearch';
 import { eventsIndexPattern } from '../../../../plugins/security_solution/common/endpoint/constants';
 import {
   ResolverTree,
@@ -20,7 +19,6 @@ import { InsertedEvents } from '../../services/resolver';
 export default function resolverAPIIntegrationTests({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const resolver = getService('resolverGenerator');
-  const es = getService('es');
   const generator = new EndpointDocGenerator('resolver');
 
   describe('Resolver handling of entity ids', () => {
@@ -38,26 +36,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC
       });
 
       it('excludes events that have an empty entity_id field', async () => {
-        // first lets get the _id of the document using the parent.process.entity_id
-        // then we'll use the API to search for that specific document
-        const res = await es.search<SearchResponse<Event>>({
-          index: genData.indices[0],
-          body: {
-            query: {
-              bool: {
-                filter: [
-                  {
-                    term: { 'process.parent.entity_id': origin.process.parent!.entity_id },
-                  },
-                ],
-              },
-            },
-          },
-        });
         const { body }: { body: ResolverEntityIndex } = await supertest.get(
           // using the same indices value here twice to force the query parameter to be an array
           // for some reason using supertest's query() function doesn't construct a parsable array
-          `/api/endpoint/resolver/entity?_id=${res.body.hits.hits[0]._id}&indices=${eventsIndexPattern}&indices=${eventsIndexPattern}`
+          `/api/endpoint/resolver/entity?_id=${genData.eventsInfo[0]._id}&indices=${eventsIndexPattern}&indices=${eventsIndexPattern}`
         );
         expect(body).to.be.empty();
       });

x-pack/test/security_solution_endpoint_api_int/services/resolver.ts

@@ -11,7 +11,7 @@ import {
 } from '../../../plugins/security_solution/common/endpoint/generate_data';
 import { FtrProviderContext } from '../ftr_provider_context';
 
-const processIndex = 'logs-endpoint.events.process-default';
+export const processEventsIndex = 'logs-endpoint.events.process-default';
 
 /**
  * Options for build a resolver tree
@@ -36,7 +36,7 @@ export interface GeneratedTrees {
  * Structure containing the events inserted into ES and the index they live in
  */
 export interface InsertedEvents {
-  events: Event[];
+  eventsInfo: Array<{ _id: string; event: Event }>;
   indices: string[];
 }
 
@@ -46,24 +46,37 @@ interface BulkCreateHeader {
   };
 }
 
+interface BulkResponse {
+  items: Array<{
+    create: {
+      _id: string;
+    };
+  }>;
+}
+
 export function ResolverGeneratorProvider({ getService }: FtrProviderContext) {
   const client = getService('es');
 
   return {
     async insertEvents(
       events: Event[],
-      eventsIndex: string = processIndex
+      eventsIndex: string = processEventsIndex
     ): Promise<InsertedEvents> {
       const body = events.reduce((array: Array<BulkCreateHeader | Event>, doc) => {
         array.push({ create: { _index: eventsIndex } }, doc);
         return array;
       }, []);
-      await client.bulk({ body, refresh: true });
-      return { events, indices: [eventsIndex] };
+      const bulkResp = await client.bulk<BulkResponse>({ body, refresh: true });
+
+      const eventsInfo = events.map((event: Event, i: number) => {
+        return { event, _id: bulkResp.body.items[i].create._id };
+      });
+
+      return { eventsInfo, indices: [eventsIndex] };
     },
     async createTrees(
       options: Options,
-      eventsIndex: string = processIndex,
+      eventsIndex: string = processEventsIndex,
       alertsIndex: string = 'logs-endpoint.alerts-default'
     ): Promise<GeneratedTrees> {
       const seed = options.seed || 'resolver-seed';