From 0f75573b0fba47296e0b872826706448fdf852cd Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Fri, 20 Aug 2021 14:03:37 +0100 Subject: [PATCH] Exception List Telemetry (#107765) --- .../server/lib/telemetry/constants.ts | 14 ++ .../server/lib/telemetry/endpoint_task.ts | 3 +- .../server/lib/telemetry/helpers.test.ts | 69 +++++++++ .../server/lib/telemetry/helpers.ts | 79 +++++++++- .../server/lib/telemetry/mocks.ts | 6 +- ...sk.test.ts => security_lists_task.test.ts} | 41 +++--- .../lib/telemetry/security_lists_task.ts | 138 ++++++++++++++++++ .../server/lib/telemetry/sender.ts | 39 ++++- .../server/lib/telemetry/trusted_apps_task.ts | 122 ---------------- .../server/lib/telemetry/types.ts | 43 ++++++ 10 files changed, 402 insertions(+), 152 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/constants.ts rename x-pack/plugins/security_solution/server/lib/telemetry/{trusted_apps_task.test.ts => security_lists_task.test.ts} (66%) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts new file mode 100644 index 000000000000..3ef45a554e7a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const TELEMETRY_CHANNEL_LISTS = 'security-lists'; + +export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata'; + +export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception'; + +export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts index 13b4ebf0b3ef..668696f0dce1 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts @@ -25,6 +25,7 @@ import { EndpointPolicyResponseAggregation, EndpointPolicyResponseDocument, } from './types'; +import { TELEMETRY_CHANNEL_ENDPOINT_META } from './constants'; export const TelemetryEndpointTaskConstants = { TIMEOUT: '5m', @@ -326,7 +327,7 @@ export class TelemetryEndpointTask { * Send the documents in a batches of 100 */ batchTelemetryRecords(telemetryPayloads, 100).forEach((telemetryBatch) => - this.sender.sendOnDemand('endpoint-metadata', telemetryBatch) + this.sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, telemetryBatch) ); return telemetryPayloads.length; } catch (err) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts index bee673fc8725..a4d11b71f2a8 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -7,12 +7,17 @@ import moment from 'moment'; import { createMockPackagePolicy } from './mocks'; +import { TrustedApp } from '../../../common/endpoint/types'; +import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER } from './constants'; import { getPreviousDiagTaskTimestamp, getPreviousEpMetaTaskTimestamp, batchTelemetryRecords, isPackagePolicyList, + templateTrustedApps, + templateEndpointExceptions, } from './helpers'; +import { EndpointExceptionListItem } from './types'; describe('test diagnostic telemetry scheduled task timing helper', () => { test('test -5 mins is returned when there is no previous task run', async () => { @@ -125,3 +130,67 @@ describe('test package policy type guard', () => { expect(result).toEqual(true); }); }); + +describe('list telemetry schema', () => { + test('trusted apps document is correctly formed', () => { + const data = [{ id: 'test_1' }] as TrustedApp[]; + const templatedItems = templateTrustedApps(data); + + expect(templatedItems[0]?.trusted_application.length).toEqual(1); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + }); + + test('trusted apps document is correctly formed with multiple entries', () => { + const data = [{ id: 'test_2' }, { id: 'test_2' }] as TrustedApp[]; + const templatedItems = templateTrustedApps(data); + + expect(templatedItems[0]?.trusted_application.length).toEqual(1); + expect(templatedItems[1]?.trusted_application.length).toEqual(1); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + }); + + test('endpoint exception document is correctly formed', () => { + const data = [{ id: 'test_3' }] as EndpointExceptionListItem[]; + const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION); + + expect(templatedItems[0]?.trusted_application.length).toEqual(0); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(1); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + }); + + test('endpoint exception document is correctly formed with multiple entries', () => { + const data = [ + { id: 'test_4' }, + { id: 'test_4' }, + { id: 'test_4' }, + ] as EndpointExceptionListItem[]; + const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION); + + expect(templatedItems[0]?.trusted_application.length).toEqual(0); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(1); + expect(templatedItems[1]?.endpoint_exception.length).toEqual(1); + expect(templatedItems[2]?.endpoint_exception.length).toEqual(1); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0); + }); + + test('endpoint event filters document is correctly formed', () => { + const data = [{ id: 'test_5' }] as EndpointExceptionListItem[]; + const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER); + + expect(templatedItems[0]?.trusted_application.length).toEqual(0); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1); + }); + + test('endpoint event filters document is correctly formed with multiple entries', () => { + const data = [{ id: 'test_6' }, { id: 'test_6' }] as EndpointExceptionListItem[]; + const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER); + + expect(templatedItems[0]?.trusted_application.length).toEqual(0); + expect(templatedItems[0]?.endpoint_exception.length).toEqual(0); + expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1); + expect(templatedItems[1]?.endpoint_event_filter.length).toEqual(1); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index 6af258a4cbe6..bb2cc4f42ca9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -6,7 +6,11 @@ */ import moment from 'moment'; +import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { TrustedApp } from '../../../common/endpoint/types'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; +import { EndpointExceptionListItem, ListTemplate } from './types'; +import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER } from './constants'; /** * Determines the when the last run was in order to execute to. @@ -84,9 +88,82 @@ export function isPackagePolicyList( return (data as PackagePolicy[])[0].inputs !== undefined; } +/** + * Maps Exception list item to parsable object + * + * @param exceptionListItem + * @returns collection of endpoint exceptions + */ +export const exceptionListItemToEndpointEntry = (exceptionListItem: ExceptionListItemSchema) => { + return { + id: exceptionListItem.id, + version: exceptionListItem._version || '', + name: exceptionListItem.name, + description: exceptionListItem.description, + created_at: exceptionListItem.created_at, + created_by: exceptionListItem.created_by, + updated_at: exceptionListItem.updated_at, + updated_by: exceptionListItem.updated_by, + entries: exceptionListItem.entries, + os_types: exceptionListItem.os_types, + } as EndpointExceptionListItem; +}; + +/** + * Constructs the lists telemetry schema from a collection of Trusted Apps + * + * @param listData + * @returns lists telemetry schema + */ +export const templateTrustedApps = (listData: TrustedApp[]) => { + return listData.map((item) => { + const template: ListTemplate = { + trusted_application: [], + endpoint_exception: [], + endpoint_event_filter: [], + }; + + template.trusted_application.push(item); + return template; + }); +}; + +/** + * Consructs the list telemetry schema from a collection of endpoint exceptions + * + * @param listData + * @param listType + * @returns lists telemetry schema + */ +export const templateEndpointExceptions = ( + listData: EndpointExceptionListItem[], + listType: string +) => { + return listData.map((item) => { + const template: ListTemplate = { + trusted_application: [], + endpoint_exception: [], + endpoint_event_filter: [], + }; + + if (listType === LIST_ENDPOINT_EXCEPTION) { + template.endpoint_exception.push(item); + return template; + } + + if (listType === LIST_ENDPOINT_EVENT_FILTER) { + template.endpoint_event_filter.push(item); + return template; + } + + return null; + }); +}; + /** * Convert counter label list to kebab case - * @params label_list the list of labels to create standardized UsageCounter from + * + * @param label_list the list of labels to create standardized UsageCounter from * @returns a string label for usage in the UsageCounter */ export function createUsageCounterLabel(labelList: string[]): string { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index 642be5fc737f..a38042e214ce 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -9,7 +9,7 @@ import { TelemetryEventsSender } from './sender'; import { TelemetryDiagTask } from './diagnostic_task'; import { TelemetryEndpointTask } from './endpoint_task'; -import { TelemetryTrustedAppsTask } from './trusted_apps_task'; +import { TelemetryExceptionListsTask } from './security_lists_task'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; /** @@ -69,8 +69,8 @@ export class MockTelemetryEndpointTask extends TelemetryEndpointTask { } /** - * Creates a mocked Telemetry trusted app Task + * Creates a mocked Telemetry exception lists Task */ -export class MockTelemetryTrustedAppTask extends TelemetryTrustedAppsTask { +export class MockExceptionListsTask extends TelemetryExceptionListsTask { public runTask = jest.fn(); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts similarity index 66% rename from x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts index 5cd67a9c9c21..20d89c9721b2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts @@ -9,10 +9,13 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { TaskStatus } from '../../../../task_manager/server'; import { taskManagerMock } from '../../../../task_manager/server/mocks'; -import { TelemetryTrustedAppsTask, TelemetryTrustedAppsTaskConstants } from './trusted_apps_task'; -import { createMockTelemetryEventsSender, MockTelemetryTrustedAppTask } from './mocks'; +import { + TelemetryExceptionListsTask, + TelemetrySecuityListsTaskConstants, +} from './security_lists_task'; +import { createMockTelemetryEventsSender, MockExceptionListsTask } from './mocks'; -describe('test trusted apps telemetry task functionality', () => { +describe('test exception list telemetry task functionality', () => { let logger: ReturnType; beforeEach(() => { @@ -20,25 +23,25 @@ describe('test trusted apps telemetry task functionality', () => { }); test('the trusted apps task can register', () => { - const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask( + const telemetryTrustedAppsTask = new TelemetryExceptionListsTask( logger, taskManagerMock.createSetup(), createMockTelemetryEventsSender(true) ); - expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryTrustedAppsTask); + expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryExceptionListsTask); }); - test('the trusted apps task should be registered', () => { + test('the exception list task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - new TelemetryTrustedAppsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); + new TelemetryExceptionListsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); }); - test('the trusted apps task should be scheduled', async () => { + test('the exception list task should be scheduled', async () => { const mockTaskManagerSetup = taskManagerMock.createSetup(); - const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask( + const telemetryTrustedAppsTask = new TelemetryExceptionListsTask( logger, mockTaskManagerSetup, createMockTelemetryEventsSender(true) @@ -49,13 +52,13 @@ describe('test trusted apps telemetry task functionality', () => { expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); }); - test('the trusted apps task should not query elastic if telemetry is not opted in', async () => { + test('the exception list task should not query elastic if telemetry is not opted in', async () => { const mockSender = createMockTelemetryEventsSender(false); const mockTaskManager = taskManagerMock.createSetup(); - new MockTelemetryTrustedAppTask(logger, mockTaskManager, mockSender); + new MockExceptionListsTask(logger, mockTaskManager, mockSender); const mockTaskInstance = { - id: TelemetryTrustedAppsTaskConstants.TYPE, + id: TelemetrySecuityListsTaskConstants.TYPE, runAt: new Date(), attempts: 0, ownerId: '', @@ -65,28 +68,28 @@ describe('test trusted apps telemetry task functionality', () => { retryAt: new Date(), params: {}, state: {}, - taskType: TelemetryTrustedAppsTaskConstants.TYPE, + taskType: TelemetrySecuityListsTaskConstants.TYPE, }; const createTaskRunner = mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ - TelemetryTrustedAppsTaskConstants.TYPE + TelemetrySecuityListsTaskConstants.TYPE ].createTaskRunner; const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); await taskRunner.run(); expect(mockSender.fetchTrustedApplications).not.toHaveBeenCalled(); }); - test('the trusted apps task should query elastic if telemetry opted in', async () => { + test('the exception list task should query elastic if telemetry opted in', async () => { const mockSender = createMockTelemetryEventsSender(true); const mockTaskManager = taskManagerMock.createSetup(); - const telemetryTrustedAppsTask = new MockTelemetryTrustedAppTask( + const telemetryTrustedAppsTask = new MockExceptionListsTask( logger, mockTaskManager, mockSender ); const mockTaskInstance = { - id: TelemetryTrustedAppsTaskConstants.TYPE, + id: TelemetrySecuityListsTaskConstants.TYPE, runAt: new Date(), attempts: 0, ownerId: '', @@ -96,11 +99,11 @@ describe('test trusted apps telemetry task functionality', () => { retryAt: new Date(), params: {}, state: {}, - taskType: TelemetryTrustedAppsTaskConstants.TYPE, + taskType: TelemetrySecuityListsTaskConstants.TYPE, }; const createTaskRunner = mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ - TelemetryTrustedAppsTaskConstants.TYPE + TelemetrySecuityListsTaskConstants.TYPE ].createTaskRunner; const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); await taskRunner.run(); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts new file mode 100644 index 000000000000..1c4dc28f1c5a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { Logger } from 'src/core/server'; +import { + ENDPOINT_LIST_ID, + ENDPOINT_EVENT_FILTERS_LIST_ID, +} from '@kbn/securitysolution-list-constants'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../task_manager/server'; +import { + LIST_ENDPOINT_EXCEPTION, + LIST_ENDPOINT_EVENT_FILTER, + TELEMETRY_CHANNEL_LISTS, +} from './constants'; +import { batchTelemetryRecords, templateEndpointExceptions, templateTrustedApps } from './helpers'; +import { TelemetryEventsSender } from './sender'; + +export const TelemetrySecuityListsTaskConstants = { + TIMEOUT: '3m', + TYPE: 'security:telemetry-lists', + INTERVAL: '24h', + VERSION: '1.0.0', +}; + +const MAX_TELEMETRY_BATCH = 1_000; + +export class TelemetryExceptionListsTask { + private readonly logger: Logger; + private readonly sender: TelemetryEventsSender; + + constructor( + logger: Logger, + taskManager: TaskManagerSetupContract, + sender: TelemetryEventsSender + ) { + this.logger = logger; + this.sender = sender; + + taskManager.registerTaskDefinitions({ + [TelemetrySecuityListsTaskConstants.TYPE]: { + title: 'Security Solution Lists Telemetry', + timeout: TelemetrySecuityListsTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const { state } = taskInstance; + + return { + run: async () => { + const taskExecutionTime = moment().utc().toISOString(); + const hits = await this.runTask(taskInstance.id); + + return { + state: { + lastExecutionTimestamp: taskExecutionTime, + runs: (state.runs || 0) + 1, + hits, + }, + }; + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (taskManager: TaskManagerStartContract) => { + try { + await taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: TelemetrySecuityListsTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: TelemetrySecuityListsTaskConstants.INTERVAL, + }, + state: { runs: 0 }, + params: { version: TelemetrySecuityListsTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.error(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${TelemetrySecuityListsTaskConstants.TYPE}:${TelemetrySecuityListsTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string) => { + if (taskId !== this.getTaskId()) { + return 0; + } + + const isOptedIn = await this.sender.isTelemetryOptedIn(); + if (!isOptedIn) { + return 0; + } + + // Lists Telemetry: Trusted Applications + + const trustedApps = await this.sender.fetchTrustedApplications(); + const trustedAppsJson = templateTrustedApps(trustedApps.data); + this.logger.debug(`Trusted Apps: ${trustedAppsJson}`); + + batchTelemetryRecords(trustedAppsJson, MAX_TELEMETRY_BATCH).forEach((batch) => + this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) + ); + + // Lists Telemetry: Endpoint Exceptions + + const epExceptions = await this.sender.fetchEndpointList(ENDPOINT_LIST_ID); + const epExceptionsJson = templateEndpointExceptions(epExceptions.data, LIST_ENDPOINT_EXCEPTION); + this.logger.debug(`EP Exceptions: ${epExceptionsJson}`); + + batchTelemetryRecords(epExceptionsJson, MAX_TELEMETRY_BATCH).forEach((batch) => + this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) + ); + + // Lists Telemetry: Endpoint Event Filters + + const epFilters = await this.sender.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID); + const epFiltersJson = templateEndpointExceptions(epFilters.data, LIST_ENDPOINT_EVENT_FILTER); + this.logger.debug(`EP Event Filters: ${epFiltersJson}`); + + batchTelemetryRecords(epFiltersJson, MAX_TELEMETRY_BATCH).forEach((batch) => + this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) + ); + + return trustedAppsJson.length + epExceptionsJson.length + epFiltersJson.length; + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 5724c61bfcee..c7bb58dd2251 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -18,14 +18,15 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { createUsageCounterLabel } from './helpers'; import { TelemetryDiagTask } from './diagnostic_task'; import { TelemetryEndpointTask } from './endpoint_task'; -import { TelemetryTrustedAppsTask } from './trusted_apps_task'; +import { TelemetryExceptionListsTask } from './security_lists_task'; import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/server'; -import { ExceptionListClient } from '../../../../lists/server'; import { getTrustedAppsList } from '../../endpoint/routes/trusted_apps/service'; +import { ExceptionListClient } from '../../../../lists/server'; +import { GetEndpointListResponse } from './types'; +import { createUsageCounterLabel, exceptionListItemToEndpointEntry } from './helpers'; type BaseSearchTypes = string | number | boolean | object; export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; @@ -63,7 +64,7 @@ export class TelemetryEventsSender { private isOptedIn?: boolean = true; // Assume true until the first check private diagTask?: TelemetryDiagTask; private epMetricsTask?: TelemetryEndpointTask; - private trustedAppsTask?: TelemetryTrustedAppsTask; + private exceptionListTask?: TelemetryExceptionListsTask; private agentService?: AgentService; private agentPolicyService?: AgentPolicyServiceInterface; private esClient?: ElasticsearchClient; @@ -86,7 +87,7 @@ export class TelemetryEventsSender { if (taskManager) { this.diagTask = new TelemetryDiagTask(this.logger, taskManager, this); this.epMetricsTask = new TelemetryEndpointTask(this.logger, taskManager, this); - this.trustedAppsTask = new TelemetryTrustedAppsTask(this.logger, taskManager, this); + this.exceptionListTask = new TelemetryExceptionListsTask(this.logger, taskManager, this); } } @@ -108,7 +109,7 @@ export class TelemetryEventsSender { this.logger.debug(`Starting diagnostic and endpoint telemetry tasks`); this.diagTask.start(taskManager); this.epMetricsTask.start(taskManager); - this.trustedAppsTask?.start(taskManager); + this.exceptionListTask?.start(taskManager); } this.logger.debug(`Starting local task`); @@ -279,6 +280,32 @@ export class TelemetryEventsSender { return getTrustedAppsList(this.exceptionListClient, { page: 1, per_page: 10_000 }); } + public async fetchEndpointList(listId: string): Promise { + if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { + throw Error('could not fetch trusted applications. exception list client not available.'); + } + + // Ensure list is created if it does not exist + await this.exceptionListClient.createTrustedAppsList(); + + const results = await this.exceptionListClient.findExceptionListItem({ + listId, + page: 1, + perPage: this.max_records, + filter: undefined, + namespaceType: 'agnostic', + sortField: 'name', + sortOrder: 'asc', + }); + + return { + data: results?.data.map(exceptionListItemToEndpointEntry) ?? [], + total: results?.total ?? 0, + page: results?.page ?? 1, + per_page: results?.per_page ?? this.max_records, + }; + } + public queueTelemetryEvents(events: TelemetryEvent[]) { const qlength = this.queue.length; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts deleted file mode 100644 index f91f3e8428d0..000000000000 --- a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import moment from 'moment'; -import { Logger } from 'src/core/server'; -import { - ConcreteTaskInstance, - TaskManagerSetupContract, - TaskManagerStartContract, -} from '../../../../task_manager/server'; - -import { getPreviousEpMetaTaskTimestamp, batchTelemetryRecords } from './helpers'; -import { TelemetryEventsSender } from './sender'; - -export const TelemetryTrustedAppsTaskConstants = { - TIMEOUT: '1m', - TYPE: 'security:trusted-apps-telemetry', - INTERVAL: '24h', - VERSION: '1.0.0', -}; - -/** Telemetry Trusted Apps Task - * - * The Trusted Apps task is a daily batch job that collects and transmits non-sensitive - * trusted apps hashes + file paths for supported operating systems. This helps test - * efficacy of our protections. - */ -export class TelemetryTrustedAppsTask { - private readonly logger: Logger; - private readonly sender: TelemetryEventsSender; - - constructor( - logger: Logger, - taskManager: TaskManagerSetupContract, - sender: TelemetryEventsSender - ) { - this.logger = logger; - this.sender = sender; - - taskManager.registerTaskDefinitions({ - [TelemetryTrustedAppsTaskConstants.TYPE]: { - title: 'Security Solution Telemetry Endpoint Metrics and Info task', - timeout: TelemetryTrustedAppsTaskConstants.TIMEOUT, - createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - const { state } = taskInstance; - - return { - run: async () => { - const taskExecutionTime = moment().utc().toISOString(); - const lastExecutionTimestamp = getPreviousEpMetaTaskTimestamp( - taskExecutionTime, - taskInstance.state?.lastExecutionTimestamp - ); - - const hits = await this.runTask( - taskInstance.id, - lastExecutionTimestamp, - taskExecutionTime - ); - - return { - state: { - lastExecutionTimestamp: taskExecutionTime, - runs: (state.runs || 0) + 1, - hits, - }, - }; - }, - cancel: async () => {}, - }; - }, - }, - }); - } - - public start = async (taskManager: TaskManagerStartContract) => { - try { - await taskManager.ensureScheduled({ - id: this.getTaskId(), - taskType: TelemetryTrustedAppsTaskConstants.TYPE, - scope: ['securitySolution'], - schedule: { - interval: TelemetryTrustedAppsTaskConstants.INTERVAL, - }, - state: { runs: 0 }, - params: { version: TelemetryTrustedAppsTaskConstants.VERSION }, - }); - } catch (e) { - this.logger.error(`Error scheduling task, received ${e.message}`); - } - }; - - private getTaskId = (): string => { - return `${TelemetryTrustedAppsTaskConstants.TYPE}:${TelemetryTrustedAppsTaskConstants.VERSION}`; - }; - - public runTask = async (taskId: string, executeFrom: string, executeTo: string) => { - if (taskId !== this.getTaskId()) { - this.logger.debug(`Outdated task running: ${taskId}`); - return 0; - } - - const isOptedIn = await this.sender.isTelemetryOptedIn(); - if (!isOptedIn) { - this.logger.debug(`Telemetry is not opted-in.`); - return 0; - } - - const response = await this.sender.fetchTrustedApplications(); - this.logger.debug(`Trusted Apps: ${response}`); - - batchTelemetryRecords(response.data, 1_000).forEach((telemetryBatch) => - this.sender.sendOnDemand('lists-trustedapps', telemetryBatch) - ); - - return response.data.length; - }; -} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 355393145fa0..d1d7740071e1 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -5,6 +5,9 @@ * 2.0. */ +import { schema, TypeOf } from '@kbn/config-schema'; +import { TrustedApp } from '../../../common/endpoint/types'; + // EP Policy Response export interface EndpointPolicyResponseAggregation { @@ -138,3 +141,43 @@ interface EndpointMetricOS { platform: string; full: string; } + +// List HTTP Types + +export const GetTrustedAppsRequestSchema = { + query: schema.object({ + page: schema.maybe(schema.number({ defaultValue: 1, min: 1 })), + per_page: schema.maybe(schema.number({ defaultValue: 20, min: 1 })), + kuery: schema.maybe(schema.string()), + }), +}; + +export type GetEndpointListRequest = TypeOf; + +export interface GetEndpointListResponse { + per_page: number; + page: number; + total: number; + data: EndpointExceptionListItem[]; +} + +// Telemetry List types + +export interface EndpointExceptionListItem { + id: string; + version: string; + name: string; + description: string; + created_at: string; + created_by: string; + updated_at: string; + updated_by: string; + entries: object; + os_types: object; +} + +export interface ListTemplate { + trusted_application: TrustedApp[]; + endpoint_exception: EndpointExceptionListItem[]; + endpoint_event_filter: EndpointExceptionListItem[]; +}