Exception List Telemetry (#107765)
This commit is contained in:
parent
fcd89703f7
commit
0f75573b0f
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const TELEMETRY_CHANNEL_LISTS = 'security-lists';
|
||||
|
||||
export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata';
|
||||
|
||||
export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception';
|
||||
|
||||
export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter';
|
|
@ -25,6 +25,7 @@ import {
|
|||
EndpointPolicyResponseAggregation,
|
||||
EndpointPolicyResponseDocument,
|
||||
} from './types';
|
||||
import { TELEMETRY_CHANNEL_ENDPOINT_META } from './constants';
|
||||
|
||||
export const TelemetryEndpointTaskConstants = {
|
||||
TIMEOUT: '5m',
|
||||
|
@ -326,7 +327,7 @@ export class TelemetryEndpointTask {
|
|||
* Send the documents in a batches of 100
|
||||
*/
|
||||
batchTelemetryRecords(telemetryPayloads, 100).forEach((telemetryBatch) =>
|
||||
this.sender.sendOnDemand('endpoint-metadata', telemetryBatch)
|
||||
this.sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, telemetryBatch)
|
||||
);
|
||||
return telemetryPayloads.length;
|
||||
} catch (err) {
|
||||
|
|
|
@ -7,12 +7,17 @@
|
|||
|
||||
import moment from 'moment';
|
||||
import { createMockPackagePolicy } from './mocks';
|
||||
import { TrustedApp } from '../../../common/endpoint/types';
|
||||
import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER } from './constants';
|
||||
import {
|
||||
getPreviousDiagTaskTimestamp,
|
||||
getPreviousEpMetaTaskTimestamp,
|
||||
batchTelemetryRecords,
|
||||
isPackagePolicyList,
|
||||
templateTrustedApps,
|
||||
templateEndpointExceptions,
|
||||
} from './helpers';
|
||||
import { EndpointExceptionListItem } from './types';
|
||||
|
||||
describe('test diagnostic telemetry scheduled task timing helper', () => {
|
||||
test('test -5 mins is returned when there is no previous task run', async () => {
|
||||
|
@ -125,3 +130,67 @@ describe('test package policy type guard', () => {
|
|||
expect(result).toEqual(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('list telemetry schema', () => {
|
||||
test('trusted apps document is correctly formed', () => {
|
||||
const data = [{ id: 'test_1' }] as TrustedApp[];
|
||||
const templatedItems = templateTrustedApps(data);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(1);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
|
||||
});
|
||||
|
||||
test('trusted apps document is correctly formed with multiple entries', () => {
|
||||
const data = [{ id: 'test_2' }, { id: 'test_2' }] as TrustedApp[];
|
||||
const templatedItems = templateTrustedApps(data);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(1);
|
||||
expect(templatedItems[1]?.trusted_application.length).toEqual(1);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
|
||||
});
|
||||
|
||||
test('endpoint exception document is correctly formed', () => {
|
||||
const data = [{ id: 'test_3' }] as EndpointExceptionListItem[];
|
||||
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
|
||||
});
|
||||
|
||||
test('endpoint exception document is correctly formed with multiple entries', () => {
|
||||
const data = [
|
||||
{ id: 'test_4' },
|
||||
{ id: 'test_4' },
|
||||
{ id: 'test_4' },
|
||||
] as EndpointExceptionListItem[];
|
||||
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
|
||||
expect(templatedItems[1]?.endpoint_exception.length).toEqual(1);
|
||||
expect(templatedItems[2]?.endpoint_exception.length).toEqual(1);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
|
||||
});
|
||||
|
||||
test('endpoint event filters document is correctly formed', () => {
|
||||
const data = [{ id: 'test_5' }] as EndpointExceptionListItem[];
|
||||
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1);
|
||||
});
|
||||
|
||||
test('endpoint event filters document is correctly formed with multiple entries', () => {
|
||||
const data = [{ id: 'test_6' }, { id: 'test_6' }] as EndpointExceptionListItem[];
|
||||
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER);
|
||||
|
||||
expect(templatedItems[0]?.trusted_application.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
|
||||
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1);
|
||||
expect(templatedItems[1]?.endpoint_event_filter.length).toEqual(1);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -6,7 +6,11 @@
|
|||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
|
||||
import { TrustedApp } from '../../../common/endpoint/types';
|
||||
import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
|
||||
import { EndpointExceptionListItem, ListTemplate } from './types';
|
||||
import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER } from './constants';
|
||||
|
||||
/**
|
||||
* Determines the when the last run was in order to execute to.
|
||||
|
@ -84,9 +88,82 @@ export function isPackagePolicyList(
|
|||
return (data as PackagePolicy[])[0].inputs !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps Exception list item to parsable object
|
||||
*
|
||||
* @param exceptionListItem
|
||||
* @returns collection of endpoint exceptions
|
||||
*/
|
||||
export const exceptionListItemToEndpointEntry = (exceptionListItem: ExceptionListItemSchema) => {
|
||||
return {
|
||||
id: exceptionListItem.id,
|
||||
version: exceptionListItem._version || '',
|
||||
name: exceptionListItem.name,
|
||||
description: exceptionListItem.description,
|
||||
created_at: exceptionListItem.created_at,
|
||||
created_by: exceptionListItem.created_by,
|
||||
updated_at: exceptionListItem.updated_at,
|
||||
updated_by: exceptionListItem.updated_by,
|
||||
entries: exceptionListItem.entries,
|
||||
os_types: exceptionListItem.os_types,
|
||||
} as EndpointExceptionListItem;
|
||||
};
|
||||
|
||||
/**
|
||||
* Constructs the lists telemetry schema from a collection of Trusted Apps
|
||||
*
|
||||
* @param listData
|
||||
* @returns lists telemetry schema
|
||||
*/
|
||||
export const templateTrustedApps = (listData: TrustedApp[]) => {
|
||||
return listData.map((item) => {
|
||||
const template: ListTemplate = {
|
||||
trusted_application: [],
|
||||
endpoint_exception: [],
|
||||
endpoint_event_filter: [],
|
||||
};
|
||||
|
||||
template.trusted_application.push(item);
|
||||
return template;
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Consructs the list telemetry schema from a collection of endpoint exceptions
|
||||
*
|
||||
* @param listData
|
||||
* @param listType
|
||||
* @returns lists telemetry schema
|
||||
*/
|
||||
export const templateEndpointExceptions = (
|
||||
listData: EndpointExceptionListItem[],
|
||||
listType: string
|
||||
) => {
|
||||
return listData.map((item) => {
|
||||
const template: ListTemplate = {
|
||||
trusted_application: [],
|
||||
endpoint_exception: [],
|
||||
endpoint_event_filter: [],
|
||||
};
|
||||
|
||||
if (listType === LIST_ENDPOINT_EXCEPTION) {
|
||||
template.endpoint_exception.push(item);
|
||||
return template;
|
||||
}
|
||||
|
||||
if (listType === LIST_ENDPOINT_EVENT_FILTER) {
|
||||
template.endpoint_event_filter.push(item);
|
||||
return template;
|
||||
}
|
||||
|
||||
return null;
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Convert counter label list to kebab case
|
||||
* @params label_list the list of labels to create standardized UsageCounter from
|
||||
*
|
||||
* @param label_list the list of labels to create standardized UsageCounter from
|
||||
* @returns a string label for usage in the UsageCounter
|
||||
*/
|
||||
export function createUsageCounterLabel(labelList: string[]): string {
|
||||
|
|
|
@ -9,7 +9,7 @@
|
|||
import { TelemetryEventsSender } from './sender';
|
||||
import { TelemetryDiagTask } from './diagnostic_task';
|
||||
import { TelemetryEndpointTask } from './endpoint_task';
|
||||
import { TelemetryTrustedAppsTask } from './trusted_apps_task';
|
||||
import { TelemetryExceptionListsTask } from './security_lists_task';
|
||||
import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
|
||||
|
||||
/**
|
||||
|
@ -69,8 +69,8 @@ export class MockTelemetryEndpointTask extends TelemetryEndpointTask {
|
|||
}
|
||||
|
||||
/**
|
||||
* Creates a mocked Telemetry trusted app Task
|
||||
* Creates a mocked Telemetry exception lists Task
|
||||
*/
|
||||
export class MockTelemetryTrustedAppTask extends TelemetryTrustedAppsTask {
|
||||
export class MockExceptionListsTask extends TelemetryExceptionListsTask {
|
||||
public runTask = jest.fn();
|
||||
}
|
||||
|
|
|
@ -9,10 +9,13 @@ import { loggingSystemMock } from 'src/core/server/mocks';
|
|||
import { TaskStatus } from '../../../../task_manager/server';
|
||||
import { taskManagerMock } from '../../../../task_manager/server/mocks';
|
||||
|
||||
import { TelemetryTrustedAppsTask, TelemetryTrustedAppsTaskConstants } from './trusted_apps_task';
|
||||
import { createMockTelemetryEventsSender, MockTelemetryTrustedAppTask } from './mocks';
|
||||
import {
|
||||
TelemetryExceptionListsTask,
|
||||
TelemetrySecuityListsTaskConstants,
|
||||
} from './security_lists_task';
|
||||
import { createMockTelemetryEventsSender, MockExceptionListsTask } from './mocks';
|
||||
|
||||
describe('test trusted apps telemetry task functionality', () => {
|
||||
describe('test exception list telemetry task functionality', () => {
|
||||
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
|
||||
|
||||
beforeEach(() => {
|
||||
|
@ -20,25 +23,25 @@ describe('test trusted apps telemetry task functionality', () => {
|
|||
});
|
||||
|
||||
test('the trusted apps task can register', () => {
|
||||
const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask(
|
||||
const telemetryTrustedAppsTask = new TelemetryExceptionListsTask(
|
||||
logger,
|
||||
taskManagerMock.createSetup(),
|
||||
createMockTelemetryEventsSender(true)
|
||||
);
|
||||
|
||||
expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryTrustedAppsTask);
|
||||
expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryExceptionListsTask);
|
||||
});
|
||||
|
||||
test('the trusted apps task should be registered', () => {
|
||||
test('the exception list task should be registered', () => {
|
||||
const mockTaskManager = taskManagerMock.createSetup();
|
||||
new TelemetryTrustedAppsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true));
|
||||
new TelemetryExceptionListsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true));
|
||||
|
||||
expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('the trusted apps task should be scheduled', async () => {
|
||||
test('the exception list task should be scheduled', async () => {
|
||||
const mockTaskManagerSetup = taskManagerMock.createSetup();
|
||||
const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask(
|
||||
const telemetryTrustedAppsTask = new TelemetryExceptionListsTask(
|
||||
logger,
|
||||
mockTaskManagerSetup,
|
||||
createMockTelemetryEventsSender(true)
|
||||
|
@ -49,13 +52,13 @@ describe('test trusted apps telemetry task functionality', () => {
|
|||
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('the trusted apps task should not query elastic if telemetry is not opted in', async () => {
|
||||
test('the exception list task should not query elastic if telemetry is not opted in', async () => {
|
||||
const mockSender = createMockTelemetryEventsSender(false);
|
||||
const mockTaskManager = taskManagerMock.createSetup();
|
||||
new MockTelemetryTrustedAppTask(logger, mockTaskManager, mockSender);
|
||||
new MockExceptionListsTask(logger, mockTaskManager, mockSender);
|
||||
|
||||
const mockTaskInstance = {
|
||||
id: TelemetryTrustedAppsTaskConstants.TYPE,
|
||||
id: TelemetrySecuityListsTaskConstants.TYPE,
|
||||
runAt: new Date(),
|
||||
attempts: 0,
|
||||
ownerId: '',
|
||||
|
@ -65,28 +68,28 @@ describe('test trusted apps telemetry task functionality', () => {
|
|||
retryAt: new Date(),
|
||||
params: {},
|
||||
state: {},
|
||||
taskType: TelemetryTrustedAppsTaskConstants.TYPE,
|
||||
taskType: TelemetrySecuityListsTaskConstants.TYPE,
|
||||
};
|
||||
const createTaskRunner =
|
||||
mockTaskManager.registerTaskDefinitions.mock.calls[0][0][
|
||||
TelemetryTrustedAppsTaskConstants.TYPE
|
||||
TelemetrySecuityListsTaskConstants.TYPE
|
||||
].createTaskRunner;
|
||||
const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance });
|
||||
await taskRunner.run();
|
||||
expect(mockSender.fetchTrustedApplications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('the trusted apps task should query elastic if telemetry opted in', async () => {
|
||||
test('the exception list task should query elastic if telemetry opted in', async () => {
|
||||
const mockSender = createMockTelemetryEventsSender(true);
|
||||
const mockTaskManager = taskManagerMock.createSetup();
|
||||
const telemetryTrustedAppsTask = new MockTelemetryTrustedAppTask(
|
||||
const telemetryTrustedAppsTask = new MockExceptionListsTask(
|
||||
logger,
|
||||
mockTaskManager,
|
||||
mockSender
|
||||
);
|
||||
|
||||
const mockTaskInstance = {
|
||||
id: TelemetryTrustedAppsTaskConstants.TYPE,
|
||||
id: TelemetrySecuityListsTaskConstants.TYPE,
|
||||
runAt: new Date(),
|
||||
attempts: 0,
|
||||
ownerId: '',
|
||||
|
@ -96,11 +99,11 @@ describe('test trusted apps telemetry task functionality', () => {
|
|||
retryAt: new Date(),
|
||||
params: {},
|
||||
state: {},
|
||||
taskType: TelemetryTrustedAppsTaskConstants.TYPE,
|
||||
taskType: TelemetrySecuityListsTaskConstants.TYPE,
|
||||
};
|
||||
const createTaskRunner =
|
||||
mockTaskManager.registerTaskDefinitions.mock.calls[0][0][
|
||||
TelemetryTrustedAppsTaskConstants.TYPE
|
||||
TelemetrySecuityListsTaskConstants.TYPE
|
||||
].createTaskRunner;
|
||||
const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance });
|
||||
await taskRunner.run();
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import { Logger } from 'src/core/server';
|
||||
import {
|
||||
ENDPOINT_LIST_ID,
|
||||
ENDPOINT_EVENT_FILTERS_LIST_ID,
|
||||
} from '@kbn/securitysolution-list-constants';
|
||||
import {
|
||||
ConcreteTaskInstance,
|
||||
TaskManagerSetupContract,
|
||||
TaskManagerStartContract,
|
||||
} from '../../../../task_manager/server';
|
||||
import {
|
||||
LIST_ENDPOINT_EXCEPTION,
|
||||
LIST_ENDPOINT_EVENT_FILTER,
|
||||
TELEMETRY_CHANNEL_LISTS,
|
||||
} from './constants';
|
||||
import { batchTelemetryRecords, templateEndpointExceptions, templateTrustedApps } from './helpers';
|
||||
import { TelemetryEventsSender } from './sender';
|
||||
|
||||
export const TelemetrySecuityListsTaskConstants = {
|
||||
TIMEOUT: '3m',
|
||||
TYPE: 'security:telemetry-lists',
|
||||
INTERVAL: '24h',
|
||||
VERSION: '1.0.0',
|
||||
};
|
||||
|
||||
const MAX_TELEMETRY_BATCH = 1_000;
|
||||
|
||||
export class TelemetryExceptionListsTask {
|
||||
private readonly logger: Logger;
|
||||
private readonly sender: TelemetryEventsSender;
|
||||
|
||||
constructor(
|
||||
logger: Logger,
|
||||
taskManager: TaskManagerSetupContract,
|
||||
sender: TelemetryEventsSender
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.sender = sender;
|
||||
|
||||
taskManager.registerTaskDefinitions({
|
||||
[TelemetrySecuityListsTaskConstants.TYPE]: {
|
||||
title: 'Security Solution Lists Telemetry',
|
||||
timeout: TelemetrySecuityListsTaskConstants.TIMEOUT,
|
||||
createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => {
|
||||
const { state } = taskInstance;
|
||||
|
||||
return {
|
||||
run: async () => {
|
||||
const taskExecutionTime = moment().utc().toISOString();
|
||||
const hits = await this.runTask(taskInstance.id);
|
||||
|
||||
return {
|
||||
state: {
|
||||
lastExecutionTimestamp: taskExecutionTime,
|
||||
runs: (state.runs || 0) + 1,
|
||||
hits,
|
||||
},
|
||||
};
|
||||
},
|
||||
cancel: async () => {},
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public start = async (taskManager: TaskManagerStartContract) => {
|
||||
try {
|
||||
await taskManager.ensureScheduled({
|
||||
id: this.getTaskId(),
|
||||
taskType: TelemetrySecuityListsTaskConstants.TYPE,
|
||||
scope: ['securitySolution'],
|
||||
schedule: {
|
||||
interval: TelemetrySecuityListsTaskConstants.INTERVAL,
|
||||
},
|
||||
state: { runs: 0 },
|
||||
params: { version: TelemetrySecuityListsTaskConstants.VERSION },
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error(`Error scheduling task, received ${e.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
private getTaskId = (): string => {
|
||||
return `${TelemetrySecuityListsTaskConstants.TYPE}:${TelemetrySecuityListsTaskConstants.VERSION}`;
|
||||
};
|
||||
|
||||
public runTask = async (taskId: string) => {
|
||||
if (taskId !== this.getTaskId()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const isOptedIn = await this.sender.isTelemetryOptedIn();
|
||||
if (!isOptedIn) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Lists Telemetry: Trusted Applications
|
||||
|
||||
const trustedApps = await this.sender.fetchTrustedApplications();
|
||||
const trustedAppsJson = templateTrustedApps(trustedApps.data);
|
||||
this.logger.debug(`Trusted Apps: ${trustedAppsJson}`);
|
||||
|
||||
batchTelemetryRecords(trustedAppsJson, MAX_TELEMETRY_BATCH).forEach((batch) =>
|
||||
this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch)
|
||||
);
|
||||
|
||||
// Lists Telemetry: Endpoint Exceptions
|
||||
|
||||
const epExceptions = await this.sender.fetchEndpointList(ENDPOINT_LIST_ID);
|
||||
const epExceptionsJson = templateEndpointExceptions(epExceptions.data, LIST_ENDPOINT_EXCEPTION);
|
||||
this.logger.debug(`EP Exceptions: ${epExceptionsJson}`);
|
||||
|
||||
batchTelemetryRecords(epExceptionsJson, MAX_TELEMETRY_BATCH).forEach((batch) =>
|
||||
this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch)
|
||||
);
|
||||
|
||||
// Lists Telemetry: Endpoint Event Filters
|
||||
|
||||
const epFilters = await this.sender.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID);
|
||||
const epFiltersJson = templateEndpointExceptions(epFilters.data, LIST_ENDPOINT_EVENT_FILTER);
|
||||
this.logger.debug(`EP Event Filters: ${epFiltersJson}`);
|
||||
|
||||
batchTelemetryRecords(epFiltersJson, MAX_TELEMETRY_BATCH).forEach((batch) =>
|
||||
this.sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch)
|
||||
);
|
||||
|
||||
return trustedAppsJson.length + epExceptionsJson.length + epFiltersJson.length;
|
||||
};
|
||||
}
|
|
@ -18,14 +18,15 @@ import {
|
|||
TaskManagerSetupContract,
|
||||
TaskManagerStartContract,
|
||||
} from '../../../../task_manager/server';
|
||||
import { createUsageCounterLabel } from './helpers';
|
||||
import { TelemetryDiagTask } from './diagnostic_task';
|
||||
import { TelemetryEndpointTask } from './endpoint_task';
|
||||
import { TelemetryTrustedAppsTask } from './trusted_apps_task';
|
||||
import { TelemetryExceptionListsTask } from './security_lists_task';
|
||||
import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
|
||||
import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/server';
|
||||
import { ExceptionListClient } from '../../../../lists/server';
|
||||
import { getTrustedAppsList } from '../../endpoint/routes/trusted_apps/service';
|
||||
import { ExceptionListClient } from '../../../../lists/server';
|
||||
import { GetEndpointListResponse } from './types';
|
||||
import { createUsageCounterLabel, exceptionListItemToEndpointEntry } from './helpers';
|
||||
|
||||
type BaseSearchTypes = string | number | boolean | object;
|
||||
export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined;
|
||||
|
@ -63,7 +64,7 @@ export class TelemetryEventsSender {
|
|||
private isOptedIn?: boolean = true; // Assume true until the first check
|
||||
private diagTask?: TelemetryDiagTask;
|
||||
private epMetricsTask?: TelemetryEndpointTask;
|
||||
private trustedAppsTask?: TelemetryTrustedAppsTask;
|
||||
private exceptionListTask?: TelemetryExceptionListsTask;
|
||||
private agentService?: AgentService;
|
||||
private agentPolicyService?: AgentPolicyServiceInterface;
|
||||
private esClient?: ElasticsearchClient;
|
||||
|
@ -86,7 +87,7 @@ export class TelemetryEventsSender {
|
|||
if (taskManager) {
|
||||
this.diagTask = new TelemetryDiagTask(this.logger, taskManager, this);
|
||||
this.epMetricsTask = new TelemetryEndpointTask(this.logger, taskManager, this);
|
||||
this.trustedAppsTask = new TelemetryTrustedAppsTask(this.logger, taskManager, this);
|
||||
this.exceptionListTask = new TelemetryExceptionListsTask(this.logger, taskManager, this);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -108,7 +109,7 @@ export class TelemetryEventsSender {
|
|||
this.logger.debug(`Starting diagnostic and endpoint telemetry tasks`);
|
||||
this.diagTask.start(taskManager);
|
||||
this.epMetricsTask.start(taskManager);
|
||||
this.trustedAppsTask?.start(taskManager);
|
||||
this.exceptionListTask?.start(taskManager);
|
||||
}
|
||||
|
||||
this.logger.debug(`Starting local task`);
|
||||
|
@ -279,6 +280,32 @@ export class TelemetryEventsSender {
|
|||
return getTrustedAppsList(this.exceptionListClient, { page: 1, per_page: 10_000 });
|
||||
}
|
||||
|
||||
public async fetchEndpointList(listId: string): Promise<GetEndpointListResponse> {
|
||||
if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) {
|
||||
throw Error('could not fetch trusted applications. exception list client not available.');
|
||||
}
|
||||
|
||||
// Ensure list is created if it does not exist
|
||||
await this.exceptionListClient.createTrustedAppsList();
|
||||
|
||||
const results = await this.exceptionListClient.findExceptionListItem({
|
||||
listId,
|
||||
page: 1,
|
||||
perPage: this.max_records,
|
||||
filter: undefined,
|
||||
namespaceType: 'agnostic',
|
||||
sortField: 'name',
|
||||
sortOrder: 'asc',
|
||||
});
|
||||
|
||||
return {
|
||||
data: results?.data.map(exceptionListItemToEndpointEntry) ?? [],
|
||||
total: results?.total ?? 0,
|
||||
page: results?.page ?? 1,
|
||||
per_page: results?.per_page ?? this.max_records,
|
||||
};
|
||||
}
|
||||
|
||||
public queueTelemetryEvents(events: TelemetryEvent[]) {
|
||||
const qlength = this.queue.length;
|
||||
|
||||
|
|
|
@ -1,122 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import { Logger } from 'src/core/server';
|
||||
import {
|
||||
ConcreteTaskInstance,
|
||||
TaskManagerSetupContract,
|
||||
TaskManagerStartContract,
|
||||
} from '../../../../task_manager/server';
|
||||
|
||||
import { getPreviousEpMetaTaskTimestamp, batchTelemetryRecords } from './helpers';
|
||||
import { TelemetryEventsSender } from './sender';
|
||||
|
||||
export const TelemetryTrustedAppsTaskConstants = {
|
||||
TIMEOUT: '1m',
|
||||
TYPE: 'security:trusted-apps-telemetry',
|
||||
INTERVAL: '24h',
|
||||
VERSION: '1.0.0',
|
||||
};
|
||||
|
||||
/** Telemetry Trusted Apps Task
|
||||
*
|
||||
* The Trusted Apps task is a daily batch job that collects and transmits non-sensitive
|
||||
* trusted apps hashes + file paths for supported operating systems. This helps test
|
||||
* efficacy of our protections.
|
||||
*/
|
||||
export class TelemetryTrustedAppsTask {
|
||||
private readonly logger: Logger;
|
||||
private readonly sender: TelemetryEventsSender;
|
||||
|
||||
constructor(
|
||||
logger: Logger,
|
||||
taskManager: TaskManagerSetupContract,
|
||||
sender: TelemetryEventsSender
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.sender = sender;
|
||||
|
||||
taskManager.registerTaskDefinitions({
|
||||
[TelemetryTrustedAppsTaskConstants.TYPE]: {
|
||||
title: 'Security Solution Telemetry Endpoint Metrics and Info task',
|
||||
timeout: TelemetryTrustedAppsTaskConstants.TIMEOUT,
|
||||
createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => {
|
||||
const { state } = taskInstance;
|
||||
|
||||
return {
|
||||
run: async () => {
|
||||
const taskExecutionTime = moment().utc().toISOString();
|
||||
const lastExecutionTimestamp = getPreviousEpMetaTaskTimestamp(
|
||||
taskExecutionTime,
|
||||
taskInstance.state?.lastExecutionTimestamp
|
||||
);
|
||||
|
||||
const hits = await this.runTask(
|
||||
taskInstance.id,
|
||||
lastExecutionTimestamp,
|
||||
taskExecutionTime
|
||||
);
|
||||
|
||||
return {
|
||||
state: {
|
||||
lastExecutionTimestamp: taskExecutionTime,
|
||||
runs: (state.runs || 0) + 1,
|
||||
hits,
|
||||
},
|
||||
};
|
||||
},
|
||||
cancel: async () => {},
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public start = async (taskManager: TaskManagerStartContract) => {
|
||||
try {
|
||||
await taskManager.ensureScheduled({
|
||||
id: this.getTaskId(),
|
||||
taskType: TelemetryTrustedAppsTaskConstants.TYPE,
|
||||
scope: ['securitySolution'],
|
||||
schedule: {
|
||||
interval: TelemetryTrustedAppsTaskConstants.INTERVAL,
|
||||
},
|
||||
state: { runs: 0 },
|
||||
params: { version: TelemetryTrustedAppsTaskConstants.VERSION },
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error(`Error scheduling task, received ${e.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
private getTaskId = (): string => {
|
||||
return `${TelemetryTrustedAppsTaskConstants.TYPE}:${TelemetryTrustedAppsTaskConstants.VERSION}`;
|
||||
};
|
||||
|
||||
public runTask = async (taskId: string, executeFrom: string, executeTo: string) => {
|
||||
if (taskId !== this.getTaskId()) {
|
||||
this.logger.debug(`Outdated task running: ${taskId}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const isOptedIn = await this.sender.isTelemetryOptedIn();
|
||||
if (!isOptedIn) {
|
||||
this.logger.debug(`Telemetry is not opted-in.`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const response = await this.sender.fetchTrustedApplications();
|
||||
this.logger.debug(`Trusted Apps: ${response}`);
|
||||
|
||||
batchTelemetryRecords(response.data, 1_000).forEach((telemetryBatch) =>
|
||||
this.sender.sendOnDemand('lists-trustedapps', telemetryBatch)
|
||||
);
|
||||
|
||||
return response.data.length;
|
||||
};
|
||||
}
|
|
@ -5,6 +5,9 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { schema, TypeOf } from '@kbn/config-schema';
|
||||
import { TrustedApp } from '../../../common/endpoint/types';
|
||||
|
||||
// EP Policy Response
|
||||
|
||||
export interface EndpointPolicyResponseAggregation {
|
||||
|
@ -138,3 +141,43 @@ interface EndpointMetricOS {
|
|||
platform: string;
|
||||
full: string;
|
||||
}
|
||||
|
||||
// List HTTP Types
|
||||
|
||||
export const GetTrustedAppsRequestSchema = {
|
||||
query: schema.object({
|
||||
page: schema.maybe(schema.number({ defaultValue: 1, min: 1 })),
|
||||
per_page: schema.maybe(schema.number({ defaultValue: 20, min: 1 })),
|
||||
kuery: schema.maybe(schema.string()),
|
||||
}),
|
||||
};
|
||||
|
||||
export type GetEndpointListRequest = TypeOf<typeof GetTrustedAppsRequestSchema.query>;
|
||||
|
||||
export interface GetEndpointListResponse {
|
||||
per_page: number;
|
||||
page: number;
|
||||
total: number;
|
||||
data: EndpointExceptionListItem[];
|
||||
}
|
||||
|
||||
// Telemetry List types
|
||||
|
||||
export interface EndpointExceptionListItem {
|
||||
id: string;
|
||||
version: string;
|
||||
name: string;
|
||||
description: string;
|
||||
created_at: string;
|
||||
created_by: string;
|
||||
updated_at: string;
|
||||
updated_by: string;
|
||||
entries: object;
|
||||
os_types: object;
|
||||
}
|
||||
|
||||
export interface ListTemplate {
|
||||
trusted_application: TrustedApp[];
|
||||
endpoint_exception: EndpointExceptionListItem[];
|
||||
endpoint_event_filter: EndpointExceptionListItem[];
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue