Insights telemetry collection -- Alert Status Updates (#115471)
* Added security plugin to signals route * Added insights payload construction to status route * Pass cloud plugin setup to route to test if cloud is enabled * Incorrectly getting username from authenticated user * Test needs cloud setup passed to mock * Mistakenly added sender to migration * Just pass cloudEnabled boolean to route * Remove cloud specific checks from telemetry forwarding * Populate sessionId from request, hash+salt with clusterID * Converted payload construction to map * Added logger to route, found that ui sometimes passes alert_ids in query * Properly pass logger into test * Change deep nested query field access to lodash get * Fixed some import issues * Addressed some comments from @pjhamptom * Added fields to mock to ensure that the testTelemetrySender has the proper interface * Wrapped awaits in Promise.all, abstract and remove async fetchClusterInfo calls since clusterInfo is immutable * Missed some references to fetchClusterInfo() * Removed references to rules, changed 'page' to 'route', made insights functions not methods Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
parent
edc43c0ff2
commit
de2ca18226
10 changed files with 166 additions and 14 deletions
|
@ -15,16 +15,21 @@ import {
|
|||
getSuccessfulSignalUpdateResponse,
|
||||
} from '../__mocks__/request_responses';
|
||||
import { requestContextMock, serverMock, requestMock } from '../__mocks__';
|
||||
import { SetupPlugins } from '../../../../plugin';
|
||||
import { createMockTelemetryEventsSender } from '../../../telemetry/__mocks__';
|
||||
import { setSignalsStatusRoute } from './open_close_signals_route';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
|
||||
import { loggingSystemMock } from 'src/core/server/mocks';
|
||||
|
||||
describe('set signal status', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let { context } = requestContextMock.createTools();
|
||||
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
|
||||
|
||||
beforeEach(() => {
|
||||
server = serverMock.create();
|
||||
logger = loggingSystemMock.createLogger();
|
||||
({ context } = requestContextMock.createTools());
|
||||
|
||||
context.core.elasticsearch.client.asCurrentUser.updateByQuery.mockResolvedValue(
|
||||
|
@ -32,8 +37,13 @@ describe('set signal status', () => {
|
|||
getSuccessfulSignalUpdateResponse()
|
||||
)
|
||||
);
|
||||
|
||||
setSignalsStatusRoute(server.router);
|
||||
const telemetrySenderMock = createMockTelemetryEventsSender();
|
||||
const securityMock = {
|
||||
authc: {
|
||||
getCurrentUser: jest.fn().mockReturnValue({ user: { username: 'my-username' } }),
|
||||
},
|
||||
} as unknown as SetupPlugins['security'];
|
||||
setSignalsStatusRoute(server.router, logger, securityMock, telemetrySenderMock);
|
||||
});
|
||||
|
||||
describe('status on signal', () => {
|
||||
|
|
|
@ -5,8 +5,10 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { get } from 'lodash';
|
||||
import { transformError } from '@kbn/securitysolution-es-utils';
|
||||
import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils';
|
||||
import { Logger } from 'src/core/server';
|
||||
import { setSignalStatusValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/set_signal_status_type_dependents';
|
||||
import {
|
||||
SetSignalsStatusSchemaDecoded,
|
||||
|
@ -15,10 +17,21 @@ import {
|
|||
import type { SecuritySolutionPluginRouter } from '../../../../types';
|
||||
import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '../../../../../common/constants';
|
||||
import { buildSiemResponse } from '../utils';
|
||||
|
||||
import { TelemetryEventsSender } from '../../../telemetry/sender';
|
||||
import { INSIGHTS_CHANNEL } from '../../../telemetry/constants';
|
||||
import { SetupPlugins } from '../../../../plugin';
|
||||
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
|
||||
import {
|
||||
getSessionIDfromKibanaRequest,
|
||||
createAlertStatusPayloads,
|
||||
} from '../../../telemetry/insights';
|
||||
|
||||
export const setSignalsStatusRoute = (router: SecuritySolutionPluginRouter) => {
|
||||
export const setSignalsStatusRoute = (
|
||||
router: SecuritySolutionPluginRouter,
|
||||
logger: Logger,
|
||||
security: SetupPlugins['security'],
|
||||
sender: TelemetryEventsSender
|
||||
) => {
|
||||
router.post(
|
||||
{
|
||||
path: DETECTION_ENGINE_SIGNALS_STATUS_URL,
|
||||
|
@ -46,6 +59,30 @@ export const setSignalsStatusRoute = (router: SecuritySolutionPluginRouter) => {
|
|||
return siemResponse.error({ statusCode: 404 });
|
||||
}
|
||||
|
||||
const clusterId = sender.getClusterID();
|
||||
const [isTelemetryOptedIn, username] = await Promise.all([
|
||||
sender.isTelemetryOptedIn(),
|
||||
security?.authc.getCurrentUser(request)?.username,
|
||||
]);
|
||||
if (isTelemetryOptedIn && clusterId) {
|
||||
// Sometimes the ids are in the query not passed in the request?
|
||||
const toSendAlertIds = get(query, 'bool.filter.terms._id') || signalIds;
|
||||
// Get Context for Insights Payloads
|
||||
const sessionId = getSessionIDfromKibanaRequest(clusterId, request);
|
||||
if (username && toSendAlertIds && sessionId && status) {
|
||||
const insightsPayloads = createAlertStatusPayloads(
|
||||
clusterId,
|
||||
toSendAlertIds,
|
||||
sessionId,
|
||||
username,
|
||||
DETECTION_ENGINE_SIGNALS_STATUS_URL,
|
||||
status
|
||||
);
|
||||
logger.debug(`Sending Insights Payloads ${JSON.stringify(insightsPayloads)}`);
|
||||
await sender.sendOnDemand(INSIGHTS_CHANNEL, insightsPayloads);
|
||||
}
|
||||
}
|
||||
|
||||
let queryObject;
|
||||
if (signalIds) {
|
||||
queryObject = { ids: { values: signalIds } };
|
||||
|
|
|
@ -21,12 +21,14 @@ export const createMockTelemetryEventsSender = (
|
|||
setup: jest.fn(),
|
||||
start: jest.fn(),
|
||||
stop: jest.fn(),
|
||||
getClusterID: jest.fn(),
|
||||
fetchTelemetryUrl: jest.fn(),
|
||||
queueTelemetryEvents: jest.fn(),
|
||||
processEvents: jest.fn(),
|
||||
isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()),
|
||||
sendIfDue: jest.fn(),
|
||||
sendEvents: jest.fn(),
|
||||
sendOnDemand: jest.fn(),
|
||||
} as unknown as jest.Mocked<TelemetryEventsSender>;
|
||||
};
|
||||
|
||||
|
@ -35,7 +37,6 @@ export const createMockTelemetryReceiver = (
|
|||
): jest.Mocked<TelemetryReceiver> => {
|
||||
return {
|
||||
start: jest.fn(),
|
||||
fetchClusterInfo: jest.fn(),
|
||||
fetchLicenseInfo: jest.fn(),
|
||||
copyLicenseFields: jest.fn(),
|
||||
fetchFleetAgents: jest.fn(),
|
||||
|
|
|
@ -24,3 +24,5 @@ export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception';
|
|||
export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter';
|
||||
|
||||
export const LIST_TRUSTED_APPLICATION = 'trusted_application';
|
||||
|
||||
export const INSIGHTS_CHANNEL = 'security-insights-v1';
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
export { getSessionIDfromKibanaRequest, createAlertStatusPayloads } from './insights';
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import { KibanaRequest } from 'src/core/server';
|
||||
import { sha256 } from 'js-sha256';
|
||||
|
||||
interface AlertContext {
|
||||
alert_id: string;
|
||||
}
|
||||
|
||||
interface AlertStatusAction {
|
||||
alert_status: string;
|
||||
action_timestamp: string;
|
||||
}
|
||||
|
||||
export interface InsightsPayload {
|
||||
state: {
|
||||
route: string;
|
||||
cluster_id: string;
|
||||
user_id: string;
|
||||
session_id: string;
|
||||
context: AlertContext;
|
||||
};
|
||||
action: AlertStatusAction;
|
||||
}
|
||||
export function getSessionIDfromKibanaRequest(clusterId: string, request: KibanaRequest): string {
|
||||
const rawCookieHeader = request.headers.cookie;
|
||||
if (!rawCookieHeader) {
|
||||
return '';
|
||||
}
|
||||
const cookieHeaders = Array.isArray(rawCookieHeader) ? rawCookieHeader : [rawCookieHeader];
|
||||
let tokenPackage: string | undefined;
|
||||
|
||||
cookieHeaders
|
||||
.flatMap((rawHeader) => rawHeader.split('; '))
|
||||
.forEach((rawCookie) => {
|
||||
const [cookieName, cookieValue] = rawCookie.split('=');
|
||||
if (cookieName === 'sid') tokenPackage = cookieValue;
|
||||
});
|
||||
|
||||
if (tokenPackage) {
|
||||
return getClusterHashSalt(clusterId, tokenPackage);
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function getClusterHashSalt(clusterId: string, toHash: string): string {
|
||||
const concatValue = toHash + clusterId;
|
||||
const sha = sha256.create().update(concatValue).hex();
|
||||
return sha;
|
||||
}
|
||||
|
||||
export function createAlertStatusPayloads(
|
||||
clusterId: string,
|
||||
alertIds: string[],
|
||||
sessionId: string,
|
||||
username: string,
|
||||
route: string,
|
||||
status: string
|
||||
): InsightsPayload[] {
|
||||
return alertIds.map((alertId) => ({
|
||||
state: {
|
||||
route,
|
||||
cluster_id: clusterId,
|
||||
user_id: getClusterHashSalt(clusterId, username),
|
||||
session_id: sessionId,
|
||||
context: {
|
||||
alert_id: alertId,
|
||||
},
|
||||
},
|
||||
action: {
|
||||
alert_status: status,
|
||||
action_timestamp: moment().toISOString(),
|
||||
},
|
||||
}));
|
||||
}
|
|
@ -38,6 +38,7 @@ export class TelemetryReceiver {
|
|||
private exceptionListClient?: ExceptionListClient;
|
||||
private soClient?: SavedObjectsClientContract;
|
||||
private kibanaIndex?: string;
|
||||
private clusterInfo?: ESClusterInfo;
|
||||
private readonly max_records = 10_000;
|
||||
|
||||
constructor(logger: Logger) {
|
||||
|
@ -57,6 +58,11 @@ export class TelemetryReceiver {
|
|||
this.exceptionListClient = exceptionListClient;
|
||||
this.soClient =
|
||||
core?.savedObjects.createInternalRepository() as unknown as SavedObjectsClientContract;
|
||||
this.clusterInfo = await this.fetchClusterInfo();
|
||||
}
|
||||
|
||||
public getClusterInfo(): ESClusterInfo | undefined {
|
||||
return this.clusterInfo;
|
||||
}
|
||||
|
||||
public async fetchFleetAgents() {
|
||||
|
@ -304,7 +310,7 @@ export class TelemetryReceiver {
|
|||
};
|
||||
}
|
||||
|
||||
public async fetchClusterInfo(): Promise<ESClusterInfo> {
|
||||
private async fetchClusterInfo(): Promise<ESClusterInfo> {
|
||||
if (this.esClient === undefined || this.esClient === null) {
|
||||
throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
|
||||
}
|
||||
|
|
|
@ -67,6 +67,10 @@ export class TelemetryEventsSender {
|
|||
}
|
||||
}
|
||||
|
||||
public getClusterID(): string | undefined {
|
||||
return this.receiver?.getClusterInfo()?.cluster_uuid;
|
||||
}
|
||||
|
||||
public start(
|
||||
telemetryStart?: TelemetryPluginStart,
|
||||
taskManager?: TaskManagerStartContract,
|
||||
|
@ -149,9 +153,10 @@ export class TelemetryEventsSender {
|
|||
return;
|
||||
}
|
||||
|
||||
const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([
|
||||
const clusterInfo = this.receiver?.getClusterInfo();
|
||||
|
||||
const [telemetryUrl, licenseInfo] = await Promise.all([
|
||||
this.fetchTelemetryUrl('alerts-endpoint'),
|
||||
this.receiver?.fetchClusterInfo(),
|
||||
this.receiver?.fetchLicenseInfo(),
|
||||
]);
|
||||
|
||||
|
@ -198,10 +203,10 @@ export class TelemetryEventsSender {
|
|||
* @param toSend telemetry events
|
||||
*/
|
||||
public async sendOnDemand(channel: string, toSend: unknown[]) {
|
||||
const clusterInfo = this.receiver?.getClusterInfo();
|
||||
try {
|
||||
const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([
|
||||
const [telemetryUrl, licenseInfo] = await Promise.all([
|
||||
this.fetchTelemetryUrl(channel),
|
||||
this.receiver?.fetchClusterInfo(),
|
||||
this.receiver?.fetchLicenseInfo(),
|
||||
]);
|
||||
|
||||
|
@ -255,6 +260,7 @@ export class TelemetryEventsSender {
|
|||
const ndjson = transformDataToNdjson(events);
|
||||
|
||||
try {
|
||||
this.logger.debug(`Sending ${events.length} telemetry events to ${channel}`);
|
||||
const resp = await axios.post(telemetryUrl, ndjson, {
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ndjson',
|
||||
|
@ -275,9 +281,7 @@ export class TelemetryEventsSender {
|
|||
});
|
||||
this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}`
|
||||
);
|
||||
this.logger.debug(`Error sending events: ${err}`);
|
||||
this.telemetryUsageCounter?.incrementCounter({
|
||||
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
|
||||
counterType: 'docs_lost',
|
||||
|
|
|
@ -236,6 +236,7 @@ export class Plugin implements ISecuritySolutionPlugin {
|
|||
config,
|
||||
plugins.encryptedSavedObjects?.canEncrypt === true,
|
||||
plugins.security,
|
||||
this.telemetryEventsSender,
|
||||
plugins.ml,
|
||||
logger,
|
||||
isRuleRegistryEnabled,
|
||||
|
|
|
@ -55,6 +55,7 @@ import { persistPinnedEventRoute } from '../lib/timeline/routes/pinned_events';
|
|||
|
||||
import { SetupPlugins } from '../plugin';
|
||||
import { ConfigType } from '../config';
|
||||
import { TelemetryEventsSender } from '../lib/telemetry/sender';
|
||||
import { installPrepackedTimelinesRoute } from '../lib/timeline/routes/prepackaged_timelines/install_prepackaged_timelines';
|
||||
import { previewRulesRoute } from '../lib/detection_engine/routes/rules/preview_rules_route';
|
||||
import { CreateRuleOptions } from '../lib/detection_engine/rule_types/types';
|
||||
|
@ -67,6 +68,7 @@ export const initRoutes = (
|
|||
config: ConfigType,
|
||||
hasEncryptionKey: boolean,
|
||||
security: SetupPlugins['security'],
|
||||
telemetrySender: TelemetryEventsSender,
|
||||
ml: SetupPlugins['ml'],
|
||||
logger: Logger,
|
||||
isRuleRegistryEnabled: boolean,
|
||||
|
@ -120,7 +122,7 @@ export const initRoutes = (
|
|||
// Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals
|
||||
// POST /api/detection_engine/signals/status
|
||||
// Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals
|
||||
setSignalsStatusRoute(router);
|
||||
setSignalsStatusRoute(router, logger, security, telemetrySender);
|
||||
querySignalsRoute(router, config);
|
||||
getSignalsMigrationStatusRoute(router);
|
||||
createSignalsMigrationRoute(router, security);
|
||||
|
|
Loading…
Reference in a new issue