maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

add policy details and update SO limit requests (#71789)

Browse source
This commit is contained in:
Michael Olorunnisola 2020-07-15 09:36:48 -04:00 committed by GitHub
parent 5f6389af60
commit ed387dd15f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 252 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/security_solution/server/usage/collector.ts
View file

@ -66,8 +66,8 @@ export const registerCollector: RegisterCollector = ({
},
policies: {
malware: {
success: { type: 'long' },
warning: { type: 'long' },
active: { type: 'long' },
inactive: { type: 'long' },
failure: { type: 'long' },
},
},

103
x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts
View file

@ -76,6 +76,108 @@ export const mockFleetObjectsResponse = (
],
});
const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
JSON.stringify({
'endpoint-security': {
Endpoint: {
configuration: {
inputs: [
{
id: '0d466df0-c60f-11ea-a5c5-151665e785c4',
policy: {
linux: {
events: {
file: true,
network: true,
process: true,
},
logging: {
file: 'info',
},
},
mac: {
events: {
file: true,
network: true,
process: true,
},
logging: {
file: 'info',
},
malware: {
mode: 'prevent',
},
},
windows: {
events: {
dll_and_driver_load: true,
dns: true,
file: true,
network: true,
process: true,
registry: true,
security: true,
},
logging: {
file: 'info',
},
malware: {
mode: 'prevent',
},
},
},
},
],
},
policy: {
applied: {
id: '0d466df0-c60f-11ea-a5c5-151665e785c4',
response: {
configurations: {
malware: {
concerned_actions: [
'load_config',
'workflow',
'download_global_artifacts',
'download_user_artifacts',
'configure_malware',
'read_malware_config',
'load_malware_model',
'read_kernel_config',
'configure_kernel',
'detect_process_events',
'detect_file_write_events',
'connect_kernel',
'detect_file_open_events',
'detect_sync_image_load_events',
],
status: `${malwareStatus}`,
},
},
},
status: `${malwareStatus}`,
},
},
},
agent: {
id: 'testAgentId',
version: '8.0.0-SNAPSHOT',
},
host: {
architecture: 'x86_64',
id: 'a4148b63-1758-ab1f-a6d3-f95075cb1a9c',
os: {
Ext: {
variant: 'Windows 10 Pro',
},
full: 'Windows 10 Pro 2004 (10.0.19041.329)',
name: 'Windows',
version: '2004 (10.0.19041.329)',
},
},
},
});
/**
*
* @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state
@ -102,6 +204,7 @@ export const mockFleetEventsObjectsResponse = (
message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${
running ? 'RUNNING' : 'FAILED'
}: `,
payload: mockPolicyPayload(running ? 'success' : 'failure'),
config_id: testConfigId,
},
references: [],

36
x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts
View file

@ -20,12 +20,12 @@ import * as fleetSavedObjects from './fleet_saved_objects';
describe('test security solution endpoint telemetry', () => {
let mockSavedObjectsRepository: jest.Mocked<ISavedObjectsRepository>;
let getFleetSavedObjectsMetadataSpy: jest.SpyInstance<Promise<SavedObjectsFindResponse<Agent>>>;
let getFleetEventsSavedObjectsSpy: jest.SpyInstance<Promise<
let getLatestFleetEndpointEventSpy: jest.SpyInstance<Promise<
SavedObjectsFindResponse<AgentEventSOAttributes>
>>;
beforeAll(() => {
getFleetEventsSavedObjectsSpy = jest.spyOn(fleetSavedObjects, 'getFleetEventsSavedObjects');
getLatestFleetEndpointEventSpy = jest.spyOn(fleetSavedObjects, 'getLatestFleetEndpointEvent');
getFleetSavedObjectsMetadataSpy = jest.spyOn(fleetSavedObjects, 'getFleetSavedObjectsMetadata');
mockSavedObjectsRepository = savedObjectsRepositoryMock.create();
});
@ -39,6 +39,13 @@ describe('test security solution endpoint telemetry', () => {
Object {
"active_within_last_24_hours": 0,
"os": Array [],
"policies": Object {
"malware": Object {
"active": 0,
"failure": 0,
"inactive": 0,
},
},
"total_installed": 0,
}
`);
@ -58,6 +65,13 @@ describe('test security solution endpoint telemetry', () => {
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
policies: {
malware: {
failure: 0,
active: 0,
inactive: 0,
},
},
});
});
});
@ -67,7 +81,7 @@ describe('test security solution endpoint telemetry', () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);
@ -85,6 +99,13 @@ describe('test security solution endpoint telemetry', () => {
count: 1,
},
],
policies: {
malware: {
failure: 1,
active: 0,
inactive: 0,
},
},
});
});
@ -92,7 +113,7 @@ describe('test security solution endpoint telemetry', () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);
@ -110,6 +131,13 @@ describe('test security solution endpoint telemetry', () => {
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 1,
inactive: 0,
},
},
});
});
});

4
x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts
View file

@ -19,17 +19,19 @@ export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObj
type: AGENT_SAVED_OBJECT_TYPE,
fields: ['packages', 'last_checkin', 'local_metadata'],
filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`,
perPage: 10000,
sortField: 'enrolled_at',
sortOrder: 'desc',
});
export const getFleetEventsSavedObjects = async (
export const getLatestFleetEndpointEvent = async (
savedObjectsClient: ISavedObjectsRepository,
agentId: string
) =>
savedObjectsClient.find<AgentEventSOAttributes>({
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`,
perPage: 1, // Get the most recent endpoint event.
sortField: 'timestamp',
sortOrder: 'desc',
search: agentId,

154
x-pack/plugins/security_solution/server/usage/endpoints/index.ts
View file

@ -6,11 +6,7 @@
import { ISavedObjectsRepository } from 'src/core/server';
import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent';
import {
getFleetSavedObjectsMetadata,
getFleetEventsSavedObjects,
FLEET_ENDPOINT_PACKAGE_CONSTANT,
} from './fleet_saved_objects';
import { getFleetSavedObjectsMetadata, getLatestFleetEndpointEvent } from './fleet_saved_objects';
export interface AgentOSMetadataTelemetry {
full_name: string;
@ -18,22 +14,25 @@ export interface AgentOSMetadataTelemetry {
version: string;
count: number;
}
export interface PolicyTelemetry {
active: number;
inactive: number;
failure: number;
}
export interface PoliciesTelemetry {
malware: {
success: number;
warning: number;
failure: number;
};
malware: PolicyTelemetry;
}
export interface EndpointUsage {
total_installed: number;
active_within_last_24_hours: number;
os: AgentOSMetadataTelemetry[];
policies?: PoliciesTelemetry; // TODO: make required when able to enable policy information
policies: PoliciesTelemetry;
}
type EndpointOSNames = 'Linux' | 'Windows' | 'macOs';
export interface AgentLocalMetadata extends AgentMetadata {
elastic: {
agent: {
@ -51,7 +50,8 @@ export interface AgentLocalMetadata extends AgentMetadata {
};
}
export type OSTracker = Record<string, AgentOSMetadataTelemetry>;
type OSTracker = Record<string, AgentOSMetadataTelemetry>;
type AgentDailyActiveTracker = Map<string, boolean>;
/**
* @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn
*/
@ -59,8 +59,18 @@ export const getDefaultEndpointTelemetry = (): EndpointUsage => ({
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
policies: {
malware: {
active: 0,
inactive: 0,
failure: 0,
},
},
});
/**
* @description this fun
*/
export const trackEndpointOSTelemetry = (
os: AgentLocalMetadata['os'],
osTracker: OSTracker
@ -82,6 +92,80 @@ export const trackEndpointOSTelemetry = (
return updatedOSTracker;
};
/**
* @description This iterates over all unique agents that currently track an endpoint package. It takes a list of agents who have checked in in the last 24 hours
* and then checks whether those agents have endpoints whose latest status is 'RUNNING' to determine an active_within_last_24_hours. Since the policy information is also tracked in these events
* we pull out the status of the current protection (malware) type. This must be done in a compound manner as the desired status is reflected in the config, and the successful application of that policy
* is tracked in the policy.applied.response.configurations[protectionsType].status. Using these two we can determine whether the policy is toggled on, off, or failed to turn on.
*/
export const addEndpointDailyActivityAndPolicyDetailsToTelemetry = async (
agentDailyActiveTracker: AgentDailyActiveTracker,
savedObjectsClient: ISavedObjectsRepository,
endpointTelemetry: EndpointUsage
): Promise<EndpointUsage> => {
const updatedEndpointTelemetry = { ...endpointTelemetry };
const policyHostTypeToPolicyType = {
Linux: 'linux',
macOs: 'mac',
Windows: 'windows',
};
const enabledMalwarePolicyTypes = ['prevent', 'detect'];
for (const agentId of agentDailyActiveTracker.keys()) {
const { saved_objects: agentEvents } = await getLatestFleetEndpointEvent(
savedObjectsClient,
agentId
);
const latestEndpointEvent = agentEvents[0];
if (latestEndpointEvent) {
/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours.
*/
const { subtype, payload } = latestEndpointEvent.attributes;
const endpointIsActive =
subtype === 'RUNNING' && agentDailyActiveTracker.get(agentId) === true;
if (endpointIsActive) {
updatedEndpointTelemetry.active_within_last_24_hours += 1;
}
// The policy details are sent as a string on the 'payload' attribute of the agent event
const endpointPolicyDetails = payload ? JSON.parse(payload) : null;
if (endpointPolicyDetails) {
// We get the setting the user desired to enable (treating prevent and detect as 'active' states) and then see if it succeded or failed.
const hostType =
policyHostTypeToPolicyType[
endpointPolicyDetails['endpoint-security']?.host?.os?.name as EndpointOSNames
];
const userDesiredMalwareState =
endpointPolicyDetails['endpoint-security'].Endpoint?.configuration?.inputs[0]?.policy[
hostType
]?.malware?.mode;
const isAnActiveMalwareState = enabledMalwarePolicyTypes.includes(userDesiredMalwareState);
const malwareStatus =
endpointPolicyDetails['endpoint-security'].Endpoint?.policy?.applied?.response
?.configurations?.malware?.status;
if (isAnActiveMalwareState && malwareStatus !== 'failure') {
updatedEndpointTelemetry.policies.malware.active += 1;
}
if (!isAnActiveMalwareState) {
updatedEndpointTelemetry.policies.malware.inactive += 1;
}
if (isAnActiveMalwareState && malwareStatus === 'failure') {
updatedEndpointTelemetry.policies.malware.failure += 1;
}
}
}
}
return updatedEndpointTelemetry;
};
/**
* @description This aggregates the telemetry details from the two fleet savedObject sources, `fleet-agents` and `fleet-agent-events` to populate
* the telemetry details for endpoint. Since we cannot access our own indices due to `kibana_system` not having access, this is the best alternative.
@ -100,8 +184,8 @@ export const getEndpointTelemetryFromFleet = async (
// Use unique hosts to prevent any potential duplicates
const uniqueHostIds: Set<string> = new Set();
// Need unique agents to get events data for those that have run in last 24 hours
const uniqueAgentIds: Set<string> = new Set();
// Need agents to get events data for those that have run in last 24 hours as well as policy details
const agentDailyActiveTracker: AgentDailyActiveTracker = new Map();
const aDayAgo = new Date();
aDayAgo.setDate(aDayAgo.getDate() - 1);
@ -110,17 +194,15 @@ export const getEndpointTelemetryFromFleet = async (
const endpointMetadataTelemetry = endpointAgents.reduce(
(metadataTelemetry, { attributes: metadataAttributes }) => {
const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
// The extended AgentMetadata is just an empty blob, so cast to account for our specific use case
const { host, os, elastic } = localMetadata as AgentLocalMetadata;
const { host, os, elastic } = localMetadata as AgentLocalMetadata; // AgentMetadata is just an empty blob, casting for our use case
if (lastCheckin && new Date(lastCheckin) > aDayAgo) {
// Get agents that have checked in within the last 24 hours to later see if their endpoints are running
uniqueAgentIds.add(elastic.agent.id);
}
if (host && uniqueHostIds.has(host.id)) {
// use hosts since new agents could potentially be re-installed on existing hosts
return metadataTelemetry;
} else {
uniqueHostIds.add(host.id);
const isActiveWithinLastDay = !!lastCheckin && new Date(lastCheckin) > aDayAgo;
agentDailyActiveTracker.set(elastic.agent.id, isActiveWithinLastDay);
osTracker = trackEndpointOSTelemetry(os, osTracker);
return metadataTelemetry;
}
@ -128,32 +210,16 @@ export const getEndpointTelemetryFromFleet = async (
endpointTelemetry
);
// All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed.
// All unique hosts with an endpoint installed.
endpointTelemetry.total_installed = uniqueHostIds.size;
// Get the objects to populate our OS Telemetry
endpointMetadataTelemetry.os = Object.values(osTracker);
// Populate endpoint telemetry with the finalized 24 hour count and policy details
const finalizedEndpointTelemetryData = await addEndpointDailyActivityAndPolicyDetailsToTelemetry(
agentDailyActiveTracker,
savedObjectsClient,
endpointMetadataTelemetry
);
// Check for agents running in the last 24 hours whose endpoints are still active
for (const agentId of uniqueAgentIds) {
const { saved_objects: agentEvents } = await getFleetEventsSavedObjects(
savedObjectsClient,
agentId
);
const lastEndpointStatus = agentEvents.find((agentEvent) =>
agentEvent.attributes.message.includes(FLEET_ENDPOINT_PACKAGE_CONSTANT)
);
/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours. If / when we get the policy response, then we can use that
instead
*/
const endpointIsActive = lastEndpointStatus?.attributes.subtype === 'RUNNING';
if (endpointIsActive) {
endpointMetadataTelemetry.active_within_last_24_hours += 1;
}
}
return endpointMetadataTelemetry;
return finalizedEndpointTelemetryData;
};

4
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -246,10 +246,10 @@
"properties": {
"malware": {
"properties": {
"success": {
"active": {
"type": "long"
},
"warning": {
"inactive": {
"type": "long"
},
"failure": {