maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution] Cleanup endpoint telemetry (#71950)

Browse source 
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Michael Olorunnisola 2020-07-20 12:28:46 -04:00 committed by GitHub
parent 96d965d4e3
commit 2094f33537
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 478 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/security_solution/server/usage/collector.ts
View file

@ -12,7 +12,7 @@ import { EndpointUsage, getEndpointTelemetryFromFleet } from './endpoints';
export type RegisterCollector = (deps: CollectorDependencies) => void;
export interface UsageData {
detections: DetectionsUsage;
endpoints: EndpointUsage;
endpoints: EndpointUsage | {};
}
export async function getInternalSavedObjectsClient(core: CoreSetup) {

60
x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts
View file

@ -14,6 +14,7 @@ import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects';
const testAgentId = 'testAgentId';
const testConfigId = 'testConfigId';
const testHostId = 'randoHostId';
/** Mock OS Platform for endpoint telemetry */
export const MockOSPlatform = 'somePlatform';
@ -30,6 +31,7 @@ export const MockOSFullName = 'somePlatformFullName';
* @description We request the install and OS related telemetry information from the 'fleet-agents' saved objects in ingest_manager. This mocks that response
*/
export const mockFleetObjectsResponse = (
hasDuplicates = true,
lastCheckIn = new Date().toISOString()
): SavedObjectsFindResponse<Agent> => ({
page: 1,
@ -56,7 +58,44 @@ export const mockFleetObjectsResponse = (
host: {
hostname: 'testDesktop',
name: 'testDesktop',
id: 'randoHostId',
id: testHostId,
},
os: {
platform: MockOSPlatform,
version: MockOSVersion,
name: MockOSName,
full: MockOSFullName,
},
},
packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
last_checkin: lastCheckIn,
},
references: [],
updated_at: lastCheckIn,
version: 'WzI4MSwxXQ==',
score: 0,
},
{
type: AGENT_SAVED_OBJECT_TYPE,
id: testAgentId,
attributes: {
active: true,
id: 'oldTestAgentId',
config_id: 'randoConfigId',
type: 'PERMANENT',
user_provided_metadata: {},
enrolled_at: lastCheckIn,
current_error_events: [],
local_metadata: {
elastic: {
agent: {
id: 'oldTestAgentId',
},
},
host: {
hostname: 'testDesktop',
name: 'testDesktop',
id: hasDuplicates ? testHostId : 'oldRandoHostId',
},
os: {
platform: MockOSPlatform,
@ -76,7 +115,10 @@ export const mockFleetObjectsResponse = (
],
});
const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
const mockPolicyPayload = (
policyStatus: 'success' | 'warning' | 'failure',
policyMode: 'prevent' | 'detect' | 'off' = 'prevent'
) =>
JSON.stringify({
'endpoint-security': {
Endpoint: {
@ -105,7 +147,7 @@ const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
file: 'info',
},
malware: {
mode: 'prevent',
mode: policyMode,
},
},
windows: {
@ -122,7 +164,7 @@ const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
file: 'info',
},
malware: {
mode: 'prevent',
mode: policyMode,
},
},
},
@ -151,11 +193,11 @@ const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
'detect_file_open_events',
'detect_sync_image_load_events',
],
status: `${malwareStatus}`,
status: `${policyStatus}`,
},
},
},
status: `${malwareStatus}`,
status: `${policyStatus}`,
},
},
},
@ -186,7 +228,9 @@ const mockPolicyPayload = (malwareStatus: 'success' | 'warning' | 'failure') =>
*/
export const mockFleetEventsObjectsResponse = (
running?: boolean,
updatedDate = new Date().toISOString()
updatedDate = new Date().toISOString(),
policyStatus: 'success' | 'failure' = running ? 'success' : 'failure',
policyMode: 'prevent' | 'detect' | 'off' = 'prevent'
): SavedObjectsFindResponse<AgentEventSOAttributes> => {
return {
page: 1,
@ -204,7 +248,7 @@ export const mockFleetEventsObjectsResponse = (
message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${
running ? 'RUNNING' : 'FAILED'
}: `,
payload: mockPolicyPayload(running ? 'success' : 'failure'),
payload: running ? mockPolicyPayload(policyStatus, policyMode) : undefined,
config_id: testConfigId,
},
references: [],

332
x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts
View file

@ -51,17 +51,31 @@ describe('test security solution endpoint telemetry', () => {
`);
});
describe('when a request for endpoint agents fails', () => {
it('should return an empty object', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.reject(Error('No agents for you'))
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(getFleetSavedObjectsMetadataSpy).toHaveBeenCalled();
expect(endpointUsage).toEqual({});
});
});
describe('when an agent has not been installed', () => {
it('should return the default shape if no agents are found', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve({ saved_objects: [], total: 0, per_page: 0, page: 0 })
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(getFleetSavedObjectsMetadataSpy).toHaveBeenCalled();
expect(emptyEndpointTelemetryData).toEqual({
expect(endpointUsage).toEqual({
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
@ -76,68 +90,274 @@ describe('test security solution endpoint telemetry', () => {
});
});
describe('when an agent has been installed', () => {
it('should show one enpoint installed but it is inactive', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);
describe('when agent(s) have been installed', () => {
describe('when a request for events has failed', () => {
it('should show only one endpoint installed but it is inactive', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.reject(Error('No events for you'))
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(emptyEndpointTelemetryData).toEqual({
total_installed: 1,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 0,
inactive: 0,
},
},
],
policies: {
malware: {
failure: 1,
active: 0,
inactive: 0,
},
},
});
});
});
it('should show one endpoint installed and it is active', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);
describe('when a request for events is successful', () => {
it('should show one endpoint installed but endpoint has failed to run', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(emptyEndpointTelemetryData).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 0,
inactive: 0,
},
},
],
policies: {
malware: {
failure: 0,
active: 1,
inactive: 0,
});
});
it('should show two endpoints installed but both endpoints have failed to run', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse(false))
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 2,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 2,
},
],
policies: {
malware: {
failure: 0,
active: 0,
inactive: 0,
},
},
},
});
});
it('should show two endpoints installed but agents have not checked in within past day', async () => {
const twoDaysAgo = new Date();
twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);
const twoDaysAgoISOString = twoDaysAgo.toISOString();
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse(false, twoDaysAgoISOString))
);
getLatestFleetEndpointEventSpy.mockImplementation(
() => Promise.resolve(mockFleetEventsObjectsResponse(true, twoDaysAgoISOString)) // agent_id doesn't matter for mock here
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 2,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 2,
},
],
policies: {
malware: {
failure: 0,
active: 2,
inactive: 0,
},
},
});
});
it('should show one endpoint installed and endpoint is running', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 1,
inactive: 0,
},
},
});
});
describe('malware policy', () => {
it('should have failed to enable', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(
mockFleetEventsObjectsResponse(true, new Date().toISOString(), 'failure')
)
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 1,
active: 0,
inactive: 0,
},
},
});
});
it('should be enabled successfully', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 1,
inactive: 0,
},
},
});
});
it('should be disabled successfully', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getLatestFleetEndpointEventSpy.mockImplementation(() =>
Promise.resolve(
mockFleetEventsObjectsResponse(true, new Date().toISOString(), 'success', 'off')
)
);
const endpointUsage = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(endpointUsage).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
policies: {
malware: {
failure: 0,
active: 0,
inactive: 1,
},
},
});
});
});
});
});

16
x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts
View file

@ -16,8 +16,16 @@ export const FLEET_ENDPOINT_PACKAGE_CONSTANT = FleetDefaultPackages.endpoint;
export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObjectsRepository) =>
savedObjectsClient.find<Agent>({
// Get up to 10000 agents with endpoint installed
type: AGENT_SAVED_OBJECT_TYPE,
fields: ['packages', 'last_checkin', 'local_metadata'],
fields: [
'packages',
'last_checkin',
'local_metadata.agent.id',
'local_metadata.host.id',
'local_metadata.elastic.agent.id',
'local_metadata.os',
],
filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`,
perPage: 10000,
sortField: 'enrolled_at',
@ -29,9 +37,11 @@ export const getLatestFleetEndpointEvent = async (
agentId: string
) =>
savedObjectsClient.find<AgentEventSOAttributes>({
// Get the most recent endpoint event.
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`,
perPage: 1, // Get the most recent endpoint event.
fields: ['agent_id', 'subtype', 'payload'],
filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`,
perPage: 1,
sortField: 'timestamp',
sortOrder: 'desc',
search: agentId,

235
x-pack/plugins/security_solution/server/usage/endpoints/index.ts
View file

@ -3,8 +3,10 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { cloneDeep } from 'lodash';
import { ISavedObjectsRepository } from 'src/core/server';
import { SavedObject } from './../../../../../../src/core/types/saved_objects';
import { Agent, NewAgentEvent } from './../../../../ingest_manager/common/types/models/agent';
import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent';
import { getFleetSavedObjectsMetadata, getLatestFleetEndpointEvent } from './fleet_saved_objects';
@ -51,7 +53,7 @@ export interface AgentLocalMetadata extends AgentMetadata {
}
type OSTracker = Record<string, AgentOSMetadataTelemetry>;
type AgentDailyActiveTracker = Map<string, boolean>;
/**
* @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn
*/
@ -69,13 +71,14 @@ export const getDefaultEndpointTelemetry = (): EndpointUsage => ({
});
/**
* @description this fun
* @description this function updates the os telemetry. We use the fullName field as the key as it contains the name and version details.
* If it has already been tracked, the count will be updated, otherwise a tracker will be initialized for that fullName.
*/
export const trackEndpointOSTelemetry = (
export const updateEndpointOSTelemetry = (
os: AgentLocalMetadata['os'],
osTracker: OSTracker
): OSTracker => {
const updatedOSTracker = { ...osTracker };
const updatedOSTracker = cloneDeep(osTracker);
const { version: osVersion, platform: osPlatform, full: osFullName } = os;
if (osFullName && osVersion) {
if (updatedOSTracker[osFullName]) updatedOSTracker[osFullName].count += 1;
@ -93,18 +96,32 @@ export const trackEndpointOSTelemetry = (
};
/**
* @description This iterates over all unique agents that currently track an endpoint package. It takes a list of agents who have checked in in the last 24 hours
* and then checks whether those agents have endpoints whose latest status is 'RUNNING' to determine an active_within_last_24_hours. Since the policy information is also tracked in these events
* we pull out the status of the current protection (malware) type. This must be done in a compound manner as the desired status is reflected in the config, and the successful application of that policy
* is tracked in the policy.applied.response.configurations[protectionsType].status. Using these two we can determine whether the policy is toggled on, off, or failed to turn on.
* @description we take the latest endpoint specific agent event, get the status of the endpoint, and if it is running
* and the agent itself has been active within the last 24 hours, we can safely assume the endpoint has been active within
* the same time span.
*/
export const addEndpointDailyActivityAndPolicyDetailsToTelemetry = async (
agentDailyActiveTracker: AgentDailyActiveTracker,
savedObjectsClient: ISavedObjectsRepository,
endpointTelemetry: EndpointUsage
): Promise<EndpointUsage> => {
const updatedEndpointTelemetry = { ...endpointTelemetry };
export const updateEndpointDailyActiveCount = (
latestEndpointEvent: SavedObject<NewAgentEvent>,
lastAgentCheckin: Agent['last_checkin'],
currentCount: number
) => {
const aDayAgo = new Date();
aDayAgo.setDate(aDayAgo.getDate() - 1);
const agentWasActiveOverLastDay = !!lastAgentCheckin && new Date(lastAgentCheckin) > aDayAgo;
return agentWasActiveOverLastDay && latestEndpointEvent.attributes.subtype === 'RUNNING'
? currentCount + 1
: currentCount;
};
/**
* @description We take the latest endpoint specific agent event, and as long as it provides the payload with policy details, we will parse that policy
* to populate the success of it's application. The policy is provided in the agent health checks.
*/
export const updateEndpointPolicyTelemetry = (
latestEndpointEvent: SavedObject<NewAgentEvent>,
policiesTracker: PoliciesTelemetry
): PoliciesTelemetry => {
const policyHostTypeToPolicyType = {
Linux: 'linux',
macOs: 'mac',
@ -112,58 +129,60 @@ export const addEndpointDailyActivityAndPolicyDetailsToTelemetry = async (
};
const enabledMalwarePolicyTypes = ['prevent', 'detect'];
for (const agentId of agentDailyActiveTracker.keys()) {
const { saved_objects: agentEvents } = await getLatestFleetEndpointEvent(
savedObjectsClient,
agentId
);
// The policy details are sent as a string on the 'payload' attribute of the agent event
const { payload } = latestEndpointEvent.attributes;
const latestEndpointEvent = agentEvents[0];
if (latestEndpointEvent) {
/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours.
*/
const { subtype, payload } = latestEndpointEvent.attributes;
const endpointIsActive =
subtype === 'RUNNING' && agentDailyActiveTracker.get(agentId) === true;
if (endpointIsActive) {
updatedEndpointTelemetry.active_within_last_24_hours += 1;
}
// The policy details are sent as a string on the 'payload' attribute of the agent event
const endpointPolicyDetails = payload ? JSON.parse(payload) : null;
if (endpointPolicyDetails) {
// We get the setting the user desired to enable (treating prevent and detect as 'active' states) and then see if it succeded or failed.
const hostType =
policyHostTypeToPolicyType[
endpointPolicyDetails['endpoint-security']?.host?.os?.name as EndpointOSNames
];
const userDesiredMalwareState =
endpointPolicyDetails['endpoint-security'].Endpoint?.configuration?.inputs[0]?.policy[
hostType
]?.malware?.mode;
const isAnActiveMalwareState = enabledMalwarePolicyTypes.includes(userDesiredMalwareState);
const malwareStatus =
endpointPolicyDetails['endpoint-security'].Endpoint?.policy?.applied?.response
?.configurations?.malware?.status;
if (isAnActiveMalwareState && malwareStatus !== 'failure') {
updatedEndpointTelemetry.policies.malware.active += 1;
}
if (!isAnActiveMalwareState) {
updatedEndpointTelemetry.policies.malware.inactive += 1;
}
if (isAnActiveMalwareState && malwareStatus === 'failure') {
updatedEndpointTelemetry.policies.malware.failure += 1;
}
}
}
if (!payload) {
// This payload may not always be provided depending on the state of the endpoint. Guard again situations where it is not sent
return policiesTracker;
}
return updatedEndpointTelemetry;
let endpointPolicyPayload;
try {
endpointPolicyPayload = JSON.parse(latestEndpointEvent.attributes.payload);
} catch (error) {
return policiesTracker;
}
// Get the platform: windows, mac, or linux
const hostType =
policyHostTypeToPolicyType[
endpointPolicyPayload['endpoint-security']?.host?.os?.name as EndpointOSNames
];
// Get whether the malware setting for the platform on the most recently provided config is active (prevent or detect is on) or off
const userDesiredMalwareState =
endpointPolicyPayload['endpoint-security'].Endpoint?.configuration?.inputs[0]?.policy[hostType]
?.malware?.mode;
// Get the status of the application of the malware protection
const malwareStatus =
endpointPolicyPayload['endpoint-security'].Endpoint?.policy?.applied?.response?.configurations
?.malware?.status;
if (!userDesiredMalwareState || !malwareStatus) {
// If we get policy information without the mode or status, then nothing to track or update
return policiesTracker;
}
const updatedPoliciesTracker = {
malware: { ...policiesTracker.malware },
};
const isAnActiveMalwareState = enabledMalwarePolicyTypes.includes(userDesiredMalwareState);
// we only check for 'not failure' as the 'warning' state for malware is still technically actively enabled (with warnings)
const successfullyEnabled = !!malwareStatus && malwareStatus !== 'failure';
const failedToEnable = !!malwareStatus && malwareStatus === 'failure';
if (isAnActiveMalwareState && successfullyEnabled) {
updatedPoliciesTracker.malware.active += 1;
} else if (!isAnActiveMalwareState && successfullyEnabled) {
updatedPoliciesTracker.malware.inactive += 1;
} else if (isAnActiveMalwareState && failedToEnable) {
updatedPoliciesTracker.malware.failure += 1;
}
return updatedPoliciesTracker;
};
/**
@ -173,53 +192,71 @@ export const addEndpointDailyActivityAndPolicyDetailsToTelemetry = async (
* to confirm whether or not the endpoint is still active
*/
export const getEndpointTelemetryFromFleet = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<EndpointUsage> => {
// Retrieve every agent that references the endpoint as an installed package. It will not be listed if it was never installed
const { saved_objects: endpointAgents } = await getFleetSavedObjectsMetadata(savedObjectsClient);
soClient: ISavedObjectsRepository
): Promise<EndpointUsage | {}> => {
// Retrieve every agent (max 10000) that references the endpoint as an installed package. It will not be listed if it was never installed
let endpointAgents;
try {
const response = await getFleetSavedObjectsMetadata(soClient);
endpointAgents = response.saved_objects;
} catch (error) {
// Better to provide an empty object rather than default telemetry as this better informs us of an error
return {};
}
const endpointAgentsCount = endpointAgents.length;
const endpointTelemetry = getDefaultEndpointTelemetry();
// If there are no installed endpoints return the default telemetry object
if (!endpointAgents || endpointAgents.length < 1) return endpointTelemetry;
if (!endpointAgents || endpointAgentsCount < 1) return endpointTelemetry;
// Use unique hosts to prevent any potential duplicates
const uniqueHostIds: Set<string> = new Set();
// Need agents to get events data for those that have run in last 24 hours as well as policy details
const agentDailyActiveTracker: AgentDailyActiveTracker = new Map();
const aDayAgo = new Date();
aDayAgo.setDate(aDayAgo.getDate() - 1);
let osTracker: OSTracker = {};
let dailyActiveCount = 0;
let policyTracker: PoliciesTelemetry = { malware: { active: 0, inactive: 0, failure: 0 } };
const endpointMetadataTelemetry = endpointAgents.reduce(
(metadataTelemetry, { attributes: metadataAttributes }) => {
const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
const { host, os, elastic } = localMetadata as AgentLocalMetadata; // AgentMetadata is just an empty blob, casting for our use case
for (let i = 0; i < endpointAgentsCount; i += 1) {
const { attributes: metadataAttributes } = endpointAgents[i];
const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
const { host, os, elastic } = localMetadata as AgentLocalMetadata; // AgentMetadata is just an empty blob, casting for our use case
if (host && uniqueHostIds.has(host.id)) {
// use hosts since new agents could potentially be re-installed on existing hosts
return metadataTelemetry;
} else {
uniqueHostIds.add(host.id);
const isActiveWithinLastDay = !!lastCheckin && new Date(lastCheckin) > aDayAgo;
agentDailyActiveTracker.set(elastic.agent.id, isActiveWithinLastDay);
osTracker = trackEndpointOSTelemetry(os, osTracker);
return metadataTelemetry;
if (!uniqueHostIds.has(host.id)) {
uniqueHostIds.add(host.id);
const agentId = elastic?.agent?.id;
osTracker = updateEndpointOSTelemetry(os, osTracker);
if (agentId) {
let agentEvents;
try {
const response = await getLatestFleetEndpointEvent(soClient, agentId);
agentEvents = response.saved_objects;
} catch (error) {
// If the request fails we do not obtain `active within last 24 hours for this agent` or policy specifics
}
// AgentEvents will have a max length of 1
if (agentEvents && agentEvents.length > 0) {
const latestEndpointEvent = agentEvents[0];
dailyActiveCount = updateEndpointDailyActiveCount(
latestEndpointEvent,
lastCheckin,
dailyActiveCount
);
policyTracker = updateEndpointPolicyTelemetry(latestEndpointEvent, policyTracker);
}
}
},
endpointTelemetry
);
}
}
// All unique hosts with an endpoint installed.
// All unique hosts with an endpoint installed, thus all unique endpoint installs
endpointTelemetry.total_installed = uniqueHostIds.size;
// Set the daily active count for the endpoints
endpointTelemetry.active_within_last_24_hours = dailyActiveCount;
// Get the objects to populate our OS Telemetry
endpointMetadataTelemetry.os = Object.values(osTracker);
// Populate endpoint telemetry with the finalized 24 hour count and policy details
const finalizedEndpointTelemetryData = await addEndpointDailyActivityAndPolicyDetailsToTelemetry(
agentDailyActiveTracker,
savedObjectsClient,
endpointMetadataTelemetry
);
endpointTelemetry.os = Object.values(osTracker);
// Provide the updated policy information
endpointTelemetry.policies = policyTracker;
return finalizedEndpointTelemetryData;
return endpointTelemetry;
};